From 36f7a89b37c8fd7b96603395b5d746057ac246aa Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 11:54:00 -0500 Subject: [PATCH] Check exact candidate eval profile before provider startup Compare expected profile fields with the built Runner before materializing context or constructing a live service. Require the flag for candidate runs; old CLIs reject it without starting a provider. Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 10 +++++ .../src/cli/eval-session-contract.test.ts | 44 ++++++++++++++++++- .../paperclip-runner/src/cli/eval-session.ts | 37 +++++++++++++++- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 36b36fbded7..bbaaf1aee5e 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -154,6 +154,16 @@ versions and patched ACP server bytes. Do not replace this step with a fresh `patchedDependencies`, so the resulting ACPX executables no longer match their qualified digests. +Candidate ACPX evals require both `--candidate-profile ` and +`--expected-acpx-profile `. The eval program supplies the exact configured +profile as the JSON argument. Before creating a runtime context or provider +service, the built CLI compares every profile field, including model, version +and command digest, with its own resolved profile. Missing, extra or mismatched +fields fail admission. Older CLIs reject the new flag before execution; there +is no retry without the check. The daemon's own profile admission and binary +digest check remain independent gates. Post-run scoring still compares the +retained profile as evidence, but cannot replace this pre-launch check. + The direct eval CLI also materializes a minimal immutable native runtime context in each isolated attempt workspace. This keeps the direct layer on the same `paperclip.native-execution-input.v3` contract as production, including diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index 7082be9ad58..7d5820bdced 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js"; import { parseNativeRuntimeContext } from "../contracts/runtime-context.js"; +import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; import { evalSessionUsage, @@ -78,11 +79,50 @@ describe("eval-session request contract", () => { it("accepts only known diagnostic flags and rejects ambiguous repeated arguments", () => { const args = ["--request", "/tmp/request.json", "--output", "/tmp/result.json"]; - expect(parseEvalSessionCliArgs([...args, "--candidate-profile", "pi"])).toMatchObject({ candidateProfile: "pi" }); + const expected = resolveQualifiedAcpxProfile("pi", "openrouter/deepseek/deepseek-v4-flash-0731"); + expect(parseEvalSessionCliArgs([...args, "--candidate-profile", "pi", "--expected-acpx-profile", JSON.stringify(expected)])) + .toMatchObject({ candidateProfile: "pi", expectedAcpxProfile: expected }); + expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile", "pi"])).toThrow("require --expected-acpx-profile"); expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile", "codex"])).toThrow("must be pi, cursor, or copilot"); expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile"])).toThrow("missing"); expect(() => parseEvalSessionCliArgs([...args, "--request", "/tmp/other.json"])).toThrow("duplicate"); expect(() => parseEvalSessionRequest(request(), { candidateProfile: "pi" })).toThrow("must match"); + for (const invalid of ["null", "[]", "true", "{", JSON.stringify("text")]) { + expect(() => parseEvalSessionCliArgs([...args, "--expected-acpx-profile", invalid])).toThrow("JSON object"); + } + expect(() => parseEvalSessionCliArgs([...args, "--expected-acpx-profile", " ".repeat(4097)])).toThrow("4096-byte bound"); + }); + + it.each(["pi", "cursor", "copilot"] as const)("rejects stale or incomplete %s identities before runtime construction", async (agent) => { + const workspace = await mkdtemp(join(tmpdir(), "paperclip-eval-profile-")); + const model = agent === "pi" ? "openrouter/deepseek/deepseek-v4-flash-0731" : "explicit-provider-model"; + const expected = resolveQualifiedAcpxProfile(agent, model); + const requestPath = join(workspace, "request.json"); + const args = ["--request", requestPath, "--output", join(workspace, "output.json"), "--candidate-profile", agent]; + const serviceFactory = vi.fn(() => { throw new Error("provider must not start"); }); + try { + await writeFile(requestPath, JSON.stringify(request({ provider: "acpx", acpxAgent: agent, model, + runnerd: { path: join(workspace, "missing-runnerd"), sha256: "a".repeat(64) }, + session: { workingDirectory: workspace }, + }))); + const { commandDigest: _removed, ...incomplete } = expected; + for (const stale of [ + {}, incomplete, { ...expected, unexpected: true }, + { ...expected, agentProfileVersion: expected.agentProfileVersion - 1 }, + { ...expected, commandDigest: `sha256:${"0".repeat(64)}` }, + { ...expected, reportedModelId: "another-model" }, + { ...expected, agentServerVersion: "previous-binary" }, + ]) { + await expect(runEvalSessionCli([...args, "--expected-acpx-profile", JSON.stringify(stale)], { serviceFactory })) + .rejects.toThrow("does not match the built runner profile"); + } + await expect(runEvalSessionCli(args, { serviceFactory })).rejects.toThrow("require --expected-acpx-profile"); + // A matching identity reaches the independent runner-binary digest gate. + await expect(runEvalSessionCli([...args, "--expected-acpx-profile", JSON.stringify(expected)], { serviceFactory })) + .rejects.toThrow("ENOENT"); + expect(serviceFactory).not.toHaveBeenCalled(); + expect(await readdir(workspace)).toEqual(["request.json"]); + } finally { await rm(workspace, { recursive: true, force: true }); } }); it("materializes a production-v3 runtime context for direct live providers", async () => { @@ -446,7 +486,7 @@ describe("eval-session budget settlement", () => { const requestPath = join(workspace, "request.json"); const outputPath = join(workspace, "output.json"); await writeFile(requestPath, JSON.stringify(input)); - const exitCode = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, ...(agent === "grok" ? [] : ["--candidate-profile", "copilot"])], { + const exitCode = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, ...(agent === "grok" ? [] : ["--candidate-profile", "copilot", "--expected-acpx-profile", JSON.stringify(resolveQualifiedAcpxProfile("copilot", model))])], { serviceFactory: () => ({ create: async () => ({ sendMessage, completeAttempt, shutdown, snapshot: () => snapshot }) }) as never, }); const artifact = JSON.parse(await readFile(outputPath, "utf8")); diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 19153159445..a7b7597f6c5 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -40,6 +40,7 @@ interface EvalSessionCliOptions { requestPath: string; outputPath: string; candidateProfile?: EvalCandidateProfile; + expectedAcpxProfile?: Record; } function argument(args: string[], name: string): string { @@ -50,7 +51,7 @@ function argument(args: string[], name: string): string { } export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { - const allowed = new Set(["--request", "--output", "--candidate-profile"]); + const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile"]); const seen = new Set(); for (let index = 0; index < args.length; index += 2) { if (!allowed.has(args[index] ?? "")) { @@ -65,8 +66,28 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { if (candidateProfile !== undefined && candidateProfile !== "pi" && candidateProfile !== "cursor" && candidateProfile !== "copilot") { throw new Error("--candidate-profile must be pi, cursor, or copilot"); } + const expectedIndex = args.indexOf("--expected-acpx-profile"); + const expectedText = expectedIndex < 0 ? undefined : args[expectedIndex + 1]; + if (candidateProfile !== undefined && expectedText === undefined) { + throw new Error("Candidate evals require --expected-acpx-profile before provider execution"); + } + let expectedAcpxProfile: Record | undefined; + if (expectedText !== undefined) { + if (Buffer.byteLength(expectedText, "utf8") > 4_096) { + throw new Error("--expected-acpx-profile exceeds its 4096-byte bound"); + } + let parsed: unknown; + try { parsed = JSON.parse(expectedText); } catch { + throw new Error("--expected-acpx-profile must be a JSON object"); + } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("--expected-acpx-profile must be a JSON object"); + } + expectedAcpxProfile = parsed as Record; + } return { ...(candidateProfile === undefined ? {} : { candidateProfile }), + ...(expectedAcpxProfile === undefined ? {} : { expectedAcpxProfile }), requestPath: argument(args, "--request"), outputPath: argument(args, "--output"), }; @@ -324,6 +345,20 @@ export async function runEvalSessionCli( JSON.parse(await readFile(cli.requestPath, "utf8")), { candidateProfile: cli.candidateProfile }, ); + // Check the built CLI's profile before constructing any runtime context, + // transport or service. Scoring after a paid turn is too late for admission. + if (cli.expectedAcpxProfile !== undefined) { + if (request.provider !== "acpx") { + throw new Error("--expected-acpx-profile requires an ACPX request"); + } + const actualProfile = resolveQualifiedAcpxProfile(request.acpxAgent ?? "codex", request.model); + const entries = Object.entries(actualProfile); + if (Object.keys(cli.expectedAcpxProfile).length !== entries.length || entries.some( + ([key, value]) => !Object.hasOwn(cli.expectedAcpxProfile!, key) || cli.expectedAcpxProfile![key] !== value, + )) { + throw new Error("--expected-acpx-profile does not match the built runner profile"); + } + } const runnerdPath = resolve(request.runnerd.path); const actualDigest = await sha256(runnerdPath); if (actualDigest !== request.runnerd.sha256.replace(/^sha256:/, "")) {