Check exact candidate eval profile before provider startup

Compare expected profile fields with the built Runner before materializing context or constructing a live service. Require the flag for candidate runs; old CLIs reject it without starting a provider.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -154,6 +154,16 @@ versions and patched ACP server bytes. Do not replace this step with a fresh
 `patchedDependencies`, so the resulting ACPX executables no longer match their
 qualified digests.
 
+Candidate ACPX evals require both `--candidate-profile <agent>` and
+`--expected-acpx-profile <JSON>`. The eval program supplies the exact configured
+profile as the JSON argument. Before creating a runtime context or provider
+service, the built CLI compares every profile field, including model, version
+and command digest, with its own resolved profile. Missing, extra or mismatched
+fields fail admission. Older CLIs reject the new flag before execution; there
+is no retry without the check. The daemon's own profile admission and binary
+digest check remain independent gates. Post-run scoring still compares the
+retained profile as evidence, but cannot replace this pre-launch check.
+
 The direct eval CLI also materializes a minimal immutable native runtime
 context in each isolated attempt workspace. This keeps the direct layer on the
 same `paperclip.native-execution-input.v3` contract as production, including

packages/paperclip-runner/src/cli/eval-session-contract.test.ts

@@ -7,6 +7,7 @@ import { describe, expect, it, vi } from "vitest";
 import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js";
 
 import { parseNativeRuntimeContext } from "../contracts/runtime-context.js";
+import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js";
 import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js";
 import {
   evalSessionUsage,
@@ -78,11 +79,50 @@ describe("eval-session request contract", () => {
 
   it("accepts only known diagnostic flags and rejects ambiguous repeated arguments", () => {
     const args = ["--request", "/tmp/request.json", "--output", "/tmp/result.json"];
-    expect(parseEvalSessionCliArgs([...args, "--candidate-profile", "pi"])).toMatchObject({ candidateProfile: "pi" });
+    const expected = resolveQualifiedAcpxProfile("pi", "openrouter/deepseek/deepseek-v4-flash-0731");
+    expect(parseEvalSessionCliArgs([...args, "--candidate-profile", "pi", "--expected-acpx-profile", JSON.stringify(expected)]))
+      .toMatchObject({ candidateProfile: "pi", expectedAcpxProfile: expected });
+    expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile", "pi"])).toThrow("require --expected-acpx-profile");
     expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile", "codex"])).toThrow("must be pi, cursor, or copilot");
     expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile"])).toThrow("missing");
     expect(() => parseEvalSessionCliArgs([...args, "--request", "/tmp/other.json"])).toThrow("duplicate");
     expect(() => parseEvalSessionRequest(request(), { candidateProfile: "pi" })).toThrow("must match");
+    for (const invalid of ["null", "[]", "true", "{", JSON.stringify("text")]) {
+      expect(() => parseEvalSessionCliArgs([...args, "--expected-acpx-profile", invalid])).toThrow("JSON object");
+    }
+    expect(() => parseEvalSessionCliArgs([...args, "--expected-acpx-profile", " ".repeat(4097)])).toThrow("4096-byte bound");
+  });
+
+  it.each(["pi", "cursor", "copilot"] as const)("rejects stale or incomplete %s identities before runtime construction", async (agent) => {
+    const workspace = await mkdtemp(join(tmpdir(), "paperclip-eval-profile-"));
+    const model = agent === "pi" ? "openrouter/deepseek/deepseek-v4-flash-0731" : "explicit-provider-model";
+    const expected = resolveQualifiedAcpxProfile(agent, model);
+    const requestPath = join(workspace, "request.json");
+    const args = ["--request", requestPath, "--output", join(workspace, "output.json"), "--candidate-profile", agent];
+    const serviceFactory = vi.fn(() => { throw new Error("provider must not start"); });
+    try {
+      await writeFile(requestPath, JSON.stringify(request({ provider: "acpx", acpxAgent: agent, model,
+        runnerd: { path: join(workspace, "missing-runnerd"), sha256: "a".repeat(64) },
+        session: { workingDirectory: workspace },
+      })));
+      const { commandDigest: _removed, ...incomplete } = expected;
+      for (const stale of [
+        {}, incomplete, { ...expected, unexpected: true },
+        { ...expected, agentProfileVersion: expected.agentProfileVersion - 1 },
+        { ...expected, commandDigest: `sha256:${"0".repeat(64)}` },
+        { ...expected, reportedModelId: "another-model" },
+        { ...expected, agentServerVersion: "previous-binary" },
+      ]) {
+        await expect(runEvalSessionCli([...args, "--expected-acpx-profile", JSON.stringify(stale)], { serviceFactory }))
+          .rejects.toThrow("does not match the built runner profile");
+      }
+      await expect(runEvalSessionCli(args, { serviceFactory })).rejects.toThrow("require --expected-acpx-profile");
+      // A matching identity reaches the independent runner-binary digest gate.
+      await expect(runEvalSessionCli([...args, "--expected-acpx-profile", JSON.stringify(expected)], { serviceFactory }))
+        .rejects.toThrow("ENOENT");
+      expect(serviceFactory).not.toHaveBeenCalled();
+      expect(await readdir(workspace)).toEqual(["request.json"]);
+    } finally { await rm(workspace, { recursive: true, force: true }); }
   });
 
   it("materializes a production-v3 runtime context for direct live providers", async () => {
@@ -446,7 +486,7 @@ describe("eval-session budget settlement", () => {
       const requestPath = join(workspace, "request.json");
       const outputPath = join(workspace, "output.json");
       await writeFile(requestPath, JSON.stringify(input));
-      const exitCode = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, ...(agent === "grok" ? [] : ["--candidate-profile", "copilot"])], {
+      const exitCode = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, ...(agent === "grok" ? [] : ["--candidate-profile", "copilot", "--expected-acpx-profile", JSON.stringify(resolveQualifiedAcpxProfile("copilot", model))])], {
         serviceFactory: () => ({ create: async () => ({ sendMessage, completeAttempt, shutdown, snapshot: () => snapshot }) }) as never,
       });
       const artifact = JSON.parse(await readFile(outputPath, "utf8"));

packages/paperclip-runner/src/cli/eval-session.ts

@@ -40,6 +40,7 @@ interface EvalSessionCliOptions {
   requestPath: string;
   outputPath: string;
   candidateProfile?: EvalCandidateProfile;
+  expectedAcpxProfile?: Record<string, unknown>;
 }
 
 function argument(args: string[], name: string): string {
@@ -50,7 +51,7 @@ function argument(args: string[], name: string): string {
 }
 
 export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {
-  const allowed = new Set(["--request", "--output", "--candidate-profile"]);
+  const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile"]);
   const seen = new Set<string>();
   for (let index = 0; index < args.length; index += 2) {
     if (!allowed.has(args[index] ?? "")) {
@@ -65,8 +66,28 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {
   if (candidateProfile !== undefined && candidateProfile !== "pi" && candidateProfile !== "cursor" && candidateProfile !== "copilot") {
     throw new Error("--candidate-profile must be pi, cursor, or copilot");
   }
+  const expectedIndex = args.indexOf("--expected-acpx-profile");
+  const expectedText = expectedIndex < 0 ? undefined : args[expectedIndex + 1];
+  if (candidateProfile !== undefined && expectedText === undefined) {
+    throw new Error("Candidate evals require --expected-acpx-profile before provider execution");
+  }
+  let expectedAcpxProfile: Record<string, unknown> | undefined;
+  if (expectedText !== undefined) {
+    if (Buffer.byteLength(expectedText, "utf8") > 4_096) {
+      throw new Error("--expected-acpx-profile exceeds its 4096-byte bound");
+    }
+    let parsed: unknown;
+    try { parsed = JSON.parse(expectedText); } catch {
+      throw new Error("--expected-acpx-profile must be a JSON object");
+    }
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+      throw new Error("--expected-acpx-profile must be a JSON object");
+    }
+    expectedAcpxProfile = parsed as Record<string, unknown>;
+  }
   return {
     ...(candidateProfile === undefined ? {} : { candidateProfile }),
+    ...(expectedAcpxProfile === undefined ? {} : { expectedAcpxProfile }),
     requestPath: argument(args, "--request"),
     outputPath: argument(args, "--output"),
   };
@@ -324,6 +345,20 @@ export async function runEvalSessionCli(
     JSON.parse(await readFile(cli.requestPath, "utf8")),
     { candidateProfile: cli.candidateProfile },
   );
+  // Check the built CLI's profile before constructing any runtime context,
+  // transport or service. Scoring after a paid turn is too late for admission.
+  if (cli.expectedAcpxProfile !== undefined) {
+    if (request.provider !== "acpx") {
+      throw new Error("--expected-acpx-profile requires an ACPX request");
+    }
+    const actualProfile = resolveQualifiedAcpxProfile(request.acpxAgent ?? "codex", request.model);
+    const entries = Object.entries(actualProfile);
+    if (Object.keys(cli.expectedAcpxProfile).length !== entries.length || entries.some(
+      ([key, value]) => !Object.hasOwn(cli.expectedAcpxProfile!, key) || cli.expectedAcpxProfile![key] !== value,
+    )) {
+      throw new Error("--expected-acpx-profile does not match the built runner profile");
+    }
+  }
   const runnerdPath = resolve(request.runnerd.path);
   const actualDigest = await sha256(runnerdPath);
   if (actualDigest !== request.runnerd.sha256.replace(/^sha256:/, "")) {