From 6b9e5f3a806b7b0a51cfb33d00748d5e1596a159 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:59:20 -0500 Subject: [PATCH] fix(evals): enforce qualified ACPX profile admission before paid turns (#17964) ## Thinking Path > - Paperclip coordinates agent work through Runner. > - Controlled evals route model calls through a budget gateway. > - The definitions pin each ACPX runtime identity. > - A stale qualified profile currently reaches a paid turn before scoring rejects it. > - The Runner CLI already checks an exact expected profile before it creates a provider. > - This change forwards that existing admission argument through the controller. > - Profile drift can then stop before model spending. ## Linked Issues or Issue Description Refs #17943 and #17929. **What happened?** The worker in run 40618022700 passed all four behavior checks and retained three final request receipts. Its grader then rejected the old Claude SDK pin. The controlled wrapper could not pass the existing `--expected-acpx-profile` argument to the built CLI. **What did you expect to happen?** The built CLI must reject a stale expected profile before provider creation. **Steps to reproduce** Run a controlled ACPX cell with a definition that pins a different SDK from the built Runner. The old wrapper accepts only request and output arguments. It cannot enforce the profile admission supplied by updated definitions. **Paperclip version** The observed run used e4b30163bba629914904b068511b45bafa21d93b. **Deployment mode** The controlled GitHub Actions Runner protocol workflow. ## What Changed - Forward bounded expected-profile JSON through the controlled wrapper to the same built CLI. - Reject invalid objects, oversized arguments, and profiles on non-ACPX requests before opening evidence. - Exercise a real child boundary and the actual qualified Claude CLI admission gate. - Document the definition/controller compatibility requirement. ## Verification - Controller suite: 77 tests passed. The new regressions fail against the original wrapper. - Hosted and portable controller suites: 8 tests passed. - Actual eval-session CLI contract suite: 39 tests passed. A stale qualified Claude SDK never creates a provider service. - Runner package typecheck passed. - The non-ACPX rejection guard also passed its focused regression after review. - No live provider request was made for this change. - Full repository checks and current-head reviews run on this PR. ## Risks - Updated definitions require this wrapper before they can pass the new argument. Merge the controller first. - Profile identity remains owned by the built Runner CLI. The controller only forwards its existing argument. - Existing callers that omit the argument retain their current behavior. - Optional review suggestions to duplicate the expected-profile hash into controller evidence and add a full built-CLI/controller integration case are deferred. The pinned definition revision retains the expected profile, and both controller entrypoints plus the actual CLI admission boundary are covered. A compatibility error fails closed without a paid fallback. ## Model Used OpenAI Codex, GPT-6 family, with tool execution and extended reasoning. The exact serving model ID and context window are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../docs/runner-protocol-live-evals.md | 6 +++ .../controlled-evals/controller-fixture.mjs | 1 + .../controlled-evals/controller.test.mjs | 40 +++++++++++++++++++ .../scripts/controlled-evals/run.mjs | 17 ++++++-- .../src/cli/eval-session-contract.test.ts | 20 ++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 94801d17540..1ef927f5825 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -149,6 +149,12 @@ startup hooks, or user homes into this route. Unsupported harnesses and mismatched descriptor identities fail before provider creation. The temporary home is removed on success and failure. +The controlled wrapper forwards the bounded `--expected-acpx-profile` JSON +argument to the same built CLI. Eval definitions pass it for qualified ACPX +lanes as well as candidates, so stale SDK or command pins fail before a paid +turn rather than only during scoring. A rejected profile never falls back to +an invocation without the admission check. + Success and failure artifacts retain `requestRouting`: controller gateway mode, provider, declared upstream route, campaign/execution/attempt IDs, and the SHA-256 of the descriptor bytes. The token and endpoint are excluded from this diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs index 230cb01ac32..3c1fcafd5ab 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs @@ -19,6 +19,7 @@ import {dirname,join} from 'node:path'; import {execFileSync} from 'node:child_process'; const args=new Map();for(let i=2;i { + const profile = { agent: 'claude', agentRuntimeVersion: '0.3.286' }; + for (const mode of ['function', 'cli']) for (const stale of [false, true]) { + const requests = mode === 'function' ? 1 : 0; + const f = await fixture(t, { providerProfile: 'claude', builtAcpxProfile: profile, requests }); + const expectedAcpxProfile = { ...profile, ...(stale ? { agentRuntimeVersion: '0.3.263' } : {}) }; + let calls = 0; + let result; + if (mode === 'function') { + result = await runControlledEval({ manifestPath: f.manifestPath, ...f.requests[0], environment: f.environment, expectedAcpxProfile, + transport: async () => { calls++; return success('anthropic'); } }); + } else { + let code = 0; + try { + await promisify(execFile)(process.execPath, [fileURLToPath(new URL('./run.mjs', runtime)), '--request', f.requests[0].requestPath, + '--output', f.requests[0].outputPath, '--expected-acpx-profile', JSON.stringify(expectedAcpxProfile)], { + env: { PATH: process.env.PATH, ANTHROPIC_API_KEY: f.environment.ANTHROPIC_API_KEY, PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST: f.manifestPath }, timeout: 15000, + }); + } catch (error) { if (typeof error.code !== 'number') throw error; code = error.code; } + result = { code, report: JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')).controllerAccounting }; + } + assert.equal(calls, stale ? 0 : requests); + assert.equal(result.code === 0, !stale); + assert.equal(result.report.accounting.requests.length, stale ? 0 : requests); + } +}); + +test('malformed profile arguments fail before opening controlled evidence', async t => { + for (const [providerProfile, profiles] of [ + ['claude', [null, [], 'profile', { overflow: 'x'.repeat(4097) }]], + ['codex', [{}]], + ]) { + const f = await fixture(t, { providerProfile }); + for (const expectedAcpxProfile of profiles) { + await assert.rejects(runControlledEval({ manifestPath: f.manifestPath, ...f.requests[0], environment: {}, expectedAcpxProfile }), /Expected ACPX profile/); + } + await assert.rejects(stat(f.manifest.outputDirectory), { code: 'ENOENT' }); + } +}); + test('real child requests reserve before dispatch, stop at the cap and retain only controller credentials', async t => { const f = await fixture(t); let calls = 0; diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 27659b161bf..c348d9f910e 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -22,6 +22,7 @@ export const attemptDirectory = (manifest, attemptId) => join(manifest.outputDir async function runChild(input) { const child = spawn(process.execPath, ['--import', input.loader, input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath, + ...(input.expectedAcpxProfile ? ['--expected-acpx-profile', input.expectedAcpxProfile] : []), ...(input.observabilityPath ? ['--observability-config', input.observabilityPath] : [])], { cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); @@ -65,11 +66,17 @@ export function controlledResultArtifact(artifact, report, attempt) { controllerAccounting: report }; } -export async function runControlledEval({ manifestPath, requestPath, outputPath, environment = process.env, signal, transport, otlpTransport }) { +export async function runControlledEval({ manifestPath, requestPath, outputPath, expectedAcpxProfile, environment = process.env, signal, transport, otlpTransport }) { if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Controlled eval process ownership requires Linux or macOS'); signal?.throwIfAborted(); const { manifest, manifestSha256 } = validateManifest(await readJson(manifestPath)); const request = await readJson(requestPath, 8_388_608), attempt = bindRequest(manifest, request); + let profileArgument; + if (expectedAcpxProfile !== undefined) { + if (request.provider !== 'acpx' || expectedAcpxProfile === null || typeof expectedAcpxProfile !== 'object' || Array.isArray(expectedAcpxProfile)) throw new Error('Expected ACPX profile requires an ACPX request and JSON object'); + profileArgument = canonical(expectedAcpxProfile); + if (Buffer.byteLength(profileArgument) > 4096) throw new Error('Expected ACPX profile exceeds its 4096-byte bound'); + } assertHostedAdmission(manifest, attempt, environment); try { await lstat(outputPath); throw new Error('Controlled output already exists'); } catch (error) { if (error.code !== 'ENOENT') throw error; } @@ -152,7 +159,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { failureStage = 'child_failed'; signal?.throwIfAborted(); if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn'); - child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, observabilityPath: observability?.descriptorPath, + child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, expectedAcpxProfile: profileArgument, observabilityPath: observability?.descriptorPath, cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => Promise.all([gateway.close(), observability?.close()]) }); failureStage = 'output_retention_failed'; for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); @@ -222,10 +229,12 @@ if (await isEntrypoint(import.meta.url)) { const cancel = () => controller.abort(); process.once('SIGINT', cancel); process.once('SIGTERM', cancel); try { - const { values } = parseArgs({ options: { request: { type: 'string' }, output: { type: 'string' } } }); + const { values } = parseArgs({ options: { request: { type: 'string' }, output: { type: 'string' }, 'expected-acpx-profile': { type: 'string' } } }); if (!values.request || !values.output || !process.env.PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST) throw new Error('A controlled manifest, request and output are required'); + const rawProfile = values['expected-acpx-profile']; + if (rawProfile !== undefined && Buffer.byteLength(rawProfile) > 4096) throw new Error('Expected ACPX profile exceeds its 4096-byte bound'); const result = await runControlledEval({ manifestPath: process.env.PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST, - requestPath: values.request, outputPath: values.output, signal: controller.signal }); + requestPath: values.request, outputPath: values.output, expectedAcpxProfile: rawProfile === undefined ? undefined : JSON.parse(rawProfile), signal: controller.signal }); process.exitCode = result.code; } catch { process.stderr.write('Controlled Runner eval admission or retention failed; inspect retained controller evidence.\n'); process.exitCode = 2; } finally { process.removeListener('SIGINT', cancel); process.removeListener('SIGTERM', cancel); } diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index 6b12436ec21..479934cc8f6 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -43,6 +43,26 @@ function request(overrides: Record = {}): unknown { }; } +it("rejects a stale qualified Claude SDK before provider creation", async () => { + const workspace = await mkdtemp(join(tmpdir(), "eval-qualified-profile-")); + try { + const path = join(workspace, "request.json"); + const expected = resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"); + await writeFile(path, JSON.stringify(request({ + provider: "acpx", driver: "acpx_runtime", acpxAgent: "claude", model: "claude-sonnet-5", + runnerd: { path: join(workspace, "missing-runnerd"), sha256: "a".repeat(64) }, + session: { workingDirectory: workspace }, + }))); + const serviceFactory = vi.fn(); + const args = ["--request", path, "--output", join(workspace, "artifact.json"), "--expected-acpx-profile"]; + await expect(runEvalSessionCli([...args, JSON.stringify({ ...expected, agentRuntimeVersion: "0.3.263" })], { serviceFactory })) + .rejects.toThrow("does not match the built runner profile"); + await expect(runEvalSessionCli([...args, JSON.stringify(expected)], { serviceFactory })).rejects.toThrow("ENOENT"); + expect(serviceFactory).not.toHaveBeenCalled(); + expect(await readdir(workspace)).toEqual(["request.json"]); + } finally { await rm(workspace, { recursive: true, force: true }); } +}); + function agentCoreProfile(overrides: Record = {}) { return { profileId: "agentcore-qualified",