diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 94801d17540..1ef927f5825 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -149,6 +149,12 @@ startup hooks, or user homes into this route. Unsupported harnesses and mismatched descriptor identities fail before provider creation. The temporary home is removed on success and failure. +The controlled wrapper forwards the bounded `--expected-acpx-profile` JSON +argument to the same built CLI. Eval definitions pass it for qualified ACPX +lanes as well as candidates, so stale SDK or command pins fail before a paid +turn rather than only during scoring. A rejected profile never falls back to +an invocation without the admission check. + Success and failure artifacts retain `requestRouting`: controller gateway mode, provider, declared upstream route, campaign/execution/attempt IDs, and the SHA-256 of the descriptor bytes. The token and endpoint are excluded from this diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs index 230cb01ac32..3c1fcafd5ab 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller-fixture.mjs @@ -19,6 +19,7 @@ import {dirname,join} from 'node:path'; import {execFileSync} from 'node:child_process'; const args=new Map();for(let i=2;i { + const profile = { agent: 'claude', agentRuntimeVersion: '0.3.286' }; + for (const mode of ['function', 'cli']) for (const stale of [false, true]) { + const requests = mode === 'function' ? 1 : 0; + const f = await fixture(t, { providerProfile: 'claude', builtAcpxProfile: profile, requests }); + const expectedAcpxProfile = { ...profile, ...(stale ? { agentRuntimeVersion: '0.3.263' } : {}) }; + let calls = 0; + let result; + if (mode === 'function') { + result = await runControlledEval({ manifestPath: f.manifestPath, ...f.requests[0], environment: f.environment, expectedAcpxProfile, + transport: async () => { calls++; return success('anthropic'); } }); + } else { + let code = 0; + try { + await promisify(execFile)(process.execPath, [fileURLToPath(new URL('./run.mjs', runtime)), '--request', f.requests[0].requestPath, + '--output', f.requests[0].outputPath, '--expected-acpx-profile', JSON.stringify(expectedAcpxProfile)], { + env: { PATH: process.env.PATH, ANTHROPIC_API_KEY: f.environment.ANTHROPIC_API_KEY, PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST: f.manifestPath }, timeout: 15000, + }); + } catch (error) { if (typeof error.code !== 'number') throw error; code = error.code; } + result = { code, report: JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')).controllerAccounting }; + } + assert.equal(calls, stale ? 0 : requests); + assert.equal(result.code === 0, !stale); + assert.equal(result.report.accounting.requests.length, stale ? 0 : requests); + } +}); + +test('malformed profile arguments fail before opening controlled evidence', async t => { + for (const [providerProfile, profiles] of [ + ['claude', [null, [], 'profile', { overflow: 'x'.repeat(4097) }]], + ['codex', [{}]], + ]) { + const f = await fixture(t, { providerProfile }); + for (const expectedAcpxProfile of profiles) { + await assert.rejects(runControlledEval({ manifestPath: f.manifestPath, ...f.requests[0], environment: {}, expectedAcpxProfile }), /Expected ACPX profile/); + } + await assert.rejects(stat(f.manifest.outputDirectory), { code: 'ENOENT' }); + } +}); + test('real child requests reserve before dispatch, stop at the cap and retain only controller credentials', async t => { const f = await fixture(t); let calls = 0; diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 27659b161bf..c348d9f910e 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -22,6 +22,7 @@ export const attemptDirectory = (manifest, attemptId) => join(manifest.outputDir async function runChild(input) { const child = spawn(process.execPath, ['--import', input.loader, input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath, + ...(input.expectedAcpxProfile ? ['--expected-acpx-profile', input.expectedAcpxProfile] : []), ...(input.observabilityPath ? ['--observability-config', input.observabilityPath] : [])], { cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); @@ -65,11 +66,17 @@ export function controlledResultArtifact(artifact, report, attempt) { controllerAccounting: report }; } -export async function runControlledEval({ manifestPath, requestPath, outputPath, environment = process.env, signal, transport, otlpTransport }) { +export async function runControlledEval({ manifestPath, requestPath, outputPath, expectedAcpxProfile, environment = process.env, signal, transport, otlpTransport }) { if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Controlled eval process ownership requires Linux or macOS'); signal?.throwIfAborted(); const { manifest, manifestSha256 } = validateManifest(await readJson(manifestPath)); const request = await readJson(requestPath, 8_388_608), attempt = bindRequest(manifest, request); + let profileArgument; + if (expectedAcpxProfile !== undefined) { + if (request.provider !== 'acpx' || expectedAcpxProfile === null || typeof expectedAcpxProfile !== 'object' || Array.isArray(expectedAcpxProfile)) throw new Error('Expected ACPX profile requires an ACPX request and JSON object'); + profileArgument = canonical(expectedAcpxProfile); + if (Buffer.byteLength(profileArgument) > 4096) throw new Error('Expected ACPX profile exceeds its 4096-byte bound'); + } assertHostedAdmission(manifest, attempt, environment); try { await lstat(outputPath); throw new Error('Controlled output already exists'); } catch (error) { if (error.code !== 'ENOENT') throw error; } @@ -152,7 +159,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { failureStage = 'child_failed'; signal?.throwIfAborted(); if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn'); - child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, observabilityPath: observability?.descriptorPath, + child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, expectedAcpxProfile: profileArgument, observabilityPath: observability?.descriptorPath, cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => Promise.all([gateway.close(), observability?.close()]) }); failureStage = 'output_retention_failed'; for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); @@ -222,10 +229,12 @@ if (await isEntrypoint(import.meta.url)) { const cancel = () => controller.abort(); process.once('SIGINT', cancel); process.once('SIGTERM', cancel); try { - const { values } = parseArgs({ options: { request: { type: 'string' }, output: { type: 'string' } } }); + const { values } = parseArgs({ options: { request: { type: 'string' }, output: { type: 'string' }, 'expected-acpx-profile': { type: 'string' } } }); if (!values.request || !values.output || !process.env.PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST) throw new Error('A controlled manifest, request and output are required'); + const rawProfile = values['expected-acpx-profile']; + if (rawProfile !== undefined && Buffer.byteLength(rawProfile) > 4096) throw new Error('Expected ACPX profile exceeds its 4096-byte bound'); const result = await runControlledEval({ manifestPath: process.env.PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST, - requestPath: values.request, outputPath: values.output, signal: controller.signal }); + requestPath: values.request, outputPath: values.output, expectedAcpxProfile: rawProfile === undefined ? undefined : JSON.parse(rawProfile), signal: controller.signal }); process.exitCode = result.code; } catch { process.stderr.write('Controlled Runner eval admission or retention failed; inspect retained controller evidence.\n'); process.exitCode = 2; } finally { process.removeListener('SIGINT', cancel); process.removeListener('SIGTERM', cancel); } diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index 6b12436ec21..479934cc8f6 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -43,6 +43,26 @@ function request(overrides: Record = {}): unknown { }; } +it("rejects a stale qualified Claude SDK before provider creation", async () => { + const workspace = await mkdtemp(join(tmpdir(), "eval-qualified-profile-")); + try { + const path = join(workspace, "request.json"); + const expected = resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"); + await writeFile(path, JSON.stringify(request({ + provider: "acpx", driver: "acpx_runtime", acpxAgent: "claude", model: "claude-sonnet-5", + runnerd: { path: join(workspace, "missing-runnerd"), sha256: "a".repeat(64) }, + session: { workingDirectory: workspace }, + }))); + const serviceFactory = vi.fn(); + const args = ["--request", path, "--output", join(workspace, "artifact.json"), "--expected-acpx-profile"]; + await expect(runEvalSessionCli([...args, JSON.stringify({ ...expected, agentRuntimeVersion: "0.3.263" })], { serviceFactory })) + .rejects.toThrow("does not match the built runner profile"); + await expect(runEvalSessionCli([...args, JSON.stringify(expected)], { serviceFactory })).rejects.toThrow("ENOENT"); + expect(serviceFactory).not.toHaveBeenCalled(); + expect(await readdir(workspace)).toEqual(["request.json"]); + } finally { await rm(workspace, { recursive: true, force: true }); } +}); + function agentCoreProfile(overrides: Record = {}) { return { profileId: "agentcore-qualified",