From 507033e48c94b167c330837f6956555706b8c346 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 09:59:26 -0700 Subject: [PATCH 1/9] feat(evals): enforce fixed Runner request allocations Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 78 +++++++ packages/paperclip-runner/package.json | 2 +- .../controlled-evals/controller.test.mjs | 200 ++++++++++++++++++ .../scripts/controlled-evals/manifest.mjs | 93 ++++++++ .../scripts/controlled-evals/report.mjs | 69 ++++++ .../scripts/controlled-evals/run.mjs | 163 ++++++++++++++ .../controlled-evals/shared-cost-runtime.ts | 3 + 7 files changed, 607 insertions(+), 1 deletion(-) create mode 100644 packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/manifest.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/report.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/run.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index a7114ed7c31..a77870cc501 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -159,6 +159,84 @@ change the hosted matrix. The controller must enforce request admission, deadlines, and limits before forwarding traffic. A local fake-API probe only proves routing; it does not qualify live provider or Braintrust behavior. + +## Controlled request budgets + +The source-checkout controller is +`packages/paperclip-runner/scripts/controlled-evals/run.mjs`. Runner/evals owns +this orchestration. It reuses the existing eval request proxy and ledger from +`tests/runner-e2e` through one import graph; it does not copy their reservation +or provider-usage rules. It requires the workspace's installed `tsx` runtime. +It is not a published Runner package entry point. + +The controller accepts the existing `--request` and `--output` contract. Set +`PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST` to an explicit reviewed manifest and +supply the selected upstream credential only to this trusted controller. For +Runner Evals, pass this controller path as `eval_program.py run --runner-cli`; +the manifest identifies the actual built `eval-session.js`. The provider child +receives only its temporary proxy credential and private home. + +The manifest schema is `paperclip.runner-eval.controller/v1`. It contains: + +- `campaignId`, `maxUsd`, `deadlineAt`, `pricingVerifiedAt`, and + `pricingExpiresAt`; admission must fit a current four-hour window. +- `sourceRevision` and `definitionsRevision`, each an exact commit ID. + These are the trusted preparer's provenance declarations. +- `runnerCli`, `runnerPackage`, and `runnerd`, each an absolute `path` and + content `sha256`. The controller checks the three files before launch. +- An absolute durable `outputDirectory`, outside the disposable case workspace. +- `route` (`direct` or `braintrust`), plus `projectId` for Braintrust. +- `models`, using the existing exact-model price, request-limit, and reviewed + full-lifecycle `provisionalBound` contract. +- `overhead`, using the existing four-component operating allowance with a + current full-lifecycle bound for every component. This is reserved once. +- `attempts`, each with a unique `executionId` and `attemptId`, + `requestSha256`, `provider`, optional `acpxAgent`, exact `model`, `maxUsd`, + `maxRequests`, `maxConcurrent`, and `timeoutMs`. Native Codex and ACPX + Claude are supported. Each execution has one explicit attempt and no retry. + +Use `requestBindingSha256` from `controlled-evals/manifest.mjs` to bind the +complete compiled request. It excludes only the temporary working directory +and native executable path; the executable digest remains bound and its path +is checked separately. The controller gives the child a retained copy of the +validated request. Changing prompts, fixtures, models, limits, or attempts +requires a new reviewed binding. + +The sum of every cell's allocation and the once-only operating allowance must +fit `maxUsd` before any cell starts. A request reserves against its cell's +ledger before forwarding. Cells cannot borrow unused allocations. Unknown +requests keep their reservations, and failed or missing cell evidence keeps +that cell's full allocation in the campaign report. An existing cell directory +or output is never reset for a retry. Cancellation closes admission, drains +upstream receipts within the proxy limit, and stops the owned CLI process +group. Unverified provider cleanup remains explicit and retains the allocation. + +Each result retains `controllerAccounting` alongside the original provider +outcome and usage. Retain the durable controller directory as private evidence: +its admission, request, gateway descriptor, request receipts, ledger, original +provider artifact, and final result are required for reconciliation. The +private program's existing immutable attempt copy includes the result metadata; +its current default history publisher does not archive this separate directory. +Do not infer receipt retention from that metadata alone. + +```sh +node packages/paperclip-runner/scripts/controlled-evals/report.mjs \ + /private/path/controller-manifest.json > /private/path/controller-report.json +``` + +The report recomputes ledger costs, checks their attempt/manifest bindings, and +preserves known partial charges. It reports operating costs as held and total +costs as provisional when evidence is intact, or incomplete otherwise. It does +not release historical holds or claim invoice +settlement. Existing provider usage is retained separately; do not add the +provider and controller cost observations as if they were separate charges. + +This local controller is not yet wired into the hosted matrix. Its durable +attempt-directory guard applies to one shared filesystem, not independent +GitHub workers. Hosted integration must bind unique cells to the prepared +allocation, reject workflow/job reruns, and preserve the controller evidence +before enabling paid work. Native OTLP routing, report integration, vendor +metering bounds, and actual paid qualification remain separate gates. ## Hosted campaign Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 8080d157740..f42661d79c4 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -85,7 +85,7 @@ "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "pnpm run test:typescript:prep && vitest run", - "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", + "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs new file mode 100644 index 00000000000..afa7e6bb6e0 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -0,0 +1,200 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; +import { attemptDirectory, runControlledEval } from './run.mjs'; +import { summarizeControlledCampaign } from './report.mjs'; + +const childScript = options => ` +import assert from 'node:assert/strict'; +import {createHash} from 'node:crypto'; +import {readFile,writeFile} from 'node:fs/promises'; +const args=new Map();for(let i=2;isetInterval(()=>{},1000)); +if(!${Boolean(options.missingArtifact)})await writeFile(args.get('--output'),JSON.stringify({schema:'paperclip-runner/eval-session-artifact/v1',attemptId:request.attemptId,requestedModel:request.model, + requestRouting:${options.missingRoute ? 'null' : "{mode:'controller_gateway',provider:descriptor.provider,declaredUpstreamRoute:descriptor.upstreamRoute,campaignId:descriptor.campaignId,executionId:descriptor.executionId,attemptId:descriptor.attemptId,descriptorSha256:createHash('sha256').update(descriptorRaw).digest('hex')}"}, + snapshot:{status:'closed',process:{runnerExited:true,runnerPid:process.pid,providerPid:process.pid}},statuses, + leaked:${options.leakToken ? 'process.env.PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN' : 'null'}})); +`; + +async function fixture(t, { count = 1, allocation = '0.01', providerProfile = 'codex', route = 'direct', ...options } = {}) { + const root = await mkdtemp(join(tmpdir(), 'runner-controller-test-')); + t.after(() => rm(root, { recursive: true, force: true })); + const cli = join(root, 'fake-cli.mjs'), binary = join(root, 'runnerd'), archive = join(root, 'runner.tgz'); + await writeFile(cli, childScript(options)); await writeFile(binary, 'synthetic daemon'); await writeFile(archive, 'synthetic package'); + const provenance = async path => ({ path, sha256: sha256(await readFile(path)) }); + const now = Date.now(), verifiedAt = new Date(now - 1000).toISOString(), expiresAt = new Date(now + 86_400_000).toISOString(); + const overhead = { version: 1, reference: 'synthetic-allowance', runner: 'github-hosted', verifiedAt, expiresAt, + components: costRuntime.EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: '0.001', reference: 'synthetic-proof' })) }; + const bound = configuration => ({ version: 1, scope: 'full_lifecycle', evidenceSha256: 'a'.repeat(64), configurationSha256: sha256(canonical(configuration)) }); + for (const component of overhead.components) component.provisionalBound = bound({ reference: overhead.reference, runner: overhead.runner, verifiedAt, expiresAt, component: { ...component } }); + const biller = providerProfile === 'claude' ? 'anthropic' : 'openai'; + const price = { provider: biller, model: 'fixture', version: 'synthetic-pricing', sourceUrl: 'https://provider.invalid/pricing', inputUsdPerMillion: '2', cachedReadUsdPerMillion: '1', outputUsdPerMillion: '10', ...(biller === 'openai' ? { cacheWrite30mUsdPerMillion: '2', cacheWrite12hUsdPerMillion: '2' } : { cacheWrite5mUsdPerMillion: '2', cacheWrite1hUsdPerMillion: '2' }) }; + const model = { provider: biller, price, maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 1000 }; + model.provisionalBound = bound({ provider: model.provider, mode: route, price, maximumInputTokens: 1000, maximumOutputTokens: 100, reservation: '0.003000000' }); + const manifest = { schema: 'paperclip.runner-eval.controller/v1', campaignId: 'synthetic-controller', maxUsd: '1', + deadlineAt: new Date(now + 60_000).toISOString(), pricingVerifiedAt: verifiedAt, pricingExpiresAt: expiresAt, overhead, + sourceRevision: 'a'.repeat(40), definitionsRevision: 'b'.repeat(40), runnerCli: await provenance(cli), runnerPackage: await provenance(archive), runnerd: await provenance(binary), + outputDirectory: join(root, 'controller'), route, ...(route === 'braintrust' ? { projectId: '11111111-1111-4111-8111-111111111111' } : {}), models: [model], attempts: [] }; + const requests = []; + for (let index = 0; index < count; index++) { + const workspace = join(root, `workspace-${index}`); await mkdir(workspace); + const request = { schema: 'paperclip-runner/eval-session-request/v1', attemptId: `attempt-${index}`, prompt: 'synthetic', provider: providerProfile === 'claude' ? 'acpx' : 'codex', ...(providerProfile === 'claude' ? { acpxAgent: 'claude' } : {}), model: 'fixture', + runnerd: { path: binary, sha256: manifest.runnerd.sha256 }, limits: { turnTimeoutMs: 5000, maxAgentTurns: 1, maxEstimatedCostNanodollars: 1_000_000_000 }, session: { workingDirectory: workspace } }; + const requestPath = join(root, `request-${index}.json`), outputPath = join(root, `output-${index}.json`); + await writeFile(requestPath, JSON.stringify(request)); requests.push({ request, requestPath, outputPath }); + manifest.attempts.push({ executionId: `execution-${index}`, attemptId: request.attemptId, requestSha256: requestBindingSha256(request), provider: request.provider, ...(request.acpxAgent ? { acpxAgent: request.acpxAgent } : {}), model: 'fixture', maxUsd: allocation, maxRequests: 2, maxConcurrent: 1, timeoutMs: 5000 }); + } + const manifestPath = join(root, 'manifest.json'); + const save = () => writeFile(manifestPath, JSON.stringify(manifest)); await save(); + const environment = { PATH: process.env.PATH, OPENAI_API_KEY: 'synthetic-upstream-private', ANTHROPIC_API_KEY: 'other-private', BRAINTRUST_API_KEY: 'other-braintrust', GH_TOKEN: 'other-github', NODE_OPTIONS: '--require must-not-inherit', PYTHONPATH: '/must-not-inherit' }; + return { root, manifest, manifestPath, requests, environment, save, + run: (index = 0, transport = () => success(biller), signal) => runControlledEval({ manifestPath, ...requests[index], environment, transport, signal }) }; +} +let sequence = 0; +function success(provider = 'openai') { + if (provider === 'anthropic') return Response.json({ type: 'message', id: `synthetic-${++sequence}`, model: 'fixture', usage: { input_tokens: 8, cache_read_input_tokens: 2, cache_creation_input_tokens: 0, output_tokens: 5, cache_creation: { ephemeral_5m_input_tokens: 0, ephemeral_1h_input_tokens: 0 } } }, { headers: { 'x-bt-used-endpoint': 'ANTHROPIC_API_KEY', 'x-bt-cached': 'MISS' } }); + return Response.json({ object: 'response', id: `synthetic-${++sequence}`, status: 'completed', model: 'fixture', usage: { + input_tokens: 10, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 5, output_tokens_details: { reasoning_tokens: 3 } } }); +} + +test('real child requests reserve before dispatch, stop at the cap and retain only controller credentials', async t => { + const f = await fixture(t); + let calls = 0; + const result = await f.run(0, async (_url, init) => { + calls++; + assert.equal(new Headers(init.headers).get('authorization'), 'Bearer synthetic-upstream-private'); + const ledger = await costRuntime.readEvalCostLedger(join(attemptDirectory(f.manifest, 'attempt-0'), 'ledger.jsonl')); + assert.equal(ledger.state.snapshot().inflight, 1); + return success(); + }); + assert.equal(calls, 2); assert.equal(result.code, 0); assert.equal(result.report.cleanupVerified, true); + const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + assert.deepEqual(artifact.statuses, [200, 200, 429]); + assert.equal(artifact.controllerAccounting.accounting.knownUsd, '0.000136000'); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.incompleteAttempts, 0); assert.equal(summary.heldUsd, '0.004000000'); assert.equal(summary.committedUsd, '0.004136000'); + for (const name of ['admission.json', 'result.json', 'ledger.jsonl', 'provider-artifact.json', 'stdout.log', 'stderr.log']) { + assert(!(await readFile(join(attemptDirectory(f.manifest, 'attempt-0'), name), 'utf8')).includes(f.environment.OPENAI_API_KEY)); + } + await assert.rejects(f.run(), /output already exists/); + await rm(f.requests[0].outputPath); + await assert.rejects(f.run(), { code: 'EEXIST' }); + assert.equal(calls, 2); +}); + +test('disjoint concurrent cells cannot borrow an allowance and unknown requests retain their reservations', async t => { + const f = await fixture(t, { count: 2, allocation: '0.003' }); f.manifest.maxUsd = '0.01'; await f.save(); + let calls = 0; + const transport = async () => { calls++; return Response.json({ error: 'synthetic refusal' }, { status: 429 }); }; + const results = await Promise.all([f.run(0, transport), f.run(1, transport)]); + assert.equal(calls, 2); assert(results.every(result => result.report.status === 'closed')); + assert(results.every(result => result.report.accounting.heldUsd === '0.003000000')); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.committedUsd, '0.010000000'); assert.equal(summary.knownUsd, '0.000000000'); assert.equal(summary.overBudget, false); + assert(summary.attempts.every(attempt => attempt.boundedUnresolvedRequests === 1)); +}); + +test('rejects oversized or unqualified allocations, changed requests and changed executables before dispatch', async t => { + const f = await fixture(t, { count: 2 }); + for (const mutate of [ + value => { value.maxUsd = '0.01'; }, value => { value.attempts[1].executionId = value.attempts[0].executionId; }, + value => { value.attempts[0].provider = 'acpx'; value.attempts[0].acpxAgent = 'codex'; }, + value => { delete value.overhead.components[0].provisionalBound; }, value => { value.models[0].secret = 'unexpected'; }, + ]) { const value = structuredClone(f.manifest); mutate(value); assert.throws(() => validateManifest(value)); } + f.requests[0].request.prompt = 'drift'; await writeFile(f.requests[0].requestPath, JSON.stringify(f.requests[0].request)); + await assert.rejects(f.run(), /allocated identity/); + await writeFile(f.manifest.runnerCli.path, 'changed CLI'); + await assert.rejects(f.run(1), /provenance changed/); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.incompleteAttempts, 2); assert.equal(summary.heldUsd, '0.024000000'); +}); + +for (const fault of ['missingArtifact', 'missingRoute', 'leakToken']) test(`${fault} keeps the allocation held and preserves the attempt failure`, async t => { + const f = await fixture(t, { [fault]: true, requests: 1 }); + const result = await f.run(); + assert.equal(result.code, 2); assert.equal(result.report.status, 'incomplete'); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.incompleteAttempts, 1); assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.costStatus, 'incomplete'); + assert.equal(summary.knownUsd, '0.000068000'); + const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + assert.equal(artifact.accountingFailure.retryable, false); + if (fault === 'leakToken') { assert.equal(result.report.secretExcluded, false); assert.equal(artifact.leaked, '[redacted]'); } +}); + +test('cancellation closes the proxy, bounds the owned child, and preserves unknown request holds', async t => { + const f = await fixture(t, { requests: 1, waitForever: true }); + const abort = new AbortController(); let calls = 0; + const result = await f.run(0, async (_url, init) => { + calls++; abort.abort(); + return new Promise((_resolve, reject) => init.signal.addEventListener('abort', () => reject(new Error('synthetic interrupted request')), { once: true })); + }, abort.signal); + assert.equal(calls, 1); assert.equal(result.code, 2); assert.equal(result.report.cleanupVerified, false); + assert.equal(result.report.accounting.unknownRequests, 1); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.incompleteAttempts, 1); +}); + +test('changed receipts cannot release allocations and expired campaigns remain inspectable', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const resultPath = join(root, 'result.json'), result = JSON.parse(await readFile(resultPath, 'utf8')); + result.accounting.knownUsd = '0.000000000'; await writeFile(resultPath, JSON.stringify(result)); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.incompleteAttempts, 1); assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.knownUsd, '0.000068000'); + assert.throws(() => validateManifest(f.manifest, Date.now() + 2 * 86_400_000)); + assert.doesNotThrow(() => validateManifest(f.manifest, Date.now() + 2 * 86_400_000, { admission: false })); +}); + + +test('a lost final report preserves known ledger charges while holding the remaining cell allowance', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + await rm(join(attemptDirectory(f.manifest, 'attempt-0'), 'result.json')); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.knownUsd, '0.000068000'); assert.equal(summary.committedUsd, '0.014000000'); + assert.equal(summary.incompleteAttempts, 1); +}); + + +test('Claude cells use the assigned Braintrust provider and keep its credential on the controller', async t => { + const f = await fixture(t, { providerProfile: 'claude', route: 'braintrust', requests: 1 }); + let calls = 0; + const result = await f.run(0, async (url, init) => { + calls++; assert.equal(url, 'https://gateway.braintrust.dev/messages'); + const headers = new Headers(init.headers); + assert.equal(headers.get('authorization'), 'Bearer other-braintrust'); + assert.equal(headers.get('x-bt-endpoint-name'), 'ANTHROPIC_API_KEY'); + assert.equal(headers.get('x-bt-project-id'), f.manifest.projectId); + return success('anthropic'); + }); + assert.equal(calls, 1); assert.equal(result.code, 0); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.incompleteAttempts, 0); assert.equal(summary.knownUsd, '0.000068000'); +}); + + +test('existing outputs and physical workspace aliases are rejected before provider work', async t => { + const f = await fixture(t); + let calls = 0; + const transport = async () => { calls++; return success(); }; + await writeFile(f.requests[0].outputPath, 'existing evidence'); + await assert.rejects(f.run(0, transport), /already exists/); + assert.equal(await readFile(f.requests[0].outputPath, 'utf8'), 'existing evidence'); + await rm(f.requests[0].outputPath); + f.manifest.outputDirectory = join(f.requests[0].request.session.workingDirectory, '..evidence'); await f.save(); + await assert.rejects(f.run(0, transport), /outside the disposable/); + f.manifest.outputDirectory = join(f.root, 'aliased-evidence'); + await symlink(f.requests[0].request.session.workingDirectory, f.manifest.outputDirectory); await f.save(); + await assert.rejects(f.run(0, transport), /resolves inside/); + assert.equal(calls, 0); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs new file mode 100644 index 00000000000..0ca1bb655d0 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -0,0 +1,93 @@ +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { readFile } from 'node:fs/promises'; +import { isAbsolute } from 'node:path'; + +const require = createRequire(new URL('../../../../server/package.json', import.meta.url)); +const { tsImport } = await import(require.resolve('tsx/esm/api')); +export const costRuntime = await tsImport('./shared-cost-runtime.ts', import.meta.url); +export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); +export const canonical = value => JSON.stringify(value, (_key, item) => item && typeof item === 'object' && !Array.isArray(item) + ? Object.fromEntries(Object.keys(item).sort().map(key => [key, item[key]])) : item); +export const usd = value => `${value / 1_000_000_000n}.${String(value % 1_000_000_000n).padStart(9, '0')}`; +export function units(value) { + if (typeof value !== 'string' || !/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(value)) throw new Error('Invalid controller USD amount'); + const [whole, fraction = ''] = value.split('.'); + return BigInt(whole) * 1_000_000_000n + BigInt(fraction.padEnd(9, '0')); +} +export function fields(value, names) { + if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(key => !names.includes(key))) throw new Error('Invalid controller fields'); +} +const hash = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +const id = value => costRuntime.validEvalLedgerId(value); +export function requestBindingSha256(request) { + const copy = structuredClone(request); + if (!copy.session || !copy.runnerd) throw new Error('Invalid controlled request'); + delete copy.session.workingDirectory; + delete copy.runnerd.path; + return sha256(canonical(copy)); +} +export async function readJson(file, maximum = 1_048_576) { + const bytes = await readFile(file); + if (bytes.length > maximum) throw new Error('Controller document exceeds its byte limit'); + return JSON.parse(bytes.toString('utf8')); +} +export function validateManifest(value, now = Date.now(), { admission = true } = {}) { + fields(value, ['schema', 'campaignId', 'maxUsd', 'deadlineAt', 'pricingVerifiedAt', 'pricingExpiresAt', 'overhead', + 'sourceRevision', 'definitionsRevision', 'runnerCli', 'runnerPackage', 'runnerd', 'outputDirectory', 'route', 'projectId', 'models', 'attempts']); + const deadline = Date.parse(value.deadlineAt); + const verified = Date.parse(value.pricingVerifiedAt), expires = Date.parse(value.pricingExpiresAt); + if (value.schema !== 'paperclip.runner-eval.controller/v1' || !id(value.campaignId) + || !Number.isFinite(now) || !Number.isFinite(deadline) || admission && (deadline <= now || deadline - now > 4 * 3_600_000) + || !Number.isFinite(verified) || !Number.isFinite(expires) || verified >= deadline || expires <= verified || admission && verified > now || expires <= deadline || expires - verified > 31 * 86_400_000 + || !/^[a-f0-9]{40}$/.test(value.sourceRevision ?? '') || !/^[a-f0-9]{40}$/.test(value.definitionsRevision ?? '') + || !isAbsolute(value.outputDirectory ?? '') || !['direct', 'braintrust'].includes(value.route) + || value.route === 'braintrust' && !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value.projectId ?? '') + || value.route === 'direct' && value.projectId !== undefined) throw new Error('Invalid controller campaign identity or validity window'); + for (const artifact of [value.runnerCli, value.runnerPackage, value.runnerd]) { + fields(artifact, ['path', 'sha256']); + if (!isAbsolute(artifact.path ?? '') || !hash(artifact.sha256)) throw new Error('Invalid controller executable provenance'); + } + const overhead = costRuntime.validateOperatingOverhead(value.overhead, value.maxUsd); + if (admission && Date.parse(value.overhead.verifiedAt) > now || Date.parse(value.overhead.expiresAt) <= deadline + || value.overhead.components.some(component => !component.provisionalBound)) throw new Error('Controller requires current full-lifecycle operating bounds'); + if (!Array.isArray(value.models) || !value.models.length || value.models.length > 16 + || !Array.isArray(value.attempts) || !value.attempts.length || value.attempts.length > 1000) throw new Error('Invalid controlled campaign selection'); + const prices = new Map(); + const rateFields = ['inputUsdPerMillion', 'cachedReadUsdPerMillion', 'outputUsdPerMillion', 'cacheWrite5mUsdPerMillion', 'cacheWrite1hUsdPerMillion', 'cacheWrite30mUsdPerMillion', 'cacheWrite12hUsdPerMillion']; + for (const model of value.models) { + fields(model, ['provider', 'price', 'maximumInputTokens', 'maximumOutputTokens', 'timeoutMs', 'provisionalBound']); + fields(model.price, ['provider', 'model', 'version', 'sourceUrl', 'longContext', ...rateFields]); + if (model.price.longContext) { fields(model.price.longContext, ['aboveInputTokens', 'rates']); fields(model.price.longContext.rates, rateFields); } + const url = new URL(model.price.sourceUrl); + const key = `${model.provider}:${model.price.model}`; + if (!['openai', 'anthropic'].includes(model.provider) || !model.provisionalBound || url.protocol !== 'https:' + || url.username || url.password || url.search || url.hash || prices.has(key)) throw new Error('Invalid controller model pricing'); + prices.set(key, units(costRuntime.evalRequestReservation({ ...model, mode: value.route, apiKey: 'validation' }))); + } + const attempts = new Set(), executions = new Set(); + let allocated = overhead; + for (const attempt of value.attempts) { + fields(attempt, ['executionId', 'attemptId', 'requestSha256', 'provider', 'acpxAgent', 'model', 'maxUsd', 'maxRequests', 'maxConcurrent', 'timeoutMs']); + const provider = attempt.provider === 'codex' && attempt.acpxAgent === undefined ? 'openai' + : attempt.provider === 'acpx' && attempt.acpxAgent === 'claude' ? 'anthropic' : null; + const reservation = prices.get(`${provider}:${attempt.model}`); + if (!provider || !id(attempt.executionId) || !id(attempt.attemptId) || !hash(attempt.requestSha256) + || attempts.has(attempt.attemptId) || executions.has(attempt.executionId) || reservation === undefined + || !Number.isSafeInteger(attempt.timeoutMs) || attempt.timeoutMs < 1 || attempt.timeoutMs > 4 * 3_600_000 + || !Number.isSafeInteger(attempt.maxRequests) || attempt.maxRequests < 1 || attempt.maxRequests > 1000 + || !Number.isSafeInteger(attempt.maxConcurrent) || attempt.maxConcurrent < 1 || attempt.maxConcurrent > 16 + || units(attempt.maxUsd) < reservation) throw new Error('Invalid controlled attempt allocation'); + attempts.add(attempt.attemptId); executions.add(attempt.executionId); allocated += units(attempt.maxUsd); + } + if (allocated > units(value.maxUsd)) throw new Error('Controlled allocations exceed the campaign budget'); + return { manifest: structuredClone(value), manifestSha256: sha256(canonical(value)), allocatedUsd: usd(allocated), overheadUsd: usd(overhead) }; +} +export function bindRequest(manifest, request) { + const attempt = manifest.attempts.find(item => item.attemptId === request.attemptId); + if (!attempt || requestBindingSha256(request) !== attempt.requestSha256 || request.provider !== attempt.provider + || request.acpxAgent !== (attempt.acpxAgent ?? null) && request.acpxAgent !== attempt.acpxAgent + || request.model !== attempt.model || request.runnerd.sha256.replace(/^sha256:/, '') !== manifest.runnerd.sha256 + || request.runnerd.path !== manifest.runnerd.path) throw new Error('Controlled request does not match its allocated identity'); + return attempt; +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs new file mode 100644 index 00000000000..dcbd145a170 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -0,0 +1,69 @@ +import { readFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { canonical, costRuntime, readJson, sha256, units, usd, validateManifest } from './manifest.mjs'; +import { attemptDirectory } from './run.mjs'; + +export async function summarizeControlledCampaign(value) { + const { manifest, manifestSha256, allocatedUsd, overheadUsd } = validateManifest(value, Date.now(), { admission: false }); + let known = 0n, committed = units(overheadUsd); + const attempts = []; + for (const allocation of manifest.attempts) { + const root = attemptDirectory(manifest, allocation.attemptId); + let snapshot, report, verified = false; + try { + const admission = await readJson(join(root, 'admission.json')); + const ledgerBytes = await readFile(join(root, 'ledger.jsonl')); + const ledger = await costRuntime.readEvalCostLedger(join(root, 'ledger.jsonl')); + const candidateSnapshot = ledger.state.snapshot(); + const started = Date.parse(admission.startedAt); + const expectedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, + deadlineAt: new Date(Math.min(Date.parse(manifest.deadlineAt), started + allocation.timeoutMs)).toISOString(), + maxRequests: allocation.maxRequests, maxConcurrent: allocation.maxConcurrent, maxAttemptsPerExecution: 1 }; + if (admission.schema !== 'paperclip.runner-eval.admission/v1' || admission.manifestSha256 !== manifestSha256 + || admission.campaignId !== manifest.campaignId || canonical(admission.attempt) !== canonical(allocation) + || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits) + || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }]) + || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger does not match its allocation'); + snapshot = candidateSnapshot; + report = await readJson(join(root, 'result.json')); + const artifactBytes = await readFile(join(root, 'provider-artifact.json')); + const artifact = JSON.parse(artifactBytes.toString('utf8')); + const descriptorBytes = await readFile(join(root, 'gateway.json')); + const descriptor = JSON.parse(descriptorBytes.toString('utf8')); + if (report.schema !== 'paperclip.runner-eval.control-result/v1' || report.manifestSha256 !== manifestSha256 + || report.campaignId !== manifest.campaignId || report.executionId !== allocation.executionId || report.attemptId !== allocation.attemptId + || report.requestSha256 !== allocation.requestSha256 || report.ledgerSha256 !== sha256(ledgerBytes) + || report.providerArtifactSha256 !== sha256(artifactBytes) || canonical(report.accounting) !== canonical(snapshot) + || report.status !== 'closed' || report.cleanupVerified !== true || report.secretExcluded !== true || report.failure !== null + || report.startedAt !== admission.startedAt || !Number.isFinite(Date.parse(report.finishedAt)) || Date.parse(report.finishedAt) < started + || !report.child || report.child.stopped !== null || artifact.snapshot?.status !== 'closed' || artifact.snapshot?.process?.runnerExited !== true + || descriptor.schema !== 'paperclip.runner-eval.gateway/v1' || descriptor.provider !== (allocation.provider === 'codex' ? 'openai' : 'anthropic') + || descriptor.model !== allocation.model || descriptor.upstreamRoute !== manifest.route + || descriptor.campaignId !== manifest.campaignId || descriptor.executionId !== allocation.executionId || descriptor.attemptId !== allocation.attemptId + || canonical(artifact.requestRouting) !== canonical({ mode: 'controller_gateway', provider: descriptor.provider, declaredUpstreamRoute: manifest.route, + campaignId: manifest.campaignId, executionId: allocation.executionId, attemptId: allocation.attemptId, descriptorSha256: sha256(descriptorBytes) }) + || artifact.attemptId !== allocation.attemptId || artifact.requestedModel !== allocation.model + || snapshot.inflight || snapshot.conflictingRequests || snapshot.overBudget + || snapshot.unknownRequests !== snapshot.boundedUnresolvedRequests) throw new Error('Incomplete controller evidence'); + verified = true; + } catch {} + const cost = snapshot ? units(snapshot.knownUsd) : 0n; + const observedCommitment = snapshot ? units(snapshot.committedUsd) : 0n; + const commitment = verified ? observedCommitment : observedCommitment > units(allocation.maxUsd) ? observedCommitment : units(allocation.maxUsd); + known += cost; committed += commitment; + attempts.push({ executionId: allocation.executionId, attemptId: allocation.attemptId, status: verified ? 'retained' : 'incomplete', + allocationUsd: allocation.maxUsd, knownUsd: usd(cost), heldUsd: usd(commitment - cost), committedUsd: usd(commitment), + unknownRequests: snapshot?.unknownRequests ?? null, boundedUnresolvedRequests: snapshot?.boundedUnresolvedRequests ?? null }); + } + return { schema: 'paperclip.runner-eval.control-campaign/v1', campaignId: manifest.campaignId, manifestSha256, + limitUsd: manifest.maxUsd, allocatedUsd, operatingHeldUsd: overheadUsd, knownUsd: usd(known), heldUsd: usd(committed - known), + committedUsd: usd(committed), overBudget: committed > units(manifest.maxUsd), incompleteAttempts: attempts.filter(attempt => attempt.status === 'incomplete').length, + costStatus: attempts.some(attempt => attempt.status === 'incomplete') || committed > units(manifest.maxUsd) ? 'incomplete' : 'provisional', attempts }; +} +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + if (process.argv.length !== 3) throw new Error('Usage: report.mjs '); + const result = await summarizeControlledCampaign(await readJson(process.argv[2])); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); + if (result.incompleteAttempts || result.overBudget) process.exitCode = 2; +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs new file mode 100644 index 00000000000..761b9e077d1 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -0,0 +1,163 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { lstat, mkdir, open, readFile, realpath, rename, writeFile } from 'node:fs/promises'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs'; + +async function atomicJson(file, value) { + const temporary = `${file}.tmp`; + const handle = await open(temporary, 'wx', 0o600); + try { await handle.writeFile(JSON.stringify(value, null, 2) + '\n'); await handle.sync(); } + finally { await handle.close(); } + await rename(temporary, file); +} +const absent = pid => { try { process.kill(pid, 0); return false; } catch (error) { return error.code === 'ESRCH'; } }; +export const attemptDirectory = (manifest, attemptId) => join(manifest.outputDirectory, sha256(attemptId)); + +async function runChild(input) { + const child = spawn(process.execPath, [input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], { + cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'], + }); + let stopped = null, force; + const output = { stdout: [], stderr: [] }, sizes = { stdout: 0, stderr: 0 }; + const stop = reason => { + if (stopped) return; + stopped = reason; + void input.closeGateway().catch(() => {}); + try { process.kill(-child.pid, 'SIGTERM'); } catch {} + force = setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }, 5000); + }; + for (const stream of ['stdout', 'stderr']) child[stream].on('data', bytes => { + const available = Math.max(0, 1_048_576 - sizes[stream]); + output[stream].push(bytes.subarray(0, available)); sizes[stream] += bytes.length; + if (sizes[stream] > 1_048_576) stop('output_limit'); + }); + const onAbort = () => stop('controller_cancelled'); + input.signal?.addEventListener('abort', onAbort, { once: true }); + const timeout = setTimeout(() => stop('attempt_deadline'), Math.max(1, input.deadlineAt - Date.now())); + if (input.signal?.aborted) onAbort(); + try { + const [code, signal] = await once(child, 'exit'); + return { code, signal, pid: child.pid, stopped, stdout: Buffer.concat(output.stdout).toString(), stderr: Buffer.concat(output.stderr).toString() }; + } finally { + clearTimeout(timeout); clearTimeout(force); input.signal?.removeEventListener('abort', onAbort); + } +} + +export async function runControlledEval({ manifestPath, requestPath, outputPath, environment = process.env, signal, transport }) { + if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Controlled eval process ownership requires Linux or macOS'); + signal?.throwIfAborted(); + const { manifest, manifestSha256 } = validateManifest(await readJson(manifestPath)); + const request = await readJson(requestPath, 8_388_608), attempt = bindRequest(manifest, request); + try { await lstat(outputPath); throw new Error('Controlled output already exists'); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + if (request.session.workingDirectory) { + const suffix = relative(resolve(request.session.workingDirectory), resolve(manifest.outputDirectory)); + if (!suffix || suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) throw new Error('Controller evidence must be outside the disposable workspace'); + } + for (const artifact of [manifest.runnerCli, manifest.runnerPackage, manifest.runnerd]) { + if (sha256(await readFile(artifact.path)) !== artifact.sha256) throw new Error('Controlled eval executable provenance changed'); + } + const provider = attempt.provider === 'codex' ? 'openai' : 'anthropic'; + const keyName = manifest.route === 'braintrust' ? 'BRAINTRUST_API_KEY' : provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY'; + const key = environment[keyName]; + if (!key || /[\r\n]/.test(key)) throw new Error('Missing controlled eval upstream credential'); + const model = manifest.models.find(item => item.provider === provider && item.price.model === attempt.model); + const root = attemptDirectory(manifest, attempt.attemptId); + await mkdir(manifest.outputDirectory, { recursive: true, mode: 0o700 }); + if (request.session.workingDirectory) { + const suffix = relative(await realpath(request.session.workingDirectory), await realpath(manifest.outputDirectory)); + if (!suffix || suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) throw new Error('Controller evidence resolves inside the disposable workspace'); + } + await mkdir(root, { mode: 0o700 }); + const started = Date.now(), startedAt = new Date(started).toISOString(); + const deadlineAt = Math.min(Date.parse(manifest.deadlineAt), started + attempt.timeoutMs); + const limits = { campaignId: manifest.campaignId, maxUsd: attempt.maxUsd, deadlineAt: new Date(deadlineAt).toISOString(), + maxRequests: attempt.maxRequests, maxConcurrent: attempt.maxConcurrent, maxAttemptsPerExecution: 1 }; + await atomicJson(join(root, 'admission.json'), { schema: 'paperclip.runner-eval.admission/v1', campaignId: manifest.campaignId, + manifestSha256, attempt, limits, startedAt }); + const retainedRequestPath = join(root, 'request.json'); + await writeFile(retainedRequestPath, JSON.stringify(request) + '\n', { flag: 'wx', mode: 0o600 }); + await mkdir(join(root, 'receipts'), { mode: 0o700 }); + await mkdir(join(root, 'home'), { mode: 0o700 }); + await mkdir(join(root, 'tmp'), { mode: 0o700 }); + const ledgerPath = join(root, 'ledger.jsonl'); + const ledger = await costRuntime.openEvalCostLedger(ledgerPath, limits); + let gateway, child, artifact, failure = null, cleanupVerified = false, secretExcluded = true; + let providerArtifactSha256; + const secrets = [key]; + const redact = raw => secrets.reduce((text, secret) => text.split(secret).join('[redacted]'), raw); + try { + await ledger.admitAttempt(attempt.executionId, attempt.attemptId); + gateway = await costRuntime.startEvalGateway({ ledger, identity: { executionId: attempt.executionId, attemptId: attempt.attemptId, phase: 'agent' }, + route: { ...model, mode: manifest.route, projectId: manifest.projectId, endpointName: provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY', apiKey: key }, + deadlineAt, transport, persist: receipt => atomicJson(join(root, 'receipts', `${receipt.requestId}.json`), receipt), + }); + secrets.push(gateway.token); + const descriptor = { schema: 'paperclip.runner-eval.gateway/v1', provider, model: attempt.model, baseUrl: gateway.baseUrl, + campaignId: manifest.campaignId, executionId: attempt.executionId, attemptId: attempt.attemptId, upstreamRoute: manifest.route }; + const descriptorPath = join(root, 'gateway.json'), descriptorRaw = JSON.stringify(descriptor) + '\n'; + await writeFile(descriptorPath, descriptorRaw, { flag: 'wx', mode: 0o600 }); + const childEnvironment = Object.fromEntries(['PATH', 'LANG', 'LANGUAGE', 'LC_ALL', 'TZ'].flatMap(name => environment[name] === undefined ? [] : [[name, environment[name]]])); + Object.assign(childEnvironment, { HOME: join(root, 'home'), TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: gateway.token }); + const artifactPath = join(root, 'provider-artifact.json'); + signal?.throwIfAborted(); + if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn'); + child = await runChild({ cli: manifest.runnerCli.path, requestPath: retainedRequestPath, artifactPath, descriptorPath, + cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => gateway.close() }); + for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); + const raw = await readFile(artifactPath, 'utf8'); + if (Buffer.byteLength(raw) > 16_777_216) throw new Error('Provider artifact exceeds its byte limit'); + secretExcluded = secrets.every(secret => !raw.includes(secret)); + if (!secretExcluded) await writeFile(artifactPath, redact(raw), { mode: 0o600 }); + providerArtifactSha256 = sha256(redact(raw)); + artifact = JSON.parse(redact(raw)); + const expected = { mode: 'controller_gateway', provider, declaredUpstreamRoute: manifest.route, campaignId: manifest.campaignId, + executionId: attempt.executionId, attemptId: attempt.attemptId, descriptorSha256: sha256(descriptorRaw) }; + if (canonical(artifact.requestRouting) !== canonical(expected) || artifact.attemptId !== attempt.attemptId || artifact.requestedModel !== attempt.model) throw new Error('Provider did not retain the assigned route'); + const processInfo = artifact.snapshot?.process; + const pids = [...new Set([child.pid, processInfo?.runnerPid, processInfo?.providerPid, processInfo?.sidecarPid, processInfo?.agentPid].filter(pid => Number.isSafeInteger(pid) && pid > 0))]; + cleanupVerified = !child.stopped && artifact.snapshot?.status === 'closed' && processInfo?.runnerExited === true + && Number.isSafeInteger(processInfo.runnerPid) && processInfo.runnerPid > 0 + && Number.isSafeInteger(processInfo.providerPid) && processInfo.providerPid > 0 && pids.every(absent); + if (!cleanupVerified || !secretExcluded) throw new Error('Controlled provider cleanup or credential exclusion is unverified'); + } catch { + failure = 'controlled_attempt_incomplete'; + } finally { + try { await gateway?.close(); } catch { failure = 'gateway_cleanup_unverified'; } + try { await ledger.close(); } catch { failure = 'ledger_close_failed'; } + } + const snapshot = ledger.snapshot(); + if (snapshot.overBudget || snapshot.conflictingRequests || snapshot.inflight || snapshot.unknownRequests !== snapshot.boundedUnresolvedRequests) failure ??= 'request_accounting_unbounded'; + const ledgerSha256 = sha256(await readFile(ledgerPath)); + const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256, + executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256, + allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure, + child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null, + cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, accounting: snapshot }; + await atomicJson(join(root, 'result.json'), report); + const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, + requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }; + if (failure) Object.assign(resultArtifact, { accountingError: 'Controlled request accounting or cleanup is incomplete', + accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } }); + resultArtifact.controllerAccounting = report; + await mkdir(dirname(resolve(outputPath)), { recursive: true, mode: 0o700 }); + await writeFile(outputPath, JSON.stringify(resultArtifact, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + return { code: failure ? 2 : child?.code ?? 2, report }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const controller = new AbortController(); + const cancel = () => controller.abort(); + process.once('SIGINT', cancel); process.once('SIGTERM', cancel); + try { + const { values } = parseArgs({ options: { request: { type: 'string' }, output: { type: 'string' } } }); + if (!values.request || !values.output || !process.env.PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST) throw new Error('A controlled manifest, request and output are required'); + const result = await runControlledEval({ manifestPath: process.env.PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST, + requestPath: values.request, outputPath: values.output, signal: controller.signal }); + process.exitCode = result.code; + } catch { process.stderr.write('Controlled Runner eval admission or retention failed; inspect retained controller evidence.\n'); process.exitCode = 2; } + finally { process.removeListener('SIGINT', cancel); process.removeListener('SIGTERM', cancel); } +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts b/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts new file mode 100644 index 00000000000..38f45e0839a --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts @@ -0,0 +1,3 @@ +export { EvalSpendState, openEvalCostLedger, readEvalCostLedger, validEvalLedgerId } from '../../../../tests/runner-e2e/cost-ledger.js'; +export { evalRequestReservation, startEvalGateway } from '../../../../tests/runner-e2e/eval-gateway.js'; +export { EVAL_OVERHEAD_COMPONENTS, validateOperatingOverhead } from '../../../../tests/runner-e2e/operating-overhead.js'; From 966f3a2b2795eaa4d8b9d6a3b9c6abacafe25dc1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 10:07:52 -0700 Subject: [PATCH 2/9] fix(evals): qualify local Codex tool namespaces Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 4 ++++ .../src/cli/eval-gateway-runtime.test.ts | 1 + .../src/cli/eval-gateway-runtime.ts | 1 + tests/runner-e2e/eval-gateway.test.ts | 23 +++++++++++++++++++ tests/runner-e2e/eval-gateway.ts | 8 +++++-- 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index a77870cc501..cf08f52de63 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -494,3 +494,7 @@ checksum-verified Grok binary before packaging the portable runtime. Targets without the Grok package keep their existing build behavior. This protocol workflow uses the explicitly selected API key; subscription credentials are not delivered by it. + +The controlled Codex profile disables web search. The shared proxy accepts local +function/custom tool namespaces and still rejects hosted tools, including any +hidden inside a namespace. These routes do not qualify web-search charges. diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts index cd1cb1e03e1..0ef09abd57f 100644 --- a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts +++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts @@ -73,6 +73,7 @@ it.each([["codex", undefined], ["acpx", "claude"]])("routes %s/%s with only its expect(config).toContain('env_key = "PAPERCLIP_AI_PROVIDER_KEY"'); expect(config).toContain("requires_openai_auth = false"); expect(config).toContain("shell_snapshot = false"); + expect(config).toContain('web_search = "disabled"'); expect(config).not.toContain(token); expect((await stat(join(env.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600); } diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts index 0d32670c3a6..a4e27f9c180 100644 --- a/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts +++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts @@ -70,6 +70,7 @@ export async function prepareEvalGatewayRuntime( environment.PAPERCLIP_AI_PROVIDER_KEY = token; await writeFile(join(codexHome, "config.toml"), [ 'model_provider = "paperclip_eval"', + 'web_search = "disabled"', `model = ${JSON.stringify(request.model)}`, '[model_providers.paperclip_eval]', 'name = "Paperclip eval controller"', diff --git a/tests/runner-e2e/eval-gateway.test.ts b/tests/runner-e2e/eval-gateway.test.ts index 6e38d356bea..aefca15a9e3 100644 --- a/tests/runner-e2e/eval-gateway.test.ts +++ b/tests/runner-e2e/eval-gateway.test.ts @@ -121,6 +121,29 @@ it.each([ await expect(forwardEvalRequest({ ...f, payload, transport })).rejects.toThrow(); expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0); }); +it("admits local function and custom tools inside a Codex namespace", async () => { + const f = await fixture(); + const tools = [{ type: "namespace", name: "local", tools: [ + { type: "function", name: "read_task", parameters: { type: "object" } }, + { type: "custom", name: "apply_patch" }, + ] }]; + const transport = vi.fn(async (_url, init) => { + expect(JSON.parse(init!.body as string).tools).toEqual(tools); + return response(); + }); + const result = await forwardEvalRequest({ ...f, payload: { ...f.payload, tools }, transport }); + expect(result.outcome).toBe("succeeded"); + expect(transport).toHaveBeenCalledOnce(); +}); +it.each([ + { type: "namespace", name: "local", tools: [{ type: "web_search" }] }, + { type: "namespace", name: "local", tools: [{ type: "namespace", name: "nested", tools: [{ type: "web_search" }] }] }, + { type: "namespace", name: "local" }, +])("rejects unqualified tools inside namespaces before reservation", async tool => { + const f = await fixture(), transport = vi.fn(); + await expect(forwardEvalRequest({ ...f, payload: { ...f.payload, tools: [tool] }, transport })).rejects.toThrow(); + expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0); +}); it("does not dispatch when the private pre-request receipt cannot be persisted", async () => { const f = await fixture(), transport = vi.fn(); await expect(forwardEvalRequest({ ...f, persist: async () => { throw new Error("disk unavailable"); }, transport })).rejects.toThrow("disk unavailable"); diff --git a/tests/runner-e2e/eval-gateway.ts b/tests/runner-e2e/eval-gateway.ts index ecd9b55ede8..d7cc20ab0db 100644 --- a/tests/runner-e2e/eval-gateway.ts +++ b/tests/runner-e2e/eval-gateway.ts @@ -71,8 +71,12 @@ function boundedPayload(value: unknown, route: EvalGatewayRoute) { if (body.background === true || body.mcp_servers !== undefined || body.service_tier !== undefined && !["auto", "default", "standard_only"].includes(String(body.service_tier))) throw new EvalRequestError("unqualified_billing_mode"); if (body.speed !== undefined && body.speed !== "standard" || body.inference_geo !== undefined) throw new EvalRequestError("unqualified_billing_mode"); if (body.tools !== undefined && (!Array.isArray(body.tools) || body.tools.some(tool => { - const type = row(tool).type; - return route.provider === "openai" ? !["function", "custom"].includes(String(type)) : type !== undefined && type !== "custom"; + const definition = row(tool), type = definition.type; + if (route.provider !== "openai") return type !== undefined && type !== "custom"; + if (type === "namespace") return typeof definition.name !== "string" || !definition.name + || !Array.isArray(definition.tools) || !definition.tools.length + || definition.tools.some(nested => !["function", "custom"].includes(String(row(nested).type))); + return !["function", "custom"].includes(String(type)); }))) throw new EvalRequestError("unqualified_hosted_tool"); const inspect = (item: unknown): void => { if (Array.isArray(item)) { item.forEach(inspect); return; } From e796f677105a1904f2915c732f444432ea5152a2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 10:19:15 -0700 Subject: [PATCH 3/9] fix(evals): retain accounting proof and pin launched bytes Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 12 ++++++ .../controlled-evals/controller.test.mjs | 41 ++++++++++++++++++- .../scripts/controlled-evals/report.mjs | 27 +++++++++--- .../scripts/controlled-evals/run.mjs | 36 ++++++++++++---- 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index cf08f52de63..4319d7f76c6 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -498,3 +498,15 @@ not delivered by it. The controlled Codex profile disables web search. The shared proxy accepts local function/custom tool namespaces and still rejects hosted tools, including any hidden inside a namespace. These routes do not qualify web-search charges. + +The controller stages the exact verified CLI and daemon bytes in its private +attempt directory. A Node module-resolution hook keeps the CLI's relative +imports pointed at the trusted source checkout; it does not reload the original +entry file. Installed dependencies still belong to that trusted checkout. The +staged copies are removed after verified process cleanup and kept if cleanup is +incomplete. This path requires the repository's Node 24.11+ runtime. + +Campaign reports verify the retained request binding and the digest of every +request receipt before reducing a cell's allocation. Missing admission, request, +or receipt evidence preserves known ledger charges and holds the rest of the +full allocation. diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index afa7e6bb6e0..bceb4a58568 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; @@ -198,3 +199,41 @@ test('existing outputs and physical workspace aliases are rejected before provid await assert.rejects(f.run(0, transport), /resolves inside/); assert.equal(calls, 0); }); + +for (const missing of ['request.json', 'admission.json', 'receipt']) test(`lost ${missing} keeps known charges and the full allocation`, async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const file = missing === 'receipt' ? join(root, 'receipts', (await readdir(join(root, 'receipts')))[0]) : join(root, missing); + await rm(file); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.knownUsd, '0.000068000'); + assert.equal(summary.committedUsd, '0.014000000'); + assert.equal(summary.incompleteAttempts, 1); +}); + +for (const changed of ['request', 'receipt']) test(`changed ${changed} evidence cannot release the allocation`, async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const file = changed === 'receipt' ? join(root, 'receipts', (await readdir(join(root, 'receipts')))[0]) : join(root, 'request.json'); + const value = JSON.parse(await readFile(file, 'utf8')); + if (changed === 'request') value.prompt = 'different request'; else value.costUsd = '0.000000000'; + await writeFile(file, JSON.stringify(value)); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.knownUsd, '0.000068000'); + assert.equal(summary.committedUsd, '0.014000000'); + assert.equal(summary.incompleteAttempts, 1); +}); + +for (const replaced of ['runnerCli', 'runnerd']) test(`launches verified bytes after the original ${replaced} path is replaced`, async t => { + const f = await fixture(t, { requests: 1 }); + await writeFile(f.manifest.runnerCli.path, childScript({ requests: 1 }) + "\nassert.equal(await readFile(request.runnerd.path, 'utf8'), 'synthetic daemon');\n"); + f.manifest.runnerCli.sha256 = sha256(await readFile(f.manifest.runnerCli.path)); await f.save(); + Object.defineProperty(f.environment, 'OPENAI_API_KEY', { get() { + writeFileSync(f.manifest[replaced].path, replaced === 'runnerCli' ? "throw new Error('replacement must not execute');" : 'replaced daemon'); + return 'synthetic-upstream-private'; + }}); + const result = await f.run(); + assert.equal(result.code, 0); assert.equal(result.report.cleanupVerified, true); + assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0); + await assert.rejects(readdir(join(attemptDirectory(f.manifest, 'attempt-0'), 'executables')), { code: 'ENOENT' }); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs index dcbd145a170..23e4f996036 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/report.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -1,7 +1,7 @@ import { readFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { canonical, costRuntime, readJson, sha256, units, usd, validateManifest } from './manifest.mjs'; +import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { attemptDirectory } from './run.mjs'; export async function summarizeControlledCampaign(value) { @@ -12,21 +12,36 @@ export async function summarizeControlledCampaign(value) { const root = attemptDirectory(manifest, allocation.attemptId); let snapshot, report, verified = false; try { - const admission = await readJson(join(root, 'admission.json')); const ledgerBytes = await readFile(join(root, 'ledger.jsonl')); const ledger = await costRuntime.readEvalCostLedger(join(root, 'ledger.jsonl')); const candidateSnapshot = ledger.state.snapshot(); + const ledgerDeadline = Date.parse(ledger.state.limits.deadlineAt); + const fixedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: ledger.state.limits.deadlineAt, + maxRequests: allocation.maxRequests, maxConcurrent: allocation.maxConcurrent, maxAttemptsPerExecution: 1 }; + if (canonical(ledger.state.limits) !== canonical(fixedLimits) || !Number.isFinite(ledgerDeadline) + || ledgerDeadline <= Date.parse(manifest.pricingVerifiedAt) || ledgerDeadline > Date.parse(manifest.deadlineAt) + || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }]) + || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger identity does not match its allocation'); + snapshot = candidateSnapshot; + const admission = await readJson(join(root, 'admission.json')); const started = Date.parse(admission.startedAt); const expectedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: new Date(Math.min(Date.parse(manifest.deadlineAt), started + allocation.timeoutMs)).toISOString(), maxRequests: allocation.maxRequests, maxConcurrent: allocation.maxConcurrent, maxAttemptsPerExecution: 1 }; if (admission.schema !== 'paperclip.runner-eval.admission/v1' || admission.manifestSha256 !== manifestSha256 || admission.campaignId !== manifest.campaignId || canonical(admission.attempt) !== canonical(allocation) - || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits) - || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }]) - || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger does not match its allocation'); - snapshot = candidateSnapshot; + || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits)) throw new Error('Ledger does not match its admission'); report = await readJson(join(root, 'result.json')); + const request = await readJson(join(root, 'request.json'), 8_388_608); + if (requestBindingSha256(request) !== allocation.requestSha256 || request.runnerd.sha256.replace(/^sha256:/, '') !== manifest.runnerd.sha256 + || canonical(Object.keys(report.receiptHashes ?? {}).sort()) !== canonical(snapshot.requests.map(item => item.requestId).sort())) throw new Error('Retained request evidence does not match'); + for (const item of snapshot.requests) { + if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(item.requestId)) throw new Error('Invalid retained receipt identity'); + const bytes = await readFile(join(root, 'receipts', `${item.requestId}.json`)); + const receipt = JSON.parse(bytes.toString('utf8')); + if (sha256(bytes) !== report.receiptHashes[item.requestId] || receipt.requestId !== item.requestId + || receipt.route !== manifest.route || receipt.costUsd !== item.costUsd) throw new Error('Retained receipt does not match its ledger'); + } const artifactBytes = await readFile(join(root, 'provider-artifact.json')); const artifact = JSON.parse(artifactBytes.toString('utf8')); const descriptorBytes = await readFile(join(root, 'gateway.json')); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 761b9e077d1..fc9fa6d217e 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process'; import { once } from 'node:events'; -import { lstat, mkdir, open, readFile, realpath, rename, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import { parseArgs } from 'node:util'; @@ -17,7 +17,7 @@ const absent = pid => { try { process.kill(pid, 0); return false; } catch (error export const attemptDirectory = (manifest, attemptId) => join(manifest.outputDirectory, sha256(attemptId)); async function runChild(input) { - const child = spawn(process.execPath, [input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], { + const child = spawn(process.execPath, ['--import', input.loader, input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], { cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); let stopped = null, force; @@ -57,8 +57,11 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, const suffix = relative(resolve(request.session.workingDirectory), resolve(manifest.outputDirectory)); if (!suffix || suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) throw new Error('Controller evidence must be outside the disposable workspace'); } - for (const artifact of [manifest.runnerCli, manifest.runnerPackage, manifest.runnerd]) { - if (sha256(await readFile(artifact.path)) !== artifact.sha256) throw new Error('Controlled eval executable provenance changed'); + const verified = {}; + for (const name of ['runnerCli', 'runnerPackage', 'runnerd']) { + const artifact = manifest[name], bytes = await readFile(artifact.path); + if (sha256(bytes) !== artifact.sha256) throw new Error('Controlled eval executable provenance changed'); + if (name !== 'runnerPackage') verified[name] = bytes; } const provider = attempt.provider === 'codex' ? 'openai' : 'anthropic'; const keyName = manifest.route === 'braintrust' ? 'BRAINTRUST_API_KEY' : provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY'; @@ -78,8 +81,19 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, maxRequests: attempt.maxRequests, maxConcurrent: attempt.maxConcurrent, maxAttemptsPerExecution: 1 }; await atomicJson(join(root, 'admission.json'), { schema: 'paperclip.runner-eval.admission/v1', campaignId: manifest.campaignId, manifestSha256, attempt, limits, startedAt }); + const executables = join(root, 'executables'); + await mkdir(executables, { mode: 0o700 }); + const stagedCli = join(executables, 'entry.mjs'), stagedRunnerd = join(executables, 'runnerd'), loader = join(executables, 'imports.mjs'); + await writeFile(stagedCli, verified.runnerCli, { flag: 'wx', mode: 0o400 }); + await writeFile(stagedRunnerd, verified.runnerd, { flag: 'wx', mode: 0o500 }); + const stagedUrl = JSON.stringify(pathToFileURL(stagedCli).href), originalUrl = JSON.stringify(pathToFileURL(manifest.runnerCli.path).href); + await writeFile(loader, `import { registerHooks } from 'node:module'; +registerHooks({ resolve(specifier, context, nextResolve) { + return nextResolve(specifier, context.parentURL === ${stagedUrl} ? { ...context, parentURL: ${originalUrl} } : context); +} });\n`, { flag: 'wx', mode: 0o400 }); const retainedRequestPath = join(root, 'request.json'); - await writeFile(retainedRequestPath, JSON.stringify(request) + '\n', { flag: 'wx', mode: 0o600 }); + const childRequest = structuredClone(request); childRequest.runnerd.path = stagedRunnerd; + await writeFile(retainedRequestPath, JSON.stringify(childRequest) + '\n', { flag: 'wx', mode: 0o600 }); await mkdir(join(root, 'receipts'), { mode: 0o700 }); await mkdir(join(root, 'home'), { mode: 0o700 }); await mkdir(join(root, 'tmp'), { mode: 0o700 }); @@ -105,7 +119,7 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, const artifactPath = join(root, 'provider-artifact.json'); signal?.throwIfAborted(); if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn'); - child = await runChild({ cli: manifest.runnerCli.path, requestPath: retainedRequestPath, artifactPath, descriptorPath, + child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => gateway.close() }); for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); const raw = await readFile(artifactPath, 'utf8'); @@ -129,14 +143,22 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, try { await gateway?.close(); } catch { failure = 'gateway_cleanup_unverified'; } try { await ledger.close(); } catch { failure = 'ledger_close_failed'; } } + if (cleanupVerified) { + try { await rm(executables, { recursive: true }); } + catch { cleanupVerified = false; failure ??= 'executable_cleanup_unverified'; } + } const snapshot = ledger.snapshot(); if (snapshot.overBudget || snapshot.conflictingRequests || snapshot.inflight || snapshot.unknownRequests !== snapshot.boundedUnresolvedRequests) failure ??= 'request_accounting_unbounded'; + const receiptHashes = {}; + try { + for (const request of snapshot.requests) receiptHashes[request.requestId] = sha256(await readFile(join(root, 'receipts', `${request.requestId}.json`))); + } catch { failure ??= 'request_receipt_missing'; } const ledgerSha256 = sha256(await readFile(ledgerPath)); const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256, executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256, allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure, child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null, - cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, accounting: snapshot }; + cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, receiptHashes, accounting: snapshot }; await atomicJson(join(root, 'result.json'), report); const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }; From 5201836602cfe7d45fd961632f99d2a96b813d77 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 10:20:59 -0700 Subject: [PATCH 4/9] fix(evals): resolve staged CLI imports through canonical paths Co-Authored-By: Paperclip --- .../scripts/controlled-evals/controller.test.mjs | 13 +++++++++++++ .../scripts/controlled-evals/run.mjs | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index bceb4a58568..92ed44792c9 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -237,3 +237,16 @@ for (const replaced of ['runnerCli', 'runnerd']) test(`launches verified bytes a assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0); await assert.rejects(readdir(join(attemptDirectory(f.manifest, 'attempt-0'), 'executables')), { code: 'ENOENT' }); }); + +test('staged CLI relative imports survive a symlinked evidence directory', async t => { + const f = await fixture(t, { requests: 1 }); + await writeFile(join(f.root, 'helper.mjs'), "export const marker = 'trusted sibling';\n"); + await writeFile(f.manifest.runnerCli.path, "import { marker } from './helper.mjs';\n" + childScript({ requests: 1 }) + "\nassert.equal(marker, 'trusted sibling');\n"); + f.manifest.runnerCli.sha256 = sha256(await readFile(f.manifest.runnerCli.path)); + const alias = join(f.root, 'evidence-alias'); + await mkdir(join(f.root, 'evidence-real')); await symlink(join(f.root, 'evidence-real'), alias); + f.manifest.outputDirectory = join(alias, 'controller'); await f.save(); + const result = await f.run(); + assert.equal(result.code, 0); + assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index fc9fa6d217e..07740fe3a29 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -57,6 +57,7 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, const suffix = relative(resolve(request.session.workingDirectory), resolve(manifest.outputDirectory)); if (!suffix || suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) throw new Error('Controller evidence must be outside the disposable workspace'); } + const originalCliUrl = pathToFileURL(await realpath(manifest.runnerCli.path)).href; const verified = {}; for (const name of ['runnerCli', 'runnerPackage', 'runnerd']) { const artifact = manifest[name], bytes = await readFile(artifact.path); @@ -86,7 +87,7 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, const stagedCli = join(executables, 'entry.mjs'), stagedRunnerd = join(executables, 'runnerd'), loader = join(executables, 'imports.mjs'); await writeFile(stagedCli, verified.runnerCli, { flag: 'wx', mode: 0o400 }); await writeFile(stagedRunnerd, verified.runnerd, { flag: 'wx', mode: 0o500 }); - const stagedUrl = JSON.stringify(pathToFileURL(stagedCli).href), originalUrl = JSON.stringify(pathToFileURL(manifest.runnerCli.path).href); + const stagedUrl = JSON.stringify(pathToFileURL(await realpath(stagedCli)).href), originalUrl = JSON.stringify(originalCliUrl); await writeFile(loader, `import { registerHooks } from 'node:module'; registerHooks({ resolve(specifier, context, nextResolve) { return nextResolve(specifier, context.parentURL === ${stagedUrl} ? { ...context, parentURL: ${originalUrl} } : context); From 4781487215116afd10a415d4db53b1ec8e004dd5 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 10:23:49 -0700 Subject: [PATCH 5/9] fix(evals): finish process group cancellation after CLI exit Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 4 +++ .../controlled-evals/controller.test.mjs | 27 +++++++++++++++++++ .../scripts/controlled-evals/run.mjs | 11 ++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 4319d7f76c6..b066a6de31f 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -510,3 +510,7 @@ Campaign reports verify the retained request binding and the digest of every request receipt before reducing a cell's allocation. Missing admission, request, or receipt evidence preserves known ledger charges and holds the rest of the full allocation. + +On cancellation, the controller keeps the scheduled process-group SIGKILL until +any remaining owned descendants receive it, even if the CLI exits first. A +forced stop still leaves cleanup unverified and retains the full allocation. diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 92ed44792c9..a4af19aa000 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -11,9 +11,17 @@ import { summarizeControlledCampaign } from './report.mjs'; const childScript = options => ` import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; +import {spawn} from 'node:child_process'; +import {once} from 'node:events'; import {readFile,writeFile} from 'node:fs/promises'; const args=new Map();for(let i=2;i{});process.stdout.write('ready');setInterval(()=>{},1000)"],{stdio:['ignore','pipe','ignore']}); + await once(descendant.stdout,'data'); + await writeFile(request.session.workingDirectory+'/descendant.pid',String(descendant.pid)); +} + const descriptorRaw=await readFile(args.get('--gateway-config'),'utf8'),descriptor=JSON.parse(descriptorRaw); for(const key of ['OPENAI_API_KEY','ANTHROPIC_API_KEY','BRAINTRUST_API_KEY','GH_TOKEN','NODE_OPTIONS','PYTHONPATH'])assert.equal(process.env[key],undefined); const statuses=[]; @@ -250,3 +258,22 @@ test('staged CLI relative imports survive a symlinked evidence directory', async assert.equal(result.code, 0); assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0); }); + +test('cancellation still kills an owned resistant descendant after the CLI exits', async t => { + const f = await fixture(t, { requests: 1, waitForever: true, resistantDescendant: true }); + f.manifest.attempts[0].timeoutMs = 15000; await f.save(); + const abort = new AbortController(); let ready; + const started = new Promise(resolve => { ready = resolve; }); + const running = f.run(0, async () => { ready(); return success(); }, abort.signal); + await started; + const pid = Number(await readFile(join(f.requests[0].request.session.workingDirectory, 'descendant.pid'), 'utf8')); + t.after(() => { try { process.kill(pid, 'SIGKILL'); } catch {} }); + abort.abort(); + const result = await running; + assert.equal(result.report.child.stopped, 'controller_cancelled'); + const absent = () => { try { process.kill(pid, 0); return false; } catch (error) { return error.code === 'ESRCH'; } }; + for (let retry = 0; retry < 100 && !absent(); retry++) await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal(absent(), true); + assert.equal(result.report.cleanupVerified, false); + assert.equal((await summarizeControlledCampaign(f.manifest)).committedUsd, '0.014000000'); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 07740fe3a29..42fda084873 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -20,14 +20,16 @@ async function runChild(input) { const child = spawn(process.execPath, ['--import', input.loader, input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], { cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); - let stopped = null, force; + let stopped = null, force, escalation; const output = { stdout: [], stderr: [] }, sizes = { stdout: 0, stderr: 0 }; const stop = reason => { if (stopped) return; stopped = reason; void input.closeGateway().catch(() => {}); try { process.kill(-child.pid, 'SIGTERM'); } catch {} - force = setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }, 5000); + escalation = new Promise(resolve => { + force = setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} finally { resolve(); } }, 5000); + }); }; for (const stream of ['stdout', 'stderr']) child[stream].on('data', bytes => { const available = Math.max(0, 1_048_576 - sizes[stream]); @@ -40,6 +42,11 @@ async function runChild(input) { if (input.signal?.aborted) onAbort(); try { const [code, signal] = await once(child, 'exit'); + if (stopped) { + let groupPresent = true; + try { process.kill(-child.pid, 0); } catch (error) { if (error.code === 'ESRCH') groupPresent = false; } + if (groupPresent) await escalation; + } return { code, signal, pid: child.pid, stopped, stdout: Buffer.concat(output.stdout).toString(), stderr: Buffer.concat(output.stderr).toString() }; } finally { clearTimeout(timeout); clearTimeout(force); input.signal?.removeEventListener('abort', onAbort); From 99e07e675de163e69999f9e2e0cb3f3b6427ca6b Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 11:13:45 -0700 Subject: [PATCH 6/9] feat(evals): retain portable Runner controller evidence Replay archived ledgers through the existing verifier, preserve conservative holds, and keep private bundles out of public reports. Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 48 ++++- .../scripts/controlled-evals/archive.mjs | 173 ++++++++++++++++++ .../controlled-evals/controller.test.mjs | 125 +++++++++++++ .../scripts/controlled-evals/report.mjs | 29 ++- .../controlled-evals/shared-cost-runtime.ts | 2 +- .../scripts/runner-protocol-eval-campaign.mjs | 31 ++++ tests/runner-e2e/cost-ledger.ts | 5 +- 7 files changed, 399 insertions(+), 14 deletions(-) create mode 100644 packages/paperclip-runner/scripts/controlled-evals/archive.mjs diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index b066a6de31f..d5fbb0e65f6 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -215,15 +215,57 @@ Each result retains `controllerAccounting` alongside the original provider outcome and usage. Retain the durable controller directory as private evidence: its admission, request, gateway descriptor, request receipts, ledger, original provider artifact, and final result are required for reconciliation. The -private program's existing immutable attempt copy includes the result metadata; -its current default history publisher does not archive this separate directory. -Do not infer receipt retention from that metadata alone. +private `controller-evidence.json` bundle can travel with the immutable attempt. +Set `PAPERCLIP_RUNNER_EVAL_ARCHIVE_CLI` to the sibling `archive.mjs` when using a +private eval program with controller retention support. That program validates +the request before launch, captures the bundle before finalizing the attempt, +and marks missing or incomplete capture as a non-retryable accounting failure. +It must use zero infrastructure retries. Older private programs do not capture +this bundle; result metadata alone is not proof of retention. ```sh node packages/paperclip-runner/scripts/controlled-evals/report.mjs \ /private/path/controller-manifest.json > /private/path/controller-report.json ``` +To retain or inspect the evidence independently of the original worker: + +```sh +node packages/paperclip-runner/scripts/controlled-evals/archive.mjs pack \ + /private/path/controller-manifest.json attempt-id \ + /private/path/runs/attempt-id/controller-evidence.json +node packages/paperclip-runner/scripts/controlled-evals/archive.mjs report-attempts \ + /private/path/runs/attempt-id +``` + +`pack` is exclusive and never overwrites evidence. The private bundle contains +the full manifest and exact bytes of the admission, request, gateway descriptor, +ledger, receipts, provider artifact, result, and bounded stdout/stderr. It omits +executables, private homes, and caches. Files carry byte lengths and SHA-256 +hashes. These detect corruption; they are not signatures or proof against a +host that can rewrite all evidence. Known configured credential values are +checked before encoding and after decoding. Keep bundles private. + +The reader accepts only bounded regular files and fixed evidence paths. A +bundle is limited to 64 MiB of decoded content plus encoding overhead; this +storage-format limit does not qualify a vendor charge or operating-cost bound. +`report` accepts standalone bundle files; `report-attempts` additionally binds +them to the surrounding immutable request and artifact. Both work after the +original worker directory is gone, using the same hash-chain ledger verifier. +They group attempts by manifest, count operating holds once, deduplicate exact +copies, reject conflicting copies, and keep full allocations for missing +selected attempts. They never consult old worker paths to fill archive gaps. + +The campaign collector preserves the bundle in private runs and returns its +independently replayed totals as `controlledCosts`. For this collector, each +manifest `executionId` must equal its catalog `cellId`, and the complete +manifest selection must match the catalog. Extra attempts are rejected. +Existing `costs` remain +legacy provider-usage observations; do not add them to `controlledCosts`. +The public attempt projection omits bundles and raw `controllerAccounting`. +Private reports label ledger-known amounts and conservative holds as provisional +or incomplete; a missing verifier or invalid evidence cannot become a zero cost. + The report recomputes ledger costs, checks their attempt/manifest bindings, and preserves known partial charges. It reports operating costs as held and total costs as provisional when evidence is intact, or incomplete otherwise. It does diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs new file mode 100644 index 00000000000..ffea93bee58 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -0,0 +1,173 @@ +import { constants } from 'node:fs'; +import { lstat, open, readdir, writeFile } from 'node:fs/promises'; +import { basename, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; +import { attemptDirectory } from './run.mjs'; +import { summarizeControlledCampaign } from './report.mjs'; + +const MiB = 1_048_576; +const MAX_RAW_BYTES = 64 * MiB; +const MAX_ARCHIVE_BYTES = 96 * MiB; +const FILE_LIMITS = new Map([ + ['admission.json', MiB], ['request.json', 8 * MiB], ['gateway.json', 8192], + ['ledger.jsonl', 16 * MiB], ['result.json', MiB], ['provider-artifact.json', 16 * MiB], + ['stdout.log', MiB], ['stderr.log', MiB], +]); +const RECEIPT = /^receipts\/[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\.json$/; +const limitFor = name => FILE_LIMITS.get(name) ?? (RECEIPT.test(name) ? MiB : 0); + +function secretValues({ environment = process.env, secrets = [] } = {}) { + return [...new Set([...secrets, ...Object.entries(environment) + .filter(([name]) => /(?:API_KEY|TOKEN|SECRET|PASSWORD|ACCESS_KEY_ID)$/.test(name)) + .map(([, value]) => value)].filter(value => typeof value === 'string' && value.length > 0))]; +} +function excludeSecrets(bytes, secrets) { + if (secrets.some(secret => bytes.includes(Buffer.from(secret)))) throw new Error('Controller evidence contains a configured credential'); +} +async function readRegularFile(path, maximum) { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > maximum) throw new Error('Controller evidence is not a bounded regular file'); + const bytes = Buffer.alloc(stat.size + 1); + let length = 0; + while (length < bytes.length) { + const { bytesRead } = await handle.read(bytes, length, bytes.length - length, null); + if (!bytesRead) break; + length += bytesRead; + } + if (length !== stat.size) throw new Error('Controller evidence changed while being read'); + return bytes.subarray(0, length); + } finally { await handle.close(); } +} +async function realDirectory(path) { + try { + const stat = await lstat(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('Controller evidence directory is not a real directory'); + return true; + } catch (error) { if (error.code === 'ENOENT') return false; throw error; } +} + +export async function packControlledAttempt({ manifest: value, attemptId, ...options }) { + const { manifest, manifestSha256 } = validateManifest(value, Date.now(), { admission: false }); + const allocation = manifest.attempts.find(attempt => attempt.attemptId === attemptId); + if (!allocation) throw new Error('Attempt is absent from the controller manifest'); + const secrets = secretValues(options); + const manifestBytes = Buffer.from(canonical(manifest)); + if (manifestBytes.length > MiB) throw new Error('Controller manifest exceeds its archive limit'); + excludeSecrets(manifestBytes, secrets); + const root = attemptDirectory(manifest, attemptId), files = {}; + let total = manifestBytes.length; + if (await realDirectory(root)) { + const names = [...FILE_LIMITS.keys()]; + if (await realDirectory(join(root, 'receipts'))) { + for (const name of await readdir(join(root, 'receipts'))) { + if (RECEIPT.test(`receipts/${name.replace(/\.tmp$/, '')}`) && name.endsWith('.tmp')) continue; + if (!RECEIPT.test(`receipts/${name}`)) throw new Error('Unexpected controller receipt path'); + names.push(`receipts/${name}`); + } + } + if (names.length > allocation.maxRequests + FILE_LIMITS.size) throw new Error('Too many controller receipts'); + for (const name of names.sort()) { + let bytes; + try { bytes = await readRegularFile(join(root, name), limitFor(name)); } + catch (error) { if (error.code === 'ENOENT') continue; throw error; } + total += bytes.length; + if (total > MAX_RAW_BYTES) throw new Error('Controller evidence exceeds its archive limit'); + excludeSecrets(bytes, secrets); + files[name] = { sizeBytes: bytes.length, sha256: sha256(bytes), base64: bytes.toString('base64') }; + } + } + return { schema: 'paperclip.runner-eval.control-evidence/v1', manifest, manifestSha256, attemptId, files }; +} + +export function decodeControlledEvidence(value, options) { + fields(value, ['schema', 'manifest', 'manifestSha256', 'attemptId', 'files']); + const { manifest, manifestSha256 } = validateManifest(value.manifest, Date.now(), { admission: false }); + const allocation = manifest.attempts.find(attempt => attempt.attemptId === value.attemptId); + if (value.schema !== 'paperclip.runner-eval.control-evidence/v1' || value.manifestSha256 !== manifestSha256 || !allocation) throw new Error('Controller archive identity mismatch'); + fields(value.files, Object.keys(value.files ?? {})); + const entries = Object.entries(value.files); + if (entries.length > allocation.maxRequests + FILE_LIMITS.size) throw new Error('Too many controller evidence files'); + const secrets = secretValues(options), manifestBytes = Buffer.from(canonical(manifest)); + if (manifestBytes.length > MiB) throw new Error('Controller manifest exceeds its archive limit'); + excludeSecrets(manifestBytes, secrets); + let total = manifestBytes.length; + const files = new Map(); + for (const [name, file] of entries) { + const limit = limitFor(name); + fields(file, ['sizeBytes', 'sha256', 'base64']); + if (!limit || !Number.isSafeInteger(file.sizeBytes) || file.sizeBytes < 0 || file.sizeBytes > limit + || typeof file.base64 !== 'string' || file.base64.length !== 4 * Math.ceil(file.sizeBytes / 3) + || typeof file.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(file.sha256)) throw new Error('Invalid controller evidence file'); + total += file.sizeBytes; + if (total > MAX_RAW_BYTES) throw new Error('Controller evidence exceeds its archive limit'); + const bytes = Buffer.from(file.base64, 'base64'); + if (bytes.length !== file.sizeBytes || bytes.toString('base64') !== file.base64 || sha256(bytes) !== file.sha256) throw new Error('Controller evidence digest mismatch'); + excludeSecrets(bytes, secrets); + files.set(name, bytes); + } + return { manifest, manifestSha256, attemptId: value.attemptId, files }; +} + +export async function readControlledEvidence(path, options) { + const value = JSON.parse((await readRegularFile(path, MAX_ARCHIVE_BYTES)).toString('utf8')); + decodeControlledEvidence(value, options); + return value; +} + +export async function readAttemptControlledEvidence(directory, options) { + const bundle = await readControlledEvidence(join(directory, 'controller-evidence.json'), options); + const artifact = JSON.parse((await readRegularFile(join(directory, 'artifact.json'), 16 * MiB)).toString('utf8')); + const request = JSON.parse((await readRegularFile(join(directory, 'request.json'), 8 * MiB)).toString('utf8')); + const allocation = bundle.manifest.attempts.find(attempt => attempt.attemptId === bundle.attemptId); + if (basename(resolve(directory)) !== bundle.attemptId || artifact.attemptId !== bundle.attemptId + || requestBindingSha256(request) !== allocation.requestSha256 + || artifact.controllerAccounting && artifact.controllerAccounting.manifestSha256 !== bundle.manifestSha256) { + throw new Error('Immutable attempt does not match its controller archive'); + } + return bundle; +} + +export async function summarizeControlledEvidence(bundles, options) { + if (!Array.isArray(bundles) || !bundles.length) throw new Error('No controller evidence supplied'); + const groups = new Map(); + for (const bundle of bundles) { + const decoded = decodeControlledEvidence(bundle, options); + let group = groups.get(decoded.manifest.campaignId); + if (group && group.manifestSha256 !== decoded.manifestSha256) throw new Error('Conflicting controller campaign manifests'); + if (!group) { + group = { ...decoded, filesByAttempt: new Map(), digests: new Map() }; + groups.set(decoded.manifest.campaignId, group); + } + const digest = sha256(canonical(bundle)); + if (group.digests.has(decoded.attemptId) && group.digests.get(decoded.attemptId) !== digest) throw new Error('Conflicting controller attempt archives'); + group.digests.set(decoded.attemptId, digest); + group.filesByAttempt.set(decoded.attemptId, decoded.files); + } + const campaigns = []; + for (const group of [...groups.values()].sort((a, b) => a.manifest.campaignId.localeCompare(b.manifest.campaignId))) { + campaigns.push(await summarizeControlledCampaign(group.manifest, { filesByAttempt: group.filesByAttempt })); + } + const sum = field => usd(campaigns.reduce((total, campaign) => total + units(campaign[field]), 0n)); + return { schema: 'paperclip.runner-eval.control-history/v1', costStatus: campaigns.some(c => c.costStatus === 'incomplete') ? 'incomplete' : 'provisional', + knownUsd: sum('knownUsd'), heldUsd: sum('heldUsd'), committedUsd: sum('committedUsd'), operatingHeldUsd: sum('operatingHeldUsd'), + incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const [command, ...args] = process.argv.slice(2); + if (command === 'verify' && args.length === 2) { + const { manifest } = validateManifest(await readJson(args[0])); + bindRequest(manifest, await readJson(args[1], 8 * MiB)); + } else if (command === 'pack' && args.length === 3) { + const bundle = await packControlledAttempt({ manifest: await readJson(args[0]), attemptId: args[1] }); + await writeFile(args[2], JSON.stringify(bundle) + '\n', { flag: 'wx', mode: 0o600 }); + } else if (['report', 'report-attempts'].includes(command) && args.length) { + const read = command === 'report' ? readControlledEvidence : readAttemptControlledEvidence; + const result = await summarizeControlledEvidence(await Promise.all(args.map(path => read(path)))); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); + if (result.costStatus === 'incomplete') process.exitCode = 2; + } else throw new Error('Usage: archive.mjs verify | pack | report ... | report-attempts ...'); +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index a4af19aa000..5fac90ff3b1 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -7,6 +7,8 @@ import { join } from 'node:path'; import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; import { attemptDirectory, runControlledEval } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; +import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; +import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; const childScript = options => ` import assert from 'node:assert/strict'; @@ -277,3 +279,126 @@ test('cancellation still kills an owned resistant descendant after the CLI exits assert.equal(result.report.cleanupVerified, false); assert.equal((await summarizeControlledCampaign(f.manifest)).committedUsd, '0.014000000'); }); + + +test('portable archives replay after worker deletion, deduplicate attempts, and hold absent cells', async t => { + const f = await fixture(t, { count: 2, requests: 1 }); + await f.run(0); + const first = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: f.environment }); + const partial = await summarizeControlledEvidence([first, first]); + assert.equal(partial.incompleteAttempts, 1); + assert.equal(partial.knownUsd, '0.000068000'); + assert.equal(partial.committedUsd, '0.014068000'); + await f.run(1); + const second = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-1' }); + const expected = await summarizeControlledCampaign(f.manifest); + await rm(f.manifest.outputDirectory, { recursive: true }); + const restored = await summarizeControlledEvidence([first, second, first]); + assert.deepEqual(restored.campaigns, [expected]); + assert.equal(restored.operatingHeldUsd, '0.004000000'); + assert.equal(restored.knownUsd, '0.000136000'); + assert.equal(restored.costStatus, 'provisional'); +}); + +test('archived partial proof preserves known costs without consulting intact worker files', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + for (const name of ['admission.json', 'request.json', 'result.json', Object.keys(bundle.files).find(name => name.startsWith('receipts/'))]) { + const partial = structuredClone(bundle); delete partial.files[name]; + const summary = await summarizeControlledEvidence([partial]); + assert.equal(summary.incompleteAttempts, 1, name); + assert.equal(summary.knownUsd, '0.000068000', name); + assert.equal(summary.committedUsd, '0.014000000', name); + } + const absent = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + absent.files = {}; + assert.equal((await summarizeControlledEvidence([absent])).committedUsd, '0.014000000'); +}); + +test('archive creation retains never-started selections and refuses nonregular or oversized proof', async t => { + const f = await fixture(t); + const empty = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + assert.deepEqual(empty.files, {}); + assert.equal((await summarizeControlledEvidence([empty])).heldUsd, '0.014000000'); + await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + await rm(join(root, 'stderr.log')); + await symlink(f.manifest.runnerCli.path, join(root, 'stderr.log')); + await assert.rejects(packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' })); + await rm(join(root, 'stderr.log')); + await writeFile(join(root, 'stderr.log'), Buffer.alloc(1_048_577)); + await assert.rejects(packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }), /bounded regular file/); +}); + +test('archive validation rejects corruption, traversal, and conflicting campaign or attempt evidence', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + for (const mutate of [ + b => { b.manifest.campaignId = 'other'; }, + b => { b.files['ledger.jsonl'].base64 = 'x' + b.files['ledger.jsonl'].base64.slice(1); }, + b => { b.files['../outside'] = b.files['ledger.jsonl']; }, + b => { b.files['stderr.log'].sizeBytes = -1; }, + ]) { const b = structuredClone(bundle); mutate(b); assert.throws(() => decodeControlledEvidence(b)); } + const conflict = structuredClone(bundle); delete conflict.files['stderr.log']; + await assert.rejects(summarizeControlledEvidence([bundle, conflict]), /Conflicting controller attempt/); + const other = structuredClone(bundle); other.manifest.maxUsd = '2'; other.manifestSha256 = sha256(canonical(other.manifest)); + await assert.rejects(summarizeControlledEvidence([bundle, other]), /Conflicting controller campaign/); +}); + +test('configured credentials are excluded before encoding and after archive decoding', async t => { + const f = await fixture(t); await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const secret = 'synthetic-braintrust-archive-secret'; + const environment = { BRAINTRUST_API_KEY: secret }; + await writeFile(join(root, 'stderr.log'), secret); + await assert.rejects(packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment }), /configured credential/); + const unchecked = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: {} }); + assert(!JSON.stringify(unchecked).includes(secret)); + assert.throws(() => decodeControlledEvidence(unchecked, { environment }), /configured credential/); + const file = join(f.root, 'archive.json'); await writeFile(file, JSON.stringify(unchecked)); + await assert.rejects(readControlledEvidence(file, { environment }), /configured credential/); +}); + +test('campaign collection retains and audits private archives, while public projection omits them', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + const download = join(f.root, 'downloads', 'cell'), attempt = join(download, 'runs', 'attempt-0'); + await mkdir(attempt, { recursive: true }); + const evalCase = { id: 'case-0', checks: [] }, config = { model: 'fixture', provider: 'codex', driver: 'codex_app_server' }; + for (const [name, value] of Object.entries({ + 'request.json': f.requests[0].request, + 'artifact.json': JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')), + 'score.json': { attemptId: 'attempt-0', caseId: evalCase.id, passed: true, disposition: 'passed', checks: [] }, + 'case.json': evalCase, 'config.json': config, 'controller-evidence.json': bundle, + })) await writeFile(join(attempt, name), JSON.stringify(value)); + const cell = { cellId: 'execution-0', caseId: evalCase.id, rosterFile: 'roster.json', rosterId: 'roster-0', ...config }; + const catalogPath = join(f.root, 'catalog.json'); + await writeFile(catalogPath, JSON.stringify({ schema: 'paperclip.runner-protocol-eval.catalog/v1', campaignId: f.manifest.campaignId, + selection: { kind: 'subset' }, cells: [cell], rosters: [{ ...cell, cases: [evalCase.id] }] })); + await writeFile(join(download, 'cell.json'), JSON.stringify({ ...cell, exitCode: 0 })); + await rm(f.manifest.outputDirectory, { recursive: true }); + const input = { catalogPath, downloadsRoot: join(f.root, 'downloads'), evalsRoot: f.root, + runsOut: join(f.root, 'collected'), campaignOut: join(f.root, 'campaign.json'), + source: { paperclip: { sha: f.manifest.sourceRevision }, evals: { sha: f.manifest.definitionsRevision } } }; + const result = await aggregateProtocolEvalCampaign(input); + assert.equal(result.controlledCosts.knownUsd, '0.000068000'); + assert.equal(result.controlledCosts.costStatus, 'provisional'); + assert.deepEqual(JSON.parse(await readFile(join(input.runsOut, 'attempt-0', 'controller-evidence.json'), 'utf8')), bundle); + const publicRoot = join(f.root, 'public'); + await sanitizeProtocolEvalRuns({ runsRoot: input.runsOut, publicRunsRoot: publicRoot }); + assert.deepEqual((await readdir(join(publicRoot, 'attempt-0'))).sort(), ['artifact.json', 'case.json', 'config.json', 'score.json']); + assert(!(await readFile(join(publicRoot, 'attempt-0', 'artifact.json'), 'utf8')).includes('controllerAccounting')); + const changedRequest = { ...f.requests[0].request, prompt: 'copied from another execution' }; + await writeFile(join(attempt, 'request.json'), JSON.stringify(changedRequest)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable attempt does not match/); + await writeFile(join(attempt, 'request.json'), JSON.stringify(f.requests[0].request)); + const drifted = structuredClone(bundle); drifted.manifest.attempts[0].executionId = 'not-the-selected-cell'; + drifted.manifestSha256 = sha256(canonical(drifted.manifest)); + await writeFile(join(attempt, 'controller-evidence.json'), JSON.stringify(drifted)); + const bareArtifact = JSON.parse(await readFile(join(attempt, 'artifact.json'), 'utf8')); delete bareArtifact.controllerAccounting; + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(bareArtifact)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + await writeFile(join(attempt, 'artifact.json'), await readFile(f.requests[0].outputPath)); + await rm(join(attempt, 'controller-evidence.json')); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs index 23e4f996036..4dc48445cf4 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/report.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -4,16 +4,27 @@ import { pathToFileURL } from 'node:url'; import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { attemptDirectory } from './run.mjs'; -export async function summarizeControlledCampaign(value) { +export async function summarizeControlledCampaign(value, { filesByAttempt } = {}) { const { manifest, manifestSha256, allocatedUsd, overheadUsd } = validateManifest(value, Date.now(), { admission: false }); let known = 0n, committed = units(overheadUsd); const attempts = []; for (const allocation of manifest.attempts) { const root = attemptDirectory(manifest, allocation.attemptId); + const readEvidence = async name => { + if (filesByAttempt === undefined) return readFile(join(root, name)); + const bytes = filesByAttempt.get(allocation.attemptId)?.get(name); + if (!Buffer.isBuffer(bytes)) throw new Error('Missing archived controller evidence'); + return bytes; + }; + const readDocument = async (name, maximum = 1_048_576) => { + const bytes = await readEvidence(name); + if (bytes.length > maximum) throw new Error('Controller document exceeds its byte limit'); + return JSON.parse(bytes.toString('utf8')); + }; let snapshot, report, verified = false; try { - const ledgerBytes = await readFile(join(root, 'ledger.jsonl')); - const ledger = await costRuntime.readEvalCostLedger(join(root, 'ledger.jsonl')); + const ledgerBytes = await readEvidence('ledger.jsonl'); + const ledger = costRuntime.parseEvalCostLedger(ledgerBytes.toString('utf8')); const candidateSnapshot = ledger.state.snapshot(); const ledgerDeadline = Date.parse(ledger.state.limits.deadlineAt); const fixedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: ledger.state.limits.deadlineAt, @@ -23,7 +34,7 @@ export async function summarizeControlledCampaign(value) { || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }]) || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger identity does not match its allocation'); snapshot = candidateSnapshot; - const admission = await readJson(join(root, 'admission.json')); + const admission = await readDocument('admission.json'); const started = Date.parse(admission.startedAt); const expectedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: new Date(Math.min(Date.parse(manifest.deadlineAt), started + allocation.timeoutMs)).toISOString(), @@ -31,20 +42,20 @@ export async function summarizeControlledCampaign(value) { if (admission.schema !== 'paperclip.runner-eval.admission/v1' || admission.manifestSha256 !== manifestSha256 || admission.campaignId !== manifest.campaignId || canonical(admission.attempt) !== canonical(allocation) || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits)) throw new Error('Ledger does not match its admission'); - report = await readJson(join(root, 'result.json')); - const request = await readJson(join(root, 'request.json'), 8_388_608); + report = await readDocument('result.json'); + const request = await readDocument('request.json', 8_388_608); if (requestBindingSha256(request) !== allocation.requestSha256 || request.runnerd.sha256.replace(/^sha256:/, '') !== manifest.runnerd.sha256 || canonical(Object.keys(report.receiptHashes ?? {}).sort()) !== canonical(snapshot.requests.map(item => item.requestId).sort())) throw new Error('Retained request evidence does not match'); for (const item of snapshot.requests) { if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(item.requestId)) throw new Error('Invalid retained receipt identity'); - const bytes = await readFile(join(root, 'receipts', `${item.requestId}.json`)); + const bytes = await readEvidence(`receipts/${item.requestId}.json`); const receipt = JSON.parse(bytes.toString('utf8')); if (sha256(bytes) !== report.receiptHashes[item.requestId] || receipt.requestId !== item.requestId || receipt.route !== manifest.route || receipt.costUsd !== item.costUsd) throw new Error('Retained receipt does not match its ledger'); } - const artifactBytes = await readFile(join(root, 'provider-artifact.json')); + const artifactBytes = await readEvidence('provider-artifact.json'); const artifact = JSON.parse(artifactBytes.toString('utf8')); - const descriptorBytes = await readFile(join(root, 'gateway.json')); + const descriptorBytes = await readEvidence('gateway.json'); const descriptor = JSON.parse(descriptorBytes.toString('utf8')); if (report.schema !== 'paperclip.runner-eval.control-result/v1' || report.manifestSha256 !== manifestSha256 || report.campaignId !== manifest.campaignId || report.executionId !== allocation.executionId || report.attemptId !== allocation.attemptId diff --git a/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts b/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts index 38f45e0839a..b8591390e09 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts +++ b/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts @@ -1,3 +1,3 @@ -export { EvalSpendState, openEvalCostLedger, readEvalCostLedger, validEvalLedgerId } from '../../../../tests/runner-e2e/cost-ledger.js'; +export { EvalSpendState, openEvalCostLedger, readEvalCostLedger, parseEvalCostLedger, validEvalLedgerId } from '../../../../tests/runner-e2e/cost-ledger.js'; export { evalRequestReservation, startEvalGateway } from '../../../../tests/runner-e2e/eval-gateway.js'; export { EVAL_OVERHEAD_COMPONENTS, validateOperatingOverhead } from '../../../../tests/runner-e2e/operating-overhead.js'; diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index 2ea5c4f7cb2..e13a218700f 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -26,6 +26,7 @@ const ATTEMPT_FILES = new Set([ "compiled-fixture.json", "case.json", "config.json", + "controller-evidence.json", ]); function json(value) { @@ -405,6 +406,7 @@ export async function aggregateProtocolEvalCampaign({ catalog.cells.map((cell) => [cell.cellId, cell]), ); const copiedAttemptIds = new Set(); + const controllerBundles = []; for (const statusPath of await findFiles(downloadsRoot, "cell.json")) { const status = await loadObject(statusPath); const cellId = safeId(status.cellId, "recorded cell ID"); @@ -453,6 +455,25 @@ export async function aggregateProtocolEvalCampaign({ ) { throw new Error(`Downloaded attempt identity drifted for ${cellId}`); } + const bundlePath = join(attemptRoot, entry.name, "controller-evidence.json"); + const bundleStat = await lstat(bundlePath).catch(error => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (bundleStat) { + const { readAttemptControlledEvidence } = await import("./controlled-evals/archive.mjs"); + const bundle = await readAttemptControlledEvidence(join(attemptRoot, entry.name)); + const allocation = bundle.manifest.attempts.find(item => item.attemptId === attemptId); + const selectedExecutions = new Set(bundle.manifest.attempts.map(item => item.executionId)); + if (bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || + selectedExecutions.size !== expectedByCell.size || [...expectedByCell.keys()].some(id => !selectedExecutions.has(id)) || + bundle.manifest.sourceRevision !== source.paperclip.sha || bundle.manifest.definitionsRevision !== source.evals.sha) { + throw new Error(`Controller archive identity drifted for ${cellId}`); + } + controllerBundles.push(bundle); + } else if (artifact.controllerAccounting) { + throw new Error(`Missing private controller evidence for ${attemptId}; cost metadata alone is insufficient`); + } await copyAttempt( join(attemptRoot, entry.name), join(runsOut, attemptId), @@ -518,6 +539,15 @@ export async function aggregateProtocolEvalCampaign({ (result) => result.disposition === "infrastructure_failure", ).length, }; + let controlledCosts; + if (controllerBundles.length) { + const { summarizeControlledEvidence } = await import("./controlled-evals/archive.mjs"); + controlledCosts = await summarizeControlledEvidence(controllerBundles); + const allocatedAttempts = new Set(controlledCosts.campaigns.flatMap(campaign => campaign.attempts.map(attempt => attempt.attemptId))); + if ([...copiedAttemptIds].some(id => !allocatedAttempts.has(id))) { + throw new Error("Retained attempt is outside the controlled campaign allocation"); + } + } const generatedAt = new Date().toISOString(); const campaign = { schema: "paperclip.runner-protocol-eval.campaign/v1", @@ -525,6 +555,7 @@ export async function aggregateProtocolEvalCampaign({ generatedAt, selection: catalog.selection, costs: sumAttemptCosts(attemptUsages), + ...(controlledCosts ? { controlledCosts } : {}), source: { paperclip: source.paperclip, evals: source.evals, diff --git a/tests/runner-e2e/cost-ledger.ts b/tests/runner-e2e/cost-ledger.ts index daa95eba7b3..576fd8a2385 100644 --- a/tests/runner-e2e/cost-ledger.ts +++ b/tests/runner-e2e/cost-ledger.ts @@ -222,7 +222,10 @@ export class EvalSpendState { } export async function readEvalCostLedger(file: string) { - const raw = await readFile(file, "utf8"); + return parseEvalCostLedger(await readFile(file, "utf8")); +} + +export function parseEvalCostLedger(raw: string) { if (!raw.endsWith("\n")) throw new Error("Eval ledger has an incomplete durable record"); const lines = raw.trimEnd().split("\n").map(value => JSON.parse(value) as Line); let previous = "", state: EvalSpendState | undefined; From b28901e420718ce2fbabaa53c1b162f58af385eb Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 12:45:39 -0700 Subject: [PATCH 7/9] fix(evals): stop surviving children after controller CLI exit Retain the full allocation and bound inherited pipes when the CLI exits before its owned provider process group. Cover both retained and absent artifacts. Co-Authored-By: Paperclip --- .../controlled-evals/controller.test.mjs | 23 +++++++++++++++++-- .../scripts/controlled-evals/run.mjs | 9 ++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 5fac90ff3b1..87759f68c81 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -19,8 +19,9 @@ import {readFile,writeFile} from 'node:fs/promises'; const args=new Map();for(let i=2;i{});process.stdout.write('ready');setInterval(()=>{},1000)"],{stdio:['ignore','pipe','ignore']}); - await once(descendant.stdout,'data'); + const orphan=${Boolean(options.orphanExit)}; + const descendant=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});"+(orphan?"process.send('ready');":"process.stdout.write('ready');")+"setInterval(()=>{},1000)"],{stdio:orphan?['ignore','inherit','inherit','ipc']:['ignore','pipe','ignore']}); + if(orphan)await once(descendant,'message');else await once(descendant.stdout,'data'); await writeFile(request.session.workingDirectory+'/descendant.pid',String(descendant.pid)); } @@ -36,6 +37,7 @@ if(!${Boolean(options.missingArtifact)})await writeFile(args.get('--output'),JSO requestRouting:${options.missingRoute ? 'null' : "{mode:'controller_gateway',provider:descriptor.provider,declaredUpstreamRoute:descriptor.upstreamRoute,campaignId:descriptor.campaignId,executionId:descriptor.executionId,attemptId:descriptor.attemptId,descriptorSha256:createHash('sha256').update(descriptorRaw).digest('hex')}"}, snapshot:{status:'closed',process:{runnerExited:true,runnerPid:process.pid,providerPid:process.pid}},statuses, leaked:${options.leakToken ? 'process.env.PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN' : 'null'}})); +if(${Boolean(options.orphanExit)})process.exit(0); `; async function fixture(t, { count = 1, allocation = '0.01', providerProfile = 'codex', route = 'direct', ...options } = {}) { @@ -143,6 +145,23 @@ for (const fault of ['missingArtifact', 'missingRoute', 'leakToken']) test(`${fa if (fault === 'leakToken') { assert.equal(result.report.secretExcluded, false); assert.equal(artifact.leaked, '[redacted]'); } }); +for (const missingArtifact of [false, true]) test(`CLI exit stops surviving owned processes, missing artifact=${missingArtifact}`, async t => { + const f = await fixture(t, { requests: 0, resistantDescendant: true, orphanExit: true, missingArtifact }); + const pidFile = join(f.requests[0].request.session.workingDirectory, 'descendant.pid'); + let pid; + t.after(() => { if (pid) { try { process.kill(pid, 'SIGKILL'); } catch {} } }); + const result = await f.run(); + pid = Number(await readFile(pidFile, 'utf8')); + assert.equal(result.report.child.stopped, 'provider_group_survived_cli_exit'); + for (let i = 0; i < 50; i++) { + try { process.kill(pid, 0); } catch (error) { if (error.code === 'ESRCH') break; throw error; } + await new Promise(resolve => setTimeout(resolve, 20)); + } + assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }); + assert.equal(result.report.status, 'incomplete'); + assert.equal((await summarizeControlledCampaign(f.manifest)).committedUsd, '0.014000000'); +}); + test('cancellation closes the proxy, bounds the owned child, and preserves unknown request holds', async t => { const f = await fixture(t, { requests: 1, waitForever: true }); const abort = new AbortController(); let calls = 0; diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 42fda084873..7945910e17b 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -42,14 +42,13 @@ async function runChild(input) { if (input.signal?.aborted) onAbort(); try { const [code, signal] = await once(child, 'exit'); - if (stopped) { - let groupPresent = true; - try { process.kill(-child.pid, 0); } catch (error) { if (error.code === 'ESRCH') groupPresent = false; } - if (groupPresent) await escalation; - } + const groupPresent = !absent(-child.pid); + if (groupPresent && !stopped) stop('provider_group_survived_cli_exit'); + if (groupPresent) await escalation; return { code, signal, pid: child.pid, stopped, stdout: Buffer.concat(output.stdout).toString(), stderr: Buffer.concat(output.stderr).toString() }; } finally { clearTimeout(timeout); clearTimeout(force); input.signal?.removeEventListener('abort', onAbort); + child.stdout.destroy(); child.stderr.destroy(); } } From 4d6673d8cedabdccbab168f33a2352f513844444 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 12:51:01 -0700 Subject: [PATCH 8/9] fix(evals): bind archived results and complete catalog allocations Compare immutable outcomes and accounting with the controller's reconstructed output. Validate provider, model, profile and driver for every selected allocation, including missing cells. Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 13 +++-- .../scripts/controlled-evals/archive.mjs | 23 +++++++- .../controlled-evals/controller.test.mjs | 53 +++++++++++++++---- .../scripts/controlled-evals/run.mjs | 14 +++-- .../scripts/runner-protocol-eval-campaign.mjs | 12 ++++- 5 files changed, 94 insertions(+), 21 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index d5fbb0e65f6..63dccf4a82c 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -209,7 +209,10 @@ requests keep their reservations, and failed or missing cell evidence keeps that cell's full allocation in the campaign report. An existing cell directory or output is never reset for a retry. Cancellation closes admission, drains upstream receipts within the proxy limit, and stops the owned CLI process -group. Unverified provider cleanup remains explicit and retains the allocation. +group. If the CLI exits while its owned group survives, the controller closes +admission, stops that group, bounds inherited output pipes, and records an +incomplete attempt. Unverified provider cleanup remains explicit and retains +the allocation. Each result retains `controllerAccounting` alongside the original provider outcome and usage. Retain the durable controller directory as private evidence: @@ -250,7 +253,10 @@ The reader accepts only bounded regular files and fixed evidence paths. A bundle is limited to 64 MiB of decoded content plus encoding overhead; this storage-format limit does not qualify a vendor charge or operating-cost bound. `report` accepts standalone bundle files; `report-attempts` additionally binds -them to the surrounding immutable request and artifact. Both work after the +them to the surrounding immutable request and reconstructed controller output, +including the provider outcome, usage, and complete accounting result. The +private program may add provenance or a stricter non-retryable retention +failure. Missing source proof remains incomplete. Both work after the original worker directory is gone, using the same hash-chain ledger verifier. They group attempts by manifest, count operating holds once, deduplicate exact copies, reject conflicting copies, and keep full allocations for missing @@ -259,7 +265,8 @@ selected attempts. They never consult old worker paths to fill archive gaps. The campaign collector preserves the bundle in private runs and returns its independently replayed totals as `controlledCosts`. For this collector, each manifest `executionId` must equal its catalog `cellId`, and the complete -manifest selection must match the catalog. Extra attempts are rejected. +manifest selection, model, provider, agent profile, and driver must match the +catalog, including cells with no retained result. Extra attempts are rejected. Existing `costs` remain legacy provider-usage observations; do not add them to `controlledCosts`. The public attempt projection omits bundles and raw `controllerAccounting`. diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index ffea93bee58..84335be989d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -3,7 +3,7 @@ import { lstat, open, readdir, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; -import { attemptDirectory } from './run.mjs'; +import { attemptDirectory, controlledResultArtifact } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; const MiB = 1_048_576; @@ -127,6 +127,27 @@ export async function readAttemptControlledEvidence(directory, options) { || artifact.controllerAccounting && artifact.controllerAccounting.manifestSha256 !== bundle.manifestSha256) { throw new Error('Immutable attempt does not match its controller archive'); } + const { files } = decodeControlledEvidence(bundle, options); + const reportBytes = files.get('result.json'), providerBytes = files.get('provider-artifact.json'); + if (reportBytes) { + let report, provider; + try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; } + catch { return bundle; } + if (providerBytes || report?.providerArtifactSha256 === null) { + const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact }; + delete expected.provenance; delete actual.provenance; + if (actual.controllerEvidenceError !== undefined) { + if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim() + || typeof actual.accountingError !== 'string' || !actual.accountingError.trim() + || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) { + throw new Error('Invalid immutable controller retention failure'); + } + delete actual.controllerEvidenceError; + if (!expected.accountingFailure) { delete actual.accountingFailure; delete actual.accountingError; } + } + if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); + } + } return bundle; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 87759f68c81..65398c8415a 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -379,7 +379,7 @@ test('configured credentials are excluded before encoding and after archive deco }); test('campaign collection retains and audits private archives, while public projection omits them', async t => { - const f = await fixture(t, { requests: 1 }); await f.run(); + const f = await fixture(t, { count: 2, requests: 1 }); await f.run(); const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); const download = join(f.root, 'downloads', 'cell'), attempt = join(download, 'runs', 'attempt-0'); await mkdir(attempt, { recursive: true }); @@ -391,9 +391,15 @@ test('campaign collection retains and audits private archives, while public proj 'case.json': evalCase, 'config.json': config, 'controller-evidence.json': bundle, })) await writeFile(join(attempt, name), JSON.stringify(value)); const cell = { cellId: 'execution-0', caseId: evalCase.id, rosterFile: 'roster.json', rosterId: 'roster-0', ...config }; + const missingCell = { ...cell, cellId: 'execution-1', caseId: 'case-1' }; + const program = join(f.root, 'evals/paperclip-runner'); + for (const folder of ['rosters', 'cases', 'configs']) await mkdir(join(program, folder), { recursive: true }); + await writeFile(join(program, 'rosters/roster.json'), JSON.stringify({ config: '../configs/config.json' })); + await writeFile(join(program, 'cases/case-1.json'), JSON.stringify({ ...evalCase, id: 'case-1' })); + await writeFile(join(program, 'configs/config.json'), JSON.stringify(config)); const catalogPath = join(f.root, 'catalog.json'); await writeFile(catalogPath, JSON.stringify({ schema: 'paperclip.runner-protocol-eval.catalog/v1', campaignId: f.manifest.campaignId, - selection: { kind: 'subset' }, cells: [cell], rosters: [{ ...cell, cases: [evalCase.id] }] })); + selection: { kind: 'subset' }, cells: [cell, missingCell], rosters: [{ ...cell, cases: [evalCase.id, missingCell.caseId] }] })); await writeFile(join(download, 'cell.json'), JSON.stringify({ ...cell, exitCode: 0 })); await rm(f.manifest.outputDirectory, { recursive: true }); const input = { catalogPath, downloadsRoot: join(f.root, 'downloads'), evalsRoot: f.root, @@ -401,7 +407,9 @@ test('campaign collection retains and audits private archives, while public proj source: { paperclip: { sha: f.manifest.sourceRevision }, evals: { sha: f.manifest.definitionsRevision } } }; const result = await aggregateProtocolEvalCampaign(input); assert.equal(result.controlledCosts.knownUsd, '0.000068000'); - assert.equal(result.controlledCosts.costStatus, 'provisional'); + assert.equal(result.controlledCosts.costStatus, 'incomplete'); + assert.equal(result.controlledCosts.committedUsd, '0.014068000'); + assert.equal(result.controlledCosts.incompleteAttempts, 1); assert.deepEqual(JSON.parse(await readFile(join(input.runsOut, 'attempt-0', 'controller-evidence.json'), 'utf8')), bundle); const publicRoot = join(f.root, 'public'); await sanitizeProtocolEvalRuns({ runsRoot: input.runsOut, publicRunsRoot: publicRoot }); @@ -411,13 +419,38 @@ test('campaign collection retains and audits private archives, while public proj await writeFile(join(attempt, 'request.json'), JSON.stringify(changedRequest)); await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable attempt does not match/); await writeFile(join(attempt, 'request.json'), JSON.stringify(f.requests[0].request)); - const drifted = structuredClone(bundle); drifted.manifest.attempts[0].executionId = 'not-the-selected-cell'; - drifted.manifestSha256 = sha256(canonical(drifted.manifest)); - await writeFile(join(attempt, 'controller-evidence.json'), JSON.stringify(drifted)); - const bareArtifact = JSON.parse(await readFile(join(attempt, 'artifact.json'), 'utf8')); delete bareArtifact.controllerAccounting; - await writeFile(join(attempt, 'artifact.json'), JSON.stringify(bareArtifact)); - await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); - await writeFile(join(attempt, 'artifact.json'), await readFile(f.requests[0].outputPath)); + const originalArtifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + for (const mutate of [ + value => { value.usage = { estimatedCostNanodollars: 0 }; }, + value => { value.turn = { status: 'completed', assistantText: 'changed outcome' }; }, + value => { value.controllerAccounting.cleanupVerified = false; }, + value => { delete value.controllerAccounting; }, + ]) { + const changed = structuredClone(originalArtifact); mutate(changed); + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(changed)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable result disagrees/); + } + await writeFile(join(attempt, 'artifact.json'), JSON.stringify({ ...originalArtifact, provenance: { runnerPackage: 'private-program-provenance' } })); + await aggregateProtocolEvalCampaign(input); + const retainedFailure = { ...originalArtifact, controllerEvidenceError: 'Retained verification was interrupted', accountingError: 'Retained verification was interrupted', + accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } }; + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(retainedFailure)); + await aggregateProtocolEvalCampaign(input); + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(originalArtifact)); + const catalog = JSON.parse(await readFile(catalogPath, 'utf8')); + for (const change of [{ model: 'wrong-model' }, { provider: 'acpx', acpxAgent: 'claude' }, { driver: 'other-driver' }]) { + const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[0], change); + await writeFile(catalogPath, JSON.stringify(driftedCatalog)); + await writeFile(join(attempt, 'config.json'), JSON.stringify({ ...config, ...change })); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + } + await writeFile(join(attempt, 'config.json'), JSON.stringify(config)); + for (const change of [{ model: 'wrong-unexecuted-model' }, { provider: 'acpx', acpxAgent: 'claude', driver: 'acpx_runtime' }]) { + const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[1], change); + await writeFile(catalogPath, JSON.stringify(driftedCatalog)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + } + await writeFile(catalogPath, JSON.stringify(catalog)); await rm(join(attempt, 'controller-evidence.json')); await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/); }); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 7945910e17b..cef1f272b0d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -52,6 +52,14 @@ async function runChild(input) { } } +export function controlledResultArtifact(artifact, report, attempt) { + return { ...(artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, + requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }), + ...(report.failure ? { accountingError: 'Controlled request accounting or cleanup is incomplete', + accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } } : {}), + controllerAccounting: report }; +} + export async function runControlledEval({ manifestPath, requestPath, outputPath, environment = process.env, signal, transport }) { if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Controlled eval process ownership requires Linux or macOS'); signal?.throwIfAborted(); @@ -167,11 +175,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null, cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, receiptHashes, accounting: snapshot }; await atomicJson(join(root, 'result.json'), report); - const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, - requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }; - if (failure) Object.assign(resultArtifact, { accountingError: 'Controlled request accounting or cleanup is incomplete', - accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } }); - resultArtifact.controllerAccounting = report; + const resultArtifact = controlledResultArtifact(artifact, report, attempt); await mkdir(dirname(resolve(outputPath)), { recursive: true, mode: 0o700 }); await writeFile(outputPath, JSON.stringify(resultArtifact, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); return { code: failure ? 2 : child?.code ?? 2, report }; diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index e13a218700f..f410814d878 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -184,6 +184,7 @@ export async function buildProtocolEvalCatalog({ configFile: relative(programRoot, configPath).split(sep).join("/"), model: String(roster.model ?? config.model ?? "unknown"), provider: String(config.provider ?? "codex"), + ...(config.provider === "acpx" ? { acpxAgent: config.acpxAgent } : {}), driver: String(config.driver ?? "codex_app_server"), credentialName, ...(config.provider === "acpx" && config.acpxAgent === "grok" @@ -214,6 +215,7 @@ export async function buildProtocolEvalCatalog({ caseId, model: roster.model, provider: roster.provider, + ...(roster.acpxAgent ? { acpxAgent: roster.acpxAgent } : {}), driver: roster.driver, credentialName: roster.credentialName, ...(roster.authenticationMode ? { authenticationMode: roster.authenticationMode } : {}), @@ -465,13 +467,19 @@ export async function aggregateProtocolEvalCampaign({ const bundle = await readAttemptControlledEvidence(join(attemptRoot, entry.name)); const allocation = bundle.manifest.attempts.find(item => item.attemptId === attemptId); const selectedExecutions = new Set(bundle.manifest.attempts.map(item => item.executionId)); - if (bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || + const allocationsMatchCatalog = bundle.manifest.attempts.every(item => { + const cell = expectedByCell.get(item.executionId); + return cell && item.model === cell.model && item.provider === cell.provider + && item.acpxAgent === cell.acpxAgent + && cell.driver === (item.provider === "codex" ? "codex_app_server" : "acpx_runtime"); + }); + if (!allocationsMatchCatalog || bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || selectedExecutions.size !== expectedByCell.size || [...expectedByCell.keys()].some(id => !selectedExecutions.has(id)) || bundle.manifest.sourceRevision !== source.paperclip.sha || bundle.manifest.definitionsRevision !== source.evals.sha) { throw new Error(`Controller archive identity drifted for ${cellId}`); } controllerBundles.push(bundle); - } else if (artifact.controllerAccounting) { + } else if ("controllerAccounting" in artifact || "controllerEvidenceError" in artifact) { throw new Error(`Missing private controller evidence for ${attemptId}; cost metadata alone is insufficient`); } await copyAttempt( From 7d4eeb572826276dfa7d091117150edc2fdd8bf0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 13:04:11 -0700 Subject: [PATCH 9/9] fix(evals): retain annotated failed controller attempts --- .../scripts/controlled-evals/archive.mjs | 3 ++- .../controlled-evals/controller.test.mjs | 21 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index 84335be989d..fc328ee72a5 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -143,7 +143,8 @@ export async function readAttemptControlledEvidence(directory, options) { throw new Error('Invalid immutable controller retention failure'); } delete actual.controllerEvidenceError; - if (!expected.accountingFailure) { delete actual.accountingFailure; delete actual.accountingError; } + if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; } + else { delete actual.accountingFailure; delete actual.accountingError; } } if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); } diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 65398c8415a..820b24280d0 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -7,7 +7,7 @@ import { join } from 'node:path'; import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; import { attemptDirectory, runControlledEval } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; -import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; +import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; const childScript = options => ` @@ -378,6 +378,25 @@ test('configured credentials are excluded before encoding and after archive deco await assert.rejects(readControlledEvidence(file, { environment }), /configured credential/); }); +test('retention annotations preserve an already-failed controller result and its known costs', async t => { + const f = await fixture(t, { requests: 1, missingRoute: true }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + assert.equal(artifact.controllerAccounting.status, 'incomplete'); + artifact.controllerEvidenceError = 'A later retention check failed'; + artifact.accountingError = artifact.controllerEvidenceError; + const directory = join(f.root, 'runs', 'attempt-0'); await mkdir(directory, { recursive: true }); + for (const [name, value] of Object.entries({ 'request.json': f.requests[0].request, 'artifact.json': artifact, 'controller-evidence.json': bundle })) { + await writeFile(join(directory, name), JSON.stringify(value)); + } + const retained = await readAttemptControlledEvidence(directory); + const report = await summarizeControlledEvidence([retained]); + assert.equal(report.costStatus, 'incomplete'); + assert.equal(report.knownUsd, '0.000068000'); + assert.equal(report.committedUsd, '0.014000000'); + assert.equal(JSON.parse(await readFile(join(directory, 'artifact.json'), 'utf8')).accountingError, artifact.controllerEvidenceError); +}); + test('campaign collection retains and audits private archives, while public projection omits them', async t => { const f = await fixture(t, { count: 2, requests: 1 }); await f.run(); const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });