From 99e07e675de163e69999f9e2e0cb3f3b6427ca6b Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 11:13:45 -0700 Subject: [PATCH] feat(evals): retain portable Runner controller evidence Replay archived ledgers through the existing verifier, preserve conservative holds, and keep private bundles out of public reports. Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 48 ++++- .../scripts/controlled-evals/archive.mjs | 173 ++++++++++++++++++ .../controlled-evals/controller.test.mjs | 125 +++++++++++++ .../scripts/controlled-evals/report.mjs | 29 ++- .../controlled-evals/shared-cost-runtime.ts | 2 +- .../scripts/runner-protocol-eval-campaign.mjs | 31 ++++ tests/runner-e2e/cost-ledger.ts | 5 +- 7 files changed, 399 insertions(+), 14 deletions(-) create mode 100644 packages/paperclip-runner/scripts/controlled-evals/archive.mjs diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index b066a6de31f..d5fbb0e65f6 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -215,15 +215,57 @@ Each result retains `controllerAccounting` alongside the original provider outcome and usage. Retain the durable controller directory as private evidence: its admission, request, gateway descriptor, request receipts, ledger, original provider artifact, and final result are required for reconciliation. The -private program's existing immutable attempt copy includes the result metadata; -its current default history publisher does not archive this separate directory. -Do not infer receipt retention from that metadata alone. +private `controller-evidence.json` bundle can travel with the immutable attempt. +Set `PAPERCLIP_RUNNER_EVAL_ARCHIVE_CLI` to the sibling `archive.mjs` when using a +private eval program with controller retention support. That program validates +the request before launch, captures the bundle before finalizing the attempt, +and marks missing or incomplete capture as a non-retryable accounting failure. +It must use zero infrastructure retries. Older private programs do not capture +this bundle; result metadata alone is not proof of retention. ```sh node packages/paperclip-runner/scripts/controlled-evals/report.mjs \ /private/path/controller-manifest.json > /private/path/controller-report.json ``` +To retain or inspect the evidence independently of the original worker: + +```sh +node packages/paperclip-runner/scripts/controlled-evals/archive.mjs pack \ + /private/path/controller-manifest.json attempt-id \ + /private/path/runs/attempt-id/controller-evidence.json +node packages/paperclip-runner/scripts/controlled-evals/archive.mjs report-attempts \ + /private/path/runs/attempt-id +``` + +`pack` is exclusive and never overwrites evidence. The private bundle contains +the full manifest and exact bytes of the admission, request, gateway descriptor, +ledger, receipts, provider artifact, result, and bounded stdout/stderr. It omits +executables, private homes, and caches. Files carry byte lengths and SHA-256 +hashes. These detect corruption; they are not signatures or proof against a +host that can rewrite all evidence. Known configured credential values are +checked before encoding and after decoding. Keep bundles private. + +The reader accepts only bounded regular files and fixed evidence paths. A +bundle is limited to 64 MiB of decoded content plus encoding overhead; this +storage-format limit does not qualify a vendor charge or operating-cost bound. +`report` accepts standalone bundle files; `report-attempts` additionally binds +them to the surrounding immutable request and artifact. Both work after the +original worker directory is gone, using the same hash-chain ledger verifier. +They group attempts by manifest, count operating holds once, deduplicate exact +copies, reject conflicting copies, and keep full allocations for missing +selected attempts. They never consult old worker paths to fill archive gaps. + +The campaign collector preserves the bundle in private runs and returns its +independently replayed totals as `controlledCosts`. For this collector, each +manifest `executionId` must equal its catalog `cellId`, and the complete +manifest selection must match the catalog. Extra attempts are rejected. +Existing `costs` remain +legacy provider-usage observations; do not add them to `controlledCosts`. +The public attempt projection omits bundles and raw `controllerAccounting`. +Private reports label ledger-known amounts and conservative holds as provisional +or incomplete; a missing verifier or invalid evidence cannot become a zero cost. + The report recomputes ledger costs, checks their attempt/manifest bindings, and preserves known partial charges. It reports operating costs as held and total costs as provisional when evidence is intact, or incomplete otherwise. It does diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs new file mode 100644 index 00000000000..ffea93bee58 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -0,0 +1,173 @@ +import { constants } from 'node:fs'; +import { lstat, open, readdir, writeFile } from 'node:fs/promises'; +import { basename, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; +import { attemptDirectory } from './run.mjs'; +import { summarizeControlledCampaign } from './report.mjs'; + +const MiB = 1_048_576; +const MAX_RAW_BYTES = 64 * MiB; +const MAX_ARCHIVE_BYTES = 96 * MiB; +const FILE_LIMITS = new Map([ + ['admission.json', MiB], ['request.json', 8 * MiB], ['gateway.json', 8192], + ['ledger.jsonl', 16 * MiB], ['result.json', MiB], ['provider-artifact.json', 16 * MiB], + ['stdout.log', MiB], ['stderr.log', MiB], +]); +const RECEIPT = /^receipts\/[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\.json$/; +const limitFor = name => FILE_LIMITS.get(name) ?? (RECEIPT.test(name) ? MiB : 0); + +function secretValues({ environment = process.env, secrets = [] } = {}) { + return [...new Set([...secrets, ...Object.entries(environment) + .filter(([name]) => /(?:API_KEY|TOKEN|SECRET|PASSWORD|ACCESS_KEY_ID)$/.test(name)) + .map(([, value]) => value)].filter(value => typeof value === 'string' && value.length > 0))]; +} +function excludeSecrets(bytes, secrets) { + if (secrets.some(secret => bytes.includes(Buffer.from(secret)))) throw new Error('Controller evidence contains a configured credential'); +} +async function readRegularFile(path, maximum) { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > maximum) throw new Error('Controller evidence is not a bounded regular file'); + const bytes = Buffer.alloc(stat.size + 1); + let length = 0; + while (length < bytes.length) { + const { bytesRead } = await handle.read(bytes, length, bytes.length - length, null); + if (!bytesRead) break; + length += bytesRead; + } + if (length !== stat.size) throw new Error('Controller evidence changed while being read'); + return bytes.subarray(0, length); + } finally { await handle.close(); } +} +async function realDirectory(path) { + try { + const stat = await lstat(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('Controller evidence directory is not a real directory'); + return true; + } catch (error) { if (error.code === 'ENOENT') return false; throw error; } +} + +export async function packControlledAttempt({ manifest: value, attemptId, ...options }) { + const { manifest, manifestSha256 } = validateManifest(value, Date.now(), { admission: false }); + const allocation = manifest.attempts.find(attempt => attempt.attemptId === attemptId); + if (!allocation) throw new Error('Attempt is absent from the controller manifest'); + const secrets = secretValues(options); + const manifestBytes = Buffer.from(canonical(manifest)); + if (manifestBytes.length > MiB) throw new Error('Controller manifest exceeds its archive limit'); + excludeSecrets(manifestBytes, secrets); + const root = attemptDirectory(manifest, attemptId), files = {}; + let total = manifestBytes.length; + if (await realDirectory(root)) { + const names = [...FILE_LIMITS.keys()]; + if (await realDirectory(join(root, 'receipts'))) { + for (const name of await readdir(join(root, 'receipts'))) { + if (RECEIPT.test(`receipts/${name.replace(/\.tmp$/, '')}`) && name.endsWith('.tmp')) continue; + if (!RECEIPT.test(`receipts/${name}`)) throw new Error('Unexpected controller receipt path'); + names.push(`receipts/${name}`); + } + } + if (names.length > allocation.maxRequests + FILE_LIMITS.size) throw new Error('Too many controller receipts'); + for (const name of names.sort()) { + let bytes; + try { bytes = await readRegularFile(join(root, name), limitFor(name)); } + catch (error) { if (error.code === 'ENOENT') continue; throw error; } + total += bytes.length; + if (total > MAX_RAW_BYTES) throw new Error('Controller evidence exceeds its archive limit'); + excludeSecrets(bytes, secrets); + files[name] = { sizeBytes: bytes.length, sha256: sha256(bytes), base64: bytes.toString('base64') }; + } + } + return { schema: 'paperclip.runner-eval.control-evidence/v1', manifest, manifestSha256, attemptId, files }; +} + +export function decodeControlledEvidence(value, options) { + fields(value, ['schema', 'manifest', 'manifestSha256', 'attemptId', 'files']); + const { manifest, manifestSha256 } = validateManifest(value.manifest, Date.now(), { admission: false }); + const allocation = manifest.attempts.find(attempt => attempt.attemptId === value.attemptId); + if (value.schema !== 'paperclip.runner-eval.control-evidence/v1' || value.manifestSha256 !== manifestSha256 || !allocation) throw new Error('Controller archive identity mismatch'); + fields(value.files, Object.keys(value.files ?? {})); + const entries = Object.entries(value.files); + if (entries.length > allocation.maxRequests + FILE_LIMITS.size) throw new Error('Too many controller evidence files'); + const secrets = secretValues(options), manifestBytes = Buffer.from(canonical(manifest)); + if (manifestBytes.length > MiB) throw new Error('Controller manifest exceeds its archive limit'); + excludeSecrets(manifestBytes, secrets); + let total = manifestBytes.length; + const files = new Map(); + for (const [name, file] of entries) { + const limit = limitFor(name); + fields(file, ['sizeBytes', 'sha256', 'base64']); + if (!limit || !Number.isSafeInteger(file.sizeBytes) || file.sizeBytes < 0 || file.sizeBytes > limit + || typeof file.base64 !== 'string' || file.base64.length !== 4 * Math.ceil(file.sizeBytes / 3) + || typeof file.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(file.sha256)) throw new Error('Invalid controller evidence file'); + total += file.sizeBytes; + if (total > MAX_RAW_BYTES) throw new Error('Controller evidence exceeds its archive limit'); + const bytes = Buffer.from(file.base64, 'base64'); + if (bytes.length !== file.sizeBytes || bytes.toString('base64') !== file.base64 || sha256(bytes) !== file.sha256) throw new Error('Controller evidence digest mismatch'); + excludeSecrets(bytes, secrets); + files.set(name, bytes); + } + return { manifest, manifestSha256, attemptId: value.attemptId, files }; +} + +export async function readControlledEvidence(path, options) { + const value = JSON.parse((await readRegularFile(path, MAX_ARCHIVE_BYTES)).toString('utf8')); + decodeControlledEvidence(value, options); + return value; +} + +export async function readAttemptControlledEvidence(directory, options) { + const bundle = await readControlledEvidence(join(directory, 'controller-evidence.json'), options); + const artifact = JSON.parse((await readRegularFile(join(directory, 'artifact.json'), 16 * MiB)).toString('utf8')); + const request = JSON.parse((await readRegularFile(join(directory, 'request.json'), 8 * MiB)).toString('utf8')); + const allocation = bundle.manifest.attempts.find(attempt => attempt.attemptId === bundle.attemptId); + if (basename(resolve(directory)) !== bundle.attemptId || artifact.attemptId !== bundle.attemptId + || requestBindingSha256(request) !== allocation.requestSha256 + || artifact.controllerAccounting && artifact.controllerAccounting.manifestSha256 !== bundle.manifestSha256) { + throw new Error('Immutable attempt does not match its controller archive'); + } + return bundle; +} + +export async function summarizeControlledEvidence(bundles, options) { + if (!Array.isArray(bundles) || !bundles.length) throw new Error('No controller evidence supplied'); + const groups = new Map(); + for (const bundle of bundles) { + const decoded = decodeControlledEvidence(bundle, options); + let group = groups.get(decoded.manifest.campaignId); + if (group && group.manifestSha256 !== decoded.manifestSha256) throw new Error('Conflicting controller campaign manifests'); + if (!group) { + group = { ...decoded, filesByAttempt: new Map(), digests: new Map() }; + groups.set(decoded.manifest.campaignId, group); + } + const digest = sha256(canonical(bundle)); + if (group.digests.has(decoded.attemptId) && group.digests.get(decoded.attemptId) !== digest) throw new Error('Conflicting controller attempt archives'); + group.digests.set(decoded.attemptId, digest); + group.filesByAttempt.set(decoded.attemptId, decoded.files); + } + const campaigns = []; + for (const group of [...groups.values()].sort((a, b) => a.manifest.campaignId.localeCompare(b.manifest.campaignId))) { + campaigns.push(await summarizeControlledCampaign(group.manifest, { filesByAttempt: group.filesByAttempt })); + } + const sum = field => usd(campaigns.reduce((total, campaign) => total + units(campaign[field]), 0n)); + return { schema: 'paperclip.runner-eval.control-history/v1', costStatus: campaigns.some(c => c.costStatus === 'incomplete') ? 'incomplete' : 'provisional', + knownUsd: sum('knownUsd'), heldUsd: sum('heldUsd'), committedUsd: sum('committedUsd'), operatingHeldUsd: sum('operatingHeldUsd'), + incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const [command, ...args] = process.argv.slice(2); + if (command === 'verify' && args.length === 2) { + const { manifest } = validateManifest(await readJson(args[0])); + bindRequest(manifest, await readJson(args[1], 8 * MiB)); + } else if (command === 'pack' && args.length === 3) { + const bundle = await packControlledAttempt({ manifest: await readJson(args[0]), attemptId: args[1] }); + await writeFile(args[2], JSON.stringify(bundle) + '\n', { flag: 'wx', mode: 0o600 }); + } else if (['report', 'report-attempts'].includes(command) && args.length) { + const read = command === 'report' ? readControlledEvidence : readAttemptControlledEvidence; + const result = await summarizeControlledEvidence(await Promise.all(args.map(path => read(path)))); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); + if (result.costStatus === 'incomplete') process.exitCode = 2; + } else throw new Error('Usage: archive.mjs verify | pack | report ... | report-attempts ...'); +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index a4af19aa000..5fac90ff3b1 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -7,6 +7,8 @@ import { join } from 'node:path'; import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; import { attemptDirectory, runControlledEval } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; +import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; +import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; const childScript = options => ` import assert from 'node:assert/strict'; @@ -277,3 +279,126 @@ test('cancellation still kills an owned resistant descendant after the CLI exits assert.equal(result.report.cleanupVerified, false); assert.equal((await summarizeControlledCampaign(f.manifest)).committedUsd, '0.014000000'); }); + + +test('portable archives replay after worker deletion, deduplicate attempts, and hold absent cells', async t => { + const f = await fixture(t, { count: 2, requests: 1 }); + await f.run(0); + const first = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: f.environment }); + const partial = await summarizeControlledEvidence([first, first]); + assert.equal(partial.incompleteAttempts, 1); + assert.equal(partial.knownUsd, '0.000068000'); + assert.equal(partial.committedUsd, '0.014068000'); + await f.run(1); + const second = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-1' }); + const expected = await summarizeControlledCampaign(f.manifest); + await rm(f.manifest.outputDirectory, { recursive: true }); + const restored = await summarizeControlledEvidence([first, second, first]); + assert.deepEqual(restored.campaigns, [expected]); + assert.equal(restored.operatingHeldUsd, '0.004000000'); + assert.equal(restored.knownUsd, '0.000136000'); + assert.equal(restored.costStatus, 'provisional'); +}); + +test('archived partial proof preserves known costs without consulting intact worker files', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + for (const name of ['admission.json', 'request.json', 'result.json', Object.keys(bundle.files).find(name => name.startsWith('receipts/'))]) { + const partial = structuredClone(bundle); delete partial.files[name]; + const summary = await summarizeControlledEvidence([partial]); + assert.equal(summary.incompleteAttempts, 1, name); + assert.equal(summary.knownUsd, '0.000068000', name); + assert.equal(summary.committedUsd, '0.014000000', name); + } + const absent = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + absent.files = {}; + assert.equal((await summarizeControlledEvidence([absent])).committedUsd, '0.014000000'); +}); + +test('archive creation retains never-started selections and refuses nonregular or oversized proof', async t => { + const f = await fixture(t); + const empty = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + assert.deepEqual(empty.files, {}); + assert.equal((await summarizeControlledEvidence([empty])).heldUsd, '0.014000000'); + await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + await rm(join(root, 'stderr.log')); + await symlink(f.manifest.runnerCli.path, join(root, 'stderr.log')); + await assert.rejects(packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' })); + await rm(join(root, 'stderr.log')); + await writeFile(join(root, 'stderr.log'), Buffer.alloc(1_048_577)); + await assert.rejects(packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }), /bounded regular file/); +}); + +test('archive validation rejects corruption, traversal, and conflicting campaign or attempt evidence', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + for (const mutate of [ + b => { b.manifest.campaignId = 'other'; }, + b => { b.files['ledger.jsonl'].base64 = 'x' + b.files['ledger.jsonl'].base64.slice(1); }, + b => { b.files['../outside'] = b.files['ledger.jsonl']; }, + b => { b.files['stderr.log'].sizeBytes = -1; }, + ]) { const b = structuredClone(bundle); mutate(b); assert.throws(() => decodeControlledEvidence(b)); } + const conflict = structuredClone(bundle); delete conflict.files['stderr.log']; + await assert.rejects(summarizeControlledEvidence([bundle, conflict]), /Conflicting controller attempt/); + const other = structuredClone(bundle); other.manifest.maxUsd = '2'; other.manifestSha256 = sha256(canonical(other.manifest)); + await assert.rejects(summarizeControlledEvidence([bundle, other]), /Conflicting controller campaign/); +}); + +test('configured credentials are excluded before encoding and after archive decoding', async t => { + const f = await fixture(t); await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const secret = 'synthetic-braintrust-archive-secret'; + const environment = { BRAINTRUST_API_KEY: secret }; + await writeFile(join(root, 'stderr.log'), secret); + await assert.rejects(packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment }), /configured credential/); + const unchecked = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: {} }); + assert(!JSON.stringify(unchecked).includes(secret)); + assert.throws(() => decodeControlledEvidence(unchecked, { environment }), /configured credential/); + const file = join(f.root, 'archive.json'); await writeFile(file, JSON.stringify(unchecked)); + await assert.rejects(readControlledEvidence(file, { environment }), /configured credential/); +}); + +test('campaign collection retains and audits private archives, while public projection omits them', async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + const download = join(f.root, 'downloads', 'cell'), attempt = join(download, 'runs', 'attempt-0'); + await mkdir(attempt, { recursive: true }); + const evalCase = { id: 'case-0', checks: [] }, config = { model: 'fixture', provider: 'codex', driver: 'codex_app_server' }; + for (const [name, value] of Object.entries({ + 'request.json': f.requests[0].request, + 'artifact.json': JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')), + 'score.json': { attemptId: 'attempt-0', caseId: evalCase.id, passed: true, disposition: 'passed', checks: [] }, + 'case.json': evalCase, 'config.json': config, 'controller-evidence.json': bundle, + })) await writeFile(join(attempt, name), JSON.stringify(value)); + const cell = { cellId: 'execution-0', caseId: evalCase.id, rosterFile: 'roster.json', rosterId: 'roster-0', ...config }; + const catalogPath = join(f.root, 'catalog.json'); + await writeFile(catalogPath, JSON.stringify({ schema: 'paperclip.runner-protocol-eval.catalog/v1', campaignId: f.manifest.campaignId, + selection: { kind: 'subset' }, cells: [cell], rosters: [{ ...cell, cases: [evalCase.id] }] })); + await writeFile(join(download, 'cell.json'), JSON.stringify({ ...cell, exitCode: 0 })); + await rm(f.manifest.outputDirectory, { recursive: true }); + const input = { catalogPath, downloadsRoot: join(f.root, 'downloads'), evalsRoot: f.root, + runsOut: join(f.root, 'collected'), campaignOut: join(f.root, 'campaign.json'), + source: { paperclip: { sha: f.manifest.sourceRevision }, evals: { sha: f.manifest.definitionsRevision } } }; + const result = await aggregateProtocolEvalCampaign(input); + assert.equal(result.controlledCosts.knownUsd, '0.000068000'); + assert.equal(result.controlledCosts.costStatus, 'provisional'); + assert.deepEqual(JSON.parse(await readFile(join(input.runsOut, 'attempt-0', 'controller-evidence.json'), 'utf8')), bundle); + const publicRoot = join(f.root, 'public'); + await sanitizeProtocolEvalRuns({ runsRoot: input.runsOut, publicRunsRoot: publicRoot }); + assert.deepEqual((await readdir(join(publicRoot, 'attempt-0'))).sort(), ['artifact.json', 'case.json', 'config.json', 'score.json']); + assert(!(await readFile(join(publicRoot, 'attempt-0', 'artifact.json'), 'utf8')).includes('controllerAccounting')); + const changedRequest = { ...f.requests[0].request, prompt: 'copied from another execution' }; + await writeFile(join(attempt, 'request.json'), JSON.stringify(changedRequest)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable attempt does not match/); + await writeFile(join(attempt, 'request.json'), JSON.stringify(f.requests[0].request)); + const drifted = structuredClone(bundle); drifted.manifest.attempts[0].executionId = 'not-the-selected-cell'; + drifted.manifestSha256 = sha256(canonical(drifted.manifest)); + await writeFile(join(attempt, 'controller-evidence.json'), JSON.stringify(drifted)); + const bareArtifact = JSON.parse(await readFile(join(attempt, 'artifact.json'), 'utf8')); delete bareArtifact.controllerAccounting; + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(bareArtifact)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + await writeFile(join(attempt, 'artifact.json'), await readFile(f.requests[0].outputPath)); + await rm(join(attempt, 'controller-evidence.json')); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs index 23e4f996036..4dc48445cf4 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/report.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -4,16 +4,27 @@ import { pathToFileURL } from 'node:url'; import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { attemptDirectory } from './run.mjs'; -export async function summarizeControlledCampaign(value) { +export async function summarizeControlledCampaign(value, { filesByAttempt } = {}) { const { manifest, manifestSha256, allocatedUsd, overheadUsd } = validateManifest(value, Date.now(), { admission: false }); let known = 0n, committed = units(overheadUsd); const attempts = []; for (const allocation of manifest.attempts) { const root = attemptDirectory(manifest, allocation.attemptId); + const readEvidence = async name => { + if (filesByAttempt === undefined) return readFile(join(root, name)); + const bytes = filesByAttempt.get(allocation.attemptId)?.get(name); + if (!Buffer.isBuffer(bytes)) throw new Error('Missing archived controller evidence'); + return bytes; + }; + const readDocument = async (name, maximum = 1_048_576) => { + const bytes = await readEvidence(name); + if (bytes.length > maximum) throw new Error('Controller document exceeds its byte limit'); + return JSON.parse(bytes.toString('utf8')); + }; let snapshot, report, verified = false; try { - const ledgerBytes = await readFile(join(root, 'ledger.jsonl')); - const ledger = await costRuntime.readEvalCostLedger(join(root, 'ledger.jsonl')); + const ledgerBytes = await readEvidence('ledger.jsonl'); + const ledger = costRuntime.parseEvalCostLedger(ledgerBytes.toString('utf8')); const candidateSnapshot = ledger.state.snapshot(); const ledgerDeadline = Date.parse(ledger.state.limits.deadlineAt); const fixedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: ledger.state.limits.deadlineAt, @@ -23,7 +34,7 @@ export async function summarizeControlledCampaign(value) { || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }]) || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger identity does not match its allocation'); snapshot = candidateSnapshot; - const admission = await readJson(join(root, 'admission.json')); + const admission = await readDocument('admission.json'); const started = Date.parse(admission.startedAt); const expectedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: new Date(Math.min(Date.parse(manifest.deadlineAt), started + allocation.timeoutMs)).toISOString(), @@ -31,20 +42,20 @@ export async function summarizeControlledCampaign(value) { if (admission.schema !== 'paperclip.runner-eval.admission/v1' || admission.manifestSha256 !== manifestSha256 || admission.campaignId !== manifest.campaignId || canonical(admission.attempt) !== canonical(allocation) || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits)) throw new Error('Ledger does not match its admission'); - report = await readJson(join(root, 'result.json')); - const request = await readJson(join(root, 'request.json'), 8_388_608); + report = await readDocument('result.json'); + const request = await readDocument('request.json', 8_388_608); if (requestBindingSha256(request) !== allocation.requestSha256 || request.runnerd.sha256.replace(/^sha256:/, '') !== manifest.runnerd.sha256 || canonical(Object.keys(report.receiptHashes ?? {}).sort()) !== canonical(snapshot.requests.map(item => item.requestId).sort())) throw new Error('Retained request evidence does not match'); for (const item of snapshot.requests) { if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(item.requestId)) throw new Error('Invalid retained receipt identity'); - const bytes = await readFile(join(root, 'receipts', `${item.requestId}.json`)); + const bytes = await readEvidence(`receipts/${item.requestId}.json`); const receipt = JSON.parse(bytes.toString('utf8')); if (sha256(bytes) !== report.receiptHashes[item.requestId] || receipt.requestId !== item.requestId || receipt.route !== manifest.route || receipt.costUsd !== item.costUsd) throw new Error('Retained receipt does not match its ledger'); } - const artifactBytes = await readFile(join(root, 'provider-artifact.json')); + const artifactBytes = await readEvidence('provider-artifact.json'); const artifact = JSON.parse(artifactBytes.toString('utf8')); - const descriptorBytes = await readFile(join(root, 'gateway.json')); + const descriptorBytes = await readEvidence('gateway.json'); const descriptor = JSON.parse(descriptorBytes.toString('utf8')); if (report.schema !== 'paperclip.runner-eval.control-result/v1' || report.manifestSha256 !== manifestSha256 || report.campaignId !== manifest.campaignId || report.executionId !== allocation.executionId || report.attemptId !== allocation.attemptId diff --git a/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts b/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts index 38f45e0839a..b8591390e09 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts +++ b/packages/paperclip-runner/scripts/controlled-evals/shared-cost-runtime.ts @@ -1,3 +1,3 @@ -export { EvalSpendState, openEvalCostLedger, readEvalCostLedger, validEvalLedgerId } from '../../../../tests/runner-e2e/cost-ledger.js'; +export { EvalSpendState, openEvalCostLedger, readEvalCostLedger, parseEvalCostLedger, validEvalLedgerId } from '../../../../tests/runner-e2e/cost-ledger.js'; export { evalRequestReservation, startEvalGateway } from '../../../../tests/runner-e2e/eval-gateway.js'; export { EVAL_OVERHEAD_COMPONENTS, validateOperatingOverhead } from '../../../../tests/runner-e2e/operating-overhead.js'; diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index 2ea5c4f7cb2..e13a218700f 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -26,6 +26,7 @@ const ATTEMPT_FILES = new Set([ "compiled-fixture.json", "case.json", "config.json", + "controller-evidence.json", ]); function json(value) { @@ -405,6 +406,7 @@ export async function aggregateProtocolEvalCampaign({ catalog.cells.map((cell) => [cell.cellId, cell]), ); const copiedAttemptIds = new Set(); + const controllerBundles = []; for (const statusPath of await findFiles(downloadsRoot, "cell.json")) { const status = await loadObject(statusPath); const cellId = safeId(status.cellId, "recorded cell ID"); @@ -453,6 +455,25 @@ export async function aggregateProtocolEvalCampaign({ ) { throw new Error(`Downloaded attempt identity drifted for ${cellId}`); } + const bundlePath = join(attemptRoot, entry.name, "controller-evidence.json"); + const bundleStat = await lstat(bundlePath).catch(error => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (bundleStat) { + const { readAttemptControlledEvidence } = await import("./controlled-evals/archive.mjs"); + const bundle = await readAttemptControlledEvidence(join(attemptRoot, entry.name)); + const allocation = bundle.manifest.attempts.find(item => item.attemptId === attemptId); + const selectedExecutions = new Set(bundle.manifest.attempts.map(item => item.executionId)); + if (bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || + selectedExecutions.size !== expectedByCell.size || [...expectedByCell.keys()].some(id => !selectedExecutions.has(id)) || + bundle.manifest.sourceRevision !== source.paperclip.sha || bundle.manifest.definitionsRevision !== source.evals.sha) { + throw new Error(`Controller archive identity drifted for ${cellId}`); + } + controllerBundles.push(bundle); + } else if (artifact.controllerAccounting) { + throw new Error(`Missing private controller evidence for ${attemptId}; cost metadata alone is insufficient`); + } await copyAttempt( join(attemptRoot, entry.name), join(runsOut, attemptId), @@ -518,6 +539,15 @@ export async function aggregateProtocolEvalCampaign({ (result) => result.disposition === "infrastructure_failure", ).length, }; + let controlledCosts; + if (controllerBundles.length) { + const { summarizeControlledEvidence } = await import("./controlled-evals/archive.mjs"); + controlledCosts = await summarizeControlledEvidence(controllerBundles); + const allocatedAttempts = new Set(controlledCosts.campaigns.flatMap(campaign => campaign.attempts.map(attempt => attempt.attemptId))); + if ([...copiedAttemptIds].some(id => !allocatedAttempts.has(id))) { + throw new Error("Retained attempt is outside the controlled campaign allocation"); + } + } const generatedAt = new Date().toISOString(); const campaign = { schema: "paperclip.runner-protocol-eval.campaign/v1", @@ -525,6 +555,7 @@ export async function aggregateProtocolEvalCampaign({ generatedAt, selection: catalog.selection, costs: sumAttemptCosts(attemptUsages), + ...(controlledCosts ? { controlledCosts } : {}), source: { paperclip: source.paperclip, evals: source.evals, diff --git a/tests/runner-e2e/cost-ledger.ts b/tests/runner-e2e/cost-ledger.ts index daa95eba7b3..576fd8a2385 100644 --- a/tests/runner-e2e/cost-ledger.ts +++ b/tests/runner-e2e/cost-ledger.ts @@ -222,7 +222,10 @@ export class EvalSpendState { } export async function readEvalCostLedger(file: string) { - const raw = await readFile(file, "utf8"); + return parseEvalCostLedger(await readFile(file, "utf8")); +} + +export function parseEvalCostLedger(raw: string) { if (!raw.endsWith("\n")) throw new Error("Eval ledger has an incomplete durable record"); const lines = raw.trimEnd().split("\n").map(value => JSON.parse(value) as Line); let previous = "", state: EvalSpendState | undefined;