diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index cf08f52de63..4319d7f76c6 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -498,3 +498,15 @@ not delivered by it. The controlled Codex profile disables web search. The shared proxy accepts local function/custom tool namespaces and still rejects hosted tools, including any hidden inside a namespace. These routes do not qualify web-search charges. + +The controller stages the exact verified CLI and daemon bytes in its private +attempt directory. A Node module-resolution hook keeps the CLI's relative +imports pointed at the trusted source checkout; it does not reload the original +entry file. Installed dependencies still belong to that trusted checkout. The +staged copies are removed after verified process cleanup and kept if cleanup is +incomplete. This path requires the repository's Node 24.11+ runtime. + +Campaign reports verify the retained request binding and the digest of every +request receipt before reducing a cell's allocation. Missing admission, request, +or receipt evidence preserves known ledger charges and holds the rest of the +full allocation. diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index afa7e6bb6e0..bceb4a58568 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; @@ -198,3 +199,41 @@ test('existing outputs and physical workspace aliases are rejected before provid await assert.rejects(f.run(0, transport), /resolves inside/); assert.equal(calls, 0); }); + +for (const missing of ['request.json', 'admission.json', 'receipt']) test(`lost ${missing} keeps known charges and the full allocation`, async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const file = missing === 'receipt' ? join(root, 'receipts', (await readdir(join(root, 'receipts')))[0]) : join(root, missing); + await rm(file); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.knownUsd, '0.000068000'); + assert.equal(summary.committedUsd, '0.014000000'); + assert.equal(summary.incompleteAttempts, 1); +}); + +for (const changed of ['request', 'receipt']) test(`changed ${changed} evidence cannot release the allocation`, async t => { + const f = await fixture(t, { requests: 1 }); await f.run(); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const file = changed === 'receipt' ? join(root, 'receipts', (await readdir(join(root, 'receipts')))[0]) : join(root, 'request.json'); + const value = JSON.parse(await readFile(file, 'utf8')); + if (changed === 'request') value.prompt = 'different request'; else value.costUsd = '0.000000000'; + await writeFile(file, JSON.stringify(value)); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.knownUsd, '0.000068000'); + assert.equal(summary.committedUsd, '0.014000000'); + assert.equal(summary.incompleteAttempts, 1); +}); + +for (const replaced of ['runnerCli', 'runnerd']) test(`launches verified bytes after the original ${replaced} path is replaced`, async t => { + const f = await fixture(t, { requests: 1 }); + await writeFile(f.manifest.runnerCli.path, childScript({ requests: 1 }) + "\nassert.equal(await readFile(request.runnerd.path, 'utf8'), 'synthetic daemon');\n"); + f.manifest.runnerCli.sha256 = sha256(await readFile(f.manifest.runnerCli.path)); await f.save(); + Object.defineProperty(f.environment, 'OPENAI_API_KEY', { get() { + writeFileSync(f.manifest[replaced].path, replaced === 'runnerCli' ? "throw new Error('replacement must not execute');" : 'replaced daemon'); + return 'synthetic-upstream-private'; + }}); + const result = await f.run(); + assert.equal(result.code, 0); assert.equal(result.report.cleanupVerified, true); + assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0); + await assert.rejects(readdir(join(attemptDirectory(f.manifest, 'attempt-0'), 'executables')), { code: 'ENOENT' }); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs index dcbd145a170..23e4f996036 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/report.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -1,7 +1,7 @@ import { readFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { canonical, costRuntime, readJson, sha256, units, usd, validateManifest } from './manifest.mjs'; +import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { attemptDirectory } from './run.mjs'; export async function summarizeControlledCampaign(value) { @@ -12,21 +12,36 @@ export async function summarizeControlledCampaign(value) { const root = attemptDirectory(manifest, allocation.attemptId); let snapshot, report, verified = false; try { - const admission = await readJson(join(root, 'admission.json')); const ledgerBytes = await readFile(join(root, 'ledger.jsonl')); const ledger = await costRuntime.readEvalCostLedger(join(root, 'ledger.jsonl')); const candidateSnapshot = ledger.state.snapshot(); + const ledgerDeadline = Date.parse(ledger.state.limits.deadlineAt); + const fixedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: ledger.state.limits.deadlineAt, + maxRequests: allocation.maxRequests, maxConcurrent: allocation.maxConcurrent, maxAttemptsPerExecution: 1 }; + if (canonical(ledger.state.limits) !== canonical(fixedLimits) || !Number.isFinite(ledgerDeadline) + || ledgerDeadline <= Date.parse(manifest.pricingVerifiedAt) || ledgerDeadline > Date.parse(manifest.deadlineAt) + || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }]) + || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger identity does not match its allocation'); + snapshot = candidateSnapshot; + const admission = await readJson(join(root, 'admission.json')); const started = Date.parse(admission.startedAt); const expectedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: new Date(Math.min(Date.parse(manifest.deadlineAt), started + allocation.timeoutMs)).toISOString(), maxRequests: allocation.maxRequests, maxConcurrent: allocation.maxConcurrent, maxAttemptsPerExecution: 1 }; if (admission.schema !== 'paperclip.runner-eval.admission/v1' || admission.manifestSha256 !== manifestSha256 || admission.campaignId !== manifest.campaignId || canonical(admission.attempt) !== canonical(allocation) - || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits) - || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }]) - || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger does not match its allocation'); - snapshot = candidateSnapshot; + || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits)) throw new Error('Ledger does not match its admission'); report = await readJson(join(root, 'result.json')); + const request = await readJson(join(root, 'request.json'), 8_388_608); + if (requestBindingSha256(request) !== allocation.requestSha256 || request.runnerd.sha256.replace(/^sha256:/, '') !== manifest.runnerd.sha256 + || canonical(Object.keys(report.receiptHashes ?? {}).sort()) !== canonical(snapshot.requests.map(item => item.requestId).sort())) throw new Error('Retained request evidence does not match'); + for (const item of snapshot.requests) { + if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(item.requestId)) throw new Error('Invalid retained receipt identity'); + const bytes = await readFile(join(root, 'receipts', `${item.requestId}.json`)); + const receipt = JSON.parse(bytes.toString('utf8')); + if (sha256(bytes) !== report.receiptHashes[item.requestId] || receipt.requestId !== item.requestId + || receipt.route !== manifest.route || receipt.costUsd !== item.costUsd) throw new Error('Retained receipt does not match its ledger'); + } const artifactBytes = await readFile(join(root, 'provider-artifact.json')); const artifact = JSON.parse(artifactBytes.toString('utf8')); const descriptorBytes = await readFile(join(root, 'gateway.json')); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 761b9e077d1..fc9fa6d217e 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process'; import { once } from 'node:events'; -import { lstat, mkdir, open, readFile, realpath, rename, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import { parseArgs } from 'node:util'; @@ -17,7 +17,7 @@ const absent = pid => { try { process.kill(pid, 0); return false; } catch (error export const attemptDirectory = (manifest, attemptId) => join(manifest.outputDirectory, sha256(attemptId)); async function runChild(input) { - const child = spawn(process.execPath, [input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], { + const child = spawn(process.execPath, ['--import', input.loader, input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], { cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); let stopped = null, force; @@ -57,8 +57,11 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, const suffix = relative(resolve(request.session.workingDirectory), resolve(manifest.outputDirectory)); if (!suffix || suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) throw new Error('Controller evidence must be outside the disposable workspace'); } - for (const artifact of [manifest.runnerCli, manifest.runnerPackage, manifest.runnerd]) { - if (sha256(await readFile(artifact.path)) !== artifact.sha256) throw new Error('Controlled eval executable provenance changed'); + const verified = {}; + for (const name of ['runnerCli', 'runnerPackage', 'runnerd']) { + const artifact = manifest[name], bytes = await readFile(artifact.path); + if (sha256(bytes) !== artifact.sha256) throw new Error('Controlled eval executable provenance changed'); + if (name !== 'runnerPackage') verified[name] = bytes; } const provider = attempt.provider === 'codex' ? 'openai' : 'anthropic'; const keyName = manifest.route === 'braintrust' ? 'BRAINTRUST_API_KEY' : provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY'; @@ -78,8 +81,19 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, maxRequests: attempt.maxRequests, maxConcurrent: attempt.maxConcurrent, maxAttemptsPerExecution: 1 }; await atomicJson(join(root, 'admission.json'), { schema: 'paperclip.runner-eval.admission/v1', campaignId: manifest.campaignId, manifestSha256, attempt, limits, startedAt }); + const executables = join(root, 'executables'); + await mkdir(executables, { mode: 0o700 }); + const stagedCli = join(executables, 'entry.mjs'), stagedRunnerd = join(executables, 'runnerd'), loader = join(executables, 'imports.mjs'); + await writeFile(stagedCli, verified.runnerCli, { flag: 'wx', mode: 0o400 }); + await writeFile(stagedRunnerd, verified.runnerd, { flag: 'wx', mode: 0o500 }); + const stagedUrl = JSON.stringify(pathToFileURL(stagedCli).href), originalUrl = JSON.stringify(pathToFileURL(manifest.runnerCli.path).href); + await writeFile(loader, `import { registerHooks } from 'node:module'; +registerHooks({ resolve(specifier, context, nextResolve) { + return nextResolve(specifier, context.parentURL === ${stagedUrl} ? { ...context, parentURL: ${originalUrl} } : context); +} });\n`, { flag: 'wx', mode: 0o400 }); const retainedRequestPath = join(root, 'request.json'); - await writeFile(retainedRequestPath, JSON.stringify(request) + '\n', { flag: 'wx', mode: 0o600 }); + const childRequest = structuredClone(request); childRequest.runnerd.path = stagedRunnerd; + await writeFile(retainedRequestPath, JSON.stringify(childRequest) + '\n', { flag: 'wx', mode: 0o600 }); await mkdir(join(root, 'receipts'), { mode: 0o700 }); await mkdir(join(root, 'home'), { mode: 0o700 }); await mkdir(join(root, 'tmp'), { mode: 0o700 }); @@ -105,7 +119,7 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, const artifactPath = join(root, 'provider-artifact.json'); signal?.throwIfAborted(); if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn'); - child = await runChild({ cli: manifest.runnerCli.path, requestPath: retainedRequestPath, artifactPath, descriptorPath, + child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => gateway.close() }); for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); const raw = await readFile(artifactPath, 'utf8'); @@ -129,14 +143,22 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, try { await gateway?.close(); } catch { failure = 'gateway_cleanup_unverified'; } try { await ledger.close(); } catch { failure = 'ledger_close_failed'; } } + if (cleanupVerified) { + try { await rm(executables, { recursive: true }); } + catch { cleanupVerified = false; failure ??= 'executable_cleanup_unverified'; } + } const snapshot = ledger.snapshot(); if (snapshot.overBudget || snapshot.conflictingRequests || snapshot.inflight || snapshot.unknownRequests !== snapshot.boundedUnresolvedRequests) failure ??= 'request_accounting_unbounded'; + const receiptHashes = {}; + try { + for (const request of snapshot.requests) receiptHashes[request.requestId] = sha256(await readFile(join(root, 'receipts', `${request.requestId}.json`))); + } catch { failure ??= 'request_receipt_missing'; } const ledgerSha256 = sha256(await readFile(ledgerPath)); const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256, executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256, allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure, child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null, - cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, accounting: snapshot }; + cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, receiptHashes, accounting: snapshot }; await atomicJson(join(root, 'result.json'), report); const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' };