fix(evals): retain accounting proof and pin launched bytes

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -498,3 +498,15 @@ not delivered by it.
 The controlled Codex profile disables web search. The shared proxy accepts local
 function/custom tool namespaces and still rejects hosted tools, including any
 hidden inside a namespace. These routes do not qualify web-search charges.
+
+The controller stages the exact verified CLI and daemon bytes in its private
+attempt directory. A Node module-resolution hook keeps the CLI's relative
+imports pointed at the trusted source checkout; it does not reload the original
+entry file. Installed dependencies still belong to that trusted checkout. The
+staged copies are removed after verified process cleanup and kept if cleanup is
+incomplete. This path requires the repository's Node 24.11+ runtime.
+
+Campaign reports verify the retained request binding and the digest of every
+request receipt before reducing a cell's allocation. Missing admission, request,
+or receipt evidence preserves known ledger charges and holds the rest of the
+full allocation.

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict';
 import { test } from 'node:test';
-import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
+import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
+import { writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs';
@@ -198,3 +199,41 @@ test('existing outputs and physical workspace aliases are rejected before provid
   await assert.rejects(f.run(0, transport), /resolves inside/);
   assert.equal(calls, 0);
 });
+
+for (const missing of ['request.json', 'admission.json', 'receipt']) test(`lost ${missing} keeps known charges and the full allocation`, async t => {
+  const f = await fixture(t, { requests: 1 }); await f.run();
+  const root = attemptDirectory(f.manifest, 'attempt-0');
+  const file = missing === 'receipt' ? join(root, 'receipts', (await readdir(join(root, 'receipts')))[0]) : join(root, missing);
+  await rm(file);
+  const summary = await summarizeControlledCampaign(f.manifest);
+  assert.equal(summary.knownUsd, '0.000068000');
+  assert.equal(summary.committedUsd, '0.014000000');
+  assert.equal(summary.incompleteAttempts, 1);
+});
+
+for (const changed of ['request', 'receipt']) test(`changed ${changed} evidence cannot release the allocation`, async t => {
+  const f = await fixture(t, { requests: 1 }); await f.run();
+  const root = attemptDirectory(f.manifest, 'attempt-0');
+  const file = changed === 'receipt' ? join(root, 'receipts', (await readdir(join(root, 'receipts')))[0]) : join(root, 'request.json');
+  const value = JSON.parse(await readFile(file, 'utf8'));
+  if (changed === 'request') value.prompt = 'different request'; else value.costUsd = '0.000000000';
+  await writeFile(file, JSON.stringify(value));
+  const summary = await summarizeControlledCampaign(f.manifest);
+  assert.equal(summary.knownUsd, '0.000068000');
+  assert.equal(summary.committedUsd, '0.014000000');
+  assert.equal(summary.incompleteAttempts, 1);
+});
+
+for (const replaced of ['runnerCli', 'runnerd']) test(`launches verified bytes after the original ${replaced} path is replaced`, async t => {
+  const f = await fixture(t, { requests: 1 });
+  await writeFile(f.manifest.runnerCli.path, childScript({ requests: 1 }) + "\nassert.equal(await readFile(request.runnerd.path, 'utf8'), 'synthetic daemon');\n");
+  f.manifest.runnerCli.sha256 = sha256(await readFile(f.manifest.runnerCli.path)); await f.save();
+  Object.defineProperty(f.environment, 'OPENAI_API_KEY', { get() {
+    writeFileSync(f.manifest[replaced].path, replaced === 'runnerCli' ? "throw new Error('replacement must not execute');" : 'replaced daemon');
+    return 'synthetic-upstream-private';
+  }});
+  const result = await f.run();
+  assert.equal(result.code, 0); assert.equal(result.report.cleanupVerified, true);
+  assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0);
+  await assert.rejects(readdir(join(attemptDirectory(f.manifest, 'attempt-0'), 'executables')), { code: 'ENOENT' });
+});

packages/paperclip-runner/scripts/controlled-evals/report.mjs

@@ -1,7 +1,7 @@
 import { readFile } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
-import { canonical, costRuntime, readJson, sha256, units, usd, validateManifest } from './manifest.mjs';
+import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
 import { attemptDirectory } from './run.mjs';
 
 export async function summarizeControlledCampaign(value) {
@@ -12,21 +12,36 @@ export async function summarizeControlledCampaign(value) {
     const root = attemptDirectory(manifest, allocation.attemptId);
     let snapshot, report, verified = false;
     try {
-      const admission = await readJson(join(root, 'admission.json'));
       const ledgerBytes = await readFile(join(root, 'ledger.jsonl'));
       const ledger = await costRuntime.readEvalCostLedger(join(root, 'ledger.jsonl'));
       const candidateSnapshot = ledger.state.snapshot();
+      const ledgerDeadline = Date.parse(ledger.state.limits.deadlineAt);
+      const fixedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd, deadlineAt: ledger.state.limits.deadlineAt,
+        maxRequests: allocation.maxRequests, maxConcurrent: allocation.maxConcurrent, maxAttemptsPerExecution: 1 };
+      if (canonical(ledger.state.limits) !== canonical(fixedLimits) || !Number.isFinite(ledgerDeadline)
+          || ledgerDeadline <= Date.parse(manifest.pricingVerifiedAt) || ledgerDeadline > Date.parse(manifest.deadlineAt)
+          || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }])
+          || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger identity does not match its allocation');
+      snapshot = candidateSnapshot;
+      const admission = await readJson(join(root, 'admission.json'));
       const started = Date.parse(admission.startedAt);
       const expectedLimits = { campaignId: manifest.campaignId, maxUsd: allocation.maxUsd,
         deadlineAt: new Date(Math.min(Date.parse(manifest.deadlineAt), started + allocation.timeoutMs)).toISOString(),
         maxRequests: allocation.maxRequests, maxConcurrent: allocation.maxConcurrent, maxAttemptsPerExecution: 1 };
       if (admission.schema !== 'paperclip.runner-eval.admission/v1' || admission.manifestSha256 !== manifestSha256
           || admission.campaignId !== manifest.campaignId || canonical(admission.attempt) !== canonical(allocation)
-          || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits)
-          || canonical(candidateSnapshot.admittedAttempts) !== canonical([{ executionId: allocation.executionId, attemptId: allocation.attemptId }])
-          || candidateSnapshot.requests.some(request => request.executionId !== allocation.executionId || request.attemptId !== allocation.attemptId || request.phase !== 'agent')) throw new Error('Ledger does not match its allocation');
-      snapshot = candidateSnapshot;
+          || canonical(admission.limits) !== canonical(expectedLimits) || canonical(ledger.state.limits) !== canonical(expectedLimits)) throw new Error('Ledger does not match its admission');
       report = await readJson(join(root, 'result.json'));
+      const request = await readJson(join(root, 'request.json'), 8_388_608);
+      if (requestBindingSha256(request) !== allocation.requestSha256 || request.runnerd.sha256.replace(/^sha256:/, '') !== manifest.runnerd.sha256
+          || canonical(Object.keys(report.receiptHashes ?? {}).sort()) !== canonical(snapshot.requests.map(item => item.requestId).sort())) throw new Error('Retained request evidence does not match');
+      for (const item of snapshot.requests) {
+        if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(item.requestId)) throw new Error('Invalid retained receipt identity');
+        const bytes = await readFile(join(root, 'receipts', `${item.requestId}.json`));
+        const receipt = JSON.parse(bytes.toString('utf8'));
+        if (sha256(bytes) !== report.receiptHashes[item.requestId] || receipt.requestId !== item.requestId
+            || receipt.route !== manifest.route || receipt.costUsd !== item.costUsd) throw new Error('Retained receipt does not match its ledger');
+      }
       const artifactBytes = await readFile(join(root, 'provider-artifact.json'));
       const artifact = JSON.parse(artifactBytes.toString('utf8'));
       const descriptorBytes = await readFile(join(root, 'gateway.json'));

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -1,6 +1,6 @@
 import { spawn } from 'node:child_process';
 import { once } from 'node:events';
-import { lstat, mkdir, open, readFile, realpath, rename, writeFile } from 'node:fs/promises';
+import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises';
 import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseArgs } from 'node:util';
@@ -17,7 +17,7 @@ const absent = pid => { try { process.kill(pid, 0); return false; } catch (error
 export const attemptDirectory = (manifest, attemptId) => join(manifest.outputDirectory, sha256(attemptId));
 
 async function runChild(input) {
-  const child = spawn(process.execPath, [input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], {
+  const child = spawn(process.execPath, ['--import', input.loader, input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], {
     cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'],
   });
   let stopped = null, force;
@@ -57,8 +57,11 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath,
     const suffix = relative(resolve(request.session.workingDirectory), resolve(manifest.outputDirectory));
     if (!suffix || suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) throw new Error('Controller evidence must be outside the disposable workspace');
   }
-  for (const artifact of [manifest.runnerCli, manifest.runnerPackage, manifest.runnerd]) {
-    if (sha256(await readFile(artifact.path)) !== artifact.sha256) throw new Error('Controlled eval executable provenance changed');
+  const verified = {};
+  for (const name of ['runnerCli', 'runnerPackage', 'runnerd']) {
+    const artifact = manifest[name], bytes = await readFile(artifact.path);
+    if (sha256(bytes) !== artifact.sha256) throw new Error('Controlled eval executable provenance changed');
+    if (name !== 'runnerPackage') verified[name] = bytes;
   }
   const provider = attempt.provider === 'codex' ? 'openai' : 'anthropic';
   const keyName = manifest.route === 'braintrust' ? 'BRAINTRUST_API_KEY' : provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY';
@@ -78,8 +81,19 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath,
     maxRequests: attempt.maxRequests, maxConcurrent: attempt.maxConcurrent, maxAttemptsPerExecution: 1 };
   await atomicJson(join(root, 'admission.json'), { schema: 'paperclip.runner-eval.admission/v1', campaignId: manifest.campaignId,
     manifestSha256, attempt, limits, startedAt });
+  const executables = join(root, 'executables');
+  await mkdir(executables, { mode: 0o700 });
+  const stagedCli = join(executables, 'entry.mjs'), stagedRunnerd = join(executables, 'runnerd'), loader = join(executables, 'imports.mjs');
+  await writeFile(stagedCli, verified.runnerCli, { flag: 'wx', mode: 0o400 });
+  await writeFile(stagedRunnerd, verified.runnerd, { flag: 'wx', mode: 0o500 });
+  const stagedUrl = JSON.stringify(pathToFileURL(stagedCli).href), originalUrl = JSON.stringify(pathToFileURL(manifest.runnerCli.path).href);
+  await writeFile(loader, `import { registerHooks } from 'node:module';
+registerHooks({ resolve(specifier, context, nextResolve) {
+  return nextResolve(specifier, context.parentURL === ${stagedUrl} ? { ...context, parentURL: ${originalUrl} } : context);
+} });\n`, { flag: 'wx', mode: 0o400 });
   const retainedRequestPath = join(root, 'request.json');
-  await writeFile(retainedRequestPath, JSON.stringify(request) + '\n', { flag: 'wx', mode: 0o600 });
+  const childRequest = structuredClone(request); childRequest.runnerd.path = stagedRunnerd;
+  await writeFile(retainedRequestPath, JSON.stringify(childRequest) + '\n', { flag: 'wx', mode: 0o600 });
   await mkdir(join(root, 'receipts'), { mode: 0o700 });
   await mkdir(join(root, 'home'), { mode: 0o700 });
   await mkdir(join(root, 'tmp'), { mode: 0o700 });
@@ -105,7 +119,7 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath,
     const artifactPath = join(root, 'provider-artifact.json');
     signal?.throwIfAborted();
     if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn');
-    child = await runChild({ cli: manifest.runnerCli.path, requestPath: retainedRequestPath, artifactPath, descriptorPath,
+    child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath,
       cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => gateway.close() });
     for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 });
     const raw = await readFile(artifactPath, 'utf8');
@@ -129,14 +143,22 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath,
     try { await gateway?.close(); } catch { failure = 'gateway_cleanup_unverified'; }
     try { await ledger.close(); } catch { failure = 'ledger_close_failed'; }
   }
+  if (cleanupVerified) {
+    try { await rm(executables, { recursive: true }); }
+    catch { cleanupVerified = false; failure ??= 'executable_cleanup_unverified'; }
+  }
   const snapshot = ledger.snapshot();
   if (snapshot.overBudget || snapshot.conflictingRequests || snapshot.inflight || snapshot.unknownRequests !== snapshot.boundedUnresolvedRequests) failure ??= 'request_accounting_unbounded';
+  const receiptHashes = {};
+  try {
+    for (const request of snapshot.requests) receiptHashes[request.requestId] = sha256(await readFile(join(root, 'receipts', `${request.requestId}.json`)));
+  } catch { failure ??= 'request_receipt_missing'; }
   const ledgerSha256 = sha256(await readFile(ledgerPath));
   const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256,
     executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256,
     allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure,
     child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null,
-    cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, accounting: snapshot };
+    cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, receiptHashes, accounting: snapshot };
   await atomicJson(join(root, 'result.json'), report);
   const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId,
     requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' };