fix(evals): bound retained proof reads and credential exclusion

Reject nonregular and oversized evidence without blocking. Share the safe reader across execution, archive and report. Exclude configured private keys and JSON config credentials, reject hidden nested tool definitions, and constrain retained attempt directory names.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -366,6 +366,12 @@ hashes. These detect corruption; they are not signatures or proof against a
 host that can rewrite all evidence. Known configured credential values are
 checked before encoding and after decoding. Keep bundles private.
 
+Controller post-child reads, archive creation, and durable reports use the same
+nonblocking, no-follow, byte-capped regular-file reader. Configured private/provider
+keys and credentials inside JSON configuration contents are checked in both raw
+and JSON-escaped form. Attempt IDs contain only letters, digits, `_`, `.`, and `-`
+so each retained attempt has one directory name.
+
 The reader accepts only bounded regular files and fixed evidence paths. A
 bundle is limited to 64 MiB of decoded content plus encoding overhead; this
 storage-format limit does not qualify a vendor charge or operating-cost bound.

packages/paperclip-runner/scripts/controlled-evals/archive.mjs

@@ -1,46 +1,38 @@
-import { constants } from 'node:fs';
-import { lstat, open, readdir, writeFile } from 'node:fs/promises';
+import { lstat, readdir, writeFile } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
 import { attemptDirectory, controlledResultArtifact } from './run.mjs';
 import { summarizeControlledCampaign } from './report.mjs';
+import { FILE_LIMITS, RECEIPT, limitFor, readRegularFile } from './evidence-files.mjs';
 
 const MiB = 1_048_576;
 const MAX_RAW_BYTES = 64 * MiB;
 const MAX_ARCHIVE_BYTES = 96 * MiB;
-const FILE_LIMITS = new Map([
-  ['admission.json', MiB], ['request.json', 8 * MiB], ['gateway.json', 8192],
-  ['ledger.jsonl', 16 * MiB], ['result.json', MiB], ['provider-artifact.json', 16 * MiB],
-  ['stdout.log', MiB], ['stderr.log', MiB],
-]);
-const RECEIPT = /^receipts\/[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\.json$/;
-const limitFor = name => FILE_LIMITS.get(name) ?? (RECEIPT.test(name) ? MiB : 0);
 
 function secretValues({ environment = process.env, secrets = [] } = {}) {
-  return [...new Set([...secrets, ...Object.entries(environment)
-    .filter(([name]) => /(?:API_KEY|TOKEN|SECRET|PASSWORD|ACCESS_KEY_ID)$/.test(name))
-    .map(([, value]) => value)].filter(value => typeof value === 'string' && value.length > 0))];
+  const configured = [...secrets, ...Object.entries(environment)
+    .filter(([name]) => /(?:KEY|TOKEN|SECRET|PASSWORD|ACCESS_KEY_ID|CONFIG_CONTENT)$/i.test(name))
+    .map(([, value]) => value)].filter(value => typeof value === 'string' && value.length > 0);
+  for (const [name, value] of Object.entries(environment)) {
+    if (!/CONFIG_CONTENT$/i.test(name) || typeof value !== 'string') continue;
+    try {
+      const visit = item => {
+        if (!item || typeof item !== 'object') return;
+        for (const [key, entry] of Object.entries(item)) {
+          if (/(?:key|token|secret|password)$/i.test(key) && typeof entry === 'string' && entry) configured.push(entry);
+          else visit(entry);
+        }
+      };
+      visit(JSON.parse(value));
+    } catch {}
+  }
+  return [...new Set(configured.flatMap(value => [value, JSON.stringify(value).slice(1, -1)]))];
 }
 function excludeSecrets(bytes, secrets) {
   if (secrets.some(secret => bytes.includes(Buffer.from(secret)))) throw new Error('Controller evidence contains a configured credential');
 }
-async function readRegularFile(path, maximum) {
-  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
-  try {
-    const stat = await handle.stat();
-    if (!stat.isFile() || stat.size > maximum) throw new Error('Controller evidence is not a bounded regular file');
-    const bytes = Buffer.alloc(stat.size + 1);
-    let length = 0;
-    while (length < bytes.length) {
-      const { bytesRead } = await handle.read(bytes, length, bytes.length - length, null);
-      if (!bytesRead) break;
-      length += bytesRead;
-    }
-    if (length !== stat.size) throw new Error('Controller evidence changed while being read');
-    return bytes.subarray(0, length);
-  } finally { await handle.close(); }
-}
+
 async function realDirectory(path) {
   try {
     const stat = await lstat(path);

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict';
 import { test } from 'node:test';
-import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
+import { execFileSync } from 'node:child_process';
+import { mkdtemp, mkdir, readFile, readdir, rm, symlink, truncate, writeFile } from 'node:fs/promises';
 import { writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
@@ -15,7 +16,9 @@ import assert from 'node:assert/strict';
 import {createHash} from 'node:crypto';
 import {spawn} from 'node:child_process';
 import {once} from 'node:events';
-import {readFile,writeFile} from 'node:fs/promises';
+import {readFile,writeFile,rm,readdir,symlink,truncate} from 'node:fs/promises';
+import {dirname,join} from 'node:path';
+import {execFileSync} from 'node:child_process';
 const args=new Map();for(let i=2;i<process.argv.length;i+=2)args.set(process.argv[i],process.argv[i+1]);
 const request=JSON.parse(await readFile(args.get('--request'),'utf8'));
 if(${Boolean(options.resistantDescendant)}) {
@@ -37,6 +40,16 @@ if(!${Boolean(options.missingArtifact)})await writeFile(args.get('--output'),JSO
  requestRouting:${options.missingRoute ? 'null' : "{mode:'controller_gateway',provider:descriptor.provider,declaredUpstreamRoute:descriptor.upstreamRoute,campaignId:descriptor.campaignId,executionId:descriptor.executionId,attemptId:descriptor.attemptId,descriptorSha256:createHash('sha256').update(descriptorRaw).digest('hex')}"},
  snapshot:{status:'closed',process:{runnerExited:true,runnerPid:process.pid,providerPid:process.pid}},statuses,
  leaked:${options.leakToken ? 'process.env.PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN' : 'null'}}));
+if(${Boolean(options.invalidEvidence)}) {
+ const root=dirname(args.get('--output'));
+ const name=${JSON.stringify(options.invalidEvidence ?? '')};
+ const file=name==='receipt'?join(root,'receipts',(await readdir(join(root,'receipts')))[0]):join(root,name);
+ await rm(file);
+ const kind=${JSON.stringify(options.invalidFileType ?? '')};
+ if(kind==='fifo')execFileSync('mkfifo',[file]);
+ else if(kind==='symlink')await symlink('/dev/zero',file);
+ else {await writeFile(file,'');await truncate(file,16*1024*1024+1);}
+}
 if(${Boolean(options.orphanExit)})process.exit(0);
 `;
 
@@ -121,6 +134,7 @@ test('disjoint concurrent cells cannot borrow an allowance and unknown requests
 test('rejects oversized or unqualified allocations, changed requests and changed executables before dispatch', async t => {
   const f = await fixture(t, { count: 2 });
   for (const mutate of [
+    value => { value.attempts[0].attemptId = 'nested/attempt'; },
     value => { value.maxUsd = '0.01'; }, value => { value.attempts[1].executionId = value.attempts[0].executionId; },
     value => { value.attempts[0].provider = 'acpx'; value.attempts[0].acpxAgent = 'codex'; },
     value => { delete value.overhead.components[0].provisionalBound; }, value => { value.models[0].secret = 'unexpected'; },
@@ -378,6 +392,51 @@ test('configured credentials are excluded before encoding and after archive deco
   await assert.rejects(readControlledEvidence(file, { environment }), /configured credential/);
 });
 
+for (const name of ['provider-artifact.json', 'receipt', 'ledger.jsonl']) for (const kind of ['fifo', 'symlink', 'oversized']) {
+  test(`post-child ${name} rejects ${kind} without blocking or releasing its allocation`, { timeout: 20_000 }, async t => {
+    const f = await fixture(t, { requests: 1, invalidEvidence: name, invalidFileType: kind });
+    const result = await f.run();
+    assert.equal(result.report.status, 'incomplete');
+    assert.equal(result.report.accounting.knownUsd, '0.000068000');
+    const summary = await summarizeControlledCampaign(f.manifest);
+    assert.equal(summary.incompleteAttempts, 1);
+    assert.equal(summary.committedUsd, '0.014000000');
+  });
+}
+
+test('durable reports bound every evidence file and retain full holds for invalid proof', { timeout: 20_000 }, async t => {
+  const f = await fixture(t, { requests: 1 }); await f.run();
+  const root = attemptDirectory(f.manifest, 'attempt-0');
+  const receipt = `receipts/${(await readdir(join(root, 'receipts')))[0]}`;
+  for (const name of ['admission.json', 'request.json', 'result.json', 'provider-artifact.json', 'gateway.json', 'ledger.jsonl', receipt]) {
+    const path = join(root, name), original = await readFile(path);
+    for (const kind of ['fifo', 'symlink', 'oversized']) {
+      await rm(path);
+      if (kind === 'fifo') execFileSync('mkfifo', [path]);
+      else if (kind === 'symlink') await symlink('/dev/zero', path);
+      else { await writeFile(path, ''); await truncate(path, 16 * 1024 * 1024 + 1); }
+      const summary = await summarizeControlledCampaign(f.manifest);
+      assert.equal(summary.incompleteAttempts, 1, `${name}:${kind}`);
+      assert.equal(summary.committedUsd, '0.014000000', `${name}:${kind}`);
+    }
+    await rm(path); await writeFile(path, original);
+  }
+  assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0);
+});
+
+for (const name of ['PAPERCLIP_AGENT_PRIVATE_KEY', 'PAPERCLIP_AI_PROVIDER_KEY', 'AWS_SECRET_ACCESS_KEY', 'OPENCODE_CONFIG_CONTENT']) {
+  test(`archive excludes configured ${name} before and after encoding`, async t => {
+    const f = await fixture(t); await f.run();
+    const secret = name === 'PAPERCLIP_AGENT_PRIVATE_KEY' ? '-----BEGIN PRIVATE KEY-----\nsynthetic-private-material\n-----END PRIVATE KEY-----' : 'synthetic-extra-credential';
+    const environment = { [name]: name === 'OPENCODE_CONFIG_CONTENT' ? JSON.stringify({ provider: { apiKey: secret } }) : secret };
+    const root = attemptDirectory(f.manifest, 'attempt-0');
+    await writeFile(join(root, 'stderr.log'), JSON.stringify({ retained: secret }));
+    await assert.rejects(packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment }), /configured credential/);
+    const unchecked = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: {} });
+    assert.throws(() => decodeControlledEvidence(unchecked, { environment }), /configured credential/);
+  });
+}
+
 test('retention annotations preserve an already-failed controller result and its known costs', async t => {
   const f = await fixture(t, { requests: 1, missingRoute: true }); await f.run();
   const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });

packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs

@@ -0,0 +1,28 @@
+import { constants } from 'node:fs';
+import { open } from 'node:fs/promises';
+
+const MiB = 1_048_576;
+export const FILE_LIMITS = new Map([
+  ['admission.json', MiB], ['request.json', 8 * MiB], ['gateway.json', 8192],
+  ['ledger.jsonl', 16 * MiB], ['result.json', MiB], ['provider-artifact.json', 16 * MiB],
+  ['stdout.log', MiB], ['stderr.log', MiB],
+]);
+export const RECEIPT = /^receipts\/[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\.json$/;
+export const limitFor = name => FILE_LIMITS.get(name) ?? (RECEIPT.test(name) ? MiB : 0);
+
+export async function readRegularFile(path, maximum) {
+  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  try {
+    const stat = await handle.stat();
+    if (!stat.isFile() || stat.size > maximum) throw new Error('Controller evidence is not a bounded regular file');
+    const bytes = Buffer.alloc(stat.size + 1);
+    let length = 0;
+    while (length < bytes.length) {
+      const { bytesRead } = await handle.read(bytes, length, bytes.length - length, null);
+      if (!bytesRead) break;
+      length += bytesRead;
+    }
+    if (length !== stat.size) throw new Error('Controller evidence changed while being read');
+    return bytes.subarray(0, length);
+  } finally { await handle.close(); }
+}

packages/paperclip-runner/scripts/controlled-evals/manifest.mjs

@@ -72,7 +72,7 @@ export function validateManifest(value, now = Date.now(), { admission = true } =
     const provider = attempt.provider === 'codex' && attempt.acpxAgent === undefined ? 'openai'
       : attempt.provider === 'acpx' && attempt.acpxAgent === 'claude' ? 'anthropic' : null;
     const reservation = prices.get(`${provider}:${attempt.model}`);
-    if (!provider || !id(attempt.executionId) || !id(attempt.attemptId) || !hash(attempt.requestSha256)
+    if (!provider || !id(attempt.executionId) || !id(attempt.attemptId) || !/^[A-Za-z0-9_.-]+$/.test(attempt.attemptId) || !hash(attempt.requestSha256)
         || attempts.has(attempt.attemptId) || executions.has(attempt.executionId) || reservation === undefined
         || !Number.isSafeInteger(attempt.timeoutMs) || attempt.timeoutMs < 1 || attempt.timeoutMs > 4 * 3_600_000
         || !Number.isSafeInteger(attempt.maxRequests) || attempt.maxRequests < 1 || attempt.maxRequests > 1000

packages/paperclip-runner/scripts/controlled-evals/report.mjs

@@ -1,8 +1,8 @@
-import { readFile } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
 import { attemptDirectory } from './run.mjs';
+import { limitFor, readRegularFile } from './evidence-files.mjs';
 
 export async function summarizeControlledCampaign(value, { filesByAttempt } = {}) {
   const { manifest, manifestSha256, allocatedUsd, overheadUsd } = validateManifest(value, Date.now(), { admission: false });
@@ -11,9 +11,11 @@ export async function summarizeControlledCampaign(value, { filesByAttempt } = {}
   for (const allocation of manifest.attempts) {
     const root = attemptDirectory(manifest, allocation.attemptId);
     const readEvidence = async name => {
-      if (filesByAttempt === undefined) return readFile(join(root, name));
+      const limit = limitFor(name);
+      if (!limit) throw new Error('Unexpected controller evidence path');
+      if (filesByAttempt === undefined) return readRegularFile(join(root, name), limit);
       const bytes = filesByAttempt.get(allocation.attemptId)?.get(name);
-      if (!Buffer.isBuffer(bytes)) throw new Error('Missing archived controller evidence');
+      if (!Buffer.isBuffer(bytes) || bytes.length > limit) throw new Error('Missing archived controller evidence');
       return bytes;
     };
     const readDocument = async (name, maximum = 1_048_576) => {

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -4,6 +4,7 @@ import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'n
 import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseArgs } from 'node:util';
+import { limitFor, readRegularFile } from './evidence-files.mjs';
 import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs';
 
 async function atomicJson(file, value) {
@@ -137,8 +138,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
     child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath,
       cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => gateway.close() });
     for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 });
-    const raw = await readFile(artifactPath, 'utf8');
-    if (Buffer.byteLength(raw) > 16_777_216) throw new Error('Provider artifact exceeds its byte limit');
+    const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8');
     secretExcluded = secrets.every(secret => !raw.includes(secret));
     if (!secretExcluded) await writeFile(artifactPath, redact(raw), { mode: 0o600 });
     providerArtifactSha256 = sha256(redact(raw));
@@ -166,9 +166,11 @@ registerHooks({ resolve(specifier, context, nextResolve) {
   if (snapshot.overBudget || snapshot.conflictingRequests || snapshot.inflight || snapshot.unknownRequests !== snapshot.boundedUnresolvedRequests) failure ??= 'request_accounting_unbounded';
   const receiptHashes = {};
   try {
-    for (const request of snapshot.requests) receiptHashes[request.requestId] = sha256(await readFile(join(root, 'receipts', `${request.requestId}.json`)));
+    for (const request of snapshot.requests) receiptHashes[request.requestId] = sha256(await readRegularFile(join(root, 'receipts', `${request.requestId}.json`), limitFor(`receipts/${request.requestId}.json`)));
   } catch { failure ??= 'request_receipt_missing'; }
-  const ledgerSha256 = sha256(await readFile(ledgerPath));
+  let ledgerSha256;
+  try { ledgerSha256 = sha256(await readRegularFile(ledgerPath, limitFor('ledger.jsonl'))); }
+  catch { failure ??= 'ledger_evidence_invalid'; }
   const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256,
     executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256,
     allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure,

tests/runner-e2e/eval-gateway.test.ts

@@ -139,6 +139,9 @@ it.each([
   { type: "namespace", name: "local", tools: [{ type: "web_search" }] },
   { type: "namespace", name: "local", tools: [{ type: "namespace", name: "nested", tools: [{ type: "web_search" }] }] },
   { type: "namespace", name: "local" },
+  { type: "namespace", name: "local", tools: [{ type: "custom", name: "x", tools: [{ type: "web_search" }] }] },
+  { type: "namespace", name: "local", tools: [{ type: "function", name: "x", tools: [{ type: "web_search" }] }] },
+  { type: "custom", name: "x", tools: [{ type: "web_search" }] },
 ])("rejects unqualified tools inside namespaces before reservation", async tool => {
   const f = await fixture(), transport = vi.fn<typeof fetch>();
   await expect(forwardEvalRequest({ ...f, payload: { ...f.payload, tools: [tool] }, transport })).rejects.toThrow();

tests/runner-e2e/eval-gateway.ts

@@ -72,11 +72,11 @@ function boundedPayload(value: unknown, route: EvalGatewayRoute) {
   if (body.speed !== undefined && body.speed !== "standard" || body.inference_geo !== undefined) throw new EvalRequestError("unqualified_billing_mode");
   if (body.tools !== undefined && (!Array.isArray(body.tools) || body.tools.some(tool => {
     const definition = row(tool), type = definition.type;
-    if (route.provider !== "openai") return type !== undefined && type !== "custom";
+    if (route.provider !== "openai") return type !== undefined && type !== "custom" || Object.hasOwn(definition, "tools");
     if (type === "namespace") return typeof definition.name !== "string" || !definition.name
       || !Array.isArray(definition.tools) || !definition.tools.length
-      || definition.tools.some(nested => !["function", "custom"].includes(String(row(nested).type)));
-    return !["function", "custom"].includes(String(type));
+      || definition.tools.some(nested => !["function", "custom"].includes(String(row(nested).type)) || Object.hasOwn(row(nested), "tools"));
+    return !["function", "custom"].includes(String(type)) || Object.hasOwn(definition, "tools");
   }))) throw new EvalRequestError("unqualified_hosted_tool");
   const inspect = (item: unknown): void => {
     if (Array.isArray(item)) { item.forEach(inspect); return; }