fix(evals): bind incomplete outputs and replace redacted proof atomically

Reconstruct immutable results even when the provider artifact is absent and reject malformed proof. Replace redacted artifacts through an exclusive temporary file and rename so swapped symlinks cannot overwrite targets.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/scripts/controlled-evals/archive.mjs

@@ -124,23 +124,22 @@ export async function readAttemptControlledEvidence(directory, options) {
   if (reportBytes) {
     let report, provider;
     try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; }
-    catch { return bundle; }
-    if (providerBytes || report?.providerArtifactSha256 === null) {
-      const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact };
-      delete expected.provenance; delete actual.provenance;
-      if (actual.controllerEvidenceError !== undefined) {
-        if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim()
-            || typeof actual.accountingError !== 'string' || !actual.accountingError.trim()
-            || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) {
-          throw new Error('Invalid immutable controller retention failure');
-        }
-        delete actual.controllerEvidenceError;
-        if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; }
-        else { delete actual.accountingFailure; delete actual.accountingError; }
+    catch { throw new Error('Malformed immutable controller result proof'); }
+    const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact };
+    delete expected.provenance; delete actual.provenance;
+    if (actual.controllerEvidenceError !== undefined) {
+      if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim()
+          || typeof actual.accountingError !== 'string' || !actual.accountingError.trim()
+          || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) {
+        throw new Error('Invalid immutable controller retention failure');
       }
-      if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
+      delete actual.controllerEvidenceError;
+      if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; }
+      else { delete actual.accountingFailure; delete actual.accountingError; }
     }
+    if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
   }
+
   return bundle;
 }
 

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -8,6 +8,7 @@ import { join } from 'node:path';
 import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs';
 import { attemptDirectory, runControlledEval } from './run.mjs';
 import { summarizeControlledCampaign } from './report.mjs';
+import { readRegularFile, replaceEvidenceFile } from './evidence-files.mjs';
 import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs';
 import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs';
 
@@ -532,3 +533,44 @@ test('campaign collection retains and audits private archives, while public proj
   await rm(join(attempt, 'controller-evidence.json'));
   await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/);
 });
+
+
+test('credential redaction replaces a swapped symlink without changing its target', async t => {
+  const f = await fixture(t, { leakToken: true, requests: 0 });
+  const result = await f.run();
+  assert.equal(result.report.secretExcluded, false);
+  const path = join(attemptDirectory(f.manifest, 'attempt-0'), 'provider-artifact.json');
+  const bytes = await readRegularFile(path, 16_777_216);
+  const target = join(f.root, 'untouched-target'); await writeFile(target, 'target must remain unchanged');
+  await rm(path); await symlink(target, path);
+  await replaceEvidenceFile(path, bytes);
+  assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged');
+  assert.deepEqual(await readRegularFile(path, 16_777_216), bytes);
+  await symlink(target, `${path}.tmp`);
+  await assert.rejects(replaceEvidenceFile(path, bytes), { code: 'EEXIST' });
+  assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged');
+});
+
+test('missing provider artifacts cannot bypass immutable result binding', async t => {
+  const f = await fixture(t, { missingArtifact: true, requests: 0 }); await f.run();
+  const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });
+  assert.equal(bundle.files['provider-artifact.json'], undefined);
+  assert.equal(JSON.parse(Buffer.from(bundle.files['result.json'].base64, 'base64')).providerArtifactSha256, undefined);
+  const directory = join(f.root, 'runs', 'attempt-0'); await mkdir(directory, { recursive: true });
+  const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8'));
+  for (const [name, value] of Object.entries({ 'request.json': f.requests[0].request, 'artifact.json': artifact, 'controller-evidence.json': bundle })) await writeFile(join(directory, name), JSON.stringify(value));
+  await readAttemptControlledEvidence(directory);
+  for (const mutate of [
+    value => { value.usage = { estimatedCostNanodollars: 0 }; },
+    value => { value.turn = { status: 'completed', assistantText: 'forged success' }; },
+    value => { value.controllerAccounting.cleanupVerified = true; },
+  ]) {
+    const changed = structuredClone(artifact); mutate(changed);
+    await writeFile(join(directory, 'artifact.json'), JSON.stringify(changed));
+    await assert.rejects(readAttemptControlledEvidence(directory), /Immutable result disagrees/);
+  }
+  const malformed = structuredClone(bundle), bytes = Buffer.from('{');
+  malformed.files['result.json'] = { sizeBytes: bytes.length, sha256: sha256(bytes), base64: bytes.toString('base64') };
+  await writeFile(join(directory, 'controller-evidence.json'), JSON.stringify(malformed));
+  await assert.rejects(readAttemptControlledEvidence(directory), /Malformed immutable controller result proof/);
+});

packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs

@@ -1,5 +1,5 @@
 import { constants } from 'node:fs';
-import { open } from 'node:fs/promises';
+import { open, rename } from 'node:fs/promises';
 
 const MiB = 1_048_576;
 export const FILE_LIMITS = new Map([
@@ -26,3 +26,11 @@ export async function readRegularFile(path, maximum) {
     return bytes.subarray(0, length);
   } finally { await handle.close(); }
 }
+
+export async function replaceEvidenceFile(path, bytes) {
+  const temporary = `${path}.tmp`;
+  const handle = await open(temporary, 'wx', 0o600);
+  try { await handle.writeFile(bytes); await handle.sync(); }
+  finally { await handle.close(); }
+  await rename(temporary, path);
+}

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -4,7 +4,7 @@ import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'n
 import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseArgs } from 'node:util';
-import { limitFor, readRegularFile } from './evidence-files.mjs';
+import { limitFor, readRegularFile, replaceEvidenceFile } from './evidence-files.mjs';
 import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs';
 
 async function atomicJson(file, value) {
@@ -140,7 +140,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
     for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 });
     const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8');
     secretExcluded = secrets.every(secret => !raw.includes(secret));
-    if (!secretExcluded) await writeFile(artifactPath, redact(raw), { mode: 0o600 });
+    if (!secretExcluded) await replaceEvidenceFile(artifactPath, redact(raw));
     providerArtifactSha256 = sha256(redact(raw));
     artifact = JSON.parse(redact(raw));
     const expected = { mode: 'controller_gateway', provider, declaredUpstreamRoute: manifest.route, campaignId: manifest.campaignId,