From 7d4eeb572826276dfa7d091117150edc2fdd8bf0 Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Sat, 24 Oct 2026 13:04:11 -0700
Subject: [PATCH] fix(evals): retain annotated failed controller attempts

---
 .../scripts/controlled-evals/archive.mjs      |  3 ++-
 .../controlled-evals/controller.test.mjs      | 21 ++++++++++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs
index 84335be989d..fc328ee72a5 100644
--- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs
+++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs
@@ -143,7 +143,8 @@ export async function readAttemptControlledEvidence(directory, options) {
           throw new Error('Invalid immutable controller retention failure');
         }
         delete actual.controllerEvidenceError;
-        if (!expected.accountingFailure) { delete actual.accountingFailure; delete actual.accountingError; }
+        if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; }
+        else { delete actual.accountingFailure; delete actual.accountingError; }
       }
       if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
     }
diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs
index 65398c8415a..820b24280d0 100644
--- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs
+++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs
@@ -7,7 +7,7 @@ import { join } from 'node:path';
 import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs';
 import { attemptDirectory, runControlledEval } from './run.mjs';
 import { summarizeControlledCampaign } from './report.mjs';
-import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, summarizeControlledEvidence } from './archive.mjs';
+import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs';
 import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs';
 
 const childScript = options => `
@@ -378,6 +378,25 @@ test('configured credentials are excluded before encoding and after archive deco
   await assert.rejects(readControlledEvidence(file, { environment }), /configured credential/);
 });
 
+test('retention annotations preserve an already-failed controller result and its known costs', async t => {
+  const f = await fixture(t, { requests: 1, missingRoute: true }); await f.run();
+  const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });
+  const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8'));
+  assert.equal(artifact.controllerAccounting.status, 'incomplete');
+  artifact.controllerEvidenceError = 'A later retention check failed';
+  artifact.accountingError = artifact.controllerEvidenceError;
+  const directory = join(f.root, 'runs', 'attempt-0'); await mkdir(directory, { recursive: true });
+  for (const [name, value] of Object.entries({ 'request.json': f.requests[0].request, 'artifact.json': artifact, 'controller-evidence.json': bundle })) {
+    await writeFile(join(directory, name), JSON.stringify(value));
+  }
+  const retained = await readAttemptControlledEvidence(directory);
+  const report = await summarizeControlledEvidence([retained]);
+  assert.equal(report.costStatus, 'incomplete');
+  assert.equal(report.knownUsd, '0.000068000');
+  assert.equal(report.committedUsd, '0.014000000');
+  assert.equal(JSON.parse(await readFile(join(directory, 'artifact.json'), 'utf8')).accountingError, artifact.controllerEvidenceError);
+});
+
 test('campaign collection retains and audits private archives, while public projection omits them', async t => {
   const f = await fixture(t, { count: 2, requests: 1 }); await f.run();
   const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });