fix(evals): bind archived results and complete catalog allocations

Compare immutable outcomes and accounting with the controller's reconstructed output. Validate provider, model, profile and driver for every selected allocation, including missing cells.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -209,7 +209,10 @@ requests keep their reservations, and failed or missing cell evidence keeps
 that cell's full allocation in the campaign report. An existing cell directory
 or output is never reset for a retry. Cancellation closes admission, drains
 upstream receipts within the proxy limit, and stops the owned CLI process
-group. Unverified provider cleanup remains explicit and retains the allocation.
+group. If the CLI exits while its owned group survives, the controller closes
+admission, stops that group, bounds inherited output pipes, and records an
+incomplete attempt. Unverified provider cleanup remains explicit and retains
+the allocation.
 
 Each result retains `controllerAccounting` alongside the original provider
 outcome and usage. Retain the durable controller directory as private evidence:
@@ -250,7 +253,10 @@ The reader accepts only bounded regular files and fixed evidence paths. A
 bundle is limited to 64 MiB of decoded content plus encoding overhead; this
 storage-format limit does not qualify a vendor charge or operating-cost bound.
 `report` accepts standalone bundle files; `report-attempts` additionally binds
-them to the surrounding immutable request and artifact. Both work after the
+them to the surrounding immutable request and reconstructed controller output,
+including the provider outcome, usage, and complete accounting result. The
+private program may add provenance or a stricter non-retryable retention
+failure. Missing source proof remains incomplete. Both work after the
 original worker directory is gone, using the same hash-chain ledger verifier.
 They group attempts by manifest, count operating holds once, deduplicate exact
 copies, reject conflicting copies, and keep full allocations for missing
@@ -259,7 +265,8 @@ selected attempts. They never consult old worker paths to fill archive gaps.
 The campaign collector preserves the bundle in private runs and returns its
 independently replayed totals as `controlledCosts`. For this collector, each
 manifest `executionId` must equal its catalog `cellId`, and the complete
-manifest selection must match the catalog. Extra attempts are rejected.
+manifest selection, model, provider, agent profile, and driver must match the
+catalog, including cells with no retained result. Extra attempts are rejected.
 Existing `costs` remain
 legacy provider-usage observations; do not add them to `controlledCosts`.
 The public attempt projection omits bundles and raw `controllerAccounting`.

packages/paperclip-runner/scripts/controlled-evals/archive.mjs

@@ -3,7 +3,7 @@ import { lstat, open, readdir, writeFile } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
-import { attemptDirectory } from './run.mjs';
+import { attemptDirectory, controlledResultArtifact } from './run.mjs';
 import { summarizeControlledCampaign } from './report.mjs';
 
 const MiB = 1_048_576;
@@ -127,6 +127,27 @@ export async function readAttemptControlledEvidence(directory, options) {
       || artifact.controllerAccounting && artifact.controllerAccounting.manifestSha256 !== bundle.manifestSha256) {
     throw new Error('Immutable attempt does not match its controller archive');
   }
+  const { files } = decodeControlledEvidence(bundle, options);
+  const reportBytes = files.get('result.json'), providerBytes = files.get('provider-artifact.json');
+  if (reportBytes) {
+    let report, provider;
+    try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; }
+    catch { return bundle; }
+    if (providerBytes || report?.providerArtifactSha256 === null) {
+      const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact };
+      delete expected.provenance; delete actual.provenance;
+      if (actual.controllerEvidenceError !== undefined) {
+        if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim()
+            || typeof actual.accountingError !== 'string' || !actual.accountingError.trim()
+            || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) {
+          throw new Error('Invalid immutable controller retention failure');
+        }
+        delete actual.controllerEvidenceError;
+        if (!expected.accountingFailure) { delete actual.accountingFailure; delete actual.accountingError; }
+      }
+      if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
+    }
+  }
   return bundle;
 }
 

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -379,7 +379,7 @@ test('configured credentials are excluded before encoding and after archive deco
 });
 
 test('campaign collection retains and audits private archives, while public projection omits them', async t => {
-  const f = await fixture(t, { requests: 1 }); await f.run();
+  const f = await fixture(t, { count: 2, requests: 1 }); await f.run();
   const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });
   const download = join(f.root, 'downloads', 'cell'), attempt = join(download, 'runs', 'attempt-0');
   await mkdir(attempt, { recursive: true });
@@ -391,9 +391,15 @@ test('campaign collection retains and audits private archives, while public proj
     'case.json': evalCase, 'config.json': config, 'controller-evidence.json': bundle,
   })) await writeFile(join(attempt, name), JSON.stringify(value));
   const cell = { cellId: 'execution-0', caseId: evalCase.id, rosterFile: 'roster.json', rosterId: 'roster-0', ...config };
+  const missingCell = { ...cell, cellId: 'execution-1', caseId: 'case-1' };
+  const program = join(f.root, 'evals/paperclip-runner');
+  for (const folder of ['rosters', 'cases', 'configs']) await mkdir(join(program, folder), { recursive: true });
+  await writeFile(join(program, 'rosters/roster.json'), JSON.stringify({ config: '../configs/config.json' }));
+  await writeFile(join(program, 'cases/case-1.json'), JSON.stringify({ ...evalCase, id: 'case-1' }));
+  await writeFile(join(program, 'configs/config.json'), JSON.stringify(config));
   const catalogPath = join(f.root, 'catalog.json');
   await writeFile(catalogPath, JSON.stringify({ schema: 'paperclip.runner-protocol-eval.catalog/v1', campaignId: f.manifest.campaignId,
-    selection: { kind: 'subset' }, cells: [cell], rosters: [{ ...cell, cases: [evalCase.id] }] }));
+    selection: { kind: 'subset' }, cells: [cell, missingCell], rosters: [{ ...cell, cases: [evalCase.id, missingCell.caseId] }] }));
   await writeFile(join(download, 'cell.json'), JSON.stringify({ ...cell, exitCode: 0 }));
   await rm(f.manifest.outputDirectory, { recursive: true });
   const input = { catalogPath, downloadsRoot: join(f.root, 'downloads'), evalsRoot: f.root,
@@ -401,7 +407,9 @@ test('campaign collection retains and audits private archives, while public proj
     source: { paperclip: { sha: f.manifest.sourceRevision }, evals: { sha: f.manifest.definitionsRevision } } };
   const result = await aggregateProtocolEvalCampaign(input);
   assert.equal(result.controlledCosts.knownUsd, '0.000068000');
-  assert.equal(result.controlledCosts.costStatus, 'provisional');
+  assert.equal(result.controlledCosts.costStatus, 'incomplete');
+  assert.equal(result.controlledCosts.committedUsd, '0.014068000');
+  assert.equal(result.controlledCosts.incompleteAttempts, 1);
   assert.deepEqual(JSON.parse(await readFile(join(input.runsOut, 'attempt-0', 'controller-evidence.json'), 'utf8')), bundle);
   const publicRoot = join(f.root, 'public');
   await sanitizeProtocolEvalRuns({ runsRoot: input.runsOut, publicRunsRoot: publicRoot });
@@ -411,13 +419,38 @@ test('campaign collection retains and audits private archives, while public proj
   await writeFile(join(attempt, 'request.json'), JSON.stringify(changedRequest));
   await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable attempt does not match/);
   await writeFile(join(attempt, 'request.json'), JSON.stringify(f.requests[0].request));
-  const drifted = structuredClone(bundle); drifted.manifest.attempts[0].executionId = 'not-the-selected-cell';
-  drifted.manifestSha256 = sha256(canonical(drifted.manifest));
-  await writeFile(join(attempt, 'controller-evidence.json'), JSON.stringify(drifted));
-  const bareArtifact = JSON.parse(await readFile(join(attempt, 'artifact.json'), 'utf8')); delete bareArtifact.controllerAccounting;
-  await writeFile(join(attempt, 'artifact.json'), JSON.stringify(bareArtifact));
-  await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/);
-  await writeFile(join(attempt, 'artifact.json'), await readFile(f.requests[0].outputPath));
+  const originalArtifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8'));
+  for (const mutate of [
+    value => { value.usage = { estimatedCostNanodollars: 0 }; },
+    value => { value.turn = { status: 'completed', assistantText: 'changed outcome' }; },
+    value => { value.controllerAccounting.cleanupVerified = false; },
+    value => { delete value.controllerAccounting; },
+  ]) {
+    const changed = structuredClone(originalArtifact); mutate(changed);
+    await writeFile(join(attempt, 'artifact.json'), JSON.stringify(changed));
+    await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable result disagrees/);
+  }
+  await writeFile(join(attempt, 'artifact.json'), JSON.stringify({ ...originalArtifact, provenance: { runnerPackage: 'private-program-provenance' } }));
+  await aggregateProtocolEvalCampaign(input);
+  const retainedFailure = { ...originalArtifact, controllerEvidenceError: 'Retained verification was interrupted', accountingError: 'Retained verification was interrupted',
+    accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } };
+  await writeFile(join(attempt, 'artifact.json'), JSON.stringify(retainedFailure));
+  await aggregateProtocolEvalCampaign(input);
+  await writeFile(join(attempt, 'artifact.json'), JSON.stringify(originalArtifact));
+  const catalog = JSON.parse(await readFile(catalogPath, 'utf8'));
+  for (const change of [{ model: 'wrong-model' }, { provider: 'acpx', acpxAgent: 'claude' }, { driver: 'other-driver' }]) {
+    const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[0], change);
+    await writeFile(catalogPath, JSON.stringify(driftedCatalog));
+    await writeFile(join(attempt, 'config.json'), JSON.stringify({ ...config, ...change }));
+    await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/);
+  }
+  await writeFile(join(attempt, 'config.json'), JSON.stringify(config));
+  for (const change of [{ model: 'wrong-unexecuted-model' }, { provider: 'acpx', acpxAgent: 'claude', driver: 'acpx_runtime' }]) {
+    const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[1], change);
+    await writeFile(catalogPath, JSON.stringify(driftedCatalog));
+    await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/);
+  }
+  await writeFile(catalogPath, JSON.stringify(catalog));
   await rm(join(attempt, 'controller-evidence.json'));
   await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/);
 });

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -52,6 +52,14 @@ async function runChild(input) {
   }
 }
 
+export function controlledResultArtifact(artifact, report, attempt) {
+  return { ...(artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId,
+    requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }),
+    ...(report.failure ? { accountingError: 'Controlled request accounting or cleanup is incomplete',
+      accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } } : {}),
+    controllerAccounting: report };
+}
+
 export async function runControlledEval({ manifestPath, requestPath, outputPath, environment = process.env, signal, transport }) {
   if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Controlled eval process ownership requires Linux or macOS');
   signal?.throwIfAborted();
@@ -167,11 +175,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
     child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null,
     cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, receiptHashes, accounting: snapshot };
   await atomicJson(join(root, 'result.json'), report);
-  const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId,
-    requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' };
-  if (failure) Object.assign(resultArtifact, { accountingError: 'Controlled request accounting or cleanup is incomplete',
-    accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } });
-  resultArtifact.controllerAccounting = report;
+  const resultArtifact = controlledResultArtifact(artifact, report, attempt);
   await mkdir(dirname(resolve(outputPath)), { recursive: true, mode: 0o700 });
   await writeFile(outputPath, JSON.stringify(resultArtifact, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
   return { code: failure ? 2 : child?.code ?? 2, report };

packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs

@@ -184,6 +184,7 @@ export async function buildProtocolEvalCatalog({
       configFile: relative(programRoot, configPath).split(sep).join("/"),
       model: String(roster.model ?? config.model ?? "unknown"),
       provider: String(config.provider ?? "codex"),
+      ...(config.provider === "acpx" ? { acpxAgent: config.acpxAgent } : {}),
       driver: String(config.driver ?? "codex_app_server"),
       credentialName,
       ...(config.provider === "acpx" && config.acpxAgent === "grok"
@@ -214,6 +215,7 @@ export async function buildProtocolEvalCatalog({
       caseId,
       model: roster.model,
       provider: roster.provider,
+      ...(roster.acpxAgent ? { acpxAgent: roster.acpxAgent } : {}),
       driver: roster.driver,
       credentialName: roster.credentialName,
       ...(roster.authenticationMode ? { authenticationMode: roster.authenticationMode } : {}),
@@ -465,13 +467,19 @@ export async function aggregateProtocolEvalCampaign({
           const bundle = await readAttemptControlledEvidence(join(attemptRoot, entry.name));
           const allocation = bundle.manifest.attempts.find(item => item.attemptId === attemptId);
           const selectedExecutions = new Set(bundle.manifest.attempts.map(item => item.executionId));
-          if (bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId ||
+          const allocationsMatchCatalog = bundle.manifest.attempts.every(item => {
+            const cell = expectedByCell.get(item.executionId);
+            return cell && item.model === cell.model && item.provider === cell.provider
+              && item.acpxAgent === cell.acpxAgent
+              && cell.driver === (item.provider === "codex" ? "codex_app_server" : "acpx_runtime");
+          });
+          if (!allocationsMatchCatalog || bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId ||
               selectedExecutions.size !== expectedByCell.size || [...expectedByCell.keys()].some(id => !selectedExecutions.has(id)) ||
               bundle.manifest.sourceRevision !== source.paperclip.sha || bundle.manifest.definitionsRevision !== source.evals.sha) {
             throw new Error(`Controller archive identity drifted for ${cellId}`);
           }
           controllerBundles.push(bundle);
-        } else if (artifact.controllerAccounting) {
+        } else if ("controllerAccounting" in artifact || "controllerEvidenceError" in artifact) {
           throw new Error(`Missing private controller evidence for ${attemptId}; cost metadata alone is insufficient`);
         }
         await copyAttempt(