From 4d6673d8cedabdccbab168f33a2352f513844444 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 12:51:01 -0700 Subject: [PATCH] fix(evals): bind archived results and complete catalog allocations Compare immutable outcomes and accounting with the controller's reconstructed output. Validate provider, model, profile and driver for every selected allocation, including missing cells. Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 13 +++-- .../scripts/controlled-evals/archive.mjs | 23 +++++++- .../controlled-evals/controller.test.mjs | 53 +++++++++++++++---- .../scripts/controlled-evals/run.mjs | 14 +++-- .../scripts/runner-protocol-eval-campaign.mjs | 12 ++++- 5 files changed, 94 insertions(+), 21 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index d5fbb0e65f6..63dccf4a82c 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -209,7 +209,10 @@ requests keep their reservations, and failed or missing cell evidence keeps that cell's full allocation in the campaign report. An existing cell directory or output is never reset for a retry. Cancellation closes admission, drains upstream receipts within the proxy limit, and stops the owned CLI process -group. Unverified provider cleanup remains explicit and retains the allocation. +group. If the CLI exits while its owned group survives, the controller closes +admission, stops that group, bounds inherited output pipes, and records an +incomplete attempt. Unverified provider cleanup remains explicit and retains +the allocation. Each result retains `controllerAccounting` alongside the original provider outcome and usage. Retain the durable controller directory as private evidence: @@ -250,7 +253,10 @@ The reader accepts only bounded regular files and fixed evidence paths. A bundle is limited to 64 MiB of decoded content plus encoding overhead; this storage-format limit does not qualify a vendor charge or operating-cost bound. `report` accepts standalone bundle files; `report-attempts` additionally binds -them to the surrounding immutable request and artifact. Both work after the +them to the surrounding immutable request and reconstructed controller output, +including the provider outcome, usage, and complete accounting result. The +private program may add provenance or a stricter non-retryable retention +failure. Missing source proof remains incomplete. Both work after the original worker directory is gone, using the same hash-chain ledger verifier. They group attempts by manifest, count operating holds once, deduplicate exact copies, reject conflicting copies, and keep full allocations for missing @@ -259,7 +265,8 @@ selected attempts. They never consult old worker paths to fill archive gaps. The campaign collector preserves the bundle in private runs and returns its independently replayed totals as `controlledCosts`. For this collector, each manifest `executionId` must equal its catalog `cellId`, and the complete -manifest selection must match the catalog. Extra attempts are rejected. +manifest selection, model, provider, agent profile, and driver must match the +catalog, including cells with no retained result. Extra attempts are rejected. Existing `costs` remain legacy provider-usage observations; do not add them to `controlledCosts`. The public attempt projection omits bundles and raw `controllerAccounting`. diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index ffea93bee58..84335be989d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -3,7 +3,7 @@ import { lstat, open, readdir, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; -import { attemptDirectory } from './run.mjs'; +import { attemptDirectory, controlledResultArtifact } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; const MiB = 1_048_576; @@ -127,6 +127,27 @@ export async function readAttemptControlledEvidence(directory, options) { || artifact.controllerAccounting && artifact.controllerAccounting.manifestSha256 !== bundle.manifestSha256) { throw new Error('Immutable attempt does not match its controller archive'); } + const { files } = decodeControlledEvidence(bundle, options); + const reportBytes = files.get('result.json'), providerBytes = files.get('provider-artifact.json'); + if (reportBytes) { + let report, provider; + try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; } + catch { return bundle; } + if (providerBytes || report?.providerArtifactSha256 === null) { + const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact }; + delete expected.provenance; delete actual.provenance; + if (actual.controllerEvidenceError !== undefined) { + if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim() + || typeof actual.accountingError !== 'string' || !actual.accountingError.trim() + || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) { + throw new Error('Invalid immutable controller retention failure'); + } + delete actual.controllerEvidenceError; + if (!expected.accountingFailure) { delete actual.accountingFailure; delete actual.accountingError; } + } + if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); + } + } return bundle; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 87759f68c81..65398c8415a 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -379,7 +379,7 @@ test('configured credentials are excluded before encoding and after archive deco }); test('campaign collection retains and audits private archives, while public projection omits them', async t => { - const f = await fixture(t, { requests: 1 }); await f.run(); + const f = await fixture(t, { count: 2, requests: 1 }); await f.run(); const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); const download = join(f.root, 'downloads', 'cell'), attempt = join(download, 'runs', 'attempt-0'); await mkdir(attempt, { recursive: true }); @@ -391,9 +391,15 @@ test('campaign collection retains and audits private archives, while public proj 'case.json': evalCase, 'config.json': config, 'controller-evidence.json': bundle, })) await writeFile(join(attempt, name), JSON.stringify(value)); const cell = { cellId: 'execution-0', caseId: evalCase.id, rosterFile: 'roster.json', rosterId: 'roster-0', ...config }; + const missingCell = { ...cell, cellId: 'execution-1', caseId: 'case-1' }; + const program = join(f.root, 'evals/paperclip-runner'); + for (const folder of ['rosters', 'cases', 'configs']) await mkdir(join(program, folder), { recursive: true }); + await writeFile(join(program, 'rosters/roster.json'), JSON.stringify({ config: '../configs/config.json' })); + await writeFile(join(program, 'cases/case-1.json'), JSON.stringify({ ...evalCase, id: 'case-1' })); + await writeFile(join(program, 'configs/config.json'), JSON.stringify(config)); const catalogPath = join(f.root, 'catalog.json'); await writeFile(catalogPath, JSON.stringify({ schema: 'paperclip.runner-protocol-eval.catalog/v1', campaignId: f.manifest.campaignId, - selection: { kind: 'subset' }, cells: [cell], rosters: [{ ...cell, cases: [evalCase.id] }] })); + selection: { kind: 'subset' }, cells: [cell, missingCell], rosters: [{ ...cell, cases: [evalCase.id, missingCell.caseId] }] })); await writeFile(join(download, 'cell.json'), JSON.stringify({ ...cell, exitCode: 0 })); await rm(f.manifest.outputDirectory, { recursive: true }); const input = { catalogPath, downloadsRoot: join(f.root, 'downloads'), evalsRoot: f.root, @@ -401,7 +407,9 @@ test('campaign collection retains and audits private archives, while public proj source: { paperclip: { sha: f.manifest.sourceRevision }, evals: { sha: f.manifest.definitionsRevision } } }; const result = await aggregateProtocolEvalCampaign(input); assert.equal(result.controlledCosts.knownUsd, '0.000068000'); - assert.equal(result.controlledCosts.costStatus, 'provisional'); + assert.equal(result.controlledCosts.costStatus, 'incomplete'); + assert.equal(result.controlledCosts.committedUsd, '0.014068000'); + assert.equal(result.controlledCosts.incompleteAttempts, 1); assert.deepEqual(JSON.parse(await readFile(join(input.runsOut, 'attempt-0', 'controller-evidence.json'), 'utf8')), bundle); const publicRoot = join(f.root, 'public'); await sanitizeProtocolEvalRuns({ runsRoot: input.runsOut, publicRunsRoot: publicRoot }); @@ -411,13 +419,38 @@ test('campaign collection retains and audits private archives, while public proj await writeFile(join(attempt, 'request.json'), JSON.stringify(changedRequest)); await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable attempt does not match/); await writeFile(join(attempt, 'request.json'), JSON.stringify(f.requests[0].request)); - const drifted = structuredClone(bundle); drifted.manifest.attempts[0].executionId = 'not-the-selected-cell'; - drifted.manifestSha256 = sha256(canonical(drifted.manifest)); - await writeFile(join(attempt, 'controller-evidence.json'), JSON.stringify(drifted)); - const bareArtifact = JSON.parse(await readFile(join(attempt, 'artifact.json'), 'utf8')); delete bareArtifact.controllerAccounting; - await writeFile(join(attempt, 'artifact.json'), JSON.stringify(bareArtifact)); - await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); - await writeFile(join(attempt, 'artifact.json'), await readFile(f.requests[0].outputPath)); + const originalArtifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + for (const mutate of [ + value => { value.usage = { estimatedCostNanodollars: 0 }; }, + value => { value.turn = { status: 'completed', assistantText: 'changed outcome' }; }, + value => { value.controllerAccounting.cleanupVerified = false; }, + value => { delete value.controllerAccounting; }, + ]) { + const changed = structuredClone(originalArtifact); mutate(changed); + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(changed)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable result disagrees/); + } + await writeFile(join(attempt, 'artifact.json'), JSON.stringify({ ...originalArtifact, provenance: { runnerPackage: 'private-program-provenance' } })); + await aggregateProtocolEvalCampaign(input); + const retainedFailure = { ...originalArtifact, controllerEvidenceError: 'Retained verification was interrupted', accountingError: 'Retained verification was interrupted', + accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } }; + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(retainedFailure)); + await aggregateProtocolEvalCampaign(input); + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(originalArtifact)); + const catalog = JSON.parse(await readFile(catalogPath, 'utf8')); + for (const change of [{ model: 'wrong-model' }, { provider: 'acpx', acpxAgent: 'claude' }, { driver: 'other-driver' }]) { + const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[0], change); + await writeFile(catalogPath, JSON.stringify(driftedCatalog)); + await writeFile(join(attempt, 'config.json'), JSON.stringify({ ...config, ...change })); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + } + await writeFile(join(attempt, 'config.json'), JSON.stringify(config)); + for (const change of [{ model: 'wrong-unexecuted-model' }, { provider: 'acpx', acpxAgent: 'claude', driver: 'acpx_runtime' }]) { + const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[1], change); + await writeFile(catalogPath, JSON.stringify(driftedCatalog)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + } + await writeFile(catalogPath, JSON.stringify(catalog)); await rm(join(attempt, 'controller-evidence.json')); await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/); }); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 7945910e17b..cef1f272b0d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -52,6 +52,14 @@ async function runChild(input) { } } +export function controlledResultArtifact(artifact, report, attempt) { + return { ...(artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, + requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }), + ...(report.failure ? { accountingError: 'Controlled request accounting or cleanup is incomplete', + accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } } : {}), + controllerAccounting: report }; +} + export async function runControlledEval({ manifestPath, requestPath, outputPath, environment = process.env, signal, transport }) { if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Controlled eval process ownership requires Linux or macOS'); signal?.throwIfAborted(); @@ -167,11 +175,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null, cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, receiptHashes, accounting: snapshot }; await atomicJson(join(root, 'result.json'), report); - const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, - requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }; - if (failure) Object.assign(resultArtifact, { accountingError: 'Controlled request accounting or cleanup is incomplete', - accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } }); - resultArtifact.controllerAccounting = report; + const resultArtifact = controlledResultArtifact(artifact, report, attempt); await mkdir(dirname(resolve(outputPath)), { recursive: true, mode: 0o700 }); await writeFile(outputPath, JSON.stringify(resultArtifact, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); return { code: failure ? 2 : child?.code ?? 2, report }; diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index e13a218700f..f410814d878 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -184,6 +184,7 @@ export async function buildProtocolEvalCatalog({ configFile: relative(programRoot, configPath).split(sep).join("/"), model: String(roster.model ?? config.model ?? "unknown"), provider: String(config.provider ?? "codex"), + ...(config.provider === "acpx" ? { acpxAgent: config.acpxAgent } : {}), driver: String(config.driver ?? "codex_app_server"), credentialName, ...(config.provider === "acpx" && config.acpxAgent === "grok" @@ -214,6 +215,7 @@ export async function buildProtocolEvalCatalog({ caseId, model: roster.model, provider: roster.provider, + ...(roster.acpxAgent ? { acpxAgent: roster.acpxAgent } : {}), driver: roster.driver, credentialName: roster.credentialName, ...(roster.authenticationMode ? { authenticationMode: roster.authenticationMode } : {}), @@ -465,13 +467,19 @@ export async function aggregateProtocolEvalCampaign({ const bundle = await readAttemptControlledEvidence(join(attemptRoot, entry.name)); const allocation = bundle.manifest.attempts.find(item => item.attemptId === attemptId); const selectedExecutions = new Set(bundle.manifest.attempts.map(item => item.executionId)); - if (bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || + const allocationsMatchCatalog = bundle.manifest.attempts.every(item => { + const cell = expectedByCell.get(item.executionId); + return cell && item.model === cell.model && item.provider === cell.provider + && item.acpxAgent === cell.acpxAgent + && cell.driver === (item.provider === "codex" ? "codex_app_server" : "acpx_runtime"); + }); + if (!allocationsMatchCatalog || bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || selectedExecutions.size !== expectedByCell.size || [...expectedByCell.keys()].some(id => !selectedExecutions.has(id)) || bundle.manifest.sourceRevision !== source.paperclip.sha || bundle.manifest.definitionsRevision !== source.evals.sha) { throw new Error(`Controller archive identity drifted for ${cellId}`); } controllerBundles.push(bundle); - } else if (artifact.controllerAccounting) { + } else if ("controllerAccounting" in artifact || "controllerEvidenceError" in artifact) { throw new Error(`Missing private controller evidence for ${attemptId}; cost metadata alone is insufficient`); } await copyAttempt(