diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index d5fbb0e65f6..63dccf4a82c 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -209,7 +209,10 @@ requests keep their reservations, and failed or missing cell evidence keeps that cell's full allocation in the campaign report. An existing cell directory or output is never reset for a retry. Cancellation closes admission, drains upstream receipts within the proxy limit, and stops the owned CLI process -group. Unverified provider cleanup remains explicit and retains the allocation. +group. If the CLI exits while its owned group survives, the controller closes +admission, stops that group, bounds inherited output pipes, and records an +incomplete attempt. Unverified provider cleanup remains explicit and retains +the allocation. Each result retains `controllerAccounting` alongside the original provider outcome and usage. Retain the durable controller directory as private evidence: @@ -250,7 +253,10 @@ The reader accepts only bounded regular files and fixed evidence paths. A bundle is limited to 64 MiB of decoded content plus encoding overhead; this storage-format limit does not qualify a vendor charge or operating-cost bound. `report` accepts standalone bundle files; `report-attempts` additionally binds -them to the surrounding immutable request and artifact. Both work after the +them to the surrounding immutable request and reconstructed controller output, +including the provider outcome, usage, and complete accounting result. The +private program may add provenance or a stricter non-retryable retention +failure. Missing source proof remains incomplete. Both work after the original worker directory is gone, using the same hash-chain ledger verifier. They group attempts by manifest, count operating holds once, deduplicate exact copies, reject conflicting copies, and keep full allocations for missing @@ -259,7 +265,8 @@ selected attempts. They never consult old worker paths to fill archive gaps. The campaign collector preserves the bundle in private runs and returns its independently replayed totals as `controlledCosts`. For this collector, each manifest `executionId` must equal its catalog `cellId`, and the complete -manifest selection must match the catalog. Extra attempts are rejected. +manifest selection, model, provider, agent profile, and driver must match the +catalog, including cells with no retained result. Extra attempts are rejected. Existing `costs` remain legacy provider-usage observations; do not add them to `controlledCosts`. The public attempt projection omits bundles and raw `controllerAccounting`. diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index ffea93bee58..84335be989d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -3,7 +3,7 @@ import { lstat, open, readdir, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; -import { attemptDirectory } from './run.mjs'; +import { attemptDirectory, controlledResultArtifact } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; const MiB = 1_048_576; @@ -127,6 +127,27 @@ export async function readAttemptControlledEvidence(directory, options) { || artifact.controllerAccounting && artifact.controllerAccounting.manifestSha256 !== bundle.manifestSha256) { throw new Error('Immutable attempt does not match its controller archive'); } + const { files } = decodeControlledEvidence(bundle, options); + const reportBytes = files.get('result.json'), providerBytes = files.get('provider-artifact.json'); + if (reportBytes) { + let report, provider; + try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; } + catch { return bundle; } + if (providerBytes || report?.providerArtifactSha256 === null) { + const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact }; + delete expected.provenance; delete actual.provenance; + if (actual.controllerEvidenceError !== undefined) { + if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim() + || typeof actual.accountingError !== 'string' || !actual.accountingError.trim() + || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) { + throw new Error('Invalid immutable controller retention failure'); + } + delete actual.controllerEvidenceError; + if (!expected.accountingFailure) { delete actual.accountingFailure; delete actual.accountingError; } + } + if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); + } + } return bundle; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 87759f68c81..65398c8415a 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -379,7 +379,7 @@ test('configured credentials are excluded before encoding and after archive deco }); test('campaign collection retains and audits private archives, while public projection omits them', async t => { - const f = await fixture(t, { requests: 1 }); await f.run(); + const f = await fixture(t, { count: 2, requests: 1 }); await f.run(); const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); const download = join(f.root, 'downloads', 'cell'), attempt = join(download, 'runs', 'attempt-0'); await mkdir(attempt, { recursive: true }); @@ -391,9 +391,15 @@ test('campaign collection retains and audits private archives, while public proj 'case.json': evalCase, 'config.json': config, 'controller-evidence.json': bundle, })) await writeFile(join(attempt, name), JSON.stringify(value)); const cell = { cellId: 'execution-0', caseId: evalCase.id, rosterFile: 'roster.json', rosterId: 'roster-0', ...config }; + const missingCell = { ...cell, cellId: 'execution-1', caseId: 'case-1' }; + const program = join(f.root, 'evals/paperclip-runner'); + for (const folder of ['rosters', 'cases', 'configs']) await mkdir(join(program, folder), { recursive: true }); + await writeFile(join(program, 'rosters/roster.json'), JSON.stringify({ config: '../configs/config.json' })); + await writeFile(join(program, 'cases/case-1.json'), JSON.stringify({ ...evalCase, id: 'case-1' })); + await writeFile(join(program, 'configs/config.json'), JSON.stringify(config)); const catalogPath = join(f.root, 'catalog.json'); await writeFile(catalogPath, JSON.stringify({ schema: 'paperclip.runner-protocol-eval.catalog/v1', campaignId: f.manifest.campaignId, - selection: { kind: 'subset' }, cells: [cell], rosters: [{ ...cell, cases: [evalCase.id] }] })); + selection: { kind: 'subset' }, cells: [cell, missingCell], rosters: [{ ...cell, cases: [evalCase.id, missingCell.caseId] }] })); await writeFile(join(download, 'cell.json'), JSON.stringify({ ...cell, exitCode: 0 })); await rm(f.manifest.outputDirectory, { recursive: true }); const input = { catalogPath, downloadsRoot: join(f.root, 'downloads'), evalsRoot: f.root, @@ -401,7 +407,9 @@ test('campaign collection retains and audits private archives, while public proj source: { paperclip: { sha: f.manifest.sourceRevision }, evals: { sha: f.manifest.definitionsRevision } } }; const result = await aggregateProtocolEvalCampaign(input); assert.equal(result.controlledCosts.knownUsd, '0.000068000'); - assert.equal(result.controlledCosts.costStatus, 'provisional'); + assert.equal(result.controlledCosts.costStatus, 'incomplete'); + assert.equal(result.controlledCosts.committedUsd, '0.014068000'); + assert.equal(result.controlledCosts.incompleteAttempts, 1); assert.deepEqual(JSON.parse(await readFile(join(input.runsOut, 'attempt-0', 'controller-evidence.json'), 'utf8')), bundle); const publicRoot = join(f.root, 'public'); await sanitizeProtocolEvalRuns({ runsRoot: input.runsOut, publicRunsRoot: publicRoot }); @@ -411,13 +419,38 @@ test('campaign collection retains and audits private archives, while public proj await writeFile(join(attempt, 'request.json'), JSON.stringify(changedRequest)); await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable attempt does not match/); await writeFile(join(attempt, 'request.json'), JSON.stringify(f.requests[0].request)); - const drifted = structuredClone(bundle); drifted.manifest.attempts[0].executionId = 'not-the-selected-cell'; - drifted.manifestSha256 = sha256(canonical(drifted.manifest)); - await writeFile(join(attempt, 'controller-evidence.json'), JSON.stringify(drifted)); - const bareArtifact = JSON.parse(await readFile(join(attempt, 'artifact.json'), 'utf8')); delete bareArtifact.controllerAccounting; - await writeFile(join(attempt, 'artifact.json'), JSON.stringify(bareArtifact)); - await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); - await writeFile(join(attempt, 'artifact.json'), await readFile(f.requests[0].outputPath)); + const originalArtifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + for (const mutate of [ + value => { value.usage = { estimatedCostNanodollars: 0 }; }, + value => { value.turn = { status: 'completed', assistantText: 'changed outcome' }; }, + value => { value.controllerAccounting.cleanupVerified = false; }, + value => { delete value.controllerAccounting; }, + ]) { + const changed = structuredClone(originalArtifact); mutate(changed); + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(changed)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Immutable result disagrees/); + } + await writeFile(join(attempt, 'artifact.json'), JSON.stringify({ ...originalArtifact, provenance: { runnerPackage: 'private-program-provenance' } })); + await aggregateProtocolEvalCampaign(input); + const retainedFailure = { ...originalArtifact, controllerEvidenceError: 'Retained verification was interrupted', accountingError: 'Retained verification was interrupted', + accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } }; + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(retainedFailure)); + await aggregateProtocolEvalCampaign(input); + await writeFile(join(attempt, 'artifact.json'), JSON.stringify(originalArtifact)); + const catalog = JSON.parse(await readFile(catalogPath, 'utf8')); + for (const change of [{ model: 'wrong-model' }, { provider: 'acpx', acpxAgent: 'claude' }, { driver: 'other-driver' }]) { + const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[0], change); + await writeFile(catalogPath, JSON.stringify(driftedCatalog)); + await writeFile(join(attempt, 'config.json'), JSON.stringify({ ...config, ...change })); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + } + await writeFile(join(attempt, 'config.json'), JSON.stringify(config)); + for (const change of [{ model: 'wrong-unexecuted-model' }, { provider: 'acpx', acpxAgent: 'claude', driver: 'acpx_runtime' }]) { + const driftedCatalog = structuredClone(catalog); Object.assign(driftedCatalog.cells[1], change); + await writeFile(catalogPath, JSON.stringify(driftedCatalog)); + await assert.rejects(aggregateProtocolEvalCampaign(input), /Controller archive identity drifted/); + } + await writeFile(catalogPath, JSON.stringify(catalog)); await rm(join(attempt, 'controller-evidence.json')); await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/); }); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 7945910e17b..cef1f272b0d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -52,6 +52,14 @@ async function runChild(input) { } } +export function controlledResultArtifact(artifact, report, attempt) { + return { ...(artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, + requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }), + ...(report.failure ? { accountingError: 'Controlled request accounting or cleanup is incomplete', + accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } } : {}), + controllerAccounting: report }; +} + export async function runControlledEval({ manifestPath, requestPath, outputPath, environment = process.env, signal, transport }) { if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Controlled eval process ownership requires Linux or macOS'); signal?.throwIfAborted(); @@ -167,11 +175,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { child: child ? { code: child.code, signal: child.signal, stopped: child.stopped } : null, cleanupVerified, secretExcluded, providerArtifactSha256, ledgerSha256, receiptHashes, accounting: snapshot }; await atomicJson(join(root, 'result.json'), report); - const resultArtifact = artifact ?? { schema: 'paperclip-runner/eval-session-artifact/v1', attemptId: attempt.attemptId, - requestedModel: attempt.model, infrastructureError: 'Controlled provider did not retain an artifact' }; - if (failure) Object.assign(resultArtifact, { accountingError: 'Controlled request accounting or cleanup is incomplete', - accountingFailure: { class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false } }); - resultArtifact.controllerAccounting = report; + const resultArtifact = controlledResultArtifact(artifact, report, attempt); await mkdir(dirname(resolve(outputPath)), { recursive: true, mode: 0o700 }); await writeFile(outputPath, JSON.stringify(resultArtifact, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); return { code: failure ? 2 : child?.code ?? 2, report }; diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index e13a218700f..f410814d878 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -184,6 +184,7 @@ export async function buildProtocolEvalCatalog({ configFile: relative(programRoot, configPath).split(sep).join("/"), model: String(roster.model ?? config.model ?? "unknown"), provider: String(config.provider ?? "codex"), + ...(config.provider === "acpx" ? { acpxAgent: config.acpxAgent } : {}), driver: String(config.driver ?? "codex_app_server"), credentialName, ...(config.provider === "acpx" && config.acpxAgent === "grok" @@ -214,6 +215,7 @@ export async function buildProtocolEvalCatalog({ caseId, model: roster.model, provider: roster.provider, + ...(roster.acpxAgent ? { acpxAgent: roster.acpxAgent } : {}), driver: roster.driver, credentialName: roster.credentialName, ...(roster.authenticationMode ? { authenticationMode: roster.authenticationMode } : {}), @@ -465,13 +467,19 @@ export async function aggregateProtocolEvalCampaign({ const bundle = await readAttemptControlledEvidence(join(attemptRoot, entry.name)); const allocation = bundle.manifest.attempts.find(item => item.attemptId === attemptId); const selectedExecutions = new Set(bundle.manifest.attempts.map(item => item.executionId)); - if (bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || + const allocationsMatchCatalog = bundle.manifest.attempts.every(item => { + const cell = expectedByCell.get(item.executionId); + return cell && item.model === cell.model && item.provider === cell.provider + && item.acpxAgent === cell.acpxAgent + && cell.driver === (item.provider === "codex" ? "codex_app_server" : "acpx_runtime"); + }); + if (!allocationsMatchCatalog || bundle.manifest.campaignId !== catalog.campaignId || allocation.executionId !== cellId || selectedExecutions.size !== expectedByCell.size || [...expectedByCell.keys()].some(id => !selectedExecutions.has(id)) || bundle.manifest.sourceRevision !== source.paperclip.sha || bundle.manifest.definitionsRevision !== source.evals.sha) { throw new Error(`Controller archive identity drifted for ${cellId}`); } controllerBundles.push(bundle); - } else if (artifact.controllerAccounting) { + } else if ("controllerAccounting" in artifact || "controllerEvidenceError" in artifact) { throw new Error(`Missing private controller evidence for ${attemptId}; cost metadata alone is insufficient`); } await copyAttempt(