diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index 93dbcb017d1..14194c48d8d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -124,23 +124,22 @@ export async function readAttemptControlledEvidence(directory, options) { if (reportBytes) { let report, provider; try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; } - catch { return bundle; } - if (providerBytes || report?.providerArtifactSha256 === null) { - const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact }; - delete expected.provenance; delete actual.provenance; - if (actual.controllerEvidenceError !== undefined) { - if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim() - || typeof actual.accountingError !== 'string' || !actual.accountingError.trim() - || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) { - throw new Error('Invalid immutable controller retention failure'); - } - delete actual.controllerEvidenceError; - if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; } - else { delete actual.accountingFailure; delete actual.accountingError; } + catch { throw new Error('Malformed immutable controller result proof'); } + const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact }; + delete expected.provenance; delete actual.provenance; + if (actual.controllerEvidenceError !== undefined) { + if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim() + || typeof actual.accountingError !== 'string' || !actual.accountingError.trim() + || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) { + throw new Error('Invalid immutable controller retention failure'); } - if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); + delete actual.controllerEvidenceError; + if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; } + else { delete actual.accountingFailure; delete actual.accountingError; } } + if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); } + return bundle; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 43dba81e415..68498bd6bb6 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -8,6 +8,7 @@ import { join } from 'node:path'; import { canonical, costRuntime, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; import { attemptDirectory, runControlledEval } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; +import { readRegularFile, replaceEvidenceFile } from './evidence-files.mjs'; import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; @@ -532,3 +533,44 @@ test('campaign collection retains and audits private archives, while public proj await rm(join(attempt, 'controller-evidence.json')); await assert.rejects(aggregateProtocolEvalCampaign(input), /Missing private controller evidence/); }); + + +test('credential redaction replaces a swapped symlink without changing its target', async t => { + const f = await fixture(t, { leakToken: true, requests: 0 }); + const result = await f.run(); + assert.equal(result.report.secretExcluded, false); + const path = join(attemptDirectory(f.manifest, 'attempt-0'), 'provider-artifact.json'); + const bytes = await readRegularFile(path, 16_777_216); + const target = join(f.root, 'untouched-target'); await writeFile(target, 'target must remain unchanged'); + await rm(path); await symlink(target, path); + await replaceEvidenceFile(path, bytes); + assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged'); + assert.deepEqual(await readRegularFile(path, 16_777_216), bytes); + await symlink(target, `${path}.tmp`); + await assert.rejects(replaceEvidenceFile(path, bytes), { code: 'EEXIST' }); + assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged'); +}); + +test('missing provider artifacts cannot bypass immutable result binding', async t => { + const f = await fixture(t, { missingArtifact: true, requests: 0 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + assert.equal(bundle.files['provider-artifact.json'], undefined); + assert.equal(JSON.parse(Buffer.from(bundle.files['result.json'].base64, 'base64')).providerArtifactSha256, undefined); + const directory = join(f.root, 'runs', 'attempt-0'); await mkdir(directory, { recursive: true }); + const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + for (const [name, value] of Object.entries({ 'request.json': f.requests[0].request, 'artifact.json': artifact, 'controller-evidence.json': bundle })) await writeFile(join(directory, name), JSON.stringify(value)); + await readAttemptControlledEvidence(directory); + for (const mutate of [ + value => { value.usage = { estimatedCostNanodollars: 0 }; }, + value => { value.turn = { status: 'completed', assistantText: 'forged success' }; }, + value => { value.controllerAccounting.cleanupVerified = true; }, + ]) { + const changed = structuredClone(artifact); mutate(changed); + await writeFile(join(directory, 'artifact.json'), JSON.stringify(changed)); + await assert.rejects(readAttemptControlledEvidence(directory), /Immutable result disagrees/); + } + const malformed = structuredClone(bundle), bytes = Buffer.from('{'); + malformed.files['result.json'] = { sizeBytes: bytes.length, sha256: sha256(bytes), base64: bytes.toString('base64') }; + await writeFile(join(directory, 'controller-evidence.json'), JSON.stringify(malformed)); + await assert.rejects(readAttemptControlledEvidence(directory), /Malformed immutable controller result proof/); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs b/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs index b0aa4cfe485..75b339258c8 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs @@ -1,5 +1,5 @@ import { constants } from 'node:fs'; -import { open } from 'node:fs/promises'; +import { open, rename } from 'node:fs/promises'; const MiB = 1_048_576; export const FILE_LIMITS = new Map([ @@ -26,3 +26,11 @@ export async function readRegularFile(path, maximum) { return bytes.subarray(0, length); } finally { await handle.close(); } } + +export async function replaceEvidenceFile(path, bytes) { + const temporary = `${path}.tmp`; + const handle = await open(temporary, 'wx', 0o600); + try { await handle.writeFile(bytes); await handle.sync(); } + finally { await handle.close(); } + await rename(temporary, path); +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 921896180c7..48187a8991e 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -4,7 +4,7 @@ import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'n import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import { parseArgs } from 'node:util'; -import { limitFor, readRegularFile } from './evidence-files.mjs'; +import { limitFor, readRegularFile, replaceEvidenceFile } from './evidence-files.mjs'; import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs'; async function atomicJson(file, value) { @@ -140,7 +140,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8'); secretExcluded = secrets.every(secret => !raw.includes(secret)); - if (!secretExcluded) await writeFile(artifactPath, redact(raw), { mode: 0o600 }); + if (!secretExcluded) await replaceEvidenceFile(artifactPath, redact(raw)); providerArtifactSha256 = sha256(redact(raw)); artifact = JSON.parse(redact(raw)); const expected = { mode: 'controller_gateway', provider, declaredUpstreamRoute: manifest.route, campaignId: manifest.campaignId,