diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index f575cb6e455..ec246123490 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -125,23 +125,22 @@ export async function readAttemptControlledEvidence(directory, options) { if (reportBytes) { let report, provider; try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; } - catch { return bundle; } - if (providerBytes || report?.providerArtifactSha256 === null) { - const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact }; - delete expected.provenance; delete actual.provenance; - if (actual.controllerEvidenceError !== undefined) { - if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim() - || typeof actual.accountingError !== 'string' || !actual.accountingError.trim() - || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) { - throw new Error('Invalid immutable controller retention failure'); - } - delete actual.controllerEvidenceError; - if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; } - else { delete actual.accountingFailure; delete actual.accountingError; } + catch { throw new Error('Malformed immutable controller result proof'); } + const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact }; + delete expected.provenance; delete actual.provenance; + if (actual.controllerEvidenceError !== undefined) { + if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim() + || typeof actual.accountingError !== 'string' || !actual.accountingError.trim() + || canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) { + throw new Error('Invalid immutable controller retention failure'); } - if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); + delete actual.controllerEvidenceError; + if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; } + else { delete actual.accountingFailure; delete actual.accountingError; } } + if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof'); } + return bundle; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 0e35f755413..7e816c16af3 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -12,6 +12,7 @@ const { canonical, costRuntime, observabilityBoundConfiguration, requestBindingS const { attemptDirectory, runControlledEval } = await import(new URL('./run.mjs', runtime)); const { summarizeControlledCampaign } = await import(new URL('./report.mjs', runtime)); const { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } = await import(new URL('./archive.mjs', runtime)); +const { readRegularFile, replaceEvidenceFile } = await import(new URL('./evidence-files.mjs', runtime)); import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; const childScript = options => ` @@ -696,3 +697,43 @@ for (const name of ['observability.json', 'otlp-native.jsonl', 'otlp-controller. assert.equal(summary.committedUsd, '0.014000000'); }); } + +test('credential redaction replaces a swapped symlink without changing its target', async t => { + const f = await fixture(t, { leakToken: true, requests: 0 }); + const result = await f.run(); + assert.equal(result.report.secretExcluded, false); + const path = join(attemptDirectory(f.manifest, 'attempt-0'), 'provider-artifact.json'); + const bytes = await readRegularFile(path, 16_777_216); + const target = join(f.root, 'untouched-target'); await writeFile(target, 'target must remain unchanged'); + await rm(path); await symlink(target, path); + await replaceEvidenceFile(path, bytes); + assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged'); + assert.deepEqual(await readRegularFile(path, 16_777_216), bytes); + await symlink(target, `${path}.tmp`); + await assert.rejects(replaceEvidenceFile(path, bytes), { code: 'EEXIST' }); + assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged'); +}); + +test('missing provider artifacts cannot bypass immutable result binding', async t => { + const f = await fixture(t, { missingArtifact: true, requests: 0 }); await f.run(); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + assert.equal(bundle.files['provider-artifact.json'], undefined); + assert.equal(JSON.parse(Buffer.from(bundle.files['result.json'].base64, 'base64')).providerArtifactSha256, undefined); + const directory = join(f.root, 'runs', 'attempt-0'); await mkdir(directory, { recursive: true }); + const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8')); + for (const [name, value] of Object.entries({ 'request.json': f.requests[0].request, 'artifact.json': artifact, 'controller-evidence.json': bundle })) await writeFile(join(directory, name), JSON.stringify(value)); + await readAttemptControlledEvidence(directory); + for (const mutate of [ + value => { value.usage = { estimatedCostNanodollars: 0 }; }, + value => { value.turn = { status: 'completed', assistantText: 'forged success' }; }, + value => { value.controllerAccounting.cleanupVerified = true; }, + ]) { + const changed = structuredClone(artifact); mutate(changed); + await writeFile(join(directory, 'artifact.json'), JSON.stringify(changed)); + await assert.rejects(readAttemptControlledEvidence(directory), /Immutable result disagrees/); + } + const malformed = structuredClone(bundle), bytes = Buffer.from('{'); + malformed.files['result.json'] = { sizeBytes: bytes.length, sha256: sha256(bytes), base64: bytes.toString('base64') }; + await writeFile(join(directory, 'controller-evidence.json'), JSON.stringify(malformed)); + await assert.rejects(readAttemptControlledEvidence(directory), /Malformed immutable controller result proof/); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs b/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs index d6a38aef844..14594399a98 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs @@ -1,5 +1,5 @@ import { constants } from 'node:fs'; -import { open } from 'node:fs/promises'; +import { open, rename } from 'node:fs/promises'; const MiB = 1_048_576; export const FILE_LIMITS = new Map([ @@ -27,3 +27,11 @@ export async function readRegularFile(path, maximum) { return bytes.subarray(0, length); } finally { await handle.close(); } } + +export async function replaceEvidenceFile(path, bytes) { + const temporary = `${path}.tmp`; + const handle = await open(temporary, 'wx', 0o600); + try { await handle.writeFile(bytes); await handle.sync(); } + finally { await handle.close(); } + await rename(temporary, path); +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs index ca8224cbac5..6e57c49e334 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -73,11 +73,11 @@ test('copied controller runs without source dependencies, retains native evidenc console.log('stdin imports are side-effect free'); `); assert.match((await stdin).stdout, /stdin imports are side-effect free/); - const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|native post-child otlp-native.jsonl rejects fifo'; + const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|native post-child otlp-native.jsonl rejects fifo|credential redaction replaces|missing provider artifacts'; const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], { cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576, }); - assert.match(behavioral.stdout, /# pass 6\b/); + assert.match(behavioral.stdout, /# pass 8\b/); const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { cwd: root, env: environment, timeout: 10_000, }).then(value => value, error => error); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index c9cae655d49..d81a0a054bb 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -6,7 +6,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import { parseArgs } from 'node:util'; import { startControlledObservability } from './observability.mjs'; -import { limitFor, readRegularFile } from './evidence-files.mjs'; +import { limitFor, readRegularFile, replaceEvidenceFile } from './evidence-files.mjs'; import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs'; async function atomicJson(file, value) { @@ -150,7 +150,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8'); secretExcluded = secrets.every(secret => !raw.includes(secret)); - if (!secretExcluded) await writeFile(artifactPath, redact(raw), { mode: 0o600 }); + if (!secretExcluded) await replaceEvidenceFile(artifactPath, redact(raw)); providerArtifactSha256 = sha256(redact(raw)); artifact = JSON.parse(redact(raw)); const expected = { mode: 'controller_gateway', provider, declaredUpstreamRoute: manifest.route, campaignId: manifest.campaignId,