Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-10-28 04:02:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge additional retained-evidence review repairs

Browse Source 
Preserve tracing behavior while incorporating immutable missing-provider binding and atomic credential-redaction replacement.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-26 09:38:34 -05:00
2 parents edc7642773 f5c1791623
commit 1197348567
4 files changed
+65 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/paperclip-runner/scripts/controlled-evals/archive.mjs
+13 -14
View File
@@ -125,23 +125,22 @@ export async function readAttemptControlledEvidence(directory, options) {
if (reportBytes) {
let report, provider;
try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; }
catch { return bundle; }
if (providerBytes || report?.providerArtifactSha256 === null) {
const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact };
delete expected.provenance; delete actual.provenance;
if (actual.controllerEvidenceError !== undefined) {
if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim()
|| typeof actual.accountingError !== 'string' || !actual.accountingError.trim()
|| canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) {
throw new Error('Invalid immutable controller retention failure');
}
delete actual.controllerEvidenceError;
if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; }
else { delete actual.accountingFailure; delete actual.accountingError; }
catch { throw new Error('Malformed immutable controller result proof'); }
const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact };
delete expected.provenance; delete actual.provenance;
if (actual.controllerEvidenceError !== undefined) {
if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim()
|| typeof actual.accountingError !== 'string' || !actual.accountingError.trim()
|| canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) {
throw new Error('Invalid immutable controller retention failure');
}
if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
delete actual.controllerEvidenceError;
if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; }
else { delete actual.accountingFailure; delete actual.accountingError; }
}
if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
}
return bundle;
}
packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs
+41
View File
@@ -8,6 +8,7 @@ import { join } from 'node:path';
import { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } from './manifest.mjs';
import { attemptDirectory, runControlledEval } from './run.mjs';
import { summarizeControlledCampaign } from './report.mjs';
import { readRegularFile, replaceEvidenceFile } from './evidence-files.mjs';
import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs';
import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs';
@@ -693,3 +694,43 @@ for (const name of ['observability.json', 'otlp-native.jsonl', 'otlp-controller.
assert.equal(summary.committedUsd, '0.014000000');
});
}
test('credential redaction replaces a swapped symlink without changing its target', async t => {
const f = await fixture(t, { leakToken: true, requests: 0 });
const result = await f.run();
assert.equal(result.report.secretExcluded, false);
const path = join(attemptDirectory(f.manifest, 'attempt-0'), 'provider-artifact.json');
const bytes = await readRegularFile(path, 16_777_216);
const target = join(f.root, 'untouched-target'); await writeFile(target, 'target must remain unchanged');
await rm(path); await symlink(target, path);
await replaceEvidenceFile(path, bytes);
assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged');
assert.deepEqual(await readRegularFile(path, 16_777_216), bytes);
await symlink(target, `${path}.tmp`);
await assert.rejects(replaceEvidenceFile(path, bytes), { code: 'EEXIST' });
assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged');
});
test('missing provider artifacts cannot bypass immutable result binding', async t => {
const f = await fixture(t, { missingArtifact: true, requests: 0 }); await f.run();
const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });
assert.equal(bundle.files['provider-artifact.json'], undefined);
assert.equal(JSON.parse(Buffer.from(bundle.files['result.json'].base64, 'base64')).providerArtifactSha256, undefined);
const directory = join(f.root, 'runs', 'attempt-0'); await mkdir(directory, { recursive: true });
const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8'));
for (const [name, value] of Object.entries({ 'request.json': f.requests[0].request, 'artifact.json': artifact, 'controller-evidence.json': bundle })) await writeFile(join(directory, name), JSON.stringify(value));
await readAttemptControlledEvidence(directory);
for (const mutate of [
value => { value.usage = { estimatedCostNanodollars: 0 }; },
value => { value.turn = { status: 'completed', assistantText: 'forged success' }; },
value => { value.controllerAccounting.cleanupVerified = true; },
]) {
const changed = structuredClone(artifact); mutate(changed);
await writeFile(join(directory, 'artifact.json'), JSON.stringify(changed));
await assert.rejects(readAttemptControlledEvidence(directory), /Immutable result disagrees/);
}
const malformed = structuredClone(bundle), bytes = Buffer.from('{');
malformed.files['result.json'] = { sizeBytes: bytes.length, sha256: sha256(bytes), base64: bytes.toString('base64') };
await writeFile(join(directory, 'controller-evidence.json'), JSON.stringify(malformed));
await assert.rejects(readAttemptControlledEvidence(directory), /Malformed immutable controller result proof/);
});
packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs
+9 -1
View File
@@ -1,5 +1,5 @@
import { constants } from 'node:fs';
import { open } from 'node:fs/promises';
import { open, rename } from 'node:fs/promises';
const MiB = 1_048_576;
export const FILE_LIMITS = new Map([
@@ -27,3 +27,11 @@ export async function readRegularFile(path, maximum) {
return bytes.subarray(0, length);
} finally { await handle.close(); }
}
export async function replaceEvidenceFile(path, bytes) {
const temporary = `${path}.tmp`;
const handle = await open(temporary, 'wx', 0o600);
try { await handle.writeFile(bytes); await handle.sync(); }
finally { await handle.close(); }
await rename(temporary, path);
}
packages/paperclip-runner/scripts/controlled-evals/run.mjs
+2 -2
View File
@@ -5,7 +5,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { pathToFileURL } from 'node:url';
import { parseArgs } from 'node:util';
import { startControlledObservability } from './observability.mjs';
import { limitFor, readRegularFile } from './evidence-files.mjs';
import { limitFor, readRegularFile, replaceEvidenceFile } from './evidence-files.mjs';
import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs';
async function atomicJson(file, value) {
@@ -149,7 +149,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 });
const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8');
secretExcluded = secrets.every(secret => !raw.includes(secret));
if (!secretExcluded) await writeFile(artifactPath, redact(raw), { mode: 0o600 });
if (!secretExcluded) await replaceEvidenceFile(artifactPath, redact(raw));
providerArtifactSha256 = sha256(redact(raw));
artifact = JSON.parse(redact(raw));
const expected = { mode: 'controller_gateway', provider, declaredUpstreamRoute: manifest.route, campaignId: manifest.campaignId,