Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-08 01:57:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge completed retained-proof review fixes into portable runtime

Browse Source 
Exercise missing-provider result binding, atomic redaction and bounded native reads through the relocated artifact as well as source.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-26 09:39:12 -05:00
2 parents ebe8910c29 1197348567
commit 8f33531c2a
5 files changed
+67 -19
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/paperclip-runner/scripts/controlled-evals/archive.mjs
+13 -14
View File
@@ -125,23 +125,22 @@ export async function readAttemptControlledEvidence(directory, options) {
if (reportBytes) {
let report, provider;
try { report = JSON.parse(reportBytes.toString('utf8')); provider = providerBytes ? JSON.parse(providerBytes.toString('utf8')) : undefined; }
catch { return bundle; }
if (providerBytes || report?.providerArtifactSha256 === null) {
const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact };
delete expected.provenance; delete actual.provenance;
if (actual.controllerEvidenceError !== undefined) {
if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim()
|| typeof actual.accountingError !== 'string' || !actual.accountingError.trim()
|| canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) {
throw new Error('Invalid immutable controller retention failure');
}
delete actual.controllerEvidenceError;
if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; }
else { delete actual.accountingFailure; delete actual.accountingError; }
catch { throw new Error('Malformed immutable controller result proof'); }
const expected = controlledResultArtifact(provider, report, allocation), actual = { ...artifact };
delete expected.provenance; delete actual.provenance;
if (actual.controllerEvidenceError !== undefined) {
if (typeof actual.controllerEvidenceError !== 'string' || !actual.controllerEvidenceError.trim()
|| typeof actual.accountingError !== 'string' || !actual.accountingError.trim()
|| canonical(actual.accountingFailure) !== canonical({ class: 'provider_budget_coverage_unknown', category: 'provider_budget', retryable: false })) {
throw new Error('Invalid immutable controller retention failure');
}
if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
delete actual.controllerEvidenceError;
if (expected.accountingFailure) { actual.accountingFailure = expected.accountingFailure; actual.accountingError = expected.accountingError; }
else { delete actual.accountingFailure; delete actual.accountingError; }
}
if (canonical(actual) !== canonical(expected)) throw new Error('Immutable result disagrees with its controller proof');
}
return bundle;
}
packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs
+41
View File
@@ -12,6 +12,7 @@ const { canonical, costRuntime, observabilityBoundConfiguration, requestBindingS
const { attemptDirectory, runControlledEval } = await import(new URL('./run.mjs', runtime));
const { summarizeControlledCampaign } = await import(new URL('./report.mjs', runtime));
const { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } = await import(new URL('./archive.mjs', runtime));
const { readRegularFile, replaceEvidenceFile } = await import(new URL('./evidence-files.mjs', runtime));
import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs';
const childScript = options => `
@@ -696,3 +697,43 @@ for (const name of ['observability.json', 'otlp-native.jsonl', 'otlp-controller.
assert.equal(summary.committedUsd, '0.014000000');
});
}
test('credential redaction replaces a swapped symlink without changing its target', async t => {
const f = await fixture(t, { leakToken: true, requests: 0 });
const result = await f.run();
assert.equal(result.report.secretExcluded, false);
const path = join(attemptDirectory(f.manifest, 'attempt-0'), 'provider-artifact.json');
const bytes = await readRegularFile(path, 16_777_216);
const target = join(f.root, 'untouched-target'); await writeFile(target, 'target must remain unchanged');
await rm(path); await symlink(target, path);
await replaceEvidenceFile(path, bytes);
assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged');
assert.deepEqual(await readRegularFile(path, 16_777_216), bytes);
await symlink(target, `${path}.tmp`);
await assert.rejects(replaceEvidenceFile(path, bytes), { code: 'EEXIST' });
assert.equal(await readFile(target, 'utf8'), 'target must remain unchanged');
});
test('missing provider artifacts cannot bypass immutable result binding', async t => {
const f = await fixture(t, { missingArtifact: true, requests: 0 }); await f.run();
const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });
assert.equal(bundle.files['provider-artifact.json'], undefined);
assert.equal(JSON.parse(Buffer.from(bundle.files['result.json'].base64, 'base64')).providerArtifactSha256, undefined);
const directory = join(f.root, 'runs', 'attempt-0'); await mkdir(directory, { recursive: true });
const artifact = JSON.parse(await readFile(f.requests[0].outputPath, 'utf8'));
for (const [name, value] of Object.entries({ 'request.json': f.requests[0].request, 'artifact.json': artifact, 'controller-evidence.json': bundle })) await writeFile(join(directory, name), JSON.stringify(value));
await readAttemptControlledEvidence(directory);
for (const mutate of [
value => { value.usage = { estimatedCostNanodollars: 0 }; },
value => { value.turn = { status: 'completed', assistantText: 'forged success' }; },
value => { value.controllerAccounting.cleanupVerified = true; },
]) {
const changed = structuredClone(artifact); mutate(changed);
await writeFile(join(directory, 'artifact.json'), JSON.stringify(changed));
await assert.rejects(readAttemptControlledEvidence(directory), /Immutable result disagrees/);
}
const malformed = structuredClone(bundle), bytes = Buffer.from('{');
malformed.files['result.json'] = { sizeBytes: bytes.length, sha256: sha256(bytes), base64: bytes.toString('base64') };
await writeFile(join(directory, 'controller-evidence.json'), JSON.stringify(malformed));
await assert.rejects(readAttemptControlledEvidence(directory), /Malformed immutable controller result proof/);
});
packages/paperclip-runner/scripts/controlled-evals/evidence-files.mjs
+9 -1
View File
@@ -1,5 +1,5 @@
import { constants } from 'node:fs';
import { open } from 'node:fs/promises';
import { open, rename } from 'node:fs/promises';
const MiB = 1_048_576;
export const FILE_LIMITS = new Map([
@@ -27,3 +27,11 @@ export async function readRegularFile(path, maximum) {
return bytes.subarray(0, length);
} finally { await handle.close(); }
}
export async function replaceEvidenceFile(path, bytes) {
const temporary = `${path}.tmp`;
const handle = await open(temporary, 'wx', 0o600);
try { await handle.writeFile(bytes); await handle.sync(); }
finally { await handle.close(); }
await rename(temporary, path);
}
packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs
+2 -2
View File
@@ -73,11 +73,11 @@ test('copied controller runs without source dependencies, retains native evidenc
console.log('stdin imports are side-effect free');
`);
assert.match((await stdin).stdout, /stdin imports are side-effect free/);
const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|native post-child otlp-native.jsonl rejects fifo';
const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|native post-child otlp-native.jsonl rejects fifo|credential redaction replaces|missing provider artifacts';
const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], {
cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576,
});
assert.match(behavioral.stdout, /# pass 6\b/);
assert.match(behavioral.stdout, /# pass 8\b/);
const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], {
cwd: root, env: environment, timeout: 10_000,
}).then(value => value, error => error);
packages/paperclip-runner/scripts/controlled-evals/run.mjs
+2 -2
View File
@@ -6,7 +6,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { pathToFileURL } from 'node:url';
import { parseArgs } from 'node:util';
import { startControlledObservability } from './observability.mjs';
import { limitFor, readRegularFile } from './evidence-files.mjs';
import { limitFor, readRegularFile, replaceEvidenceFile } from './evidence-files.mjs';
import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs';
async function atomicJson(file, value) {
@@ -150,7 +150,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 });
const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8');
secretExcluded = secrets.every(secret => !raw.includes(secret));
if (!secretExcluded) await writeFile(artifactPath, redact(raw), { mode: 0o600 });
if (!secretExcluded) await replaceEvidenceFile(artifactPath, redact(raw));
providerArtifactSha256 = sha256(redact(raw));
artifact = JSON.parse(redact(raw));
const expected = { mode: 'controller_gateway', provider, declaredUpstreamRoute: manifest.route, campaignId: manifest.campaignId,