fix(evals): keep proven key-shaped SDK declarations out of the sealed upload

The second paid run got every plugin-SDK cell past the carrier check, then
the private result sealer rejected the tree with "Packaged evidence contains
credential contamination". Its packaging scan checks text for provider key
shapes without the SDK public-type proof, and
packages/shared/dist/validators/agent.d.ts declares
`OPENROUTER_API_KEY: z.ZodOptional<z.ZodString>`. secrets.least-scope-delivery
was even recorded as a secret_leak.

copyRetainedSourceSdk now skips files whose bytes match a key shape. Retention
already proved each such span a public schema type, and the receipt still
records the file's size and hash. Add a test that retains, copies and seals
the real built SDK with the real sealer (skipped when the SDK is not built);
it reproduces both CI failures without the two fixes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-08 22:23:44 -06:00
1 parent 49051ccae2
commit 1fece5bbd9
2 files changed
+23

No files matched your search

@@ -4,6 +4,7 @@ import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { existsSync } from "node:fs";
import { afterEach, expect, it } from "vitest";
import { stagePrivateDiagnostics } from "./private-diagnostics.js";
import { prepareSourceServerDependencies } from "./source-startup.js";
@@ -109,3 +110,19 @@ it("retains declarations and source maps under names the private result sealer a
for (const file of receipt.files) expect(hash(await readFile(path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS, file.retainedPath)))).toBe(file.sha256);
await expect(sealManualResultEvidence(context, [], results)).resolves.not.toThrow();
});
// Needs the built SDK (scripts/ensure-plugin-build-deps.mjs) and the pinned public source.
const builtSdk = ["packages/shared/dist", "packages/plugins/sdk/dist"].every(target =>
existsSync(new URL(`../../${target}/.paperclip-build-complete`, import.meta.url)));
it.skipIf(!builtSdk)("seals the real built SDK upload, keeping proven key-shaped declarations out", async () => {
const repositoryRoot = await realpath(path.resolve(import.meta.dirname, "../.."));
const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "source-sdk-real-unit-"))); roots.push(root);
const temporaryRoot = path.join(root, "attempt"), results = path.join(root, "results"), uploadDir = path.join(results, "campaign", "attempt-1");
await mkdir(path.join(temporaryRoot, "artifacts-private"), { recursive: true, mode: 0o700 }); await mkdir(path.join(temporaryRoot, "selected-private-inputs"), { recursive: true, mode: 0o700 });
const secrets = ["synthetic-unit-provider-credential-0123456789"]; stagePrivateDiagnostics(temporaryRoot, [executionId], secrets);
await retainPreparedSourceSdk({ repositoryRoot, temporaryRoot, executionIds: [executionId], sourceRevision, installed: false });
await copyRetainedSourceSdk(temporaryRoot, executionId, sourceRevision, uploadDir, secrets, repositoryRoot);
await expect(stat(path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS, "packages/shared/dist/validators/agent.d.ts.txt"))).rejects.toThrow();
await expect(stat(path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS, "packages/shared/dist/validators/agent.js"))).resolves.toBeTruthy();
const context = { root, protocol: MANUAL_PRIVATE_PROTOCOL, private: true, workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", attempt: "1", executionIds: [MANUAL_PRIVATE_EXECUTIONS[0]!] } as ManualPrivateHandoff;
await expect(sealManualResultEvidence(context, secrets, results)).resolves.not.toThrow();
}, 120_000);
+6
View File
@@ -136,6 +136,12 @@ export async function copyRetainedSourceSdk(root: string, executionId: string, s
const { files } = await verifyRetainedSourceSdk(root, executionId, sourceRevision, knownSecrets, repositoryRoot);
const destination = path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS); await mkdir(destination, { recursive: true, mode: 0o700 });
for (const file of files) {
// Verification proved every secret-shaped span in these bytes is a public
// schema type, but the private result sealer scans packaged text for key
// shapes without that proof (e.g. `OPENROUTER_API_KEY: z.ZodOptional` in
// validators/agent.d.ts) and rejects the whole attempt. Keep such files out
// of the upload; the retained receipt still records their size and hash.
if (file.file !== "receipt.json" && findSecretLeak(file.bytes, [])) continue;
const filename = path.join(destination, retainedPath(file.file)); await mkdir(path.dirname(filename), { recursive: true, mode: 0o700 });
await writeFile(filename, file.bytes, { flag: "wx", mode: 0o600 }); await chmod(filename, 0o600);
if (hash(await readFile(filename)) !== file.sha256) throw new Error("Source SDK published bytes changed");