mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): keep proven key-shaped SDK declarations out of the sealed upload
The second paid run got every plugin-SDK cell past the carrier check, then the private result sealer rejected the tree with "Packaged evidence contains credential contamination". Its packaging scan checks text for provider key shapes without the SDK public-type proof, and packages/shared/dist/validators/agent.d.ts declares `OPENROUTER_API_KEY: z.ZodOptional<z.ZodString>`. secrets.least-scope-delivery was even recorded as a secret_leak. copyRetainedSourceSdk now skips files whose bytes match a key shape. Retention already proved each such span a public schema type, and the receipt still records the file's size and hash. Add a test that retains, copies and seals the real built SDK with the real sealer (skipped when the SDK is not built); it reproduces both CI failures without the two fixes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
49051ccae2
commit
1fece5bbd9
2 files changed
+23
No files matched your search
@@ -4,6 +4,7 @@ import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { existsSync } from "node:fs";
|
||||
import { afterEach, expect, it } from "vitest";
|
||||
import { stagePrivateDiagnostics } from "./private-diagnostics.js";
|
||||
import { prepareSourceServerDependencies } from "./source-startup.js";
|
||||
@@ -109,3 +110,19 @@ it("retains declarations and source maps under names the private result sealer a
|
||||
for (const file of receipt.files) expect(hash(await readFile(path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS, file.retainedPath)))).toBe(file.sha256);
|
||||
await expect(sealManualResultEvidence(context, [], results)).resolves.not.toThrow();
|
||||
});
|
||||
// Needs the built SDK (scripts/ensure-plugin-build-deps.mjs) and the pinned public source.
|
||||
const builtSdk = ["packages/shared/dist", "packages/plugins/sdk/dist"].every(target =>
|
||||
existsSync(new URL(`../../${target}/.paperclip-build-complete`, import.meta.url)));
|
||||
it.skipIf(!builtSdk)("seals the real built SDK upload, keeping proven key-shaped declarations out", async () => {
|
||||
const repositoryRoot = await realpath(path.resolve(import.meta.dirname, "../.."));
|
||||
const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "source-sdk-real-unit-"))); roots.push(root);
|
||||
const temporaryRoot = path.join(root, "attempt"), results = path.join(root, "results"), uploadDir = path.join(results, "campaign", "attempt-1");
|
||||
await mkdir(path.join(temporaryRoot, "artifacts-private"), { recursive: true, mode: 0o700 }); await mkdir(path.join(temporaryRoot, "selected-private-inputs"), { recursive: true, mode: 0o700 });
|
||||
const secrets = ["synthetic-unit-provider-credential-0123456789"]; stagePrivateDiagnostics(temporaryRoot, [executionId], secrets);
|
||||
await retainPreparedSourceSdk({ repositoryRoot, temporaryRoot, executionIds: [executionId], sourceRevision, installed: false });
|
||||
await copyRetainedSourceSdk(temporaryRoot, executionId, sourceRevision, uploadDir, secrets, repositoryRoot);
|
||||
await expect(stat(path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS, "packages/shared/dist/validators/agent.d.ts.txt"))).rejects.toThrow();
|
||||
await expect(stat(path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS, "packages/shared/dist/validators/agent.js"))).resolves.toBeTruthy();
|
||||
const context = { root, protocol: MANUAL_PRIVATE_PROTOCOL, private: true, workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", attempt: "1", executionIds: [MANUAL_PRIVATE_EXECUTIONS[0]!] } as ManualPrivateHandoff;
|
||||
await expect(sealManualResultEvidence(context, secrets, results)).resolves.not.toThrow();
|
||||
}, 120_000);
|
||||
@@ -136,6 +136,12 @@ export async function copyRetainedSourceSdk(root: string, executionId: string, s
|
||||
const { files } = await verifyRetainedSourceSdk(root, executionId, sourceRevision, knownSecrets, repositoryRoot);
|
||||
const destination = path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS); await mkdir(destination, { recursive: true, mode: 0o700 });
|
||||
for (const file of files) {
|
||||
// Verification proved every secret-shaped span in these bytes is a public
|
||||
// schema type, but the private result sealer scans packaged text for key
|
||||
// shapes without that proof (e.g. `OPENROUTER_API_KEY: z.ZodOptional` in
|
||||
// validators/agent.d.ts) and rejects the whole attempt. Keep such files out
|
||||
// of the upload; the retained receipt still records their size and hash.
|
||||
if (file.file !== "receipt.json" && findSecretLeak(file.bytes, [])) continue;
|
||||
const filename = path.join(destination, retainedPath(file.file)); await mkdir(path.dirname(filename), { recursive: true, mode: 0o700 });
|
||||
await writeFile(filename, file.bytes, { flag: "wx", mode: 0o600 }); await chmod(filename, 0o600);
|
||||
if (hash(await readFile(filename)) !== file.sha256) throw new Error("Source SDK published bytes changed");
|
||||
|
||||
Reference in new issue
Block a user