From ed24a4a4449d56631494b4c886fd0ccd73fea60a Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 09:29:20 -0500 Subject: [PATCH] Retain certified source SDK evidence for plugin fixtures --- tests/runner-e2e/README.md | 12 ++- tests/runner-e2e/launch.ts | 3 + tests/runner-e2e/server.ts | 5 +- tests/runner-e2e/source-sdk-evidence.test.ts | 97 +++++++++++++++++++ tests/runner-e2e/source-sdk-evidence.ts | 99 ++++++++++++++++++++ tests/runner-e2e/source-startup.ts | 3 + 6 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 tests/runner-e2e/source-sdk-evidence.test.ts create mode 100644 tests/runner-e2e/source-sdk-evidence.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 41af5f0e87d..e2f63782a9b 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -80,8 +80,16 @@ one attempt, two expected provider admissions, and 1,000-cent company and agent hard stops. Automatic infrastructure retries are disabled. Existing campaign reports retain failures, timings, usage, runtime costs, and source identity. Source-server startup prepares the existing shared/plugin SDK build dependencies -before importing native tool authority. Installed-package checks never execute -checkout build helpers. +before importing native tool authority. The six source-local plugin-fixture +journeys retain the actual certified shared/SDK outputs before server listen, +fingerprinting, agent setup and human dispatch. Exact bytes, completion markers, +source/lock/helper hashes remain in Actions-only `diagnostics/source-plugin-sdk`. +The original hidden completion-marker bytes use the explicit non-hidden archival +name `build-completion-receipt.json`, with original/retained paths and hashes +recorded so bounded upload staging preserves them. The public report does not +publish those files. Startup failure does not trigger +a post-failure rebuild or manufacture missing output evidence. Installed-package +checks never execute checkout build helpers. ## Configured repository and conversion checks diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 1b9d70316c0..c63f48b18f2 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,4 +1,5 @@ import { copyHealthHelloDetail } from "./chat-readiness-detail.js"; +import { copyRetainedSourceSdk } from "./source-sdk-evidence.js"; import { prepareChatRemoteHost } from "./chat-remote-host.js"; import { POSTGRES_ARTIFACT_ENV, POSTGRES_IMAGE_ID_ENV } from "./docker-postgres.js"; import { prepareOwnedPublicOrigin, PublicOriginCleanupError, PUBLIC_ORIGIN_CONTROL_ENV, PUBLIC_ORIGIN_RECEIPT_ENV } from "./chat-public-origin.js"; @@ -916,6 +917,7 @@ async function runAttempt(input: { }); } await copyHealthHelloDetail(temporaryRoot, candidate.id, uploadDir, credentials); + await copyRetainedSourceSdk(temporaryRoot, candidate.id, result.source?.sha ?? null, uploadDir, credentials); console.log( `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`, ); @@ -948,6 +950,7 @@ async function runAttempt(input: { schemaVersion: 1, executionId: candidate.id, originalResult: original, packaging: "failed; raw diagnostics retained privately", error: error instanceof Error ? error.message : String(error), }, attemptSecrets), null, 2)}\n`, { mode: 0o600 }); + await copyRetainedSourceSdk(temporaryRoot, candidate.id, result.source?.sha ?? null, uploadDir, attemptSecrets).catch(() => {}); if (rawCleanupResults.length < executions.length) rawCleanupResults.push({ cleanup: original.cleanup, synthetic: original === fallback, status: original.status }); publishedResults.push(result); publishedResultPaths.set(candidate.id, resultPath); } diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 60b9490f208..97cbb6dc25b 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -17,6 +17,7 @@ import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server- import { runnerE2ETypeScriptProcessArgs } from "./web-server-command.js"; import { qualifyLegacyClaudeCli } from "./legacy-claude-cli.js"; import { prepareSourceServerDependencies } from "./source-startup.js"; +import { resolveRunnerE2ESource } from "./source.js"; import { spawn, type ChildProcess } from "node:child_process"; import { createWriteStream } from "node:fs"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; @@ -122,7 +123,9 @@ const definedServerEnvironment = Object.fromEntries( ), ); -await prepareSourceServerDependencies(repositoryRoot, definedServerEnvironment, usesInstalledCli(process.env)); +await prepareSourceServerDependencies(repositoryRoot, definedServerEnvironment, usesInstalledCli(process.env), { + temporaryRoot, executionIds, sourceRevision: resolveRunnerE2ESource().sha, +}); await Promise.all([ mkdir(path.dirname(logPath), { recursive: true }), diff --git a/tests/runner-e2e/source-sdk-evidence.test.ts b/tests/runner-e2e/source-sdk-evidence.test.ts new file mode 100644 index 00000000000..307e8a6b4c5 --- /dev/null +++ b/tests/runner-e2e/source-sdk-evidence.test.ts @@ -0,0 +1,97 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, it } from "vitest"; +import { stagePrivateDiagnostics } from "./private-diagnostics.js"; +import { prepareSourceServerDependencies } from "./source-startup.js"; +import { copyRetainedSourceSdk, retainPreparedSourceSdk, SOURCE_SDK_DIAGNOSTICS } from "./source-sdk-evidence.js"; +import { packageEvidence } from "./evidence.js"; +import { isHistoricalBundlePathAllowed } from "./history-publish.js"; + +const roots: string[] = [], executionId = "chat-features.runner-opencode.local.plugins.install-plugin", sourceRevision = "a".repeat(40); +const hash = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture(secrets: string[] = []) { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "source-sdk-synthetic-unit-"))); roots.push(root); + const repositoryRoot = path.join(root, "repo"), temporaryRoot = path.join(root, "attempt"), uploadDir = path.join(root, "uploaded"); + const put = async (file: string, value: string | Buffer) => { await mkdir(path.dirname(path.join(repositoryRoot, file)), { recursive: true }); await writeFile(path.join(repositoryRoot, file), value); }; + await put("scripts/ensure-plugin-build-deps.mjs", await readFile(new URL("../../scripts/ensure-plugin-build-deps.mjs", import.meta.url))); + await put("pnpm-lock.yaml", "synthetic unit lock\n"); + for (const directory of ["packages/shared", "packages/plugins/sdk"]) { + await put(`${directory}/src/index.ts`, "export const syntheticFixture = true;\n"); await put(`${directory}/tsconfig.json`, "{}"); + } + await put("node_modules/typescript/bin/tsc", `const fs=require('node:fs'),path=require('node:path');if(process.env.FAIL_COMPILER==='1')process.exit(7);const dir=path.join(path.dirname(process.argv[3]),'dist');fs.mkdirSync(path.join(dir,'nested'),{recursive:true});fs.writeFileSync(path.join(dir,'index.js'),process.env.SYNTHETIC_OUTPUT_CONTENT||'export const syntheticFixture = true;\\n');fs.writeFileSync(path.join(dir,'nested/child.js'),'export {};\\n');`); + await mkdir(path.join(temporaryRoot, "artifacts-private"), { recursive: true, mode: 0o700 }); + await mkdir(path.join(temporaryRoot, "selected-private-inputs"), { recursive: true, mode: 0o700 }); + stagePrivateDiagnostics(temporaryRoot, [executionId], secrets); + const input = { repositoryRoot, temporaryRoot, executionIds: [executionId], sourceRevision, installed: false }; + const prepare = (environment: NodeJS.ProcessEnv = {}) => prepareSourceServerDependencies(repositoryRoot, environment, false, { temporaryRoot, executionIds: [executionId], sourceRevision }); + return { root, repositoryRoot, temporaryRoot, uploadDir, input, prepare }; +} + +it.each([true, false])("retains exact synthetic compiler output before admission and survives normal packaging/cleanup (pass=%s)", async pass => { + const f = await fixture(); + await expect(readFile(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"))).rejects.toThrow(); + await f.prepare(); + const receipt = JSON.parse(await readFile(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build/receipt.json"), "utf8")); + expect(receipt).toMatchObject({ schema: "paperclip.source-plugin-sdk/v1", executionId, sourceRevision }); expect(receipt.certifiedTargets).toHaveLength(2); + expect(receipt.buildHelperSha256).toBe(hash(await readFile(path.join(f.repositoryRoot, "scripts/ensure-plugin-build-deps.mjs")))); + const packaged = await packageEvidence({ privateDir: path.join(f.temporaryRoot, "case"), uploadDir: f.uploadDir, secrets: [], expectPassScreenshot: pass }); + expect(packaged.leaks).toEqual([]); + await copyRetainedSourceSdk(f.temporaryRoot, executionId, sourceRevision, f.uploadDir, []); + const bounded = path.join(f.root, "bounded"); + await promisify(execFile)(process.execPath, [new URL("./artifact-budget.mjs", import.meta.url).pathname, "stage", "evidence", f.uploadDir, bounded]); + await rm(f.temporaryRoot, { recursive: true }); + for (const file of receipt.files) { + const retained = path.join(f.uploadDir, SOURCE_SDK_DIAGNOSTICS, file.retainedPath); + expect(hash(await readFile(retained))).toBe(file.sha256); expect((await stat(retained)).mode & 0o777).toBe(0o600); + expect(file.retainedSha256).toBe(file.sha256); + expect(hash(await readFile(path.join(bounded, SOURCE_SDK_DIAGNOSTICS, file.retainedPath)))).toBe(file.sha256); + expect(isHistoricalBundlePathAllowed(`evidence/${executionId}/attempt-1/${SOURCE_SDK_DIAGNOSTICS}/${file.retainedPath}`, true)).toBe(false); + } + expect(isHistoricalBundlePathAllowed(`evidence/${executionId}/attempt-1/${SOURCE_SDK_DIAGNOSTICS}/receipt.json`, true)).toBe(false); +}); +it("cannot retain output before existing helper certification or after compilation failure; never rebuilds during packaging", async () => { + const f = await fixture(); + await expect(retainPreparedSourceSdk(f.input)).rejects.toThrow(); + await expect(f.prepare({ FAIL_COMPILER: "1" })).rejects.toMatchObject({ code: 7 }); + await expect(readFile(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build/receipt.json"))).rejects.toThrow(); + await expect(copyRetainedSourceSdk(f.temporaryRoot, executionId, sourceRevision, f.uploadDir, [])).rejects.toThrow(); + await expect(readFile(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"))).rejects.toThrow(); +}); +it("rejects output changed after certification instead of copying a stale completion marker", async () => { + const f = await fixture(); await f.prepare(); + await rm(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"), { recursive: true }); + await writeFile(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"), "changed after certification"); + await expect(retainPreparedSourceSdk(f.input)).rejects.toThrow("no longer matches"); + await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow(); +}); +it("requires the existing owned redaction registry before retaining compiled bytes", async () => { + const f = await fixture(); await rm(path.join(f.temporaryRoot, "selected-private-inputs/diagnostic-redaction.json")); + await expect(f.prepare()).rejects.toThrow(); + await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow(); +}); +it.each(["raw", "base64"])("refuses registered %s credential bytes before any retention write", async encoding => { + const secret = "synthetic-private-upstream-credential-value", f = await fixture([secret]); + const value = encoding === "base64" ? Buffer.from(secret).toString("base64") : secret; + await expect(f.prepare({ SYNTHETIC_OUTPUT_CONTENT: value })).rejects.toThrow("privacy check failed"); + await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow(); +}); +it.each(["wrong source", "wrong selected case", "tampered bytes", "new secret", "symlink"])("fails closed for %s in retained custody", async invalid => { + const f = await fixture(); await f.prepare(); + const staged = path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"), file = path.join(staged, "packages/plugins/sdk/dist/index.js"); + if (invalid === "tampered bytes") await writeFile(file, "changed compiler bytes"); + if (invalid === "symlink") { await rm(file); await symlink(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"), file); } + await expect(copyRetainedSourceSdk(f.temporaryRoot, invalid === "wrong selected case" ? "chat-features.runner-opencode.local.plugins.configure-plugin" : executionId, + invalid === "wrong source" ? "b".repeat(40) : sourceRevision, f.uploadDir, invalid === "new secret" ? ["syntheticFixture"] : [])).rejects.toThrow(); +}); +it("ordinary/multi-cell/installed lanes do not retain or require SDK evidence", async () => { + const f = await fixture(); + for (const changed of [{ executionIds: ["chat-features.runner-opencode.local.tasks.website-task"] }, { executionIds: [executionId, executionId] }, { installed: true }]) + await expect(retainPreparedSourceSdk({ ...f.input, ...changed })).resolves.toBeUndefined(); + await expect(copyRetainedSourceSdk(f.temporaryRoot, "unselected", null, f.uploadDir, [])).resolves.toBeUndefined(); + await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow(); +}); diff --git a/tests/runner-e2e/source-sdk-evidence.ts b/tests/runner-e2e/source-sdk-evidence.ts new file mode 100644 index 00000000000..fae0175957a --- /dev/null +++ b/tests/runner-e2e/source-sdk-evidence.ts @@ -0,0 +1,99 @@ +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { diagnosticSecrets, readPrivateDiagnosticSecrets } from "./private-diagnostics.js"; +import { findSecretLeak } from "./redaction.js"; + +const selectedIds = new Set(["plugins.configure-plugin", "plugins.disable-plugin", "plugins.install-plugin", "plugins.use-plugin-tool", "routines.plugin-managed-routine", "secrets.least-scope-delivery"].map(id => `chat-features.runner-opencode.local.${id}`)); +export const SOURCE_SDK_DIAGNOSTICS = "diagnostics/source-plugin-sdk"; +const targets = ["packages/shared/dist", "packages/plugins/sdk/dist"] as const; +const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); +const hex = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); +const staging = (root: string) => path.join(root, "artifacts-private", "source-sdk-build"); +interface BuiltFile { file: string; size: number; sha256: string } +const retainedPath = (file: string) => file.replace(/\/\.paperclip-build-complete$/, "/build-completion-receipt.json"); +interface RetainedBuild { + schema: "paperclip.source-plugin-sdk/v1"; executionId: string; sourceRevision: string; retainedAt: string; + buildHelperSha256: string; lockSha256: string; files: Array; + certifiedTargets: Array<{ directory: string; sources: string; outputs: string }>; +} +function selected(ids: readonly string[]) { return ids.length === 1 && selectedIds.has(ids[0]!); } +function checkSecret(bytes: Buffer, secrets: readonly string[]) { + if (findSecretLeak(bytes.toString("utf8"), secrets)) throw new Error("Source SDK evidence privacy check failed; bytes withheld"); +} +async function boundedTree(root: string) { + const files: Array = []; let total = 0; + async function visit(directory: string) { + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const filename = path.join(directory, entry.name); + if (entry.isDirectory()) await visit(filename); + else { + const info = await lstat(filename); + if (!info.isFile() || info.isSymbolicLink() || info.size > 8 * 1024 * 1024) throw new Error("Source SDK output custody invalid"); + const bytes = await readFile(filename); total += bytes.length; + if (files.length >= 4096 || total > 64 * 1024 * 1024) throw new Error("Source SDK output exceeds evidence bound"); + files.push({ file: path.relative(root, filename).split(path.sep).join("/"), size: bytes.length, sha256: hash(bytes), bytes }); + } + } + } + const info = await lstat(root); if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Source SDK output root invalid"); + await visit(root); return files; +} +function outputHash(files: Array<{ file: string; bytes: Buffer }>) { + const digest = createHash("sha256"); + for (const file of files.filter(row => row.file !== ".paperclip-build-complete")) { + digest.update(JSON.stringify([file.file, file.bytes.length])); digest.update(file.bytes); + } + return digest.digest("hex"); +} +export async function retainPreparedSourceSdk(input: { repositoryRoot: string; temporaryRoot: string; executionIds: readonly string[]; sourceRevision: string | null; installed: boolean }) { + if (input.installed || !selected(input.executionIds)) return; + if (!/^[a-f0-9]{40}$/.test(input.sourceRevision ?? "")) throw new Error("Source SDK retention requires exact source revision"); + const root = await realpath(input.temporaryRoot), repository = await realpath(input.repositoryRoot); + if (await realpath(path.join(root, "artifacts-private")) !== path.join(root, "artifacts-private")) throw new Error("Source SDK staging custody invalid"); + const secrets = diagnosticSecrets(readPrivateDiagnosticSecrets(path.join(root, "selected-private-inputs", "diagnostic-redaction.json"), root)).values; + const buffers: Array = [], certifiedTargets: RetainedBuild["certifiedTargets"] = []; + for (const directory of targets) { + const files = await boundedTree(path.join(repository, directory)); + const marker = files.find(row => row.file === ".paperclip-build-complete"), entry = files.find(row => row.file === "index.js"); + if (!marker || !entry) throw new Error("Source SDK build completion evidence missing"); + const certificate = JSON.parse(marker.bytes.toString("utf8")); + if (!hex(certificate.sources) || !hex(certificate.outputs) || outputHash(files) !== certificate.outputs) throw new Error("Source SDK build output no longer matches its completed compiler receipt"); + certifiedTargets.push({ directory, sources: certificate.sources, outputs: certificate.outputs }); + for (const file of files) { checkSecret(file.bytes, secrets); buffers.push({ ...file, file: `${directory}/${file.file}` }); } + } + if (buffers.length >= 4096 || buffers.reduce((total, file) => total + file.size, 0) > 63 * 1024 * 1024) throw new Error("Source SDK output exceeds combined evidence bound"); + const receipt: RetainedBuild = { schema: "paperclip.source-plugin-sdk/v1", executionId: input.executionIds[0]!, sourceRevision: input.sourceRevision!, retainedAt: new Date().toISOString(), + buildHelperSha256: hash(await readFile(path.join(repository, "scripts/ensure-plugin-build-deps.mjs"))), lockSha256: hash(await readFile(path.join(repository, "pnpm-lock.yaml"))), + certifiedTargets, files: buffers.map(({ bytes: _, ...row }) => ({ ...row, retainedPath: retainedPath(row.file), retainedSha256: row.sha256 })) }; + if (new Set(receipt.files.map(file => file.retainedPath)).size !== receipt.files.length) throw new Error("Source SDK retained filename collision"); + const receiptBytes = Buffer.from(JSON.stringify(receipt, null, 2) + "\n"); checkSecret(receiptBytes, secrets); + const destination = staging(root); await mkdir(destination, { mode: 0o700 }); + try { + for (const file of buffers) { + const filename = path.join(destination, file.file); await mkdir(path.dirname(filename), { recursive: true, mode: 0o700 }); + await writeFile(filename, file.bytes, { flag: "wx", mode: 0o600 }); + } + await writeFile(path.join(destination, "receipt.json"), receiptBytes, { flag: "wx", mode: 0o600 }); + } catch (error) { await rm(destination, { recursive: true, force: true }); throw error; } +} +export async function copyRetainedSourceSdk(root: string, executionId: string, sourceRevision: string | null, uploadDir: string, knownSecrets: readonly string[]) { + if (!selected([executionId])) return; + if (await realpath(staging(root)) !== staging(root)) throw new Error("Source SDK retained custody invalid"); + const secrets = diagnosticSecrets(knownSecrets).values, files = await boundedTree(staging(root)); + const receiptFile = files.find(row => row.file === "receipt.json"); if (!receiptFile) throw new Error("Source SDK retained receipt missing"); + const receipt = JSON.parse(receiptFile.bytes.toString("utf8")) as RetainedBuild; + if (receipt.schema !== "paperclip.source-plugin-sdk/v1" || receipt.executionId !== executionId || receipt.sourceRevision !== sourceRevision + || !hex(receipt.buildHelperSha256) || !hex(receipt.lockSha256) || !Array.isArray(receipt.files)) throw new Error("Source SDK retained identity mismatch"); + const actual = files.filter(row => row.file !== "receipt.json"); + if (actual.length !== receipt.files.length || actual.some(row => !targets.some(target => row.file.startsWith(`${target}/`)) + || !receipt.files.some(expected => expected.file === row.file && expected.retainedPath === retainedPath(row.file) && expected.size === row.size && expected.sha256 === row.sha256 && expected.retainedSha256 === row.sha256))) throw new Error("Source SDK retained bytes changed"); + for (const file of files) checkSecret(file.bytes, secrets); + const destination = path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS); await mkdir(destination, { recursive: true, mode: 0o700 }); + for (const file of files) { + const filename = path.join(destination, retainedPath(file.file)); await mkdir(path.dirname(filename), { recursive: true, mode: 0o700 }); + await writeFile(filename, file.bytes, { flag: "wx", mode: 0o600 }); await chmod(filename, 0o600); + if (hash(await readFile(filename)) !== file.sha256) throw new Error("Source SDK published bytes changed"); + } +} diff --git a/tests/runner-e2e/source-startup.ts b/tests/runner-e2e/source-startup.ts index 6c6aafdcf1f..807fb93ab16 100644 --- a/tests/runner-e2e/source-startup.ts +++ b/tests/runner-e2e/source-startup.ts @@ -1,11 +1,13 @@ import { execFile } from "node:child_process"; import path from "node:path"; import { promisify } from "node:util"; +import { retainPreparedSourceSdk } from "./source-sdk-evidence.js"; export async function prepareSourceServerDependencies( repositoryRoot: string, environment: NodeJS.ProcessEnv, installed: boolean, + evidence?: { temporaryRoot: string; executionIds: readonly string[]; sourceRevision: string | null }, ) { if (installed) return; await promisify(execFile)(process.execPath, [path.join(repositoryRoot, "scripts/ensure-plugin-build-deps.mjs")], { @@ -14,4 +16,5 @@ export async function prepareSourceServerDependencies( timeout: 120_000, maxBuffer: 1024 * 1024, }); + if (evidence) await retainPreparedSourceSdk({ repositoryRoot, ...evidence, installed }); }