Retain certified source SDK evidence for plugin fixtures

tests/runner-e2e/README.md

@@ -80,8 +80,16 @@ one attempt, two expected provider admissions, and 1,000-cent company and agent
 hard stops. Automatic infrastructure retries are disabled. Existing campaign
 reports retain failures, timings, usage, runtime costs, and source identity.
 Source-server startup prepares the existing shared/plugin SDK build dependencies
-before importing native tool authority. Installed-package checks never execute
-checkout build helpers.
+before importing native tool authority. The six source-local plugin-fixture
+journeys retain the actual certified shared/SDK outputs before server listen,
+fingerprinting, agent setup and human dispatch. Exact bytes, completion markers,
+source/lock/helper hashes remain in Actions-only `diagnostics/source-plugin-sdk`.
+The original hidden completion-marker bytes use the explicit non-hidden archival
+name `build-completion-receipt.json`, with original/retained paths and hashes
+recorded so bounded upload staging preserves them. The public report does not
+publish those files. Startup failure does not trigger
+a post-failure rebuild or manufacture missing output evidence. Installed-package
+checks never execute checkout build helpers.
 
 ## Configured repository and conversion checks
 

tests/runner-e2e/launch.ts

@@ -1,4 +1,5 @@
 import { copyHealthHelloDetail } from "./chat-readiness-detail.js";
+import { copyRetainedSourceSdk } from "./source-sdk-evidence.js";
 import { prepareChatRemoteHost } from "./chat-remote-host.js";
 import { POSTGRES_ARTIFACT_ENV, POSTGRES_IMAGE_ID_ENV } from "./docker-postgres.js";
 import { prepareOwnedPublicOrigin, PublicOriginCleanupError, PUBLIC_ORIGIN_CONTROL_ENV, PUBLIC_ORIGIN_RECEIPT_ENV } from "./chat-public-origin.js";
@@ -916,6 +917,7 @@ async function runAttempt(input: {
         });
       }
       await copyHealthHelloDetail(temporaryRoot, candidate.id, uploadDir, credentials);
+      await copyRetainedSourceSdk(temporaryRoot, candidate.id, result.source?.sha ?? null, uploadDir, credentials);
       console.log(
         `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`,
       );
@@ -948,6 +950,7 @@ async function runAttempt(input: {
         schemaVersion: 1, executionId: candidate.id, originalResult: original,
         packaging: "failed; raw diagnostics retained privately", error: error instanceof Error ? error.message : String(error),
       }, attemptSecrets), null, 2)}\n`, { mode: 0o600 });
+      await copyRetainedSourceSdk(temporaryRoot, candidate.id, result.source?.sha ?? null, uploadDir, attemptSecrets).catch(() => {});
       if (rawCleanupResults.length < executions.length) rawCleanupResults.push({ cleanup: original.cleanup, synthetic: original === fallback, status: original.status });
       publishedResults.push(result); publishedResultPaths.set(candidate.id, resultPath);
     }

tests/runner-e2e/server.ts

@@ -17,6 +17,7 @@ import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server-
 import { runnerE2ETypeScriptProcessArgs } from "./web-server-command.js";
 import { qualifyLegacyClaudeCli } from "./legacy-claude-cli.js";
 import { prepareSourceServerDependencies } from "./source-startup.js";
+import { resolveRunnerE2ESource } from "./source.js";
 import { spawn, type ChildProcess } from "node:child_process";
 import { createWriteStream } from "node:fs";
 import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
@@ -122,7 +123,9 @@ const definedServerEnvironment = Object.fromEntries(
   ),
 );
 
-await prepareSourceServerDependencies(repositoryRoot, definedServerEnvironment, usesInstalledCli(process.env));
+await prepareSourceServerDependencies(repositoryRoot, definedServerEnvironment, usesInstalledCli(process.env), {
+  temporaryRoot, executionIds, sourceRevision: resolveRunnerE2ESource().sha,
+});
 
 await Promise.all([
   mkdir(path.dirname(logPath), { recursive: true }),

tests/runner-e2e/source-sdk-evidence.test.ts

@@ -0,0 +1,97 @@
+import { createHash } from "node:crypto";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { afterEach, expect, it } from "vitest";
+import { stagePrivateDiagnostics } from "./private-diagnostics.js";
+import { prepareSourceServerDependencies } from "./source-startup.js";
+import { copyRetainedSourceSdk, retainPreparedSourceSdk, SOURCE_SDK_DIAGNOSTICS } from "./source-sdk-evidence.js";
+import { packageEvidence } from "./evidence.js";
+import { isHistoricalBundlePathAllowed } from "./history-publish.js";
+
+const roots: string[] = [], executionId = "chat-features.runner-opencode.local.plugins.install-plugin", sourceRevision = "a".repeat(40);
+const hash = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex");
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+async function fixture(secrets: string[] = []) {
+  const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "source-sdk-synthetic-unit-"))); roots.push(root);
+  const repositoryRoot = path.join(root, "repo"), temporaryRoot = path.join(root, "attempt"), uploadDir = path.join(root, "uploaded");
+  const put = async (file: string, value: string | Buffer) => { await mkdir(path.dirname(path.join(repositoryRoot, file)), { recursive: true }); await writeFile(path.join(repositoryRoot, file), value); };
+  await put("scripts/ensure-plugin-build-deps.mjs", await readFile(new URL("../../scripts/ensure-plugin-build-deps.mjs", import.meta.url)));
+  await put("pnpm-lock.yaml", "synthetic unit lock\n");
+  for (const directory of ["packages/shared", "packages/plugins/sdk"]) {
+    await put(`${directory}/src/index.ts`, "export const syntheticFixture = true;\n"); await put(`${directory}/tsconfig.json`, "{}");
+  }
+  await put("node_modules/typescript/bin/tsc", `const fs=require('node:fs'),path=require('node:path');if(process.env.FAIL_COMPILER==='1')process.exit(7);const dir=path.join(path.dirname(process.argv[3]),'dist');fs.mkdirSync(path.join(dir,'nested'),{recursive:true});fs.writeFileSync(path.join(dir,'index.js'),process.env.SYNTHETIC_OUTPUT_CONTENT||'export const syntheticFixture = true;\\n');fs.writeFileSync(path.join(dir,'nested/child.js'),'export {};\\n');`);
+  await mkdir(path.join(temporaryRoot, "artifacts-private"), { recursive: true, mode: 0o700 });
+  await mkdir(path.join(temporaryRoot, "selected-private-inputs"), { recursive: true, mode: 0o700 });
+  stagePrivateDiagnostics(temporaryRoot, [executionId], secrets);
+  const input = { repositoryRoot, temporaryRoot, executionIds: [executionId], sourceRevision, installed: false };
+  const prepare = (environment: NodeJS.ProcessEnv = {}) => prepareSourceServerDependencies(repositoryRoot, environment, false, { temporaryRoot, executionIds: [executionId], sourceRevision });
+  return { root, repositoryRoot, temporaryRoot, uploadDir, input, prepare };
+}
+
+it.each([true, false])("retains exact synthetic compiler output before admission and survives normal packaging/cleanup (pass=%s)", async pass => {
+  const f = await fixture();
+  await expect(readFile(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"))).rejects.toThrow();
+  await f.prepare();
+  const receipt = JSON.parse(await readFile(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build/receipt.json"), "utf8"));
+  expect(receipt).toMatchObject({ schema: "paperclip.source-plugin-sdk/v1", executionId, sourceRevision }); expect(receipt.certifiedTargets).toHaveLength(2);
+  expect(receipt.buildHelperSha256).toBe(hash(await readFile(path.join(f.repositoryRoot, "scripts/ensure-plugin-build-deps.mjs"))));
+  const packaged = await packageEvidence({ privateDir: path.join(f.temporaryRoot, "case"), uploadDir: f.uploadDir, secrets: [], expectPassScreenshot: pass });
+  expect(packaged.leaks).toEqual([]);
+  await copyRetainedSourceSdk(f.temporaryRoot, executionId, sourceRevision, f.uploadDir, []);
+  const bounded = path.join(f.root, "bounded");
+  await promisify(execFile)(process.execPath, [new URL("./artifact-budget.mjs", import.meta.url).pathname, "stage", "evidence", f.uploadDir, bounded]);
+  await rm(f.temporaryRoot, { recursive: true });
+  for (const file of receipt.files) {
+    const retained = path.join(f.uploadDir, SOURCE_SDK_DIAGNOSTICS, file.retainedPath);
+    expect(hash(await readFile(retained))).toBe(file.sha256); expect((await stat(retained)).mode & 0o777).toBe(0o600);
+    expect(file.retainedSha256).toBe(file.sha256);
+    expect(hash(await readFile(path.join(bounded, SOURCE_SDK_DIAGNOSTICS, file.retainedPath)))).toBe(file.sha256);
+    expect(isHistoricalBundlePathAllowed(`evidence/${executionId}/attempt-1/${SOURCE_SDK_DIAGNOSTICS}/${file.retainedPath}`, true)).toBe(false);
+  }
+  expect(isHistoricalBundlePathAllowed(`evidence/${executionId}/attempt-1/${SOURCE_SDK_DIAGNOSTICS}/receipt.json`, true)).toBe(false);
+});
+it("cannot retain output before existing helper certification or after compilation failure; never rebuilds during packaging", async () => {
+  const f = await fixture();
+  await expect(retainPreparedSourceSdk(f.input)).rejects.toThrow();
+  await expect(f.prepare({ FAIL_COMPILER: "1" })).rejects.toMatchObject({ code: 7 });
+  await expect(readFile(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build/receipt.json"))).rejects.toThrow();
+  await expect(copyRetainedSourceSdk(f.temporaryRoot, executionId, sourceRevision, f.uploadDir, [])).rejects.toThrow();
+  await expect(readFile(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"))).rejects.toThrow();
+});
+it("rejects output changed after certification instead of copying a stale completion marker", async () => {
+  const f = await fixture(); await f.prepare();
+  await rm(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"), { recursive: true });
+  await writeFile(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"), "changed after certification");
+  await expect(retainPreparedSourceSdk(f.input)).rejects.toThrow("no longer matches");
+  await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow();
+});
+it("requires the existing owned redaction registry before retaining compiled bytes", async () => {
+  const f = await fixture(); await rm(path.join(f.temporaryRoot, "selected-private-inputs/diagnostic-redaction.json"));
+  await expect(f.prepare()).rejects.toThrow();
+  await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow();
+});
+it.each(["raw", "base64"])("refuses registered %s credential bytes before any retention write", async encoding => {
+  const secret = "synthetic-private-upstream-credential-value", f = await fixture([secret]);
+  const value = encoding === "base64" ? Buffer.from(secret).toString("base64") : secret;
+  await expect(f.prepare({ SYNTHETIC_OUTPUT_CONTENT: value })).rejects.toThrow("privacy check failed");
+  await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow();
+});
+it.each(["wrong source", "wrong selected case", "tampered bytes", "new secret", "symlink"])("fails closed for %s in retained custody", async invalid => {
+  const f = await fixture(); await f.prepare();
+  const staged = path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"), file = path.join(staged, "packages/plugins/sdk/dist/index.js");
+  if (invalid === "tampered bytes") await writeFile(file, "changed compiler bytes");
+  if (invalid === "symlink") { await rm(file); await symlink(path.join(f.repositoryRoot, "packages/plugins/sdk/dist/index.js"), file); }
+  await expect(copyRetainedSourceSdk(f.temporaryRoot, invalid === "wrong selected case" ? "chat-features.runner-opencode.local.plugins.configure-plugin" : executionId,
+    invalid === "wrong source" ? "b".repeat(40) : sourceRevision, f.uploadDir, invalid === "new secret" ? ["syntheticFixture"] : [])).rejects.toThrow();
+});
+it("ordinary/multi-cell/installed lanes do not retain or require SDK evidence", async () => {
+  const f = await fixture();
+  for (const changed of [{ executionIds: ["chat-features.runner-opencode.local.tasks.website-task"] }, { executionIds: [executionId, executionId] }, { installed: true }])
+    await expect(retainPreparedSourceSdk({ ...f.input, ...changed })).resolves.toBeUndefined();
+  await expect(copyRetainedSourceSdk(f.temporaryRoot, "unselected", null, f.uploadDir, [])).resolves.toBeUndefined();
+  await expect(stat(path.join(f.temporaryRoot, "artifacts-private/source-sdk-build"))).rejects.toThrow();
+});

tests/runner-e2e/source-sdk-evidence.ts

@@ -0,0 +1,99 @@
+import { createHash } from "node:crypto";
+import { chmod, lstat, mkdir, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { diagnosticSecrets, readPrivateDiagnosticSecrets } from "./private-diagnostics.js";
+import { findSecretLeak } from "./redaction.js";
+
+const selectedIds = new Set(["plugins.configure-plugin", "plugins.disable-plugin", "plugins.install-plugin", "plugins.use-plugin-tool", "routines.plugin-managed-routine", "secrets.least-scope-delivery"].map(id => `chat-features.runner-opencode.local.${id}`));
+export const SOURCE_SDK_DIAGNOSTICS = "diagnostics/source-plugin-sdk";
+const targets = ["packages/shared/dist", "packages/plugins/sdk/dist"] as const;
+const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
+const hex = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+const staging = (root: string) => path.join(root, "artifacts-private", "source-sdk-build");
+interface BuiltFile { file: string; size: number; sha256: string }
+const retainedPath = (file: string) => file.replace(/\/\.paperclip-build-complete$/, "/build-completion-receipt.json");
+interface RetainedBuild {
+  schema: "paperclip.source-plugin-sdk/v1"; executionId: string; sourceRevision: string; retainedAt: string;
+  buildHelperSha256: string; lockSha256: string; files: Array<BuiltFile & { retainedPath: string; retainedSha256: string }>;
+  certifiedTargets: Array<{ directory: string; sources: string; outputs: string }>;
+}
+function selected(ids: readonly string[]) { return ids.length === 1 && selectedIds.has(ids[0]!); }
+function checkSecret(bytes: Buffer, secrets: readonly string[]) {
+  if (findSecretLeak(bytes.toString("utf8"), secrets)) throw new Error("Source SDK evidence privacy check failed; bytes withheld");
+}
+async function boundedTree(root: string) {
+  const files: Array<BuiltFile & { bytes: Buffer }> = []; let total = 0;
+  async function visit(directory: string) {
+    const entries = await readdir(directory, { withFileTypes: true });
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+      const filename = path.join(directory, entry.name);
+      if (entry.isDirectory()) await visit(filename);
+      else {
+        const info = await lstat(filename);
+        if (!info.isFile() || info.isSymbolicLink() || info.size > 8 * 1024 * 1024) throw new Error("Source SDK output custody invalid");
+        const bytes = await readFile(filename); total += bytes.length;
+        if (files.length >= 4096 || total > 64 * 1024 * 1024) throw new Error("Source SDK output exceeds evidence bound");
+        files.push({ file: path.relative(root, filename).split(path.sep).join("/"), size: bytes.length, sha256: hash(bytes), bytes });
+      }
+    }
+  }
+  const info = await lstat(root); if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Source SDK output root invalid");
+  await visit(root); return files;
+}
+function outputHash(files: Array<{ file: string; bytes: Buffer }>) {
+  const digest = createHash("sha256");
+  for (const file of files.filter(row => row.file !== ".paperclip-build-complete")) {
+    digest.update(JSON.stringify([file.file, file.bytes.length])); digest.update(file.bytes);
+  }
+  return digest.digest("hex");
+}
+export async function retainPreparedSourceSdk(input: { repositoryRoot: string; temporaryRoot: string; executionIds: readonly string[]; sourceRevision: string | null; installed: boolean }) {
+  if (input.installed || !selected(input.executionIds)) return;
+  if (!/^[a-f0-9]{40}$/.test(input.sourceRevision ?? "")) throw new Error("Source SDK retention requires exact source revision");
+  const root = await realpath(input.temporaryRoot), repository = await realpath(input.repositoryRoot);
+  if (await realpath(path.join(root, "artifacts-private")) !== path.join(root, "artifacts-private")) throw new Error("Source SDK staging custody invalid");
+  const secrets = diagnosticSecrets(readPrivateDiagnosticSecrets(path.join(root, "selected-private-inputs", "diagnostic-redaction.json"), root)).values;
+  const buffers: Array<BuiltFile & { bytes: Buffer }> = [], certifiedTargets: RetainedBuild["certifiedTargets"] = [];
+  for (const directory of targets) {
+    const files = await boundedTree(path.join(repository, directory));
+    const marker = files.find(row => row.file === ".paperclip-build-complete"), entry = files.find(row => row.file === "index.js");
+    if (!marker || !entry) throw new Error("Source SDK build completion evidence missing");
+    const certificate = JSON.parse(marker.bytes.toString("utf8"));
+    if (!hex(certificate.sources) || !hex(certificate.outputs) || outputHash(files) !== certificate.outputs) throw new Error("Source SDK build output no longer matches its completed compiler receipt");
+    certifiedTargets.push({ directory, sources: certificate.sources, outputs: certificate.outputs });
+    for (const file of files) { checkSecret(file.bytes, secrets); buffers.push({ ...file, file: `${directory}/${file.file}` }); }
+  }
+  if (buffers.length >= 4096 || buffers.reduce((total, file) => total + file.size, 0) > 63 * 1024 * 1024) throw new Error("Source SDK output exceeds combined evidence bound");
+  const receipt: RetainedBuild = { schema: "paperclip.source-plugin-sdk/v1", executionId: input.executionIds[0]!, sourceRevision: input.sourceRevision!, retainedAt: new Date().toISOString(),
+    buildHelperSha256: hash(await readFile(path.join(repository, "scripts/ensure-plugin-build-deps.mjs"))), lockSha256: hash(await readFile(path.join(repository, "pnpm-lock.yaml"))),
+    certifiedTargets, files: buffers.map(({ bytes: _, ...row }) => ({ ...row, retainedPath: retainedPath(row.file), retainedSha256: row.sha256 })) };
+  if (new Set(receipt.files.map(file => file.retainedPath)).size !== receipt.files.length) throw new Error("Source SDK retained filename collision");
+  const receiptBytes = Buffer.from(JSON.stringify(receipt, null, 2) + "\n"); checkSecret(receiptBytes, secrets);
+  const destination = staging(root); await mkdir(destination, { mode: 0o700 });
+  try {
+    for (const file of buffers) {
+      const filename = path.join(destination, file.file); await mkdir(path.dirname(filename), { recursive: true, mode: 0o700 });
+      await writeFile(filename, file.bytes, { flag: "wx", mode: 0o600 });
+    }
+    await writeFile(path.join(destination, "receipt.json"), receiptBytes, { flag: "wx", mode: 0o600 });
+  } catch (error) { await rm(destination, { recursive: true, force: true }); throw error; }
+}
+export async function copyRetainedSourceSdk(root: string, executionId: string, sourceRevision: string | null, uploadDir: string, knownSecrets: readonly string[]) {
+  if (!selected([executionId])) return;
+  if (await realpath(staging(root)) !== staging(root)) throw new Error("Source SDK retained custody invalid");
+  const secrets = diagnosticSecrets(knownSecrets).values, files = await boundedTree(staging(root));
+  const receiptFile = files.find(row => row.file === "receipt.json"); if (!receiptFile) throw new Error("Source SDK retained receipt missing");
+  const receipt = JSON.parse(receiptFile.bytes.toString("utf8")) as RetainedBuild;
+  if (receipt.schema !== "paperclip.source-plugin-sdk/v1" || receipt.executionId !== executionId || receipt.sourceRevision !== sourceRevision
+    || !hex(receipt.buildHelperSha256) || !hex(receipt.lockSha256) || !Array.isArray(receipt.files)) throw new Error("Source SDK retained identity mismatch");
+  const actual = files.filter(row => row.file !== "receipt.json");
+  if (actual.length !== receipt.files.length || actual.some(row => !targets.some(target => row.file.startsWith(`${target}/`))
+    || !receipt.files.some(expected => expected.file === row.file && expected.retainedPath === retainedPath(row.file) && expected.size === row.size && expected.sha256 === row.sha256 && expected.retainedSha256 === row.sha256))) throw new Error("Source SDK retained bytes changed");
+  for (const file of files) checkSecret(file.bytes, secrets);
+  const destination = path.join(uploadDir, SOURCE_SDK_DIAGNOSTICS); await mkdir(destination, { recursive: true, mode: 0o700 });
+  for (const file of files) {
+    const filename = path.join(destination, retainedPath(file.file)); await mkdir(path.dirname(filename), { recursive: true, mode: 0o700 });
+    await writeFile(filename, file.bytes, { flag: "wx", mode: 0o600 }); await chmod(filename, 0o600);
+    if (hash(await readFile(filename)) !== file.sha256) throw new Error("Source SDK published bytes changed");
+  }
+}

tests/runner-e2e/source-startup.ts

@@ -1,11 +1,13 @@
 import { execFile } from "node:child_process";
 import path from "node:path";
 import { promisify } from "node:util";
+import { retainPreparedSourceSdk } from "./source-sdk-evidence.js";
 
 export async function prepareSourceServerDependencies(
   repositoryRoot: string,
   environment: NodeJS.ProcessEnv,
   installed: boolean,
+  evidence?: { temporaryRoot: string; executionIds: readonly string[]; sourceRevision: string | null },
 ) {
   if (installed) return;
   await promisify(execFile)(process.execPath, [path.join(repositoryRoot, "scripts/ensure-plugin-build-deps.mjs")], {
@@ -14,4 +16,5 @@ export async function prepareSourceServerDependencies(
     timeout: 120_000,
     maxBuffer: 1024 * 1024,
   });
+  if (evidence) await retainPreparedSourceSdk({ repositoryRoot, ...evidence, installed });
 }