diff --git a/tests/runner-e2e/generated-report-carriers.test.ts b/tests/runner-e2e/generated-report-carriers.test.ts new file mode 100644 index 00000000000..080bb08cee2 --- /dev/null +++ b/tests/runner-e2e/generated-report-carriers.test.ts @@ -0,0 +1,50 @@ +import { afterEach, expect, it } from "vitest"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { assertGeneratedReportCarriers } from "./manual-private-transfer.js"; +import { sealManualResultEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { canonicalGeneratedReportAsset } from "./manual-private-report-assets.js"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +import { privateRefusalCode } from "./private-refusal-code.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function report(files: Record) { + const root = await mkdtemp(path.join(os.tmpdir(), "generated-report-carriers-")); roots.push(root); + const source = path.join(root, "runner-e2e-merged-report"); + for (const [name, data] of Object.entries({ "html/index.html": "", "normalized/summary.md": "summary", ...files })) { + await mkdir(path.dirname(path.join(source, name)), { recursive: true }); + await writeFile(path.join(source, name), data); + } + return { root, source }; +} +const cell = "normalized/evidence/chat-features.runner-opencode.local.case/attempt-1/html-report/trace"; + +it("accepts standard carriers and authenticated trace viewer assets", async () => { + const logo = await readFile(canonicalGeneratedReportAsset("html/trace/playwright-logo.svg")!); + const { source } = await report({ "html/trace/playwright-logo.svg": logo, "html/data/0123456789abcdef0123456789abcdef01234567.zip": "zip" }); + await expect(assertGeneratedReportCarriers(source)).resolves.toBeUndefined(); +}); + +it.each([ + [`${cell}/codicon.DCmgc-ay.ttf`, "font"], + [`${cell}/playwright-logo.svg`, "svg"], + [`${cell}/manifest.webmanifest`, "webmanifest"], + ["html/data/0123456789abcdef0123456789abcdef01234567.jpeg", "image"], + ["normalized/report.js.map", "source-map"], + ["normalized/LICENSE", "no-extension"], + ["normalized/archive.tar", "other"], + ["html/trace/index.js.map", "trace-viewer-asset"], + ["html/trace/playwright-logo.svg", "trace-viewer-bytes"], +])("names a refused %s carrier by its closed category, never its path", async (name, category) => { + const { root, source } = await report({ [name]: "synthetic carrier" }); + const refusal = await assertGeneratedReportCarriers(source).then(() => undefined, (error: unknown) => error); + const code = privateRefusalCode(refusal); + expect(code).toBe(`unsupported-generated-report-carrier-${category}`); + expect(code).not.toContain("chat-features"); + // The authoritative seal refuses the same tree, so the preflight never hides an admissible report. + const context = { root, private: true, protocol: MANUAL_PRIVATE_PROTOCOL, workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", + attempt: "1", executionIds: [MANUAL_PRIVATE_EXECUTIONS[0]!] } as ManualPrivateHandoff; + await expect(sealManualResultEvidence(context, [], source, "generated-report")).rejects.toThrow("Unsupported packaged evidence carrier"); +}); diff --git a/tests/runner-e2e/manual-private-transfer.ts b/tests/runner-e2e/manual-private-transfer.ts index 81f77bb9ca4..e6b5e3f6967 100644 --- a/tests/runner-e2e/manual-private-transfer.ts +++ b/tests/runner-e2e/manual-private-transfer.ts @@ -206,6 +206,39 @@ async function verifyDownloads(env: NodeJS.ProcessEnv) { } await writeFile(path.join(root, "manual-report-input-completeness.json"), JSON.stringify({ meaning: "available-files-and-transfer-only", doesNotAttestFeatureOutcome: true, cells }), { mode: 0o600, flag: "wx" }); } +/** + * Names the first carrier the generated-report seal would refuse with a closed + * category, so a refused report is diagnosable without its paths. The seal stays + * authoritative; this only runs before it and never admits anything. + */ +export async function assertGeneratedReportCarriers(source: string) { + let visited = 0; + async function visit(relative: string): Promise { + for (const entry of await readdir(path.join(source, relative), { withFileTypes: true })) { + if (++visited > 4096) return; + const name = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.isDirectory() && !entry.isSymbolicLink()) { if (name.split("/").length < 24) await visit(name); continue; } + if (!entry.isFile() || entry.isSymbolicLink()) continue; + const canonical = canonicalGeneratedReportAsset(name); + if (name.startsWith("html/trace/") || canonical !== undefined) { + if (!canonical) throw new Error("Unsupported generated report carrier (trace viewer asset)"); + if (!await isTrustedGeneratedReportAsset(name, await readFile(path.join(source, name)))) + throw new Error("Unsupported generated report carrier (trace viewer bytes)"); + continue; + } + if (isStandardEvidenceFile(`html-report/${name}`)) continue; + const extension = path.extname(name).toLowerCase(); + if ([".ttf", ".otf", ".woff", ".woff2"].includes(extension)) throw new Error("Unsupported generated report carrier (font)"); + if (extension === ".svg") throw new Error("Unsupported generated report carrier (svg)"); + if (extension === ".webmanifest") throw new Error("Unsupported generated report carrier (webmanifest)"); + if ([".jpg", ".jpeg", ".gif", ".webp", ".avif", ".bmp", ".ico"].includes(extension)) throw new Error("Unsupported generated report carrier (image)"); + if (extension === ".map") throw new Error("Unsupported generated report carrier (source map)"); + if (!extension) throw new Error("Unsupported generated report carrier (no extension)"); + throw new Error("Unsupported generated report carrier (other)"); + } + } + await visit(""); +} export async function uploadReport(env: NodeJS.ProcessEnv) { requireSession(env); const { approval, bucket } = request(env), source = await realpath(env.PRIVATE_REPORT_SOURCE ?? ""), root = await realpath(env.PRIVATE_REPORT_ROOT ?? ""); if (!source.startsWith(`${root}${path.sep}`)) throw new Error("Private report escaped owned staging"); @@ -213,6 +246,7 @@ export async function uploadReport(env: NodeJS.ProcessEnv) { if (evidenceCompleteness.meaning !== "available-files-and-transfer-only" || evidenceCompleteness.doesNotAttestFeatureOutcome !== true || evidenceCompleteness.cells.map(cell => cell.executionId).sort().join() !== [...approval.executionIds].sort().join()) throw new Error("Private report lacks exact cell completeness receipts"); const secrets = normalizedSecrets([...CREDENTIAL_NAMES.map(name => env[name]), env.AWS_ACCESS_KEY_ID, env.AWS_SECRET_ACCESS_KEY, env.AWS_SESSION_TOKEN]); + await assertGeneratedReportCarriers(source); await sealManualResultEvidence({ root, protocol: MANUAL_PRIVATE_PROTOCOL, private: true, workflowSha: approval.workflowSha, targetSha: approval.targetSha, runId: approval.runId, attempt: approval.attempt, executionIds: [...approval.executionIds] }, secrets, source, "generated-report"); const sealed = path.join(root, "sealed-results"); diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index c65610db9d3..8a492b5156a 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -697,6 +697,33 @@ describe("runner E2E report aggregation", () => { ).toBe("warm-turn-png"); }); + it("does not stage a private cell's trace viewer font, SVG or manifest into the merged report", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-report-carriers-")); + cleanupDirectories.push(root); + const executionId = "chat-features.runner-opencode.local.runs-adapters.inspect-failed-run"; + const directory = path.join(root, "attempt-1"); + const staged = ["final-state.png", "html-report/index.html", "html-report/trace/index.html", "html-report/trace/index.KZ4wOW1K.js", "html-report/data/0123456789abcdef0123456789abcdef01234567.zip"]; + const refused = ["html-report/trace/codicon.DCmgc-ay.ttf", "html-report/trace/manifest.webmanifest", "html-report/trace/playwright-logo.svg"]; + for (const file of [...staged, ...refused]) { + await mkdir(path.join(directory, path.dirname(file)), { recursive: true }); + await writeFile(path.join(directory, file), `synthetic ${file}`); + } + await writeFile(path.join(directory, "result.json"), JSON.stringify({ + schema: "paperclip.runner-e2e.result/v1", executionId, attempt: 1, status: "passed", + profileId: "runner-opencode", environmentId: "local", caseId: "runs-adapters.inspect-failed-run", provider: "opencode", model: "fixture-model", + runtimeMode: "native", startedAt: "2026-11-07T00:00:00.000Z", finishedAt: "2026-11-07T00:00:01.000Z", durationMs: 1_000, cleanup: "passed", + } satisfies RunnerE2EResult)); + await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: [...staged, ...refused], leaks: [], missing: [] })); + const output = path.join(root, "merged"); + await execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, + env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]) }, + }).catch(error => { throw new Error(String(error.stderr).slice(-1500)); }); + const evidence = path.join(output, "evidence", executionId, "attempt-1"); + for (const file of staged) expect(await readFile(path.join(evidence, file), "utf8")).toBe(`synthetic ${file}`); + for (const file of refused) await expect(readFile(path.join(evidence, file))).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("constructs the public root JUnit from fixed markup and escaped fields", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-report-junit-test-"), diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 7167e7316c1..7a71042e642 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -24,6 +24,7 @@ import { upgradeRunnerResult, } from "./history.js"; import { resolveRunnerE2ESource } from "./source.js"; +import { isStandardEvidenceFile } from "./evidence.js"; import { runnerE2ESummaryLinks } from "./summary-links.js"; import type { RunnerE2EResult } from "./types.js"; import { gradeRetainedDependableResults, type DependableQualification } from "./dependable-qualification.js"; @@ -120,6 +121,12 @@ async function stageDashboardEvidence( for (const relative of entry.evidence?.files ?? []) { const segments = safeEvidenceRelative(relative); if (!segments) continue; + // A private cell also lists its HTML report's trace viewer program files + // (font, SVG, web manifest). They are not evidence: the merged report has + // its own authenticated viewer, and the private report seal refuses these + // carrier types anywhere else, losing the whole report. Apply that same + // carrier rule here; the cell archive still keeps the files. + if (!isStandardEvidenceFile(path.join("html-report", ...segments))) continue; const source = path.join(entry.directory, ...segments); const destination = path.join(output, ...baseSegments, ...segments); const didCopy = await mkdir(path.dirname(destination), {