From 8eb98eb52728f07609644c9d1d0c4b7d57ed911a Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 12:26:09 -0500 Subject: [PATCH] test(e2e): retain sanitized setup health failure detail Co-Authored-By: Paperclip --- .../runner-e2e/chat-readiness-detail.test.ts | 46 +++++++++++++++++ tests/runner-e2e/chat-readiness-detail.ts | 49 +++++++++++++++++++ tests/runner-e2e/chat-readiness-observer.ts | 8 +-- tests/runner-e2e/launch.ts | 2 + 4 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 tests/runner-e2e/chat-readiness-detail.test.ts create mode 100644 tests/runner-e2e/chat-readiness-detail.ts diff --git a/tests/runner-e2e/chat-readiness-detail.test.ts b/tests/runner-e2e/chat-readiness-detail.test.ts new file mode 100644 index 00000000000..92e6535a19d --- /dev/null +++ b/tests/runner-e2e/chat-readiness-detail.test.ts @@ -0,0 +1,46 @@ +import { afterEach, expect, it } from "vitest"; +import { mkdir, mkdtemp, readFile, realpath, rm, stat } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { diagnosticSecrets, stagePrivateDiagnostics } from "./private-diagnostics.js"; +import { copyHealthHelloDetail, HEALTH_DETAIL_ID, HEALTH_DETAIL_PATH, retainHealthHelloDetail } from "./chat-readiness-detail.js"; +import { isHistoricalBundlePathAllowed } from "./history-publish.js"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture(secrets: string[] = []) { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "health-detail-"))); roots.push(root); + await mkdir(path.join(root, "artifacts-private"), { mode: 0o700 }); + await mkdir(path.join(root, "selected-private-inputs"), { mode: 0o700 }); + stagePrivateDiagnostics(root, [HEALTH_DETAIL_ID], secrets); + return { root, uploadDir: path.join(root, "results") }; +} +const result = (detail: string) => ({ checks: [{ code: "opencode_hello_probe_failed", level: "error", message: "OpenCode hello probe failed.", detail }] }); +it("retains useful unknown returned detail only in nonpublic Actions diagnostics", async () => { + const f = await fixture(); + expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result("Provider rejected request: unsupported structured response mode"))).toBe("retained"); + await copyHealthHelloDetail(f.root, HEALTH_DETAIL_ID, f.uploadDir, []); + const file = path.join(f.uploadDir, HEALTH_DETAIL_PATH); + expect(await readFile(file, "utf8")).toContain("unsupported structured response mode"); + expect((await stat(file)).mode & 0o777).toBe(0o600); + expect(isHistoricalBundlePathAllowed(`campaign/chat-setup-health/runner-opencode/local/setup-only/attempt-1/${HEALTH_DETAIL_PATH}`, true)).toBe(false); +}); +it("scrubs registered encodings plus credential headers, URLs and email before any write", async () => { + const secret = 'private-key-"with-sensitive-chars'; + const f = await fixture([secret]); + const variants = diagnosticSecrets([secret]).values; + const detail = [...variants, "https://person:password@provider.invalid/path?token=another", "person@example.invalid", "Authorization: Basic unregistered-header-value"].join("\n"); + expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result(detail))).toBe("retained"); + const original = await readFile(path.join(f.root, "artifacts-private/setup-health-hello.jsonl"), "utf8"); + for (const value of variants) expect(original).not.toContain(value); + expect(original).not.toMatch(/person@example|provider\.invalid|unregistered-header-value/); + await copyHealthHelloDetail(f.root, HEALTH_DETAIL_ID, f.uploadDir, [secret]); + expect(await readFile(path.join(f.uploadDir, HEALTH_DETAIL_PATH), "utf8")).toBe(original); +}); +it("withholds missing registry and oversized details, and never observes ordinary catalog cases", async () => { + const f = await fixture(); + expect(await retainHealthHelloDetail(f.root, ["chat-features.runner-opencode.local.create-backlog"], 1, result("not retained"))).toBeUndefined(); + expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result("x".repeat(2049)))).toBe("withheld"); + await rm(path.join(f.root, "selected-private-inputs/diagnostic-redaction.json")); + expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result("unregistered detail"))).toBe("withheld"); + await expect(readFile(path.join(f.root, "artifacts-private/setup-health-hello.jsonl"))).rejects.toThrow(); +}); diff --git a/tests/runner-e2e/chat-readiness-detail.ts b/tests/runner-e2e/chat-readiness-detail.ts new file mode 100644 index 00000000000..066ef9830b3 --- /dev/null +++ b/tests/runner-e2e/chat-readiness-detail.ts @@ -0,0 +1,49 @@ +import { constants } from "node:fs"; +import { chmod, mkdir, open, readFile } from "node:fs/promises"; +import path from "node:path"; +import { diagnosticSecrets, readPrivateDiagnosticSecrets } from "./private-diagnostics.js"; +import { findSecretLeak, redactText } from "./redaction.js"; + +export const HEALTH_DETAIL_ID = "chat-setup-health.runner-opencode.local.setup-only"; +export const HEALTH_DETAIL_PATH = "diagnostics/setup-health-hello.jsonl"; +const privateFile = (root: string) => path.join(root, "artifacts-private", "setup-health-hello.jsonl"); + +function scrub(value: unknown, secrets: readonly string[]) { + if (typeof value !== "string" || value.length > 2048) throw new Error("Bounded diagnostic unavailable"); + const safe = redactText(value, secrets) + .replace(/\bhttps?:\/\/[^\s<>"']+/gi, "[URL REDACTED]") + .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[EMAIL REDACTED]") + .replace(/\b(?:authorization|proxy-authorization|cookie|set-cookie|x-api-key)\s*:[^\r\n]*/gi, "[HEADER REDACTED]"); + if (findSecretLeak(safe, secrets)) throw new Error("Diagnostic withheld"); + return safe; +} +/** Returned hello check only, never raw process output, config or caught errors. */ +export async function retainHealthHelloDetail(root: string, ids: readonly string[], sequence: number, result: unknown): Promise<"retained" | "withheld" | undefined> { + if (ids.length !== 1 || ids[0] !== HEALTH_DETAIL_ID) return undefined; + try { + const secrets = diagnosticSecrets(readPrivateDiagnosticSecrets(path.join(root, "selected-private-inputs", "diagnostic-redaction.json"), root)).values; + const checks = (result as { checks?: unknown[] })?.checks; + const failures = checks?.filter((check: any) => check?.code === "opencode_hello_probe_failed" && check.level === "error") ?? []; + if (!failures.length) return undefined; + const rows = failures.slice(0, 4).map((check: any) => ({ sequence, code: "opencode_hello_probe_failed", message: scrub(check.message, secrets), detail: scrub(check.detail ?? "", secrets) })); + const file = await open(privateFile(root), constants.O_WRONLY | constants.O_CREAT | constants.O_APPEND | constants.O_NOFOLLOW, 0o600); + try { + const info = await file.stat(); + if (!info.isFile() || info.uid !== process.getuid?.() || info.mode & 0o077) throw new Error("Diagnostic custody invalid"); + await file.writeFile(rows.map(row => JSON.stringify(row)).join("\n") + "\n"); + } finally { await file.close(); } + return "retained"; + } catch { return "withheld"; } +} +/** Separate access-controlled Actions artifact entry; never public evidence. */ +export async function copyHealthHelloDetail(root: string, executionId: string, uploadDir: string, knownSecrets: readonly string[]) { + if (executionId !== HEALTH_DETAIL_ID) return; + try { + const bytes = await readFile(privateFile(root), "utf8"); + if (bytes.length > 32768 || findSecretLeak(bytes, diagnosticSecrets(knownSecrets).values)) return; + const destination = path.join(uploadDir, HEALTH_DETAIL_PATH); + await mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); + const file = await open(destination, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { await file.writeFile(bytes); await chmod(destination, 0o600); } finally { await file.close(); } + } catch { /* No raw detail or filesystem error enters reporters. */ } +} diff --git a/tests/runner-e2e/chat-readiness-observer.ts b/tests/runner-e2e/chat-readiness-observer.ts index a44250171b6..9a1de0f6109 100644 --- a/tests/runner-e2e/chat-readiness-observer.ts +++ b/tests/runner-e2e/chat-readiness-observer.ts @@ -1,3 +1,4 @@ +import { retainHealthHelloDetail } from "./chat-readiness-detail.js"; import { constants } from "node:fs"; import { mkdir, open, readFile, realpath, stat } from "node:fs/promises"; import path from "node:path"; @@ -51,7 +52,8 @@ export async function installChatReadinessObserver(input: { let sequence = 0, writes = Promise.resolve(); const retain = async (companyId: string, result: any, thrown: boolean) => { if (++sequence > 100) return; - const receipt = { schema: "paperclip.chat-readiness-probe/v1", companyId, sequence, + const detailRetention = thrown ? undefined : await retainHealthHelloDetail(root, input.executionIds, sequence, result); + const receipt = { ...(detailRetention ? { detailRetention } : {}), schema: "paperclip.chat-readiness-probe/v1", companyId, sequence, status: ["pass", "warn", "fail"].includes(result?.status) ? result.status : "unknown", thrown, checks: Array.isArray(result?.checks) ? result.checks.slice(0, 64).map((check: any) => closedCheck({ ...check, category: readinessErrorCategory(`${check?.message ?? ""}\n${check?.detail ?? ""}`) })) : [], @@ -79,7 +81,7 @@ export async function installChatReadinessObserver(input: { /** Re-project even the private file. No arbitrary file keys/strings enter evidence. */ export interface ClosedReadinessReceipt { schema: "paperclip.chat-readiness-probe/v1"; companyId: string; sequence: number; status: string; thrown: boolean; - checks: Array<{ code: string; level: string; category: Category }>; errorCategory: Category; + checks: Array<{ code: string; level: string; category: Category }>; errorCategory: Category; detailRetention?: "retained" | "withheld"; } export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string): Promise { if (!temporaryRoot || !uuid(companyId)) return []; @@ -90,7 +92,7 @@ export async function readChatReadinessReceipts(temporaryRoot: string, companyId return bytes.trim().split("\n").filter(Boolean).slice(0, 100).flatMap(line => { let row; try { row = JSON.parse(line); } catch { return []; } if (!row || row.schema !== "paperclip.chat-readiness-probe/v1" || row.companyId !== companyId || !Number.isInteger(row.sequence) || row.sequence < 1 || row.sequence > 100) return []; - return [{ schema: row.schema, companyId, sequence: row.sequence, + return [{ ...(row.detailRetention === "retained" || row.detailRetention === "withheld" ? { detailRetention: row.detailRetention } : {}), schema: row.schema, companyId, sequence: row.sequence, status: ["pass", "warn", "fail"].includes(row.status) ? row.status : "unknown", thrown: row.thrown === true, checks: Array.isArray(row.checks) ? row.checks.slice(0, 64).map(closedCheck) : [], errorCategory: categories.includes(row.errorCategory) ? row.errorCategory as Category : "unknown" }]; diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index fe97c927747..de94ee37131 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,3 +1,4 @@ +import { copyHealthHelloDetail } from "./chat-readiness-detail.js"; import { prepareChatRemoteHost } from "./chat-remote-host.js"; import { POSTGRES_ARTIFACT_ENV, POSTGRES_IMAGE_ID_ENV } from "./docker-postgres.js"; import { prepareOwnedPublicOrigin, PublicOriginCleanupError, PUBLIC_ORIGIN_CONTROL_ENV, PUBLIC_ORIGIN_RECEIPT_ENV } from "./chat-public-origin.js"; @@ -911,6 +912,7 @@ async function runAttempt(input: { expectPassScreenshot: false, }); } + await copyHealthHelloDetail(temporaryRoot, candidate.id, uploadDir, credentials); console.log( `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`, );