test(e2e): retain sanitized setup health failure detail

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-readiness-detail.test.ts

@@ -0,0 +1,46 @@
+import { afterEach, expect, it } from "vitest";
+import { mkdir, mkdtemp, readFile, realpath, rm, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { diagnosticSecrets, stagePrivateDiagnostics } from "./private-diagnostics.js";
+import { copyHealthHelloDetail, HEALTH_DETAIL_ID, HEALTH_DETAIL_PATH, retainHealthHelloDetail } from "./chat-readiness-detail.js";
+import { isHistoricalBundlePathAllowed } from "./history-publish.js";
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+async function fixture(secrets: string[] = []) {
+  const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "health-detail-"))); roots.push(root);
+  await mkdir(path.join(root, "artifacts-private"), { mode: 0o700 });
+  await mkdir(path.join(root, "selected-private-inputs"), { mode: 0o700 });
+  stagePrivateDiagnostics(root, [HEALTH_DETAIL_ID], secrets);
+  return { root, uploadDir: path.join(root, "results") };
+}
+const result = (detail: string) => ({ checks: [{ code: "opencode_hello_probe_failed", level: "error", message: "OpenCode hello probe failed.", detail }] });
+it("retains useful unknown returned detail only in nonpublic Actions diagnostics", async () => {
+  const f = await fixture();
+  expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result("Provider rejected request: unsupported structured response mode"))).toBe("retained");
+  await copyHealthHelloDetail(f.root, HEALTH_DETAIL_ID, f.uploadDir, []);
+  const file = path.join(f.uploadDir, HEALTH_DETAIL_PATH);
+  expect(await readFile(file, "utf8")).toContain("unsupported structured response mode");
+  expect((await stat(file)).mode & 0o777).toBe(0o600);
+  expect(isHistoricalBundlePathAllowed(`campaign/chat-setup-health/runner-opencode/local/setup-only/attempt-1/${HEALTH_DETAIL_PATH}`, true)).toBe(false);
+});
+it("scrubs registered encodings plus credential headers, URLs and email before any write", async () => {
+  const secret = 'private-key-"with-sensitive-chars';
+  const f = await fixture([secret]);
+  const variants = diagnosticSecrets([secret]).values;
+  const detail = [...variants, "https://person:password@provider.invalid/path?token=another", "person@example.invalid", "Authorization: Basic unregistered-header-value"].join("\n");
+  expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result(detail))).toBe("retained");
+  const original = await readFile(path.join(f.root, "artifacts-private/setup-health-hello.jsonl"), "utf8");
+  for (const value of variants) expect(original).not.toContain(value);
+  expect(original).not.toMatch(/person@example|provider\.invalid|unregistered-header-value/);
+  await copyHealthHelloDetail(f.root, HEALTH_DETAIL_ID, f.uploadDir, [secret]);
+  expect(await readFile(path.join(f.uploadDir, HEALTH_DETAIL_PATH), "utf8")).toBe(original);
+});
+it("withholds missing registry and oversized details, and never observes ordinary catalog cases", async () => {
+  const f = await fixture();
+  expect(await retainHealthHelloDetail(f.root, ["chat-features.runner-opencode.local.create-backlog"], 1, result("not retained"))).toBeUndefined();
+  expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result("x".repeat(2049)))).toBe("withheld");
+  await rm(path.join(f.root, "selected-private-inputs/diagnostic-redaction.json"));
+  expect(await retainHealthHelloDetail(f.root, [HEALTH_DETAIL_ID], 1, result("unregistered detail"))).toBe("withheld");
+  await expect(readFile(path.join(f.root, "artifacts-private/setup-health-hello.jsonl"))).rejects.toThrow();
+});

tests/runner-e2e/chat-readiness-detail.ts

@@ -0,0 +1,49 @@
+import { constants } from "node:fs";
+import { chmod, mkdir, open, readFile } from "node:fs/promises";
+import path from "node:path";
+import { diagnosticSecrets, readPrivateDiagnosticSecrets } from "./private-diagnostics.js";
+import { findSecretLeak, redactText } from "./redaction.js";
+
+export const HEALTH_DETAIL_ID = "chat-setup-health.runner-opencode.local.setup-only";
+export const HEALTH_DETAIL_PATH = "diagnostics/setup-health-hello.jsonl";
+const privateFile = (root: string) => path.join(root, "artifacts-private", "setup-health-hello.jsonl");
+
+function scrub(value: unknown, secrets: readonly string[]) {
+  if (typeof value !== "string" || value.length > 2048) throw new Error("Bounded diagnostic unavailable");
+  const safe = redactText(value, secrets)
+    .replace(/\bhttps?:\/\/[^\s<>"']+/gi, "[URL REDACTED]")
+    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[EMAIL REDACTED]")
+    .replace(/\b(?:authorization|proxy-authorization|cookie|set-cookie|x-api-key)\s*:[^\r\n]*/gi, "[HEADER REDACTED]");
+  if (findSecretLeak(safe, secrets)) throw new Error("Diagnostic withheld");
+  return safe;
+}
+/** Returned hello check only, never raw process output, config or caught errors. */
+export async function retainHealthHelloDetail(root: string, ids: readonly string[], sequence: number, result: unknown): Promise<"retained" | "withheld" | undefined> {
+  if (ids.length !== 1 || ids[0] !== HEALTH_DETAIL_ID) return undefined;
+  try {
+    const secrets = diagnosticSecrets(readPrivateDiagnosticSecrets(path.join(root, "selected-private-inputs", "diagnostic-redaction.json"), root)).values;
+    const checks = (result as { checks?: unknown[] })?.checks;
+    const failures = checks?.filter((check: any) => check?.code === "opencode_hello_probe_failed" && check.level === "error") ?? [];
+    if (!failures.length) return undefined;
+    const rows = failures.slice(0, 4).map((check: any) => ({ sequence, code: "opencode_hello_probe_failed", message: scrub(check.message, secrets), detail: scrub(check.detail ?? "", secrets) }));
+    const file = await open(privateFile(root), constants.O_WRONLY | constants.O_CREAT | constants.O_APPEND | constants.O_NOFOLLOW, 0o600);
+    try {
+      const info = await file.stat();
+      if (!info.isFile() || info.uid !== process.getuid?.() || info.mode & 0o077) throw new Error("Diagnostic custody invalid");
+      await file.writeFile(rows.map(row => JSON.stringify(row)).join("\n") + "\n");
+    } finally { await file.close(); }
+    return "retained";
+  } catch { return "withheld"; }
+}
+/** Separate access-controlled Actions artifact entry; never public evidence. */
+export async function copyHealthHelloDetail(root: string, executionId: string, uploadDir: string, knownSecrets: readonly string[]) {
+  if (executionId !== HEALTH_DETAIL_ID) return;
+  try {
+    const bytes = await readFile(privateFile(root), "utf8");
+    if (bytes.length > 32768 || findSecretLeak(bytes, diagnosticSecrets(knownSecrets).values)) return;
+    const destination = path.join(uploadDir, HEALTH_DETAIL_PATH);
+    await mkdir(path.dirname(destination), { recursive: true, mode: 0o700 });
+    const file = await open(destination, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
+    try { await file.writeFile(bytes); await chmod(destination, 0o600); } finally { await file.close(); }
+  } catch { /* No raw detail or filesystem error enters reporters. */ }
+}

tests/runner-e2e/chat-readiness-observer.ts

@@ -1,3 +1,4 @@
+import { retainHealthHelloDetail } from "./chat-readiness-detail.js";
 import { constants } from "node:fs";
 import { mkdir, open, readFile, realpath, stat } from "node:fs/promises";
 import path from "node:path";
@@ -51,7 +52,8 @@ export async function installChatReadinessObserver(input: {
   let sequence = 0, writes = Promise.resolve();
   const retain = async (companyId: string, result: any, thrown: boolean) => {
     if (++sequence > 100) return;
-    const receipt = { schema: "paperclip.chat-readiness-probe/v1", companyId, sequence,
+    const detailRetention = thrown ? undefined : await retainHealthHelloDetail(root, input.executionIds, sequence, result);
+    const receipt = { ...(detailRetention ? { detailRetention } : {}), schema: "paperclip.chat-readiness-probe/v1", companyId, sequence,
       status: ["pass", "warn", "fail"].includes(result?.status) ? result.status : "unknown", thrown,
       checks: Array.isArray(result?.checks) ? result.checks.slice(0, 64).map((check: any) => closedCheck({ ...check,
         category: readinessErrorCategory(`${check?.message ?? ""}\n${check?.detail ?? ""}`) })) : [],
@@ -79,7 +81,7 @@ export async function installChatReadinessObserver(input: {
 /** Re-project even the private file. No arbitrary file keys/strings enter evidence. */
 export interface ClosedReadinessReceipt {
   schema: "paperclip.chat-readiness-probe/v1"; companyId: string; sequence: number; status: string; thrown: boolean;
-  checks: Array<{ code: string; level: string; category: Category }>; errorCategory: Category;
+  checks: Array<{ code: string; level: string; category: Category }>; errorCategory: Category; detailRetention?: "retained" | "withheld";
 }
 export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string): Promise<ClosedReadinessReceipt[]> {
   if (!temporaryRoot || !uuid(companyId)) return [];
@@ -90,7 +92,7 @@ export async function readChatReadinessReceipts(temporaryRoot: string, companyId
   return bytes.trim().split("\n").filter(Boolean).slice(0, 100).flatMap(line => {
     let row; try { row = JSON.parse(line); } catch { return []; }
     if (!row || row.schema !== "paperclip.chat-readiness-probe/v1" || row.companyId !== companyId || !Number.isInteger(row.sequence) || row.sequence < 1 || row.sequence > 100) return [];
-    return [{ schema: row.schema, companyId, sequence: row.sequence,
+    return [{ ...(row.detailRetention === "retained" || row.detailRetention === "withheld" ? { detailRetention: row.detailRetention } : {}), schema: row.schema, companyId, sequence: row.sequence,
       status: ["pass", "warn", "fail"].includes(row.status) ? row.status : "unknown", thrown: row.thrown === true,
       checks: Array.isArray(row.checks) ? row.checks.slice(0, 64).map(closedCheck) : [],
       errorCategory: categories.includes(row.errorCategory) ? row.errorCategory as Category : "unknown" }];

tests/runner-e2e/launch.ts

@@ -1,3 +1,4 @@
+import { copyHealthHelloDetail } from "./chat-readiness-detail.js";
 import { prepareChatRemoteHost } from "./chat-remote-host.js";
 import { POSTGRES_ARTIFACT_ENV, POSTGRES_IMAGE_ID_ENV } from "./docker-postgres.js";
 import { prepareOwnedPublicOrigin, PublicOriginCleanupError, PUBLIC_ORIGIN_CONTROL_ENV, PUBLIC_ORIGIN_RECEIPT_ENV } from "./chat-public-origin.js";
@@ -911,6 +912,7 @@ async function runAttempt(input: {
           expectPassScreenshot: false,
         });
       }
+      await copyHealthHelloDetail(temporaryRoot, candidate.id, uploadDir, credentials);
       console.log(
         `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`,
       );