test(evals): retain closed same-call readiness diagnostics

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-feature-flow.ts

@@ -1,3 +1,4 @@
+import { readChatReadinessReceipts } from "./chat-readiness-observer.js";
 import { legacyChatFeatureFingerprint } from "./chat-catalog-fingerprint.js";
 import { runChatCatalogFlow } from "./chat-catalog-flow.js";
 import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
@@ -79,7 +80,8 @@ export async function runChatFeatureFlow(context: {
   async function saveEvidence() {
     await input.evidence("chat-feature.json", { schema: CHAT_FEATURE_GRADER, scenarioId: scenario!.id,
       capabilities: scenario!.capabilities, observation, grade: gradeChatFeature(observation),
-      setupObservations, setupUsageCoverage: "Production agent hello probes are outside heartbeat usage; their detailed tokens and cost are unavailable." });
+      setupObservations, setupProbeReceipts: await readChatReadinessReceipts(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "", f.company.id),
+      setupUsageCoverage: "Production agent hello probes are outside heartbeat usage; their detailed tokens and cost are unavailable." });
   }
   try {
     expect(await api.get(`${company}/issues?includeAgentCreatedTasks=true`)).toEqual([]);
@@ -87,7 +89,8 @@ export async function runChatFeatureFlow(context: {
       await pollUntil({ label: `chat feature agent readiness ${agentId}`, deadlineAt: Date.now() + 45_000, intervalMs: 500,
         load: async () => {
           const agent = await api.get<{ lifecycleState: string; status: string; lifecycleError?: string; lifecycleHolds?: string[] }>(`/api/agents/${agentId}`);
-          setupObservations.push({ agentId, lifecycleState: agent.lifecycleState, status: agent.status });
+          setupObservations.push({ agentId, lifecycleState: agent.lifecycleState, status: agent.status,
+            lifecycleErrorPresent: !!agent.lifecycleError, lifecycleHeld: !!agent.lifecycleHolds?.length });
           return agent;
         },
         accept: agent => agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status),

tests/runner-e2e/chat-readiness-observer.test.ts

@@ -0,0 +1,83 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import type { ServerAdapterModule } from "../../server/src/adapters/types.js";
+import { installChatReadinessObserver, readChatReadinessReceipts, readinessErrorCategory } from "./chat-readiness-observer.js";
+const roots: string[] = [];
+const companyId = "11111111-1111-4111-8111-111111111111";
+const ids = ["chat-features.runner-opencode.local.create-backlog"];
+const context = { companyId, adapterType: "paperclip_runner", config: { provider: "opencode", env: { PRIVATE_KEY: "private-config-canary" } } };
+const result = () => ({ adapterType: "paperclip_runner", status: "fail" as const, testedAt: "2026-10-27T14:00:00Z",
+  checks: [{ code: "opencode_hello_probe_failed", level: "error" as const, message: "OpenCode hello probe failed.",
+    detail: "HTTP 429 private-provider-canary", hint: "private-hint-canary" }] });
+async function fixture() {
+  const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "readiness-observer-"))); roots.push(temporaryRoot);
+  const testEnvironment = vi.fn(async () => result());
+  const adapter = { testEnvironment } as Pick<ServerAdapterModule, "testEnvironment">;
+  await installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter });
+  return { temporaryRoot, adapter, testEnvironment, receipts: () => readChatReadinessReceipts(temporaryRoot, companyId) };
+}
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+describe("transparent same-call readiness retention", () => {
+  it("calls exactly once with the original args/this and returns the identical result", async () => {
+    const f = await fixture(), original = result(); f.testEnvironment.mockResolvedValue(original);
+    expect(await f.adapter.testEnvironment(context)).toBe(original);
+    expect(f.testEnvironment).toHaveBeenCalledExactlyOnceWith(context);
+    expect(f.testEnvironment.mock.contexts[0]).toBe(f.adapter);
+    const receipts = await f.receipts();
+    expect(receipts).toMatchObject([{ companyId, sequence: 1, status: "fail", thrown: false,
+      checks: [{ code: "opencode_hello_probe_failed", level: "error", category: "rate-limit" }] }]);
+    const bytes = await readFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), "utf8");
+    expect(bytes + JSON.stringify(receipts)).not.toMatch(/private-|detail|message|hint|env|config/);
+    expect(receipts[0]).not.toHaveProperty("agentId");
+  });
+  it("accepts the existing launcher directory beneath its private root", async () => {
+    const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "readiness-observer-"))); roots.push(temporaryRoot);
+    await mkdir(path.join(temporaryRoot, "artifacts-private"), { mode: 0o755 });
+    const adapter: Pick<ServerAdapterModule, "testEnvironment"> = { testEnvironment: vi.fn(async () => result()) };
+    await installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter });
+    await adapter.testEnvironment(context);
+    expect(await readChatReadinessReceipts(temporaryRoot, companyId)).toHaveLength(1);
+    await chmod(temporaryRoot, 0o755);
+    await expect(installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter })).rejects.toThrow("owned private attempt root");
+  });
+  it("rethrows the identical error without another call or raw error output", async () => {
+    const f = await fixture(), error = new Error("Cannot find package private-package-canary");
+    f.testEnvironment.mockRejectedValue(error);
+    await expect(f.adapter.testEnvironment(context)).rejects.toBe(error);
+    expect(f.testEnvironment).toHaveBeenCalledTimes(1);
+    expect(await f.receipts()).toMatchObject([{ status: "unknown", thrown: true, errorCategory: "module-missing" }]);
+    expect(JSON.stringify(await f.receipts())).not.toContain("private-package-canary");
+  });
+  it("never replaces the original result when diagnostic storage fails", async () => {
+    const f = await fixture(), original = result(); f.testEnvironment.mockResolvedValue(original);
+    await writeFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), "", { mode: 0o644 });
+    expect(await f.adapter.testEnvironment(context)).toBe(original);
+    expect(f.testEnvironment).toHaveBeenCalledTimes(1);
+  });
+  it("keeps sequence only within the attempt and filters foreign companies", async () => {
+    const f = await fixture(); await f.adapter.testEnvironment(context); await f.adapter.testEnvironment(context);
+    expect((await f.receipts()).map(row => row.sequence)).toEqual([1, 2]);
+    expect(await readChatReadinessReceipts(f.temporaryRoot, "22222222-2222-4222-8222-222222222222")).toEqual([]);
+  });
+  it("reprojects arbitrary keys, status, codes and categories as unknown", async () => {
+    const f = await fixture();
+    await writeFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), JSON.stringify({ schema: "paperclip.chat-readiness-probe/v1", companyId, sequence: 1,
+      status: "private-status", secret: "private-extra", checks: [{ code: "private-code", level: "private-level", category: "private-category", detail: "private-detail" }] }));
+    const receipts = await f.receipts();
+    expect(receipts).toMatchObject([{ status: "unknown", checks: [{ code: "unknown", level: "unknown", category: "unknown" }] }]);
+    expect(JSON.stringify(receipts)).not.toContain("private-");
+  });
+  it("does not observe an unrelated provider or execution", async () => {
+    const f = await fixture(); await f.adapter.testEnvironment({ ...context, config: { provider: "codex" } });
+    expect(await f.receipts()).toEqual([]);
+    const original = vi.fn(async () => result()), adapter = { testEnvironment: original };
+    await installChatReadinessObserver({ executionIds: ["other.runner-opencode.local.case"], temporaryRoot: "", adapter });
+    expect(adapter.testEnvironment).toBe(original);
+  });
+  it.each([
+    ["HTTP 401 private", "authentication"], ["ECONNREFUSED private", "network"], ["EPERM private", "permission"],
+    ["ENOENT private", "command-missing"], ["timed out private", "timeout"], ["unrecognized private", "unknown"],
+  ])("classifies only a closed category for %s", (value, category) => expect(readinessErrorCategory(value)).toBe(category));
+});

tests/runner-e2e/chat-readiness-observer.ts

@@ -0,0 +1,93 @@
+import { constants } from "node:fs";
+import { mkdir, open, readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import type { ServerAdapterModule } from "../../server/src/adapters/types.js";
+
+// Exact source-defined codes from OpenCode's existing environment test. Messages,
+// hints, command output, config and credentials never cross this boundary.
+const codes = new Set([
+  "opencode_environment_target", "opencode_cwd_valid", "opencode_cwd_invalid",
+  "opencode_openai_api_key_missing", "opencode_headless_permissions_enabled",
+  "opencode_command_skipped", "opencode_command_resolvable", "opencode_command_unresolvable",
+  "opencode_model_validation_via_probe", "opencode_model_invalid", "opencode_model_validation_skipped_remote",
+  "opencode_models_discovered", "opencode_models_empty", "opencode_models_discovery_failed",
+  "opencode_model_configured", "opencode_hello_probe_timed_out", "opencode_hello_probe_passed",
+  "opencode_hello_probe_unexpected_output", "opencode_hello_probe_model_unavailable",
+  "opencode_hello_probe_auth_required", "opencode_hello_probe_failed",
+  "paperclip_runner_provider_unsupported",
+]);
+const categories = ["authentication", "rate-limit", "network", "permission", "command-missing", "module-missing", "timeout", "unknown"] as const;
+type Category = typeof categories[number];
+const uuid = (v: unknown): v is string => typeof v === "string" && /^[a-f0-9-]{36}$/i.test(v);
+const selected = (ids: readonly string[]) => ids.length > 0 && ids.every(id => /^chat-features\.runner-opencode\.local\.[a-z0-9.-]+$/.test(id));
+const filename = (root: string) => path.join(root, "artifacts-private", "chat-readiness-probes.jsonl");
+export function readinessErrorCategory(value: unknown): Category {
+  const text = typeof value === "string" ? value.slice(0, 8192) : "";
+  if (/\b(?:401|403|unauthorized|authentication|invalid.api.key)\b/i.test(text)) return "authentication";
+  if (/\b429\b|rate.?limit/i.test(text)) return "rate-limit";
+  if (/EACCES|EPERM|permission denied/i.test(text)) return "permission";
+  if (/ERR_MODULE_NOT_FOUND|Cannot find (?:module|package)/i.test(text)) return "module-missing";
+  if (/ENOENT|command not found/i.test(text)) return "command-missing";
+  if (/ETIMEDOUT|timed? ?out/i.test(text)) return "timeout";
+  if (/ECONNREFUSED|ENOTFOUND|ECONNRESET|fetch failed|TLS|certificate/i.test(text)) return "network";
+  return "unknown";
+}
+function closedCheck(value: any) {
+  return { code: codes.has(value?.code) ? value.code as string : "unknown",
+    level: ["info", "warn", "error"].includes(value?.level) ? value.level as string : "unknown",
+    category: categories.includes(value?.category) ? value.category as Category : "unknown" };
+}
+export async function installChatReadinessObserver(input: {
+  executionIds: readonly string[]; temporaryRoot: string; adapter: Pick<ServerAdapterModule, "testEnvironment">;
+}) {
+  if (!selected(input.executionIds)) return;
+  const root = await realpath(input.temporaryRoot), info = await stat(root);
+  if (root !== input.temporaryRoot || !info.isDirectory() || info.uid !== process.getuid?.() || (info.mode & 0o077)) throw new Error("Readiness observer requires its owned private attempt root");
+  const directory = path.dirname(filename(root)); await mkdir(directory, { recursive: true, mode: 0o700 });
+  const directoryInfo = await stat(directory);
+  if (await realpath(directory) !== directory || directoryInfo.uid !== info.uid || (directoryInfo.mode & 0o022)) throw new Error("Readiness observer directory is not private and owned");
+  const original = input.adapter.testEnvironment;
+  let sequence = 0, writes = Promise.resolve();
+  const retain = async (companyId: string, result: any, thrown: boolean) => {
+    if (++sequence > 100) return;
+    const receipt = { schema: "paperclip.chat-readiness-probe/v1", companyId, sequence,
+      status: ["pass", "warn", "fail"].includes(result?.status) ? result.status : "unknown", thrown,
+      checks: Array.isArray(result?.checks) ? result.checks.slice(0, 64).map((check: any) => closedCheck({ ...check,
+        category: readinessErrorCategory(`${check?.message ?? ""}\n${check?.detail ?? ""}`) })) : [],
+      errorCategory: thrown ? readinessErrorCategory(result instanceof Error ? result.message : "") : "unknown" };
+    writes = writes.then(async () => {
+      const file = await open(filename(root), constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
+      try {
+        const fileInfo = await file.stat();
+        if (!fileInfo.isFile() || fileInfo.uid !== info.uid || (fileInfo.mode & 0o077)) throw new Error("Unsafe readiness receipt file");
+        await file.writeFile(`${JSON.stringify(receipt)}\n`);
+      } finally { await file.close(); }
+    }).catch(() => { /* Diagnostic failure must never replace the actual probe result/error. */ });
+    await writes;
+  };
+  input.adapter.testEnvironment = async function (...args) {
+    const context = args[0];
+    if (context.adapterType !== "paperclip_runner" || context.config.provider !== "opencode" || !uuid(context.companyId)) return original.apply(this, args);
+    let result;
+    try { result = await original.apply(this, args); }
+    catch (error) { await retain(context.companyId, error, true); throw error; }
+    await retain(context.companyId, result, false);
+    return result;
+  };
+}
+/** Re-project even the private file. No arbitrary file keys/strings enter evidence. */
+export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string) {
+  if (!temporaryRoot || !uuid(companyId)) return [];
+  let bytes: string;
+  try { bytes = await readFile(filename(temporaryRoot), "utf8"); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw new Error("Readiness receipt unavailable"); }
+  if (bytes.length > 1_000_000) throw new Error("Readiness receipt exceeds bound");
+  return bytes.trim().split("\n").filter(Boolean).slice(0, 100).flatMap(line => {
+    let row; try { row = JSON.parse(line); } catch { return []; }
+    if (!row || row.schema !== "paperclip.chat-readiness-probe/v1" || row.companyId !== companyId || !Number.isInteger(row.sequence) || row.sequence < 1 || row.sequence > 100) return [];
+    return [{ schema: row.schema, companyId, sequence: row.sequence,
+      status: ["pass", "warn", "fail"].includes(row.status) ? row.status : "unknown", thrown: row.thrown === true,
+      checks: Array.isArray(row.checks) ? row.checks.slice(0, 64).map(closedCheck) : [],
+      errorCategory: categories.includes(row.errorCategory) ? row.errorCategory as Category : "unknown" }];
+  });
+}

tests/runner-e2e/server-entry.ts

@@ -1,3 +1,4 @@
+import { installChatReadinessObserver } from "./chat-readiness-observer.js";
 // This entrypoint is used only by isolated Runner E2E instances. Production
 // service code has no test flag, delay, altered prompt, or private test API.
 import { ServerResponse } from "node:http";
@@ -8,6 +9,11 @@ import { installOwnedEmailResponseLoss } from "./chat-catalog-domains/runtime-em
 import { installOwnedEmailReceiptDuplicate } from "./chat-catalog-domains/runtime-email-duplicate.js";
 
 const ids: string[] = JSON.parse(process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]");
+if (ids.length && ids.every(id => id.startsWith("chat-features.runner-opencode.local."))) {
+  const { requireServerAdapter } = await import("../../server/src/adapters/index.js");
+  await installChatReadinessObserver({ executionIds: ids, temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "",
+    adapter: requireServerAdapter("paperclip_runner") });
+}
 await installOwnedEmailResponseLoss({ executionIds: ids, attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ?? "", temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "" });
 await installOwnedEmailReceiptDuplicate({ executionIds: ids, attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ?? "", temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "" });
 const contextCommentGate = contextCommentGateSelected(ids);