diff --git a/tests/runner-e2e/chat-feature-flow.ts b/tests/runner-e2e/chat-feature-flow.ts index 1de906a1f32..3a3aa674526 100644 --- a/tests/runner-e2e/chat-feature-flow.ts +++ b/tests/runner-e2e/chat-feature-flow.ts @@ -1,3 +1,4 @@ +import { readChatReadinessReceipts } from "./chat-readiness-observer.js"; import { legacyChatFeatureFingerprint } from "./chat-catalog-fingerprint.js"; import { runChatCatalogFlow } from "./chat-catalog-flow.js"; import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; @@ -79,7 +80,8 @@ export async function runChatFeatureFlow(context: { async function saveEvidence() { await input.evidence("chat-feature.json", { schema: CHAT_FEATURE_GRADER, scenarioId: scenario!.id, capabilities: scenario!.capabilities, observation, grade: gradeChatFeature(observation), - setupObservations, setupUsageCoverage: "Production agent hello probes are outside heartbeat usage; their detailed tokens and cost are unavailable." }); + setupObservations, setupProbeReceipts: await readChatReadinessReceipts(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "", f.company.id), + setupUsageCoverage: "Production agent hello probes are outside heartbeat usage; their detailed tokens and cost are unavailable." }); } try { expect(await api.get(`${company}/issues?includeAgentCreatedTasks=true`)).toEqual([]); @@ -87,7 +89,8 @@ export async function runChatFeatureFlow(context: { await pollUntil({ label: `chat feature agent readiness ${agentId}`, deadlineAt: Date.now() + 45_000, intervalMs: 500, load: async () => { const agent = await api.get<{ lifecycleState: string; status: string; lifecycleError?: string; lifecycleHolds?: string[] }>(`/api/agents/${agentId}`); - setupObservations.push({ agentId, lifecycleState: agent.lifecycleState, status: agent.status }); + setupObservations.push({ agentId, lifecycleState: agent.lifecycleState, status: agent.status, + lifecycleErrorPresent: !!agent.lifecycleError, lifecycleHeld: !!agent.lifecycleHolds?.length }); return agent; }, accept: agent => agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status), diff --git a/tests/runner-e2e/chat-readiness-observer.test.ts b/tests/runner-e2e/chat-readiness-observer.test.ts new file mode 100644 index 00000000000..81164e5404f --- /dev/null +++ b/tests/runner-e2e/chat-readiness-observer.test.ts @@ -0,0 +1,83 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { ServerAdapterModule } from "../../server/src/adapters/types.js"; +import { installChatReadinessObserver, readChatReadinessReceipts, readinessErrorCategory } from "./chat-readiness-observer.js"; +const roots: string[] = []; +const companyId = "11111111-1111-4111-8111-111111111111"; +const ids = ["chat-features.runner-opencode.local.create-backlog"]; +const context = { companyId, adapterType: "paperclip_runner", config: { provider: "opencode", env: { PRIVATE_KEY: "private-config-canary" } } }; +const result = () => ({ adapterType: "paperclip_runner", status: "fail" as const, testedAt: "2026-10-27T14:00:00Z", + checks: [{ code: "opencode_hello_probe_failed", level: "error" as const, message: "OpenCode hello probe failed.", + detail: "HTTP 429 private-provider-canary", hint: "private-hint-canary" }] }); +async function fixture() { + const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "readiness-observer-"))); roots.push(temporaryRoot); + const testEnvironment = vi.fn(async () => result()); + const adapter = { testEnvironment } as Pick; + await installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter }); + return { temporaryRoot, adapter, testEnvironment, receipts: () => readChatReadinessReceipts(temporaryRoot, companyId) }; +} +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +describe("transparent same-call readiness retention", () => { + it("calls exactly once with the original args/this and returns the identical result", async () => { + const f = await fixture(), original = result(); f.testEnvironment.mockResolvedValue(original); + expect(await f.adapter.testEnvironment(context)).toBe(original); + expect(f.testEnvironment).toHaveBeenCalledExactlyOnceWith(context); + expect(f.testEnvironment.mock.contexts[0]).toBe(f.adapter); + const receipts = await f.receipts(); + expect(receipts).toMatchObject([{ companyId, sequence: 1, status: "fail", thrown: false, + checks: [{ code: "opencode_hello_probe_failed", level: "error", category: "rate-limit" }] }]); + const bytes = await readFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), "utf8"); + expect(bytes + JSON.stringify(receipts)).not.toMatch(/private-|detail|message|hint|env|config/); + expect(receipts[0]).not.toHaveProperty("agentId"); + }); + it("accepts the existing launcher directory beneath its private root", async () => { + const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "readiness-observer-"))); roots.push(temporaryRoot); + await mkdir(path.join(temporaryRoot, "artifacts-private"), { mode: 0o755 }); + const adapter: Pick = { testEnvironment: vi.fn(async () => result()) }; + await installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter }); + await adapter.testEnvironment(context); + expect(await readChatReadinessReceipts(temporaryRoot, companyId)).toHaveLength(1); + await chmod(temporaryRoot, 0o755); + await expect(installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter })).rejects.toThrow("owned private attempt root"); + }); + it("rethrows the identical error without another call or raw error output", async () => { + const f = await fixture(), error = new Error("Cannot find package private-package-canary"); + f.testEnvironment.mockRejectedValue(error); + await expect(f.adapter.testEnvironment(context)).rejects.toBe(error); + expect(f.testEnvironment).toHaveBeenCalledTimes(1); + expect(await f.receipts()).toMatchObject([{ status: "unknown", thrown: true, errorCategory: "module-missing" }]); + expect(JSON.stringify(await f.receipts())).not.toContain("private-package-canary"); + }); + it("never replaces the original result when diagnostic storage fails", async () => { + const f = await fixture(), original = result(); f.testEnvironment.mockResolvedValue(original); + await writeFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), "", { mode: 0o644 }); + expect(await f.adapter.testEnvironment(context)).toBe(original); + expect(f.testEnvironment).toHaveBeenCalledTimes(1); + }); + it("keeps sequence only within the attempt and filters foreign companies", async () => { + const f = await fixture(); await f.adapter.testEnvironment(context); await f.adapter.testEnvironment(context); + expect((await f.receipts()).map(row => row.sequence)).toEqual([1, 2]); + expect(await readChatReadinessReceipts(f.temporaryRoot, "22222222-2222-4222-8222-222222222222")).toEqual([]); + }); + it("reprojects arbitrary keys, status, codes and categories as unknown", async () => { + const f = await fixture(); + await writeFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), JSON.stringify({ schema: "paperclip.chat-readiness-probe/v1", companyId, sequence: 1, + status: "private-status", secret: "private-extra", checks: [{ code: "private-code", level: "private-level", category: "private-category", detail: "private-detail" }] })); + const receipts = await f.receipts(); + expect(receipts).toMatchObject([{ status: "unknown", checks: [{ code: "unknown", level: "unknown", category: "unknown" }] }]); + expect(JSON.stringify(receipts)).not.toContain("private-"); + }); + it("does not observe an unrelated provider or execution", async () => { + const f = await fixture(); await f.adapter.testEnvironment({ ...context, config: { provider: "codex" } }); + expect(await f.receipts()).toEqual([]); + const original = vi.fn(async () => result()), adapter = { testEnvironment: original }; + await installChatReadinessObserver({ executionIds: ["other.runner-opencode.local.case"], temporaryRoot: "", adapter }); + expect(adapter.testEnvironment).toBe(original); + }); + it.each([ + ["HTTP 401 private", "authentication"], ["ECONNREFUSED private", "network"], ["EPERM private", "permission"], + ["ENOENT private", "command-missing"], ["timed out private", "timeout"], ["unrecognized private", "unknown"], + ])("classifies only a closed category for %s", (value, category) => expect(readinessErrorCategory(value)).toBe(category)); +}); diff --git a/tests/runner-e2e/chat-readiness-observer.ts b/tests/runner-e2e/chat-readiness-observer.ts new file mode 100644 index 00000000000..ce7e44b34ce --- /dev/null +++ b/tests/runner-e2e/chat-readiness-observer.ts @@ -0,0 +1,93 @@ +import { constants } from "node:fs"; +import { mkdir, open, readFile, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import type { ServerAdapterModule } from "../../server/src/adapters/types.js"; + +// Exact source-defined codes from OpenCode's existing environment test. Messages, +// hints, command output, config and credentials never cross this boundary. +const codes = new Set([ + "opencode_environment_target", "opencode_cwd_valid", "opencode_cwd_invalid", + "opencode_openai_api_key_missing", "opencode_headless_permissions_enabled", + "opencode_command_skipped", "opencode_command_resolvable", "opencode_command_unresolvable", + "opencode_model_validation_via_probe", "opencode_model_invalid", "opencode_model_validation_skipped_remote", + "opencode_models_discovered", "opencode_models_empty", "opencode_models_discovery_failed", + "opencode_model_configured", "opencode_hello_probe_timed_out", "opencode_hello_probe_passed", + "opencode_hello_probe_unexpected_output", "opencode_hello_probe_model_unavailable", + "opencode_hello_probe_auth_required", "opencode_hello_probe_failed", + "paperclip_runner_provider_unsupported", +]); +const categories = ["authentication", "rate-limit", "network", "permission", "command-missing", "module-missing", "timeout", "unknown"] as const; +type Category = typeof categories[number]; +const uuid = (v: unknown): v is string => typeof v === "string" && /^[a-f0-9-]{36}$/i.test(v); +const selected = (ids: readonly string[]) => ids.length > 0 && ids.every(id => /^chat-features\.runner-opencode\.local\.[a-z0-9.-]+$/.test(id)); +const filename = (root: string) => path.join(root, "artifacts-private", "chat-readiness-probes.jsonl"); +export function readinessErrorCategory(value: unknown): Category { + const text = typeof value === "string" ? value.slice(0, 8192) : ""; + if (/\b(?:401|403|unauthorized|authentication|invalid.api.key)\b/i.test(text)) return "authentication"; + if (/\b429\b|rate.?limit/i.test(text)) return "rate-limit"; + if (/EACCES|EPERM|permission denied/i.test(text)) return "permission"; + if (/ERR_MODULE_NOT_FOUND|Cannot find (?:module|package)/i.test(text)) return "module-missing"; + if (/ENOENT|command not found/i.test(text)) return "command-missing"; + if (/ETIMEDOUT|timed? ?out/i.test(text)) return "timeout"; + if (/ECONNREFUSED|ENOTFOUND|ECONNRESET|fetch failed|TLS|certificate/i.test(text)) return "network"; + return "unknown"; +} +function closedCheck(value: any) { + return { code: codes.has(value?.code) ? value.code as string : "unknown", + level: ["info", "warn", "error"].includes(value?.level) ? value.level as string : "unknown", + category: categories.includes(value?.category) ? value.category as Category : "unknown" }; +} +export async function installChatReadinessObserver(input: { + executionIds: readonly string[]; temporaryRoot: string; adapter: Pick; +}) { + if (!selected(input.executionIds)) return; + const root = await realpath(input.temporaryRoot), info = await stat(root); + if (root !== input.temporaryRoot || !info.isDirectory() || info.uid !== process.getuid?.() || (info.mode & 0o077)) throw new Error("Readiness observer requires its owned private attempt root"); + const directory = path.dirname(filename(root)); await mkdir(directory, { recursive: true, mode: 0o700 }); + const directoryInfo = await stat(directory); + if (await realpath(directory) !== directory || directoryInfo.uid !== info.uid || (directoryInfo.mode & 0o022)) throw new Error("Readiness observer directory is not private and owned"); + const original = input.adapter.testEnvironment; + let sequence = 0, writes = Promise.resolve(); + const retain = async (companyId: string, result: any, thrown: boolean) => { + if (++sequence > 100) return; + const receipt = { schema: "paperclip.chat-readiness-probe/v1", companyId, sequence, + status: ["pass", "warn", "fail"].includes(result?.status) ? result.status : "unknown", thrown, + checks: Array.isArray(result?.checks) ? result.checks.slice(0, 64).map((check: any) => closedCheck({ ...check, + category: readinessErrorCategory(`${check?.message ?? ""}\n${check?.detail ?? ""}`) })) : [], + errorCategory: thrown ? readinessErrorCategory(result instanceof Error ? result.message : "") : "unknown" }; + writes = writes.then(async () => { + const file = await open(filename(root), constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); + try { + const fileInfo = await file.stat(); + if (!fileInfo.isFile() || fileInfo.uid !== info.uid || (fileInfo.mode & 0o077)) throw new Error("Unsafe readiness receipt file"); + await file.writeFile(`${JSON.stringify(receipt)}\n`); + } finally { await file.close(); } + }).catch(() => { /* Diagnostic failure must never replace the actual probe result/error. */ }); + await writes; + }; + input.adapter.testEnvironment = async function (...args) { + const context = args[0]; + if (context.adapterType !== "paperclip_runner" || context.config.provider !== "opencode" || !uuid(context.companyId)) return original.apply(this, args); + let result; + try { result = await original.apply(this, args); } + catch (error) { await retain(context.companyId, error, true); throw error; } + await retain(context.companyId, result, false); + return result; + }; +} +/** Re-project even the private file. No arbitrary file keys/strings enter evidence. */ +export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string) { + if (!temporaryRoot || !uuid(companyId)) return []; + let bytes: string; + try { bytes = await readFile(filename(temporaryRoot), "utf8"); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw new Error("Readiness receipt unavailable"); } + if (bytes.length > 1_000_000) throw new Error("Readiness receipt exceeds bound"); + return bytes.trim().split("\n").filter(Boolean).slice(0, 100).flatMap(line => { + let row; try { row = JSON.parse(line); } catch { return []; } + if (!row || row.schema !== "paperclip.chat-readiness-probe/v1" || row.companyId !== companyId || !Number.isInteger(row.sequence) || row.sequence < 1 || row.sequence > 100) return []; + return [{ schema: row.schema, companyId, sequence: row.sequence, + status: ["pass", "warn", "fail"].includes(row.status) ? row.status : "unknown", thrown: row.thrown === true, + checks: Array.isArray(row.checks) ? row.checks.slice(0, 64).map(closedCheck) : [], + errorCategory: categories.includes(row.errorCategory) ? row.errorCategory as Category : "unknown" }]; + }); +} diff --git a/tests/runner-e2e/server-entry.ts b/tests/runner-e2e/server-entry.ts index da4002e2b37..3e0a7ef3a3f 100644 --- a/tests/runner-e2e/server-entry.ts +++ b/tests/runner-e2e/server-entry.ts @@ -1,3 +1,4 @@ +import { installChatReadinessObserver } from "./chat-readiness-observer.js"; // This entrypoint is used only by isolated Runner E2E instances. Production // service code has no test flag, delay, altered prompt, or private test API. import { ServerResponse } from "node:http"; @@ -8,6 +9,11 @@ import { installOwnedEmailResponseLoss } from "./chat-catalog-domains/runtime-em import { installOwnedEmailReceiptDuplicate } from "./chat-catalog-domains/runtime-email-duplicate.js"; const ids: string[] = JSON.parse(process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]"); +if (ids.length && ids.every(id => id.startsWith("chat-features.runner-opencode.local."))) { + const { requireServerAdapter } = await import("../../server/src/adapters/index.js"); + await installChatReadinessObserver({ executionIds: ids, temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "", + adapter: requireServerAdapter("paperclip_runner") }); +} await installOwnedEmailResponseLoss({ executionIds: ids, attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ?? "", temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "" }); await installOwnedEmailReceiptDuplicate({ executionIds: ids, attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ?? "", temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "" }); const contextCommentGate = contextCommentGateSelected(ids);