From 139d83b7779ea0ec7edb49446221f50719d67d17 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 11:46:28 -0500 Subject: [PATCH] test(evals): add explicit zero-work chat setup health Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 17 +++ tests/runner-e2e/catalog.ts | 19 ++- .../chat-readiness-observer.test.ts | 25 ++++ tests/runner-e2e/chat-readiness-observer.ts | 13 ++- tests/runner-e2e/chat-remote-host.test.ts | 9 ++ tests/runner-e2e/chat-remote-host.ts | 5 +- tests/runner-e2e/chat-setup-health.test.ts | 102 ++++++++++++++++ tests/runner-e2e/chat-setup-health.ts | 109 ++++++++++++++++++ tests/runner-e2e/live-fixtures.ts | 3 +- tests/runner-e2e/runner.spec.ts | 13 ++- tests/runner-e2e/server-entry.ts | 3 +- tests/runner-e2e/server.ts | 5 + tests/runner-e2e/types.ts | 1 + 13 files changed, 310 insertions(+), 14 deletions(-) create mode 100644 tests/runner-e2e/chat-setup-health.test.ts create mode 100644 tests/runner-e2e/chat-setup-health.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 9aa0500b740..3edf7f3df5a 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -3021,3 +3021,20 @@ a preflight rejection does not establish that prerequisite. Provider/account qualification, paid-job approval, and owned AWS Spot capacity must be established before remote admission. Discovery and credential-free controls establish none of those live outcomes. + + +### Explicit chat provider setup health + +`chat-setup-health.runner-opencode.local.setup-only` is a diagnostic cell, outside +all 244 chat catalog scenarios and excluded from `--all`. It reuses the exact +chat OpenCode/OpenRouter/DeepSeek profile, ordinary trusted fixture setup, owned +Docker PostgreSQL and normal teardown. One agent's ordinary automatic hello is +expected; it makes no explicit environment-test call and sends no chat/task +request. No heartbeat runs, issues, conversations or comments may exist. + +Select this cell explicitly with retries zero through the existing Product E2E +workflow. A failed readiness check fails health and retains only closed check +codes/categories. Public pause fences automatic setup retries; failed pause +requests the existing owned supervisor's shutdown before further API observations. +Multiple probe receipts fail health. Setup-probe usage/cost is unavailable, never +zero. This diagnostic cannot qualify a catalog scenario or rewrite prior results. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 417fbf07848..0624b2fde1a 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,3 +1,4 @@ +import { CHAT_SETUP_HEALTH_SUITE, chatSetupHealthTask, chatSetupHealthDigest } from "./chat-setup-health.js"; import { CHAT_OPENCODE_MODEL, CHAT_OPENCODE_PROFILE } from "./chat-runtime-profile.js"; import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js"; import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js"; @@ -1157,6 +1158,12 @@ const repositoryModelProfiles = [ }, ]; +const chatFeatureExecutionProfile = chatFeatureProfile(nativeProfile({ + id: CHAT_OPENCODE_PROFILE, label: "Native OpenCode OpenRouter DeepSeek V4.1 Flash 1017", + provider: "opencode", model: CHAT_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"], + modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-catalog:opencode-1.18.34:deepseek-v4.1-flash-1017:remote-initial" }, +})); + export const runnerSuites: readonly RunnerSuiteFixture[] = [ { id: "daytona-restart-continuity", label: "Daytona controller outage continuity", manualOnly: true, @@ -1605,13 +1612,19 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment], tasks: firstTaskTasks, expectedMatrixSize: 52, definitionMetadata: { version: 8, runGrading: "evidenced-nonexecution-and-refusal", credentialPersistenceCheck: false, questionChoiceMinimum: 2, nativeSetup: "qualified-platform-wizard-saved-default", claudeDefaultTargets: ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"], legacySetup: "explicit-advanced-runner-choice", productionInstructions: true, qualityGrading: "informational" }, }, + { + id: CHAT_SETUP_HEALTH_SUITE, label: "Chat provider setup health", manualOnly: true, + description: "Diagnostic only: one normal lifecycle hello, zero chat/task work; excluded from the 244 catalog scenarios.", + groups: [], profiles: [chatFeatureExecutionProfile], environments: [localEnvironment], tasks: [chatSetupHealthTask], expectedMatrixSize: 1, + definitionMetadata: { version: 1, sourceDigest: chatSetupHealthDigest, scheduling: "explicit-only", catalogScenario: false, + maximumAttemptsPerCell: 1, maximumAutomaticSetupHellos: 1, expectedHeartbeatRuns: 0, canonicalQueries: 0, + setupProbeUsage: "unavailable-outside-heartbeat-accounting" }, + }, { id: CHAT_FEATURE_SUITE, label: "Chat feature journeys", manualOnly: true, description: "Executable catalog requests through Agent Chat, with independently graded persisted state.", groups: ["chat", "native"], - profiles: [chatFeatureProfile(nativeProfile({ id: CHAT_OPENCODE_PROFILE, label: "Native OpenCode OpenRouter DeepSeek V4.1 Flash 1017", - provider: "opencode", model: CHAT_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"], - modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-catalog:opencode-1.18.34:deepseek-v4.1-flash-1017:remote-initial" } }))], + profiles: [chatFeatureExecutionProfile], environments: [localEnvironment], tasks: [...chatFeatureTasks, ...chatCatalogTasks], expectedMatrixSize: 2 + chatCatalogTasks.length, definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, inventoryDigest: chatCatalogInventoryDigest, frozenScenarioCount: 244, scheduling: "explicit-only", instructions: "production-role-defaults", companyAndAgentBudgetCents: CHAT_FEATURE_BUDGET_CENTS, diff --git a/tests/runner-e2e/chat-readiness-observer.test.ts b/tests/runner-e2e/chat-readiness-observer.test.ts index 81164e5404f..cb8248fe178 100644 --- a/tests/runner-e2e/chat-readiness-observer.test.ts +++ b/tests/runner-e2e/chat-readiness-observer.test.ts @@ -76,6 +76,31 @@ describe("transparent same-call readiness retention", () => { await installChatReadinessObserver({ executionIds: ["other.runner-opencode.local.case"], temporaryRoot: "", adapter }); expect(adapter.testEnvironment).toBe(original); }); + it.each([ + ["agent_runner_config_incompatible", "unknown"], ["paperclip_runner_mode_invalid", "unknown"], ["paperclip_runner_opencode_model_invalid", "unknown"], + ])("preserves source profile-validation code %s without its message", async (code, category) => { + const f = await fixture(); f.testEnvironment.mockResolvedValue({ ...result(), checks: [{ code, level: "error", message: "private-profile-canary", detail: "private-canary", hint: "private-canary" }] }); + await f.adapter.testEnvironment(context); + expect((await f.receipts())[0]!.checks).toEqual([{ code, level: "error", category }]); + expect(JSON.stringify(await f.receipts())).not.toContain("private-"); + }); + // Actual check.message/detail layouts emitted by opencode-local/server/test.ts: + // summarizeProbeDetail selects parsed error, first stderr or first stdout line. + it.each([ + ["opencode_command_unresolvable", "Command is not executable: ENOENT", "opencode", "command-missing"], + ["opencode_openai_api_key_missing", "OPENAI_API_KEY override is empty.", "", "unknown"], + ["opencode_hello_probe_auth_required", "OpenCode is installed, but provider authentication is not ready.", "HTTP 401 Unauthorized private-canary", "authentication"], + ["opencode_hello_probe_model_unavailable", "The configured model was not found by the provider.", "ProviderModelNotFoundError private-canary", "model-unavailable"], + ["opencode_hello_probe_failed", "OpenCode hello probe failed.", '{"name":"APIError","data":{"statusCode":429,"message":"private-canary"}}', "rate-limit"], + ["opencode_hello_probe_failed", "OpenCode hello probe failed.", "fetch failed ECONNREFUSED private-canary", "network"], + ["opencode_hello_probe_timed_out", "OpenCode hello probe timed out.", "", "timeout"], + ["opencode_hello_probe_failed", "OpenCode hello probe failed.", "private-unknown-canary", "unknown"], + ])("retains closed actual-source diagnostic shape %s/%s", async (code, message, detail, category) => { + const f = await fixture(); f.testEnvironment.mockResolvedValue({ ...result(), checks: [{ code, level: "error", message, detail, hint: "private-hint-canary" }] }); + await f.adapter.testEnvironment(context); + expect((await f.receipts())[0]!.checks).toEqual([{ code, level: "error", category }]); + expect(JSON.stringify(await f.receipts())).not.toMatch(/private-|APIError|statusCode|detail|message|hint/); + }); it.each([ ["HTTP 401 private", "authentication"], ["ECONNREFUSED private", "network"], ["EPERM private", "permission"], ["ENOENT private", "command-missing"], ["timed out private", "timeout"], ["unrecognized private", "unknown"], diff --git a/tests/runner-e2e/chat-readiness-observer.ts b/tests/runner-e2e/chat-readiness-observer.ts index ce7e44b34ce..a44250171b6 100644 --- a/tests/runner-e2e/chat-readiness-observer.ts +++ b/tests/runner-e2e/chat-readiness-observer.ts @@ -14,15 +14,16 @@ const codes = new Set([ "opencode_model_configured", "opencode_hello_probe_timed_out", "opencode_hello_probe_passed", "opencode_hello_probe_unexpected_output", "opencode_hello_probe_model_unavailable", "opencode_hello_probe_auth_required", "opencode_hello_probe_failed", - "paperclip_runner_provider_unsupported", + "paperclip_runner_provider_unsupported", "agent_runner_config_incompatible", "paperclip_runner_mode_invalid", "paperclip_runner_opencode_model_invalid", ]); -const categories = ["authentication", "rate-limit", "network", "permission", "command-missing", "module-missing", "timeout", "unknown"] as const; +const categories = ["authentication", "rate-limit", "network", "permission", "command-missing", "module-missing", "model-unavailable", "timeout", "unknown"] as const; type Category = typeof categories[number]; const uuid = (v: unknown): v is string => typeof v === "string" && /^[a-f0-9-]{36}$/i.test(v); -const selected = (ids: readonly string[]) => ids.length > 0 && ids.every(id => /^chat-features\.runner-opencode\.local\.[a-z0-9.-]+$/.test(id)); +const selected = (ids: readonly string[]) => ids.length > 0 && ids.every(id => id === "chat-setup-health.runner-opencode.local.setup-only" || /^chat-features\.runner-opencode\.local\.[a-z0-9.-]+$/.test(id)); const filename = (root: string) => path.join(root, "artifacts-private", "chat-readiness-probes.jsonl"); export function readinessErrorCategory(value: unknown): Category { const text = typeof value === "string" ? value.slice(0, 8192) : ""; + if (/ProviderModelNotFoundError|model.{0,40}not found/i.test(text)) return "model-unavailable"; if (/\b(?:401|403|unauthorized|authentication|invalid.api.key)\b/i.test(text)) return "authentication"; if (/\b429\b|rate.?limit/i.test(text)) return "rate-limit"; if (/EACCES|EPERM|permission denied/i.test(text)) return "permission"; @@ -76,7 +77,11 @@ export async function installChatReadinessObserver(input: { }; } /** Re-project even the private file. No arbitrary file keys/strings enter evidence. */ -export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string) { +export interface ClosedReadinessReceipt { + schema: "paperclip.chat-readiness-probe/v1"; companyId: string; sequence: number; status: string; thrown: boolean; + checks: Array<{ code: string; level: string; category: Category }>; errorCategory: Category; +} +export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string): Promise { if (!temporaryRoot || !uuid(companyId)) return []; let bytes: string; try { bytes = await readFile(filename(temporaryRoot), "utf8"); } diff --git a/tests/runner-e2e/chat-remote-host.test.ts b/tests/runner-e2e/chat-remote-host.test.ts index d7f6afe17d8..f3278df1890 100644 --- a/tests/runner-e2e/chat-remote-host.test.ts +++ b/tests/runner-e2e/chat-remote-host.test.ts @@ -1,3 +1,4 @@ +import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js"; import { afterEach, describe, expect, it, vi } from "vitest"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import os from "node:os"; @@ -111,6 +112,14 @@ describe("remote-only chat host qualification without native/provider calls", () await expect(prepareChatRemoteHost(f)).rejects.toThrow("owned-linux-amd64-postgres-image-unqualified"); expect(await f.receipt()).toMatchObject({ qualificationStage: "postgres-image-validate", evaluationAdmitted: false, imageChecks: { [predicate]: false } }); }); + it("keeps the one exact health cell on the same AWS/Docker policy", async () => { + const f = await fixture("on-demand"); f.executions[0]!.id = CHAT_SETUP_HEALTH_ID; f.executions[0]!.suite.id = CHAT_SETUP_HEALTH_SUITE; + expect(selectsRemoteChat(f.executions)).toBe(true); + expect(await prepareChatRemoteHost(f)).toMatchObject({ databaseLane: "owned-docker-postgres", maxAutomaticRetries: 0, maxParallel: 2 }); + expect(f.docker).toHaveBeenCalled(); + f.executions[0]!.id = "chat-setup-health.runner-opencode.local.wrong"; + expect(selectsRemoteChat(f.executions)).toBe(false); + }); it("does not change historical/other profiles or require IMDS for their discovery", async () => { const f = await fixture(); f.executions[0]!.profile.id = "runner-codex-subscription"; expect(selectsRemoteChat(f.executions)).toBe(false); expect(await prepareChatRemoteHost(f)).toBeUndefined(); expect(f.fetch).not.toHaveBeenCalled(); expect(f.docker).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/chat-remote-host.ts b/tests/runner-e2e/chat-remote-host.ts index c73ef00d045..73ee705dd5e 100644 --- a/tests/runner-e2e/chat-remote-host.ts +++ b/tests/runner-e2e/chat-remote-host.ts @@ -1,3 +1,4 @@ +import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js"; import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; @@ -19,7 +20,7 @@ const defaults: RemoteHostDependencies = { fetch: globalThis.fetch, docker: runD sourceRevision: async root => (await promisify(execFile)("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5000 })).stdout.trim() }; /** This new campaign lane is remote-only. Existing profiles/local defaults remain unchanged. */ export function selectsRemoteChat(executions: readonly Selection[]) { - return executions.some(e => e.suite.id === "chat-features" && e.profile.id === "runner-opencode"); + return executions.some(e => (e.suite.id === "chat-features" || e.id === CHAT_SETUP_HEALTH_ID) && e.profile.id === "runner-opencode"); } /** No native runner, model, or database workload is started here. IMDS and image acquisition * run only after the actual trusted GitHub AWS host identity has been checked. */ @@ -36,7 +37,7 @@ export async function prepareChatRemoteHost(input: { executions: readonly Select try { if (env.GITHUB_ACTIONS !== "true" || env.GITHUB_REPOSITORY !== "paperclipai/paperclip" || env.GITHUB_REF !== "refs/heads/master" || env.GITHUB_WORKFLOW_REF !== workflow || !/^\d+$/.test(env.GITHUB_RUN_ID ?? "") || !/^\d+$/.test(env.GITHUB_RUN_ATTEMPT ?? "") || deps.platform !== "linux" || deps.arch !== "x64") throw new Error(reason); - if (input.executions.some(e => e.suite.id !== "chat-features" || e.profile.id !== "runner-opencode" || !reviewedChatProfile(e.profile) || e.environment.id !== "local" || e.task.automaticRetryPolicy !== "single_attempt")) throw new Error(reason); + if (input.executions.some(e => (e.suite.id !== "chat-features" && (e.suite.id !== CHAT_SETUP_HEALTH_SUITE || e.id !== CHAT_SETUP_HEALTH_ID)) || e.profile.id !== "runner-opencode" || !reviewedChatProfile(e.profile) || e.environment.id !== "local" || e.task.automaticRetryPolicy !== "single_attempt")) throw new Error(reason); const source = await deps.sourceRevision(input.repositoryRoot); if (!/^[a-f0-9]{40}$/.test(source) || env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA !== source) throw new Error(reason); Object.assign(receipt, { workflowRef: workflow, workflowRunId: env.GITHUB_RUN_ID, workflowAttempt: Number(env.GITHUB_RUN_ATTEMPT), sourceRevision: source }); diff --git a/tests/runner-e2e/chat-setup-health.test.ts b/tests/runner-e2e/chat-setup-health.test.ts new file mode 100644 index 00000000000..00770afdf04 --- /dev/null +++ b/tests/runner-e2e/chat-setup-health.test.ts @@ -0,0 +1,102 @@ +import { appendFile, mkdtemp, mkdir, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { RunnerApi } from "./api.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { runnerMatrix, runnerSuites } from "./catalog.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; +import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE, gradeSetupHealth, runChatSetupHealth, requestSetupHealthStop, stopSetupHealthIfRequested, SetupHealthAbort } from "./chat-setup-health.js"; +const valid = (): Parameters[0] => ({ ready: true, agentCount: 1, configMatches: true, issues: 0, conversations: 0, comments: 0, humanComments: 0, runs: 0, + probes: [{ schema: "paperclip.chat-readiness-probe/v1", companyId: "company", sequence: 1, status: "pass", thrown: false, + checks: [{ code: "opencode_hello_probe_passed", level: "info", category: "unknown" as const }], errorCategory: "unknown" as const }] }); +describe("explicit setup-only diagnostic cell", () => { + it("selects one exact-profile retry-zero cell outside the catalog and --all", () => { + const cell = runnerMatrix.find(row => row.id === CHAT_SETUP_HEALTH_ID)!; + expect(cell).toBeDefined(); + expect(cell.profile).toBe(runnerSuites.find(row => row.id === "chat-features")!.profiles[0]); + expect(cell).toMatchObject({ suite: { id: CHAT_SETUP_HEALTH_SUITE, manualOnly: true }, profile: { + id: "runner-opencode", model: CHAT_OPENCODE_MODEL, provider: "opencode", credential: "OPENROUTER_API_KEY" }, + task: { flow: "setup_health", expectedRunCount: 0, automaticRetryPolicy: "single_attempt" } }); + expect(cell.task.buildPrompt("nonce")).toBe(""); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"]), runnerMatrix).some(row => row.id === cell.id)).toBe(false); + expect(selectRunnerExecutions(parseRunnerSelectors(["--id", CHAT_SETUP_HEALTH_ID]), runnerMatrix)).toEqual([cell]); + expect(runnerMatrix.filter(row => row.suite.id === "chat-features").length).toBe(244); + }); + it("accepts one real successful setup without any chat/task/native work", () => expect(gradeSetupHealth(valid()).every(check => check.passed)).toBe(true)); + it.each(["ready", "configMatches", "agentCount", "issues", "conversations", "comments", "humanComments", "runs"])("rejects wrong %s", field => { + const state = valid(); Object.assign(state, { [field]: ["ready", "configMatches"].includes(field) ? false : 2 }); + expect(gradeSetupHealth(state).some(check => !check.passed)).toBe(true); + }); + it("rejects missing/repeated/failed/unknown or non-hello probes", () => { + for (const probes of [[], [...valid().probes, ...valid().probes], [{ ...valid().probes[0]!, status: "fail" }], + [{ ...valid().probes[0]!, status: "unknown" }], [{ ...valid().probes[0]!, checks: [] }]]) { + expect(gradeSetupHealth({ ...valid(), probes }).some(check => !check.passed)).toBe(true); + } + }); +}); + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +it.each([[false, false], [true, false], [false, true]])("uses only readiness GET plus cleanup; failure=%s lateProbe=%s", async (failed, lateProbe) => { + const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "setup-health-offline-"))); roots.push(temporaryRoot); + await mkdir(path.join(temporaryRoot, "artifacts-private")); + const companyId = "11111111-1111-4111-8111-111111111111"; + const probe = { ...valid().probes[0]!, companyId, status: failed ? "fail" : "pass" }; + await writeFile(path.join(temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), JSON.stringify(probe)); + const agent = { id: "agent", companyId, defaultEnvironmentId: "environment", adapterType: "paperclip_runner", + adapterConfig: { provider: "opencode", model: CHAT_OPENCODE_MODEL }, status: "idle", lifecycleState: failed ? "verifying" : "ready", + lifecycleError: failed ? "Execution configuration check failed" : null }; + const get = vi.fn(async (route: string) => { + if (route === "/api/agents/agent") return agent; + if (route.endsWith("/agents")) return [agent]; + if (route === "/api/instance/settings/experimental") return { enableAgentChat: true }; + return []; + }); + const post = vi.fn(async (route: string) => { + expect(route).toBe("/api/agents/agent/pause"); + if (lateProbe) await appendFile(path.join(temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), `\n${JSON.stringify({ ...probe, sequence: 2 })}`); + return {}; + }); + const evidence = vi.fn(async (_name: string, _value: unknown) => {}); + const operation = runChatSetupHealth({ api: { get, post } as unknown as RunnerApi, + fixtures: { company: { id: companyId }, agent: { id: "agent" }, environment: { id: "environment" } } as LiveFixtureValues, + temporaryRoot, evidence }); + if (failed || lateProbe) await expect(operation).rejects.toThrow("Setup health failed"); else await operation; + expect(post).toHaveBeenCalledTimes(1); + expect(get.mock.calls.every(([route]) => !route.includes("test-environment") && !route.includes("/chats/"))).toBe(true); + expect(evidence).toHaveBeenCalledTimes(2); + expect((evidence.mock.calls[1]![1] as { state: { probes: unknown[] } }).state.probes).toHaveLength(lateProbe ? 2 : 1); + expect(evidence.mock.calls[1]![1]).toMatchObject({ canonicalQueries: 0, state: { issues: 0, conversations: 0, runs: 0 } }); +}); + +it("only an exact owned health stop request invokes the existing cleanup callback", async () => { + const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "setup-health-stop-"))); roots.push(temporaryRoot); + await mkdir(path.join(temporaryRoot, "artifacts-private")); + const stop = vi.fn(async () => {}); + await requestSetupHealthStop(temporaryRoot); + expect(await stopSetupHealthIfRequested({ executionIds: ["chat-features.runner-opencode.local.create-backlog"], temporaryRoot, stop })).toBe(false); + expect(stop).not.toHaveBeenCalled(); + await writeFile(path.join(temporaryRoot, "artifacts-private", "chat-setup-health-stop.json"), JSON.stringify({ executionId: CHAT_SETUP_HEALTH_ID, temporaryRoot: "wrong-root" })); + expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(false); + expect(stop).not.toHaveBeenCalled(); + await rm(path.join(temporaryRoot, "artifacts-private", "chat-setup-health-stop.json")); + await requestSetupHealthStop(temporaryRoot); + expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(true); + expect(stop).toHaveBeenCalledTimes(1); +}); +it("pause failure requests owned stop before extra oracle reads, even if initial evidence fails", async () => { + const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "setup-health-stop-"))); roots.push(temporaryRoot); + await mkdir(path.join(temporaryRoot, "artifacts-private")); + const get = vi.fn(async () => ({ lifecycleState: "ready", status: "idle" })); + const post = vi.fn(async () => { throw new Error("private-api-error"); }); + const evidence = vi.fn(async () => { throw new Error("private-storage-error"); }); + await expect(runChatSetupHealth({ api: { get, post } as unknown as RunnerApi, + fixtures: { company: { id: "11111111-1111-4111-8111-111111111111" }, agent: { id: "agent" } } as LiveFixtureValues, + temporaryRoot, evidence })).rejects.toBeInstanceOf(SetupHealthAbort); + expect(get).toHaveBeenCalledTimes(1); expect(post).toHaveBeenCalledTimes(1); + const stop = vi.fn(async () => {}); + expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(true); + expect(stop).toHaveBeenCalledTimes(1); +}); diff --git a/tests/runner-e2e/chat-setup-health.ts b/tests/runner-e2e/chat-setup-health.ts new file mode 100644 index 00000000000..838b989acea --- /dev/null +++ b/tests/runner-e2e/chat-setup-health.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; +import { constants, readFileSync } from "node:fs"; +import { open, realpath, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; +import { readChatReadinessReceipts } from "./chat-readiness-observer.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; +import type { RunnerTaskFixture } from "./types.js"; +export const CHAT_SETUP_HEALTH_SUITE = "chat-setup-health"; +export const CHAT_SETUP_HEALTH_ID = "chat-setup-health.runner-opencode.local.setup-only"; +export class SetupHealthAbort extends Error { + constructor() { super("Setup health cleanup failed; owned supervisor stop requested, no further API observations"); } +} +const stopPath = (root: string) => path.join(root, "artifacts-private", "chat-setup-health-stop.json"); +export async function requestSetupHealthStop(root: string) { + await writeFile(stopPath(root), JSON.stringify({ executionId: CHAT_SETUP_HEALTH_ID, temporaryRoot: root }), { flag: "wx", mode: 0o600 }); +} +/** Exact one-cell request, consumed only by the existing owned supervisor. */ +export async function stopSetupHealthIfRequested(input: { executionIds: readonly string[]; temporaryRoot: string; stop(): Promise }) { + if (input.executionIds.length !== 1 || input.executionIds[0] !== CHAT_SETUP_HEALTH_ID) return false; + let file; + try { + const root = await realpath(input.temporaryRoot), rootInfo = await stat(root); + if (root !== input.temporaryRoot || rootInfo.uid !== process.getuid?.() || (rootInfo.mode & 0o077)) return false; + file = await open(stopPath(root), constants.O_RDONLY | constants.O_NOFOLLOW); + const info = await file.stat(); + if (!info.isFile() || info.uid !== rootInfo.uid || (info.mode & 0o077) || info.size > 1024) return false; + const row = JSON.parse(await file.readFile("utf8")); + if (row?.executionId !== CHAT_SETUP_HEALTH_ID || row?.temporaryRoot !== root) return false; + } catch { return false; } + finally { await file?.close(); } + await input.stop(); + return true; +} +export const chatSetupHealthTask: RunnerTaskFixture = { + id: "setup-only", label: "One normal OpenCode setup verification; no catalog request", groups: [], flow: "setup_health", workMode: "standard", + expectedRunCount: 0, automaticRetryPolicy: "single_attempt", turnTimeoutMs: 60_000, + attemptTimeoutMs: { local: 180_000, daytona: 180_000 }, expectedTerminalState: { issue: "in_review", run: "succeeded" }, + buildTitle: () => "Setup health only", buildPrompt: () => "", buildVisibleMarker: () => "", + buildMatchers: () => [], +}; +export const chatSetupHealthDigest = createHash("sha256").update([ + "chat-setup-health.ts", "chat-readiness-observer.ts", "server-entry.ts", "runner.spec.ts", "live-fixtures.ts", "chat-remote-host.ts", "server.ts", +].map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n")).digest("hex"); +interface HealthAgent { id: string; companyId: string; lifecycleState?: string; lifecycleError?: string | null; status: string; + defaultEnvironmentId?: string | null; adapterType: string; adapterConfig?: Record } +export function gradeSetupHealth(state: { + ready: boolean; agentCount: number; configMatches: boolean; issues: number; conversations: number; comments: number; humanComments: number; runs: number; + probes: Awaited>; +}) { + return [ + { id: "ordinary-agent-ready", passed: state.ready && state.agentCount === 1 && state.configMatches }, + { id: "one-ordinary-hello", passed: state.probes.length === 1 && !state.probes[0]!.thrown + && ["pass", "warn"].includes(state.probes[0]!.status) && state.probes[0]!.checks.some(check => check.code === "opencode_hello_probe_passed" && check.level === "info") }, + { id: "zero-chat-or-task-work", passed: state.issues === 0 && state.conversations === 0 && state.comments === 0 && state.humanComments === 0 && state.runs === 0 }, + ]; +} +export async function runChatSetupHealth(input: { api: RunnerApi; fixtures: LiveFixtureValues; temporaryRoot: string; + evidence(name: string, value: unknown): Promise }) { + const { api, fixtures: f } = input; + let ready = false; + try { + await pollUntil({ label: "ordinary setup-health agent readiness", deadlineAt: Date.now() + 75_000, intervalMs: 250, + load: () => api.get(`/api/agents/${f.agent.id}`, { timeout: 5_000 }), + accept: agent => agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status), + reject: agent => agent.lifecycleError ? "Actual setup-health lifecycle verification failed" : undefined }); + ready = true; + } finally { + // GET only: no chat resolution/creation, task dispatch or explicit second probe. + const company = `/api/companies/${f.company.id}`; + let diagnosticFailure = false; + try { + const probes = await readChatReadinessReceipts(input.temporaryRoot, f.company.id); + await input.evidence("chat-setup-health.json", { schema: "paperclip.chat-setup-health/v1", scope: "diagnostic-outside-244", + companyId: f.company.id, agentId: f.agent.id, probes, publicState: "not-yet-verified", + setupProbeUsage: "unavailable outside heartbeat accounting; never zero" }); + } catch { diagnosticFailure = true; } + // Always fence future automatic verification, even when diagnostic I/O fails. + try { await api.post(`/api/agents/${f.agent.id}/pause`, {}, { timeout: 5_000 }); } + catch { + let stopRequested = false; + try { await requestSetupHealthStop(input.temporaryRoot); stopRequested = true; } catch { /* Preserve uncertain cleanup. */ } + try { await input.evidence("chat-setup-health-cleanup.json", { scope: "diagnostic-outside-244", pauseFailed: true, stopRequested, cleanup: "failed" }); } catch { /* Do not delay owned stop for diagnostics. */ } + throw new SetupHealthAbort(); + } + if (diagnosticFailure) throw new Error("Setup health diagnostics unavailable after normal public pause"); + // Read after the pause fence too: a late second completion must fail the count. + const probes = await readChatReadinessReceipts(input.temporaryRoot, f.company.id); + const settings = await api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental", { timeout: 5_000 }); + if (!settings.enableAgentChat) await api.patch("/api/instance/settings/experimental", { enableAgentChat: true }); + const [agents, issues, conversations, runs] = await Promise.all([ + api.get(`${company}/agents`, { timeout: 5_000 }), api.get>(`${company}/issues?includeAgentCreatedTasks=true&limit=100`, { timeout: 5_000 }), + api.get>(`${company}/chats`, { timeout: 5_000 }), api.get(`${company}/heartbeat-runs?limit=100`, { timeout: 5_000 }), + ]); + const sourceIds = [...new Set([...issues, ...conversations].map(row => row.id))]; + const comments = (await Promise.all(sourceIds.map(id => api.get>(`/api/issues/${id}/comments?order=asc`, { timeout: 5_000 })))).flat(); + const agent = agents.find(row => row.id === f.agent.id); + const state = { ready, agentCount: agents.length, configMatches: agent?.companyId === f.company.id && agent.defaultEnvironmentId === f.environment.id + && agent.adapterType === "paperclip_runner" && agent.adapterConfig?.provider === "opencode" && agent.adapterConfig?.model === CHAT_OPENCODE_MODEL, + issues: issues.length, conversations: conversations.length, comments: comments.length, humanComments: comments.filter(row => row.authorUserId).length, + runs: runs.length, probes }; + const checks = gradeSetupHealth(state); + await input.evidence("chat-setup-health.json", { schema: "paperclip.chat-setup-health/v1", scope: "diagnostic-outside-244", companyId: f.company.id, + agentId: f.agent.id, state, checks, canonicalQueries: state.humanComments, + setupProbeUsage: "One ordinary automatic lifecycle hello is expected. Provider tokens and cost are unavailable outside heartbeat accounting; never reported as zero." }); + if (checks.some(check => !check.passed)) throw new Error("Setup health failed; inspect retained closed readiness diagnostics"); + } +} diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 7f7c04838a5..38c74db43c2 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,3 +1,4 @@ +import { CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js"; import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js"; import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js"; @@ -168,7 +169,7 @@ export async function setupLiveFixtures(input: { const company = await api.post("/api/companies", { name: `Runner E2E ${execution.id} ${input.executionNonce}`, description: "Ephemeral paid full-stack runner acceptance fixture", - budgetMonthlyCents: execution.suite.id === CHAT_FEATURE_SUITE ? CHAT_FEATURE_BUDGET_CENTS + budgetMonthlyCents: [CHAT_FEATURE_SUITE, CHAT_SETUP_HEALTH_SUITE].includes(execution.suite.id) ? CHAT_FEATURE_BUDGET_CENTS : receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index e8cea8f098e..106586ec829 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,4 @@ +import { SetupHealthAbort, runChatSetupHealth } from "./chat-setup-health.js"; import { bindInvitationEnrollmentPrerequisite } from "./chat-catalog-domains/identity-invite-prerequisite.js"; import { verifyInvitationSourceProof } from "./chat-catalog-domains/identity-invite-source-proof.js"; import { readChatPrivateInputs, chatPrivateSetupBlockers, chatPrivateInputRequirements } from "./chat-private-inputs.js"; @@ -1335,6 +1336,10 @@ for (const execution of executions) { }); issue = contextIntegrity.issue as unknown as IssueRecord; selectedRuns = contextIntegrity.runs as unknown as RunRecord[]; + } else if (execution.task.flow === "setup_health") { + await runChatSetupHealth({ api, fixtures, temporaryRoot: temporaryRoot!, evidence: completionEvidence }); + matcherResults = [{ matcher: { kind: "json_path", path: "setupHealth.closed-checks", expected: true }, passed: true, + detail: "One genuine ordinary setup hello, exact saved configuration, zero chat/task work." }]; } else if (execution.task.flow === "agent_chat") { const chat = await runChatFlow({ privateSecrets: secrets, privateInputs, @@ -3146,7 +3151,7 @@ for (const execution of executions) { failureClassOverride = "permanent_infrastructure"; } try { - await captureFailureApiState(); + if (!(error instanceof SetupHealthAbort)) await captureFailureApiState(); } catch (captureError) { const failureClass = classifyFailure(captureError); if (failureClass === "secret_leak") { @@ -3164,7 +3169,7 @@ for (const execution of executions) { }); } } - if (!page.isClosed()) { + if (!(error instanceof SetupHealthAbort) && !page.isClosed()) { if (isReviewedFixtureScreenshotRoute()) { await captureScreenshot( "failure", @@ -3178,6 +3183,7 @@ for (const execution of executions) { } } } finally { + const healthAbort = primaryError instanceof SetupHealthAbort; await workerTiming.enter("cleanup"); trace?.phase("cleanup", Boolean(primaryError)); await responseTiming?.stop(); @@ -3202,7 +3208,8 @@ for (const execution of executions) { failureClassOverride = "secret_leak"; } browserBootstrap.dispose(); - if (fixtures) { + if (healthAbort) { cleanup = "failed"; failureClassOverride = "cleanup_failure"; } + if (fixtures && !healthAbort) { await captureRuntimeLeases().catch((error) => { networkDiagnostics.push({ runtimeBillingCaptureError: diff --git a/tests/runner-e2e/server-entry.ts b/tests/runner-e2e/server-entry.ts index 3e0a7ef3a3f..e2175485d66 100644 --- a/tests/runner-e2e/server-entry.ts +++ b/tests/runner-e2e/server-entry.ts @@ -1,3 +1,4 @@ +import { CHAT_SETUP_HEALTH_ID } from "./chat-setup-health.js"; import { installChatReadinessObserver } from "./chat-readiness-observer.js"; // This entrypoint is used only by isolated Runner E2E instances. Production // service code has no test flag, delay, altered prompt, or private test API. @@ -9,7 +10,7 @@ import { installOwnedEmailResponseLoss } from "./chat-catalog-domains/runtime-em import { installOwnedEmailReceiptDuplicate } from "./chat-catalog-domains/runtime-email-duplicate.js"; const ids: string[] = JSON.parse(process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]"); -if (ids.length && ids.every(id => id.startsWith("chat-features.runner-opencode.local."))) { +if (ids.length && ids.every(id => id === CHAT_SETUP_HEALTH_ID || id.startsWith("chat-features.runner-opencode.local."))) { const { requireServerAdapter } = await import("../../server/src/adapters/index.js"); await installChatReadinessObserver({ executionIds: ids, temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "", adapter: requireServerAdapter("paperclip_runner") }); diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index fe36f6f3b69..60b9490f208 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -1,3 +1,4 @@ +import { stopSetupHealthIfRequested } from "./chat-setup-health.js"; import { publishInvitationSourceProof, invitationSourceProofPath } from "./chat-catalog-domains/identity-invite-source-proof.js"; import { readSelectedVaultServerEnvironment } from "./chat-vault-private-server.js"; import { readValidatedPublicOriginReceipt, publicOriginServerOverrides, PUBLIC_ORIGIN_RECEIPT_ENV, PUBLIC_ORIGIN_CONTROL_ENV } from "./chat-public-origin.js"; @@ -447,6 +448,10 @@ async function supervise() { await startServer(); let lastRestartRequestId: string | null = null; while (!shutdownRequested()) { + if (await stopSetupHealthIfRequested({ executionIds, temporaryRoot, stop: () => stopServer.stopAll("SIGTERM") })) { + shutdownSignal = "SIGTERM"; + break; + } if (unexpectedChildFailure) throw unexpectedChildFailure; const request = await readRestartRequest(); if (request && request.requestId !== lastRestartRequestId) { diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index fe9b0ce238f..9b8154a85de 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -37,6 +37,7 @@ export type RunnerTaskFlow = | "continuation" | "first_task" | "agent_chat" + | "setup_health" | "governed_tool_review" | "single_turn" | "plan_revision_acceptance"