test(evals): add explicit zero-work chat setup health

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -3021,3 +3021,20 @@ a preflight rejection does not establish that prerequisite. Provider/account
 qualification, paid-job approval, and owned AWS Spot capacity must be established
 before remote admission. Discovery and credential-free controls establish none
 of those live outcomes.
+
+
+### Explicit chat provider setup health
+
+`chat-setup-health.runner-opencode.local.setup-only` is a diagnostic cell, outside
+all 244 chat catalog scenarios and excluded from `--all`. It reuses the exact
+chat OpenCode/OpenRouter/DeepSeek profile, ordinary trusted fixture setup, owned
+Docker PostgreSQL and normal teardown. One agent's ordinary automatic hello is
+expected; it makes no explicit environment-test call and sends no chat/task
+request. No heartbeat runs, issues, conversations or comments may exist.
+
+Select this cell explicitly with retries zero through the existing Product E2E
+workflow. A failed readiness check fails health and retains only closed check
+codes/categories. Public pause fences automatic setup retries; failed pause
+requests the existing owned supervisor's shutdown before further API observations.
+Multiple probe receipts fail health. Setup-probe usage/cost is unavailable, never
+zero. This diagnostic cannot qualify a catalog scenario or rewrite prior results.

tests/runner-e2e/catalog.ts

@@ -1,3 +1,4 @@
+import { CHAT_SETUP_HEALTH_SUITE, chatSetupHealthTask, chatSetupHealthDigest } from "./chat-setup-health.js";
 import { CHAT_OPENCODE_MODEL, CHAT_OPENCODE_PROFILE } from "./chat-runtime-profile.js";
 import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js";
 import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js";
@@ -1157,6 +1158,12 @@ const repositoryModelProfiles = [
   },
 ];
 
+const chatFeatureExecutionProfile = chatFeatureProfile(nativeProfile({
+  id: CHAT_OPENCODE_PROFILE, label: "Native OpenCode OpenRouter DeepSeek V4.1 Flash 1017",
+  provider: "opencode", model: CHAT_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"],
+  modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-catalog:opencode-1.18.34:deepseek-v4.1-flash-1017:remote-initial" },
+}));
+
 export const runnerSuites: readonly RunnerSuiteFixture[] = [
   {
     id: "daytona-restart-continuity", label: "Daytona controller outage continuity", manualOnly: true,
@@ -1605,13 +1612,19 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
     environments: [localEnvironment], tasks: firstTaskTasks, expectedMatrixSize: 52,
     definitionMetadata: { version: 8, runGrading: "evidenced-nonexecution-and-refusal", credentialPersistenceCheck: false, questionChoiceMinimum: 2, nativeSetup: "qualified-platform-wizard-saved-default", claudeDefaultTargets: ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"], legacySetup: "explicit-advanced-runner-choice", productionInstructions: true, qualityGrading: "informational" },
   },
+  {
+    id: CHAT_SETUP_HEALTH_SUITE, label: "Chat provider setup health", manualOnly: true,
+    description: "Diagnostic only: one normal lifecycle hello, zero chat/task work; excluded from the 244 catalog scenarios.",
+    groups: [], profiles: [chatFeatureExecutionProfile], environments: [localEnvironment], tasks: [chatSetupHealthTask], expectedMatrixSize: 1,
+    definitionMetadata: { version: 1, sourceDigest: chatSetupHealthDigest, scheduling: "explicit-only", catalogScenario: false,
+      maximumAttemptsPerCell: 1, maximumAutomaticSetupHellos: 1, expectedHeartbeatRuns: 0, canonicalQueries: 0,
+      setupProbeUsage: "unavailable-outside-heartbeat-accounting" },
+  },
   {
     id: CHAT_FEATURE_SUITE, label: "Chat feature journeys", manualOnly: true,
     description: "Executable catalog requests through Agent Chat, with independently graded persisted state.",
     groups: ["chat", "native"],
-    profiles: [chatFeatureProfile(nativeProfile({ id: CHAT_OPENCODE_PROFILE, label: "Native OpenCode OpenRouter DeepSeek V4.1 Flash 1017",
-      provider: "opencode", model: CHAT_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"],
-      modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-catalog:opencode-1.18.34:deepseek-v4.1-flash-1017:remote-initial" } }))],
+    profiles: [chatFeatureExecutionProfile],
     environments: [localEnvironment], tasks: [...chatFeatureTasks, ...chatCatalogTasks], expectedMatrixSize: 2 + chatCatalogTasks.length,
     definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, inventoryDigest: chatCatalogInventoryDigest, frozenScenarioCount: 244, scheduling: "explicit-only",
       instructions: "production-role-defaults", companyAndAgentBudgetCents: CHAT_FEATURE_BUDGET_CENTS,

tests/runner-e2e/chat-readiness-observer.test.ts

@@ -76,6 +76,31 @@ describe("transparent same-call readiness retention", () => {
     await installChatReadinessObserver({ executionIds: ["other.runner-opencode.local.case"], temporaryRoot: "", adapter });
     expect(adapter.testEnvironment).toBe(original);
   });
+  it.each([
+    ["agent_runner_config_incompatible", "unknown"], ["paperclip_runner_mode_invalid", "unknown"], ["paperclip_runner_opencode_model_invalid", "unknown"],
+  ])("preserves source profile-validation code %s without its message", async (code, category) => {
+    const f = await fixture(); f.testEnvironment.mockResolvedValue({ ...result(), checks: [{ code, level: "error", message: "private-profile-canary", detail: "private-canary", hint: "private-canary" }] });
+    await f.adapter.testEnvironment(context);
+    expect((await f.receipts())[0]!.checks).toEqual([{ code, level: "error", category }]);
+    expect(JSON.stringify(await f.receipts())).not.toContain("private-");
+  });
+  // Actual check.message/detail layouts emitted by opencode-local/server/test.ts:
+  // summarizeProbeDetail selects parsed error, first stderr or first stdout line.
+  it.each([
+    ["opencode_command_unresolvable", "Command is not executable: ENOENT", "opencode", "command-missing"],
+    ["opencode_openai_api_key_missing", "OPENAI_API_KEY override is empty.", "", "unknown"],
+    ["opencode_hello_probe_auth_required", "OpenCode is installed, but provider authentication is not ready.", "HTTP 401 Unauthorized private-canary", "authentication"],
+    ["opencode_hello_probe_model_unavailable", "The configured model was not found by the provider.", "ProviderModelNotFoundError private-canary", "model-unavailable"],
+    ["opencode_hello_probe_failed", "OpenCode hello probe failed.", '{"name":"APIError","data":{"statusCode":429,"message":"private-canary"}}', "rate-limit"],
+    ["opencode_hello_probe_failed", "OpenCode hello probe failed.", "fetch failed ECONNREFUSED private-canary", "network"],
+    ["opencode_hello_probe_timed_out", "OpenCode hello probe timed out.", "", "timeout"],
+    ["opencode_hello_probe_failed", "OpenCode hello probe failed.", "private-unknown-canary", "unknown"],
+  ])("retains closed actual-source diagnostic shape %s/%s", async (code, message, detail, category) => {
+    const f = await fixture(); f.testEnvironment.mockResolvedValue({ ...result(), checks: [{ code, level: "error", message, detail, hint: "private-hint-canary" }] });
+    await f.adapter.testEnvironment(context);
+    expect((await f.receipts())[0]!.checks).toEqual([{ code, level: "error", category }]);
+    expect(JSON.stringify(await f.receipts())).not.toMatch(/private-|APIError|statusCode|detail|message|hint/);
+  });
   it.each([
     ["HTTP 401 private", "authentication"], ["ECONNREFUSED private", "network"], ["EPERM private", "permission"],
     ["ENOENT private", "command-missing"], ["timed out private", "timeout"], ["unrecognized private", "unknown"],

tests/runner-e2e/chat-readiness-observer.ts

@@ -14,15 +14,16 @@ const codes = new Set([
   "opencode_model_configured", "opencode_hello_probe_timed_out", "opencode_hello_probe_passed",
   "opencode_hello_probe_unexpected_output", "opencode_hello_probe_model_unavailable",
   "opencode_hello_probe_auth_required", "opencode_hello_probe_failed",
-  "paperclip_runner_provider_unsupported",
+  "paperclip_runner_provider_unsupported", "agent_runner_config_incompatible", "paperclip_runner_mode_invalid", "paperclip_runner_opencode_model_invalid",
 ]);
-const categories = ["authentication", "rate-limit", "network", "permission", "command-missing", "module-missing", "timeout", "unknown"] as const;
+const categories = ["authentication", "rate-limit", "network", "permission", "command-missing", "module-missing", "model-unavailable", "timeout", "unknown"] as const;
 type Category = typeof categories[number];
 const uuid = (v: unknown): v is string => typeof v === "string" && /^[a-f0-9-]{36}$/i.test(v);
-const selected = (ids: readonly string[]) => ids.length > 0 && ids.every(id => /^chat-features\.runner-opencode\.local\.[a-z0-9.-]+$/.test(id));
+const selected = (ids: readonly string[]) => ids.length > 0 && ids.every(id => id === "chat-setup-health.runner-opencode.local.setup-only" || /^chat-features\.runner-opencode\.local\.[a-z0-9.-]+$/.test(id));
 const filename = (root: string) => path.join(root, "artifacts-private", "chat-readiness-probes.jsonl");
 export function readinessErrorCategory(value: unknown): Category {
   const text = typeof value === "string" ? value.slice(0, 8192) : "";
+  if (/ProviderModelNotFoundError|model.{0,40}not found/i.test(text)) return "model-unavailable";
   if (/\b(?:401|403|unauthorized|authentication|invalid.api.key)\b/i.test(text)) return "authentication";
   if (/\b429\b|rate.?limit/i.test(text)) return "rate-limit";
   if (/EACCES|EPERM|permission denied/i.test(text)) return "permission";
@@ -76,7 +77,11 @@ export async function installChatReadinessObserver(input: {
   };
 }
 /** Re-project even the private file. No arbitrary file keys/strings enter evidence. */
-export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string) {
+export interface ClosedReadinessReceipt {
+  schema: "paperclip.chat-readiness-probe/v1"; companyId: string; sequence: number; status: string; thrown: boolean;
+  checks: Array<{ code: string; level: string; category: Category }>; errorCategory: Category;
+}
+export async function readChatReadinessReceipts(temporaryRoot: string, companyId: string): Promise<ClosedReadinessReceipt[]> {
   if (!temporaryRoot || !uuid(companyId)) return [];
   let bytes: string;
   try { bytes = await readFile(filename(temporaryRoot), "utf8"); }

tests/runner-e2e/chat-remote-host.test.ts

@@ -1,3 +1,4 @@
+import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { mkdtemp, readFile, rm } from "node:fs/promises";
 import os from "node:os";
@@ -111,6 +112,14 @@ describe("remote-only chat host qualification without native/provider calls", ()
     await expect(prepareChatRemoteHost(f)).rejects.toThrow("owned-linux-amd64-postgres-image-unqualified");
     expect(await f.receipt()).toMatchObject({ qualificationStage: "postgres-image-validate", evaluationAdmitted: false, imageChecks: { [predicate]: false } });
   });
+  it("keeps the one exact health cell on the same AWS/Docker policy", async () => {
+    const f = await fixture("on-demand"); f.executions[0]!.id = CHAT_SETUP_HEALTH_ID; f.executions[0]!.suite.id = CHAT_SETUP_HEALTH_SUITE;
+    expect(selectsRemoteChat(f.executions)).toBe(true);
+    expect(await prepareChatRemoteHost(f)).toMatchObject({ databaseLane: "owned-docker-postgres", maxAutomaticRetries: 0, maxParallel: 2 });
+    expect(f.docker).toHaveBeenCalled();
+    f.executions[0]!.id = "chat-setup-health.runner-opencode.local.wrong";
+    expect(selectsRemoteChat(f.executions)).toBe(false);
+  });
   it("does not change historical/other profiles or require IMDS for their discovery", async () => {
     const f = await fixture(); f.executions[0]!.profile.id = "runner-codex-subscription";
     expect(selectsRemoteChat(f.executions)).toBe(false); expect(await prepareChatRemoteHost(f)).toBeUndefined(); expect(f.fetch).not.toHaveBeenCalled(); expect(f.docker).not.toHaveBeenCalled();

tests/runner-e2e/chat-remote-host.ts

@@ -1,3 +1,4 @@
+import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js";
 import { createHash } from "node:crypto";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
@@ -19,7 +20,7 @@ const defaults: RemoteHostDependencies = { fetch: globalThis.fetch, docker: runD
   sourceRevision: async root => (await promisify(execFile)("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5000 })).stdout.trim() };
 /** This new campaign lane is remote-only. Existing profiles/local defaults remain unchanged. */
 export function selectsRemoteChat(executions: readonly Selection[]) {
-  return executions.some(e => e.suite.id === "chat-features" && e.profile.id === "runner-opencode");
+  return executions.some(e => (e.suite.id === "chat-features" || e.id === CHAT_SETUP_HEALTH_ID) && e.profile.id === "runner-opencode");
 }
 /** No native runner, model, or database workload is started here. IMDS and image acquisition
  * run only after the actual trusted GitHub AWS host identity has been checked. */
@@ -36,7 +37,7 @@ export async function prepareChatRemoteHost(input: { executions: readonly Select
   try {
     if (env.GITHUB_ACTIONS !== "true" || env.GITHUB_REPOSITORY !== "paperclipai/paperclip" || env.GITHUB_REF !== "refs/heads/master" || env.GITHUB_WORKFLOW_REF !== workflow
       || !/^\d+$/.test(env.GITHUB_RUN_ID ?? "") || !/^\d+$/.test(env.GITHUB_RUN_ATTEMPT ?? "") || deps.platform !== "linux" || deps.arch !== "x64") throw new Error(reason);
-    if (input.executions.some(e => e.suite.id !== "chat-features" || e.profile.id !== "runner-opencode" || !reviewedChatProfile(e.profile) || e.environment.id !== "local" || e.task.automaticRetryPolicy !== "single_attempt")) throw new Error(reason);
+    if (input.executions.some(e => (e.suite.id !== "chat-features" && (e.suite.id !== CHAT_SETUP_HEALTH_SUITE || e.id !== CHAT_SETUP_HEALTH_ID)) || e.profile.id !== "runner-opencode" || !reviewedChatProfile(e.profile) || e.environment.id !== "local" || e.task.automaticRetryPolicy !== "single_attempt")) throw new Error(reason);
     const source = await deps.sourceRevision(input.repositoryRoot);
     if (!/^[a-f0-9]{40}$/.test(source) || env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA !== source) throw new Error(reason);
     Object.assign(receipt, { workflowRef: workflow, workflowRunId: env.GITHUB_RUN_ID, workflowAttempt: Number(env.GITHUB_RUN_ATTEMPT), sourceRevision: source });

tests/runner-e2e/chat-setup-health.test.ts

@@ -0,0 +1,102 @@
+import { appendFile, mkdtemp, mkdir, realpath, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import type { RunnerApi } from "./api.js";
+import type { LiveFixtureValues } from "./live-fixtures.js";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { runnerMatrix, runnerSuites } from "./catalog.js";
+import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
+import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
+import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE, gradeSetupHealth, runChatSetupHealth, requestSetupHealthStop, stopSetupHealthIfRequested, SetupHealthAbort } from "./chat-setup-health.js";
+const valid = (): Parameters<typeof gradeSetupHealth>[0] => ({ ready: true, agentCount: 1, configMatches: true, issues: 0, conversations: 0, comments: 0, humanComments: 0, runs: 0,
+  probes: [{ schema: "paperclip.chat-readiness-probe/v1", companyId: "company", sequence: 1, status: "pass", thrown: false,
+    checks: [{ code: "opencode_hello_probe_passed", level: "info", category: "unknown" as const }], errorCategory: "unknown" as const }] });
+describe("explicit setup-only diagnostic cell", () => {
+  it("selects one exact-profile retry-zero cell outside the catalog and --all", () => {
+    const cell = runnerMatrix.find(row => row.id === CHAT_SETUP_HEALTH_ID)!;
+    expect(cell).toBeDefined();
+    expect(cell.profile).toBe(runnerSuites.find(row => row.id === "chat-features")!.profiles[0]);
+    expect(cell).toMatchObject({ suite: { id: CHAT_SETUP_HEALTH_SUITE, manualOnly: true }, profile: {
+      id: "runner-opencode", model: CHAT_OPENCODE_MODEL, provider: "opencode", credential: "OPENROUTER_API_KEY" },
+      task: { flow: "setup_health", expectedRunCount: 0, automaticRetryPolicy: "single_attempt" } });
+    expect(cell.task.buildPrompt("nonce")).toBe("");
+    expect(selectRunnerExecutions(parseRunnerSelectors(["--all"]), runnerMatrix).some(row => row.id === cell.id)).toBe(false);
+    expect(selectRunnerExecutions(parseRunnerSelectors(["--id", CHAT_SETUP_HEALTH_ID]), runnerMatrix)).toEqual([cell]);
+    expect(runnerMatrix.filter(row => row.suite.id === "chat-features").length).toBe(244);
+  });
+  it("accepts one real successful setup without any chat/task/native work", () => expect(gradeSetupHealth(valid()).every(check => check.passed)).toBe(true));
+  it.each(["ready", "configMatches", "agentCount", "issues", "conversations", "comments", "humanComments", "runs"])("rejects wrong %s", field => {
+    const state = valid(); Object.assign(state, { [field]: ["ready", "configMatches"].includes(field) ? false : 2 });
+    expect(gradeSetupHealth(state).some(check => !check.passed)).toBe(true);
+  });
+  it("rejects missing/repeated/failed/unknown or non-hello probes", () => {
+    for (const probes of [[], [...valid().probes, ...valid().probes], [{ ...valid().probes[0]!, status: "fail" }],
+      [{ ...valid().probes[0]!, status: "unknown" }], [{ ...valid().probes[0]!, checks: [] }]]) {
+      expect(gradeSetupHealth({ ...valid(), probes }).some(check => !check.passed)).toBe(true);
+    }
+  });
+});
+
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+it.each([[false, false], [true, false], [false, true]])("uses only readiness GET plus cleanup; failure=%s lateProbe=%s", async (failed, lateProbe) => {
+  const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "setup-health-offline-"))); roots.push(temporaryRoot);
+  await mkdir(path.join(temporaryRoot, "artifacts-private"));
+  const companyId = "11111111-1111-4111-8111-111111111111";
+  const probe = { ...valid().probes[0]!, companyId, status: failed ? "fail" : "pass" };
+  await writeFile(path.join(temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), JSON.stringify(probe));
+  const agent = { id: "agent", companyId, defaultEnvironmentId: "environment", adapterType: "paperclip_runner",
+    adapterConfig: { provider: "opencode", model: CHAT_OPENCODE_MODEL }, status: "idle", lifecycleState: failed ? "verifying" : "ready",
+    lifecycleError: failed ? "Execution configuration check failed" : null };
+  const get = vi.fn(async (route: string) => {
+    if (route === "/api/agents/agent") return agent;
+    if (route.endsWith("/agents")) return [agent];
+    if (route === "/api/instance/settings/experimental") return { enableAgentChat: true };
+    return [];
+  });
+  const post = vi.fn(async (route: string) => {
+    expect(route).toBe("/api/agents/agent/pause");
+    if (lateProbe) await appendFile(path.join(temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), `\n${JSON.stringify({ ...probe, sequence: 2 })}`);
+    return {};
+  });
+  const evidence = vi.fn(async (_name: string, _value: unknown) => {});
+  const operation = runChatSetupHealth({ api: { get, post } as unknown as RunnerApi,
+    fixtures: { company: { id: companyId }, agent: { id: "agent" }, environment: { id: "environment" } } as LiveFixtureValues,
+    temporaryRoot, evidence });
+  if (failed || lateProbe) await expect(operation).rejects.toThrow("Setup health failed"); else await operation;
+  expect(post).toHaveBeenCalledTimes(1);
+  expect(get.mock.calls.every(([route]) => !route.includes("test-environment") && !route.includes("/chats/"))).toBe(true);
+  expect(evidence).toHaveBeenCalledTimes(2);
+  expect((evidence.mock.calls[1]![1] as { state: { probes: unknown[] } }).state.probes).toHaveLength(lateProbe ? 2 : 1);
+  expect(evidence.mock.calls[1]![1]).toMatchObject({ canonicalQueries: 0, state: { issues: 0, conversations: 0, runs: 0 } });
+});
+
+it("only an exact owned health stop request invokes the existing cleanup callback", async () => {
+  const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "setup-health-stop-"))); roots.push(temporaryRoot);
+  await mkdir(path.join(temporaryRoot, "artifacts-private"));
+  const stop = vi.fn(async () => {});
+  await requestSetupHealthStop(temporaryRoot);
+  expect(await stopSetupHealthIfRequested({ executionIds: ["chat-features.runner-opencode.local.create-backlog"], temporaryRoot, stop })).toBe(false);
+  expect(stop).not.toHaveBeenCalled();
+  await writeFile(path.join(temporaryRoot, "artifacts-private", "chat-setup-health-stop.json"), JSON.stringify({ executionId: CHAT_SETUP_HEALTH_ID, temporaryRoot: "wrong-root" }));
+  expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(false);
+  expect(stop).not.toHaveBeenCalled();
+  await rm(path.join(temporaryRoot, "artifacts-private", "chat-setup-health-stop.json"));
+  await requestSetupHealthStop(temporaryRoot);
+  expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(true);
+  expect(stop).toHaveBeenCalledTimes(1);
+});
+it("pause failure requests owned stop before extra oracle reads, even if initial evidence fails", async () => {
+  const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "setup-health-stop-"))); roots.push(temporaryRoot);
+  await mkdir(path.join(temporaryRoot, "artifacts-private"));
+  const get = vi.fn(async () => ({ lifecycleState: "ready", status: "idle" }));
+  const post = vi.fn(async () => { throw new Error("private-api-error"); });
+  const evidence = vi.fn(async () => { throw new Error("private-storage-error"); });
+  await expect(runChatSetupHealth({ api: { get, post } as unknown as RunnerApi,
+    fixtures: { company: { id: "11111111-1111-4111-8111-111111111111" }, agent: { id: "agent" } } as LiveFixtureValues,
+    temporaryRoot, evidence })).rejects.toBeInstanceOf(SetupHealthAbort);
+  expect(get).toHaveBeenCalledTimes(1); expect(post).toHaveBeenCalledTimes(1);
+  const stop = vi.fn(async () => {});
+  expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(true);
+  expect(stop).toHaveBeenCalledTimes(1);
+});

tests/runner-e2e/chat-setup-health.ts

@@ -0,0 +1,109 @@
+import { createHash } from "node:crypto";
+import { constants, readFileSync } from "node:fs";
+import { open, realpath, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { pollUntil, type RunnerApi } from "./api.js";
+import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
+import { readChatReadinessReceipts } from "./chat-readiness-observer.js";
+import type { LiveFixtureValues } from "./live-fixtures.js";
+import type { RunnerTaskFixture } from "./types.js";
+export const CHAT_SETUP_HEALTH_SUITE = "chat-setup-health";
+export const CHAT_SETUP_HEALTH_ID = "chat-setup-health.runner-opencode.local.setup-only";
+export class SetupHealthAbort extends Error {
+  constructor() { super("Setup health cleanup failed; owned supervisor stop requested, no further API observations"); }
+}
+const stopPath = (root: string) => path.join(root, "artifacts-private", "chat-setup-health-stop.json");
+export async function requestSetupHealthStop(root: string) {
+  await writeFile(stopPath(root), JSON.stringify({ executionId: CHAT_SETUP_HEALTH_ID, temporaryRoot: root }), { flag: "wx", mode: 0o600 });
+}
+/** Exact one-cell request, consumed only by the existing owned supervisor. */
+export async function stopSetupHealthIfRequested(input: { executionIds: readonly string[]; temporaryRoot: string; stop(): Promise<void> }) {
+  if (input.executionIds.length !== 1 || input.executionIds[0] !== CHAT_SETUP_HEALTH_ID) return false;
+  let file;
+  try {
+    const root = await realpath(input.temporaryRoot), rootInfo = await stat(root);
+    if (root !== input.temporaryRoot || rootInfo.uid !== process.getuid?.() || (rootInfo.mode & 0o077)) return false;
+    file = await open(stopPath(root), constants.O_RDONLY | constants.O_NOFOLLOW);
+    const info = await file.stat();
+    if (!info.isFile() || info.uid !== rootInfo.uid || (info.mode & 0o077) || info.size > 1024) return false;
+    const row = JSON.parse(await file.readFile("utf8"));
+    if (row?.executionId !== CHAT_SETUP_HEALTH_ID || row?.temporaryRoot !== root) return false;
+  } catch { return false; }
+  finally { await file?.close(); }
+  await input.stop();
+  return true;
+}
+export const chatSetupHealthTask: RunnerTaskFixture = {
+  id: "setup-only", label: "One normal OpenCode setup verification; no catalog request", groups: [], flow: "setup_health", workMode: "standard",
+  expectedRunCount: 0, automaticRetryPolicy: "single_attempt", turnTimeoutMs: 60_000,
+  attemptTimeoutMs: { local: 180_000, daytona: 180_000 }, expectedTerminalState: { issue: "in_review", run: "succeeded" },
+  buildTitle: () => "Setup health only", buildPrompt: () => "", buildVisibleMarker: () => "",
+  buildMatchers: () => [],
+};
+export const chatSetupHealthDigest = createHash("sha256").update([
+  "chat-setup-health.ts", "chat-readiness-observer.ts", "server-entry.ts", "runner.spec.ts", "live-fixtures.ts", "chat-remote-host.ts", "server.ts",
+].map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n")).digest("hex");
+interface HealthAgent { id: string; companyId: string; lifecycleState?: string; lifecycleError?: string | null; status: string;
+  defaultEnvironmentId?: string | null; adapterType: string; adapterConfig?: Record<string, unknown> }
+export function gradeSetupHealth(state: {
+  ready: boolean; agentCount: number; configMatches: boolean; issues: number; conversations: number; comments: number; humanComments: number; runs: number;
+  probes: Awaited<ReturnType<typeof readChatReadinessReceipts>>;
+}) {
+  return [
+    { id: "ordinary-agent-ready", passed: state.ready && state.agentCount === 1 && state.configMatches },
+    { id: "one-ordinary-hello", passed: state.probes.length === 1 && !state.probes[0]!.thrown
+      && ["pass", "warn"].includes(state.probes[0]!.status) && state.probes[0]!.checks.some(check => check.code === "opencode_hello_probe_passed" && check.level === "info") },
+    { id: "zero-chat-or-task-work", passed: state.issues === 0 && state.conversations === 0 && state.comments === 0 && state.humanComments === 0 && state.runs === 0 },
+  ];
+}
+export async function runChatSetupHealth(input: { api: RunnerApi; fixtures: LiveFixtureValues; temporaryRoot: string;
+  evidence(name: string, value: unknown): Promise<void> }) {
+  const { api, fixtures: f } = input;
+  let ready = false;
+  try {
+    await pollUntil({ label: "ordinary setup-health agent readiness", deadlineAt: Date.now() + 75_000, intervalMs: 250,
+      load: () => api.get<HealthAgent>(`/api/agents/${f.agent.id}`, { timeout: 5_000 }),
+      accept: agent => agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status),
+      reject: agent => agent.lifecycleError ? "Actual setup-health lifecycle verification failed" : undefined });
+    ready = true;
+  } finally {
+    // GET only: no chat resolution/creation, task dispatch or explicit second probe.
+    const company = `/api/companies/${f.company.id}`;
+    let diagnosticFailure = false;
+    try {
+      const probes = await readChatReadinessReceipts(input.temporaryRoot, f.company.id);
+      await input.evidence("chat-setup-health.json", { schema: "paperclip.chat-setup-health/v1", scope: "diagnostic-outside-244",
+        companyId: f.company.id, agentId: f.agent.id, probes, publicState: "not-yet-verified",
+        setupProbeUsage: "unavailable outside heartbeat accounting; never zero" });
+    } catch { diagnosticFailure = true; }
+    // Always fence future automatic verification, even when diagnostic I/O fails.
+    try { await api.post(`/api/agents/${f.agent.id}/pause`, {}, { timeout: 5_000 }); }
+    catch {
+      let stopRequested = false;
+      try { await requestSetupHealthStop(input.temporaryRoot); stopRequested = true; } catch { /* Preserve uncertain cleanup. */ }
+      try { await input.evidence("chat-setup-health-cleanup.json", { scope: "diagnostic-outside-244", pauseFailed: true, stopRequested, cleanup: "failed" }); } catch { /* Do not delay owned stop for diagnostics. */ }
+      throw new SetupHealthAbort();
+    }
+    if (diagnosticFailure) throw new Error("Setup health diagnostics unavailable after normal public pause");
+    // Read after the pause fence too: a late second completion must fail the count.
+    const probes = await readChatReadinessReceipts(input.temporaryRoot, f.company.id);
+    const settings = await api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental", { timeout: 5_000 });
+    if (!settings.enableAgentChat) await api.patch("/api/instance/settings/experimental", { enableAgentChat: true });
+    const [agents, issues, conversations, runs] = await Promise.all([
+      api.get<HealthAgent[]>(`${company}/agents`, { timeout: 5_000 }), api.get<Array<{ id: string }>>(`${company}/issues?includeAgentCreatedTasks=true&limit=100`, { timeout: 5_000 }),
+      api.get<Array<{ id: string }>>(`${company}/chats`, { timeout: 5_000 }), api.get<unknown[]>(`${company}/heartbeat-runs?limit=100`, { timeout: 5_000 }),
+    ]);
+    const sourceIds = [...new Set([...issues, ...conversations].map(row => row.id))];
+    const comments = (await Promise.all(sourceIds.map(id => api.get<Array<{ authorUserId?: string | null }>>(`/api/issues/${id}/comments?order=asc`, { timeout: 5_000 })))).flat();
+    const agent = agents.find(row => row.id === f.agent.id);
+    const state = { ready, agentCount: agents.length, configMatches: agent?.companyId === f.company.id && agent.defaultEnvironmentId === f.environment.id
+      && agent.adapterType === "paperclip_runner" && agent.adapterConfig?.provider === "opencode" && agent.adapterConfig?.model === CHAT_OPENCODE_MODEL,
+      issues: issues.length, conversations: conversations.length, comments: comments.length, humanComments: comments.filter(row => row.authorUserId).length,
+      runs: runs.length, probes };
+    const checks = gradeSetupHealth(state);
+    await input.evidence("chat-setup-health.json", { schema: "paperclip.chat-setup-health/v1", scope: "diagnostic-outside-244", companyId: f.company.id,
+      agentId: f.agent.id, state, checks, canonicalQueries: state.humanComments,
+      setupProbeUsage: "One ordinary automatic lifecycle hello is expected. Provider tokens and cost are unavailable outside heartbeat accounting; never reported as zero." });
+    if (checks.some(check => !check.passed)) throw new Error("Setup health failed; inspect retained closed readiness diagnostics");
+  }
+}

tests/runner-e2e/live-fixtures.ts

@@ -1,3 +1,4 @@
+import { CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js";
 import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js";
 import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
 import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js";
@@ -168,7 +169,7 @@ export async function setupLiveFixtures(input: {
       const company = await api.post<CompanyRecord>("/api/companies", {
         name: `Runner E2E ${execution.id} ${input.executionNonce}`,
         description: "Ephemeral paid full-stack runner acceptance fixture",
-        budgetMonthlyCents: execution.suite.id === CHAT_FEATURE_SUITE ? CHAT_FEATURE_BUDGET_CENTS
+        budgetMonthlyCents: [CHAT_FEATURE_SUITE, CHAT_SETUP_HEALTH_SUITE].includes(execution.suite.id) ? CHAT_FEATURE_BUDGET_CENTS
           : receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS
           : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id)
           || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS

tests/runner-e2e/runner.spec.ts

@@ -1,3 +1,4 @@
+import { SetupHealthAbort, runChatSetupHealth } from "./chat-setup-health.js";
 import { bindInvitationEnrollmentPrerequisite } from "./chat-catalog-domains/identity-invite-prerequisite.js";
 import { verifyInvitationSourceProof } from "./chat-catalog-domains/identity-invite-source-proof.js";
 import { readChatPrivateInputs, chatPrivateSetupBlockers, chatPrivateInputRequirements } from "./chat-private-inputs.js";
@@ -1335,6 +1336,10 @@ for (const execution of executions) {
         });
         issue = contextIntegrity.issue as unknown as IssueRecord;
         selectedRuns = contextIntegrity.runs as unknown as RunRecord[];
+      } else if (execution.task.flow === "setup_health") {
+        await runChatSetupHealth({ api, fixtures, temporaryRoot: temporaryRoot!, evidence: completionEvidence });
+        matcherResults = [{ matcher: { kind: "json_path", path: "setupHealth.closed-checks", expected: true }, passed: true,
+          detail: "One genuine ordinary setup hello, exact saved configuration, zero chat/task work." }];
       } else if (execution.task.flow === "agent_chat") {
         const chat = await runChatFlow({
           privateSecrets: secrets, privateInputs,
@@ -3146,7 +3151,7 @@ for (const execution of executions) {
         failureClassOverride = "permanent_infrastructure";
       }
       try {
-        await captureFailureApiState();
+        if (!(error instanceof SetupHealthAbort)) await captureFailureApiState();
       } catch (captureError) {
         const failureClass = classifyFailure(captureError);
         if (failureClass === "secret_leak") {
@@ -3164,7 +3169,7 @@ for (const execution of executions) {
           });
         }
       }
-      if (!page.isClosed()) {
+      if (!(error instanceof SetupHealthAbort) && !page.isClosed()) {
         if (isReviewedFixtureScreenshotRoute()) {
           await captureScreenshot(
             "failure",
@@ -3178,6 +3183,7 @@ for (const execution of executions) {
         }
       }
     } finally {
+      const healthAbort = primaryError instanceof SetupHealthAbort;
       await workerTiming.enter("cleanup");
       trace?.phase("cleanup", Boolean(primaryError));
       await responseTiming?.stop();
@@ -3202,7 +3208,8 @@ for (const execution of executions) {
         failureClassOverride = "secret_leak";
       }
       browserBootstrap.dispose();
-      if (fixtures) {
+      if (healthAbort) { cleanup = "failed"; failureClassOverride = "cleanup_failure"; }
+      if (fixtures && !healthAbort) {
         await captureRuntimeLeases().catch((error) => {
           networkDiagnostics.push({
             runtimeBillingCaptureError:

tests/runner-e2e/server-entry.ts

@@ -1,3 +1,4 @@
+import { CHAT_SETUP_HEALTH_ID } from "./chat-setup-health.js";
 import { installChatReadinessObserver } from "./chat-readiness-observer.js";
 // This entrypoint is used only by isolated Runner E2E instances. Production
 // service code has no test flag, delay, altered prompt, or private test API.
@@ -9,7 +10,7 @@ import { installOwnedEmailResponseLoss } from "./chat-catalog-domains/runtime-em
 import { installOwnedEmailReceiptDuplicate } from "./chat-catalog-domains/runtime-email-duplicate.js";
 
 const ids: string[] = JSON.parse(process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]");
-if (ids.length && ids.every(id => id.startsWith("chat-features.runner-opencode.local."))) {
+if (ids.length && ids.every(id => id === CHAT_SETUP_HEALTH_ID || id.startsWith("chat-features.runner-opencode.local."))) {
   const { requireServerAdapter } = await import("../../server/src/adapters/index.js");
   await installChatReadinessObserver({ executionIds: ids, temporaryRoot: process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "",
     adapter: requireServerAdapter("paperclip_runner") });

tests/runner-e2e/server.ts

@@ -1,3 +1,4 @@
+import { stopSetupHealthIfRequested } from "./chat-setup-health.js";
 import { publishInvitationSourceProof, invitationSourceProofPath } from "./chat-catalog-domains/identity-invite-source-proof.js";
 import { readSelectedVaultServerEnvironment } from "./chat-vault-private-server.js";
 import { readValidatedPublicOriginReceipt, publicOriginServerOverrides, PUBLIC_ORIGIN_RECEIPT_ENV, PUBLIC_ORIGIN_CONTROL_ENV } from "./chat-public-origin.js";
@@ -447,6 +448,10 @@ async function supervise() {
   await startServer();
   let lastRestartRequestId: string | null = null;
   while (!shutdownRequested()) {
+    if (await stopSetupHealthIfRequested({ executionIds, temporaryRoot, stop: () => stopServer.stopAll("SIGTERM") })) {
+      shutdownSignal = "SIGTERM";
+      break;
+    }
     if (unexpectedChildFailure) throw unexpectedChildFailure;
     const request = await readRestartRequest();
     if (request && request.requestId !== lastRestartRequestId) {

tests/runner-e2e/types.ts

@@ -37,6 +37,7 @@ export type RunnerTaskFlow =
   | "continuation"
   | "first_task"
   | "agent_chat"
+  | "setup_health"
   | "governed_tool_review"
   | "single_turn"
   | "plan_revision_acceptance"