test(evals): cover disabled chat and actual failed native history phases

tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json

@@ -111,8 +111,8 @@
     },
     {
       "scenarioId": "agent-chat.chat-disabled",
-      "readinessClass": "boundary",
-      "needs": "Actual disabled Agent Chat context needs a narrowly allowlisted unavailable-surface browser-boundary case; ordinary Agent Chat cannot send its canonical query. No conference-only unrelated-gate substitute."
+      "readinessClass": "implemented",
+      "needs": "Actual allowlisted authenticated-harness boundary; disable Agent Chat itself; real Morgan route unavailable/no composer, agent settings denial, create-chat 404, usable Morgan profile and zero tasks/chats/native admissions."
     },
     {
       "scenarioId": "runs-adapters.list-model-options",
@@ -126,8 +126,8 @@
     },
     {
       "scenarioId": "runs-adapters.inspect-failed-run",
-      "readinessClass": "local",
-      "needs": "Generate real failed website run and later successful one; retain exact failure diagnostics and missing cost/log branches."
+      "readinessClass": "implemented",
+      "needs": "Public unavailable-model configuration must actually admit one native failed run with model/task/actor/terminal-event evidence; restore exact configuration, one actual human Retry, later successful HTML artifact, retained failure/source links and truthful missing logs/cost."
     },
     {
       "scenarioId": "runs-adapters.reset-session",
@@ -201,8 +201,8 @@
     },
     {
       "scenarioId": "recovery.browser-reconnect",
-      "readinessClass": "partial",
-      "needs": "Real browser offline/server completion plus second actual still-running phase; phase-bound status replies and no duplicate sends/runs. Failed-native-run reconnect remains a concrete coverage gap; no live runtime attempts yet."
+      "readinessClass": "implemented",
+      "needs": "Three executable real offline phases: native completion, independently observed still-running work, and mandatory actually admitted failed native attempt under explicit unavailable-model configuration; each answer bound to its own persisted human/native run. No live qualification claimed."
     },
     {
       "scenarioId": "steering.queue-follow-up",

tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts

@@ -0,0 +1,11 @@
+import { it, expect } from "vitest";
+import { gradeDisabledChat } from "./runtime-chat-disabled.js";
+function valid() { return { companyId: "company", agentId: "morgan", personId: "actual-human", canonicalIntent: "Open a chat with Morgan.", agentChatEnabled: false,
+  unavailableVisible: true, composerAbsent: true, supportedOperatorStepVisible: true, profileAccessible: true, agentSettingsStatus: 403,
+  createChatStatus: 404, gateAfterProbe: false, beforeTaskIds: [], afterTaskIds: [] as string[], chatIds: [] as string[], runIds: [] as string[] }; }
+it("accepts the actual unavailable Morgan Chat surface with no admission", () => expect(gradeDisabledChat(valid()).passed).toBe(true));
+it("rejects an unrelated disabled conference gate while Agent Chat is enabled", () => { const o = valid(); o.agentChatEnabled = true; expect(gradeDisabledChat(o).passed).toBe(false); });
+it("rejects missing canonical unavailable context", () => expect(gradeDisabledChat(undefined).passed).toBe(false));
+it("rejects a disguised task created while Chat is disabled", () => { const o = valid(); o.afterTaskIds = ["workaround"]; expect(gradeDisabledChat(o).passed).toBe(false); });
+it("rejects an ordinary agent enabling the instance setting", () => { const o = valid(); o.agentSettingsStatus = 200; o.gateAfterProbe = true; expect(gradeDisabledChat(o).passed).toBe(false); });
+it("rejects native work started by unavailable navigation", () => { const o = valid(); o.runIds = ["unrequested"]; expect(gradeDisabledChat(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts

@@ -0,0 +1,80 @@
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import { expect } from "@playwright/test";
+import { RunnerApi } from "../api.js";
+import { setupLiveFixtures, type LiveFixtureValues } from "../live-fixtures.js";
+import { createAuthenticatedPerson } from "./identity-auth.js";
+import { checks } from "./internal-support.js";
+import type { IdentityBoundaryCase, IdentityBoundaryContext } from "./identity-boundary-contract.js";
+type Person = Awaited<ReturnType<typeof createAuthenticatedPerson>>;
+const owned = new WeakMap<IdentityBoundaryContext, { person: Person; api: RunnerApi; companyId?: string; fixtures?: LiveFixtureValues; keyId?: string }>();
+interface DisabledObservation { companyId: string; agentId: string; personId: string; canonicalIntent: string; agentChatEnabled: boolean;
+  unavailableVisible: boolean; composerAbsent: boolean; supportedOperatorStepVisible: boolean; profileAccessible: boolean;
+  agentSettingsStatus: number; createChatStatus: number; gateAfterProbe: boolean; beforeTaskIds: string[]; afterTaskIds: string[]; chatIds: string[]; runIds: string[] }
+export function gradeDisabledChat(value: unknown) {
+  const c = checks(); const o = value as DisabledObservation | undefined;
+  c.add("actual-disabled-canonical-context", !!o && !!o.companyId && !!o.agentId && !!o.personId && o.personId !== "local-board"
+    && o.canonicalIntent === "Open a chat with Morgan." && o.agentChatEnabled === false && o.unavailableVisible && o.composerAbsent,
+  "The real signed-in human opens the intended Morgan Agent Chat target with its actual gate disabled; canonical intent remains unsent.");
+  c.add("clear-supported-human-path", !!o && o.supportedOperatorStepVisible && o.profileAccessible,
+  "The unavailable surface identifies Experimental settings as the human management step, and the actual Morgan profile remains usable.");
+  c.add("ordinary-agent-cannot-change-instance-gate", !!o && o.agentSettingsStatus === 403 && o.gateAfterProbe === false
+    && o.createChatStatus === 404, "The real ordinary agent cannot enable instance Chat, and the public create-chat path enforces the same disabled gate.");
+  c.add("no-disguised-task-or-admission", !!o && Array.isArray(o.beforeTaskIds) && Array.isArray(o.afterTaskIds)
+    && JSON.stringify(o.beforeTaskIds) === JSON.stringify(o.afterTaskIds) && Array.isArray(o.chatIds) && !o.chatIds.length
+    && Array.isArray(o.runIds) && !o.runIds.length,
+  "Unavailable navigation admits no native work, duplicate standing chat or silently created task.");
+  return c.result();
+}
+export const runtimeDisabledChatCases: readonly IdentityBoundaryCase[] = [{
+  scenarioId: "agent-chat.chat-disabled", journeyKind: "browser-boundary", boundaryTarget: "authenticated-harness", authentication: "human-session",
+  oracleVersion: "paperclip.chat-catalog.runtime.v2", expectedRunCount: 0, expectedHumanMessageCount: 0, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000,
+  sourceFiles: ["chat-catalog-domains/runtime-chat-disabled.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "live-fixtures.ts", "chat-scenarios/agent-chat.yaml"],
+  grade: gradeDisabledChat,
+  async run(ctx) {
+    const browser = ctx.page.context().browser(); if (!browser) throw new Error("Unavailable Agent Chat boundary requires the real human browser");
+    const person = await createAuthenticatedPerson({ api: ctx.api, browser, secrets: ctx.privateSecrets, name: "Alex" });
+    const api = new RunnerApi(person.context.request); person.bindApiSession(api); const resource: { person: Person; api: RunnerApi; companyId?: string; fixtures?: LiveFixtureValues; keyId?: string } = { person, api }; owned.set(ctx, resource);
+    await api.post("/api/bootstrap/claim");
+    const workspacePath = path.join(ctx.privateDir, "disabled-chat-owned-workspace"); await mkdir(workspacePath, { recursive: true, mode: 0o700 });
+    const fixtures = await setupLiveFixtures({ api, execution: ctx.execution, executionNonce: ctx.nonce, workspacePath, credentials: ctx.credentials,
+      async onCompanyCreated(id) { resource.companyId = id; ctx.observeResources({ companyId: id, runIds: [] }); } });
+    resource.fixtures = fixtures;
+    await api.patch(`/api/agents/${fixtures.agent.id}`, { name: "Morgan" });
+    await api.patch("/api/instance/settings/experimental", { enableAgentChat: false });
+    const settings = await api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental");
+    const root = `/api/companies/${fixtures.company.id}`;
+    const beforeTaskIds = (await api.get<Array<{ id: string }>>(`${root}/issues?includeAgentCreatedTasks=true&limit=1000`)).map(row => row.id).sort();
+    const page = await person.context.newPage();
+    await page.goto(`/${fixtures.company.issuePrefix}/chats/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" });
+    await expect(page.getByText(/Agent Chat is disabled\. Enable it in Experimental settings/)).toBeVisible();
+    await expect(page.getByTestId("task-chat-composer-input")).toHaveCount(0);
+    const unavailableVisible = await page.getByText(/Agent Chat is disabled/).isVisible();
+    const supportedOperatorStepVisible = await page.getByText(/Enable it in Experimental settings/).isVisible();
+    await ctx.capture("disabled-agent-chat", "Morgan Chat is genuinely unavailable; canonical intent remains unsent", "runtime-disabled-agent-chat.png");
+    const key = await api.postSensitive<{ id: string; token: string }>(`/api/agents/${fixtures.agent.id}/keys`, { name: "disabled-chat-permission-probe" });
+    ctx.privateSecrets.push(key.token); resource.keyId = key.id;
+    const denied = await fetch(new URL("/api/instance/settings/experimental", api.baseURL), { method: "PATCH", redirect: "error", signal: AbortSignal.timeout(15_000),
+      headers: { Authorization: `Bearer ${key.token}`, "content-type": "application/json" }, body: JSON.stringify({ enableAgentChat: true }) });
+    const agentSettingsStatus = denied.status; await denied.body?.cancel();
+    const rejected = await api.request.post(`${root}/chats/${fixtures.agent.id}`, { data: {} }); const createChatStatus = rejected.status(); await rejected.dispose();
+    const gateAfterProbe = (await api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental")).enableAgentChat;
+    await page.goto(`/${fixtures.company.issuePrefix}/agents/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" });
+    await page.reload({ waitUntil: "domcontentloaded" }); await expect(page.getByRole("heading", { name: "Morgan", exact: true }).first()).toBeVisible();
+    const tasks = await api.get<Array<{ id: string; conversationAgentId?: string | null }>>(`${root}/issues?includeAgentCreatedTasks=true&limit=1000`);
+    const runIds = (await api.get<Array<{ id: string }>>(`${root}/heartbeat-runs?limit=100`)).map(row => row.id);
+    ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds });
+    const observation: DisabledObservation = { companyId: fixtures.company.id, agentId: fixtures.agent.id, personId: person.id, canonicalIntent: ctx.scenario.query,
+      agentChatEnabled: settings.enableAgentChat, unavailableVisible, composerAbsent: true, supportedOperatorStepVisible, profileAccessible: true,
+      agentSettingsStatus, createChatStatus, gateAfterProbe, beforeTaskIds, afterTaskIds: tasks.map(row => row.id).sort(),
+      chatIds: tasks.filter(row => row.conversationAgentId).map(row => row.id), runIds };
+    await ctx.evidence("runtime-disabled-chat-boundary.json", observation); return observation;
+  },
+  async cleanup(ctx) {
+    const resource = owned.get(ctx); if (!resource) return;
+    try { if (resource.keyId && resource.fixtures) await resource.api.delete(`/api/agents/${resource.fixtures.agent.id}/keys/${resource.keyId}`, { allowNotFound: true });
+      await resource.fixtures?.teardown();
+      if (resource.companyId) await resource.api.delete(`/api/companies/${resource.companyId}`, { allowNotFound: true });
+    } finally { await resource.person.dispose(); owned.delete(ctx); }
+  },
+}];

tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts

@@ -0,0 +1,17 @@
+import { it, expect } from "vitest";
+import { gradeFailedAttempt } from "./runtime-failed-runs.js";
+function valid() { const failedRun = { id: "failed-model", status: "failed", errorCode: "native_provider_model_rejected", error: "Selected model unavailable", costUsd: null as number | null, logRef: "actual-log" as string | null };
+  return { scenarioId: "runs-adapters.inspect-failed-run", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
+    state: { failedRun, retainedFailure: { ...failedRun }, successfulRun: { id: "actual-success", status: "succeeded" }, successArtifactVerified: true, task: { id: "website" }, failureLogAvailable: true },
+    agents: [], tasks: [{ id: "website" }], activity: [], humanQueries: [], links: ["/ACORN/agents/morgan/runs/failed-model"],
+    reply: "The failed-model attempt failed because its selected model was rejected by the provider. The later attempt succeeded; cost data for the failure is unpriced and unavailable.",
+    runs: [{ id: "failed-model", status: "failed" }, { id: "actual-success", status: "succeeded" }, { id: "answer", status: "succeeded" }] }; }
+it("accepts distinct retained native failure and later success with honest unavailable billing", () => expect(gradeFailedAttempt(valid()).passed).toBe(true));
+it("rejects absent genuine failure evidence", () => expect(gradeFailedAttempt(undefined).passed).toBe(false));
+it("rejects erasing the failure after later success", () => { const o = valid(); o.state.retainedFailure.status = "succeeded"; expect(gradeFailedAttempt(o).passed).toBe(false); });
+it("rejects explaining or linking a similarly named successful attempt", () => { const o = valid(); o.links = ["/ACORN/agents/morgan/runs/actual-success"]; expect(gradeFailedAttempt(o).passed).toBe(false); });
+it("rejects treating unavailable billing as zero", () => { const o = valid(); o.reply += " The cost was $0.00."; expect(gradeFailedAttempt(o).passed).toBe(false); });
+it("requires a missing output disclosure when logs are unavailable", () => { const o = valid(); o.state.failedRun.logRef = null; o.state.failureLogAvailable = false;
+  expect(gradeFailedAttempt(o).passed).toBe(false); o.reply += " Retained log output is unavailable."; expect(gradeFailedAttempt(o).passed).toBe(true); });
+it("accepts an actually recorded zero cost without calling it unavailable", () => { const o = valid(); o.state.failedRun.costUsd = 0; o.reply = "The failed-model run failed with a provider model error. The recorded cost was $0.00."; expect(gradeFailedAttempt(o).passed).toBe(true); });
+it("rejects invented successful output without verified saved bytes", () => { const o = valid(); o.state.successArtifactVerified = false; expect(gradeFailedAttempt(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts

@@ -0,0 +1,73 @@
+import { expect } from "@playwright/test";
+import { pollUntil } from "../api.js";
+import { readRegisteredArtifacts } from "../registered-artifact.js";
+import { checks, type Entity } from "./internal-support.js";
+import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js";
+import { unavailableNativeModelFailure } from "./runtime-native-failure.js";
+import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js";
+export function gradeFailedAttempt(value: unknown) {
+  const c = checks(); const o = runtimeObservation(value);
+  c.add("actual-failed-and-successful-native-history", !!o, "An actually admitted native failure and a later real successful execution must remain separately readable."); if (!o) return c.result();
+  const failed = o.state.failedRun as HeartbeatRun | undefined, retained = o.state.retainedFailure as HeartbeatRun | undefined;
+  const success = o.state.successfulRun as HeartbeatRun | undefined;
+  c.add("original-failure-not-erased-by-success", !!failed && !!retained && failed.id === retained.id && failed.status === "failed"
+    && retained.status === "failed" && retained.errorCode === failed.errorCode && retained.error === failed.error && !!failed.errorCode
+    && !!success && success.status === "succeeded" && success.id !== failed.id && o.state.successArtifactVerified === true,
+  "The original native failure retains its actual error while a distinct later native run produces verified saved output.");
+  c.add("correct-attempt-and-observed-error", !!failed && o.reply.includes(failed.id) && /fail|reject|error|could not|couldn't/i.test(o.reply)
+    && /model|configuration|provider|runtime/i.test(o.reply) && !/failed (?:attempt|run).*(?:was|is) (?:actually )?success/i.test(o.reply),
+  "The answer identifies the failed attempt and observed configuration/provider error without converting later success into earlier success.");
+  c.add("real-run-source-link", !!failed && o.links.some(href => { try { const url = new URL(href, o.origin);
+    return url.origin === o.origin && url.pathname.split("/").map(decodeURIComponent).includes(failed.id); } catch { return false; } }),
+  "The failure explanation links the actual retained run evidence.");
+  const missingCost = failed?.costUsd == null; const missingLog = !failed?.logRef || o.state.failureLogAvailable === false;
+  c.add("unavailable-data-not-zero", (!missingCost || /cost|usage|token/i.test(o.reply) && /unavailable|unknown|not (?:available|recorded)|no (?:reported|recorded|priced)|unpriced/i.test(o.reply))
+    && (!missingLog || /log|output/i.test(o.reply) && /unavailable|not (?:available|recorded)|no (?:retained|recorded)|missing/i.test(o.reply))
+    && !(missingCost && /\$0(?:\.00)?\b|zero (?:cost|dollars)|cost (?:was|is) 0/i.test(o.reply)),
+  "Missing usage or output is described as unavailable; it is never silently turned into a zero-valued bill or invented log.");
+  c.add("no-new-work-to-explain-history", o.tasks.length === 1 && o.tasks[0]?.id === (o.state.task as Entity).id
+    && o.runs.length === 3 && o.runs.filter(run => run.status === "failed").length === 1 && o.runs.filter(run => run.status === "succeeded").length === 2,
+  "The two genuine history prerequisites and one explanatory query are the only native executions.");
+  return c.result();
+}
+export const runtimeFailedRunCases = [runtimeDefinition("runs-adapters.inspect-failed-run", async ctx => {
+  const worker = await runtimeWorker(ctx); let task: Entity | undefined;
+  const failure = await unavailableNativeModelFailure(ctx, worker, async () => {
+    task = await ctx.input.api.post<Entity>(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id,
+      description: `Create a small semantic HTML launch website with headline Acorn Notes and reference ${ctx.input.nonce}. Save it as website.html and register that actual file as a downloadable artifact work product titled website.html. Verify it and complete this task normally. No deployment, email, new task or remote operation is authorized.` });
+    return { issueId: task.id };
+  });
+  if (!task) throw new Error("Actual failed native task was not created");
+  const source = ctx.input.page.url();
+  await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
+  const retry = ctx.input.page.getByTestId("task-chat-run-failed-try-again").last(); await expect(retry).toBeVisible();
+  await retry.click();
+  const done = await pollUntil({ label: "Real successful task retry after restoring the admitted model", deadlineAt: Date.now() + 240_000,
+    load: async () => ({ task: await ctx.input.api.get<Entity>(`/api/issues/${task!.id}`), runs: await ctx.allRuns() }),
+    accept: value => value.task.status === "done" && value.runs.length === 2 && value.runs.some(run => run.id !== failure.failedRun.id && run.status === "succeeded"),
+    reject: value => value.runs.length > 2 || value.runs.some(run => run.id !== failure.failedRun.id && ["failed", "cancelled", "timed_out"].includes(run.status)) ? "The one declared successful history retry did not succeed" : undefined });
+  const successfulRun = done.runs.find(run => run.id !== failure.failedRun.id)!;
+  const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, successfulRun.id, ["website.html"]);
+  if (!artifact?.contentVerified || !artifact.content.includes(ctx.input.nonce) || !/<html|<!doctype html/i.test(artifact.content)) throw new Error("The later run did not produce the genuine expected HTML delivery");
+  const response = await ctx.input.api.request.get(`/api/heartbeat-runs/${failure.failedRun.id}/log?offset=0&limitBytes=4096`);
+  const failureLogAvailable = response.ok(); await response.dispose();
+  await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  return { worker, task: done.task, ...failure, successfulRun, successArtifactVerified: true, failureLogAvailable };
+}, gradeFailedAttempt, {
+  sourceFiles: ["chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml"],
+  expectedRunCount: 3, attemptTimeoutMs: 720_000,
+  runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 2 } },
+  async observe(ctx, state) {
+    state.retainedFailure = await ctx.input.api.get(`/api/heartbeat-runs/${(state.failedRun as { id: string }).id}`);
+    return observeRuntime(ctx, state);
+  },
+  async verifyResult(ctx, state) {
+    const source = ctx.input.page.url(); const failed = state.failedRun as { id: string }; const worker = state.worker as Entity;
+    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/agents/${worker.id}/runs/${failed.id}`, { waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByText(/failed/i).first()).toBeVisible();
+    const actual = await ctx.input.api.get<HeartbeatRun>(`/api/heartbeat-runs/${failed.id}`);
+    if (actual.id !== failed.id || actual.status !== "failed") throw new Error("Failure result UI did not retain the intended failed attempt");
+    await ctx.input.evidence("runtime-retained-failed-attempt-ui.json", { runId: failed.id, errorCode: actual.errorCode, taskId: (state.task as Entity).id, url: ctx.input.page.url() });
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+})];

tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts

@@ -23,10 +23,13 @@ function reconnect() {
       workerWasReallyWaiting: true, offlineObserved: true, workerStatusBeforeDisconnect: "running", workerStatusWhileOffline: "succeeded",
       taskStatusWhileOffline: "done", serverAdvancedWhileOffline: true, reconnected: true, historyRetained: true,
       offlineCommentsRetained: true, queryReply: "The website ACORN-1 finished while you were away.",
-      remainingPhase: { attributed: true, reply: "That follow-up is still running and waiting for the brief.", taskId: "followup", runId: "remaining", offlineObserved: true, runningWhileOffline: true, reconnected: true } },
-    agents: [], tasks: [{ id: "website", status: "done" }, { id: "followup", status: "in_progress" }], documents: [{ id: "delivery", title: "Website delivery", createdByRunId: "worker" }],
-    links: [], activity: [], reply: "", humanQueries: ["Track this", "Did you finish while I was disconnected?", "Is the follow-up finished?"],
-    runs: [{ id: "worker", agentId: "morgan", status: "succeeded" }, { id: "prior", agentId: "ceo", status: "succeeded" }, { id: "query", agentId: "ceo", status: "succeeded" }, { id: "remaining", agentId: "morgan", status: "running" }, { id: "status", agentId: "ceo", status: "succeeded" }] };
+      remainingPhase: { attributed: true, reply: "That follow-up is still running and waiting for the brief.", taskId: "followup", runId: "remaining", offlineObserved: true, runningWhileOffline: true, reconnected: true, runStatusAtReply: "running", taskStatusAtReply: "in_progress" },
+      failedReconnectPhase: { attributed: true, reply: "Riley's attempt failed with a provider model configuration error.", runId: "real-failure", taskId: "failed-task", offlineObserved: true, failedWhileOffline: true, reconnected: true } },
+    agents: [], tasks: [{ id: "website", status: "done" }, { id: "followup", status: "done" }, { id: "failed-task", status: "blocked" }], documents: [{ id: "delivery", title: "Website delivery", latestRevisionId: "saved" }],
+    revisions: [{ id: "saved", documentId: "delivery", createdByRunId: "worker", createdByAgentId: "morgan" }],
+    links: [], activity: [], reply: "", humanQueries: ["Track this", "Did you finish while I was disconnected?", "Is the follow-up finished?", "Did Riley finish?"],
+    runs: [{ id: "worker", agentId: "morgan", status: "succeeded" }, { id: "prior", agentId: "ceo", status: "succeeded" }, { id: "query", agentId: "ceo", status: "succeeded" }, { id: "remaining", agentId: "morgan", status: "succeeded" }, { id: "status", agentId: "ceo", status: "succeeded" },
+      { id: "real-failure", agentId: "riley", status: "failed", runtimeMode: "native", errorCode: "native_provider_model_rejected" }, { id: "failed-status", agentId: "ceo", status: "succeeded" }] };
 }
 it("accepts actual server completion through a real browser disconnect", () => expect(gradeBrowserReconnect(reconnect()).passed).toBe(true));
 it("rejects missing disconnect evidence", () => expect(gradeBrowserReconnect(undefined).passed).toBe(false));
@@ -37,3 +40,6 @@ it("rejects losing messages whose live stream events were missed", () => { const
 it("rejects a status claim only present in the old phase", () => { const o = reconnect(); o.reply = o.state.queryReply; o.state.queryReply = "Hello."; expect(gradeBrowserReconnect(o).passed).toBe(false); });
 it("rejects a completion claim for genuinely remaining work", () => { const o = reconnect(); o.state.remainingPhase.reply = "It has finished."; expect(gradeBrowserReconnect(o).passed).toBe(false); });
 it("rejects another disconnect without independent server activity", () => { const o = reconnect(); o.state.remainingPhase.runningWhileOffline = false; expect(gradeBrowserReconnect(o).passed).toBe(false); });
+it("rejects document title matches without actual latest-revision run provenance", () => { const o = reconnect(); o.revisions[0]!.createdByRunId = "other-run"; expect(gradeBrowserReconnect(o).passed).toBe(false); });
+it("rejects reporting a genuinely failed offline attempt as completed", () => { const o = reconnect(); o.state.failedReconnectPhase.reply = "That attempt has finished successfully."; expect(gradeBrowserReconnect(o).passed).toBe(false); });
+it("rejects cancellation substituted for a genuine native failed phase", () => { const o = reconnect(); o.runs.find(run => run.id === "real-failure")!.status = "cancelled"; expect(gradeBrowserReconnect(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-history.ts

@@ -5,8 +5,9 @@ import { sendChatMessage } from "../chat-flow.js";
 import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { checks, type Entity } from "./internal-support.js";
 import { openStandingChat } from "./identity-support.js";
-import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
+import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js";
 import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
+import { unavailableNativeModelFailure } from "./runtime-native-failure.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 type Artifact = Awaited<ReturnType<typeof readRegisteredArtifacts>>[number];
 
@@ -73,22 +74,31 @@ export function gradeBrowserReconnect(value: unknown) {
   "The actual existing worker completes on the server while the browser is disconnected.");
   c.add("reconnected-persisted-state", o.state.reconnected === true && o.state.historyRetained === true
     && o.state.offlineCommentsRetained === true && current?.status === "done" && workerRun?.status === "succeeded"
-    && o.documents.some(row => row.createdByRunId === workerRun.id || /brief|website/i.test(row.title ?? "")),
+    && o.documents.some(doc => o.revisions.some(revision => revision.id === doc.latestRevisionId && revision.documentId === doc.id
+      && revision.createdByRunId === workerRun.id && revision.createdByAgentId === (o.state.worker as Entity).id)),
   "Reloaded UI and public reads recover the persisted conversation, real task completion and delivered document.");
   c.add("honest-canonical-status", /done|complete|finished/i.test(reply) && !/still (?:working|running)|restart(?:ed|ing).*disconnect/i.test(reply)
     && (reply.includes(task.identifier ?? task.id) || /website/i.test(reply)),
   "The canonical reply reports actual completion rather than inferring a restart from reconnecting.");
-  const remaining = o.state.remainingPhase as { attributed?: boolean; reply?: string; taskId?: string; runId?: string; offlineObserved?: boolean; runningWhileOffline?: boolean; reconnected?: boolean } | undefined;
+  const remaining = o.state.remainingPhase as { attributed?: boolean; reply?: string; taskId?: string; runId?: string; offlineObserved?: boolean; runningWhileOffline?: boolean; reconnected?: boolean; runStatusAtReply?: string; taskStatusAtReply?: string } | undefined;
   const active = o.runs.find(run => run.id === remaining?.runId); const remainingTask = o.tasks.find(row => row.id === remaining?.taskId);
   c.add("remaining-work-reported-after-second-disconnect", remaining?.attributed === true && remaining.offlineObserved === true
-    && remaining.runningWhileOffline === true && remaining.reconnected === true && active?.status === "running"
-    && remainingTask?.status !== "done" && /running|working|waiting|not (?:yet )?(?:done|finished|complete)|still|in progress/i.test(remaining.reply ?? "")
+    && remaining.runningWhileOffline === true && remaining.reconnected === true && !!active && !!remainingTask
+    && remaining.runStatusAtReply === "running" && typeof remaining.taskStatusAtReply === "string" && remaining.taskStatusAtReply !== "done"
+    && /running|working|waiting|not (?:yet )?(?:done|finished|complete)|still|in progress/i.test(remaining.reply ?? "")
     && !/(?:is|has|have) (?:now )?(?:finished|completed)|restarted.*disconnect/i.test(remaining.reply ?? ""),
   "Another genuine worker phase remains active through a second disconnect; only that phase's attributed answer reports the actual remaining work.");
-  c.add("no-duplicate-send-or-worker", o.runs.length === 5 && o.runs.filter(run => run.agentId === (o.state.worker as Entity).id).length === 2
-    && o.humanQueries.length === 3 && o.tasks.length === 2 && new Set(o.tasks.map(row => row.id)).size === 2
+  const failed = o.state.failedReconnectPhase as { attributed?: boolean; reply?: string; runId?: string; taskId?: string; offlineObserved?: boolean; failedWhileOffline?: boolean; reconnected?: boolean } | undefined;
+  const failedRun = o.runs.find(run => run.id === failed?.runId);
+  c.add("genuine-failed-run-not-promoted-to-success", failed?.attributed === true && failed.offlineObserved === true && failed.failedWhileOffline === true
+    && failed.reconnected === true && failedRun?.runtimeMode === "native" && failedRun.status === "failed" && !!failedRun.errorCode
+    && /fail|error|reject|could not|couldn't|did not finish/i.test(failed.reply ?? "")
+    && !/(?:is|has) (?:now )?(?:finished|completed|succeeded)|restarted.*disconnect/i.test(failed.reply ?? ""),
+  "A third genuine native failure occurs while offline; its own phase-attributed answer retains that failure instead of inferring success from missed events.");
+  c.add("no-duplicate-send-or-worker", o.runs.length === 7 && o.runs.filter(run => run.agentId === (o.state.worker as Entity).id).length === 2
+    && o.humanQueries.length === 4 && o.tasks.length === 3 && new Set(o.tasks.map(row => row.id)).size === 3
     && o.tasks.some(row => row.id === task.id) && !!remaining?.taskId && remaining.taskId !== task.id,
-  "The two genuine worker tasks and three explicit source turns remain the only work; reconnecting neither replays sends nor restarts either worker.");
+  "The three genuine worker tasks and four explicit source turns remain the only work; reconnecting neither replays sends nor restarts a worker.");
   return c.result();
 }
 
@@ -155,10 +165,9 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
     await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
   }
 }, gradeBrowserReconnect, {
-  sourceFiles: ["chat-catalog-domains/runtime-history.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/recovery.yaml"],
-  setupHumanMessageCount: 1, expectedHumanMessageCount: 3, expectedRunCount: 5, attemptTimeoutMs: 900_000,
-  coverageGaps: ["Reconnect after a genuinely failed native run remains unexercised; successful and still-running native phases plus failed-evidence negative controls are authored."],
-  runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 },
+  sourceFiles: ["chat-catalog-domains/runtime-history.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/recovery.yaml"],
+  setupHumanMessageCount: 1, expectedHumanMessageCount: 4, expectedRunCount: 7, attemptTimeoutMs: 1_200_000,
+  runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 6, failed: 1 } },
   async act(ctx, state) {
     const canonical = await sourceSettled(ctx, 3);
     state.canonicalRunIds = canonical.runs.filter(run => !(state.setupRunIds as string[]).includes(run.id)).map(run => run.id);
@@ -179,8 +188,37 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
     const prior = (await ctx.allRuns()).map(run => run.id);
     const message = `I was disconnected again while Morgan worked on the new Website task ${next.task.identifier ?? next.task.id}. Is that follow-up website finished? Please report its actual current state; do not start another task or interrupt it.`;
     await sendChatMessage(ctx.input.page, message); await sourceSettled(ctx, 5);
+    const remainingRun = (await ctx.allRuns()).find(run => run.id === next.workerRunId);
+    const remainingTask = await ctx.input.api.get<Entity>(`/api/issues/${next.task.id}`);
     state.remainingPhase = { ...await attributedPhase(ctx, prior, message), taskId: next.task.id, runId: next.workerRunId,
-      offlineObserved, runningWhileOffline, reconnected: await ctx.input.page.evaluate(() => navigator.onLine) };
+      offlineObserved, runningWhileOffline, reconnected: await ctx.input.page.evaluate(() => navigator.onLine),
+      runStatusAtReply: remainingRun?.status, taskStatusAtReply: remainingTask.status };
+    await releaseRuntimeWorker(next);
+    await pollUntil({ label: "Remaining phase settles before separate failed-native phase", deadlineAt: Date.now() + 180_000,
+      load: () => ctx.allRuns(), accept: rows => rows.length === 5 && rows.every(run => run.status === "succeeded") });
+    const failureWorker = await runtimeWorker(ctx, "Riley"); let failedTask: Entity | undefined;
+    let failedOffline = false, failure: Awaited<ReturnType<typeof unavailableNativeModelFailure>> | undefined;
+    await ctx.input.page.context().setOffline(true);
+    try {
+      failedOffline = await ctx.input.page.evaluate(() => navigator.onLine === false);
+      failure = await unavailableNativeModelFailure(ctx, failureWorker, async () => {
+        failedTask = await ctx.input.api.post<Entity>(`/api/companies/${ctx.input.fixtures.company.id}/issues`, {
+          title: "Website configuration-failure attempt", status: "todo", assigneeAgentId: failureWorker.id,
+          description: "Draft a single-page website and save a task document. Do not create another task or external side effect." });
+        return { issueId: failedTask.id };
+      });
+    } finally {
+      await ctx.input.page.context().setOffline(false);
+      await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
+      await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
+    }
+    if (!failure || !failedTask) throw new Error("The offline fixture produced no genuine native failure");
+    const failureBoundary = (await ctx.allRuns()).map(run => run.id);
+    const failedMessage = `I was disconnected during Riley's Website configuration-failure attempt ${failedTask!.identifier ?? failedTask!.id}. Did that attempt finish? Read its actual retained run state and explain it without starting another attempt.`;
+    await sendChatMessage(ctx.input.page, failedMessage); await sourceSettled(ctx, 7);
+    state.failedReconnectEvidence = failure;
+    state.failedReconnectPhase = { ...await attributedPhase(ctx, failureBoundary, failedMessage), runId: failure.failedRun.id, taskId: failedTask!.id,
+      offlineObserved: failedOffline, failedWhileOffline: failure.failedRun.status === "failed", reconnected: await ctx.input.page.evaluate(() => navigator.onLine) };
   },
   async observe(ctx, state) {
     state.reconnected = await ctx.input.page.evaluate(() => navigator.onLine);

tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts

@@ -31,6 +31,9 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor
     || original.adapterType !== "paperclip_runner" || original.adapterConfig?.provider !== "codex")
     throw new Error("Native failure prerequisite requires an actually ready native Codex worker");
   const before = await ctx.allRuns(); const model = `paperclip-eval-unavailable-${ctx.input.nonce}`;
+  for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
+    await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true,
+      isActive: true, unpricedUsagePolicy: "allow" });
   let failed: NativeFailureRun | undefined;
   try {
     const configured = await ctx.input.api.patch<RuntimeAgent>(endpoint, { adapterConfig: { ...original.adapterConfig, model } });