From 6e41ca33d84edd334a2305f06bbcc57dd1ce6315 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 20:24:50 -0500 Subject: [PATCH] test(evals): cover disabled chat and actual failed native history phases --- .../runtime-capability-audit.json | 12 +-- .../runtime-chat-disabled.test.ts | 11 +++ .../runtime-chat-disabled.ts | 80 +++++++++++++++++++ .../runtime-failed-runs.test.ts | 17 ++++ .../runtime-failed-runs.ts | 73 +++++++++++++++++ .../runtime-history.test.ts | 14 +++- .../chat-catalog-domains/runtime-history.ts | 64 ++++++++++++--- .../runtime-native-failure.ts | 3 + 8 files changed, 251 insertions(+), 23 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json index 77b05a9927f..3bff465a305 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -111,8 +111,8 @@ }, { "scenarioId": "agent-chat.chat-disabled", - "readinessClass": "boundary", - "needs": "Actual disabled Agent Chat context needs a narrowly allowlisted unavailable-surface browser-boundary case; ordinary Agent Chat cannot send its canonical query. No conference-only unrelated-gate substitute." + "readinessClass": "implemented", + "needs": "Actual allowlisted authenticated-harness boundary; disable Agent Chat itself; real Morgan route unavailable/no composer, agent settings denial, create-chat 404, usable Morgan profile and zero tasks/chats/native admissions." }, { "scenarioId": "runs-adapters.list-model-options", @@ -126,8 +126,8 @@ }, { "scenarioId": "runs-adapters.inspect-failed-run", - "readinessClass": "local", - "needs": "Generate real failed website run and later successful one; retain exact failure diagnostics and missing cost/log branches." + "readinessClass": "implemented", + "needs": "Public unavailable-model configuration must actually admit one native failed run with model/task/actor/terminal-event evidence; restore exact configuration, one actual human Retry, later successful HTML artifact, retained failure/source links and truthful missing logs/cost." }, { "scenarioId": "runs-adapters.reset-session", @@ -201,8 +201,8 @@ }, { "scenarioId": "recovery.browser-reconnect", - "readinessClass": "partial", - "needs": "Real browser offline/server completion plus second actual still-running phase; phase-bound status replies and no duplicate sends/runs. Failed-native-run reconnect remains a concrete coverage gap; no live runtime attempts yet." + "readinessClass": "implemented", + "needs": "Three executable real offline phases: native completion, independently observed still-running work, and mandatory actually admitted failed native attempt under explicit unavailable-model configuration; each answer bound to its own persisted human/native run. No live qualification claimed." }, { "scenarioId": "steering.queue-follow-up", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts new file mode 100644 index 00000000000..80b8bf6c295 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts @@ -0,0 +1,11 @@ +import { it, expect } from "vitest"; +import { gradeDisabledChat } from "./runtime-chat-disabled.js"; +function valid() { return { companyId: "company", agentId: "morgan", personId: "actual-human", canonicalIntent: "Open a chat with Morgan.", agentChatEnabled: false, + unavailableVisible: true, composerAbsent: true, supportedOperatorStepVisible: true, profileAccessible: true, agentSettingsStatus: 403, + createChatStatus: 404, gateAfterProbe: false, beforeTaskIds: [], afterTaskIds: [] as string[], chatIds: [] as string[], runIds: [] as string[] }; } +it("accepts the actual unavailable Morgan Chat surface with no admission", () => expect(gradeDisabledChat(valid()).passed).toBe(true)); +it("rejects an unrelated disabled conference gate while Agent Chat is enabled", () => { const o = valid(); o.agentChatEnabled = true; expect(gradeDisabledChat(o).passed).toBe(false); }); +it("rejects missing canonical unavailable context", () => expect(gradeDisabledChat(undefined).passed).toBe(false)); +it("rejects a disguised task created while Chat is disabled", () => { const o = valid(); o.afterTaskIds = ["workaround"]; expect(gradeDisabledChat(o).passed).toBe(false); }); +it("rejects an ordinary agent enabling the instance setting", () => { const o = valid(); o.agentSettingsStatus = 200; o.gateAfterProbe = true; expect(gradeDisabledChat(o).passed).toBe(false); }); +it("rejects native work started by unavailable navigation", () => { const o = valid(); o.runIds = ["unrequested"]; expect(gradeDisabledChat(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts new file mode 100644 index 00000000000..d5fa721162c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts @@ -0,0 +1,80 @@ +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { RunnerApi } from "../api.js"; +import { setupLiveFixtures, type LiveFixtureValues } from "../live-fixtures.js"; +import { createAuthenticatedPerson } from "./identity-auth.js"; +import { checks } from "./internal-support.js"; +import type { IdentityBoundaryCase, IdentityBoundaryContext } from "./identity-boundary-contract.js"; +type Person = Awaited>; +const owned = new WeakMap(); +interface DisabledObservation { companyId: string; agentId: string; personId: string; canonicalIntent: string; agentChatEnabled: boolean; + unavailableVisible: boolean; composerAbsent: boolean; supportedOperatorStepVisible: boolean; profileAccessible: boolean; + agentSettingsStatus: number; createChatStatus: number; gateAfterProbe: boolean; beforeTaskIds: string[]; afterTaskIds: string[]; chatIds: string[]; runIds: string[] } +export function gradeDisabledChat(value: unknown) { + const c = checks(); const o = value as DisabledObservation | undefined; + c.add("actual-disabled-canonical-context", !!o && !!o.companyId && !!o.agentId && !!o.personId && o.personId !== "local-board" + && o.canonicalIntent === "Open a chat with Morgan." && o.agentChatEnabled === false && o.unavailableVisible && o.composerAbsent, + "The real signed-in human opens the intended Morgan Agent Chat target with its actual gate disabled; canonical intent remains unsent."); + c.add("clear-supported-human-path", !!o && o.supportedOperatorStepVisible && o.profileAccessible, + "The unavailable surface identifies Experimental settings as the human management step, and the actual Morgan profile remains usable."); + c.add("ordinary-agent-cannot-change-instance-gate", !!o && o.agentSettingsStatus === 403 && o.gateAfterProbe === false + && o.createChatStatus === 404, "The real ordinary agent cannot enable instance Chat, and the public create-chat path enforces the same disabled gate."); + c.add("no-disguised-task-or-admission", !!o && Array.isArray(o.beforeTaskIds) && Array.isArray(o.afterTaskIds) + && JSON.stringify(o.beforeTaskIds) === JSON.stringify(o.afterTaskIds) && Array.isArray(o.chatIds) && !o.chatIds.length + && Array.isArray(o.runIds) && !o.runIds.length, + "Unavailable navigation admits no native work, duplicate standing chat or silently created task."); + return c.result(); +} +export const runtimeDisabledChatCases: readonly IdentityBoundaryCase[] = [{ + scenarioId: "agent-chat.chat-disabled", journeyKind: "browser-boundary", boundaryTarget: "authenticated-harness", authentication: "human-session", + oracleVersion: "paperclip.chat-catalog.runtime.v2", expectedRunCount: 0, expectedHumanMessageCount: 0, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000, + sourceFiles: ["chat-catalog-domains/runtime-chat-disabled.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "live-fixtures.ts", "chat-scenarios/agent-chat.yaml"], + grade: gradeDisabledChat, + async run(ctx) { + const browser = ctx.page.context().browser(); if (!browser) throw new Error("Unavailable Agent Chat boundary requires the real human browser"); + const person = await createAuthenticatedPerson({ api: ctx.api, browser, secrets: ctx.privateSecrets, name: "Alex" }); + const api = new RunnerApi(person.context.request); person.bindApiSession(api); const resource: { person: Person; api: RunnerApi; companyId?: string; fixtures?: LiveFixtureValues; keyId?: string } = { person, api }; owned.set(ctx, resource); + await api.post("/api/bootstrap/claim"); + const workspacePath = path.join(ctx.privateDir, "disabled-chat-owned-workspace"); await mkdir(workspacePath, { recursive: true, mode: 0o700 }); + const fixtures = await setupLiveFixtures({ api, execution: ctx.execution, executionNonce: ctx.nonce, workspacePath, credentials: ctx.credentials, + async onCompanyCreated(id) { resource.companyId = id; ctx.observeResources({ companyId: id, runIds: [] }); } }); + resource.fixtures = fixtures; + await api.patch(`/api/agents/${fixtures.agent.id}`, { name: "Morgan" }); + await api.patch("/api/instance/settings/experimental", { enableAgentChat: false }); + const settings = await api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental"); + const root = `/api/companies/${fixtures.company.id}`; + const beforeTaskIds = (await api.get>(`${root}/issues?includeAgentCreatedTasks=true&limit=1000`)).map(row => row.id).sort(); + const page = await person.context.newPage(); + await page.goto(`/${fixtures.company.issuePrefix}/chats/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" }); + await expect(page.getByText(/Agent Chat is disabled\. Enable it in Experimental settings/)).toBeVisible(); + await expect(page.getByTestId("task-chat-composer-input")).toHaveCount(0); + const unavailableVisible = await page.getByText(/Agent Chat is disabled/).isVisible(); + const supportedOperatorStepVisible = await page.getByText(/Enable it in Experimental settings/).isVisible(); + await ctx.capture("disabled-agent-chat", "Morgan Chat is genuinely unavailable; canonical intent remains unsent", "runtime-disabled-agent-chat.png"); + const key = await api.postSensitive<{ id: string; token: string }>(`/api/agents/${fixtures.agent.id}/keys`, { name: "disabled-chat-permission-probe" }); + ctx.privateSecrets.push(key.token); resource.keyId = key.id; + const denied = await fetch(new URL("/api/instance/settings/experimental", api.baseURL), { method: "PATCH", redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { Authorization: `Bearer ${key.token}`, "content-type": "application/json" }, body: JSON.stringify({ enableAgentChat: true }) }); + const agentSettingsStatus = denied.status; await denied.body?.cancel(); + const rejected = await api.request.post(`${root}/chats/${fixtures.agent.id}`, { data: {} }); const createChatStatus = rejected.status(); await rejected.dispose(); + const gateAfterProbe = (await api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental")).enableAgentChat; + await page.goto(`/${fixtures.company.issuePrefix}/agents/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" }); + await page.reload({ waitUntil: "domcontentloaded" }); await expect(page.getByRole("heading", { name: "Morgan", exact: true }).first()).toBeVisible(); + const tasks = await api.get>(`${root}/issues?includeAgentCreatedTasks=true&limit=1000`); + const runIds = (await api.get>(`${root}/heartbeat-runs?limit=100`)).map(row => row.id); + ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds }); + const observation: DisabledObservation = { companyId: fixtures.company.id, agentId: fixtures.agent.id, personId: person.id, canonicalIntent: ctx.scenario.query, + agentChatEnabled: settings.enableAgentChat, unavailableVisible, composerAbsent: true, supportedOperatorStepVisible, profileAccessible: true, + agentSettingsStatus, createChatStatus, gateAfterProbe, beforeTaskIds, afterTaskIds: tasks.map(row => row.id).sort(), + chatIds: tasks.filter(row => row.conversationAgentId).map(row => row.id), runIds }; + await ctx.evidence("runtime-disabled-chat-boundary.json", observation); return observation; + }, + async cleanup(ctx) { + const resource = owned.get(ctx); if (!resource) return; + try { if (resource.keyId && resource.fixtures) await resource.api.delete(`/api/agents/${resource.fixtures.agent.id}/keys/${resource.keyId}`, { allowNotFound: true }); + await resource.fixtures?.teardown(); + if (resource.companyId) await resource.api.delete(`/api/companies/${resource.companyId}`, { allowNotFound: true }); + } finally { await resource.person.dispose(); owned.delete(ctx); } + }, +}]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts new file mode 100644 index 00000000000..3876b29f230 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts @@ -0,0 +1,17 @@ +import { it, expect } from "vitest"; +import { gradeFailedAttempt } from "./runtime-failed-runs.js"; +function valid() { const failedRun = { id: "failed-model", status: "failed", errorCode: "native_provider_model_rejected", error: "Selected model unavailable", costUsd: null as number | null, logRef: "actual-log" as string | null }; + return { scenarioId: "runs-adapters.inspect-failed-run", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { failedRun, retainedFailure: { ...failedRun }, successfulRun: { id: "actual-success", status: "succeeded" }, successArtifactVerified: true, task: { id: "website" }, failureLogAvailable: true }, + agents: [], tasks: [{ id: "website" }], activity: [], humanQueries: [], links: ["/ACORN/agents/morgan/runs/failed-model"], + reply: "The failed-model attempt failed because its selected model was rejected by the provider. The later attempt succeeded; cost data for the failure is unpriced and unavailable.", + runs: [{ id: "failed-model", status: "failed" }, { id: "actual-success", status: "succeeded" }, { id: "answer", status: "succeeded" }] }; } +it("accepts distinct retained native failure and later success with honest unavailable billing", () => expect(gradeFailedAttempt(valid()).passed).toBe(true)); +it("rejects absent genuine failure evidence", () => expect(gradeFailedAttempt(undefined).passed).toBe(false)); +it("rejects erasing the failure after later success", () => { const o = valid(); o.state.retainedFailure.status = "succeeded"; expect(gradeFailedAttempt(o).passed).toBe(false); }); +it("rejects explaining or linking a similarly named successful attempt", () => { const o = valid(); o.links = ["/ACORN/agents/morgan/runs/actual-success"]; expect(gradeFailedAttempt(o).passed).toBe(false); }); +it("rejects treating unavailable billing as zero", () => { const o = valid(); o.reply += " The cost was $0.00."; expect(gradeFailedAttempt(o).passed).toBe(false); }); +it("requires a missing output disclosure when logs are unavailable", () => { const o = valid(); o.state.failedRun.logRef = null; o.state.failureLogAvailable = false; + expect(gradeFailedAttempt(o).passed).toBe(false); o.reply += " Retained log output is unavailable."; expect(gradeFailedAttempt(o).passed).toBe(true); }); +it("accepts an actually recorded zero cost without calling it unavailable", () => { const o = valid(); o.state.failedRun.costUsd = 0; o.reply = "The failed-model run failed with a provider model error. The recorded cost was $0.00."; expect(gradeFailedAttempt(o).passed).toBe(true); }); +it("rejects invented successful output without verified saved bytes", () => { const o = valid(); o.state.successArtifactVerified = false; expect(gradeFailedAttempt(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts new file mode 100644 index 00000000000..ab129b9026e --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts @@ -0,0 +1,73 @@ +import { expect } from "@playwright/test"; +import { pollUntil } from "../api.js"; +import { readRegisteredArtifacts } from "../registered-artifact.js"; +import { checks, type Entity } from "./internal-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import { unavailableNativeModelFailure } from "./runtime-native-failure.js"; +import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; +export function gradeFailedAttempt(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("actual-failed-and-successful-native-history", !!o, "An actually admitted native failure and a later real successful execution must remain separately readable."); if (!o) return c.result(); + const failed = o.state.failedRun as HeartbeatRun | undefined, retained = o.state.retainedFailure as HeartbeatRun | undefined; + const success = o.state.successfulRun as HeartbeatRun | undefined; + c.add("original-failure-not-erased-by-success", !!failed && !!retained && failed.id === retained.id && failed.status === "failed" + && retained.status === "failed" && retained.errorCode === failed.errorCode && retained.error === failed.error && !!failed.errorCode + && !!success && success.status === "succeeded" && success.id !== failed.id && o.state.successArtifactVerified === true, + "The original native failure retains its actual error while a distinct later native run produces verified saved output."); + c.add("correct-attempt-and-observed-error", !!failed && o.reply.includes(failed.id) && /fail|reject|error|could not|couldn't/i.test(o.reply) + && /model|configuration|provider|runtime/i.test(o.reply) && !/failed (?:attempt|run).*(?:was|is) (?:actually )?success/i.test(o.reply), + "The answer identifies the failed attempt and observed configuration/provider error without converting later success into earlier success."); + c.add("real-run-source-link", !!failed && o.links.some(href => { try { const url = new URL(href, o.origin); + return url.origin === o.origin && url.pathname.split("/").map(decodeURIComponent).includes(failed.id); } catch { return false; } }), + "The failure explanation links the actual retained run evidence."); + const missingCost = failed?.costUsd == null; const missingLog = !failed?.logRef || o.state.failureLogAvailable === false; + c.add("unavailable-data-not-zero", (!missingCost || /cost|usage|token/i.test(o.reply) && /unavailable|unknown|not (?:available|recorded)|no (?:reported|recorded|priced)|unpriced/i.test(o.reply)) + && (!missingLog || /log|output/i.test(o.reply) && /unavailable|not (?:available|recorded)|no (?:retained|recorded)|missing/i.test(o.reply)) + && !(missingCost && /\$0(?:\.00)?\b|zero (?:cost|dollars)|cost (?:was|is) 0/i.test(o.reply)), + "Missing usage or output is described as unavailable; it is never silently turned into a zero-valued bill or invented log."); + c.add("no-new-work-to-explain-history", o.tasks.length === 1 && o.tasks[0]?.id === (o.state.task as Entity).id + && o.runs.length === 3 && o.runs.filter(run => run.status === "failed").length === 1 && o.runs.filter(run => run.status === "succeeded").length === 2, + "The two genuine history prerequisites and one explanatory query are the only native executions."); + return c.result(); +} +export const runtimeFailedRunCases = [runtimeDefinition("runs-adapters.inspect-failed-run", async ctx => { + const worker = await runtimeWorker(ctx); let task: Entity | undefined; + const failure = await unavailableNativeModelFailure(ctx, worker, async () => { + task = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, + description: `Create a small semantic HTML launch website with headline Acorn Notes and reference ${ctx.input.nonce}. Save it as website.html and register that actual file as a downloadable artifact work product titled website.html. Verify it and complete this task normally. No deployment, email, new task or remote operation is authorized.` }); + return { issueId: task.id }; + }); + if (!task) throw new Error("Actual failed native task was not created"); + const source = ctx.input.page.url(); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); + const retry = ctx.input.page.getByTestId("task-chat-run-failed-try-again").last(); await expect(retry).toBeVisible(); + await retry.click(); + const done = await pollUntil({ label: "Real successful task retry after restoring the admitted model", deadlineAt: Date.now() + 240_000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task!.id}`), runs: await ctx.allRuns() }), + accept: value => value.task.status === "done" && value.runs.length === 2 && value.runs.some(run => run.id !== failure.failedRun.id && run.status === "succeeded"), + reject: value => value.runs.length > 2 || value.runs.some(run => run.id !== failure.failedRun.id && ["failed", "cancelled", "timed_out"].includes(run.status)) ? "The one declared successful history retry did not succeed" : undefined }); + const successfulRun = done.runs.find(run => run.id !== failure.failedRun.id)!; + const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, successfulRun.id, ["website.html"]); + if (!artifact?.contentVerified || !artifact.content.includes(ctx.input.nonce) || !/(`/api/heartbeat-runs/${failed.id}`); + if (actual.id !== failed.id || actual.status !== "failed") throw new Error("Failure result UI did not retain the intended failed attempt"); + await ctx.input.evidence("runtime-retained-failed-attempt-ui.json", { runId: failed.id, errorCode: actual.errorCode, taskId: (state.task as Entity).id, url: ctx.input.page.url() }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts index b085295d80a..0c464656eab 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts @@ -23,10 +23,13 @@ function reconnect() { workerWasReallyWaiting: true, offlineObserved: true, workerStatusBeforeDisconnect: "running", workerStatusWhileOffline: "succeeded", taskStatusWhileOffline: "done", serverAdvancedWhileOffline: true, reconnected: true, historyRetained: true, offlineCommentsRetained: true, queryReply: "The website ACORN-1 finished while you were away.", - remainingPhase: { attributed: true, reply: "That follow-up is still running and waiting for the brief.", taskId: "followup", runId: "remaining", offlineObserved: true, runningWhileOffline: true, reconnected: true } }, - agents: [], tasks: [{ id: "website", status: "done" }, { id: "followup", status: "in_progress" }], documents: [{ id: "delivery", title: "Website delivery", createdByRunId: "worker" }], - links: [], activity: [], reply: "", humanQueries: ["Track this", "Did you finish while I was disconnected?", "Is the follow-up finished?"], - runs: [{ id: "worker", agentId: "morgan", status: "succeeded" }, { id: "prior", agentId: "ceo", status: "succeeded" }, { id: "query", agentId: "ceo", status: "succeeded" }, { id: "remaining", agentId: "morgan", status: "running" }, { id: "status", agentId: "ceo", status: "succeeded" }] }; + remainingPhase: { attributed: true, reply: "That follow-up is still running and waiting for the brief.", taskId: "followup", runId: "remaining", offlineObserved: true, runningWhileOffline: true, reconnected: true, runStatusAtReply: "running", taskStatusAtReply: "in_progress" }, + failedReconnectPhase: { attributed: true, reply: "Riley's attempt failed with a provider model configuration error.", runId: "real-failure", taskId: "failed-task", offlineObserved: true, failedWhileOffline: true, reconnected: true } }, + agents: [], tasks: [{ id: "website", status: "done" }, { id: "followup", status: "done" }, { id: "failed-task", status: "blocked" }], documents: [{ id: "delivery", title: "Website delivery", latestRevisionId: "saved" }], + revisions: [{ id: "saved", documentId: "delivery", createdByRunId: "worker", createdByAgentId: "morgan" }], + links: [], activity: [], reply: "", humanQueries: ["Track this", "Did you finish while I was disconnected?", "Is the follow-up finished?", "Did Riley finish?"], + runs: [{ id: "worker", agentId: "morgan", status: "succeeded" }, { id: "prior", agentId: "ceo", status: "succeeded" }, { id: "query", agentId: "ceo", status: "succeeded" }, { id: "remaining", agentId: "morgan", status: "succeeded" }, { id: "status", agentId: "ceo", status: "succeeded" }, + { id: "real-failure", agentId: "riley", status: "failed", runtimeMode: "native", errorCode: "native_provider_model_rejected" }, { id: "failed-status", agentId: "ceo", status: "succeeded" }] }; } it("accepts actual server completion through a real browser disconnect", () => expect(gradeBrowserReconnect(reconnect()).passed).toBe(true)); it("rejects missing disconnect evidence", () => expect(gradeBrowserReconnect(undefined).passed).toBe(false)); @@ -37,3 +40,6 @@ it("rejects losing messages whose live stream events were missed", () => { const it("rejects a status claim only present in the old phase", () => { const o = reconnect(); o.reply = o.state.queryReply; o.state.queryReply = "Hello."; expect(gradeBrowserReconnect(o).passed).toBe(false); }); it("rejects a completion claim for genuinely remaining work", () => { const o = reconnect(); o.state.remainingPhase.reply = "It has finished."; expect(gradeBrowserReconnect(o).passed).toBe(false); }); it("rejects another disconnect without independent server activity", () => { const o = reconnect(); o.state.remainingPhase.runningWhileOffline = false; expect(gradeBrowserReconnect(o).passed).toBe(false); }); +it("rejects document title matches without actual latest-revision run provenance", () => { const o = reconnect(); o.revisions[0]!.createdByRunId = "other-run"; expect(gradeBrowserReconnect(o).passed).toBe(false); }); +it("rejects reporting a genuinely failed offline attempt as completed", () => { const o = reconnect(); o.state.failedReconnectPhase.reply = "That attempt has finished successfully."; expect(gradeBrowserReconnect(o).passed).toBe(false); }); +it("rejects cancellation substituted for a genuine native failed phase", () => { const o = reconnect(); o.runs.find(run => run.id === "real-failure")!.status = "cancelled"; expect(gradeBrowserReconnect(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-history.ts b/tests/runner-e2e/chat-catalog-domains/runtime-history.ts index a8606a60125..986c840ff20 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-history.ts @@ -5,8 +5,9 @@ import { sendChatMessage } from "../chat-flow.js"; import { readRegisteredArtifacts } from "../registered-artifact.js"; import { checks, type Entity } from "./internal-support.js"; import { openStandingChat } from "./identity-support.js"; -import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { unavailableNativeModelFailure } from "./runtime-native-failure.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; type Artifact = Awaited>[number]; @@ -73,22 +74,31 @@ export function gradeBrowserReconnect(value: unknown) { "The actual existing worker completes on the server while the browser is disconnected."); c.add("reconnected-persisted-state", o.state.reconnected === true && o.state.historyRetained === true && o.state.offlineCommentsRetained === true && current?.status === "done" && workerRun?.status === "succeeded" - && o.documents.some(row => row.createdByRunId === workerRun.id || /brief|website/i.test(row.title ?? "")), + && o.documents.some(doc => o.revisions.some(revision => revision.id === doc.latestRevisionId && revision.documentId === doc.id + && revision.createdByRunId === workerRun.id && revision.createdByAgentId === (o.state.worker as Entity).id)), "Reloaded UI and public reads recover the persisted conversation, real task completion and delivered document."); c.add("honest-canonical-status", /done|complete|finished/i.test(reply) && !/still (?:working|running)|restart(?:ed|ing).*disconnect/i.test(reply) && (reply.includes(task.identifier ?? task.id) || /website/i.test(reply)), "The canonical reply reports actual completion rather than inferring a restart from reconnecting."); - const remaining = o.state.remainingPhase as { attributed?: boolean; reply?: string; taskId?: string; runId?: string; offlineObserved?: boolean; runningWhileOffline?: boolean; reconnected?: boolean } | undefined; + const remaining = o.state.remainingPhase as { attributed?: boolean; reply?: string; taskId?: string; runId?: string; offlineObserved?: boolean; runningWhileOffline?: boolean; reconnected?: boolean; runStatusAtReply?: string; taskStatusAtReply?: string } | undefined; const active = o.runs.find(run => run.id === remaining?.runId); const remainingTask = o.tasks.find(row => row.id === remaining?.taskId); c.add("remaining-work-reported-after-second-disconnect", remaining?.attributed === true && remaining.offlineObserved === true - && remaining.runningWhileOffline === true && remaining.reconnected === true && active?.status === "running" - && remainingTask?.status !== "done" && /running|working|waiting|not (?:yet )?(?:done|finished|complete)|still|in progress/i.test(remaining.reply ?? "") + && remaining.runningWhileOffline === true && remaining.reconnected === true && !!active && !!remainingTask + && remaining.runStatusAtReply === "running" && typeof remaining.taskStatusAtReply === "string" && remaining.taskStatusAtReply !== "done" + && /running|working|waiting|not (?:yet )?(?:done|finished|complete)|still|in progress/i.test(remaining.reply ?? "") && !/(?:is|has|have) (?:now )?(?:finished|completed)|restarted.*disconnect/i.test(remaining.reply ?? ""), "Another genuine worker phase remains active through a second disconnect; only that phase's attributed answer reports the actual remaining work."); - c.add("no-duplicate-send-or-worker", o.runs.length === 5 && o.runs.filter(run => run.agentId === (o.state.worker as Entity).id).length === 2 - && o.humanQueries.length === 3 && o.tasks.length === 2 && new Set(o.tasks.map(row => row.id)).size === 2 + const failed = o.state.failedReconnectPhase as { attributed?: boolean; reply?: string; runId?: string; taskId?: string; offlineObserved?: boolean; failedWhileOffline?: boolean; reconnected?: boolean } | undefined; + const failedRun = o.runs.find(run => run.id === failed?.runId); + c.add("genuine-failed-run-not-promoted-to-success", failed?.attributed === true && failed.offlineObserved === true && failed.failedWhileOffline === true + && failed.reconnected === true && failedRun?.runtimeMode === "native" && failedRun.status === "failed" && !!failedRun.errorCode + && /fail|error|reject|could not|couldn't|did not finish/i.test(failed.reply ?? "") + && !/(?:is|has) (?:now )?(?:finished|completed|succeeded)|restarted.*disconnect/i.test(failed.reply ?? ""), + "A third genuine native failure occurs while offline; its own phase-attributed answer retains that failure instead of inferring success from missed events."); + c.add("no-duplicate-send-or-worker", o.runs.length === 7 && o.runs.filter(run => run.agentId === (o.state.worker as Entity).id).length === 2 + && o.humanQueries.length === 4 && o.tasks.length === 3 && new Set(o.tasks.map(row => row.id)).size === 3 && o.tasks.some(row => row.id === task.id) && !!remaining?.taskId && remaining.taskId !== task.id, - "The two genuine worker tasks and three explicit source turns remain the only work; reconnecting neither replays sends nor restarts either worker."); + "The three genuine worker tasks and four explicit source turns remain the only work; reconnecting neither replays sends nor restarts a worker."); return c.result(); } @@ -155,10 +165,9 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }, gradeBrowserReconnect, { - sourceFiles: ["chat-catalog-domains/runtime-history.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/recovery.yaml"], - setupHumanMessageCount: 1, expectedHumanMessageCount: 3, expectedRunCount: 5, attemptTimeoutMs: 900_000, - coverageGaps: ["Reconnect after a genuinely failed native run remains unexercised; successful and still-running native phases plus failed-evidence negative controls are authored."], - runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, + sourceFiles: ["chat-catalog-domains/runtime-history.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/recovery.yaml"], + setupHumanMessageCount: 1, expectedHumanMessageCount: 4, expectedRunCount: 7, attemptTimeoutMs: 1_200_000, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 6, failed: 1 } }, async act(ctx, state) { const canonical = await sourceSettled(ctx, 3); state.canonicalRunIds = canonical.runs.filter(run => !(state.setupRunIds as string[]).includes(run.id)).map(run => run.id); @@ -179,8 +188,37 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f const prior = (await ctx.allRuns()).map(run => run.id); const message = `I was disconnected again while Morgan worked on the new Website task ${next.task.identifier ?? next.task.id}. Is that follow-up website finished? Please report its actual current state; do not start another task or interrupt it.`; await sendChatMessage(ctx.input.page, message); await sourceSettled(ctx, 5); + const remainingRun = (await ctx.allRuns()).find(run => run.id === next.workerRunId); + const remainingTask = await ctx.input.api.get(`/api/issues/${next.task.id}`); state.remainingPhase = { ...await attributedPhase(ctx, prior, message), taskId: next.task.id, runId: next.workerRunId, - offlineObserved, runningWhileOffline, reconnected: await ctx.input.page.evaluate(() => navigator.onLine) }; + offlineObserved, runningWhileOffline, reconnected: await ctx.input.page.evaluate(() => navigator.onLine), + runStatusAtReply: remainingRun?.status, taskStatusAtReply: remainingTask.status }; + await releaseRuntimeWorker(next); + await pollUntil({ label: "Remaining phase settles before separate failed-native phase", deadlineAt: Date.now() + 180_000, + load: () => ctx.allRuns(), accept: rows => rows.length === 5 && rows.every(run => run.status === "succeeded") }); + const failureWorker = await runtimeWorker(ctx, "Riley"); let failedTask: Entity | undefined; + let failedOffline = false, failure: Awaited> | undefined; + await ctx.input.page.context().setOffline(true); + try { + failedOffline = await ctx.input.page.evaluate(() => navigator.onLine === false); + failure = await unavailableNativeModelFailure(ctx, failureWorker, async () => { + failedTask = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { + title: "Website configuration-failure attempt", status: "todo", assigneeAgentId: failureWorker.id, + description: "Draft a single-page website and save a task document. Do not create another task or external side effect." }); + return { issueId: failedTask.id }; + }); + } finally { + await ctx.input.page.context().setOffline(false); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + } + if (!failure || !failedTask) throw new Error("The offline fixture produced no genuine native failure"); + const failureBoundary = (await ctx.allRuns()).map(run => run.id); + const failedMessage = `I was disconnected during Riley's Website configuration-failure attempt ${failedTask!.identifier ?? failedTask!.id}. Did that attempt finish? Read its actual retained run state and explain it without starting another attempt.`; + await sendChatMessage(ctx.input.page, failedMessage); await sourceSettled(ctx, 7); + state.failedReconnectEvidence = failure; + state.failedReconnectPhase = { ...await attributedPhase(ctx, failureBoundary, failedMessage), runId: failure.failedRun.id, taskId: failedTask!.id, + offlineObserved: failedOffline, failedWhileOffline: failure.failedRun.status === "failed", reconnected: await ctx.input.page.evaluate(() => navigator.onLine) }; }, async observe(ctx, state) { state.reconnected = await ctx.input.page.evaluate(() => navigator.onLine); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts index f43c1851b79..173b505a361 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts @@ -31,6 +31,9 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor || original.adapterType !== "paperclip_runner" || original.adapterConfig?.provider !== "codex") throw new Error("Native failure prerequisite requires an actually ready native Codex worker"); const before = await ctx.allRuns(); const model = `paperclip-eval-unavailable-${ctx.input.nonce}`; + for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, + isActive: true, unpricedUsagePolicy: "allow" }); let failed: NativeFailureRun | undefined; try { const configured = await ctx.input.api.patch(endpoint, { adapterConfig: { ...original.adapterConfig, model } });