diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index f434bc8b16d..82e9ba76041 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -27,7 +27,8 @@ Agent access: [companion](../agent-access-map/recovery.md). no duplicate dispatch, and later recovery after preparation leaves the registry. - `source-action`: `recovery/source-action` — source-scoped recovery actions remain attached to the task; independent repair work may have its own issue and return owner. -- `exhaustion`: `recovery/exhaustion` — an exhausted infrastructure attempt offers ordinary Retry. +- `exhaustion`: `recovery/exhaustion` — a run stopped for a reason only a person can resolve offers ordinary Retry; + exhausted infrastructure recovery instead continues in one fresh run from saved work. Internal repair stays system-owned; genuine user decisions retain their specific action. - `workspace`: `recovery/workspace` — workspace divergence/export failures show the applicable repair, diff --git a/tests/runner-e2e/chat-catalog-contract.test.ts b/tests/runner-e2e/chat-catalog-contract.test.ts index a0e92f3f6f8..fa74dd03fd1 100644 --- a/tests/runner-e2e/chat-catalog-contract.test.ts +++ b/tests/runner-e2e/chat-catalog-contract.test.ts @@ -36,7 +36,7 @@ describe("catalog oracle contract", () => { // execution-environments.choose-capable-target: the second computer is a saved SSH target advertised for Claude Code but never tested. // questions-and-approvals.necessary-question: the answer may shape the saved deliverable as well as the brief. // tasks.assignment-conflict: the query names the Website task, since "this task" in a chat can mean the chat. - expect(chatCatalogInventoryDigest).toBe("9686d865ec65ae3c05652b7ce586286f9e7a5f79950b2dc0a3ec0f4512addccc"); + expect(chatCatalogInventoryDigest).toBe("dcda709b646b20bdd75c833468d886a111040f130e6699c94fa437ce073bf853"); for (const implementation of chatCatalogCases) expect(chatCatalogScenarios.some(row => row.id === implementation.scenarioId)).toBe(true); }); }); diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 5a5de805c1c..924891de680 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -323,9 +323,10 @@ "registered-artifact.ts", "chat-scenarios/agent-chat.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-multiple-handoffs.v4", + "oracleVersion": "paperclip.chat-catalog.runtime-multiple-handoffs.v5", "coverageGaps": [], "expectedRunCount": 5, + "minimumRunCount": 4, "setupHumanMessageCount": 1, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, @@ -2087,7 +2088,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.fast-responses.v3", + "oracleVersion": "paperclip.chat-catalog.identity.fast-responses.v4", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, @@ -2130,7 +2131,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.prompt-injection-guard.v2", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, @@ -4432,24 +4433,24 @@ "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v5", + "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v6", "coverageGaps": [], - "expectedRunCount": 2, + "expectedRunCount": 3, "turnTimeoutMs": 120000, - "attemptTimeoutMs": 1200000, + "attemptTimeoutMs": 1500000, "setupPolicy": { "readiness": "preserve-scenario", "invariantCheckIds": [ "lead-ready", - "real-native-infrastructure-exhaustion", - "original-source-action-owner" + "real-native-infrastructure-loss", + "fresh-continuation-without-person" ] }, "runPolicy": { "kind": "terminal-counts", "terminalStatusCounts": { "failed": 1, - "succeeded": 1 + "succeeded": 2 } }, "modulePath": "./chat-catalog-domains/runtime-infrastructure.js", @@ -4505,13 +4506,14 @@ "chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/runtime-native-fault.ts", + "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.retry-exhausted.v5", + "oracleVersion": "paperclip.chat-catalog.runtime.retry-exhausted.v6", "coverageGaps": [], "expectedRunCount": 5, "expectedHumanMessageCount": 3, @@ -4522,8 +4524,7 @@ "readiness": "preserve-scenario", "invariantCheckIds": [ "lead-ready", - "real-native-infrastructure-exhaustion", - "original-source-action-owner" + "person-owned-failure-awaits-retry" ] }, "runPolicy": { @@ -4548,24 +4549,24 @@ "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v5", + "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v6", "coverageGaps": [], - "expectedRunCount": 2, + "expectedRunCount": 3, "turnTimeoutMs": 120000, - "attemptTimeoutMs": 1200000, + "attemptTimeoutMs": 1500000, "setupPolicy": { "readiness": "preserve-scenario", "invariantCheckIds": [ "lead-ready", - "real-native-infrastructure-exhaustion", - "original-source-action-owner" + "real-native-infrastructure-loss", + "fresh-continuation-without-person" ] }, "runPolicy": { "kind": "terminal-counts", "terminalStatusCounts": { "failed": 1, - "succeeded": 1 + "succeeded": 2 } }, "modulePath": "./chat-catalog-domains/runtime-infrastructure.js", @@ -5523,9 +5524,10 @@ "chat-rendered-links.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v4", + "oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v5", "coverageGaps": [], "expectedRunCount": 3, + "minimumRunCount": 2, "turnTimeoutMs": 120000, "attemptTimeoutMs": 720000, "authentication": "human-session", @@ -5537,6 +5539,15 @@ "worker-task-scope" ] }, + "runPolicy": { + "kind": "terminal-counts", + "terminalStatusCounts": { + "succeeded": 2 + }, + "maximumTerminalStatusCounts": { + "succeeded": 3 + } + }, "modulePath": "./chat-catalog-domains/runtime-queue-controls.js", "exportName": "runtimeQueueControlCases" }, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts index 79179bb3e02..a1ff480aa50 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { companyModelIdentityCases, requestedAuxiliaryServiceModel, saveFastResponseModelBeforeTest } from "./identity-company-models.js"; +import { companyModelIdentityCases, requestedAuxiliaryServiceModel, saveServiceModelBeforeTest } from "./identity-company-models.js"; import type { IdentityObservation } from "./identity-support.js"; vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal(), probeAgentPermission: async () => 403, @@ -66,15 +66,28 @@ describe("company model service configuration and actual tests", () => { it.each(["companies.prompt-injection-guard", "companies.fast-responses"])("%s actual human act stops before test if saved model differs", async id => { const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!; const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), test = vi.fn(async () => {}); - const section = { waitFor: async () => {}, getByLabel: () => ({ fill }), getByRole: (_role: string, options: { name: string }) => ({ + const section = { waitFor: async () => {}, getByLabel: () => ({ fill, inputValue: async () => requestedAuxiliaryServiceModel }), getByRole: (_role: string, options: { name: string }) => ({ click: options.name.startsWith("Test") ? test : save, setChecked: async () => {}, }) }; const get = vi.fn(async () => ({ settings: { enabled: false, model: "openai/wrong-default" } })); const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }], - observationDeadlineAt: Date.now() + 390_000, input: { evidence: async () => {}, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; + observationDeadlineAt: Date.now() + 390_000, input: { evidence: async () => {}, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, + request: () => ({ postDataJSON: () => ({ model: requestedAuxiliaryServiceModel }) }), json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; await expect(definition.act!(ctx as never, valid(id).state)).rejects.toThrow(/Requested auxiliary model saved before any service test/); expect(fill).toHaveBeenCalledExactlyOnceWith(requestedAuxiliaryServiceModel); expect(save).toHaveBeenCalledTimes(2); - expect(get).toHaveBeenCalledTimes(id.endsWith("fast-responses") ? 1 : 2); expect(test).not.toHaveBeenCalled(); + expect(get).toHaveBeenCalledOnce(); expect(test).not.toHaveBeenCalled(); + }); + it.each(["companies.prompt-injection-guard", "companies.fast-responses"])("%s does not save when the model field lost the requested model", async id => { + const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!; + const save = vi.fn(async () => {}), test = vi.fn(async () => {}), waitForResponse = vi.fn(); + const section = { waitFor: async () => {}, getByLabel: () => ({ fill: async () => {}, inputValue: async () => "openai/gpt-oss-safeguard-20b" }), getByRole: (_role: string, options: { name: string }) => ({ + click: options.name.startsWith("Test") ? test : save, setChecked: async () => {}, + }) }; + const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }], + observationDeadlineAt: Date.now() + 390_000, input: { evidence: async () => {}, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get: async () => ({ settings: { enabled: false } }) }, + page: { waitForResponse, url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; + await expect(definition.act!(ctx as never, valid(id).state)).rejects.toThrow(/requested model is entered before saving/); + expect(waitForResponse).not.toHaveBeenCalled(); expect(test).not.toHaveBeenCalled(); }); it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); }); it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); }); @@ -107,8 +120,8 @@ describe("company model service configuration and actual tests", () => { it("rejects an acknowledgement whose independent invocation belongs to a different human", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); (o.state.history as Array<{ responsibleUserId: string }>)[0]!.responsibleUserId = "other-person"; expect(d.grade(o).passed).toBe(false); }); }); -describe("fast settings save completion before any service invocation", () => { - function fixture() { +describe.each(["fast-response", "prompt-injection-prevention"] as const)("%s settings save completion before any service invocation", service => { + function fixture(sentModel: string = requestedAuxiliaryServiceModel) { const saved = { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model: requestedAuxiliaryServiceModel }; let resolve!: (value: unknown) => void; const pending = new Promise(resolveValue => { resolve = resolveValue; }); @@ -116,43 +129,47 @@ describe("fast settings save completion before any service invocation", () => { const ctx = { observationDeadlineAt: Date.now() + 390_000, input: { fixtures: { company: { id: "acorn" } }, api: { get }, page: { url: () => "https://fixture.test", waitForResponse: vi.fn((predicate: (row: unknown) => boolean) => { - expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => "https://fixture.test/api/companies/acorn/fast-response" })).toBe(true); - for (const [method, path] of [["POST", "/api/companies/acorn/fast-response/test"], ["PUT", "/api/companies/foreign/fast-response"]]) + expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => `https://fixture.test/api/companies/acorn/${service}` })).toBe(true); + for (const [method, path] of [["POST", `/api/companies/acorn/${service}/test`], ["PUT", `/api/companies/foreign/${service}`], ["PUT", "/api/companies/acorn/decision-model"]]) expect(predicate({ request: () => ({ method: () => method }), url: () => `https://fixture.test${path}` })).toBe(false); - expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => "https://foreign.test/api/companies/acorn/fast-response" })).toBe(false); + expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => `https://foreign.test/api/companies/acorn/${service}` })).toBe(false); return pending; }) } } }; - const reply = (status = 200, body = saved) => resolve({ status: () => status, json: async () => body }); + const reply = (status = 200, body = saved) => resolve({ status: () => status, request: () => ({ postDataJSON: () => ({ ...saved, model: sentModel }) }), json: async () => body }); return { saved, ctx, get, reply, connection: { id: "shared", grantId: "grant" } }; } it("waits for the real matching delayed PUT before reading persisted model", async () => { const f = fixture(); const click = vi.fn(async () => {}); f.get.mockImplementation(async () => ({ settings: { ...f.saved, model: "stale" } })); - const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, click); + const pending = saveServiceModelBeforeTest(f.ctx as never, service, f.connection as never, click); await Promise.resolve(); expect(f.get).not.toHaveBeenCalled(); f.get.mockResolvedValue({ settings: f.saved }); f.reply(); await expect(pending).resolves.toEqual(f.saved); expect(click).toHaveBeenCalledOnce(); expect(f.get).toHaveBeenCalledOnce(); }); it.each([403, 422, 500])("does not read or test after rejected save %s", async status => { - const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); f.reply(status); - await expect(pending).rejects.toThrow(/save completed successfully/); expect(f.get).not.toHaveBeenCalled(); + const f = fixture(); const pending = saveServiceModelBeforeTest(f.ctx as never, service, f.connection as never, async () => {}); f.reply(status); + await expect(pending).rejects.toThrow(/settings save completed successfully/); expect(f.get).not.toHaveBeenCalled(); }); it("rejects wrong PUT model and independently wrong persisted model", async () => { for (const responseWrong of [true, false]) { - const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); + const f = fixture(); const pending = saveServiceModelBeforeTest(f.ctx as never, service, f.connection as never, async () => {}); if (!responseWrong) f.get.mockResolvedValue({ settings: { ...f.saved, model: "wrong" } }); f.reply(200, responseWrong ? { ...f.saved, model: "wrong" } : f.saved); await expect(pending).rejects.toThrow(/Requested auxiliary model saved/); } }); it.each([{ companyId: "foreign" }, { connectionId: "foreign" }, { grantId: "foreign" }, { enabled: false }])("rejects wrong saved identity or disabled state %j", async wrong => { - const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); + const f = fixture(); const pending = saveServiceModelBeforeTest(f.ctx as never, service, f.connection as never, async () => {}); f.reply(200, { ...f.saved, ...wrong }); await expect(pending).rejects.toThrow(/Requested auxiliary model saved/); expect(f.get).not.toHaveBeenCalled(); }); + it("rejects a save that sent a different model than the one entered", async () => { + const f = fixture("openai/gpt-oss-safeguard-20b"); const pending = saveServiceModelBeforeTest(f.ctx as never, service, f.connection as never, async () => {}); + f.reply(); await expect(pending).rejects.toThrow(/sent the requested auxiliary model/); expect(f.get).not.toHaveBeenCalled(); + }); it("never grants a fresh or unlimited deadline after delayed save", async () => { const f = fixture(); const now = Date.now(); const clock = vi.spyOn(Date, "now").mockReturnValue(now); - const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); + const pending = saveServiceModelBeforeTest(f.ctx as never, service, f.connection as never, async () => {}); clock.mockReturnValue(now + 400_000); f.reply(); await expect(pending).rejects.toThrow(/budget exhausted/); expect(f.get).not.toHaveBeenCalled(); clock.mockRestore(); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index be93f54dc19..3c3721b3e68 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -18,22 +18,23 @@ function remainingCompanyServiceObservationBudget(ctx: ChatCatalogContext): numb return deadline! - now; } -/** The settings click starts an async PUT; observe its completion before independent readback. */ -export async function saveFastResponseModelBeforeTest(ctx: ChatCatalogContext, connection: AiManagedConnectionSummary, - clickSave: (options: { timeout: number }) => Promise): Promise { +export async function saveServiceModelBeforeTest(ctx: ChatCatalogContext, service: "fast-response" | "prompt-injection-prevention", connection: AiManagedConnectionSummary, + clickSave: (options: { timeout: number }) => Promise): Promise { const remaining = () => remainingCompanyServiceObservationBudget(ctx); - const path = `/api/companies/${ctx.input.fixtures.company.id}/fast-response`; + const path = `/api/companies/${ctx.input.fixtures.company.id}/${service}`; const origin = new URL(ctx.input.page.url()).origin; const response = ctx.input.page.waitForResponse(row => row.request().method() === "PUT" && new URL(row.url()).origin === origin && new URL(row.url()).pathname === path, { timeout: remaining() }); const [, receipt] = await Promise.all([clickSave({ timeout: remaining() }), response]); remaining(); - expect(receipt.status(), "Actual fast response save completed successfully before service test").toBe(200); - const saved = await receipt.json() as FastResponseSettings; + expect(receipt.status(), "Actual service settings save completed successfully before service test").toBe(200); + const sent = receipt.request().postDataJSON() as { model?: unknown } | null; + expect(sent?.model, "The settings save sent the requested auxiliary model").toBe(requestedAuxiliaryServiceModel); + const saved = await receipt.json() as FastResponseSettings | PromptInjectionPreventionSettings; const expected = { companyId: ctx.input.fixtures.company.id, enabled: true, connectionId: connection.id, grantId: connection.grantId, model: requestedAuxiliaryServiceModel }; expect(saved, "Requested auxiliary model saved before any service test").toMatchObject(expected); - const persisted = (await ctx.input.api.get<{ settings: FastResponseSettings }>(path, { timeout: remaining() })).settings; + const persisted = (await ctx.input.api.get<{ settings: FastResponseSettings | PromptInjectionPreventionSettings }>(path, { timeout: remaining() })).settings; remaining(); expect(persisted, "Requested auxiliary model saved before any service test").toMatchObject(expected); return persisted; @@ -125,7 +126,8 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s } catch (error) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } throw error; } }, gradeCompanyModel, { sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000, - ...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v3" } : {}), + ...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v4" } + : spec.id === "companies.prompt-injection-guard" ? { oracleVersion: "paperclip.chat-catalog.identity.prompt-injection-guard.v2" } : {}), async act(ctx, state) { await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; @@ -145,15 +147,12 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click(); if (spec.path !== "decision-model") await section.getByLabel("Model", { exact: true }).fill(requestedAuxiliaryServiceModel); await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true); - if (spec.path === "fast-response") { - const persisted = await saveFastResponseModelBeforeTest(ctx, state.connection as AiManagedConnectionSummary, + if (spec.path === "decision-model") await section.getByRole("button", { name: spec.save, exact: true }).click(); + else { + expect(await section.getByLabel("Model", { exact: true }).inputValue(), "The requested model is entered before saving").toBe(requestedAuxiliaryServiceModel); + const persisted = await saveServiceModelBeforeTest(ctx, spec.path, state.connection as AiManagedConnectionSummary, options => section.getByRole("button", { name: spec.save, exact: true }).click(options)); state.modelBeforeTest = persisted.model; - } else await section.getByRole("button", { name: spec.save, exact: true }).click(); - if (spec.path === "prompt-injection-prevention") { - const persisted = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings; - expect(persisted.model, "Requested auxiliary model saved before any service test").toBe(requestedAuxiliaryServiceModel); - state.modelBeforeTest = persisted.model; } const testOptions = () => spec.path === "fast-response" ? { timeout: remainingCompanyServiceObservationBudget(ctx) } : {}; await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled(testOptions()); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts index 75d322f7de5..7e1d75e657e 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { createHash } from "node:crypto"; -import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import { mkdir, readFile, realpath, stat, writeFile } from "node:fs/promises"; +import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; import { resolveDefaultAgentWorkspaceDir } from "../../../server/src/home-paths.js"; import { prepareChatBrief } from "../chat-stories.js"; import { pollUntil } from "../api.js"; @@ -87,3 +88,23 @@ export async function releaseRuntimeWorker(state: Record) { const gate = state.gate as { gate?: string } | undefined; if (gate?.gate) await writeFile(gate.gate, "The human ended this disposable fixture. Preserve any saved work and finish normally.", "utf8"); } +export function retriesOriginalRun(run: Pick, originalRunId: string) { + if (run.retryOfRunId === originalRunId) return true; + const context = (run.contextSnapshot ?? {}) as { previousRunId?: unknown; executionContinuation?: { trigger?: { reason?: unknown; sourceRunId?: unknown } } }; + const trigger = context.executionContinuation?.trigger; + return context.previousRunId === originalRunId && trigger?.reason === "task_retry" && trigger.sourceRunId === originalRunId; +} +export async function releasedRetryListener(run: HeartbeatRun, expected: { companyId: string; workerId: string; taskId: string; originalRunId: string; gateName: string }, + root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT) { + const profile = (run as { runnerProfileJson?: { nativeExecutionInput?: { workspace?: { cwd?: unknown }; binding?: { executionWorkspaceId?: unknown } } } }).runnerProfileJson; + const cwd = profile?.nativeExecutionInput?.workspace?.cwd, workspaceId = profile?.nativeExecutionInput?.binding?.executionWorkspaceId; + if (!root || run.companyId !== expected.companyId || run.agentId !== expected.workerId || !retriesOriginalRun(run, expected.originalRunId) + || ![run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].includes(expected.taskId) || typeof cwd !== "string" || typeof workspaceId !== "string") return null; + const real = await realpath(cwd).catch(() => null), ownedRoot = await realpath(root); + if (!real || !real.startsWith(`${ownedRoot}${path.sep}`)) return null; + const ready = await realpath(path.join(real, `${expected.gateName}.waiting`)).catch(() => null); + if (!ready || path.dirname(ready) !== real || (await readFile(ready, "utf8")).trim() !== "waiting") return null; + const started = Date.parse(String(run.startedAt ?? "")); + if (!Number.isFinite(started) || (await stat(ready)).mtimeMs < started - 1_000) return null; + return { gate: path.join(real, expected.gateName), workspaceId }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.test.ts index 864aaeb40dc..2e74102de60 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.test.ts @@ -6,9 +6,9 @@ import { gradeInfrastructureRetry, isHumanRetryRequest, releasedRetryListener, r function valid() { return { scenarioId: "recovery.retry-exhausted", origin: "http://paperclip.test", companyId: "company", leadId: "ceo", agents: [], links: ["/COM/issues/COM-2"], humanQueries: [], reply: "", tasks: [{ id: "website", status: "done" }], activity: [], runs: [{ id: "exhausted", status: "failed" }, ...["canonical", "held", "retry", "final"].map(id => ({ id, status: "succeeded" }))], - state: { task: { id: "website", identifier: "COM-2" }, queryRecoveryRead: true, heldRecoveryRead: true, finalRecoveryRead: true, worker: { id: "morgan" }, failure: { id: "exhausted", status: "failed", nativePhase: "terminal_failure" }, - recoveryBefore: { id: "recovery", companyId: "company", sourceIssueId: "website", kind: "active_run_watchdog", cause: "native_session_retry_exhausted", ownerType: "board", ownerAgentId: null, returnOwnerAgentId: "morgan", recoveryIssueId: null, - evidence: { runId: "exhausted", coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted" }, maxAttempts: 3, wakePolicy: null, nextAction: "Inspect provider trace and explicitly choose replacement." }, queryReply: "Use the Retry button on the original failed run; this requires your human board session and rechecks current gates.", retryVisible: true, + state: { task: { id: "website", identifier: "COM-2", status: "blocked" }, queryRecoveryRead: true, heldRecoveryRead: true, finalRecoveryRead: true, worker: { id: "morgan" }, + failure: { id: "exhausted", companyId: "company", agentId: "morgan", status: "failed", errorCode: "native_provider_model_rejected" }, freshContinuationsBefore: 0, successorRunsBefore: 0, + queryReply: "Use the Retry button on the original failed run; this requires your human board session and rechecks current gates.", retryVisible: true, budgetHeldBeforeRetry: true, heldRetryStatus: 202, noDuplicateRetryWhileHeld: true, noAdmissionUnderBudget: true, failedRunVisibleWhileHeld: true, heldReplyAttributed: true, heldReply: "The retry could not start because Morgan's current budget hold blocks admission. The previous native failure remains visible.", budgetRestoredByHuman: true, retryBoundToOriginal: true, retryRunCount: 1, retryAdmissionAfterGateClear: true, @@ -31,7 +31,16 @@ it("rejects completion without the actual retry's saved output", () => { const o it("rejects borrowing the pre-retry reply for final completion", () => { const o = valid(); o.state.finalReplyAttributed = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); it("rejects absent native retry evidence", () => expect(gradeInfrastructureRetry(undefined).passed).toBe(false)); -it("rejects substituting a terminal model failure for actual infrastructure exhaustion", () => { const o = valid(); o.state.recoveryBefore.cause = "native_provider_model_rejected"; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects an infrastructure loss, which continues automatically, in place of a person-owned failure", () => { const o = valid(); o.state.failure.errorCode = "native_runner_process_exited"; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects a person-owned failure the system continued without the person", () => { + const continued = valid(); continued.state.freshContinuationsBefore = 1; expect(gradeInfrastructureRetry(continued).passed).toBe(false); + const started = valid(); started.state.successorRunsBefore = 1; expect(gradeInfrastructureRetry(started).passed).toBe(false); +}); +it("rejects a failed task that is not waiting for the person", () => { const o = valid(); o.state.task.status = "todo"; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects another agent's or company's failure", () => { + const agent = valid(); agent.state.failure.agentId = "riley"; expect(gradeInfrastructureRetry(agent).passed).toBe(false); + const company = valid(); company.state.failure.companyId = "other"; expect(gradeInfrastructureRetry(company).passed).toBe(false); +}); it("accepts the task's own Retry request as the human Retry, for this task and assignee only", () => { const request = (method: string, path: string) => ({ method: () => method, url: () => `http://paperclip.test${path}` }); @@ -60,7 +69,7 @@ it("binds the brief to the released Retry's own workspace, and only that one", a await writeFile(path.join(fresh, "brief.txt.waiting"), "waiting"); await writeFile(path.join(outside, "brief.txt.waiting"), "waiting"); const expected = { companyId: "company", workerId: "morgan", taskId: "website", originalRunId: "exhausted", gateName: "brief.txt" }; - const run = (cwd: string, patch: Record = {}) => ({ id: "retry", companyId: "company", agentId: "morgan", retryOfRunId: null, + const run = (cwd: string, patch: Record = {}) => ({ id: "retry", companyId: "company", agentId: "morgan", retryOfRunId: null, startedAt: new Date(Date.now() - 60_000).toISOString(), contextSnapshot: { issueId: "website", previousRunId: "exhausted", executionContinuation: { trigger: { reason: "task_retry", sourceRunId: "exhausted" } } }, runnerProfileJson: { nativeExecutionInput: { workspace: { cwd }, binding: { executionWorkspaceId: "fresh-ws" } } }, ...patch }) as never; expect(await releasedRetryListener(run(fresh), expected, root)).toEqual({ gate: path.join(await import("node:fs/promises").then(fs => fs.realpath(fresh)), "brief.txt"), workspaceId: "fresh-ws" }); expect(await releasedRetryListener(run(fresh, { contextSnapshot: { issueId: "website", previousRunId: "other", @@ -69,6 +78,8 @@ it("binds the brief to the released Retry's own workspace, and only that one", a expect(await releasedRetryListener(run(fresh, { contextSnapshot: { issueId: "other" } }), expected, root)).toBeNull(); expect(await releasedRetryListener(run(outside), expected, root)).toBeNull(); expect(await releasedRetryListener(run(fresh), { ...expected, gateName: "missing.txt" }, root)).toBeNull(); + expect(await releasedRetryListener(run(fresh, { startedAt: new Date(Date.now() + 60_000).toISOString() }), expected, root)).toBeNull(); + expect(await releasedRetryListener(run(fresh, { startedAt: null }), expected, root)).toBeNull(); } finally { await rm(root, { recursive: true, force: true }); await rm(outside, { recursive: true, force: true }); } }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.ts index c684b417118..4360e89ba23 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.ts @@ -1,66 +1,68 @@ import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js"; import { expect, type Page } from "@playwright/test"; -import { readFile, realpath, writeFile } from "node:fs/promises"; -import path from "node:path"; +import { writeFile } from "node:fs/promises"; import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; -import type { Issue, IssueRecoveryAction } from "../../../packages/shared/src/types/issue.js"; +import type { Issue } from "../../../packages/shared/src/types/issue.js"; import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { checks, type Entity } from "./internal-support.js"; -import { releaseRuntimeWorker, type RuntimeBriefGate } from "./runtime-busy-worker.js"; -import { controllerReadRecovery, exhaustedNativeInfrastructure, isExhaustedNativeRecovery, linksSourceTask, runtimeInfrastructureCases } from "./runtime-infrastructure.js"; -import type { NativeFaultRun } from "./runtime-native-fault.js"; +import { prepareRuntimeWorkerGate, releasedRetryListener, releaseRuntimeWorker, retriesOriginalRun, type RuntimeBriefGate } from "./runtime-busy-worker.js"; +import { controllerReadRecovery, linksSourceTask, sourceRunRecovery, type SourceRecoveryActions } from "./runtime-infrastructure.js"; +import { unavailableNativeModelFailure } from "./runtime-native-failure.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js"; -/** The person's Retry for the stopped task: the failed-run marker in the - * thread, or the task's own Retry when automatic recovery has stopped and the - * run is held for recovery (the thread then omits the marker's Retry). */ export function humanRetryControl(page: Page) { return page.getByTestId("task-chat-run-failed-try-again").or(page.getByTestId("task-execution-retry")).last(); } -/** Either supported Retry request for this task and its assignee. */ export function isHumanRetryRequest(request: { method(): string; url(): string }, taskId: string, workerId: string) { const path = new URL(request.url()).pathname; return request.method() === "POST" && (path === `/api/agents/${workerId}/wakeup` || path === `/api/issues/${taskId}/retry`); } -/** - * Whether a run is a Retry of the original run. A person's task Retry names its - * source in the execution continuation (`task_retry` from the original run); - * an automatic scheduled retry names it in `retryOfRunId`. - */ -export function retriesOriginalRun(run: Pick, originalRunId: string) { - if (run.retryOfRunId === originalRunId) return true; - const context = (run.contextSnapshot ?? {}) as { previousRunId?: unknown; executionContinuation?: { trigger?: { reason?: unknown; sourceRunId?: unknown } } }; - const trigger = context.executionContinuation?.trigger; - return context.previousRunId === originalRunId && trigger?.reason === "task_retry" && trigger.sourceRunId === originalRunId; +export { releasedRetryListener, retriesOriginalRun }; +export const PERSON_OWNED_RETRY_FAILURE_CODE = "native_provider_model_rejected"; +export function isPersonOwnedRetryFailure(input: { run?: Pick & { nativePhase?: string | null }; companyId: string; agentId: string; + taskStatus?: string; freshContinuations: number; successorRuns: number }) { + const run = input.run; + return !!run && run.status === "failed" && run.errorCode === PERSON_OWNED_RETRY_FAILURE_CODE && run.companyId === input.companyId && run.agentId === input.agentId + && input.taskStatus === "blocked" && input.freshContinuations === 0 && input.successorRuns === 0; } -/** - * The released Retry's own brief listener. A Retry can start in a fresh - * isolated workspace, so read the directory from that run's saved execution - * input, never from the task's earlier binding. It must be the worker's one - * retry of the original run for this task, inside the owned attempt root, and - * its script must already wait for the brief there. - */ -export async function releasedRetryListener(run: HeartbeatRun, expected: { companyId: string; workerId: string; taskId: string; originalRunId: string; gateName: string }, - root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT) { - const profile = (run as { runnerProfileJson?: { nativeExecutionInput?: { workspace?: { cwd?: unknown }; binding?: { executionWorkspaceId?: unknown } } } }).runnerProfileJson; - const cwd = profile?.nativeExecutionInput?.workspace?.cwd, workspaceId = profile?.nativeExecutionInput?.binding?.executionWorkspaceId; - if (!root || run.companyId !== expected.companyId || run.agentId !== expected.workerId || !retriesOriginalRun(run, expected.originalRunId) - || ![run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].includes(expected.taskId) || typeof cwd !== "string" || typeof workspaceId !== "string") return null; - const real = await realpath(cwd).catch(() => null), ownedRoot = await realpath(root); - if (!real || !real.startsWith(`${ownedRoot}${path.sep}`)) return null; - const ready = await realpath(path.join(real, `${expected.gateName}.waiting`)).catch(() => null); - if (!ready || path.dirname(ready) !== real || (await readFile(ready, "utf8")).trim() !== "waiting") return null; - return { gate: path.join(real, expected.gateName), workspaceId }; +export async function personOwnedNativeFailure(ctx: ChatCatalogContext) { + const { worker, gate, company } = await prepareRuntimeWorkerGate(ctx, { taskBound: true }); + let task: Entity | undefined; + const failure = await unavailableNativeModelFailure(ctx, worker, async () => { + task = await ctx.input.api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, allowDuplicate: true, + description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` }); + return { issueId: task.id }; + }); + if (!task || failure.failedIssueId !== task.id) throw new Error("Person-owned failure did not stop the fixture's own Website task"); + const companyId = ctx.input.fixtures.company.id, failedRunId = failure.failedRun.id, websiteId = task.id; + const load = async () => { + const [current, run, recovery, runs] = await Promise.all([ctx.input.api.get(`/api/issues/${websiteId}`), ctx.input.api.get(`/api/heartbeat-runs/${failedRunId}`), + sourceRunRecovery(ctx, websiteId, failedRunId), ctx.allRuns()]); + return { current, run, recovery, freshContinuations: (recovery.actions ?? []).filter(action => !!(action.evidence as Record).freshContinuation).length, + successorRuns: runs.filter(row => row.agentId === worker.id && row.id !== failedRunId).length }; + }; + const accepted = (v: Awaited>) => isPersonOwnedRetryFailure({ run: v.run, companyId, agentId: worker.id, taskStatus: v.current.status, + freshContinuations: v.freshContinuations, successorRuns: v.successorRuns }); + await pollUntil({ label: "The person-owned failure waits for the person's Retry", deadlineAt: Date.now() + 90_000, load, accept: accepted, + reject: v => v.successorRuns > 0 || v.freshContinuations > 0 ? "A person-owned failure was continued without the person" : undefined }); + const settled = await pollUntil({ label: "No automatic continuation follows the person-owned failure", deadlineAt: Date.now() + 60_000, load, + accept: v => accepted(v) && Date.now() - Date.parse(String(v.run.finishedAt)) >= 45_000, + reject: v => accepted(v) ? undefined : "A person-owned failure was continued without the person" }); + const before = await ctx.input.api.get(`/api/issues/${websiteId}/recovery-actions`); + await ctx.input.evidence("runtime-person-owned-failure.json", { runId: failedRunId, errorCode: settled.run.errorCode, taskStatus: settled.current.status, + activeRecovery: before.active ? { id: before.active.id, ownerType: before.active.ownerType, cause: before.active.cause, status: before.active.status } : null, + freshContinuations: settled.freshContinuations, successorRuns: settled.successorRuns }); + return { worker, gate, task: settled.current, workerRunId: failedRunId, failure: settled.run, recoveryBefore: before.active ?? null, + freshContinuationsBefore: settled.freshContinuations, successorRunsBefore: settled.successorRuns, failureKind: failure.failureKind }; } export function gradeInfrastructureRetry(value: unknown) { const c = checks(), o = runtimeObservation(value); - c.add("actual-exhausted-retry-and-current-gates", !!o, "The genuine exhausted native run, real human Retry controls and independently observed admission gates are required."); if (!o) return c.result(); + c.add("actual-person-owned-failure-retry-and-current-gates", !!o, "The genuine person-owned native failure, real human Retry controls and independently observed admission gates are required."); if (!o) return c.result(); const s = o.state; - c.add("actual-original-infrastructure-exhaustion", !!s.failure && isExhaustedNativeRecovery(s.recoveryBefore as IssueRecoveryAction, s.failure as NativeFaultRun, o.companyId, (s.task as Entity).id, (s.worker as Entity).id), - "The original task really exposes the exhausted three-attempt native infrastructure action; an unavailable-model failure is insufficient."); - // Reply wording is not graded: each declared phase is bound to its own - // controller run, which must have read the task's saved recovery record or - // runs before answering. A person reads the replies for meaning. + c.add("actual-person-owned-failure-awaits-retry", isPersonOwnedRetryFailure({ run: s.failure as HeartbeatRun, companyId: o.companyId, agentId: (s.worker as Entity)?.id, + taskStatus: (s.task as Issue)?.status, freshContinuations: Number(s.freshContinuationsBefore), successorRuns: Number(s.successorRunsBefore) }), + "The original task stopped for a reason only the person can resolve, stays blocked for the person, and the system starts no continuation of its own; an infrastructure loss continues automatically and is insufficient."); c.add("ordinary-human-retry-guidance", s.queryRecoveryRead === true && s.retryVisible === true && linksSourceTask(o, s.task as Entity), "The canonical reply comes from a run that read the saved recovery record and links the original task, where the real human Retry control is visible."); c.add("actual-closed-gate-prevents-new-execution", s.budgetHeldBeforeRetry === true && [200, 202, 409].includes(Number(s.heldRetryStatus)) @@ -78,14 +80,21 @@ export function gradeInfrastructureRetry(value: unknown) { "A separately declared factual follow-up receives its own native reply from a run that read the task's actual runs."); return c.result(); } -export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retry-exhausted", exhaustedNativeInfrastructure, gradeInfrastructureRetry, { - oracleVersion: "paperclip.chat-catalog.runtime.retry-exhausted.v5", authentication: "human-session", expectedHumanMessageCount: 3, expectedRunCount: 5, attemptTimeoutMs: 1_800_000, - setupPolicy: runtimeInfrastructureCases[0]!.setupPolicy, verifySetup: runtimeInfrastructureCases[0]!.verifySetup, +export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retry-exhausted", personOwnedNativeFailure, gradeInfrastructureRetry, { + oracleVersion: "paperclip.chat-catalog.runtime.retry-exhausted.v6", authentication: "human-session", expectedHumanMessageCount: 3, expectedRunCount: 5, attemptTimeoutMs: 1_800_000, + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "person-owned-failure-awaits-retry"] }, + async verifySetup(ctx, state) { const c = checks(), lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`), task = state.task as Issue, worker = state.worker as Entity; + const [current, run, recovery, runs] = await Promise.all([ctx.input.api.get(`/api/issues/${task.id}`), ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), + sourceRunRecovery(ctx, task.id, String(state.workerRunId)), ctx.allRuns()]); + c.add("lead-ready", lead.lifecycleState === "ready" && ["active", "idle"].includes(lead.status ?? ""), "The actual controller remains independently admitted."); + c.add("person-owned-failure-awaits-retry", isPersonOwnedRetryFailure({ run, companyId: ctx.input.fixtures.company.id, agentId: worker.id, taskStatus: current.status, + freshContinuations: (recovery.actions ?? []).filter(action => !!(action.evidence as Record).freshContinuation).length, + successorRuns: runs.filter(row => row.agentId === worker.id && row.id !== run.id).length }), "The original task waits for the person's Retry after a failure only the person can resolve."); return c.result(); }, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 4 } }, - sourceFiles: ["chat-catalog-domains/runtime-infrastructure-retry.ts", "chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-infrastructure-retry.ts", "chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml"], async act(ctx, state) { const task = state.task as Issue, worker = state.worker as RuntimeAgent; - const initial = await pollUntil({ label: "Canonical human Retry guidance after actual infrastructure exhaustion", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), + const initial = await pollUntil({ label: "Canonical human Retry guidance after the person-owned failure", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) }); state.queryReply = (await observeRuntime(ctx, state)).reply; const source = ctx.input.page.url(), root = `/api/companies/${ctx.input.fixtures.company.id}`; @@ -106,7 +115,6 @@ export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retr state.budgetHeldBeforeRetry = held.lifecycleHolds?.includes("budget") === true; const rejected = ctx.input.page.waitForResponse(r => isHumanRetryRequest(r.request(), task.id, worker.id)); await retry.click(); state.heldRetryStatus = (await rejected).status(); - // The worker's runs other than the exhausted one. None may exist while held. const workerRetryRuns = async () => (await ctx.allRuns()).filter(row => row.agentId === worker.id && row.id !== state.workerRunId).length; const holdSamples = [await workerRetryRuns()]; // Wait on the open page as a person would, without reloading: the task @@ -168,7 +176,7 @@ export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retr state.retryBoundToOriginal = !!full && retriesOriginalRun(full, String(state.workerRunId)) && [full.contextSnapshot?.issueId, full.contextSnapshot?.taskId].includes(task.id); state.retryAdmissionAfterGateClear = !!full && Date.parse(String(full.createdAt)) >= Number(state.gatesClearedAt); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); - const finalQuery = `I restored Morgan's budget and used the supported Retry for Website ${task.identifier ?? task.id}. What is the actual result now, and is the earlier exhausted attempt still retained?`; + const finalQuery = `I restored Morgan's budget and used the supported Retry for Website ${task.identifier ?? task.id}. What is the actual result now, and is the earlier failed attempt still retained?`; await sendChatMessage(ctx.input.page, finalQuery, chatCatalogNativePhaseRemainingBudget(ctx)); const final = await pollUntil({ label: "Own final source reply reports actual eligible Retry result", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.filter(row => row.id !== state.workerRunId).every(row => row.status === "succeeded")) }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts index 9ba6687dbf4..03d5a942e79 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts @@ -1,26 +1,65 @@ import { expect, it } from "vitest"; -import { exhaustedSourceRecovery, gradeInfrastructureRecovery, readSourceRecoveryRecord, sameRecoveryAction } from "./runtime-infrastructure.js"; +import { freshContinuationRecord, gradeInfrastructureRecovery, readSourceRecoveryRecord } from "./runtime-infrastructure.js"; import { savedCall } from "./api-read-receipts.test-fixtures.js"; -function valid(id = "recovery.explain-stall") { - const recovery = { id: "recovery", companyId: "company", sourceIssueId: "website", kind: "active_run_watchdog", cause: "native_session_retry_exhausted", ownerType: "board", ownerAgentId: null, - returnOwnerAgentId: "morgan", recoveryIssueId: null, evidence: { runId: "run", coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted" }, maxAttempts: 3, wakePolicy: null, nextAction: "Inspect the provider trace and explicitly choose a replacement run or provider configuration; automatic provider work is stopped." }; - return { scenarioId: id, origin: "http://paperclip.test", companyId: "company", leadId: "ceo", agents: [], links: ["/COM/issues/COM-2"], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], - runs: [{ id: "run", agentId: "morgan", status: "failed" }, { id: "source", agentId: "ceo", status: "succeeded" }], - state: { task: { id: "website", identifier: "COM-2" }, queryRecoveryRead: true, worker: { id: "morgan" }, failure: { id: "run", status: "failed", nativePhase: "terminal_failure" }, recoveryBefore: recovery, recoveryAfter: { ...recovery }, - setupFaults: [1, 2, 3].map(pid => ({ runId: "run", pid, errorCode: "native_runner_process_exited" })), sourceRecoveryVisible: true, - queryReply: "Website stopped after three native Runner infrastructure attempts exhausted recovery. The original task's recovery action belongs to the board: an operator must inspect its provider trace and explicitly choose a replacement run or provider configuration." } }; +function record(overrides: Record = {}) { + return { id: "recovery", companyId: "company", sourceIssueId: "website", kind: "active_run_watchdog", status: "resolved", outcome: "restored", cause: "native_session_retry_exhausted", + ownerType: "system", ownerAgentId: null, ownerUserId: null, returnOwnerAgentId: "morgan", recoveryIssueId: null, wakePolicy: null, + nextAction: "The task continues in a fresh run from its last saved work.", resolutionNote: "continued_fresh_from_last_saved", + evidence: { runId: "run", continuationDelivery: "delivered", continuationRunId: "next", + freshContinuation: { schema: "paperclip.fresh-continuation.v1", sourceRunId: "run", trigger: "recovery_refused", failureCode: "native_runner_process_exited", delivery: "recorded", isolation: "fresh_isolated_workspace" }, + freshContinuationSettlement: { schema: "paperclip.fresh-continuation.v1", actorType: "system", actorId: "fresh-continuation", decision: "continue_in_fresh_run", replay: "none", sourceRunId: "run" } }, ...overrides }; } -it("accepts actual bounded process failure and truthful source-scoped diagnosis", () => expect(gradeInfrastructureRecovery(valid()).passed).toBe(true)); -it("accepts a natural factual explanation", () => { const o = valid(); o.state.queryReply = "The Website's native session hit its limit of 3 resume attempts after process loss. You can open the run details and inspect the provider trace before authorizing a retry; this needs operator action."; expect(gradeInfrastructureRecovery(o).passed).toBe(true); }); -it("accepts actual original source recovery identification", () => { const o = valid("recovery.source-action"); o.state.queryReply = "Use the recovery action on the original Website task. Its board owner must inspect the run details and choose an explicit retry. A separate repair task is unnecessary."; expect(gradeInfrastructureRecovery(o).passed).toBe(true); }); -it("rejects relabelled unavailable-model failure as infrastructure loss", () => { const o = valid(); o.state.setupFaults[0]!.errorCode = "native_provider_model_rejected"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); -it("rejects an invented exhaustion counter", () => { const o = valid(); o.state.recoveryBefore.evidence.coordinatorAttempt = 1; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); -it("rejects three receipts for one reused PID", () => { const o = valid(); o.state.setupFaults[1]!.pid = 1; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); -it("rejects silently transferring the action owner", () => { const o = valid(); o.state.recoveryAfter.ownerType = "agent"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); -it("rejects a foreign source action", () => { const o = valid(); o.state.recoveryAfter.sourceIssueId = "other"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); -it("does not grade reply wording", () => { const o = valid(); o.state.queryReply = "Short answer: see the task."; expect(gradeInfrastructureRecovery(o).passed).toBe(true); }); -it("rejects an answer whose run never read the saved recovery record", () => { const o = valid(); o.state.queryRecoveryRead = false; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); -it("rejects an answer that does not link the original task", () => { const o = valid(); o.links = ["/COM/issues/COM-9"]; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); +function valid(id = "recovery.explain-stall") { + return { scenarioId: id, origin: "http://paperclip.test", companyId: "company", leadId: "ceo", agents: [], links: ["/COM/issues/COM-2"], humanQueries: [], reply: "", tasks: [{ id: "website", status: "done", assigneeAgentId: "morgan" }], activity: [], + runs: [{ id: "run", agentId: "morgan", status: "failed" }, { id: "next", agentId: "morgan", status: "succeeded" }, { id: "source", agentId: "ceo", status: "succeeded" }], + state: { task: { id: "website", identifier: "COM-2" }, queryRecoveryRead: true, worker: { id: "morgan" }, + failure: { id: "run", status: "failed", resultJson: { freshContinuationSupersededAt: "2026-11-12T21:50:53.372Z" } }, + continuation: record(), recoveryAfter: { active: null, actions: [record()] }, continuationRunId: "next", + continuationRun: { id: "next", companyId: "company", agentId: "morgan", retryOfRunId: "run", contextSnapshot: { issueId: "website" } }, + activeHoldAtReply: null as unknown, taskStatusAtReply: "in_progress", continuationResult: true, + setupFaults: [1, 2, 3].map(pid => ({ runId: "run", pid, errorCode: "native_runner_process_exited" })), + queryReply: "Morgan's Website run lost its Runner three times. Paperclip continued the task in a fresh run from its saved work; nothing is needed from you." } }; +} +const grade = (o: ReturnType) => gradeInfrastructureRecovery(o).passed; +it("accepts one system-owned fresh continuation that finishes the original task", () => expect(grade(valid())).toBe(true)); +it("accepts the same contract for the source-action question", () => expect(grade(valid("recovery.source-action"))).toBe(true)); +it("does not grade reply wording", () => { const o = valid(); o.state.queryReply = "It got cut off and picked back up on its own; see the task."; expect(grade(o)).toBe(true); }); +it("accepts the recovery record that keeps an earlier hold as the continuation's primary record", () => { + const o = valid(), hold = record({ kind: "active_run_watchdog", cause: "native_session_retry_exhausted", evidence: { ...record().evidence, automaticRecoveryHistory: { policy: "preserve_without_replay_v1" } } }); + o.state.continuation = hold; o.state.recoveryAfter = { active: null, actions: [hold] }; expect(grade(o)).toBe(true); +}); +it("accepts a system-owned active hold at the reply", () => { const o = valid(); o.state.activeHoldAtReply = { ownerType: "system" }; expect(grade(o)).toBe(true); }); +it("rejects a task left blocked for a person", () => { const o = valid(); o.state.taskStatusAtReply = "blocked"; expect(grade(o)).toBe(false); }); +it("rejects a board-owned recovery action the person must act on", () => { const o = valid(); o.state.activeHoldAtReply = { ownerType: "board" }; expect(grade(o)).toBe(false); }); +it("rejects a continuation record owned by a person", () => { const o = valid(), owned = record({ ownerType: "board" }); o.state.recoveryAfter = { active: null, actions: [owned] }; expect(grade(o)).toBe(false); }); +it("rejects a continuation that replays the interrupted turn", () => { + const o = valid(), replay = record({ evidence: { ...record().evidence, freshContinuationSettlement: { ...record().evidence.freshContinuationSettlement, replay: "exact" } } }); + o.state.recoveryAfter = { active: null, actions: [replay] }; expect(grade(o)).toBe(false); +}); +it("rejects two continuation records for one interrupted run", () => { const o = valid(); o.state.recoveryAfter = { active: null, actions: [record(), record({ id: "second" })] }; expect(grade(o)).toBe(false); }); +it("rejects a replaced continuation record", () => { const o = valid(); o.state.recoveryAfter = { active: null, actions: [record({ id: "replacement" })] }; expect(grade(o)).toBe(false); }); +it("rejects missing continuation evidence", () => { const o = valid(); o.state.recoveryAfter = { active: null, actions: [] }; expect(grade(o)).toBe(false); }); +it("rejects an interrupted run that keeps its write authority", () => { const o = valid(); o.state.failure.resultJson = {} as never; expect(grade(o)).toBe(false); }); +it("rejects a second continuation run", () => { const o = valid(); o.runs.push({ id: "again", agentId: "morgan", status: "succeeded" }); expect(grade(o)).toBe(false); }); +it("rejects a run that does not continue the interrupted one", () => { const o = valid(); o.state.continuationRun.retryOfRunId = "other"; expect(grade(o)).toBe(false); }); +it("rejects a continuation that never finishes", () => { const o = valid(); o.runs[1]!.status = "failed"; expect(grade(o)).toBe(false); }); +it("rejects a continuation that never saves its result", () => { const o = valid(); o.state.continuationResult = false; expect(grade(o)).toBe(false); }); +it("rejects an unfinished original task", () => { const o = valid(); o.tasks[0]!.status = "in_progress"; expect(grade(o)).toBe(false); }); +it("rejects relabelled unavailable-model failure as infrastructure loss", () => { const o = valid(); o.state.setupFaults[0]!.errorCode = "native_provider_model_rejected"; expect(grade(o)).toBe(false); }); +it("rejects a continuation recorded for a person-owned failure", () => { + const o = valid(), person = record({ evidence: { ...record().evidence, freshContinuation: { ...record().evidence.freshContinuation, failureCode: "native_provider_model_rejected" } } }); + o.state.recoveryAfter = { active: null, actions: [person] }; expect(grade(o)).toBe(false); +}); +it("rejects three receipts for one reused PID", () => { const o = valid(); o.state.setupFaults[1]!.pid = 1; expect(grade(o)).toBe(false); }); +it("rejects an answer whose run never read the saved recovery record", () => { const o = valid(); o.state.queryRecoveryRead = false; expect(grade(o)).toBe(false); }); +it("rejects an answer that does not link the original task", () => { const o = valid(); o.links = ["/COM/issues/COM-9"]; expect(grade(o)).toBe(false); }); +it("selects only the continuation of this run in this task", () => { + const run = { id: "run", status: "failed", resultJson: { freshContinuationSupersededAt: "2026-11-12T21:50:53.372Z" } } as never; + expect(freshContinuationRecord({ actions: [record() as never] }, run, "company", "website", "morgan")?.id).toBe("recovery"); + expect(freshContinuationRecord({ actions: [record() as never] }, run, "company", "other", "morgan")).toBeUndefined(); + expect(freshContinuationRecord({ actions: [record() as never] }, run, "company", "website", "riley")).toBeUndefined(); + expect(freshContinuationRecord({ actions: [record({ status: "active" }) as never] }, run, "company", "website", "morgan")).toBeUndefined(); +}); it("accepts a genuine call_api recovery read after the port redacts its body, bound to the source task or failed run", () => { const read = (operationId: string, pathParams: Record, status = 200) => savedCall({ operationId, pathParams }, status, "3", "run-1", { sourceIssueId: "website", runId: "run" }); expect(readSourceRecoveryRecord(read("GET /api/issues/{id}/recovery-actions", { id: "website" }), "website", "run", "COM-1")).toBe(true); @@ -28,6 +67,8 @@ it("accepts a genuine call_api recovery read after the port redacts its body, bo expect(readSourceRecoveryRecord(read("GET /api/heartbeat-runs/{runId}", { runId: "run" }), "website", "run", "COM-1")).toBe(true); expect(readSourceRecoveryRecord(read("GET /api/issues/{id}", { id: "COM-2" }), "website", "run", "COM-1")).toBe(false); expect(readSourceRecoveryRecord(read("GET /api/heartbeat-runs/{runId}", { runId: "other" }), "website", "run", "COM-1")).toBe(false); + expect(readSourceRecoveryRecord(read("GET /api/heartbeat-runs/{runId}", { runId: "next" }), "website", "run", "COM-1", "next")).toBe(true); + expect(readSourceRecoveryRecord(read("GET /api/heartbeat-runs/{runId}", { runId: "next" }), "website", "run", "COM-1")).toBe(false); expect(readSourceRecoveryRecord(read("GET /api/issues/{id}/recovery-actions", { id: "website" }, 403), "website", "run", "COM-1")).toBe(false); }); it("reads the recovery record only from exact product tool output", () => { @@ -44,49 +85,10 @@ it("reads the recovery record only from exact product tool output", () => { const task = (body: string) => `{"apiOperationId":"GET /api/issues/{id}","data":${body}}`; expect(readSourceRecoveryRecord([event(task('{"id":"website","executionBlocker":{"runId":"run","cause":"native_session_retry_exhausted"}}'))], "website", "run")).toBe(true); expect(readSourceRecoveryRecord([event(task('{"id":"website","executionBlocker":null}'))], "website", "run")).toBe(false); + expect(readSourceRecoveryRecord([event(task('{"id":"website","executionRunId":"next"}'))], "website", "run", null, "next")).toBe(true); expect(readSourceRecoveryRecord([event(task('{"id":"other","executionBlocker":{"runId":"run"}}'))], "website", "run")).toBe(false); expect(readSourceRecoveryRecord([event(task('{"id":"website","executionBlocker":{"runId":"run"}}'), "failed")], "website", "run")).toBe(false); expect(readSourceRecoveryRecord([event('{"apiOperationId":"GET /api/issues/{id}/documents/{key}","data":"website run"}')], "website", "run")).toBe(false); }); -it("rejects duplicate repair chains", () => { const o = valid(); o.tasks.push({ id: "repair" }); expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); -it("rejects concealing resumed work as still failed", () => { const o = valid(); o.runs[0]!.status = "running"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); }); +it("rejects duplicate repair chains", () => { const o = valid(); o.tasks.push({ id: "repair", status: "todo", assigneeAgentId: "morgan" }); expect(grade(o)).toBe(false); }); it("rejects missing actual source recovery evidence", () => expect(gradeInfrastructureRecovery(undefined).passed).toBe(false)); - -// The product settles an exhausted native action as resolved/blocked under -// preserve_without_replay_v1 and blocks the task, where the human Retry lives. -function settled(overrides: Record = {}, automatic: Record = {}) { - const { recoveryBefore } = valid().state; - return { ...recoveryBefore, status: "resolved", outcome: "blocked", createdAt: "2026-11-09T15:13:11.000Z", wakePolicy: null, - nextAction: "Automatic recovery stopped. Recorded work is preserved; actions with unverified outcomes will not be repeated.", - evidence: { ...recoveryBefore.evidence, automaticRecovery: { policy: "preserve_without_replay_v1", runId: "run", replay: "blocked", ...automatic } }, ...overrides }; -} -const run = { id: "run", status: "failed", nativePhase: "terminal_failure" } as never; -const select = (actions: unknown[], status = "blocked", active?: unknown) => - exhaustedSourceRecovery({ active: active as never, actions: actions as never }, { status } as never, run, "company", "website", "morgan"); -it("accepts the exhausted action settled as blocked without replay on a blocked task", () => { - const action = settled(); - expect(select([action])?.id).toBe("recovery"); - const o = valid(); o.state.recoveryBefore = action as never; o.state.recoveryAfter = { ...action } as never; - expect(gradeInfrastructureRecovery(o).passed).toBe(true); -}); -it("still accepts the exhausted action while it is active", () => expect(select([], "blocked", { ...valid().state.recoveryBefore, status: "active" })?.id).toBe("recovery")); -it.each([ - ["a cancelled settlement", settled({ outcome: "cancelled" })], - ["a settlement without the preserve-without-replay marker", settled({ evidence: { ...valid().state.recoveryBefore.evidence } })], - ["a settlement for another run", settled({}, { runId: "other" })], - ["a settlement under another policy", settled({}, { policy: "replay_v1" })], - ["a settled action that kept a wake policy", settled({ wakePolicy: { kind: "resume_native_run", runId: "run" } })], - ["a settled action for another attempt count", settled({ evidence: { ...settled().evidence, coordinatorAttempt: 2 } })], -])("rejects %s", (_label, action) => expect(select([action])).toBeUndefined()); -it("rejects a settled exhaustion while the task is not blocked", () => expect(select([settled()], "todo")).toBeUndefined()); -it("judges only the newest source action", () => { - const older = settled({ id: "older", createdAt: "2026-11-09T15:00:00.000Z" }), newer = settled({ id: "newer", outcome: "cancelled" }); - expect(select([older, newer])).toBeUndefined(); -}); -it("reads before and after as the same action by ID, so a replacement is detected", () => { - const before = settled(), replacement = settled({ id: "replacement", createdAt: "2026-11-09T15:20:00.000Z" }); - expect(sameRecoveryAction({ actions: [before as never] }, "recovery")?.id).toBe("recovery"); - expect(sameRecoveryAction({ actions: [replacement as never] }, "recovery")).toBeUndefined(); - const o = valid(); o.state.recoveryBefore = before as never; o.state.recoveryAfter = replacement as never; - expect(gradeInfrastructureRecovery(o).passed).toBe(false); -}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts index ab271dd9856..e4d9a5b8821 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts @@ -1,15 +1,17 @@ import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; +import { writeFile } from "node:fs/promises"; +import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; import type { Issue, IssueRecoveryAction } from "../../../packages/shared/src/types/issue.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; import { collectRunEvents } from "../run-observations.js"; import { readRecord, recordParams } from "./api-read-receipts.js"; import { checks, hasLink, type Entity } from "./internal-support.js"; -import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { busyRuntimeWorker, releasedRetryListener, releaseRuntimeWorker, type RuntimeBriefGate } from "./runtime-busy-worker.js"; import { faultOwnedNativeProcess, NATIVE_INFRASTRUCTURE_FAULT_CODES, type NativeFaultRun } from "./runtime-native-fault.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type RuntimeObservation } from "./runtime-support.js"; -export const INFRASTRUCTURE_RECOVERY_ORACLE = "paperclip.chat-catalog.runtime.infrastructure-recovery.v5"; +export const INFRASTRUCTURE_RECOVERY_ORACLE = "paperclip.chat-catalog.runtime.infrastructure-recovery.v6"; const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; /** * The chat run read the source task's saved recovery record through a product @@ -17,62 +19,57 @@ const objectRecord = (value: unknown): Record => value && typeo * whose execution blocker names that failed run. Tool outputs * are product data and are matched exactly; the agent's wording never is. */ -export function readSourceRecoveryRecord(events: readonly Record[], taskId: string, failedRunId: string, identifier?: string | null) { +export function readSourceRecoveryRecord(events: readonly Record[], taskId: string, failedRunId: string, identifier?: string | null, continuationRunId?: string | null) { + const runIds = [failedRunId, continuationRunId].filter((id): id is string => !!id); // call_api bodies are redacted before saving: bind those reads by their input digest and a successful result. const task = recordParams("id", taskId, identifier); if (taskId && failedRunId && readRecord(events, [ { operationId: "GET /api/issues/{id}/recovery-actions", pathParams: task, paged: true }, { operationId: "GET /api/issues/{id}/runs", pathParams: task, paged: true }, { operationId: "GET /api/issues/{id}", pathParams: task }, - { operationId: "GET /api/heartbeat-runs/{runId}", pathParams: recordParams("runId", failedRunId) }, + { operationId: "GET /api/heartbeat-runs/{runId}", pathParams: runIds.flatMap(id => recordParams("runId", id)) }, ])) return true; return !!taskId && !!failedRunId && events.some(row => { const envelope = objectRecord(objectRecord(row.payload).prpEvent), payload = objectRecord(envelope.payload); const output = typeof payload.output === "string" ? payload.output : ""; return row.eventType === "tool.execution.completed" && envelope.sourceKind === "runner" && payload.status === "completed" && (/apiOperationId\\*"\s*:\s*\\*"GET \/api\/issues\/\{id\}\/(?:recovery-actions|runs)(?![\w/{])/.test(output) && output.includes(taskId) - || /apiOperationId\\*"\s*:\s*\\*"GET \/api\/heartbeat-runs\/\{runId\}(?![\w/{])/.test(output) && output.includes(failedRunId) + || /apiOperationId\\*"\s*:\s*\\*"GET \/api\/heartbeat-runs\/\{runId\}(?![\w/{])/.test(output) && runIds.some(id => output.includes(id)) // The task read carries its execution blocker: the saved recovery // record, bound to the failed run it stopped on. - || /apiOperationId\\*"\s*:\s*\\*"GET \/api\/issues\/\{id\}\\*"/.test(output) && output.includes(taskId) && output.includes(failedRunId)); + || /apiOperationId\\*"\s*:\s*\\*"GET \/api\/issues\/\{id\}\\*"/.test(output) && output.includes(taskId) && runIds.some(id => output.includes(id))); }); } /** Whether one controller run read the recovery record; computed from its own retained events. */ -export async function controllerReadRecovery(ctx: ChatCatalogContext, runId: string | undefined, taskId: string, failedRunId: string, identifier?: string | null) { +export async function controllerReadRecovery(ctx: ChatCatalogContext, runId: string | undefined, taskId: string, failedRunId: string, identifier?: string | null, continuationRunId?: string | null) { if (!runId) return false; const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=1000`)); - return readSourceRecoveryRecord(events, taskId, failedRunId, identifier); + return readSourceRecoveryRecord(events, taskId, failedRunId, identifier, continuationRunId); } -/** The reply links the original task, where its recovery and Retry controls live. */ export function linksSourceTask(o: RuntimeObservation, task: Entity) { return hasLink(o, "issues", task); } -/** The product either keeps the exhausted action active, or settles it for - * this run as `blocked` under its preserve-without-replay policy, leaving the - * human Retry on the blocked source task. Any other settlement is not it. */ -function exhaustedActionLive(action: IssueRecoveryAction, run: NativeFaultRun) { - if (action.status !== "resolved") return action.status === undefined || action.status === "active" || action.status === "escalated"; - const automatic = objectRecord(action.evidence.automaticRecovery); - return action.outcome === "blocked" && automatic.policy === "preserve_without_replay_v1" && automatic.runId === run.id; -} -export function isExhaustedNativeRecovery(action: IssueRecoveryAction | undefined, run: NativeFaultRun, companyId: string, issueId: string, agentId: string) { - return !!action && exhaustedActionLive(action, run) && action.companyId === companyId && action.sourceIssueId === issueId && action.kind === "active_run_watchdog" && action.cause === "native_session_retry_exhausted" - && action.ownerType === "board" && action.ownerAgentId == null && action.returnOwnerAgentId === agentId && action.recoveryIssueId == null - && action.evidence.runId === run.id && action.evidence.coordinatorAttempt === 3 && action.evidence.recoveryDisposition === "native_session_retry_exhausted" - && action.maxAttempts === 3 && action.wakePolicy == null && action.nextAction.trim().length > 0 && run.status === "failed" && run.nativePhase === "terminal_failure"; -} export type SourceRecoveryActions = { active?: IssueRecoveryAction | null; actions?: IssueRecoveryAction[] }; -const newest = (actions: readonly IssueRecoveryAction[]) => [...actions].sort((a, b) => Date.parse(String(b.createdAt)) - Date.parse(String(a.createdAt)))[0]; -/** The source task's exhausted action: the active one, or else its newest - * action. The task itself must be blocked, where the human Retry lives. */ -export function exhaustedSourceRecovery(recovery: SourceRecoveryActions, task: Pick, run: NativeFaultRun, companyId: string, issueId: string, agentId: string) { - const action = recovery.active ?? newest(recovery.actions ?? []); - return task.status === "blocked" && isExhaustedNativeRecovery(action, run, companyId, issueId, agentId) ? action : undefined; +export const FRESH_CONTINUATION_SCHEMA = "paperclip.fresh-continuation.v1"; +export function sourceRunRecovery(ctx: ChatCatalogContext, taskId: string, runId: string) { + return ctx.input.api.get(`/api/issues/${taskId}/recovery-actions?inspectSourceRunId=${encodeURIComponent(runId)}`); } -/** The same action, read again by ID, so a replaced or duplicated action is detected. */ -export function sameRecoveryAction(recovery: SourceRecoveryActions, id: string) { - return [recovery.active, ...(recovery.actions ?? [])].find(action => action?.id === id) ?? undefined; +type FailedRun = Pick; +export function freshContinuationRecord(recovery: SourceRecoveryActions | undefined, run: FailedRun | undefined, companyId: string, issueId: string, agentId: string) { + if (!run) return undefined; + const records = (recovery?.actions ?? []).filter(action => objectRecord(objectRecord(action.evidence).freshContinuation).sourceRunId === run.id); + const action = records.length === 1 ? records[0]! : undefined; + if (!action) return undefined; + const fresh = objectRecord(action.evidence.freshContinuation), settlement = objectRecord(action.evidence.freshContinuationSettlement); + return action.companyId === companyId && action.sourceIssueId === issueId && action.status === "resolved" && action.outcome === "restored" + && action.ownerType === "system" && action.ownerAgentId == null && action.ownerUserId == null && action.recoveryIssueId == null && action.returnOwnerAgentId === agentId + && fresh.schema === FRESH_CONTINUATION_SCHEMA && NATIVE_INFRASTRUCTURE_FAULT_CODES.includes(String(fresh.failureCode)) + && settlement.actorType === "system" && settlement.decision === "continue_in_fresh_run" && settlement.replay === "none" && settlement.sourceRunId === run.id + && run.status === "failed" && typeof objectRecord(run.resultJson).freshContinuationSupersededAt === "string" ? action : undefined; } -/** Three actual coordinator attempts on the SAME native heartbeat; never status patches or fake event rows. */ -export async function exhaustedNativeInfrastructure(ctx: ChatCatalogContext) { +export function continuesFailedRun(run: Pick | undefined, failedRunId: string, companyId: string, issueId: string, agentId: string) { + return !!run && run.id !== failedRunId && run.companyId === companyId && run.agentId === agentId && run.retryOfRunId === failedRunId + && [run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].includes(issueId); +} +export async function nativeInfrastructureLoss(ctx: ChatCatalogContext) { const state = await busyRuntimeWorker(ctx), task = state.task as Entity; const faults: Array<{ runId: string; pid: number | null; errorCode: string | null }> = []; let previousPid: number | null = null; @@ -80,76 +77,114 @@ export async function exhaustedNativeInfrastructure(ctx: ChatCatalogContext) { const active = await pollUntil({ label: `Real native coordinator attempt ${attempt} on the owned original heartbeat`, deadlineAt: Date.now() + 120_000, load: () => ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), accept: run => run.status === "running" && !!run.processPid && run.processPid !== previousPid, - reject: run => run.nativePhase === "terminal_failure" ? "Native infrastructure became terminal before three real bounded attempts; this is not retry exhaustion" : undefined }); + reject: run => run.nativePhase === "terminal_failure" ? "Native infrastructure became terminal before three real bounded attempts" : undefined }); const failed = await faultOwnedNativeProcess(ctx, { runId: active.id, issueId: task.id, agentId: state.worker.id }); faults.push({ runId: failed.id, pid: failed.processPid, errorCode: failed.errorCode }); previousPid = active.processPid; await ctx.input.evidence(`runtime-infrastructure-attempt-${attempt}.json`, { runId: active.id, attempt, processPid: active.processPid, errorCode: failed.errorCode, status: failed.status, nativePhase: failed.nativePhase }); if ((await ctx.allRuns()).length !== 1) throw new Error("Infrastructure setup created an undeclared replacement heartbeat instead of same-run coordinator recovery"); } - const result = await pollUntil({ label: "Actual exhausted source recovery action with no automatic wake", deadlineAt: Date.now() + 30_000, - load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task.id}`), run: await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), - recovery: await ctx.input.api.get(`/api/issues/${task.id}/recovery-actions`) }), - accept: v => !!exhaustedSourceRecovery(v.recovery, v.task, v.run, ctx.input.fixtures.company.id, task.id, state.worker.id) }); - const recoveryBefore = exhaustedSourceRecovery(result.recovery, result.task, result.run, ctx.input.fixtures.company.id, task.id, state.worker.id)!; - return { ...state, task: result.task, failure: result.run, recoveryBefore, setupFaults: faults }; + return { ...state, setupFaults: faults }; +} +export async function freshlyContinuedNativeInfrastructure(ctx: ChatCatalogContext) { + const state = await nativeInfrastructureLoss(ctx), task = state.task as Entity, worker = state.worker as RuntimeAgent, gate = state.gate as RuntimeBriefGate; + const companyId = ctx.input.fixtures.company.id, lostAt = Date.now(); + const continued = await pollUntil({ label: "One fresh continuation from saved work starts and waits for the human brief", deadlineAt: Date.now() + 300_000, + load: async () => { + const run = await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`); + const recovery = await sourceRunRecovery(ctx, task.id, run.id); + const successors = (await ctx.allRuns()).filter(row => row.agentId === worker.id && row.id !== run.id); + const detail = successors.length === 1 ? await ctx.input.api.get(`/api/heartbeat-runs/${successors[0]!.id}`) : undefined; + const listener = detail && continuesFailedRun(detail, run.id, companyId, task.id, worker.id) + ? await releasedRetryListener(detail, { companyId, workerId: worker.id, taskId: task.id, originalRunId: run.id, gateName: gate.gateName! }) : null; + return { run, recovery, successors: successors.length, detail, listener }; + }, + accept: v => !!freshContinuationRecord(v.recovery, v.run, companyId, task.id, worker.id) && v.successors === 1 && v.detail?.status === "running" && !!v.listener, + reject: v => v.successors > 1 ? "More than one run continued the interrupted Website run" + : v.detail && ["failed", "cancelled", "timed_out", "succeeded"].includes(v.detail.status) ? "The fresh continuation ended before its human brief" : undefined }); + const record = freshContinuationRecord(continued.recovery, continued.run, companyId, task.id, worker.id)!, fresh = objectRecord(record.evidence.freshContinuation); + const current = await ctx.input.api.get(`/api/issues/${task.id}`); + await ctx.input.evidence("runtime-fresh-continuation.json", { sourceRunId: continued.run.id, failureCode: continued.run.errorCode, sourceFinishedAt: continued.run.finishedAt, + recordedAt: fresh.recordedAt, isolation: fresh.isolation, delivery: fresh.delivery, recoveryActionId: record.id, continuationRunId: continued.detail!.id, + continuationCreatedAt: continued.detail!.createdAt, continuationStartedAt: continued.detail!.startedAt, listenerObservedMs: Date.now() - lostAt, + continuationWorkspaceId: continued.listener!.workspaceId, originalWorkspaceId: gate.boundWorkspaceId ?? null }); + return { ...state, task: current, failure: continued.run, continuation: record, continuationRunId: continued.detail!.id, + continuationGate: continued.listener!.gate, continuationWorkspaceId: continued.listener!.workspaceId }; } export function gradeInfrastructureRecovery(value: unknown) { const c = checks(), o = runtimeObservation(value); - c.add("real-infrastructure-and-source-action-evidence", !!o, "Three actual verified native Runner losses and the resulting public source-scoped recovery action are required."); if (!o) return c.result(); - const s = o.state, task = s.task as Issue, run = s.failure as NativeFaultRun, before = s.recoveryBefore as IssueRecoveryAction, after = s.recoveryAfter as IssueRecoveryAction; - c.add("genuine-bounded-infrastructure-exhaustion", !!run && isExhaustedNativeRecovery(before, run, o.companyId, task.id, (s.worker as Entity).id) - && Array.isArray(s.setupFaults) && s.setupFaults.length === 3 && new Set(s.setupFaults.map(row => (row as { pid: number }).pid)).size === 3 + c.add("real-infrastructure-and-continuation-evidence", !!o, "Three actual verified native Runner losses and the resulting public recovery record are required."); if (!o) return c.result(); + const s = o.state, task = s.task as Issue, worker = s.worker as Entity, run = s.failure as NativeFaultRun, recorded = s.continuation as IssueRecoveryAction | undefined; + const after = freshContinuationRecord(s.recoveryAfter as SourceRecoveryActions | undefined, run, o.companyId, task?.id, worker?.id); + c.add("genuine-infrastructure-loss", !!run && run.status === "failed" && Array.isArray(s.setupFaults) && s.setupFaults.length === 3 + && new Set(s.setupFaults.map(row => (row as { pid: number }).pid)).size === 3 && (s.setupFaults as Array<{ runId: string; errorCode: string }>).every(row => row.runId === run.id && NATIVE_INFRASTRUCTURE_FAULT_CODES.includes(row.errorCode)), - "The actual original native run exhausts three coordinator attempts with real process loss evidence; model failure, cancellation or a fabricated attempt counter cannot qualify."); - c.add("retained-source-owner-cause-and-next-action", !!after && after.id === before.id && after.sourceIssueId === task.id && after.recoveryIssueId == null - && after.ownerType === "board" && after.cause === before.cause && after.nextAction === before.nextAction && after.evidence.runId === run.id - && s.sourceRecoveryVisible === true && o.tasks.length === 1 && o.runs.length === 2 - && o.runs.find(row => row.id === run.id)?.status === "failed" && o.runs.filter(row => row.agentId === o.leadId).every(row => row.status === "succeeded"), - "The original task retains its exact recovery action, responsible operator and failure; explanatory chat creates no duplicate repair chain or premature worker."); - // Reply wording is not graded. The controller must have read the saved - // recovery record, and the reply must link the original task where the - // recovery lives. A person reads the reply for meaning. + "The actual original native run loses its Runner three times with real process loss evidence; a model failure, cancellation or fabricated attempt cannot qualify."); + const continuation = s.continuationRun as HeartbeatRun | undefined; + c.add("one-fresh-continuation-without-person-action", !!after && after.id === recorded?.id && s.taskStatusAtReply !== "blocked" + && (s.activeHoldAtReply == null || (s.activeHoldAtReply as IssueRecoveryAction).ownerType === "system") + && continuesFailedRun(continuation, run.id, o.companyId, task.id, worker.id) && continuation?.id === s.continuationRunId + && o.runs.filter(row => row.agentId === worker.id && row.id !== run.id).length === 1, + "The system records exactly one fresh continuation of the interrupted run from saved work, owned by the system, and no person must act before it starts."); + c.add("continuation-finishes-from-saved-work", s.continuationResult === true && o.tasks.length === 1 + && o.tasks.some(row => row.id === task.id && row.status === "done" && row.assigneeAgentId === worker.id) + && o.runs.length === 3 && o.runs.find(row => row.id === run.id)?.status === "failed" && o.runs.find(row => row.id === s.continuationRunId)?.status === "succeeded" + && o.runs.filter(row => row.agentId === o.leadId).every(row => row.status === "succeeded"), + "The continuation actually runs to completion on the original task and saves its result; no repair task, duplicate run or Retry is introduced."); c.add("read-actual-recovery-record", s.queryRecoveryRead === true, - "The controller's own run read the original task's saved recovery record or the failed run before answering."); - c.add("links-original-task", linksSourceTask(o, s.task as Entity), - "The reply links the original task that holds the recovery action, rather than a new repair task."); + "The controller's own run read the original task's saved recovery record, its runs, or the interrupted or continuing run before answering."); + c.add("links-original-task", linksSourceTask(o, task as unknown as Entity), + "The reply links the original task that is continuing, rather than a new repair task."); return c.result(); } function infrastructureCase(id: "recovery.explain-stall" | "recovery.source-action") { - return runtimeDefinition(id, exhaustedNativeInfrastructure, gradeInfrastructureRecovery, { - oracleVersion: INFRASTRUCTURE_RECOVERY_ORACLE, expectedRunCount: 2, attemptTimeoutMs: 1_200_000, - runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 1 } }, - setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-native-infrastructure-exhaustion", "original-source-action-owner"] }, + return runtimeDefinition(id, freshlyContinuedNativeInfrastructure, gradeInfrastructureRecovery, { + oracleVersion: INFRASTRUCTURE_RECOVERY_ORACLE, expectedRunCount: 3, attemptTimeoutMs: 1_500_000, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 2 } }, + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-native-infrastructure-loss", "fresh-continuation-without-person"] }, async verifySetup(ctx, state) { const c = checks(), lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); - const run = await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), task = state.task as Issue; - const recovery = await ctx.input.api.get(`/api/issues/${task.id}/recovery-actions`); - const current = await ctx.input.api.get(`/api/issues/${task.id}`); - const action = exhaustedSourceRecovery(recovery, current, run, ctx.input.fixtures.company.id, task.id, (state.worker as Entity).id); + const run = await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), task = state.task as Issue, worker = state.worker as Entity; + const record = freshContinuationRecord(await sourceRunRecovery(ctx, task.id, run.id), run, ctx.input.fixtures.company.id, task.id, worker.id); + const continuation = await ctx.input.api.get(`/api/heartbeat-runs/${state.continuationRunId}`); c.add("lead-ready", lead.lifecycleState === "ready" && ["active", "idle"].includes(lead.status ?? ""), "The actual controller remains independently admitted."); - c.add("real-native-infrastructure-exhaustion", !!action && action.id === (state.recoveryBefore as IssueRecoveryAction).id, "The saved native run really exhausted its own three attempts."); - c.add("original-source-action-owner", action?.sourceIssueId === task.id && action.ownerType === "board" && action.recoveryIssueId == null, "The original task exposes the real operator-owned recovery without a separate repair task."); return c.result(); }, + c.add("real-native-infrastructure-loss", run.status === "failed" && NATIVE_INFRASTRUCTURE_FAULT_CODES.includes(String(run.errorCode)), "The saved native run really failed after losing its Runner."); + c.add("fresh-continuation-without-person", !!record && record.id === (state.continuation as IssueRecoveryAction).id + && continuesFailedRun(continuation, run.id, ctx.input.fixtures.company.id, task.id, worker.id) && continuation.status === "running", + "One system-owned fresh continuation of the interrupted run is running."); return c.result(); }, sourceFiles: ["chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml"], async act(ctx, state) { - await pollUntil({ label: "Canonical source explanation while native infrastructure remains failed", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), + const task = state.task as Issue, failedRunId = String(state.workerRunId), continuationRunId = String(state.continuationRunId); + await pollUntil({ label: "Canonical explanation while the fresh continuation waits for its brief", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) }); + accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.some(row => row.id === failedRunId && row.status === "failed") + && v.runs.some(row => row.id === continuationRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) }); const answered = await observeRuntime(ctx, state); state.queryReply = answered.reply; - state.queryRecoveryRead = await controllerReadRecovery(ctx, answered.runs.find(row => row.agentId === ctx.input.fixtures.agent.id)?.id, (state.task as Issue).id, String(state.workerRunId), (state.task as Issue).identifier); - const source = ctx.input.page.url(), task = state.task as Issue; + state.queryRecoveryRead = await controllerReadRecovery(ctx, answered.runs.find(row => row.agentId === ctx.input.fixtures.agent.id)?.id, task.id, failedRunId, task.identifier, continuationRunId); + state.activeHoldAtReply = (await ctx.input.api.get(`/api/issues/${task.id}/recovery-actions`)).active ?? null; + state.taskStatusAtReply = (await ctx.input.api.get(`/api/issues/${task.id}`)).status; + const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: task.title, exact: true }).first()).toBeVisible(); - await expect(ctx.input.page.getByText(/recovery|retry|failed|interrupted/i).first()).toBeVisible(); state.sourceRecoveryVisible = true; - await ctx.input.capture("actual-exhausted-source-recovery", "Original Website retains actual exhausted native infrastructure and operator recovery", "runtime-infrastructure-recovery.png"); + await ctx.input.capture("actual-fresh-continuation", "Original Website continues in a fresh run from saved work after the Runner loss", "runtime-infrastructure-recovery.png"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + await writeFile(String(state.continuationGate), `The human brief is ready. Save a non-plan website-delivery document with the exact reference Continued website ${ctx.input.nonce}, a semantic single-page Acorn website outline and an accessible signup action, then complete this original task normally. Do not create more tasks or external effects.`, "utf8"); + await pollUntil({ label: "The fresh continuation finishes the original Website task", deadlineAt: Date.now() + 240_000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns() }), + accept: v => v.task.status === "done" && v.runs.length === 3 && v.runs.some(row => row.id === continuationRunId && row.status === "succeeded"), + reject: v => v.runs.length > 3 || v.runs.some(row => row.id === continuationRunId && ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The continuation failed or duplicated Website work" : undefined }); + }, + async observe(ctx, state) { + const task = state.task as Issue; + state.recoveryAfter = await sourceRunRecovery(ctx, task.id, String(state.workerRunId)); + state.continuationRun = await ctx.input.api.get(`/api/heartbeat-runs/${state.continuationRunId}`); + const o = await observeRuntime(ctx, state); + state.continuationResult = o.revisions.some(row => row.createdByRunId === state.continuationRunId && row.createdByAgentId === (state.worker as Entity).id + && (row.body ?? "").includes(`Continued website ${ctx.input.nonce}`) && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); + return o; }, - async observe(ctx, state) { const recovery = await ctx.input.api.get(`/api/issues/${(state.task as Entity).id}/recovery-actions`); - // A newer action on the source task would be a second repair chain. - const before = (state.recoveryBefore as IssueRecoveryAction).id, latest = recovery.active ?? newest(recovery.actions ?? []); - state.recoveryAfter = latest?.id === before ? sameRecoveryAction(recovery, before) : latest; return observeRuntime(ctx, state); }, async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Issue; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); - await expect(ctx.input.page.getByRole("heading", { name: task.title, exact: true }).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, - cleanup: async (_ctx, state) => releaseRuntimeWorker(state), + await expect(ctx.input.page.getByRole("button", { name: "Change status (current: Done)", exact: true }).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, + cleanup: async (_ctx, state) => { if (state.continuationGate) await writeFile(String(state.continuationGate), "The human ended this disposable fixture. Preserve any saved work and finish normally.", "utf8"); await releaseRuntimeWorker(state); }, }); } export const runtimeInfrastructureCases = [infrastructureCase("recovery.explain-stall"), infrastructureCase("recovery.source-action")]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts index 650406ae041..e334d976efd 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts @@ -47,3 +47,26 @@ it("finds a delivered comment in a native continuation that embeds it as JSON", expect(promptCarriesComment(prompt.replace("Launch sooner", "Launch later"), comment)).toBe(false); expect(promptCarriesComment(undefined, comment)).toBe(false); }); +function inTurn() { + const o = valid(); o.runs.pop(); + Object.assign(o.state, { savedComment: { id: "headline", updatedAt: "2026-11-12T21:50:58.894Z" }, workerHistoryReads: { headline: "2026-11-12T21:50:58.894Z" }, + nativeDeliveryBound: false, finalHeadlineRevision: false, originalWebsiteRevisionRetained: false, inTurnHeadlineRevision: true }); + return o; +} +it("accepts the running Website turn reading the current headline feedback and saving it, with no continuation", () => expect(gradeMultipleHandoffs(inTurn()).passed).toBe(true)); +it("accepts the same in-turn consumption with a later read of the same revision", () => { + const o = inTurn(); (o.state as Record).workerHistoryReads = { headline: "2026-11-12T21:51:25.000Z" }; expect(gradeMultipleHandoffs(o).passed).toBe(true); +}); +it("rejects an in-turn read of an earlier revision with no continuation", () => { + const o = inTurn(); (o.state as Record).workerHistoryReads = { headline: "2026-11-12T21:50:00.000Z" }; expect(gradeMultipleHandoffs(o).passed).toBe(false); +}); +it("rejects an in-turn read whose headline was never saved", () => { const o = inTurn(); (o.state as Record).inTurnHeadlineRevision = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); +it("rejects a double application: an in-turn read and a continuation", () => { + const o = valid(); Object.assign(o.state, { savedComment: { id: "headline", updatedAt: "2026-11-12T21:50:58.894Z" }, workerHistoryReads: { headline: "2026-11-12T21:50:58.894Z" }, inTurnHeadlineRevision: true }); + expect(gradeMultipleHandoffs(o).passed).toBe(false); +}); +it("rejects two Website continuations", () => { const o = valid(); o.runs.push({ id: "again", agentId: "morgan", status: "succeeded" }); expect(gradeMultipleHandoffs(o).passed).toBe(false); }); +it("rejects an in-turn read of a different comment", () => { + const o = inTurn(); (o.state as Record).savedComment = { id: "other", updatedAt: "2026-11-12T21:50:58.894Z" }; (o.state as Record).workerHistoryReads = { other: "2026-11-12T21:50:58.894Z" }; + expect(gradeMultipleHandoffs(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts index 0e6210ce077..f4b75bdf310 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts @@ -7,7 +7,7 @@ import { readRegisteredArtifacts } from "../registered-artifact.js"; import { checks, sameTask, type Entity } from "./internal-support.js"; import { openStandingChat } from "./identity-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; -import { promptCarriesComment, replyNamesTask, runtimeQueueCases } from "./runtime-queue.js"; +import { promptCarriesComment, readCurrentRevision, replyNamesTask, runHistoryReads, runtimeQueueCases, settledQueue } from "./runtime-queue.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js"; export { promptCarriesComment } from "./runtime-queue.js"; type Artifact = Awaited>[number]; @@ -16,11 +16,16 @@ export function gradeMultipleHandoffs(value: unknown) { c.add("actual-two-task-history-and-delivery-evidence", !!o, "Separate real website/logo work, saved source references and exact consuming native input are required."); if (!o) return c.result(); const s = o.state, queue = s.queueAtReply as IssueQueuedCommentQueue, target = queue?.entries[0], logoBefore = s.logoBefore as Entity, logo = s.logoAfter as Entity; const before = s.logoArtifactBefore as Artifact, after = s.logoArtifactAfter as Artifact; + const worker = s.worker as Entity | undefined, websiteRuns = o.runs.filter(row => row.agentId === worker?.id); + const continuations = websiteRuns.filter(row => row.id !== s.workerRunId), saved = s.savedComment as { id: string; updatedAt?: unknown } | undefined; + const readInTurn = !!saved && saved.id === target?.comment.id && readCurrentRevision(s.workerHistoryReads as Record | undefined, saved); + const consumedInTurn = readInTurn && continuations.length === 0 && s.inTurnHeadlineRevision === true; + const consumedByContinuation = !readInTurn && continuations.length === 1 && s.nativeDeliveryBound === true && s.finalHeadlineRevision === true && s.originalWebsiteRevisionRetained === true; c.add("existing-website-worker-receives-one-saved-requirement", s.sourceHistoryBound === true && queue?.issueId === (s.task as Entity).id && queue.targetRunId === s.workerRunId && queue.entries.length === 1 && ["queued", "deferred"].includes(queue.state ?? "") && target?.comment.authorAgentId === o.leadId && target.comment.createdByRunId === s.canonicalRunId && target.comment.body.includes("Launch sooner") - && s.workerRunningAtReply === true && s.nativeDeliveryBound === true && s.queueEmptyAfterDelivery === true, - "One CEO-attributed headline instruction is saved against the actual active Website owner, then reaches exactly one ordinary native continuation."); + && s.workerRunningAtReply === true && (consumedInTurn || consumedByContinuation) && s.queueEmptyAfterDelivery === true, + "One CEO-attributed headline instruction is saved against the actual active Website owner and is consumed once: read at its current revision and applied by the running turn with no continuation, or delivered to exactly one ordinary continuation."); c.add("logo-task-and-delivered-logo-unchanged", !!logoBefore && !!logo && sameTask(logoBefore, logo) && logo.status === "done" && logo.assigneeAgentId === (s.logoWorker as Entity).id && before?.contentVerified === true && after?.contentVerified === true && before.attachmentId === after.attachmentId && before.sha256 === after.sha256 && before.content === after.content && after.content.includes(String(s.logoMarker)) && / row.id === (s.task as Entity).id && row.status === "done" && row.assigneeAgentId === (s.worker as Entity).id) - && o.runs.length === 5 && o.runs.every(row => row.status === "succeeded") && o.runs.filter(row => row.agentId === (s.worker as Entity).id).length === 2 + c.add("real-final-headline-and-original-work-retained", (consumedInTurn || consumedByContinuation) + && o.tasks.length === 2 && o.tasks.some(row => row.id === (s.task as Entity).id && row.status === "done" && row.assigneeAgentId === worker?.id) + && o.runs.length === 3 + websiteRuns.length && o.runs.every(row => row.status === "succeeded") && o.runs.filter(row => row.agentId === (s.logoWorker as Entity).id).length === 1, "The final actual Website revision contains the requested headline, preserves prior work, and no duplicate task, logo run or wrong-worker execution is introduced."); return c.result(); @@ -61,7 +66,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip sourceHistoryBound: comments.filter(row => !row.authorAgentId).length === 1 && comments.some(row => !row.authorAgentId && row.body?.includes(busy.task.identifier ?? busy.task.id) && row.body.includes(logo.identifier ?? logo.id) && row.body.includes(artifact.attachmentId)), setupRunIds: saved.runs.map(row => row.id) }; }, gradeMultipleHandoffs, { - setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-multiple-handoffs.v4", + setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-multiple-handoffs.v5", sourceFiles: ["chat-catalog-domains/runtime-multiple-handoffs.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/agent-chat.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { @@ -74,16 +79,21 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); state.queryReply = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === state.canonicalRunId).map(row => row.body).join("\n"); state.queueAtReply = first.queue; state.workerRunningAtReply = true; - await writeFile((state.gate as { gate: string }).gate, "The original human brief is now ready. Save the current website plan as a task document with headline Launch later and accessible labels, then finish this current turn normally. Leave the saved headline follow-up intact for its normal continuation.", "utf8"); - const final = await pollUntil({ label: "One normal Website continuation consumes the exact saved headline feedback", deadlineAt: Date.now() + 240_000, + await writeFile((state.gate as { gate: string }).gate, "The original human brief is now ready. Save the current website plan as a task document with headline Launch later and accessible labels, then finish this current turn normally.", "utf8"); + const consumedInTurn = settledQueue((v: { runs: Array<{ status: string }>; task: Entity; queue: IssueQueuedCommentQueue }) => v.runs.length === 4 + && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, 30_000); + const final = await pollUntil({ label: "The saved headline feedback is consumed once, in the running turn or one ordinary continuation", deadlineAt: Date.now() + 300_000, load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get(`/api/issues/${task.id}`), queue: await ctx.input.api.get(queuePath) }), - accept: v => v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, + accept: v => consumedInTurn(v) || v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, reject: v => v.runs.length > 5 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Headline continuation failed or duplicated/wrong-task work" : undefined }); - const consumer = final.runs.find(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId)!; - const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`), saved = first.queue.entries[0]!.comment; - state.nativeDeliveryBound = detail.contextSnapshot?.wakeCommentIds?.length === 1 && detail.contextSnapshot.wakeCommentIds[0] === saved.id + const saved = first.queue.entries[0]!.comment, current = (await ctx.input.api.get(`/api/issues/${task.id}/comments?order=asc`)).find(row => row.id === saved.id); + state.savedComment = current ? { id: current.id, updatedAt: current.updatedAt } : null; + state.workerHistoryReads = await runHistoryReads(ctx, String(state.workerRunId)); + const consumer = final.runs.find(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); + const detail = consumer && await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`); + state.nativeDeliveryBound = detail?.contextSnapshot?.wakeCommentIds?.length === 1 && detail.contextSnapshot.wakeCommentIds[0] === saved.id && promptCarriesComment(detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt, saved); - state.queueEmptyAfterDelivery = true; state.finalWebsiteRunId = consumer.id; + state.queueEmptyAfterDelivery = final.queue.entries.length === 0; state.finalWebsiteRunId = consumer?.id ?? null; }, async observe(ctx, state) { const o = await observeRuntime(ctx, state), logo = state.logoBefore as Entity; @@ -94,8 +104,9 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip state.logoRevisionsUnchanged = JSON.stringify(documents.map(doc => [doc.id, doc.latestRevisionId]).sort()) === JSON.stringify((state.logoDocumentsBefore as Entity[]).map(doc => [doc.id, doc.latestRevisionId]).sort()) && revisions.length === (state.logoRevisionsBefore as Entity[]).length && (state.logoRevisionsBefore as Entity[]).every(prior => revisions.some(row => row.id === prior.id && row.body === prior.body && row.createdByRunId === prior.createdByRunId)); state.originalWebsiteRevisionRetained = o.revisions.some(row => row.createdByRunId === state.workerRunId && /Launch later/.test(row.body ?? "")); - state.finalHeadlineRevision = o.revisions.some(row => row.createdByRunId === state.finalWebsiteRunId && row.createdByAgentId === (state.worker as Entity).id && /Launch sooner/.test(row.body ?? "") - && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); return o; + const latestHeadline = (runId: unknown) => !!runId && o.revisions.some(row => row.createdByRunId === runId && row.createdByAgentId === (state.worker as Entity).id && /Launch sooner/.test(row.body ?? "") + && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); + state.finalHeadlineRevision = latestHeadline(state.finalWebsiteRunId); state.inTurnHeadlineRevision = latestHeadline(state.workerRunId); return o; }, async verifyResult(ctx, state) { const source = ctx.input.page.url(); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.test.ts index 44306d2f373..3dd4fb93e71 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.test.ts @@ -6,8 +6,10 @@ function valid() { { position: 1, canEdit: true, comment: { id: "color", body: "Make the website red. Keep the existing note about accessible labels.", authorUserId: "human", authorAgentId: null } }, ] }; return { scenarioId: "steering.edit-queued-message", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", agents: [], tasks: [], activity: [], links: [], humanQueries: [], reply: "", - runs: [{ id: "prior", status: "succeeded" }, { id: "guide", status: "succeeded" }, { id: "consumer", status: "succeeded" }], - state: { task: { id: "website" }, queueBefore: before, targetId: "color", workerRunId: "prior", humanId: "human", concurrentMarker: "Preserve the human note.", + runs: [{ id: "prior", agentId: "morgan", status: "succeeded" }, { id: "guide", agentId: "ceo", status: "succeeded" }, { id: "consumer", agentId: "morgan", status: "succeeded" }], + state: { task: { id: "website" }, worker: { id: "morgan" }, queueBefore: before, targetId: "color", workerRunId: "prior", humanId: "human", concurrentMarker: "Preserve the human note.", + queuedComments: [{ id: "color", updatedAt: "2026-11-12T21:52:30.494Z" }, { id: "font", updatedAt: "2026-11-12T21:50:24.404Z" }], + workerHistoryReads: {} as Record, taskDone: true, inTurnOutcomeApplied: false, queueAfter: { ...before, entries: [{ ...before.entries[1]!, position: 0, comment: { ...before.entries[1]!.comment, body: "Make the website blue. Keep the existing note about accessible labels. Preserve the human note." } }, { ...before.entries[0]!, position: 1 }] }, queryReply: "Your pending red request is in the Website queue. Open its message menu, edit the full request to blue, then move it first.", @@ -46,3 +48,29 @@ it("rejects losing the initial native work", () => { const o = valid(); o.state. it.each([200, 403, 500])("rejects late edit status %s rather than replacing the real 409 assertion", status => { const o = valid(); o.state.dispatchedStatus = status; expect(gradeQueueEdit(o).passed).toBe(false); }); + +function inTurn() { + const o = valid(); o.runs.pop(); + o.state.workerHistoryReads = { color: "2026-11-12T21:52:30.494Z", font: "2026-11-12T21:50:24.404Z" }; + Object.assign(o.state, { consumingRunCount: 0, nativeDeliveryBound: false, inTurnOutcomeApplied: true }); + return o; +} +it("accepts the running turn consuming the current edited revision with no continuation", () => expect(gradeQueueEdit(inTurn()).passed).toBe(true)); +it("accepts an in-turn read made after the last edit", () => { const o = inTurn(); o.state.workerHistoryReads.color = "2026-11-12T21:52:39.000Z"; expect(gradeQueueEdit(o).passed).toBe(true); }); +it("still accepts one continuation when the running turn read only the earlier revision", () => { + const o = valid(); o.state.workerHistoryReads = { color: "2026-11-12T21:50:32.657Z" }; expect(gradeQueueEdit(o).passed).toBe(true); +}); +it("rejects an in-turn read of the earlier revision only", () => { const o = inTurn(); o.state.workerHistoryReads.color = "2026-11-12T21:50:32.657Z"; expect(gradeQueueEdit(o).passed).toBe(false); }); +it("rejects an in-turn read of only one queued message", () => { const o = inTurn(); delete (o.state.workerHistoryReads as Record).font; expect(gradeQueueEdit(o).passed).toBe(false); }); +it("rejects an in-turn read whose change was never saved", () => { const o = inTurn(); o.state.inTurnOutcomeApplied = false; expect(gradeQueueEdit(o).passed).toBe(false); }); +it("rejects a double application: an in-turn read and a continuation", () => { + const o = valid(); o.state.workerHistoryReads = { color: "2026-11-12T21:52:30.494Z", font: "2026-11-12T21:50:24.404Z" }; o.state.inTurnOutcomeApplied = true; + expect(gradeQueueEdit(o).passed).toBe(false); +}); +it("rejects two continuations", () => { + const o = valid(); o.runs.push({ id: "again", agentId: "morgan", status: "succeeded" }); o.state.consumingRunCount = 2; expect(gradeQueueEdit(o).passed).toBe(false); +}); +it("rejects consumption that leaves the queue or task unfinished", () => { + const queued = inTurn(); queued.state.queueEmptyAfterDelivery = false; expect(gradeQueueEdit(queued).passed).toBe(false); + const open = inTurn(); open.state.taskDone = false; expect(gradeQueueEdit(open).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts index 311469df81e..dcc858c8c5a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts @@ -8,7 +8,7 @@ import { submitTaskReply } from "../user-actions.js"; import { checks, hasLink, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes, waitForTurnReplyLinks } from "./identity-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; -import { keyboardMoveQueuedMessageUp, runtimeQueueCases } from "./runtime-queue.js"; +import { keyboardMoveQueuedMessageUp, readCurrentRevision, runHistoryReads, runtimeQueueCases, settledQueue } from "./runtime-queue.js"; import { observeRuntime, runtimeDefinition, runtimeObservation } from "./runtime-support.js"; const otherText = "Use a readable font with accessible labels."; @@ -70,17 +70,24 @@ export function gradeQueueEdit(value: unknown) { "An actual held old-revision human request loses to a real second-session edit and shows its conflict, retaining the winner's text."); c.add("dispatched-edit-is-visible-new-direction", s.dispatchedStatus === 409 && s.dispatchedWarningVisible === true && s.originalCommentStillReadable === true, "After actual dispatch, the old edit rejects and the real composer offers a new message rather than rewriting consumed history."); - c.add("exactly-one-normal-native-consumer", s.nativeDeliveryBound === true && s.queueEmptyAfterDelivery === true && o.runs.length === 3 - && o.runs.every(row => row.status === "succeeded") && s.consumingRunCount === 1 && s.initialRevisionRetained === true, - "One actual ordinary continuation receives the edited message once, in its final order, while original work remains preserved."); + const queued = Array.isArray(s.queuedComments) ? s.queuedComments as Array<{ id: string; updatedAt?: unknown }> : []; + const reads = s.workerHistoryReads as Record | undefined, worker = s.worker as Entity | undefined; + const continuations = o.runs.filter(row => row.agentId === worker?.id && row.id !== s.workerRunId); + const targetRead = readCurrentRevision(reads, queued.find(row => row.id === s.targetId)); + const consumedInTurn = queued.length === 2 && queued.every(row => readCurrentRevision(reads, row)) && continuations.length === 0 && o.runs.length === 2 + && s.inTurnOutcomeApplied === true; + const consumedByContinuation = !targetRead && continuations.length === 1 && s.consumingRunCount === 1 && s.nativeDeliveryBound === true && o.runs.length === 3; + c.add("edited-message-consumed-once", queued.length === 2 && queued[0]?.id === s.targetId && (consumedInTurn || consumedByContinuation) + && s.queueEmptyAfterDelivery === true && s.taskDone === true && o.runs.every(row => row.status === "succeeded") && s.initialRevisionRetained === true, + "The edited message is consumed exactly once: the running turn read its current revision and saved the change with no continuation, or one ordinary continuation receives it in its final order."); return c.result(); } export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued-message", ctx => humanQueuedRuntime(ctx), gradeQueueEdit, { sourceFiles: ["chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/steering.yaml"], - // v3: guidance graded on the task link, refused agent writes and the unchanged queue at the reply, not keywords. - oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v4", - authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 720_000, + oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v5", + authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 2 }, maximumTerminalStatusCounts: { succeeded: 3 } }, setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { await pollUntil({ label: "Canonical editing guidance while the real worker remains active", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), @@ -129,28 +136,36 @@ export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await edit(ctx.input.page, targetId, `${blueText} ${state.concurrentMarker}`); const pending = state.queueAfter as IssueQueuedCommentQueue; - await writeFile((state.gate as { gate: string }).gate, "The human brief is ready. Save the current website brief as a task document, then finish this turn normally. Leave saved follow-up directions intact for the normal continuation.", "utf8"); - const delivered = await pollUntil({ label: "Edited messages reach one ordinary native continuation", deadlineAt: Date.now() + 240_000, + await writeFile((state.gate as { gate: string }).gate, "The human brief is ready. Save the current website brief as a task document, then finish this turn normally.", "utf8"); + const consumedInTurn = settledQueue((v: { runs: Array<{ status: string }>; queue: IssueQueuedCommentQueue; task: Entity }) => v.runs.length === 2 + && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", 30_000); + const delivered = await pollUntil({ label: "Edited messages are consumed once, in the running turn or one ordinary continuation", deadlineAt: Date.now() + 300_000, load: async () => ({ runs: await ctx.allRuns(), queue: await queue(ctx, state), task: await ctx.input.api.get(`/api/issues/${(state.task as Entity).id}`) }), - accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", + accept: v => consumedInTurn(v) || v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared edited-message delivery failed or duplicated work" : undefined }); await expect(ctx.input.page.getByText("Queued message changed", { exact: true })).toBeVisible(); await expect(ctx.input.page.getByRole("button", { name: "Queue as new message", exact: true })).toBeVisible(); state.dispatchedWarningVisible = true; state.dispatchedStatus = await probeConsumedHumanQueue(ctx, String(state.humanId), mutation, "PATCH", { queueId: pending.queueId, revision: pending.revision, body: "Late stale edit must not replace history" }); const comments = await ctx.input.api.get(`/api/issues/${(state.task as Entity).id}/comments?order=asc`); state.originalCommentStillReadable = comments.some(row => row.id === targetId && row.body === `${blueText} ${state.concurrentMarker}` && row.authorUserId === state.humanId); + const otherId = initial.entries.find(row => row.comment.id !== targetId)?.comment.id; + state.queuedComments = [targetId, otherId].map(id => comments.find(row => row.id === id)).filter(Boolean).map(row => ({ id: row!.id, updatedAt: row!.updatedAt, body: row!.body })); + state.workerHistoryReads = await runHistoryReads(ctx, String(state.workerRunId)); + const unread = (state.queuedComments as Array<{ id: string; updatedAt?: unknown; body?: string }>).filter(row => !readCurrentRevision(state.workerHistoryReads as Record, row)); const consumers = delivered.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); const run = consumers[0] && await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumers[0].id}`); const ids = run?.contextSnapshot?.wakeCommentIds ?? [], prompt = run?.runnerProfileJson?.nativeExecutionInput?.task?.prompt ?? ""; - state.consumingRunCount = consumers.length; state.nativeDeliveryBound = ids.length === 2 && ids[0] === targetId - && ids[1] === initial.entries.find(row => row.comment.id !== targetId)?.comment.id && prompt.includes(`${blueText} ${state.concurrentMarker}`) && prompt.includes(otherText); - state.queueEmptyAfterDelivery = delivered.queue.entries.length === 0; + state.consumingRunCount = consumers.length; state.nativeDeliveryBound = unread.length > 0 && JSON.stringify(ids) === JSON.stringify(unread.map(row => row.id)) + && unread.every(row => prompt.includes(String(row.body))); + state.queueEmptyAfterDelivery = delivered.queue.entries.length === 0; state.taskDone = delivered.task.status === "done"; await ctx.input.capture("actual-queued-edit-stale-dispatch", "Consumed human direction remains history and stale editing offers a new message", "runtime-queued-edit.png"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, async observe(ctx, state) { const o = await observeRuntime(ctx, state); state.initialRevisionRetained = o.revisions.some(row => row.createdByRunId === state.workerRunId && row.createdByAgentId === (state.worker as Entity).id); + state.inTurnOutcomeApplied = o.revisions.some(row => row.createdByRunId === state.workerRunId && row.createdByAgentId === (state.worker as Entity).id + && /\bblue\b/i.test(row.body ?? "") && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); return o; }, async cleanup(ctx, state) { try { await cleanupIdentityProbes(ctx); } finally { await releaseRuntimeWorker(state); } }, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts index b633bc0faa9..bb20ee85147 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts @@ -1,6 +1,6 @@ -import { it, expect } from "vitest"; +import { describe, it, expect } from "vitest"; import { deflateRawSync } from "node:zlib"; -import { gradeQueuedFollowup, keyboardMoveQueuedMessageUp, promptCarriesComment, zipEntryTexts } from "./runtime-queue.js"; +import { gradeQueuedFollowup, historyReadRevisions, keyboardMoveQueuedMessageUp, promptCarriesComment, readCurrentRevision, zipEntryTexts } from "./runtime-queue.js"; /** A minimal zip with one deflated and one stored entry, laid out as zip tools write it. */ function zip(entries: Array<{ name: string; text: string; deflate: boolean }>) { const locals: Buffer[] = [], centrals: Buffer[] = []; let offset = 0; @@ -87,3 +87,19 @@ it("fails when the handle never enables or the gesture sends no order request", const handle = { ...(silent.handle as object), press: async () => undefined }; await expect(keyboardMoveQueuedMessageUp(silent.page, handle as never, 200, 20)).rejects.toThrow(/sent no queue order request/); }); + +describe("task history reads", () => { + const event = (comments: unknown, eventType = "task_history.read") => ({ eventType, payload: { comments } }); + it("keeps the newest revision each comment was read at, from history-read events only", () => { + expect(historyReadRevisions([event({ a: "2026-11-12T21:50:00.000Z" }), event({ a: "2026-11-12T21:52:30.494Z", b: "2026-11-12T21:50:24.404Z" }), + event({ a: "2026-11-12T21:49:00.000Z" }), event({ c: "2026-11-12T21:59:00.000Z" }, "tool.execution.completed"), event(["a"]), event({ d: "not a date" })])) + .toEqual({ a: "2026-11-12T21:52:30.494Z", b: "2026-11-12T21:50:24.404Z" }); + }); + it("counts a read only at or after the comment's current revision", () => { + const reads = { a: "2026-11-12T21:52:30.494Z" }; + expect(readCurrentRevision(reads, { id: "a", updatedAt: "2026-11-12T21:52:30.494Z" })).toBe(true); + expect(readCurrentRevision(reads, { id: "a", updatedAt: "2026-11-12T21:52:31.000Z" })).toBe(false); + expect(readCurrentRevision(reads, { id: "b", updatedAt: "2026-11-12T21:50:00.000Z" })).toBe(false); + expect(readCurrentRevision(reads, undefined)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts index b0950c6d01f..a5c0da82243 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -4,6 +4,8 @@ import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { inflateRawSync } from "node:zlib"; import { pollUntil } from "../api.js"; +import { collectRunEvents } from "../run-observations.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { checks, type Entity } from "./internal-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js"; @@ -58,6 +60,31 @@ function jsonCarriesComment(value: unknown, comment: { id: string; body: string return (row.id === comment.id && row.body === comment.body) || Object.values(row).some(item => jsonCarriesComment(item, comment, depth + 1)); } /** Native continuations embed new comments as JSON (an envelope may nest it again as a string), so a body with quotes or newlines appears escaped. */ +export const TASK_HISTORY_READ_EVENT = "task_history.read"; +export function historyReadRevisions(events: ReadonlyArray<{ eventType?: unknown; payload?: unknown }>) { + const reads: Record = {}; + for (const row of events) { + if (row.eventType !== TASK_HISTORY_READ_EVENT) continue; + const comments = (row.payload as { comments?: unknown } | null | undefined)?.comments; + if (!comments || typeof comments !== "object" || Array.isArray(comments)) continue; + for (const [id, at] of Object.entries(comments as Record)) + if (typeof at === "string" && Number.isFinite(Date.parse(at)) && !(Date.parse(reads[id] ?? "") >= Date.parse(at))) reads[id] = at; + } + return reads; +} +export function readCurrentRevision(reads: Record | undefined, comment: { id: string; updatedAt?: unknown } | undefined) { + if (!comment) return false; + const read = Date.parse(String(reads?.[comment.id] ?? "")), current = Date.parse(String(comment.updatedAt ?? "")); + return Number.isFinite(read) && Number.isFinite(current) && read >= current; +} +export async function runHistoryReads(ctx: ChatCatalogContext, runId: string) { + const events = await collectRunEvents<{ seq?: number; eventType?: unknown; payload?: unknown }>((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + return historyReadRevisions(events); +} +export function settledQueue(accept: (value: T) => boolean, quietMs: number) { + let since: number | undefined; + return (value: T) => { if (!accept(value)) { since = undefined; return false; } since ??= Date.now(); return Date.now() - since >= quietMs; }; +} export function promptCarriesComment(prompt: string | undefined, comment: { id: string; body: string }) { return typeof prompt === "string" && prompt.includes(comment.id) && (prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1)) || jsonCarriesComment(prompt, comment)); diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index d26aa355104..18e7be05787 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1188,6 +1188,36 @@ "field": "originalQuery", "previous": "Move this task to Riley now.", "current": "Move the Website task to Riley now." + }, + { + "key": "recovery.explain-stall", + "field": "originalContext", + "previous": "The linked task shows a retained recovery cause, attempt count and next action.", + "current": "The website run lost its Runner and could not be resumed exactly; the linked task records the interruption and continues in one fresh run from its saved work." + }, + { + "key": "recovery.explain-stall", + "field": "expectedOutcome", + "previous": "The agent reads the actual task and run evidence, explains the recorded interruption and identifies who or what can proceed. It distinguishes infrastructure recovery, user input, budget holds and deliberate stops. The person gets a concrete next step rather than a raw-log dump or an invented diagnosis.", + "current": "The agent reads the actual task and run evidence and explains the recorded interruption: an infrastructure fault cut the run off, and the task continues automatically in one fresh run from its saved work, so nobody needs to act. It distinguishes infrastructure recovery, user input, budget holds and deliberate stops, and gives no raw-log dump or invented diagnosis. Chat starts no second run and files no repair task, and the continuation finishes the original task." + }, + { + "key": "recovery.retry-exhausted", + "field": "originalContext", + "previous": "Automatic infrastructure attempts are exhausted and the source task exposes ordinary Retry to its authorized human.", + "current": "The website run stopped for a reason only a person can resolve (its selected model was rejected), and the person has since restored the configuration. The source task exposes ordinary Retry to its authorized human; an infrastructure fault would instead continue automatically." + }, + { + "key": "recovery.source-action", + "field": "originalContext", + "previous": "A failed task has a source-scoped recovery action; a separate repair issue exists only if independent work is genuinely required.", + "current": "An infrastructure fault interrupted the website run; the source task's recovery record shows the system continuing it in one fresh run from saved work. A separate repair issue exists only if independent work is genuinely required." + }, + { + "key": "recovery.source-action", + "field": "expectedOutcome", + "previous": "The agent identifies the recovery action attached to the original task and its responsible owner. Ordinary repair stays with the harness; any independent repair work links back to the source. Chat does not spawn a chain of duplicate repair tasks or claim the original task succeeded when only a repair was filed.", + "current": "The agent identifies the recovery record attached to the original task and its owner: the system is already continuing the work in a fresh run from saved work, so no person must act. Any independent repair work links back to the source. Chat does not press Retry, spawn a chain of duplicate repair tasks or claim the original task succeeded before the continuation finishes it." } ], "candidateDescriptorSnapshot": "fixtures/chat-repair-candidate-descriptors.json", @@ -10599,8 +10629,8 @@ "externalInput": null, "installedSource": null, "originalQuery": "Why did work on the website stop?", - "expectedOutcome": "The agent reads the actual task and run evidence, explains the recorded interruption and identifies who or what can proceed. It distinguishes infrastructure recovery, user input, budget holds and deliberate stops. The person gets a concrete next step rather than a raw-log dump or an invented diagnosis.", - "originalContext": "The linked task shows a retained recovery cause, attempt count and next action.", + "expectedOutcome": "The agent reads the actual task and run evidence and explains the recorded interruption: an infrastructure fault cut the run off, and the task continues automatically in one fresh run from its saved work, so nobody needs to act. It distinguishes infrastructure recovery, user input, budget holds and deliberate stops, and gives no raw-log dump or invented diagnosis. Chat starts no second run and files no repair task, and the continuation finishes the original task.", + "originalContext": "The website run lost its Runner and could not be resumed exactly; the linked task records the interruption and continues in one fresh run from its saved work.", "capabilities": [ "recovery/diagnosis", "runs-adapters/runs" @@ -10699,7 +10729,7 @@ "installedSource": null, "originalQuery": "Try the website task again.", "expectedOutcome": "The response directs or presents the supported human Retry action and explains the current state. One accepted retry rechecks execution gates and preserves earlier attempt evidence. The person is not asked to certify internal infrastructure facts, and failure remains visible if the new attempt cannot start.", - "originalContext": "Automatic infrastructure attempts are exhausted and the source task exposes ordinary Retry to its authorized human.", + "originalContext": "The website run stopped for a reason only a person can resolve (its selected model was rejected), and the person has since restored the configuration. The source task exposes ordinary Retry to its authorized human; an infrastructure fault would instead continue automatically.", "capabilities": [ "recovery/exhaustion" ], @@ -10744,8 +10774,8 @@ "externalInput": null, "installedSource": null, "originalQuery": "Help me get this website task unstuck.", - "expectedOutcome": "The agent identifies the recovery action attached to the original task and its responsible owner. Ordinary repair stays with the harness; any independent repair work links back to the source. Chat does not spawn a chain of duplicate repair tasks or claim the original task succeeded when only a repair was filed.", - "originalContext": "A failed task has a source-scoped recovery action; a separate repair issue exists only if independent work is genuinely required.", + "expectedOutcome": "The agent identifies the recovery record attached to the original task and its owner: the system is already continuing the work in a fresh run from saved work, so no person must act. Any independent repair work links back to the source. Chat does not press Retry, spawn a chain of duplicate repair tasks or claim the original task succeeded before the continuation finishes it.", + "originalContext": "An infrastructure fault interrupted the website run; the source task's recovery record shows the system continuing it in one fresh run from saved work. A separate repair issue exists only if independent work is genuinely required.", "capabilities": [ "recovery/source-action" ], diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index d55c4c1ef1c..b0ae99f389b 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -450,6 +450,26 @@ test('repair source metadata binds the corrected candidate and explicit changed [ "tasks.assignment-conflict", "originalQuery" + ], + [ + "recovery.explain-stall", + "originalContext" + ], + [ + "recovery.explain-stall", + "expectedOutcome" + ], + [ + "recovery.retry-exhausted", + "originalContext" + ], + [ + "recovery.source-action", + "originalContext" + ], + [ + "recovery.source-action", + "expectedOutcome" ] ]); for (const change of repair.changedFields) { diff --git a/tests/runner-e2e/chat-scenarios/recovery.yaml b/tests/runner-e2e/chat-scenarios/recovery.yaml index 59be891ea99..44bed14bfc3 100644 --- a/tests/runner-e2e/chat-scenarios/recovery.yaml +++ b/tests/runner-e2e/chat-scenarios/recovery.yaml @@ -9,14 +9,16 @@ scenarios: - recovery/diagnosis - runs-adapters/runs context: >- - The linked task shows a retained recovery cause, attempt count and next action. + The website run lost its Runner and could not be resumed exactly; the linked task records the + interruption and continues in one fresh run from its saved work. query: >- Why did work on the website stop? expected_outcome: >- - The agent reads the actual task and run evidence, explains the recorded interruption and - identifies who or what can proceed. It distinguishes infrastructure recovery, user input, - budget holds and deliberate stops. The person gets a concrete next step rather than a - raw-log dump or an invented diagnosis. + The agent reads the actual task and run evidence and explains the recorded interruption: an + infrastructure fault cut the run off, and the task continues automatically in one fresh run + from its saved work, so nobody needs to act. It distinguishes infrastructure recovery, user + input, budget holds and deliberate stops, and gives no raw-log dump or invented diagnosis. Chat + starts no second run and files no repair task, and the continuation finishes the original task. - id: recovery.controller-restart capabilities: - recovery/bounded-continuation @@ -35,21 +37,25 @@ scenarios: capabilities: - recovery/source-action context: >- - A failed task has a source-scoped recovery action; a separate repair issue exists only if - independent work is genuinely required. + An infrastructure fault interrupted the website run; the source task's recovery record shows + the system continuing it in one fresh run from saved work. A separate repair issue exists + only if independent work is genuinely required. query: >- Help me get this website task unstuck. expected_outcome: >- - The agent identifies the recovery action attached to the original task and its responsible - owner. Ordinary repair stays with the harness; any independent repair work links back to - the source. Chat does not spawn a chain of duplicate repair tasks or claim the original - task succeeded when only a repair was filed. + The agent identifies the recovery record attached to the original task and its owner: the + system is already continuing the work in a fresh run from saved work, so no person must act. + Any independent repair work links back to the source. Chat does not press Retry, spawn a chain + of duplicate repair tasks or claim the original task succeeded before the continuation + finishes it. - id: recovery.retry-exhausted capabilities: - recovery/exhaustion context: >- - Automatic infrastructure attempts are exhausted and the source task exposes ordinary Retry - to its authorized human. + The website run stopped for a reason only a person can resolve (its selected model was + rejected), and the person has since restored the configuration. The source task exposes + ordinary Retry to its authorized human; an infrastructure fault would instead continue + automatically. query: >- Try the website task again. expected_outcome: >-