diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json index 09ec96629b9..7fceef251ad 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -181,8 +181,8 @@ }, { "scenarioId": "recovery.retry-exhausted", - "readinessClass": "local", - "needs": "Genuine exhausted infrastructure attempts and human Retry; previous evidence retained and new admission gates independently read." + "readinessClass": "implemented", + "needs": "Same genuine three owned native-process faults/coordinator exhaustion fixture; initial ordinary human Retry guidance. Actual hard-stop budget prevents first Retry admission and retains visible failure; explicit follow-up gets its own actual gate report. Actual operator restores budget/clears recorded error only if needed, one eligible human Retry with server-owned original failed context admits one native result; exact retryOfRunId/createdAt/current gate/original failed row/latest output verified. Final explicit human query/native reply reports actual outcome. Both blocked and eligible phases authored; no live fault or Retry qualification claimed." }, { "scenarioId": "recovery.workspace-repair", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.test.ts new file mode 100644 index 00000000000..fa21c56964a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from "vitest"; +import { gradeInfrastructureRetry } from "./runtime-infrastructure-retry.js"; +function valid() { + return { scenarioId: "recovery.retry-exhausted", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website", status: "done" }], activity: [], + runs: [{ id: "exhausted", status: "failed" }, ...["canonical", "held", "retry", "final"].map(id => ({ id, status: "succeeded" }))], + state: { task: { id: "website" }, worker: { id: "morgan" }, failure: { id: "exhausted", status: "failed", nativePhase: "terminal_failure" }, + recoveryBefore: { id: "recovery", companyId: "company", sourceIssueId: "website", kind: "active_run_watchdog", cause: "native_session_retry_exhausted", ownerType: "board", ownerAgentId: null, returnOwnerAgentId: "morgan", recoveryIssueId: null, + evidence: { runId: "exhausted", coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted" }, maxAttempts: 3, wakePolicy: null, nextAction: "Inspect provider trace and explicitly choose replacement." }, queryReply: "Use the Retry button on the original failed run; this requires your human board session and rechecks current gates.", retryVisible: true, + budgetHeldBeforeRetry: true, heldRetryStatus: 409, noAdmissionUnderBudget: true, failedRunVisibleWhileHeld: true, heldReplyAttributed: true, + heldReply: "The retry could not start because Morgan's current budget hold blocks admission. The previous native failure remains visible.", + budgetRestoredByHuman: true, eligibleRetryStatus: 202, retryBoundToOriginal: true, retryRunCount: 1, retryAdmissionAfterGateClear: true, + originalFailureRetained: true, actualRetryResult: true, finalReplyAttributed: true, finalReply: "Website is complete after the eligible retry. The original exhausted attempt remains inspectable." } }; +} +it("accepts actual budget rejection, one eligible retry and retained failure", () => expect(gradeInfrastructureRetry(valid()).passed).toBe(true)); +it("accepts substantive natural human guidance and later factual reports", () => { const o = valid(); o.state.queryReply = "You can try again using the task's supported Retry control. It checks today's execution conditions."; o.state.heldReply = "Morgan has reached the spending cap, so no new execution was admitted."; o.state.finalReply = "The Website delivery is ready; its earlier failed attempts remain in the run history."; expect(gradeInfrastructureRetry(o).passed).toBe(true); }); +it("rejects asking the person to certify internal cleanup", () => { const o = valid(); o.state.queryReply += " First attest that infrastructure is healthy."; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects a retry admitted under the actual hard stop", () => { const o = valid(); o.state.noAdmissionUnderBudget = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects a hidden original failure while Retry cannot start", () => { const o = valid(); o.state.failedRunVisibleWhileHeld = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects falsely reporting the held attempt as successful", () => { const o = valid(); o.state.heldReply = "The retry succeeded and Website is now complete despite its budget limit."; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects an unbound held-phase reply", () => { const o = valid(); o.state.heldReplyAttributed = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects retrying a different task or failed attempt", () => { const o = valid(); o.state.retryBoundToOriginal = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects duplicate eligible retries", () => { const o = valid(); o.state.retryRunCount = 2; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects admission predating actual gate restoration", () => { const o = valid(); o.state.retryAdmissionAfterGateClear = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects rewriting the exhausted attempt into success", () => { const o = valid(); o.state.originalFailureRetained = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects completion without the actual retry's saved output", () => { const o = valid(); o.state.actualRetryResult = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects borrowing the pre-retry reply for final completion", () => { const o = valid(); o.state.finalReplyAttributed = false; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); +it("rejects absent native retry evidence", () => expect(gradeInfrastructureRetry(undefined).passed).toBe(false)); + +it("rejects substituting a terminal model failure for actual infrastructure exhaustion", () => { const o = valid(); o.state.recoveryBefore.cause = "native_provider_model_rejected"; expect(gradeInfrastructureRetry(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.ts new file mode 100644 index 00000000000..86e7d0db5f6 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure-retry.ts @@ -0,0 +1,103 @@ +import { expect } from "@playwright/test"; +import { writeFile } from "node:fs/promises"; +import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; +import type { Issue, IssueRecoveryAction } from "../../../packages/shared/src/types/issue.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import { releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { exhaustedNativeInfrastructure, isExhaustedNativeRecovery, runtimeInfrastructureCases } from "./runtime-infrastructure.js"; +import type { NativeFaultRun } from "./runtime-native-fault.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js"; +export function gradeInfrastructureRetry(value: unknown) { + const c = checks(), o = runtimeObservation(value); + c.add("actual-exhausted-retry-and-current-gates", !!o, "The genuine exhausted native run, real human Retry controls and independently observed admission gates are required."); if (!o) return c.result(); + const s = o.state; + c.add("actual-original-infrastructure-exhaustion", !!s.failure && isExhaustedNativeRecovery(s.recoveryBefore as IssueRecoveryAction, s.failure as NativeFaultRun, o.companyId, (s.task as Entity).id, (s.worker as Entity).id), + "The original task really exposes the exhausted three-attempt native infrastructure action; an unavailable-model failure is insufficient."); + c.add("ordinary-human-retry-guidance", /retry|try again/i.test(String(s.queryReply)) && /human|board|operator|you|button|control/i.test(String(s.queryReply)) + && !/certify|attest|confirm.{0,30}(?:infrastructure|runner|cleanup).{0,30}(?:healthy|safe|stopped)/i.test(String(s.queryReply)) && s.retryVisible === true, + "The canonical reply presents ordinary human Retry, without requiring the person to certify internal infrastructure facts."); + c.add("actual-closed-gate-prevents-new-execution", s.budgetHeldBeforeRetry === true && [202, 409].includes(Number(s.heldRetryStatus)) + && s.noAdmissionUnderBudget === true && s.failedRunVisibleWhileHeld === true && s.heldReplyAttributed === true + && /budget|spend|cap|limit/i.test(String(s.heldReply)) && !/successfully retried|website (?:is|has been) (?:now )?(?:done|complete)|retry (?:has )?succeeded/i.test(String(s.heldReply)), + "The first real Retry cannot start under the independently saved budget hold, retains failure visibly, and its own subsequent chat reply reports that actual gate."); + c.add("one-accepted-eligible-retry-preserves-original-failure", s.budgetRestoredByHuman === true && s.eligibleRetryStatus === 202 && s.retryBoundToOriginal === true && s.retryRunCount === 1 + && s.retryAdmissionAfterGateClear === true && s.originalFailureRetained === true && s.actualRetryResult === true + && o.tasks.length === 1 && o.tasks[0]?.id === (s.task as Entity).id && o.tasks[0]?.status === "done" + && o.runs.length === 5 && o.runs.filter(row => row.status === "failed").length === 1 && o.runs.filter(row => row.status === "succeeded").length === 4, + "Once the actual human restores the gate, exactly one eligible native retry uses the server-owned failed task context, retains that original failure and saves the real Website result."); + c.add("own-final-phase-reports-actual-result", s.finalReplyAttributed === true && /website/i.test(String(s.finalReply)) && /done|complete|finish|ready|delivered/i.test(String(s.finalReply)) + && !/still (?:blocked|waiting)|not yet (?:done|finished)/i.test(String(s.finalReply)), + "A separately declared factual follow-up receives its own native reply describing the actual completed retry result."); + return c.result(); +} +export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retry-exhausted", exhaustedNativeInfrastructure, gradeInfrastructureRetry, { + authentication: "human-session", expectedHumanMessageCount: 3, expectedRunCount: 5, attemptTimeoutMs: 1_800_000, + setupPolicy: runtimeInfrastructureCases[0]!.setupPolicy, verifySetup: runtimeInfrastructureCases[0]!.verifySetup, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 4 } }, + sourceFiles: ["chat-catalog-domains/runtime-infrastructure-retry.ts", "chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml"], + async act(ctx, state) { + const task = state.task as Issue, worker = state.worker as RuntimeAgent; + const initial = await pollUntil({ label: "Canonical human Retry guidance after actual infrastructure exhaustion", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); + state.queryReply = (await observeRuntime(ctx, state)).reply; const source = ctx.input.page.url(), root = `/api/companies/${ctx.input.fixtures.company.id}`; + const taskURL = `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`; + await ctx.input.page.goto(taskURL, { waitUntil: "domcontentloaded" }); + const retry = ctx.input.page.getByTestId("task-chat-run-failed-try-again").last(); await expect(retry).toBeVisible(); state.retryVisible = true; + await ctx.input.api.post(`${root}/budgets/policies`, { scopeType: "agent", scopeId: worker.id, amount: 0, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "block" }); + const held = await pollUntil({ label: "Real current budget hold before first human Retry", deadlineAt: Date.now() + 45_000, load: () => ctx.input.api.get(`/api/agents/${worker.id}`), accept: row => row.lifecycleHolds?.includes("budget") === true }); + state.budgetHeldBeforeRetry = held.lifecycleHolds?.includes("budget") === true; + const rejected = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `/api/agents/${worker.id}/wakeup`); + await retry.click(); state.heldRetryStatus = (await rejected).status(); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-run-failed-try-again").last()).toBeVisible(); state.failedRunVisibleWhileHeld = true; + state.noAdmissionUnderBudget = (await ctx.allRuns()).length === 2; + await ctx.input.capture("actual-retry-gate-held", "Original native failure remains visible when current budget prevents Retry admission", "runtime-retry-budget-held.png"); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + const heldQuery = `I tried the supported Retry on Website ${task.identifier ?? task.id} while Morgan's actual budget is held. Did a new attempt start, and what current gate is stopping it?`; + await sendChatMessage(ctx.input.page, heldQuery); + const heldPhase = await pollUntil({ label: "Own chat phase reports actual non-admission under current budget", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.filter(row => row.agentId === ctx.input.fixtures.agent.id).every(row => row.status === "succeeded") }); + const phaseReply = async (query: string, runId: string) => { + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = comments.filter(row => !row.authorAgentId && row.body === query), replies = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === runId); + const detail = await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${runId}`); + return { attributed: human.length === 1 && replies.length > 0 && detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id) === true, body: replies.map(row => row.body).join("\n") }; + }; + const heldRun = heldPhase.runs.find(row => !initial.runs.some(prior => prior.id === row.id) && row.agentId === ctx.input.fixtures.agent.id); + if (!heldRun) throw new Error("Held retry response lacks an actual subsequent native turn"); + const heldReply = await phaseReply(heldQuery, heldRun.id); state.heldReply = heldReply.body; state.heldReplyAttributed = heldReply.attributed; + await ctx.input.api.post(`${root}/budgets/policies`, { scopeType: "agent", scopeId: worker.id, amount: 500, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "allow" }); + let restored = await ctx.input.api.get(`/api/agents/${worker.id}`); + if (restored.status === "error") { await ctx.input.api.post(`/api/agents/${worker.id}/clear-error`, {}); restored = await ctx.input.api.get(`/api/agents/${worker.id}`); } + state.budgetRestoredByHuman = !restored.lifecycleHolds?.includes("budget"); state.gatesClearedAt = Date.now(); + await writeFile((state.gate as { gate: string }).gate, `The authorized human has restored the execution gate. Use the preserved task context and save a non-plan website-delivery document with the exact reference Retry website ${ctx.input.nonce}, a semantic single-page Acorn website outline and an accessible signup action, then complete this original task normally. Do not create more tasks or external effects.`, "utf8"); + await ctx.input.page.goto(taskURL, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + const accepted = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `/api/agents/${worker.id}/wakeup`); + await ctx.input.page.getByTestId("task-chat-run-failed-try-again").last().click(); const acceptedResponse = await accepted; state.eligibleRetryStatus = acceptedResponse.status(); + const completed = await pollUntil({ label: "One eligible ordinary native Retry after actual gate restoration", deadlineAt: Date.now() + 240_000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns() }), + accept: v => v.task.status === "done" && v.runs.length === 4 && v.runs.filter(row => row.id !== state.workerRunId).every(row => row.status === "succeeded"), + reject: v => v.runs.length > 4 || v.runs.some(row => row.id !== state.workerRunId && ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Eligible Retry failed or duplicated native work" : undefined }); + const retried = completed.runs.filter(row => row.agentId === worker.id && row.id !== state.workerRunId); state.retryRunCount = retried.length; + const full = retried[0] && await ctx.input.api.get(`/api/heartbeat-runs/${retried[0].id}`); state.retryRunId = full?.id; + state.retryBoundToOriginal = full?.retryOfRunId === state.workerRunId && [full.contextSnapshot?.issueId, full.contextSnapshot?.taskId].includes(task.id); + state.retryAdmissionAfterGateClear = !!full && Date.parse(String(full.createdAt)) >= Number(state.gatesClearedAt); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + const finalQuery = `I restored Morgan's budget and used the supported Retry for Website ${task.identifier ?? task.id}. What is the actual result now, and is the earlier exhausted attempt still retained?`; + await sendChatMessage(ctx.input.page, finalQuery); + const final = await pollUntil({ label: "Own final source reply reports actual eligible Retry result", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.filter(row => row.id !== state.workerRunId).every(row => row.status === "succeeded") }); + const finalRun = final.runs.find(row => !completed.runs.some(prior => prior.id === row.id))!; + const finalReply = await phaseReply(finalQuery, finalRun.id); state.finalReply = finalReply.body; state.finalReplyAttributed = finalReply.attributed; + }, + async observe(ctx, state) { const o = await observeRuntime(ctx, state); + const before = state.failure as HeartbeatRun, retained = await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`); + state.originalFailureRetained = retained.id === before.id && retained.status === "failed" && retained.errorCode === before.errorCode && retained.finishedAt === before.finishedAt; + state.actualRetryResult = o.revisions.some(row => row.createdByRunId === state.retryRunId && row.createdByAgentId === (state.worker as Entity).id && (row.body ?? "").includes(`Retry website ${ctx.input.nonce}`) + && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); return o; }, + async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Issue; + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("button", { name: "Change status (current: Done)", exact: true }).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, + cleanup: async (_ctx, state) => releaseRuntimeWorker(state), +})];