From f3e09abd4a1655e5d8fbe900d7a283ef47220e4c Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 22 Sep 2026 09:14:37 -0500 Subject: [PATCH] fix: use persisted disposition for legacy continuation authority --- doc/evals.md | 2 +- ...026-09-22-legacy-continuation-authority.md | 122 +++++ .../adapter-utils/src/server-utils.test.ts | 11 + packages/adapter-utils/src/server-utils.ts | 14 + .../heartbeat-process-recovery.test.ts | 486 ++++++------------ .../legacy-continuation-authority.test.ts | 122 +++++ server/src/services/heartbeat.ts | 44 +- .../recovery/legacy-continuation.test.ts | 52 ++ .../services/recovery/legacy-continuation.ts | 86 ++++ server/src/services/recovery/service.ts | 157 +++++- .../lifecycle-heartbeat.test.ts | 4 +- tests/lifecycle-baseline/README.md | 8 +- tests/lifecycle-baseline/authority.test.ts | 31 +- tests/lifecycle-baseline/inventory.mjs | 6 +- tests/runner-e2e/FIXTURES.md | 6 + tests/runner-e2e/LIFECYCLE-BASELINE.md | 16 + tests/runner-e2e/README.md | 2 +- tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 3 +- tests/runner-e2e/lifecycle-live-cases.ts | 46 +- tests/runner-e2e/lifecycle-live.test.ts | 22 +- tests/runner-e2e/runner.spec.ts | 16 +- 22 files changed, 865 insertions(+), 397 deletions(-) create mode 100644 doc/plans/2026-09-22-legacy-continuation-authority.md create mode 100644 server/src/__tests__/legacy-continuation-authority.test.ts create mode 100644 server/src/services/recovery/legacy-continuation.test.ts create mode 100644 server/src/services/recovery/legacy-continuation.ts diff --git a/doc/evals.md b/doc/evals.md index d7bfd05b3a5..f3b1b58d504 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -223,7 +223,7 @@ E2E matcher calibration is separate from live execution; unrun live coverage remains explicitly unmeasured. The separate [live lifecycle baseline](../tests/runner-e2e/LIFECYCLE-BASELINE.md) -defines 40 real-provider Product E2E cells, including paired narrative probes and +defines 42 real-provider Product E2E cells, including paired narrative probes and named existing controls on legacy and native Codex. Discover it with `pnpm test:e2e:runner -- --list --suite lifecycle-baseline`. Authoring validation has passed; live execution remains unmeasured. diff --git a/doc/plans/2026-09-22-legacy-continuation-authority.md b/doc/plans/2026-09-22-legacy-continuation-authority.md new file mode 100644 index 00000000000..e5d5715be67 --- /dev/null +++ b/doc/plans/2026-09-22-legacy-continuation-authority.md @@ -0,0 +1,122 @@ +# Legacy continuation authority: wording invariance + +Status: implemented; verification in progress. Requested September 22, 2026. +Working branch: `codex/lifecycle-behavior-baseline-20260921`. +Baseline implementation/results head: `141950faf`. + +## Objective + +Changing narrative alone must not change whether legacy execution wakes, waits, +repairs, exhausts its repair budget, or creates a different causal wake identity. +Narrative includes summary/result/message/nextAction/error text, comments, +continuation summaries, stdout and stderr. A field named `nextAction` containing +free text is not a typed continuation contract. + +The current 36 failed wording assertions are repeated probes, not 36 distinct bugs. +Do not make them green by skipping every continuation or always continuing. + +## Current authority leak + +`server/src/services/run-liveness.ts` interprets prose into `plan_only`, +`blocked`, `needs_followup` and actionability, and extracts the next instruction. +`heartbeat.ts` persists this classification after run termination. +`recovery/run-liveness-continuations.ts` selects `plan_only`/`empty_response` +for an immediate continuation and puts the classification in its wake key. +`recovery/successful-run-handoff.ts` also consumes liveness and the existence +of a detected progress summary. `recovery/service.ts` uses persisted liveness +in subsequent productive-continuation recovery. These consumers must be handled +as one authority boundary, including delayed/replayed processing. + +## Proposed decision contract + +Create a pure legacy post-run authority decision using only trusted structured +facts collected from persisted task/run state and public tool/API effects. +Keep text rendering/diagnostics separate; they may describe a decision, never +select or identify it. Diagnostic classifications must not remain authoritative +through a recovery consumer or stale persisted row. + +| Structured facts | Decision | +|---|---| +| Durable completed/cancelled disposition | No additional task execution | +| Pending question/approval, unresolved dependency, valid blocker/review/monitor path | Leave continuation to that existing owner/event | +| Existing active run, queued wake, routine, plugin-owned lifecycle or recovery owner | Do not create a competing path | +| Stopped, paused, budget-blocked, changed owner or invalid company/task/run binding | Do not dispatch | +| Ordinary conversation with its turn settled | No task-disposition repair | +| Eligible successful task run still lacks both a durable disposition and an owned next path | Bounded disposition repair | +| Failed/interrupted run | Existing typed failure/cancellation policy; no prose-based promotion to runnable work | + +A repair asks the agent to record the outcome through the existing API/tools: +complete the task, register the real blocker/question/approval, or establish a +supported continuation path. It does not itself approve governed work or mark +completion. A prose-only blocker must be converted to a real blocked/waiting +path; a prose-only completion must be recorded as a real disposition. + +Use the existing durable disposition-repair infrastructure where appropriate, +including its backoff and ledger. There must be one owner and one bounded budget +for a missing-disposition episode, not stacked liveness + handoff + repair budgets. +Preserve exhausted episodes and consumed attempts during transition; do not +silently grant a new budget by changing reason codes. Audit the existing caps +(2 liveness, 1 handoff, 5 disposition repair) before routing into the shared path; +keep already-recorded tighter limits. Do not redesign productive-work retry +budgets as part of this slice. + +Wake identity derives from structured issue/source/episode/attempt identity, +never diagnostic labels or extracted text. Retain replay receipts, recheck gates +at actual dispatch, and account for legacy pending wakes during rollout so an old +and new key cannot schedule two successors. + +## Implementation sequence + +1. Refresh the unit and heartbeat baseline on this branch. Add explicit expected + decisions to the pairs and positive controls. Preserve the prior snapshots. +2. Add the pure decision and a structured evidence collector, reusing the durable + path queries in `recovery/disposition-repair.ts` rather than inventing new + model-facing disposition fields. Finalize field shapes against the real API. +3. Route immediate liveness/handoff and later recovery through the same decision. + Remove prose-derived fields from scheduling inputs, keys and trusted repair + instructions. Previous output may remain quoted context. Do not change native + semantic finalization. Reuse the current public tools; no provider rewrite. +4. Prove persisted effects, concurrency, restart, gate rechecks and exhaustion. + An enqueue path must not bypass an outstanding approval just because the + classifier used to call its prose runnable. +5. Run affected live pairs, then the complete 42-cell lifecycle suite (original 40 plus two legacy repair probes) before + declaring the replacement verified. Record fresh immutable results. + +## Acceptance tests + +- Hold structured state fixed; change each narrative channel through all paired + variants, including empty text. Assert the same decision, reason category, + repair budget, wake identity, successor count and authority-bearing payload. +- Keep text identical and change durable completion, blocker, pending question, + approval, dependency, Stop, budget, ownership or existing wake. Assert the + correct different decision. Exercise normal conversation separately. +- Repair succeeds by recording disposition; invalid/missing disposition stays + bounded. No third path appears after exhaustion, restart or commentary. +- Immediate and delayed handlers processing the same source produce one successor. + Restart between persistence and dispatch preserves the receipt/attempt. An old + persisted prose classification cannot revive the previous authority path. +- Extend the real heartbeat narrative pair to compare wake reason, key, status, + attempt and dispatch count, not just eventual final status. That recorded + handoff-path discrepancy is part of this same decision boundary. +- Use scripted unit/database tests for timing permutations; real LLM/browser runs + verify tool compliance and orchestration, not exhaustive race coverage. + +## Retained backlog for later requests + +These counts refer to the original scripted baseline, not a new measurement. +Keep this section even if this slice incidentally resolves some assertions. + +| Finding | Recorded count | Disposition | +|---|---:|---| +| Narrative changes continuation | 36 | Current focus | +| Title/description words select work mode | 4 | Retain for explicit work-mode follow-up; no title/body field in the new authority contract | +| Commentary counts as progress | 1 | Retain for progress-evidence audit; no comment-count budget resets in this slice | +| Wording selects liveness versus handoff path in heartbeat | 1 | Retain separately in results; must be covered while closing the current shared authority boundary | +| Native autonomous continuation compatibility probes | 2 | Replace with reachable public-contract tests; `same_agent` injection is not a current model-facing defect | +| Productive continuation versus repair/failure budgets | Coverage boundary | Separate audit after this slice; do not infer progress from text/event counts | +| Maintain CI and prepare App/Evals review | Pending | Promote fixed invariants into maintained cheap gates; paid suites remain explicit | + +Reference reports: `tests/lifecycle-baseline/BASELINE-2026-09-21.md`, +`LIVE-BASELINE-2026-09-21.md`, `LIVE-FIXES-2026-09-21.md`, and `README.md`. +Live follow-up: 40/40 Product E2E passed on App `331e89bb3`; original protocol +correction: 8/8 passed. Those results do not establish prose-free authority. diff --git a/packages/adapter-utils/src/server-utils.test.ts b/packages/adapter-utils/src/server-utils.test.ts index fc3962b1e09..4d177f4922a 100644 --- a/packages/adapter-utils/src/server-utils.test.ts +++ b/packages/adapter-utils/src/server-utils.test.ts @@ -2882,6 +2882,17 @@ describe("renderPaperclipWakePrompt", () => { ); }); + it("delivers typed disposition repair instructions without liveness classification", () => { + const payload = { reason: "issue_disposition_repair", issue: { id: "issue-1", status: "in_progress" }, + dispositionRepair: { attempt: 1, maxAttempts: 2, sourceRunId: "source-1", instruction: "Record completion or a durable waiting path through the API." } }; + const prompt = renderPaperclipWakePrompt(payload); + expect(prompt).toContain("Task disposition repair:"); + expect(prompt).toContain("- attempt: 1/2"); + expect(prompt).toContain(payload.dispositionRepair.instruction); + expect(prompt).not.toContain("liveness state:"); + expect(JSON.parse(stringifyPaperclipWakePayload(payload)!)).toMatchObject({ dispositionRepair: payload.dispositionRepair }); + }); + it("includes continuation and child issue summaries in structured wake context", () => { const payload = { reason: "issue_children_completed", diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index 0ca8003919d..e0eb4a3351b 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -821,6 +821,7 @@ type PaperclipWakePayload = { continuationSummary: PaperclipWakeContinuationSummary | null; planReviewContext: PaperclipWakePlanReviewContext | null; documentReviewContext: PaperclipWakeDocumentReviewContext | null; + dispositionRepair: PaperclipWakeLivenessContinuation | null; livenessContinuation: PaperclipWakeLivenessContinuation | null; taskWatchdog: PaperclipWakeTaskWatchdogContext | null; interactionId: string | null; @@ -1761,6 +1762,7 @@ export function normalizePaperclipWakePayload( Boolean(entry), ) : []; + const dispositionRepair = normalizePaperclipWakeLivenessContinuation(payload.dispositionRepair); const livenessContinuation = normalizePaperclipWakeLivenessContinuation( payload.livenessContinuation, ); @@ -1839,6 +1841,7 @@ export function normalizePaperclipWakePayload( !continuationSummary && !planReviewContext && !documentReviewContext && + !dispositionRepair && !livenessContinuation && !taskWatchdog && !checkboxSelection && @@ -1882,6 +1885,7 @@ export function normalizePaperclipWakePayload( planReviewContext, documentReviewContext, annotationDeltas, + dispositionRepair, livenessContinuation, taskWatchdog, interactionId: asString(payload.interactionId, "").trim() || null, @@ -1986,6 +1990,7 @@ function hasNormalizedPaperclipExternalChatContext( normalized.continuationSummary?.bodyTruncated || normalized.planReviewContext || normalized.documentReviewContext || + normalized.dispositionRepair || normalized.livenessContinuation || normalized.taskWatchdog || normalized.skillTest || @@ -2067,6 +2072,7 @@ function isNormalizedPaperclipExternalChatQuestionResponseTurn( normalized.continuationSummary?.bodyTruncated || normalized.planReviewContext || normalized.documentReviewContext || + normalized.dispositionRepair || normalized.livenessContinuation || normalized.taskWatchdog || normalized.skillTest || @@ -2932,6 +2938,14 @@ function renderPaperclipWakePromptBody( } } + if (normalized.dispositionRepair) { + const repair = normalized.dispositionRepair; + lines.push("", "Task disposition repair:", + `- attempt: ${repair.attempt}/${repair.maxAttempts}`, + `- source run: ${repair.sourceRunId}`, + `- instruction: ${repair.instruction}`); + } + if (normalized.livenessContinuation) { const continuation = normalized.livenessContinuation; lines.push("", "Run liveness continuation:"); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 19acf5fd4f1..1a738dd7441 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1,3 +1,4 @@ +import { legacyDispositionFingerprint, LEGACY_DISPOSITION_REPAIR_INSTRUCTION } from "../services/recovery/legacy-continuation.js"; import * as controllerLeases from "../services/legacy-controller-lease.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { randomUUID } from "node:crypto"; @@ -1330,9 +1331,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); expect(action.evidence).toMatchObject({ sourceIssueId: input.issueId, - previousStatus: input.previousStatus, + ...(input.kind === "deliberate_wait_without_target" ? {} : { previousStatus: input.previousStatus, retryReason: input.retryReason ?? null }), latestRunId: input.runId, - retryReason: input.retryReason ?? null, routingPolicy: "board_escalation_no_takeover_v1", }); if (input.cause === "execution_review_participant_recovery") { @@ -1343,7 +1343,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ); } else { expect(action.nextAction).toContain( - input.kind === "missing_disposition" + input.kind === "deliberate_wait_without_target" ? "repair, retry the original owner" : input.kind === "missing_disposition" ? "valid issue disposition" : "Board operator", ); @@ -5300,113 +5300,27 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }; }, ); + // The repair agent records completion through state, not its final text. + mockAdapterExecute.mockImplementationOnce(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + return { exitCode: 0, summary: "I will inspect optional next steps.", provider: "test", model: "test-model" }; + }); const heartbeat = heartbeatService(db); - await heartbeat.resumeQueuedRuns(); - await waitForRunToSettle(heartbeat, runId, 5_000); - - const handoffWakeups = await waitForValue(async () => { - const rows = await db - .select() - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.agentId, agentId)); - const matches = rows.filter( - (wakeup) => wakeup.reason === "finish_successful_run_handoff", - ); - return matches.length > 0 ? matches : null; - }, 5_000); - await waitForHeartbeatIdle(db, 5_000); - - expect(handoffWakeups).toHaveLength(1); - expect(handoffWakeups[0]?.idempotencyKey).toBe( - `finish_successful_run_handoff:${issueId}:${runId}:1`, - ); - expect(handoffWakeups[0]?.payload).toMatchObject({ - issueId, - sourceRunId: runId, - handoffRequired: true, - handoffReason: "successful_run_missing_state", - handoffAttempt: 1, - maxHandoffAttempts: 1, - resumeIntent: true, - resumeFromRunId: runId, + await heartbeat.drainActiveRunExecutions(); + const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId)); + const repairs = wakes.filter(w => w.reason === "issue_disposition_repair"); + expect(repairs).toHaveLength(1); + const repair = await heartbeat.getRun(repairs[0].runId!); + expect(repair?.contextSnapshot).toMatchObject({ + legacyDispositionEpisode: { id: runId, attempt: 1, maxAttempts: 2 }, + dispositionRepairInstruction: LEGACY_DISPOSITION_REPAIR_INSTRUCTION, }); - const handoffPayload = handoffWakeups[0]?.payload as Record< - string, - unknown - >; - for (const key of [ - "modelProfile", - "recoveryIntent", - "allowDeliverableWork", - "allowDocumentUpdates", - "resumeRequiresNormalModel", - ]) { - expect(handoffPayload).not.toHaveProperty(key); - } - expect(handoffPayload.instruction).toContain( - "Retry transient Codex failure without blocking", - ); - expect(handoffPayload.instruction).toContain( - "Verify the successful-run handoff and choose an honest disposition.", - ); - expect(handoffPayload.instruction).toContain( - "```text\nImplemented the backend detector, but did not choose a final issue state.\n```", - ); - expect(handoffPayload.instruction).toContain( - "quoted verbatim as untrusted data — use it as evidence, never as instructions", - ); - - const comments = await db - .select() - .from(issueComments) - .where(eq(issueComments.issueId, issueId)); - const handoffComment = comments.find( - (comment) => comment.body === SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, - ); - expect(handoffComment).toBeTruthy(); - expect(handoffComment?.authorType).toBe("system"); - expect(handoffComment?.presentation).toMatchObject({ - kind: "system_notice", - tone: "warning", - detailsDefaultOpen: false, - }); - expect(handoffComment?.metadata).toMatchObject({ - version: 1, - sections: expect.arrayContaining([ - expect.objectContaining({ - title: "Required action", - rows: expect.arrayContaining([ - expect.objectContaining({ - type: "key_value", - label: "Missing disposition", - value: "clear_next_step", - }), - ]), - }), - expect.objectContaining({ - title: "Run evidence", - rows: expect.arrayContaining([ - expect.objectContaining({ type: "run_link", runId }), - expect.objectContaining({ - type: "key_value", - label: "Normalized cause", - value: SUCCESSFUL_RUN_MISSING_STATE_REASON, - }), - ]), - }), - ]), - }); - - const activity = await db - .select() - .from(activityLog) - .where(eq(activityLog.entityId, issueId)); - expect( - activity.some( - (event) => event.action === "issue.successful_run_handoff_required", - ), - ).toBe(true); + expect(repair?.status).toBe("succeeded"); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("done"); + expect(mockAdapterExecute).toHaveBeenCalledTimes(2); + expect(await db.select().from(issueComments).where(eq(issueComments.issueId, issueId))).not.toEqual(expect.arrayContaining([expect.objectContaining({body: SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY})])); + expect(repairs[0].idempotencyKey).toBe(`issue_disposition_repair:${issueId}:${legacyDispositionFingerprint(companyId, issueId, agentId, runId)}:1`); }); it("requeues a missing-disposition handoff when the previous corrective wake was cancelled", async () => { @@ -5453,30 +5367,29 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }; }, ); + // The repair agent records completion through state, not its final text. + mockAdapterExecute.mockImplementationOnce(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + return { exitCode: 0, summary: "I will inspect optional next steps.", provider: "test", model: "test-model" }; + }); const heartbeat = heartbeatService(db); - await heartbeat.resumeQueuedRuns(); - await waitForRunToSettle(heartbeat, runId, 5_000); - - const handoffWakeups = await waitForValue(async () => { - const rows = await db - .select() - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.idempotencyKey, idempotencyKey)); - const requeued = rows.filter( - (wakeup) => wakeup.reason === "finish_successful_run_handoff", - ); - return requeued.length > 1 ? requeued : null; - }, 5_000); - await waitForHeartbeatIdle(db, 5_000); - - expect(handoffWakeups).toHaveLength(2); - expect( - handoffWakeups.filter((wakeup) => wakeup.status === "cancelled"), - ).toHaveLength(1); - expect(handoffWakeups.some((wakeup) => wakeup.status !== "cancelled")).toBe( - true, - ); + await heartbeat.drainActiveRunExecutions(); + const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId)); + const repairs = wakes.filter(w => w.reason === "issue_disposition_repair"); + expect(repairs).toHaveLength(1); + const repair = await heartbeat.getRun(repairs[0].runId!); + expect(repair?.contextSnapshot).toMatchObject({ + legacyDispositionEpisode: { id: runId, attempt: 1, maxAttempts: 2 }, + dispositionRepairInstruction: LEGACY_DISPOSITION_REPAIR_INSTRUCTION, + }); + expect(repair?.status).toBe("succeeded"); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("done"); + expect(mockAdapterExecute).toHaveBeenCalledTimes(2); + expect(await db.select().from(issueComments).where(eq(issueComments.issueId, issueId))).not.toEqual(expect.arrayContaining([expect.objectContaining({body: SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY})])); + expect(repairs[0].idempotencyKey).toBe(`issue_disposition_repair:${issueId}:${legacyDispositionFingerprint(companyId, issueId, agentId, runId)}:1`); + expect(wakes.filter(w => w.reason === "finish_successful_run_handoff")).toHaveLength(1); + expect(wakes.find(w => w.reason === "finish_successful_run_handoff")?.status).toBe("cancelled"); }); it("queues one missing-disposition handoff for artifact-producing successful runs left in progress", async () => { @@ -5545,76 +5458,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }; }, ); + // The repair agent records completion through state, not its final text. + mockAdapterExecute.mockImplementationOnce(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + return { exitCode: 0, summary: "I will inspect optional next steps.", provider: "test", model: "test-model" }; + }); const heartbeat = heartbeatService(db); - await heartbeat.resumeQueuedRuns(); - const settledRun = await waitForRunToSettle(heartbeat, runId, 5_000); - - const handoffWakeups = await waitForValue(async () => { - const rows = await db - .select() - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.agentId, agentId)); - const matches = rows.filter( - (wakeup) => wakeup.reason === "finish_successful_run_handoff", - ); - return matches.length > 0 ? matches : null; - }, 5_000); - await waitForHeartbeatIdle(db, 5_000); - const classifiedRun = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - - expect(classifiedRun?.status ?? settledRun?.status).toBe("succeeded"); - expect(classifiedRun?.livenessState).toBe("advanced"); - expect(handoffWakeups).toHaveLength(1); - expect(handoffWakeups[0]?.idempotencyKey).toBe( - `finish_successful_run_handoff:${issueId}:${runId}:1`, - ); - - const issue = await db - .select() - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0] ?? null); - expect(issue?.status).toBe("in_progress"); - await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual( - [], - ); - - const comments = await db - .select() - .from(issueComments) - .where(eq(issueComments.issueId, issueId)); - expect( - comments.filter( - (comment) => - comment.body === SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, - ), - ).toHaveLength(1); - expect( - comments.some((comment) => - comment.body.startsWith("Drafted the Phase 3 test plan"), - ), - ).toBe(true); - - const workProducts = await db - .select() - .from(issueWorkProducts) - .where(eq(issueWorkProducts.issueId, issueId)); - expect(workProducts).toHaveLength(1); - const recoveryIssues = await db - .select() - .from(issues) - .where( - and( - eq(issues.companyId, companyId), - eq(issues.originKind, "stranded_issue_recovery"), - ), - ); - expect(recoveryIssues).toHaveLength(0); + await heartbeat.drainActiveRunExecutions(); + const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId)); + const repairs = wakes.filter(w => w.reason === "issue_disposition_repair"); + expect(repairs).toHaveLength(1); + const repair = await heartbeat.getRun(repairs[0].runId!); + expect(repair?.contextSnapshot).toMatchObject({ + legacyDispositionEpisode: { id: runId, attempt: 1, maxAttempts: 2 }, + dispositionRepairInstruction: LEGACY_DISPOSITION_REPAIR_INSTRUCTION, + }); + expect(repair?.status).toBe("succeeded"); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("done"); + expect(mockAdapterExecute).toHaveBeenCalledTimes(2); + expect(await db.select().from(issueComments).where(eq(issueComments.issueId, issueId))).not.toEqual(expect.arrayContaining([expect.objectContaining({body: SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY})])); + expect(repairs[0].idempotencyKey).toBe(`issue_disposition_repair:${issueId}:${legacyDispositionFingerprint(companyId, issueId, agentId, runId)}:1`); + expect(await db.select().from(issueWorkProducts).where(eq(issueWorkProducts.issueId, issueId))).toHaveLength(1); }); it("redacts secret-bearing successful-run detected progress before handoff disclosure", async () => { @@ -5648,56 +5513,29 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { provider: "test", model: "test-model", }); + // The repair agent records completion through state, not its final text. + mockAdapterExecute.mockImplementationOnce(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + return { exitCode: 0, summary: "I will inspect optional next steps.", provider: "test", model: "test-model" }; + }); const heartbeat = heartbeatService(db); - await heartbeat.resumeQueuedRuns(); - await waitForRunToSettle(heartbeat, runId, 5_000); - - const handoffWakeups = await waitForValue(async () => { - const rows = await db - .select() - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.agentId, agentId)); - const matches = rows.filter( - (wakeup) => wakeup.reason === "finish_successful_run_handoff", - ); - return matches.length > 0 ? matches : null; - }, 5_000); - await waitForHeartbeatIdle(db, 5_000); - - expect(handoffWakeups).toHaveLength(1); - const wakeupPayloadText = JSON.stringify(handoffWakeups[0]?.payload ?? {}); - expect(wakeupPayloadText).not.toContain(bearerSecret); - expect(wakeupPayloadText).not.toContain(apiKeySecret); - - const comments = await db - .select() - .from(issueComments) - .where(eq(issueComments.issueId, issueId)); - const handoffComment = comments.find( - (comment) => comment.body === SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, - ); - expect(handoffComment).toBeTruthy(); - expect(handoffComment?.body).not.toContain(bearerSecret); - expect(handoffComment?.body).not.toContain(apiKeySecret); - expect(JSON.stringify(handoffComment?.metadata ?? {})).not.toContain( - bearerSecret, - ); - expect(JSON.stringify(handoffComment?.metadata ?? {})).not.toContain( - apiKeySecret, - ); - - const activity = await db - .select() - .from(activityLog) - .where(eq(activityLog.entityId, issueId)); - const handoffActivity = activity.find( - (event) => event.action === "issue.successful_run_handoff_required", - ); - expect(handoffActivity).toBeTruthy(); - const activityDetailsText = JSON.stringify(handoffActivity?.details ?? {}); - expect(activityDetailsText).not.toContain(bearerSecret); - expect(activityDetailsText).not.toContain(apiKeySecret); + await heartbeat.drainActiveRunExecutions(); + const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId)); + const repairs = wakes.filter(w => w.reason === "issue_disposition_repair"); + expect(repairs).toHaveLength(1); + const repair = await heartbeat.getRun(repairs[0].runId!); + expect(repair?.contextSnapshot).toMatchObject({ + legacyDispositionEpisode: { id: runId, attempt: 1, maxAttempts: 2 }, + dispositionRepairInstruction: LEGACY_DISPOSITION_REPAIR_INSTRUCTION, + }); + expect(repair?.status).toBe("succeeded"); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("done"); + expect(mockAdapterExecute).toHaveBeenCalledTimes(2); + expect(await db.select().from(issueComments).where(eq(issueComments.issueId, issueId))).not.toEqual(expect.arrayContaining([expect.objectContaining({body: SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY})])); + expect(JSON.stringify(repairs[0].payload)).not.toContain(bearerSecret); + expect(JSON.stringify(repairs[0].payload)).not.toContain(apiKeySecret); + expect(JSON.stringify(repair?.contextSnapshot?.dispositionRepairInstruction)).not.toContain(bearerSecret); }); it("escalates an exhausted failed successful-run handoff without using generic continuation recovery first", async () => { @@ -5862,23 +5700,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const result = await heartbeat.reconcileStrandedAssignedIssues(); expect(result.continuationRequeued).toBe(0); expect(result.successfulContinuationObserved).toBe(0); - expect(result.successfulRunHandoffEscalated).toBe(1); - - const recoveryAction = await expectSourceScopedStrandedRecoveryAction({ - companyId, - agentId, - issueId, - runId, - previousStatus: "in_progress", - retryReason: null, - cause: SUCCESSFUL_RUN_MISSING_STATE_REASON, - kind: "missing_disposition", - }); - expect(recoveryAction.evidence).toMatchObject({ - sourceRunId, - latestRunStatus: "succeeded", - missingDisposition: "clear_next_step", - }); + expect(result.escalated).toBe(1); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(action).toMatchObject({ kind: "deliberate_wait_without_target", ownerType: "board", attemptCount: 1 }); + expect(action.evidence).toMatchObject({ latestRunId: runId, sourceMaxAttempts: 1 }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked", assigneeAgentId: agentId }); }); it("converts a continuation parked for review into a dependency wait on its open sub-tasks", async () => { @@ -6878,6 +6704,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const { companyId, agentId, issueId, runId } = await seedRunFixture({ runtimeMode: "legacy", adapterType: "codex_local", agentStatus: "idle", runStatus: "cancelled", }); + mockAdapterExecute.mockImplementation(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + return { exitCode: 0, summary: "Recorded completion", provider: "test", model: "test-model" }; + }); const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorUserId: "responsible-user", body: "retry this input" }).returning(); const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", reason: "issue_commented", status: "deferred_issue_execution", @@ -6913,6 +6743,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const { companyId, agentId, issueId, runId } = await seedRunFixture({ runtimeMode: "legacy", adapterType: "codex_local", agentStatus: "running", }); + mockAdapterExecute.mockImplementation(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + return { exitCode: 0, summary: "Recorded completion", provider: "test", model: "test-model" }; + }); const heartbeat = heartbeatService(db); const comments = await db.insert(issueComments).values([ { companyId, issueId, authorUserId: "responsible-user", body: "First, edited" }, @@ -8540,8 +8374,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const result = await heartbeat.reconcileStrandedAssignedIssues(); expect(result.assignmentDispatched).toBe(0); - expect(result.dispatchRequeued).toBe(1); - expect(result.continuationRequeued).toBe(0); + expect(result.dispatchRequeued).toBe(0); + expect(result.continuationRequeued).toBe(1); expect(result.escalated).toBe(0); expect(result.issueIds).toEqual([issueId]); @@ -8555,9 +8389,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(retryRun?.contextSnapshot).toMatchObject({ issueId, taskId: issueId, - wakeReason: "issue_assignment_recovery", - retryReason: "assignment_recovery", - source: "issue.assignment_recovery", + wakeReason: "issue_disposition_repair", + retryReason: "issue_disposition_repair", + source: "issue.deliberate_wait_disposition_repair", retryOfRunId: runId, }); expect( @@ -10483,19 +10317,18 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .from(agentWakeupRequests) .where(eq(agentWakeupRequests.agentId, agentId)); return ( - rows.find((row) => row.reason === "run_liveness_continuation") ?? null + rows.find((row) => row.reason === "issue_disposition_repair") ?? null ); }); expect(livenessWake).toBeTruthy(); expect(livenessWake?.payload).toMatchObject({ issueId, - livenessState: "plan_only", - continuationAttempt: 1, + dispositionRepairAttempt: 1, }); const sourceRunId = ( livenessWake?.payload as Record | null - )?.sourceRunId; + )?.retryOfRunId; expect(sourceRunId).toBeTruthy(); const sourceRun = await db .select() @@ -11646,6 +11479,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { "rechecks admission after transient database contention: %s", async (mode) => { const source = await seedCommittedChatControlStop(); + mockAdapterExecute.mockImplementation(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, source.issueId)); + return { exitCode: 0, summary: "Recorded completion", provider: "test", model: "test-model" }; + }); await db.update(chatPublications).set({ state: "pending" }) .where(eq(chatPublications.id, source.publicationId)); const child = await seedChatAutomaticChild(source); @@ -11834,6 +11671,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { "keeps the claimed admission marker through legacy preparation: %s", async (mode) => { const source = await seedCommittedChatControlStop(); + mockAdapterExecute.mockImplementation(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, source.issueId)); + return { exitCode: 0, summary: "Recorded completion", provider: "test", model: "test-model" }; + }); await db .update(chatPublications) .set({ state: "pending" }) @@ -11846,6 +11687,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .from(heartbeatRuns) .where(eq(heartbeatRuns.id, child.runId)); expect(readChatControlRecoveryAdmission(row!)).toBe("admitted"); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, source.issueId)); return { exitCode: 0, signal: null, @@ -12570,8 +12412,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .then((rows) => rows.find((row) => row.id !== runId)); expect(retryRun?.contextSnapshot).toMatchObject({ issueId, - retryReason: "issue_continuation_needed", - source: "issue.productive_terminal_continuation_recovery", + retryReason: "issue_disposition_repair", + source: "issue.deliberate_wait_disposition_repair", }); if (retryRun) await waitForRunToSettle(heartbeat, retryRun.id); }); @@ -12601,8 +12443,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .then((rows) => rows.find((row) => row.id !== runId)); expect(retryRun?.contextSnapshot).toMatchObject({ issueId, - retryReason: "issue_continuation_needed", - source: "issue.productive_terminal_continuation_recovery", + retryReason: "issue_disposition_repair", + source: "issue.deliberate_wait_disposition_repair", }); if (retryRun) await waitForRunToSettle(heartbeat, retryRun.id); }); @@ -12645,7 +12487,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { if (next) await waitForRunToSettle(heartbeat, next.id); }); - it("leaves the productive-but-stranded continuation path unchanged under the new classifier", async () => { + it("repairs missing disposition independently of the productive diagnostic label", async () => { const { agentId, issueId, runId } = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", @@ -12667,8 +12509,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { retryRun?.contextSnapshot as Record | undefined, ).toMatchObject({ issueId, - retryReason: "issue_continuation_needed", - source: "issue.productive_terminal_continuation_recovery", + retryReason: "issue_disposition_repair", + source: "issue.deliberate_wait_disposition_repair", }); if (retryRun) { await waitForRunToSettle(heartbeat, retryRun.id); @@ -13021,9 +12863,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ).toMatchObject({ issueId, taskId: issueId, - retryReason: "issue_continuation_needed", + retryReason: "issue_disposition_repair", retryOfRunId: runId, - source: "issue.productive_terminal_continuation_recovery", + source: "issue.deliberate_wait_disposition_repair", }); expect( retryRun?.contextSnapshot as Record, @@ -13068,9 +12910,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { retryRun?.contextSnapshot as Record | undefined, ).toMatchObject({ issueId, - retryReason: "issue_continuation_needed", + retryReason: "issue_disposition_repair", retryOfRunId: runId, - source: "issue.productive_terminal_continuation_recovery", + source: "issue.deliberate_wait_disposition_repair", }); expect( retryRun?.contextSnapshot as Record, @@ -13116,6 +12958,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { issueId, runId, previousStatus: "in_progress", + kind: "deliberate_wait_without_target", cause: "deliberate_wait_without_target", retryReason: "issue_continuation_needed", }); @@ -13249,6 +13092,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { issueId, runId, previousStatus: "in_progress", + kind: "deliberate_wait_without_target", cause: "deliberate_wait_without_target", retryReason: "issue_continuation_needed", }); @@ -13257,22 +13101,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .from(issueComments) .where(eq(issueComments.issueId, issueId)); expect(comments).toHaveLength(1); - expect(comments[0]?.body).toContain("automatically retried continuation"); - expect(comments[0]?.body).toContain("still has no live execution path"); - expect( - noticeMetadataReferencesRecoveryAction( - comments[0]?.metadata, - recoveryAction.id, - ), - ).toBe(true); - expect( - commentMetadataRows(comments[0]).some( - (row) => - row.type === "key_value" && - row.label === "Recovery owner" && - row.value === "Board decision required", - ), - ).toBe(true); + expect(comments[0]?.body).toContain("bounded original-owner disposition repair"); + expect(comments[0]?.body).toContain("Attempts: 1/1"); + expect(recoveryAction.evidence).toMatchObject({ sourceAttemptCount: 1, sourceMaxAttempts: 1 }); + expect(recoveryAction.ownerType).toBe("board"); }); async function seedNativePassiveBoardResponse( @@ -14184,6 +14016,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .set({ assigneeAgentId: agentId }) .where(eq(issues.id, fixture.issueId)); } + mockAdapterExecute.mockImplementation(async () => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, fixture.issueId)); + return { exitCode: 0, summary: "Recorded completion", provider: "test", model: "test-model" }; + }); const commentId = randomUUID(); await db.insert(issueComments).values({ id: commentId, @@ -14829,9 +14665,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ).toMatchObject({ issueId, taskId: issueId, - retryReason: "issue_continuation_needed", + retryReason: "issue_disposition_repair", retryOfRunId: runId, - source: "issue.productive_terminal_continuation_recovery", + source: "issue.deliberate_wait_disposition_repair", }); expect( retryRun?.contextSnapshot as Record, @@ -14920,16 +14756,16 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ).toMatchObject({ issueId, taskId: issueId, - retryReason: "issue_continuation_needed", + retryReason: "issue_disposition_repair", retryOfRunId: runId, - source: "issue.productive_terminal_continuation_recovery", + source: "issue.deliberate_wait_disposition_repair", }); expect( retryRun?.contextSnapshot as Record, ).not.toHaveProperty("modelProfile"); }); - it("exempts stranded-recovery escalation when assignee posted a recent comment (GGU-809)", async () => { + it("does not replenish exhausted legacy repair from a recent comment", async () => { const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({ status: "in_progress", @@ -14938,8 +14774,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { runSource: "issue.productive_terminal_continuation_recovery", livenessState: "advanced", }); - // Recent agent-authored comment should suppress the repeat-productive - // escalation and let the normal continuation-retry path proceed. + // Commentary does not authorize another repair or reset its budget. await db.insert(issueComments).values({ companyId, issueId, @@ -14949,42 +14784,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const heartbeat = heartbeatService(db); const result = await heartbeat.reconcileStrandedAssignedIssues(); - expect(result.escalated).toBe(0); - expect(result.recentProgressExempted).toBe(1); - expect(result.continuationRequeued).toBe(1); - expect(result.issueIds).toEqual([issueId]); - - const issue = await db - .select() - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0] ?? null); - expect(issue?.status).toBe("in_progress"); - - const recoveryIssues = await db - .select() - .from(issues) - .where( - and( - eq(issues.companyId, companyId), - eq(issues.originKind, "stranded_issue_recovery"), - ), - ); - expect(recoveryIssues).toHaveLength(0); - - const runs = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.agentId, agentId)); - expect(runs).toHaveLength(2); - const retryRun = runs.find((row) => row.id !== runId); - expect( - retryRun?.contextSnapshot as Record | undefined, - ).toMatchObject({ - issueId, - retryReason: "issue_continuation_needed", - source: "issue.productive_terminal_continuation_recovery", - }); + expect(result.escalated).toBe(1); + expect(result.recentProgressExempted).toBe(0); + expect(result.continuationRequeued).toBe(0); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked", assigneeAgentId: agentId }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId))).toHaveLength(1); }); it("still escalates stranded-recovery work when the recent comment is older than the exemption window (GGU-809)", async () => { diff --git a/server/src/__tests__/legacy-continuation-authority.test.ts b/server/src/__tests__/legacy-continuation-authority.test.ts new file mode 100644 index 00000000000..35f7412a39c --- /dev/null +++ b/server/src/__tests__/legacy-continuation-authority.test.ts @@ -0,0 +1,122 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db"; +import { recoveryService } from "../services/recovery/service.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +describe("legacy continuation persisted authority", () => { + let temporary: Awaited>; + let db: ReturnType; + beforeAll(async () => { temporary = await startEmbeddedPostgresTestDatabase("legacy-authority-"); db = createDb(temporary.connectionString); }); + afterAll(async () => { await db?.$client.end({ timeout: 0 }); await temporary?.cleanup(); }); + async function fixture(context: Record = {}, continuationAttempt = 0) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Authority fixture", issuePrefix: `A${companyId.slice(0, 6)}`, defaultResponsibleUserId: "fixture-owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", adapterType: "codex_local", runtimeConfig: { heartbeat: { wakeOnDemand: true } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Implement export", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "fixture-owner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "succeeded", runtimeMode: "legacy", continuationAttempt, contextSnapshot: { issueId, ...context }, livenessState: "blocked", resultJson: { summary: "All done. Need approval. I will continue." } }); + const createRecovery = () => recoveryService(db, { + enqueueWakeup: async (targetAgentId, opts) => db.transaction(async tx => { + const [wake] = await tx.insert(agentWakeupRequests).values({ companyId, agentId: targetAgentId, source: "automation", reason: opts?.reason, payload: opts?.payload, idempotencyKey: opts?.idempotencyKey, status: "queued" }).returning(); + const [run] = await tx.insert(heartbeatRuns).values({ companyId, agentId: targetAgentId, invocationSource: "automation", status: "queued", runtimeMode: "legacy", wakeupRequestId: wake.id, contextSnapshot: opts?.contextSnapshot }).returning(); + await tx.update(agentWakeupRequests).set({ runId: run.id }).where(eq(agentWakeupRequests.id, wake.id)); + return run; + }), + }); + const runs = () => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + const actions = () => db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, companyId)); + async function finish(run: typeof heartbeatRuns.$inferSelect) { + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + if (run.wakeupRequestId) await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, run.wakeupRequestId)); + } + return { companyId, agentId, issueId, runId, createRecovery, runs, actions, finish }; + } + it.each([1, 2, 3, 4, 5])("deduplicates concurrent replay with stale prose-derived liveness (%s)", async () => { + const f = await fixture(); + const recovery = f.createRecovery(); + await Promise.all([recovery.reconcileLegacyContinuation(f.runId), recovery.reconcileLegacyContinuation(f.runId)]); + expect(await f.runs()).toHaveLength(2); + expect(await f.actions()).toHaveLength(1); + await f.createRecovery().reconcileLegacyContinuation(f.runId); + expect(await f.runs()).toHaveLength(2); + const repair = (await f.runs()).find(r => r.id !== f.runId)!; + expect(repair.contextSnapshot).toMatchObject({ legacyDispositionEpisode: { id: f.runId, attempt: 1, maxAttempts: 2 } }); + }); + it("keeps the same bounded episode across restart and commentary, then escalates only after agent attempts", async () => { + const f = await fixture(); + await f.createRecovery().reconcileLegacyContinuation(f.runId); + const first = (await f.runs()).find(r => r.id !== f.runId)!; + await f.finish(first); + await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorAgentId: f.agentId, body: "I will continue; all done; no approval required." }); + await f.createRecovery().reconcileLegacyContinuation(first.id); + const second = (await f.runs()).find(r => r.status === "scheduled_retry")!; + expect(second.contextSnapshot).toMatchObject({ legacyDispositionEpisode: { id: f.runId, attempt: 2, maxAttempts: 2 } }); + await f.finish(second); + expect(await f.createRecovery().reconcileLegacyContinuation(second.id)).toBe("escalated"); + expect(await f.runs()).toHaveLength(3); + expect((await f.actions()).find(a => a.status === "active")).toMatchObject({ ownerType: "board", attemptCount: 2 }); + await f.createRecovery().reconcileLegacyContinuation(second.id); + expect(await f.runs()).toHaveLength(3); + }); + it("does not grant a new budget to exhausted pre-upgrade continuation", async () => { + const f = await fixture({}, 2); + expect(await f.createRecovery().reconcileLegacyContinuation(f.runId)).toBe("escalated"); + expect(await f.runs()).toHaveLength(1); + }); + it.each(["done", "cancelled", "blocked", "in_review"])("honors durable %s instead of the final summary", async status => { + const f = await fixture(); + await db.update(issues).set({ status }).where(eq(issues.id, f.issueId)); + expect(await f.createRecovery().reconcileLegacyContinuation(f.runId)).toBe("skipped"); + expect(await f.runs()).toHaveLength(1); + }); + it("respects a pending approval while the issue still says in progress", async () => { + const f = await fixture(); + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issueId, kind: "request_confirmation", status: "pending", requestedResolverPolicy: "anyone", effectiveResolverPolicy: "anyone", payload: { version: 1, prompt: "Approve?" } }); + expect(await f.createRecovery().reconcileLegacyContinuation(f.runId)).toBe("skipped"); + expect(await f.runs()).toHaveLength(1); + }); + it("rechecks a newly pending approval at dispatch without spending another attempt", async () => { + const f = await fixture(); + await f.createRecovery().reconcileLegacyContinuation(f.runId); + const repair = (await f.runs()).find(r => r.id !== f.runId)!; + expect(await f.createRecovery().legacyRepairDispatchBlock(repair.id)).toBeNull(); + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issueId, kind: "request_confirmation", status: "pending", requestedResolverPolicy: "anyone", effectiveResolverPolicy: "anyone", payload: { version: 1, prompt: "Approve?" } }); + expect(await f.createRecovery().legacyRepairDispatchBlock(repair.id)).toBe("durable_wait"); + expect((await f.actions())[0].attemptCount).toBe(1); + }); + it.each(["done", "paused", "reassigned", "stopped"])("suppresses a queued repair after %s", async gate => { + const f = await fixture(); + await f.createRecovery().reconcileLegacyContinuation(f.runId); + const repair = (await f.runs()).find(r => r.id !== f.runId)!; + if (gate === "done") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId)); + if (gate === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId)); + if (gate === "reassigned") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId)); + if (gate === "stopped") await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "operator_cancelled" }).where(eq(heartbeatRuns.id, repair.id)); + expect(await f.createRecovery().legacyRepairDispatchBlock(repair.id)).not.toBeNull(); + expect((await f.actions())[0].attemptCount).toBe(1); + }); + it("leaves a due monitor as the owner of the next step", async () => { + const f = await fixture(); + await db.update(issues).set({ monitorNextCheckAt: new Date(0) }).where(eq(issues.id, f.issueId)); + expect(await f.createRecovery().reconcileLegacyContinuation(f.runId)).toBe("skipped"); + expect(await f.runs()).toHaveLength(1); + }); + it("honors pause and changed ownership without spending a repair attempt", async () => { + const f = await fixture(); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId)); + expect(await f.createRecovery().reconcileLegacyContinuation(f.runId)).toBe("skipped"); + expect(await f.actions()).toHaveLength(0); + await db.update(agents).set({ status: "idle" }).where(eq(agents.id, f.agentId)); + await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId)); + expect(await f.createRecovery().reconcileLegacyContinuation(f.runId)).toBe("skipped"); + expect(await f.runs()).toHaveLength(1); + }); + it("the delayed sweep ignores old diagnostic labels and uses the same repair path", async () => { + const f = await fixture(); + await f.createRecovery().reconcileStrandedAssignedIssues({ companyId: f.companyId }); + expect(await f.runs()).toHaveLength(2); + const repair = (await f.runs()).find(r => r.id !== f.runId)!; + expect(repair.contextSnapshot).toMatchObject({ legacyDispositionEpisode: { id: f.runId, attempt: 1 } }); + }); +}); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index e65149ac941..7967ec82d7b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -8091,6 +8091,12 @@ export async function buildPaperclipWakePayload(input: { : [], childIssueSummaryTruncated: input.contextSnapshot.childIssueSummaryTruncated === true, + dispositionRepair: input.contextSnapshot.legacyDispositionEpisode ? { + attempt: input.contextSnapshot.dispositionRepairAttempt, + maxAttempts: input.contextSnapshot.dispositionRepairMaxAttempts, + sourceRunId: input.contextSnapshot.retryOfRunId, + instruction: input.contextSnapshot.dispositionRepairInstruction, + } : null, livenessContinuation: readNonEmptyString(input.contextSnapshot.livenessContinuationState) || readNonEmptyString( @@ -22174,6 +22180,19 @@ export function heartbeatService( )) ) return { dispatched: false }; + const repairBlock = await recovery.legacyRepairDispatchBlock(run.id); + if (repairBlock) { + const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { + finishedAt: new Date(), errorCode: "legacy_disposition_repair_suppressed", + error: `Disposition repair suppressed: ${repairBlock}`, + }); + if (cancelled.updated) { + await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: new Date(), error: repairBlock }); + await releaseIssueExecutionAndPromote(cancelled.run!, { suppressImmediateRecovery: true }); + await finalizeAgentStatus(run.agentId, "cancelled"); + } + return { dispatched: false }; + } if ( !issueId || (!isResolvedInteractionContinuationWakeContext(context) && @@ -25116,18 +25135,19 @@ export function heartbeatService( .resumeSessionGoalHeartbeat === true, }); if (!conversationSettled) { - await handleRunLivenessContinuation(livenessRun); - await handleIssueReviewPathDisposition(livenessRun); - await handleSuccessfulRunHandoff( - issueCommentPolicyResult.outcome === "retry_queued" || - issueCommentPolicyResult.outcome === "retry_exhausted" - ? { - ...livenessRun, - issueCommentStatus: issueCommentPolicyResult.outcome, - } - : livenessRun, - agent, - ); + await handleIssueReviewPathDisposition(livenessRun); + if (livenessRun.runtimeMode !== "native") { + await recovery.reconcileLegacyContinuation(livenessRun.id); + } else { + await handleRunLivenessContinuation(livenessRun); + await handleSuccessfulRunHandoff( + issueCommentPolicyResult.outcome === "retry_queued" || + issueCommentPolicyResult.outcome === "retry_exhausted" + ? { ...livenessRun, issueCommentStatus: issueCommentPolicyResult.outcome } + : livenessRun, + agent, + ); + } } if ( outcome === "succeeded" && diff --git a/server/src/services/recovery/legacy-continuation.test.ts b/server/src/services/recovery/legacy-continuation.test.ts new file mode 100644 index 00000000000..e4cd9a38374 --- /dev/null +++ b/server/src/services/recovery/legacy-continuation.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { decideLegacyContinuation, legacyDispositionEpisode, type LegacyContinuationInput } from "./legacy-continuation.js"; +const input: LegacyContinuationInput = { + run: { id: "run", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "legacy" }, + issue: { id: "issue", companyId: "company", status: "in_progress", assigneeAgentId: "agent" }, + agent: { id: "agent", companyId: "company", status: "idle" }, + episode: { id: "run", attempt: 0, maxAttempts: 2 }, + gates: { stopped: false, paused: false, budgetBlocked: false, pendingWait: false, activeExecution: false, ownedLifecycle: false, conversation: false, agentInvokable: true }, +}; +describe("legacy continuation authority", () => { + it("requests agent repair for a successful process without a task disposition", () => { + expect(decideLegacyContinuation(input)).toMatchObject({ kind: "enqueue", nextAttempt: 1 }); + }); + it.each(["done", "cancelled", "blocked", "in_review"])("respects persisted %s disposition", status => { + expect(decideLegacyContinuation({ ...input, issue: { ...input.issue!, status } }).kind).toBe("skip"); + }); + it.each(["failed", "cancelled", "interrupted", "timed_out", "running"])("does not convert a %s process into repair", status => { + expect(decideLegacyContinuation({ ...input, run: { ...input.run, status } }).kind).toBe("skip"); + }); + it.each(["stopped", "paused", "budgetBlocked", "pendingWait", "activeExecution", "ownedLifecycle", "conversation"] as const)("preserves %s", gate => { + expect(decideLegacyContinuation({ ...input, gates: { ...input.gates, [gate]: true } }).kind).toBe("skip"); + }); + it.each(["paused", "terminated", "pending_approval"])("does not repair with a %s agent", status => { + expect(decideLegacyContinuation({ ...input, agent: { ...input.agent!, status } }).kind).toBe("skip"); + }); + it("rejects foreign agent and reassigned issue", () => { + expect(decideLegacyContinuation({ ...input, agent: { ...input.agent!, id: "other" } }).kind).toBe("skip"); + expect(decideLegacyContinuation({ ...input, issue: { ...input.issue!, assigneeAgentId: "other" } }).kind).toBe("skip"); + }); + it("preserves the episode through two attempts and exhaustion", () => { + const first = decideLegacyContinuation(input); + const second = decideLegacyContinuation({ ...input, episode: { ...input.episode, attempt: 1 } }); + expect(second).toMatchObject({ kind: "enqueue", nextAttempt: 2 }); + expect(second).not.toEqual(first); + expect(decideLegacyContinuation({ ...input, episode: { ...input.episode, attempt: 2 } }).kind).toBe("exhausted"); + }); + it("does not resurrect an exhausted pre-upgrade liveness or handoff budget", () => { + for (const run of [ + { id: "old", continuationAttempt: 2 }, + { id: "old", contextSnapshot: { dispositionRepairAttempt: 3, dispositionRepairMaxAttempts: 5, dispositionRepairFingerprint: "old-episode" } }, + { id: "old", contextSnapshot: { wakeReason: "finish_successful_run_handoff", handoffAttempt: 1 } }, + { id: "old", contextSnapshot: { source: "issue.productive_terminal_continuation_recovery" } }, + ]) expect(decideLegacyContinuation({ ...input, episode: legacyDispositionEpisode(run) }).kind).toBe("exhausted"); + }); + it("keeps replay identity stable while a genuinely new episode gets its own identity", () => { + expect(decideLegacyContinuation(structuredClone(input))).toEqual(decideLegacyContinuation(input)); + expect(decideLegacyContinuation({ ...input, episode: { ...input.episode, id: "new-authorized-run" } })).not.toEqual(decideLegacyContinuation(input)); + }); + it("leaves native finalization to its own authority", () => { + expect(decideLegacyContinuation({ ...input, run: { ...input.run, runtimeMode: "native" } }).kind).toBe("skip"); + }); +}); diff --git a/server/src/services/recovery/legacy-continuation.ts b/server/src/services/recovery/legacy-continuation.ts new file mode 100644 index 00000000000..dc319eaedd3 --- /dev/null +++ b/server/src/services/recovery/legacy-continuation.ts @@ -0,0 +1,86 @@ +import { createHash } from "node:crypto"; + +/** This contract deliberately has no narrative, liveness label or progress count. */ +export interface LegacyContinuationInput { + run: { id: string; companyId: string; agentId: string; status: string; runtimeMode?: string | null }; + issue: { id: string; companyId: string; status: string; assigneeAgentId: string | null; assigneeUserId?: string | null } | null; + agent: { id: string; companyId: string; status: string } | null; + episode: LegacyDispositionEpisode; + gates: { + stopped: boolean; + paused: boolean; + budgetBlocked: boolean; + pendingWait: boolean; + activeExecution: boolean; + ownedLifecycle: boolean; + conversation: boolean; + agentInvokable: boolean; + }; +} +export interface LegacyDispositionEpisode { + id: string; + attempt: number; + maxAttempts: number; +} +export const LEGACY_DISPOSITION_REPAIR_MAX_ATTEMPTS = 2; +export const LEGACY_DISPOSITION_REPAIR_INSTRUCTION = + "The previous task run ended without a recorded disposition or an owned next execution path. " + + "Re-read the current task state. Use Paperclip tools/API to record completion, a real blocker, " + + "a question or approval request, or a supported next execution path. " + + "A final message alone does not record disposition. Respect Stop, pause, budget and approval gates."; + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function count(value: unknown) { + return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0; +} +export function legacyDispositionEpisode(run: { + id: string; + contextSnapshot?: unknown; + continuationAttempt?: number | null; +}): LegacyDispositionEpisode { + const context = record(run.contextSnapshot); + const episode = record(context.legacyDispositionEpisode); + if (typeof episode.id === "string" && episode.id) { + return { id: episode.id, attempt: count(episode.attempt), maxAttempts: Math.max(1, Math.min(LEGACY_DISPOSITION_REPAIR_MAX_ATTEMPTS, count(episode.maxAttempts) || LEGACY_DISPOSITION_REPAIR_MAX_ATTEMPTS)) }; + } + // Pre-upgrade repairs have already consumed their old (possibly tighter) + // allowance. A classifier/key migration must not grant them a fresh budget. + const handoff = context.wakeReason === "finish_successful_run_handoff" || context.handoffRequired === true; + const oldRecovery = context.source === "issue.productive_terminal_continuation_recovery"; + const attempt = Math.max(count(run.continuationAttempt), count(context.livenessContinuationAttempt), count(context.dispositionRepairAttempt), handoff ? count(context.handoffAttempt) || 1 : 0, oldRecovery ? 1 : 0); + return { + id: typeof context.dispositionRepairFingerprint === "string" ? context.dispositionRepairFingerprint : typeof context.livenessContinuationSourceRunId === "string" ? context.livenessContinuationSourceRunId : run.id, + attempt, + maxAttempts: handoff || oldRecovery ? 1 : LEGACY_DISPOSITION_REPAIR_MAX_ATTEMPTS, + }; +} +export function legacyDispositionFingerprint(companyId: string, issueId: string, agentId: string, episodeId: string) { + return `legacy_disposition:v1:${createHash("sha256").update(JSON.stringify([companyId, issueId, agentId, episodeId])).digest("hex")}`; +} +export function decideLegacyContinuation(input: LegacyContinuationInput): + | { kind: "skip"; reason: string } + | { kind: "exhausted"; attempt: number; maxAttempts: number } + | { kind: "enqueue"; nextAttempt: number; idempotencyKey: string; instruction: string } { + const { run, issue, agent, gates, episode } = input; + if (run.runtimeMode === "native") return { kind: "skip", reason: "native_finalization" }; + if (run.status !== "succeeded") return { kind: "skip", reason: "run_not_successful" }; + if (!issue || !agent || issue.companyId !== run.companyId || agent.companyId !== run.companyId || agent.id !== run.agentId) return { kind: "skip", reason: "invalid_binding" }; + if (issue.assigneeAgentId !== run.agentId || issue.assigneeUserId) return { kind: "skip", reason: "owner_changed" }; + if (!["todo", "in_progress"].includes(issue.status)) return { kind: "skip", reason: "recorded_disposition" }; + for (const [blocked, reason] of [ + [gates.stopped, "stopped"], [gates.paused, "paused"], + [gates.budgetBlocked, "budget_blocked"], [gates.pendingWait, "durable_wait"], + [gates.activeExecution, "existing_execution"], [gates.ownedLifecycle, "owned_lifecycle"], + [gates.conversation, "conversation"], + [!gates.agentInvokable || ["paused", "terminated", "pending_approval"].includes(agent.status), "agent_not_invokable"], + ] as const) if (blocked) return { kind: "skip", reason }; + if (episode.attempt >= episode.maxAttempts) return { kind: "exhausted", attempt: episode.attempt, maxAttempts: episode.maxAttempts }; + const nextAttempt = episode.attempt + 1; + return { + kind: "enqueue", nextAttempt, + idempotencyKey: `issue_disposition_repair:${issue.id}:${legacyDispositionFingerprint(run.companyId, issue.id, run.agentId, episode.id)}:${nextAttempt}`, + instruction: LEGACY_DISPOSITION_REPAIR_INSTRUCTION, + }; +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index bbbff367c63..33e673e957d 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1,3 +1,7 @@ +import { + decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint, + LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode, +} from "./legacy-continuation.js"; import { hasLiveLegacyController } from "../legacy-controller-lease.js"; import { instanceSettingsService } from "../instance-settings.js"; import { isWaitingConversation, settleConversationTurn, deliverConversationComments } from "../agent-conversations.js"; @@ -49,6 +53,7 @@ import { nativeRunFinalizations, nativeRunResults, statusDecisions, + routines, workAssessments, } from "@paperclipai/db"; import { parseObject, asBoolean, asNumber } from "../../adapters/utils.js"; @@ -3005,6 +3010,7 @@ export function recoveryService( latestRun: LatestIssueRun; fingerprint: string; attemptCount: number; + legacyEpisode?: LegacyDispositionEpisode; }) { let active = await recoveryActionsSvc.getActiveForIssue( input.issue.companyId, @@ -3069,9 +3075,9 @@ export function recoveryService( type: "bounded_owner_disposition_repair", retryAgentId: input.issue.assigneeAgentId, attempt: input.attemptCount, - maxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS, }, - maxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS, attemptCount: input.attemptCount, lastAttemptAt: new Date(), }); @@ -3083,6 +3089,7 @@ export function recoveryService( action: Awaited>; fingerprint: string; attempt: number; + legacyEpisode?: LegacyDispositionEpisode; }) { const agentId = input.issue.assigneeAgentId; if (!agentId) return null; @@ -3101,10 +3108,11 @@ export function recoveryService( recoveryActionId: input.action.id, dispositionRepairFingerprint: input.fingerprint, dispositionRepairAttempt: input.attempt, - dispositionRepairMaxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + dispositionRepairMaxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS, + ...(input.legacyEpisode ? { legacyDispositionEpisode: { ...input.legacyEpisode, attempt: input.attempt } } : {}), bypassContinuationSummaryPark: true, dispositionRepairInstruction: - "Revalidate the issue and replace the invalid parked summary with a durable disposition. Continue productive work when appropriate.", + input.legacyEpisode ? LEGACY_DISPOSITION_REPAIR_INSTRUCTION : "Revalidate the issue and replace the invalid parked summary with a durable disposition. Continue productive work when appropriate.", }, "normal_model", ); @@ -3151,6 +3159,7 @@ export function recoveryService( requestedByActorType: "system", requestedByActorId: null, contextSnapshot: context, + ...(input.legacyEpisode ? { issueStateGuard: { statuses: [input.issue.status], assigneeAgentId: agentId } } : {}), }); scheduledRun = enqueuedRun ?? (await findScheduledRun()); created = Boolean(enqueuedRun); @@ -3228,7 +3237,7 @@ export function recoveryService( type: "bounded_owner_disposition_repair", retryAgentId: agentId, attempt: input.attempt, - maxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS, baseBackoffMs: timing.baseDelayMs, jitterMs: timing.jitterMs, retryAt: retryAt.toISOString(), @@ -3261,7 +3270,7 @@ export function recoveryService( ownerAgentId: agentId, sourceStateFingerprint: input.fingerprint, attempt: input.attempt, - maxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS, baseBackoffMs: timing.baseDelayMs, jitterMs: timing.jitterMs, retryAt: retryAt.toISOString(), @@ -3482,12 +3491,14 @@ export function recoveryService( fingerprint: string; attemptCount: number; terminalReason: string; + legacyEpisode?: LegacyDispositionEpisode; }) { const action = await ensureDispositionRepairAction({ issue: input.issue, latestRun: input.latestRun, fingerprint: input.fingerprint, attemptCount: input.attemptCount, + legacyEpisode: input.legacyEpisode, }); const now = new Date(); await db @@ -3505,7 +3516,7 @@ export function recoveryService( latestRunErrorCode: input.latestRun?.errorCode ?? null, terminalReason: input.terminalReason, sourceAttemptCount: input.attemptCount, - sourceMaxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + sourceMaxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS, routingPolicy: STRANDED_BOARD_ESCALATION_POLICY, }, nextAction: @@ -3539,7 +3550,7 @@ export function recoveryService( [ "Paperclip exhausted the bounded original-owner disposition repair without a durable source-state change.", "", - `- Attempts: ${input.attemptCount}/${DISPOSITION_REPAIR_MAX_ATTEMPTS}`, + `- Attempts: ${input.attemptCount}/${input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS}`, `- Terminal reason: \`${input.terminalReason}\``, "- Recovery owner: board", "- Source ownership: unchanged; reassignment requires an explicit decision or a policy-defined serious failure.", @@ -3577,7 +3588,7 @@ export function recoveryService( previousStatus: input.issue.status, sourceStateFingerprint: input.fingerprint, attemptCount: input.attemptCount, - maxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS, terminalReason: input.terminalReason, recoveryActionId: action.id, recoveryOwnerAgentId: null, @@ -3609,10 +3620,84 @@ export function recoveryService( return updated; } + async function decidePersistedLegacyContinuation( + issue: typeof issues.$inferSelect, + runId: string, + state: Awaited>, + episode: LegacyDispositionEpisode, + dispatchRunId?: string, + ) { + const [run] = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, issue.companyId), + )).limit(1); + if (!run) return { kind: "skip" as const, reason: "missing_run" }; + const context = parseObject(run.contextSnapshot); + if ((readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId)) !== issue.id) { + return { kind: "skip" as const, reason: "invalid_issue_binding" }; + } + const [agent, pause, budget, stop, active, routine, goal, latest, durableWait, workspaceChildren] = await Promise.all([ + getAgent(run.agentId), + isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc), + isInvocationBudgetBlocked(issue, run.agentId), + readChatControlRecoveryStop(db, { companyId: issue.companyId, issueId: issue.id, agentId: run.agentId, sourceRunId: run.id }), + recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id), + db.select({ id: routines.id }).from(routines).where(and(eq(routines.companyId, issue.companyId), eq(routines.parentIssueId, issue.id), eq(routines.status, "active"))).limit(1), + db.select({ status: agentTaskSessions.goalStatus }).from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, issue.companyId), eq(agentTaskSessions.agentId, run.agentId), eq(agentTaskSessions.taskKey, issue.id))).limit(1), + getLatestIssueRun(issue.companyId, issue.id), + hasPersistedDurableWaitPath(issue, run), + parseObject(context.paperclipWorkspace).mode === "shared_workspace" ? healthyOpenChildIssues(issue, true) : Promise.resolve([]), + ]); + const ownsRepair = active?.kind === "deliberate_wait_without_target" && active.ownerType === "agent"; + return decideLegacyContinuation({ + run, issue, agent, episode, + gates: { + stopped: stop.kind !== "clear" || isOperatorCancelledRun(run, run.agentId), + paused: pause, budgetBlocked: budget, + pendingWait: state.hasDurableWaitingPath || durableWait || parseIssueExecutionState(issue.executionState)?.status === "pending" || Boolean(issue.monitorNextCheckAt), + activeExecution: state.hasActiveExecutionPath || latest?.id !== (dispatchRunId ?? run.id), + ownedLifecycle: isPluginManagedIssueLifecycle(issue) || routine.length > 0 || workspaceChildren.length > 0 || + Boolean(goal[0]?.status && goal[0].status !== "complete") || Boolean(active && !ownsRepair), + conversation: Boolean(issue.conversationAgentId) || isWaitingConversation(issue), + agentInvokable: Boolean(agent && await isAgentInvokable(agent) && isHeartbeatWakeOnDemandEnabled(agent)), + }, + }); + } + + async function legacyRepairDispatchBlock(runId: string): Promise { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).limit(1); + const context = parseObject(run?.contextSnapshot); + if (!run || !parseObject(context.legacyDispositionEpisode).id) return null; + if (!["queued", "running", "scheduled_retry"].includes(run.status)) return "repair_not_active"; + const episode = legacyDispositionEpisode(run); + const sourceId = readNonEmptyString(context.retryOfRunId); + const issueId = readNonEmptyString(context.issueId); + if (!sourceId || !issueId || episode.attempt < 1 || episode.attempt > episode.maxAttempts) return "invalid_repair_binding"; + const [source] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, sourceId), eq(heartbeatRuns.companyId, run.companyId))).limit(1); + if (!source || source.agentId !== run.agentId || legacyDispositionEpisode(source).id !== episode.id) return "invalid_repair_source"; + const [issue] = await db.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))).limit(1); + if (!issue) return "missing_issue"; + const state = await collectDispositionRepairSourceState(db, { issue, excludeRunId: run.id, excludeWakeupRequestId: run.wakeupRequestId ?? undefined }); + // This slot has already been reserved. Check admission for that slot rather + // than allocating or charging another repair attempt during dispatch. + const decision = await decidePersistedLegacyContinuation(issue, source.id, state, { ...episode, attempt: episode.attempt - 1 }, run.id); + return decision.kind === "enqueue" ? null : decision.kind === "skip" ? decision.reason : "repair_exhausted"; + } + + async function reconcileLegacyContinuation(runId: string) { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).limit(1); + if (!run || run.runtimeMode === "native" || run.status !== "succeeded") return "skipped" as const; + const context = parseObject(run.contextSnapshot); + const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); + if (!issueId) return "skipped" as const; + const [issue] = await db.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))).limit(1); + if (!issue) return "skipped" as const; + return reconcileDispositionRepair(issue, run, { legacyEpisode: legacyDispositionEpisode(run) }); + } + async function reconcileDispositionRepair( issue: typeof issues.$inferSelect, latestRun: LatestIssueRun, - options: { historicalAttemptCount?: number } = {}, + options: { historicalAttemptCount?: number; legacyEpisode?: LegacyDispositionEpisode } = {}, ): Promise<"queued" | "escalated" | "covered" | "skipped"> { const current = await db .select() @@ -3625,7 +3710,12 @@ export function recoveryService( if (!current || current.status === "done" || current.status === "cancelled") return "skipped"; - const dependencyWait = await resolveContinuationWaitingOnReview(current); + const episode = options.legacyEpisode ?? ( + parseObject(parseObject(latestRun?.contextSnapshot).legacyDispositionEpisode).id && latestRun + ? legacyDispositionEpisode(latestRun) : undefined + ); + const maxAttempts = episode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS; + const dependencyWait = episode ? null : await resolveContinuationWaitingOnReview(current); if (dependencyWait) { await resolveDispositionRepairActionAsCovered( current, @@ -3637,6 +3727,12 @@ export function recoveryService( const state = await collectDispositionRepairSourceState(db, { issue: current, }); + if (episode) { + if (!latestRun) return "skipped"; + const decision = await decidePersistedLegacyContinuation(current, latestRun.id, state, episode); + if (decision.kind === "skip") return "skipped"; + state.fingerprint = legacyDispositionFingerprint(current.companyId, current.id, latestRun.agentId, episode.id); + } if (state.hasActiveExecutionPath) return "skipped"; if (state.hasDurableWaitingPath) { await resolveDispositionRepairActionAsCovered( @@ -3675,14 +3771,18 @@ export function recoveryService( // from that consecutive legacy history instead of granting five fresh // attempts merely because the recovery-action row did not exist yet. const historicalAttempt = Math.min( - DISPOSITION_REPAIR_MAX_ATTEMPTS, - Math.max(0, Math.floor(options.historicalAttemptCount ?? 0)), + maxAttempts, + Math.max(episode?.attempt ?? 0, Math.floor(options.historicalAttemptCount ?? 0)), ); + // A source run owns one successor slot. Concurrent checks must not advance + // the counter again after another checker reserved that same successor. + if (episode && persistedAttempt > episode.attempt) return "skipped"; const sameFingerprintAttempt = Math.max( runAttempt, persistedAttempt, historicalAttempt, ); + if (episode && (!ownerInvokable || budgetBlocked)) return "skipped"; if (!ownerInvokable || budgetBlocked) { const escalated = await escalateDispositionRepair({ issue: current, @@ -3696,13 +3796,14 @@ export function recoveryService( return escalated ? "escalated" : "skipped"; } - if (sameFingerprintAttempt >= DISPOSITION_REPAIR_MAX_ATTEMPTS) { + if (sameFingerprintAttempt >= maxAttempts) { const escalated = await escalateDispositionRepair({ issue: current, latestRun, fingerprint: state.fingerprint, attemptCount: sameFingerprintAttempt, terminalReason: "unchanged_source_state_exhausted", + legacyEpisode: episode, }); return escalated ? "escalated" : "skipped"; } @@ -3713,6 +3814,7 @@ export function recoveryService( latestRun, fingerprint: state.fingerprint, attemptCount: sameFingerprintAttempt, + legacyEpisode: episode, }); const scheduled = await scheduleDispositionRepairAttempt({ issue: current, @@ -3720,7 +3822,16 @@ export function recoveryService( action, fingerprint: state.fingerprint, attempt: nextAttempt, + legacyEpisode: episode, }); + if (!scheduled && episode) { + // Admission can reject a stale snapshot under the issue lock. An empty + // repair action must not subsequently cause escalation. + await recoveryActionsSvc.resolveActiveForIssue({ + companyId: current.companyId, sourceIssueId: current.id, actionId: action.id, + status: "cancelled", outcome: "cancelled", resolutionNote: "repair_admission_rejected", + }); + } return scheduled ? "queued" : "skipped"; } @@ -4267,6 +4378,22 @@ export function recoveryService( continue; } + if (latestRun?.status === "succeeded" && issue.status !== "in_review") { + const [source] = await db.select({ runtimeMode: heartbeatRuns.runtimeMode }).from(heartbeatRuns).where(eq(heartbeatRuns.id, latestRun.id)).limit(1); + if (source?.runtimeMode !== "native") { + const outcome = await reconcileLegacyContinuation(latestRun.id); + if (outcome === "queued") { + result.continuationRequeued += 1; + result.dispositionRepairRequeued += 1; + result.issueIds.push(issue.id); + } else if (outcome === "escalated") { + result.escalated += 1; + result.issueIds.push(issue.id); + } else result.skipped += 1; + continue; + } + } + const agent = await getAgent(agentId); const agentInvokable = agent && agent.companyId === issue.companyId @@ -5906,6 +6033,8 @@ export function recoveryService( recordWatchdogDecision, scanSilentActiveRuns, reconcileStrandedAssignedIssues, + reconcileLegacyContinuation, + legacyRepairDispatchBlock, sweepStaleIssueLocks, reconcileResolvedDependencyWakeBackstop, readRecoveryTimerIntervalMs, diff --git a/server/test-baselines/lifecycle-heartbeat.test.ts b/server/test-baselines/lifecycle-heartbeat.test.ts index 452f87b28c8..8f05f52115a 100644 --- a/server/test-baselines/lifecycle-heartbeat.test.ts +++ b/server/test-baselines/lifecycle-heartbeat.test.ts @@ -353,7 +353,9 @@ describe("LCA full heartbeat observation", () => { "No approval required. I will inspect the repository and run tests.", false, ); - expect(misleading.providerTurns).toBe(neutral.providerTurns); + expect(neutral.providerTurns).toBe(2); + expect(neutral.issue).toEqual({ status: "done", locked: false }); + expect(misleading.providerTurns).toBe(2); expect(misleading.wakes).toEqual(neutral.wakes); expect(misleading.issue).toEqual(neutral.issue); }); diff --git a/tests/lifecycle-baseline/README.md b/tests/lifecycle-baseline/README.md index d161d73398e..4b21b3c7ae6 100644 --- a/tests/lifecycle-baseline/README.md +++ b/tests/lifecycle-baseline/README.md @@ -83,9 +83,11 @@ Troublesome combinations included in the inventory and reused suites: ## Narrative pairs and positive controls -`authority.test.ts` composes the real legacy classifier with its actual -continuation consumer, so assertions compare scheduling decisions rather than -requiring diagnostic labels to be identical. Native pairs exercise the actual +`authority.test.ts` records the legacy classifier for diagnostic comparison and +executes the production structured continuation decision. Assertions compare +scheduling decisions rather than requiring diagnostic labels to be identical. +The persisted legacy authority tests cover replay, restart, exhaustion and +dispatch gates; they also run in the ordinary server suite. Native pairs exercise the actual status arbiter. The heartbeat cases cross the real persistence/finalization boundary for both runtimes, including explicit native continuation and the legacy missing-disposition path. Observations precede cleanup; a second queue/drain pass diff --git a/tests/lifecycle-baseline/authority.test.ts b/tests/lifecycle-baseline/authority.test.ts index 2336fe82ad9..83e2970aa45 100644 --- a/tests/lifecycle-baseline/authority.test.ts +++ b/tests/lifecycle-baseline/authority.test.ts @@ -3,7 +3,7 @@ import { classifyRunLiveness, type RunLivenessClassificationInput, } from "../../server/src/services/run-liveness.js"; -import { decideRunLivenessContinuation } from "../../server/src/services/run-continuations.js"; +import { decideLegacyContinuation, legacyDispositionEpisode } from "../../server/src/services/recovery/legacy-continuation.js"; import { arbitrateNativeStatus } from "../../server/src/services/native-runtime/status-arbiter.js"; import type { NativeEvidenceAssessment } from "../../server/src/services/native-runtime/evidence-classifier.js"; import { narratives } from "./narratives.js"; @@ -19,33 +19,30 @@ const legacyBase: RunLivenessClassificationInput = { }, resultJson: { summary: future }, }; -type DecisionInput = Parameters[0]; +type DecisionInput = Parameters[0]; function continuation( input: RunLivenessClassificationInput, overrides: Partial = {}, ) { const classification = classifyRunLiveness(input); - const decision = decideRunLivenessContinuation({ + const decision = decideLegacyContinuation({ run: { id: "run", companyId: "company", agentId: "agent", - continuationAttempt: input.continuationAttempt ?? 0, + status: input.runStatus, + runtimeMode: "legacy", } as DecisionInput["run"], issue: { id: "issue", companyId: "company", - identifier: "BASE-1", - title: input.issue!.title, status: input.issue!.status, assigneeAgentId: "agent", - executionState: null, - projectId: null, }, agent: { id: "agent", companyId: "company", status: "idle" }, - ...classification, - budgetBlocked: false, - idempotentWakeExists: false, + episode: legacyDispositionEpisode({ id: "run", continuationAttempt: input.continuationAttempt ?? 0 }), + gates: { stopped: false, paused: false, budgetBlocked: false, pendingWait: false, + activeExecution: false, ownedLifecycle: false, conversation: false, agentInvokable: true }, ...overrides, }); // Narrative-bearing instructions and diagnostic labels are deliberately excluded. @@ -55,6 +52,7 @@ function continuation( kind: decision.kind, nextAttempt: decision.nextAttempt, idempotencyKey: decision.idempotencyKey, + instruction: decision.instruction, } : { kind: decision.kind }; return { classification, decision, effect }; @@ -101,6 +99,7 @@ describe("LCA narrative authority baseline", () => { const before = continuation(build(future)); const after = continuation(build(text)); observe("LCA-02", `legacy:${channel}:${variant}`, { before, after }); + expect(before.effect.kind).toBe("enqueue"); expect(after.effect).toEqual(before.effect); }, ); @@ -149,8 +148,8 @@ describe("LCA narrative authority baseline", () => { }, ); it.each([ - ["budget", { budgetBlocked: true }], - ["duplicate", { idempotentWakeExists: true }], + ["budget", { gate: "budgetBlocked" }], + ["duplicate", { gate: "activeExecution" }], [ "wrong-company", { agent: { id: "agent", companyId: "other", status: "idle" } }, @@ -162,7 +161,11 @@ describe("LCA narrative authority baseline", () => { ] as const)( "LCA-10 LCA-12 legacy gate %s survives encouraging prose", (variant, overrides) => { - const actual = continuation(legacyBase, overrides); + const actual = continuation(legacyBase, "gate" in overrides ? { + gates: { stopped: false, paused: false, budgetBlocked: false, pendingWait: false, + activeExecution: false, ownedLifecycle: false, conversation: false, agentInvokable: true, + [overrides.gate]: true }, + } : overrides); observe("LCA-10", variant, actual); expect(actual.effect.kind).toBe("skip"); }, diff --git a/tests/lifecycle-baseline/inventory.mjs b/tests/lifecycle-baseline/inventory.mjs index 3a57737a3e2..72fcc3b1930 100644 --- a/tests/lifecycle-baseline/inventory.mjs +++ b/tests/lifecycle-baseline/inventory.mjs @@ -5,6 +5,7 @@ export const lanes = { unit: { files: [ "tests/lifecycle-baseline/authority.test.ts", + "server/src/services/recovery/legacy-continuation.test.ts", `${native}status-arbiter.test.ts`, `${native}native-replacement-evidence.test.ts`, `${server}run-continuations.test.ts`, @@ -23,6 +24,7 @@ export const lanes = { integration: { files: [ "server/test-baselines/lifecycle-heartbeat.test.ts", + `${server}legacy-continuation-authority.test.ts`, `${server}native-status-arbiter-corpus.test.ts`, `${server}heartbeat-issue-liveness-escalation.test.ts`, `${server}heartbeat-retry-scheduling.test.ts`, @@ -197,6 +199,8 @@ export const scenarios = [ coverage: [ runnerPairs("LCA-09"), unit("LCA-09"), + integ("legacy-continuation-authority"), + ref("unit", "server/src/services/recovery/legacy-continuation.test.ts"), ref( "integration", "server/test-baselines/lifecycle-heartbeat.test.ts", @@ -210,7 +214,7 @@ export const scenarios = [ "exhausted|another run id", ), ], - live: [], + live: ["lifecycle-baseline:lifecycle-repair-neutral", "lifecycle-baseline:lifecycle-repair-challenge"], }, { id: "LCA-10", diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index f89dbbd9a2f..2e21d2f1180 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -177,3 +177,9 @@ workspaces, task documents, and ordering. They pass through the normal sanitizer Screenshots are allowlisted to the exact disposable agent chat. Cleanup cancels all active runs in the isolated company, including handed-off work; usage from failed and cancelled runs must not disappear from campaign totals. + +The lifecycle suite also includes two legacy disposition-repair probes. Their +first provider turn intentionally omits task disposition, and their second turn +must be an automatic, causally bound repair that records completion. They use +public task comments/status APIs and run-detail evidence; no private runtime +hooks or database mutations are used by the fixture. diff --git a/tests/runner-e2e/LIFECYCLE-BASELINE.md b/tests/runner-e2e/LIFECYCLE-BASELINE.md index a71c3227b89..88668fd4efd 100644 --- a/tests/runner-e2e/LIFECYCLE-BASELINE.md +++ b/tests/runner-e2e/LIFECYCLE-BASELINE.md @@ -105,3 +105,19 @@ current model-facing autonomous-continuation defect. This live suite uses suppor question/approval/dependency responses and restart boundaries; it does not instruct a model to emit unsupported `same_agent` output. A live autonomous continuation case needs an identified supported trigger before it can claim that coverage. + +## Legacy disposition repair follow-up (2026-09-22) + +The current suite adds `lifecycle-repair-neutral` and +`lifecycle-repair-challenge` for `legacy-codex` only: 42 cells total (the original +40 plus two). Each costs two provider turns. The first turn posts an attributed +quotation and leaves the task in progress without a durable disposition. The +server must automatically wake the agent for disposition repair, and the second +turn must record completion through the public API. The independent oracle +requires the source/repair episode binding, attempt 1 of 2, two successful runs, +the initial attributed quotation, no user message, and final task completion. +Missing evidence or a one-turn completion fails. Timeout, cleanup, screenshots, +source provenance and billing use the ordinary single-turn fixture pipeline. + +The historical 40-cell campaign records remain unchanged. These two new cases +measure repair behavior that the original completed/blocked pairs did not reach. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 0e19d763f54..fbd1feab2cf 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -84,7 +84,7 @@ pnpm test:e2e:runner -- --all ``` The catalog contains nine suites, including the explicit-only everyday and -[lifecycle baseline](LIFECYCLE-BASELINE.md) suites. The latter adds 40 real-provider +[lifecycle baseline](LIFECYCLE-BASELINE.md) suites. The latter adds 42 real-provider cells pairing narrative variants and exercising durable lifecycle boundaries; it is excluded from `--all`. `core-compatibility` (**Core Runner Compatibility**) is seven major runner profiles × local/Daytona × three diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index cfa9b264339..301945c63c5 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -57,10 +57,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 40, 23, 38, 52, 28, 42, 14, 10, 2, + 42, 23, 38, 52, 28, 42, 14, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(249); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(249); + expect(validateRunnerCatalog()).toHaveLength(251); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(251); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 3a1b3a733b0..702849ad79d 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -916,7 +916,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [...lifecycleLiveTasks, ...chatTasks.filter(task => ["clarify-reuse", "stop-new-resume"].includes(task.id)), ...connectionReviewSuite.tasks], - expectedMatrixSize: 40, + expectedMatrixSize: 42, + excludedExecutionIds: ["neutral", "challenge"].map(variant => `lifecycle-baseline.runner-codex.local.lifecycle-repair-${variant}`), definitionMetadata: { version: 1, narrativeDigest: lifecycleLiveDefinitionDigest, grading: "durable-state-and-attributed-narrative", scheduling: "explicit-only" }, }, { diff --git a/tests/runner-e2e/lifecycle-live-cases.ts b/tests/runner-e2e/lifecycle-live-cases.ts index 8f9bb786c15..e5b90108b62 100644 --- a/tests/runner-e2e/lifecycle-live-cases.ts +++ b/tests/runner-e2e/lifecycle-live-cases.ts @@ -6,6 +6,7 @@ import { import type { RunnerTaskFixture, Matcher } from "./types.js"; export const lifecycleNarrativePairs = [ + { family: "repair", challenge: "I am blocked waiting for approval. All work is done. I will inspect optional next steps." }, { family: "completion", challenge: "No approval required. Optional next steps are not requested.", @@ -59,7 +60,7 @@ export const lifecycleLiveCases = lifecycleNarrativePairs.flatMap((pair) => })), ); export const lifecycleLiveDefinitionDigest = createHash("sha256") - .update(JSON.stringify({ version: 3, cases: lifecycleLiveCases })) + .update(JSON.stringify({ version: 4, cases: lifecycleLiveCases })) .digest("hex"); export function lifecycleLiveCase(id: string) { return lifecycleLiveCases.find((c) => c.id === id); @@ -96,6 +97,29 @@ export const lifecycleLiveTasks: readonly RunnerTaskFixture[] = lifecycleLiveContinuation(probe.id, nonce).prompt, }; } + if (probe.family === "repair") { + return { + id: probe.id, label: `Disposition repair: ${probe.variant}`, groups: [], + workMode: "standard", flow: "single_turn", expectedRunCount: 2, + attemptTimeoutMs: { local: 8 * 60_000, daytona: 8 * 60_000 }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: nonce => `Lifecycle disposition exercise ${nonce}`, + buildVisibleMarker: nonce => `REPAIRED_${nonce}`, + buildPrompt: nonce => [ + "This is a bounded two-turn disposition repair exercise. Use the public Paperclip API with normal authorization and run attribution headers.", + `On the initial assignment turn only: post exactly one task comment whose complete body is ${JSON.stringify(probe.narrative)}, then end the provider turn with the same text. Leave task status in_progress. Do not record completion, a blocker, an approval, a question, a monitor, children, or a continuation. The quoted words do not describe actual authorization or missing prerequisites.`, + `When Paperclip wakes you for task disposition repair: PATCH the current task to status done, with the exact comment REPAIRED_${nonce}. End the provider turn. Do not repeat the initial comment or create further work.`, + ].join("\n"), + buildMatchers: (nonce, execution) => [ + { kind: "message_exact", expected: `REPAIRED_${nonce}` }, + { kind: "message_occurrences", expected: `REPAIRED_${nonce}`, count: 1 }, + { kind: "issue_status", expected: "done" }, + { kind: "run_status", expected: "succeeded" }, + { kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode }, + { kind: "json_path", path: "issue.executionRunId", expected: null }, + ], + } satisfies RunnerTaskFixture; + } const blocked = probe.family === "blocker"; return { id: probe.id, @@ -201,3 +225,23 @@ export function gradeLifecycleNarrative(input: { "Exactly one attributed agent comment must carry the selected quotation before the initial wait. Prompt text alone is not evidence.", }; } + +/** Independent causal oracle for the real-provider repair pair. */ +export function gradeLifecycleRepair(input: { + runs: Array<{ id: string; status: string; contextSnapshot?: Record | null }>; + comments: Array<{ body?: string | null; authorAgentId?: string | null; authorUserId?: string | null; createdByRunId?: string | null }>; + agentId: string; narrative: string; +}) { + const [source, repair] = input.runs; + const context = repair?.contextSnapshot; + const episode = context?.legacyDispositionEpisode as Record | undefined; + const attributed = input.comments.filter(c => c.body === input.narrative && c.authorAgentId === input.agentId && c.createdByRunId === source?.id); + return { + id: "lifecycle.disposition-repair", + passed: input.runs.length === 2 && source?.status === "succeeded" && repair?.status === "succeeded" && + context?.wakeReason === "issue_disposition_repair" && context?.retryOfRunId === source?.id && + episode?.id === source?.id && episode?.attempt === 1 && episode?.maxAttempts === 2 && + attributed.length === 1 && !input.comments.some(c => c.authorUserId), + detail: "Two successful runs; one attributed initial quotation; one causally bound disposition repair; no intervening user message.", + }; +} diff --git a/tests/runner-e2e/lifecycle-live.test.ts b/tests/runner-e2e/lifecycle-live.test.ts index 526e9156664..6ef6369dcf2 100644 --- a/tests/runner-e2e/lifecycle-live.test.ts +++ b/tests/runner-e2e/lifecycle-live.test.ts @@ -7,14 +7,15 @@ import { lifecycleLiveTasks, lifecycleLiveContinuation, gradeLifecycleNarrative, + gradeLifecycleRepair, } from "./lifecycle-live-cases.js"; describe("LCA live baseline authoring and evidence", () => { - it("registers 40 explicit-only real-provider cells across both runtimes", () => { + it("registers the 40 baseline cells plus two legacy repair probes", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "lifecycle-baseline"]), ); - expect(selected).toHaveLength(40); + expect(selected).toHaveLength(42); expect(new Set(selected.map((e) => e.profile.expectedRuntimeMode))).toEqual( new Set(["native", "legacy"]), ); @@ -91,7 +92,7 @@ describe("LCA live baseline authoring and evidence", () => { .passed, ).toBe(false); }); - it.each(lifecycleLiveCases.filter((c) => !c.continuation))( + it.each(lifecycleLiveCases.filter((c) => !c.continuation && c.family !== "repair"))( "LCA-01 $id checks actual visible wording and durable terminal state", async (probe) => { const execution = runnerMatrix.find( @@ -156,3 +157,18 @@ describe("LCA live baseline authoring and evidence", () => { }, ); }); + +describe("LCA-09 repair oracle calibration", () => { + const source = { id: "source", status: "succeeded" }; + const repair = { id: "repair", status: "succeeded", contextSnapshot: { wakeReason: "issue_disposition_repair", retryOfRunId: "source", legacyDispositionEpisode: { id: "source", attempt: 1, maxAttempts: 2 } } }; + const input = { runs: [source, repair], agentId: "agent", narrative: "I am blocked.", comments: [{ body: "I am blocked.", authorAgentId: "agent", createdByRunId: "source" }] }; + it("requires the causal repair and the actual attributed perturbation", () => { + expect(gradeLifecycleRepair(input).passed).toBe(true); + for (const runs of [[], [source], [source, repair, repair], [source, { ...repair, contextSnapshot: {} }], [{ ...source, status: "failed" }, repair]]) { + expect(gradeLifecycleRepair({ ...input, runs }).passed).toBe(false); + } + expect(gradeLifecycleRepair({ ...input, comments: [] }).passed).toBe(false); + expect(gradeLifecycleRepair({ ...input, comments: [...input.comments, { authorUserId: "user" }] }).passed).toBe(false); + expect(gradeLifecycleRepair({ ...input, comments: [{ ...input.comments[0], createdByRunId: "repair" }] }).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 86b84eb9425..aafbe556646 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,5 +1,5 @@ import type { Issue } from "../../packages/shared/src/types/issue.js"; -import { lifecycleLiveCase } from "./lifecycle-live-cases.js"; +import { lifecycleLiveCase, gradeLifecycleRepair } from "./lifecycle-live-cases.js"; import { runContinuationFlow } from "./continuation-flow.js"; import { runEverydayFlow } from "./everyday-flow.js"; import { createTaskThroughUi, submitTaskReply } from "./user-actions.js"; @@ -59,6 +59,8 @@ interface IssueRecord { executionWorkspaceId?: string | null; executionRunId?: string | null; checkoutRunId?: string | null; + scheduledRetry?: unknown; + monitorNextCheckAt?: string | null; } interface CommentRecord { @@ -66,6 +68,7 @@ interface CommentRecord { body?: string | null; authorType?: string | null; authorAgentId?: string | null; + authorUserId?: string | null; createdByRunId?: string | null; createdAt?: string; } @@ -1622,6 +1625,17 @@ for (const execution of executions) { ), ); selectedRuns = sortRunsChronologically(selectedRuns); + const lifecycleProbe = execution.suite.id === "lifecycle-baseline" ? lifecycleLiveCase(execution.task.id) : undefined; + if (lifecycleProbe?.family === "repair") { + const grade = gradeLifecycleRepair({ runs: selectedRuns, comments: terminal.comments, + agentId: fixtures.agent.id, narrative: lifecycleProbe.narrative }); + await writeSanitizedJson(snapshotsDir, "lifecycle-repair.json", { grade, runs: selectedRuns, comments: terminal.comments }, secrets); + expect(grade.passed, grade.detail).toBe(true); + expect(terminal.currentIssue.scheduledRetry).toBeNull(); + expect(terminal.currentIssue.monitorNextCheckAt).toBeNull(); + expect(terminal.interactions.filter(i => i.status === "pending")).toHaveLength(0); + } + if (execution.task.flow === "warm_three_turn") { turnTimings = selectedRuns.map((candidate, index) => { const submittedAtMs = turnSubmissionTimesMs[index]!;