diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 4674753f237..5084351556e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1456,3 +1456,31 @@ Contracts reference the existing brief and answers instead of copying them again Resumed sessions keep the existing message-delta path; fresh sessions receive the full covered history. Stable wording and bounded references avoid adding another full brief on each comment, but provider cache hits must be measured separately. + +### Native finalization recovery display + +A native finalization retry uses the existing recovery record but does not imply +that an agent turn is running. Task and inbox surfaces show recovery in progress +only while its recorded retry is still due or its matching retry run is verified +live. An expired or missing retry, exhausted budget, or board-owned failure shows +recovery needed rather than “Observing active run.” + +Native recovery reads include a read-only `nativeRunActivity` projection, bound +to the company, source issue, and exact `resume_native_run.runId`. It reports a +queued/running native heartbeat, or a running `workspace_finalize` operation +owned by `native_workspace_finalizer` whose actual callback is still executing +in this server process. A persisted running row alone is not activity evidence: a +crash may leave it behind. The in-process claim ends when the callback joins and +is empty after restart, with no expiry that can misclassify a slow export. This +read-only presentation check never grants takeover authority. It remains authoritative while +the original heartbeat still records its previous failure. Completed operations, +unrelated runs, and other companies do not establish activity. These retries do +not require or synthesize a legacy `scheduledRetryReason`. + +The card describes recovery of the existing run. It does not describe a fresh +owner disposition turn. This also covers native bootstrap/session recovery, +which shares the same resume policy. Inbox rows, source cards, and blocker chips +use the same activity projection. Board-owned repairs stay actionable until a +verified native retry actually starts. Once an explicit board retry is running, +its live activity takes precedence over the prior owner and exhausted budget. +Resolved and cancelled actions remain resolved. diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index fbccc1aa554..197c3c883ac 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -538,6 +538,12 @@ export interface IssueUnblockDescriptor { } export interface IssueRecoveryAction { + /** Read-only activity of the exact native run named by the wake policy. */ + nativeRunActivity?: { + runId: string; + status: "queued" | "running"; + workspaceOperationId: string | null; + } | null; id: string; companyId: string; sourceIssueId: string; diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 1160932558b..357c3cf38f3 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -1,4 +1,7 @@ import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import express from "express"; import request from "supertest"; import { and, eq } from "drizzle-orm"; @@ -19,6 +22,7 @@ import { issueRecoveryActions, issueRelations, issueThreadInteractions, + workspaceOperations, issues, } from "@paperclipai/db"; import { @@ -29,6 +33,7 @@ import { errorHandler } from "../middleware/index.js"; import { issueRoutes } from "../routes/issues.js"; import { buildPaperclipWakePayload, heartbeatService } from "../services/heartbeat.js"; import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js"; +import { workspaceOperationService, isNativeWorkspaceFinalizationOperationActive } from "../services/workspace-operations.js"; import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; import { recoveryService } from "../services/recovery/service.js"; import { noticeMetadataReferencesRecoveryAction } from "../services/recovery/successful-run-handoff.js"; @@ -132,13 +137,18 @@ if (!embeddedPostgresSupport.supported) { describeEmbeddedPostgres("issue recovery actions", () => { let tempDb: Awaited> | null = null; let db: ReturnType; + let operationLogRoot: string; + const priorOperationLogRoot = process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; beforeAll(async () => { + operationLogRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-recovery-operation-logs-")); + process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = operationLogRoot; tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-"); db = createDb(tempDb.connectionString); }, 30_000); afterEach(async () => { + await db.delete(workspaceOperations); await db.delete(issueThreadInteractions); await db.delete(issueRecoveryActions); await db.delete(issueComments); @@ -157,6 +167,9 @@ describeEmbeddedPostgres("issue recovery actions", () => { afterAll(async () => { await tempDb?.cleanup(); + if (priorOperationLogRoot === undefined) delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; + else process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = priorOperationLogRoot; + await fs.rm(operationLogRoot, { recursive: true, force: true }); }); async function seedCompany() { @@ -243,6 +256,137 @@ describeEmbeddedPostgres("issue recovery actions", () => { return app; } + async function seedNativeFinalizationRecovery(status: string) { + const fixture = await seedCompany(); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId: fixture.companyId, + agentId: fixture.coderId, + status, + runtimeMode: "native", + nativeIssueId: fixture.sourceIssueId, + nativePhase: "finalizing", + // Native coordinator retries resume this original run, without creating + // the legacy scheduled_retry_reason projection. + scheduledRetryReason: null, + contextSnapshot: { issueId: fixture.sourceIssueId }, + }); + const svc = issueRecoveryActionService(db); + await svc.upsertSourceScoped({ + companyId: fixture.companyId, + sourceIssueId: fixture.sourceIssueId, + kind: "active_run_watchdog", + ownerType: "agent", + ownerAgentId: fixture.coderId, + cause: "native_finalization_invalid", + fingerprint: `native:${runId}`, + nextAction: "Finish the existing run.", + wakePolicy: { kind: "resume_native_run", runId, notBefore: "2020-01-01T00:00:00Z" }, + maxAttempts: 3, + }); + return { ...fixture, runId, svc }; + } + + it.each(["queued", "running"])("projects the exact %s native run without a legacy scheduled retry", async (status) => { + const { companyId, sourceIssueId, runId, svc } = await seedNativeFinalizationRecovery(status); + const expected = { runId, status, workspaceOperationId: null }; + expect((await svc.getActiveForIssue(companyId, sourceIssueId))?.nativeRunActivity).toEqual(expected); + expect((await svc.listActiveForIssues(companyId, [sourceIssueId])).get(sourceIssueId)?.nativeRunActivity).toEqual(expected); + const response = await request(createApp()).get(`/api/issues/${sourceIssueId}`).expect(200); + expect(response.body.scheduledRetry).toBeNull(); + expect(response.body.activeRecoveryAction.nativeRunActivity).toEqual(expected); + }); + + it("projects an actual long-running export while its original heartbeat remains failed, then clears it", async () => { + const { companyId, sourceIssueId, runId, svc } = await seedNativeFinalizationRecovery("failed"); + let release!: () => void; + let started!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const entered = new Promise((resolve) => { started = resolve; }); + const work = workspaceOperationService(db).createRecorder({ companyId, issueId: sourceIssueId, heartbeatRunId: runId }).recordOperation({ + phase: "workspace_finalize", metadata: { owningService: "native_workspace_finalizer" }, + run: async () => { started(); await held; return { status: "succeeded" }; }, + }); + await entered; + let operationId: string; + try { + const [operation] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, runId)); + operationId = operation!.id; + // A slow callback remains live independently of elapsed wall-clock time. + await db.update(workspaceOperations).set({ startedAt: new Date("2000-01-01T00:00:00Z") }).where(eq(workspaceOperations.id, operationId)); + const scope = { companyId, issueId: sourceIssueId, runId, operationId }; + expect(isNativeWorkspaceFinalizationOperationActive(scope)).toBe(true); + expect(isNativeWorkspaceFinalizationOperationActive({ ...scope, companyId: randomUUID() })).toBe(false); + expect((await svc.getActiveForIssue(companyId, sourceIssueId))?.nativeRunActivity).toEqual({ + runId, status: "running", workspaceOperationId: operationId, + }); + } finally { release(); await work; } + expect((await svc.getActiveForIssue(companyId, sourceIssueId))?.nativeRunActivity).toBeNull(); + // Even a stranded DB row cannot turn a joined callback back into activity. + await db.update(workspaceOperations).set({ status: "running", finishedAt: null }).where(eq(workspaceOperations.id, operationId!)); + expect((await svc.getActiveForIssue(companyId, sourceIssueId))?.nativeRunActivity).toBeNull(); + }); + + it("keeps the activity claim until a timed-out export callback actually joins", async () => { + const { companyId, sourceIssueId, runId } = await seedNativeFinalizationRecovery("failed"); + let release!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const work = workspaceOperationService(db).createRecorder({ companyId, issueId: sourceIssueId, heartbeatRunId: runId }).recordOperation({ + phase: "workspace_finalize", metadata: { owningService: "native_workspace_finalizer" }, timeoutMs: 10, + run: async () => { await held; return { status: "succeeded" }; }, + }); + try { + await expect(work).rejects.toThrow("exceeded"); + const [operation] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, runId)); + const scope = { companyId, issueId: sourceIssueId, runId, operationId: operation!.id }; + expect(isNativeWorkspaceFinalizationOperationActive(scope)).toBe(true); + release(); + await held; + await new Promise((resolve) => setImmediate(resolve)); + expect(isNativeWorkspaceFinalizationOperationActive(scope)).toBe(false); + } finally { release(); } + }); + + it("does not treat an orphaned running export record as live after controller loss", async () => { + const { companyId, sourceIssueId, runId, svc } = await seedNativeFinalizationRecovery("failed"); + await db.insert(workspaceOperations).values({ + companyId, issueId: sourceIssueId, heartbeatRunId: runId, + phase: "workspace_finalize", status: "running", + metadata: { owningService: "native_workspace_finalizer" }, + }); + expect((await svc.getActiveForIssue(companyId, sourceIssueId))?.nativeRunActivity).toBeNull(); + }); + + it.each(["company", "issue", "legacy", "finished"])("rejects an apparently live run with the wrong %s authority", async (mismatch) => { + const { companyId, sourceIssueId, runId, svc } = await seedNativeFinalizationRecovery("running"); + const other = await seedCompany(); + await db.update(heartbeatRuns).set({ + ...(mismatch === "company" ? { companyId: other.companyId, agentId: other.coderId } : {}), + ...(mismatch === "issue" ? { nativeIssueId: other.sourceIssueId } : {}), + ...(mismatch === "legacy" ? { runtimeMode: "legacy" } : {}), + ...(mismatch === "finished" ? { finishedAt: new Date() } : {}), + }).where(eq(heartbeatRuns.id, runId)); + expect((await svc.getActiveForIssue(companyId, sourceIssueId))?.nativeRunActivity).toBeNull(); + }); + + it.each(["company", "issue", "run", "phase", "owner", "finished"])( + "does not borrow native finalization activity with a different %s binding", async (mismatch) => { + const { companyId, sourceIssueId, runId, svc } = await seedNativeFinalizationRecovery("failed"); + const other = await seedNativeFinalizationRecovery("running"); + await db.insert(workspaceOperations).values({ + companyId: mismatch === "company" ? other.companyId : companyId, + issueId: mismatch === "issue" ? other.sourceIssueId : sourceIssueId, + heartbeatRunId: mismatch === "run" ? other.runId : runId, + phase: mismatch === "phase" ? "workspace_prepare" : "workspace_finalize", + status: "running", finishedAt: mismatch === "finished" ? new Date() : null, + metadata: { owningService: mismatch === "owner" ? "different_service" : "native_workspace_finalizer" }, + }); + expect((await svc.getActiveForIssue(companyId, sourceIssueId))?.nativeRunActivity).toBeNull(); + expect((await svc.listActiveForIssues(companyId, [sourceIssueId, other.sourceIssueId])).get(sourceIssueId)?.nativeRunActivity).toBeNull(); + }, + ); + it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => { const { companyId, managerId, sourceIssueId } = await seedCompany(); const svc = issueRecoveryActionService(db); diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 51d5df49adf..aeb7384bee3 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,6 +1,7 @@ -import { and, desc, eq, inArray } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, isNotNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { issueRecoveryActions } from "@paperclipai/db"; +import { heartbeatRuns, issueRecoveryActions, workspaceOperations } from "@paperclipai/db"; +import { isUuidLike } from "@paperclipai/shared"; import type { IssueRecoveryAction, IssueRecoveryActionKind, @@ -9,6 +10,8 @@ import type { IssueRecoveryActionStatus, } from "@paperclipai/shared"; +import { isNativeWorkspaceFinalizationOperationActive } from "./workspace-operations.js"; + const ACTIVE_RECOVERY_ACTION_STATUSES = ["active", "escalated"] as const satisfies readonly IssueRecoveryActionStatus[]; const MAX_UPSERT_RETRIES = 3; @@ -149,6 +152,68 @@ export function issueRecoveryActionService(db: Db) { } } + // Coordinator retries reuse the original native run. They are not legacy + // scheduled retries, and workspace export can run while the heartbeat still + // records its earlier failure. Project their actual activity for every reader. + async function projectNativeRunActivity( + companyId: string, + actions: IssueRecoveryAction[], + dbOrTx: DbOrTransaction, + ) { + const nativeActions = actions.filter((action) => action.wakePolicy?.kind === "resume_native_run"); + for (const action of nativeActions) action.nativeRunActivity = null; + const runIds = [...new Set(nativeActions.flatMap((action) => { + const runId = action.wakePolicy?.runId; + return typeof runId === "string" && isUuidLike(runId) ? [runId] : []; + }))]; + if (runIds.length === 0) return; + const activity = await dbOrTx.select({ + runId: heartbeatRuns.id, + issueId: heartbeatRuns.nativeIssueId, + status: heartbeatRuns.status, + finishedAt: heartbeatRuns.finishedAt, + workspaceOperationId: workspaceOperations.id, + }).from(heartbeatRuns).leftJoin(workspaceOperations, and( + eq(workspaceOperations.companyId, heartbeatRuns.companyId), + eq(workspaceOperations.heartbeatRunId, heartbeatRuns.id), + eq(workspaceOperations.issueId, heartbeatRuns.nativeIssueId), + eq(workspaceOperations.phase, "workspace_finalize"), + eq(workspaceOperations.status, "running"), + isNull(workspaceOperations.finishedAt), + sql`${workspaceOperations.metadata}->>'owningService' = 'native_workspace_finalizer'`, + )).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.runtimeMode, "native"), + inArray(heartbeatRuns.id, runIds), + or( + and(inArray(heartbeatRuns.status, ["queued", "running"]), isNull(heartbeatRuns.finishedAt)), + isNotNull(workspaceOperations.id), + ), + )).orderBy(desc(workspaceOperations.startedAt)); + const activityByRun = new Map(); + for (const row of activity) { + const liveOperation = row.workspaceOperationId && row.issueId + && isNativeWorkspaceFinalizationOperationActive({ + operationId: row.workspaceOperationId, companyId, runId: row.runId, issueId: row.issueId, + }); + const liveHeartbeat = ["queued", "running"].includes(row.status) && row.finishedAt === null; + if (!liveHeartbeat && !liveOperation) continue; + const projected = { ...row, workspaceOperationId: liveOperation ? row.workspaceOperationId : null }; + if (!activityByRun.has(row.runId) || (!activityByRun.get(row.runId)?.workspaceOperationId && liveOperation)) { + activityByRun.set(row.runId, projected); + } + } + for (const action of nativeActions) { + const row = activityByRun.get(String(action.wakePolicy?.runId).toLowerCase()); + if (!row || row.issueId !== action.sourceIssueId) continue; + action.nativeRunActivity = { + runId: row.runId, + status: row.workspaceOperationId || row.status === "running" ? "running" : "queued", + workspaceOperationId: row.workspaceOperationId, + }; + } + } + async function getActiveForIssue( companyId: string, sourceIssueId: string, @@ -167,7 +232,10 @@ export function issueRecoveryActionService(db: Db) { .orderBy(desc(issueRecoveryActions.updatedAt)) .limit(1) .then((rows) => rows[0] ?? null); - return row ? toReadModel(row) : null; + if (!row) return null; + const action = toReadModel(row); + await projectNativeRunActivity(companyId, [action], dbOrTx); + return action; } async function listActiveForIssues(companyId: string, sourceIssueIds: string[]) { @@ -187,6 +255,7 @@ export function issueRecoveryActionService(db: Db) { for (const row of rows) { if (!result.has(row.sourceIssueId)) result.set(row.sourceIssueId, toReadModel(row)); } + await projectNativeRunActivity(companyId, [...result.values()], db); return result; } diff --git a/server/src/services/workspace-operations.ts b/server/src/services/workspace-operations.ts index 773744a40d7..233773bb110 100644 --- a/server/src/services/workspace-operations.ts +++ b/server/src/services/workspace-operations.ts @@ -96,6 +96,22 @@ function readRuntimeControlOwner(metadata: unknown): RuntimeControlOwnerStamp | */ const liveRuntimeControlOperationIds = new Set(); +type NativeFinalizationActivityScope = { + operationId: string; + companyId: string; + runId: string; + issueId: string; +}; +// Presentation-only positive proof. Persisted running rows can outlive a crash; +// this registry only exists while this process awaits the actual export callback. +const liveNativeFinalizationOperations = new Map(); + +export function isNativeWorkspaceFinalizationOperationActive(scope: NativeFinalizationActivityScope): boolean { + const active = liveNativeFinalizationOperations.get(scope.operationId); + return Boolean(active && active.companyId === scope.companyId + && active.runId === scope.runId && active.issueId === scope.issueId); +} + /** * Whether the process that stamped an operation is still alive on this host. A dead owner can * never heartbeat again, so its operation is recoverable immediately instead of after the @@ -579,8 +595,24 @@ export function workspaceOperationService(db: Db) { }; const timeoutMs = recordInput.timeoutMs ?? defaultRuntimeControlTimeoutMs(runtimeControlAction); + const runWithActivity = async () => { + const nativeFinalization = recordInput.phase === "workspace_finalize" + && recordInput.metadata?.owningService === "native_workspace_finalizer" + && input.heartbeatRunId && input.issueId; + if (nativeFinalization) liveNativeFinalizationOperations.set(id, { + operationId: id, companyId: input.companyId, + runId: input.heartbeatRunId!, issueId: input.issueId!, + }); + try { + return await recordInput.run(reportProgress); + } finally { + // Join the underlying callback, even if the recorder's timeout + // already fired. A deadline cannot prove physical export stopped. + liveNativeFinalizationOperations.delete(id); + } + }; const settle = async () => { - if (!timeoutMs || timeoutMs <= 0) return await recordInput.run(reportProgress); + if (!timeoutMs || timeoutMs <= 0) return await runWithActivity(); const timeout = new Promise((_resolve, reject) => { timeoutTimer = setTimeout( () => reject(new WorkspaceOperationTimeoutError(timeoutMs, runtimeControlAction)), @@ -588,7 +620,7 @@ export function workspaceOperationService(db: Db) { ); timeoutTimer.unref?.(); }); - return await Promise.race([recordInput.run(reportProgress), timeout]); + return await Promise.race([runWithActivity(), timeout]); }; try { diff --git a/ui/src/components/IssueBlockedNotice.test.tsx b/ui/src/components/IssueBlockedNotice.test.tsx index 5bfa04246ae..8651ff7ac05 100644 --- a/ui/src/components/IssueBlockedNotice.test.tsx +++ b/ui/src/components/IssueBlockedNotice.test.tsx @@ -844,6 +844,17 @@ describe("IssueBlockedNotice", () => { ).querySelector('[data-testid="issue-blocked-notice-recovery-indicator"]'); } + it("uses native export activity on a blocker without a scheduled retry", () => { + const action = buildDispositionRepairAction({ + kind: "resume_native_run", runId: "native-run", notBefore: "2020-01-01T00:00:00Z", + }, { + kind: "active_run_watchdog", cause: "native_finalization_invalid", + nativeRunActivity: { runId: "native-run", status: "running", workspaceOperationId: "export-operation" }, + }); + expect(renderBlockerChip(action)?.getAttribute("data-recovery-state")).toBe("in_progress"); + expect(deriveRecoveryCardState(action)).toBe("in_progress"); + }); + it("reports the same liveness state as the source task's recovery card", () => { const liveAction = buildDispositionRepairAction({ type: "bounded_owner_disposition_repair", diff --git a/ui/src/components/IssueRecoveryActionCard.test.tsx b/ui/src/components/IssueRecoveryActionCard.test.tsx index d8480fb51c5..75e4b3330cf 100644 --- a/ui/src/components/IssueRecoveryActionCard.test.tsx +++ b/ui/src/components/IssueRecoveryActionCard.test.tsx @@ -134,6 +134,34 @@ describe("deriveRecoveryCardState", () => { }); describe("IssueRecoveryActionCard", () => { + it.each(["running", "scheduled", "missed"] as const)("describes native %s finalization without claiming another agent turn", (state) => { + const node = render(); + expect(node.querySelector("section")?.getAttribute("data-recovery-state")).toBe(state === "missed" ? "needed" : "in_progress"); + expect(node.textContent).toContain("existing run"); + expect(node.textContent).not.toContain("retrying the original owner"); + expect(node.textContent).not.toContain("retrying itself"); + expect(node.textContent).not.toContain("repairs the next step only"); + expect(node.textContent).not.toContain("stopped to wait"); + expect(node.querySelector('[data-testid="recovery-retry-progress"]')?.getAttribute("data-recovery-lane")).toBe("native_run"); + }); + + it("describes actual export progress after the board retries an exhausted repair", () => { + const node = render(); + expect(node.querySelector("section")?.getAttribute("data-recovery-state")).toBe("in_progress"); + expect(node.textContent).toContain("Paperclip is recovering the existing run"); + expect(node.querySelector('[data-testid="recovery-recovery-owner"]')?.textContent).toContain("Paperclip"); + expect(node.textContent).not.toContain("Automatic retries are finished"); + }); + it("renders state and kind attributes with owner names and the recorded next action", () => { const node = render( - {recoveryOwnerIsSourceOwner ? ( + {lineage.lane === "native_run" && (action.ownerType !== "board" || Boolean(lineage.liveRunId)) ? ( + <> + Paperclip + recovers the existing run + + ) : recoveryOwnerIsSourceOwner ? ( Original owner — retrying itself ) : action.ownerType === "agent" && action.ownerAgentId ? ( <> diff --git a/ui/src/components/IssueRow.test.tsx b/ui/src/components/IssueRow.test.tsx index be507931c5d..188ece5f703 100644 --- a/ui/src/components/IssueRow.test.tsx +++ b/ui/src/components/IssueRow.test.tsx @@ -771,6 +771,18 @@ describe("IssueRow", () => { return snapshot; } + it("shows native export activity from the action even without a scheduled retry", () => { + const issue = recoveryIssue(at(-5 * 60_000)); + Object.assign(issue.activeRecoveryAction!, { + kind: "active_run_watchdog", cause: "native_finalization_invalid", + wakePolicy: { kind: "resume_native_run", runId: "native-run", notBefore: at(-5 * 60_000) }, + nativeRunActivity: { runId: "native-run", status: "running", workspaceOperationId: "export-operation" }, + }); + const chip = renderChip(issue); + expect(chip?.getAttribute("data-recovery-state")).toBe("in_progress"); + expect(chip?.getAttribute("aria-label")).not.toContain("missed"); + }); + it("stays calm while the next attempt is still ahead", () => { const chip = renderChip(recoveryIssue(at(3 * 60_000))); expect(chip?.getAttribute("data-recovery-state")).toBe("in_progress"); diff --git a/ui/src/lib/recovery-display.test.ts b/ui/src/lib/recovery-display.test.ts index 8e3e03ef121..8c6275febf6 100644 --- a/ui/src/lib/recovery-display.test.ts +++ b/ui/src/lib/recovery-display.test.ts @@ -117,6 +117,74 @@ describe("deriveRecoveryDisplayState", () => { }, ); + describe("native finalization recovery", () => { + const now = Date.parse("2026-09-27T02:00:00.000Z"); + const action = { + ...base, + kind: "active_run_watchdog" as const, + ownerType: "agent" as const, + cause: "native_finalization_invalid", + attemptCount: 1, + maxAttempts: 3, + evidence: { runId: "finalizing-run", coordinatorAttempt: 1 }, + wakePolicy: { + kind: "resume_native_run", + runId: "finalizing-run", + notBefore: "2026-09-27T01:48:53.988Z", + }, + }; + + it("shows a missed native finalization retry as recovery needed", () => { + const state = deriveRecoveryDisplayState(action, { now }); + expect(state).toBe("needed"); + expect(recoveryChipLabel(state as "needed", action.kind)).toBe("Recovery needed"); + }); + + it("describes a future finalization retry as recovery, not an active agent turn", () => { + expect(deriveRecoveryDisplayState({ + ...action, + wakePolicy: { ...action.wakePolicy, notBefore: "2026-09-27T02:01:00.000Z" }, + }, { now })).toBe("in_progress"); + }); + + it.each(["queued", "running"] as const)("uses the API's %s finalization projection without a scheduled retry", (status) => { + expect(deriveRecoveryDisplayState({ + ...action, + nativeRunActivity: { runId: "finalizing-run", status, workspaceOperationId: null }, + }, { now })).toBe("in_progress"); + expect(deriveRecoveryDisplayState({ + ...action, + nativeRunActivity: { runId: "different-run", status, workspaceOperationId: null }, + }, { now })).toBe("needed"); + }); + + it("does not borrow an unrelated legacy retry to claim native finalization activity", () => { + expect(deriveRecoveryDisplayState(action, { + now, scheduledRetry: { runId: "finalizing-run", status: "running" }, + })).toBe("needed"); + }); + + it.each([null, { kind: "resume_native_run" }, { kind: "resume_native_run", notBefore: "invalid" }])( + "does not claim observation without a usable finalization retry (%j)", + (wakePolicy) => { + expect(deriveRecoveryDisplayState({ ...action, wakePolicy }, { now })).toBe("needed"); + }, + ); + + it.each(["active", "escalated"] as const)("shows a board-admitted export as progress despite prior %s repair ownership", (status) => { + expect(deriveRecoveryDisplayState({ + ...action, status, ownerType: "board", attemptCount: 3, + nativeRunActivity: { runId: "finalizing-run", status: "running", workspaceOperationId: "export-operation" }, + }, { now })).toBe("in_progress"); + }); + + it("keeps exhausted retries and board-owned failures actionable", () => { + const wakePolicy = { ...action.wakePolicy, notBefore: "2026-09-27T02:01:00.000Z" }; + expect(deriveRecoveryDisplayState({ ...action, wakePolicy, attemptCount: 3 }, { now })).toBe("needed"); + expect(deriveRecoveryDisplayState({ ...action, wakePolicy, ownerType: "board" }, { now })).toBe("needed"); + }); + }); + it("preserves observation for an agent-owned watchdog", () => { expect( deriveRecoveryDisplayState({ diff --git a/ui/src/lib/recovery-display.ts b/ui/src/lib/recovery-display.ts index c43e16382dd..83458672d9c 100644 --- a/ui/src/lib/recovery-display.ts +++ b/ui/src/lib/recovery-display.ts @@ -55,12 +55,14 @@ export type RecoveryDisplayInput = Pick< Partial< Pick< IssueRecoveryAction, + | "cause" | "ownerType" | "wakePolicy" | "evidence" | "attemptCount" | "maxAttempts" | "timeoutAt" + | "nativeRunActivity" > >; @@ -69,13 +71,7 @@ export function deriveRecoveryDisplayState( context?: RecoveryLivenessContext, ): RecoveryDisplayState { if (action.status === "resolved") return "resolved"; - if (action.status === "escalated") return "escalated"; if (action.status === "cancelled") return "resolved"; - if (action.kind === "active_run_watchdog") { - // Native terminal failures also use this kind. Board ownership means a - // human must choose recovery; it is not evidence of a still-running turn. - return action.ownerType === "board" ? "needed" : "observe_only"; - } // A bounded retry lineage still holding a durable path is work the server will do on its // own. Shouting "recovery needed" over it would ask a human to fix something nobody has to // fix yet, so the calm tone is reserved for a lane with an attempt genuinely still coming. @@ -87,7 +83,22 @@ export function deriveRecoveryDisplayState( attemptCount: action.attemptCount, maxAttempts: action.maxAttempts, timeoutAt: action.timeoutAt, + nativeRunActivity: action.nativeRunActivity, }, context); + // An explicit board retry may retain its old owner/budget while the exact + // native run is already making progress. Actual activity wins over that + // historical repair state, but a merely scheduled board retry does not. + if (lineage?.lane === "native_run" && lineage.liveRunId) return "in_progress"; + if (action.status === "escalated") return "escalated"; + if (action.kind === "active_run_watchdog") { + // Native finalization shares the watchdog kind, but resumes a failed + // coordinator rather than observing a live agent turn. Preserve board + // ownership and only describe recovery as active while its retry is live. + if (action.ownerType === "board") return "needed"; + if (lineage) return lineage.hasDurablePath ? "in_progress" : "needed"; + if (action.cause?.startsWith("native_")) return "needed"; + return "observe_only"; + } if (lineage && lineage.lane !== "board" && lineage.hasDurablePath) return "in_progress"; if (action.outcome === "delegated") return "in_progress"; return "needed"; diff --git a/ui/src/lib/recovery-lineage.ts b/ui/src/lib/recovery-lineage.ts index efb2809a1b2..1bd8e61edf3 100644 --- a/ui/src/lib/recovery-lineage.ts +++ b/ui/src/lib/recovery-lineage.ts @@ -10,7 +10,7 @@ import { formatMonitorOffset } from "./issue-monitor"; * therefore the canonical lane signal — every surface reads the same stored value rather * than re-deriving liveness on its own. */ -export type RecoveryRetryLane = "source_owner" | "recovery_owner" | "board"; +export type RecoveryRetryLane = "source_owner" | "recovery_owner" | "native_run" | "board"; export interface RecoveryRetryLineage { lane: RecoveryRetryLane; @@ -55,6 +55,7 @@ export interface RecoveryRetryLineage { const SOURCE_LANE_POLICY = "bounded_owner_disposition_repair"; const RECOVERY_LANE_POLICY = "bounded_recovery_owner"; const BOARD_LANE_POLICY = "board_escalation"; +const NATIVE_RESUME_POLICY = "resume_native_run"; /** Scheduled-run states that mean the parked attempt is actually in flight right now. */ const LIVE_SCHEDULED_RUN_STATUSES = new Set(["queued", "running"]); @@ -70,7 +71,7 @@ const LIVE_SCHEDULED_RUN_STATUSES = new Set(["queued", "running"]); const RETRY_EXPIRY_GRACE_MS = 30_000; export type RecoveryLineageInput = Pick & - Partial>; + Partial>; /** * The source issue's scheduled-retry record, which is what lets a surface tell a run that is @@ -143,13 +144,15 @@ export function readRecoveryRetryLineage( const policy = asRecord(action.wakePolicy); if (!policy) return null; const type = asNonEmptyString(policy.type); + const nativeResume = policy.kind === NATIVE_RESUME_POLICY; const preservesSourceAssignee = policy.preservesSourceAssignee === true; const evidence = asRecord(action.evidence) ?? {}; const evidenceSourceAttempt = asCount(evidence.sourceAttemptCount); const evidenceSourceMaxAttempts = asCount(evidence.sourceMaxAttempts); let lane: RecoveryRetryLane; - if (type === SOURCE_LANE_POLICY) lane = "source_owner"; + if (nativeResume) lane = "native_run"; + else if (type === SOURCE_LANE_POLICY) lane = "source_owner"; else if (type === RECOVERY_LANE_POLICY) lane = "recovery_owner"; else if ( type === BOARD_LANE_POLICY && @@ -160,11 +163,18 @@ export function readRecoveryRetryLineage( const attempt = asCount(policy.attempt) ?? asCount(action.attemptCount) ?? 0; const maxAttempts = asCount(policy.maxAttempts) ?? asCount(action.maxAttempts); - const scheduledRunId = asNonEmptyString(policy.scheduledRunId); - const storedRetryAt = asIsoDate(policy.retryAt) ?? asIsoDate(action.timeoutAt); + const scheduledRunId = asNonEmptyString(nativeResume ? policy.runId : policy.scheduledRunId); + // Native recovery resumes an existing run; its scheduling fields + // differ from an agent wake. A retry without that run identity is not a path. + if (nativeResume && scheduledRunId === null) return null; + const storedRetryAt = nativeResume + ? asIsoDate(policy.notBefore) + : asIsoDate(policy.retryAt) ?? asIsoDate(action.timeoutAt); const exhausted = lane === "board" || (maxAttempts !== null && attempt >= maxAttempts); const nextRetryAt = exhausted ? null : storedRetryAt; - const liveRunId = resolveLiveRunId(scheduledRunId, context?.scheduledRetry); + const liveRunId = nativeResume + ? resolveLiveRunId(scheduledRunId, action.nativeRunActivity) + : resolveLiveRunId(scheduledRunId, context?.scheduledRetry); // A run that is already executing is the reason its due time is behind us, so a confirmed // live run is never "missed" — it is the attempt happening. const retryExpired = @@ -184,11 +194,12 @@ export function readRecoveryRetryLineage( scheduledRunId, liveRunId, retryAgentId: asNonEmptyString(policy.retryAgentId) ?? asNonEmptyString(policy.ownerAgentId), - preservesSourceAssignee: preservesSourceAssignee || lane === "source_owner", - sourceAttempt: lane === "source_owner" ? attempt : evidenceSourceAttempt, - sourceMaxAttempts: lane === "source_owner" ? maxAttempts : evidenceSourceMaxAttempts, + preservesSourceAssignee: preservesSourceAssignee || lane === "source_owner" || nativeResume, + sourceAttempt: nativeResume ? null : lane === "source_owner" ? attempt : evidenceSourceAttempt, + sourceMaxAttempts: nativeResume ? null : lane === "source_owner" ? maxAttempts : evidenceSourceMaxAttempts, hasDurablePath: - !exhausted && (liveRunId !== null || (nextRetryAt !== null && !retryExpired)), + (nativeResume && liveRunId !== null) || + (!exhausted && (liveRunId !== null || (nextRetryAt !== null && !retryExpired))), }; }