diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 28def0f87c9..9c18ab22cea 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -299,7 +299,9 @@ tests this host; it is component coverage, not an end-to-end repair. Manual: prepare a failed run with error code `workspace_validation_failed`, a linked source task, and a live `workspace_validation` recovery action on that -task. Open that exact run and compare its recovery diagnosis to the source task. +task. For retained-file export recovery, use a matching active source action and +terminal run; compare Recover saved files on the task and run inspector. +Open that exact run and compare its recovery diagnosis to the source task. Use an eligible action from the run surface, return to the task, and verify the same action state, persisted outcome, and restored execution. Repeat with an actor who lacks authority. Verify that the run-page controls are diff --git a/ui/src/components/RunWorkspaceRecoverySurface.test.tsx b/ui/src/components/RunWorkspaceRecoverySurface.test.tsx index 6c6514696b9..5b04b9c8e71 100644 --- a/ui/src/components/RunWorkspaceRecoverySurface.test.tsx +++ b/ui/src/components/RunWorkspaceRecoverySurface.test.tsx @@ -16,6 +16,7 @@ const resolveRecoveryMock = vi.hoisted(() => vi.fn()); const reconcileMock = vi.hoisted(() => vi.fn()); const boardAccessMock = vi.hoisted(() => vi.fn()); const recoveryActionsMock = vi.hoisted(() => vi.fn()); +const recoverWorkspaceMock = vi.hoisted(() => vi.fn()); vi.mock("@/lib/router", () => ({ useNavigate: () => navigateMock, @@ -30,6 +31,7 @@ vi.mock("../api/issues", () => ({ create: issueCreateMock, resolveRecoveryAction: resolveRecoveryMock, recoveryActions: recoveryActionsMock, + recoverWorkspace: recoverWorkspaceMock, }, })); @@ -151,10 +153,11 @@ function buildIssue(action: IssueRecoveryAction | null): Issue { } as unknown as Issue; } -async function renderSurface(run: HeartbeatRun) { +async function renderSurface(run: HeartbeatRun, viewerId?: string) { const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, }); + if (viewerId) client.setQueryData(["auth", "session"], { user: { id: viewerId } }); container = document.createElement("div"); document.body.appendChild(container); root = createRoot(container); @@ -278,6 +281,34 @@ describe("RunWorkspaceRecoverySurface", () => { expect(choices()).toHaveLength(2); }); + it("offers the same governed file preservation from the exact failed run without resolving its hold", async () => { + const run = buildRun({ errorCode: "native_workspace_sync_out_retry_exhausted" }); + const action = { ...buildRecoveryAction(), kind: "active_run_watchdog", ownerType: "system", + cause: "native_workspace_sync_out_retry_exhausted", evidence: { runId: run.id, workspaceRecovery: { + schema: "paperclip.workspace-recovery.v1", runId: run.id, status: "blocked", + stage: "restoring_files", retryable: true, requestIds: [], + }, systemRecovery: { state: "exhausted", preservationOperationIds: [] } } } as IssueRecoveryAction; + boardAccessMock.mockResolvedValue({ source: "session", companyIds: ["company-1"], userId: "run-operator", + memberships: [{ companyId: "company-1", membershipRole: "operator", status: "active" }] }); + recoveryActionsMock.mockResolvedValue({ active: action, actions: [action], sourceInspection: { + actionId: action.id, runId: run.id, scope: "recorded_source_identity", status: "verified", + observedAt: "2026-11-03T12:00:00Z", + } }); + recoverWorkspaceMock.mockResolvedValue({ actionId: action.id, status: "queued" }); + const node = await renderSurface(run, "run-operator"); + click([...node.querySelectorAll("button")].find(button => button.textContent === "Inspect recovery")!); + await flush(); + const preserve = [...node.querySelectorAll("button")].find(button => button.textContent === "Recover saved files"); + expect(preserve).toBeDefined(); + click(preserve!); await flush(); + expect(recoverWorkspaceMock).toHaveBeenCalledExactlyOnceWith("issue-1", { + actionId: action.id, runId: run.id, idempotencyKey: expect.any(String), + }); + expect(resolveRecoveryMock).not.toHaveBeenCalled(); + expect(issueCreateMock).not.toHaveBeenCalled(); + expect(node.textContent).not.toContain("Saved files recovered"); + }); + it("renders nothing for a run that is not a workspace-validation failure", async () => { issueGetMock.mockResolvedValue(buildIssue(buildRecoveryAction())); boardAccessMock.mockResolvedValue(undefined); diff --git a/ui/src/components/RunWorkspaceRecoverySurface.tsx b/ui/src/components/RunWorkspaceRecoverySurface.tsx index d57b8a4cb12..150d892f568 100644 --- a/ui/src/components/RunWorkspaceRecoverySurface.tsx +++ b/ui/src/components/RunWorkspaceRecoverySurface.tsx @@ -57,7 +57,7 @@ export function RunWorkspaceRecoverySurface({ run }: { run: HeartbeatRun }) { const issueId = readRunIssueId(run); const cause = run.execution?.cause ?? run.errorCode; if (issueId && (isNativeWorkspaceExportRepairCause(cause) || cause === "native_workspace_sync_out_unrecoverable")) - return ; + return ; return ; } diff --git a/ui/src/components/RunWorkspaceSourceInspection.tsx b/ui/src/components/RunWorkspaceSourceInspection.tsx index 9cea2578912..0ba5812fd62 100644 --- a/ui/src/components/RunWorkspaceSourceInspection.tsx +++ b/ui/src/components/RunWorkspaceSourceInspection.tsx @@ -3,6 +3,7 @@ import { ENVIRONMENT_LEASE_POLICIES, WORKSPACE_SOURCE_VERIFICATION_REASONS } fro import { issuesApi } from "../api/issues"; import { Button } from "./ui/button"; import { formatDateTime } from "../lib/utils"; +import { WorkspaceExportRecovery } from "./WorkspaceExportRecovery"; function recordedFailure(evidence: Record): string { const recovery = evidence.systemRecovery; @@ -19,7 +20,7 @@ function recordedFailure(evidence: Record): string { return [failure.code, detail.reason, ...(allowedPolicy ? [`lease policy ${policy}`] : [])].join(" · "); } -export function RunWorkspaceSourceInspection({ issueId, runId }: { issueId: string; runId: string }) { +export function RunWorkspaceSourceInspection({ issueId, runId, companyId }: { issueId: string; runId: string; companyId: string }) { const listing = useMutation({ mutationFn: () => issuesApi.recoveryActions(issueId, { runId }) }); const inspection = useMutation({ mutationFn: (actionId: string) => issuesApi.recoveryActions(issueId, { runId, actionId }) }); const pending = listing.isPending || inspection.isPending; @@ -41,6 +42,8 @@ export function RunWorkspaceSourceInspection({ issueId, runId }: { issueId: stri + {action.evidence.runId === runId ? : null} ))} {current ?
diff --git a/ui/src/components/WorkspaceExportRecovery.control.test.tsx b/ui/src/components/WorkspaceExportRecovery.control.test.tsx new file mode 100644 index 00000000000..3762dbbedbe --- /dev/null +++ b/ui/src/components/WorkspaceExportRecovery.control.test.tsx @@ -0,0 +1,261 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { IssueRecoveryAction } from "@paperclipai/shared"; +import { queryKeys } from "../lib/queryKeys"; +import { WorkspaceExportRecovery } from "./WorkspaceExportRecovery"; + +const api = vi.hoisted(() => ({ inspect: vi.fn(), recover: vi.fn() })); +vi.mock("../api/issues", () => ({ issuesApi: { recoveryActions: api.inspect, recoverWorkspace: api.recover } })); +(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const action = { + id: "action-1", companyId: "company-1", sourceIssueId: "issue-1", + cause: "native_workspace_sync_out_retry_exhausted", kind: "active_run_watchdog", + status: "active", ownerType: "system", wakePolicy: null, + evidence: { runId: "run-1", workspaceRecovery: { + schema: "paperclip.workspace-recovery.v1", runId: "run-1", status: "blocked", + stage: "restoring_files", attempt: 6, requestIds: [], retryable: true, + }, systemRecovery: { state: "exhausted", preservationOperationIds: [] } }, +} as unknown as IssueRecoveryAction; + +function inspection(value = action) { + return { active: value, actions: [value], sourceInspection: { + schema: "paperclip.workspace-source-inspection.v1", actionId: value.id, runId: "run-1", + status: "verified", scope: "recorded_source_identity", observedAt: "2026-11-03T12:00:00Z", + } }; +} + +describe("governed retained-file recovery", () => { + let root: Root, container: HTMLDivElement, client: QueryClient; + beforeEach(() => { + vi.resetAllMocks(); localStorage.clear(); + api.inspect.mockResolvedValue(inspection()); + api.recover.mockResolvedValue({ actionId: action.id, status: "queued" }); + client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: Infinity }, mutations: { retry: false } } }); + client.setQueryData(queryKeys.auth.session, { user: { id: "board-1" } }); + client.setQueryData(queryKeys.access.currentBoardAccess, { + userId: "board-1", companyIds: ["company-1"], source: "session", isInstanceAdmin: false, + memberships: [{ companyId: "company-1", membershipRole: "operator", status: "active" }], + }); + container = document.createElement("div"); document.body.append(container); root = createRoot(container); + }); + afterEach(async () => { + await act(async () => root.unmount()); container.remove(); client.clear(); localStorage.clear(); + vi.restoreAllMocks(); + }); + async function mount(value: IssueRecoveryAction | null = action) { + const props = { action: value, issueId: "issue-1", companyId: "company-1" }; + await act(async () => root.render( + + )); + await settle(); + } + async function settle() { + await act(async () => { await new Promise(resolve => setTimeout(resolve, 0)); }); + } + function button(label: string) { return [...container.querySelectorAll("button")].find(b => b.textContent === label); } + async function click(label: string) { + const target = button(label); expect(target, `missing ${label}`).toBeDefined(); + await act(async () => target!.click()); await settle(); + } + + it("offers the existing preservation operation for a stopped, exhausted system-owned export", async () => { + await mount(); + await click("Recover saved files"); + expect(api.inspect).toHaveBeenCalledWith("issue-1", { runId: "run-1", actionId: "action-1" }); + expect(api.recover).toHaveBeenCalledTimes(1); + expect(api.recover).toHaveBeenCalledWith("issue-1", { + actionId: "action-1", runId: "run-1", idempotencyKey: expect.any(String), + }); + expect(container.textContent).toContain("queued"); + expect(container.textContent).not.toContain("Saved files recovered"); + }); + + it("requests file preservation on plain HTTP without crypto.randomUUID", async () => { + const originalCrypto = globalThis.crypto; + vi.stubGlobal("crypto", { getRandomValues: originalCrypto.getRandomValues.bind(originalCrypto) }); + try { + await mount(); await click("Recover saved files"); + expect(api.recover).toHaveBeenCalledExactlyOnceWith("issue-1", { + actionId: "action-1", runId: "run-1", idempotencyKey: expect.stringMatching( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/), + }); + expect(container.textContent).not.toContain("Saved files recovered"); + } finally { vi.unstubAllGlobals(); } + }); + + it.each(["viewer", "suspended", "foreign-company", "other-account"])("does not offer mutation with %s access", async kind => { + client.setQueryData(queryKeys.access.currentBoardAccess, { + userId: kind === "other-account" ? "board-2" : "board-1", source: "session", isInstanceAdmin: false, + companyIds: kind === "foreign-company" ? ["company-2"] : ["company-1"], + memberships: [{ companyId: kind === "foreign-company" ? "company-2" : "company-1", + membershipRole: kind === "viewer" ? "viewer" : "operator", status: kind === "suspended" ? "suspended" : "active" }], + }); + await mount(); + expect(button("Recover saved files")).toBeUndefined(); + expect(api.inspect).not.toHaveBeenCalled(); + expect(api.recover).not.toHaveBeenCalled(); + }); + + it.each(["resolved", "cancelled", "foreign-company", "foreign-task", "unsafe-archive"])("does not offer a %s source", async kind => { + await mount({ ...action, ...(kind === "foreign-company" ? { companyId: "company-2" } + : kind === "foreign-task" ? { sourceIssueId: "issue-2" } + : kind === "unsafe-archive" ? { cause: "native_workspace_sync_out_unsafe_archive" } + : { status: kind }) } as IssueRecoveryAction); + expect(button("Recover saved files")).toBeUndefined(); + expect(api.recover).not.toHaveBeenCalled(); + }); + + it("rejects a source/action/run binding that changed after the control was rendered", async () => { + api.inspect.mockResolvedValueOnce(inspection()).mockResolvedValueOnce({ + ...inspection(), sourceInspection: { ...inspection().sourceInspection, runId: "foreign-run" }, + }); + await mount(); await click("Recover saved files"); + expect(api.recover).not.toHaveBeenCalled(); + expect(container.textContent).toContain("File recovery changed"); + }); + + it("does not offer an unverified source", async () => { + api.inspect.mockResolvedValue({ ...inspection(), sourceInspection: { + ...inspection().sourceInspection, status: "blocked", failureCode: "source_binding_invalid", + } }); + await mount(); + expect(button("Recover saved files")).toBeUndefined(); + expect(container.textContent).toContain("source is unavailable or has changed"); + }); + + it("persists the exact request before POST and prevents a double click", async () => { + let finish!: (value: unknown) => void; + api.recover.mockImplementation((_issue, body) => { + expect(localStorage.getItem("workspace-file-recovery:company-1:issue-1:action-1:run-1:board-1")).toContain(body.idempotencyKey); + return new Promise(resolve => { finish = resolve; }); + }); + await mount(); + const target = button("Recover saved files")!; + await act(async () => { target.click(); target.click(); }); await settle(); + expect(api.recover).toHaveBeenCalledTimes(1); + await act(async () => finish({ actionId: action.id, status: "queued" })); await settle(); + expect(button("Recover saved files")).toBeUndefined(); + expect(container.textContent).not.toContain("Saved files recovered"); + }); + + it("reconciles a lost accepted response through GET without another POST", async () => { + let current = action; + api.inspect.mockImplementation(async () => inspection(current)); + api.recover.mockImplementation(async (_issue, body) => { + current = { ...action, evidence: { ...action.evidence, workspaceRecovery: { + ...(action.evidence.workspaceRecovery as object), status: "queued", requestIds: [body.idempotencyKey], + } } }; + throw Object.assign(new Error("Network reply lost"), { status: 503 }); + }); + await mount(); await click("Recover saved files"); + expect(container.textContent).toContain("could not be confirmed"); + await click("Check file recovery status"); + expect(api.recover).toHaveBeenCalledTimes(1); + expect(container.textContent).toContain("Recovering saved files"); + expect(container.textContent).not.toContain("could not be confirmed"); + expect(button("Recover saved files")).toBeUndefined(); + }); + + it("reuses the stored key after remount and a GET that has no acceptance receipt", async () => { + api.recover.mockRejectedValueOnce(Object.assign(new Error("Network unavailable"), { status: 503 })); + await mount(); await click("Recover saved files"); + const first = api.recover.mock.calls[0][1].idempotencyKey; + await act(async () => root.render(null)); + await mount(); + expect(button("Recover saved files")).toBeUndefined(); + await click("Check file recovery status"); + await click("Recover saved files"); + expect(api.recover).toHaveBeenCalledTimes(2); + expect(api.recover.mock.calls[1][1].idempotencyKey).toBe(first); + }); + + it("does not POST when durable storage is unavailable", async () => { + vi.spyOn(Storage.prototype, "setItem").mockImplementation(() => { throw new Error("storage denied"); }); + await mount(); await click("Recover saved files"); + expect(api.recover).not.toHaveBeenCalled(); + expect(container.textContent).toContain("No request was sent"); + }); + + it("does not send an old account's request after a source read completes", async () => { + let finish!: (value: unknown) => void; + api.inspect.mockResolvedValueOnce(inspection()).mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + await mount(); await click("Recover saved files"); + await act(async () => client.setQueryData(queryKeys.auth.session, { user: { id: "board-2" } })); + await act(async () => finish(inspection())); await settle(); + expect(api.recover).not.toHaveBeenCalled(); + }); + + it("shows a confirmed blocked attempt as failed preservation, not queued or recovered", async () => { + let current = action; + api.inspect.mockImplementation(async () => inspection(current)); + api.recover.mockImplementation(async (_issue, body) => { + current = { ...action, evidence: { ...action.evidence, workspaceRecovery: { + ...(action.evidence.workspaceRecovery as object), requestIds: [body.idempotencyKey], + } } }; + return { actionId: action.id, status: "queued" }; + }); + await mount(); await click("Recover saved files"); + expect(container.textContent).toContain("File recovery has not finished"); + expect(container.textContent).not.toContain("File recovery is queued"); + expect(container.textContent).not.toContain("Saved files recovered"); + expect(api.recover).toHaveBeenCalledTimes(1); + }); + + it("offers a new explicit request only after the prior accepted attempt is confirmed exhausted", async () => { + let current = action; + const accepted: string[] = []; + api.inspect.mockImplementation(async () => inspection(current)); + api.recover.mockImplementation(async (_issue, body) => { + accepted.push(body.idempotencyKey); + current = { ...action, evidence: { ...action.evidence, workspaceRecovery: { + ...(action.evidence.workspaceRecovery as object), requestIds: [...accepted], + } } }; + return { actionId: action.id, status: "queued" }; + }); + await mount(); await click("Recover saved files"); + expect(container.textContent).toContain("File recovery has not finished"); + expect(api.recover).toHaveBeenCalledTimes(1); + await click("Try file recovery again"); + expect(api.recover).toHaveBeenCalledTimes(2); + expect(accepted[1]).not.toBe(accepted[0]); + expect(container.textContent).not.toContain("Saved files recovered"); + }); + + it("does not replace the confirmed failed key if another recovery is active at the fresh read", async () => { + let current = action; + api.inspect.mockImplementation(async () => inspection(current)); + api.recover.mockImplementation(async (_issue, body) => { + current = { ...action, evidence: { ...action.evidence, workspaceRecovery: { + ...(action.evidence.workspaceRecovery as object), requestIds: [body.idempotencyKey], + } } }; + return { actionId: action.id, status: "queued" }; + }); + await mount(); await click("Recover saved files"); + const previous = api.recover.mock.calls[0][1].idempotencyKey; + current = { ...current, evidence: { ...current.evidence, workspaceRecovery: { + ...(current.evidence.workspaceRecovery as object), status: "recovering", + } } }; + await click("Try file recovery again"); + expect(api.recover).toHaveBeenCalledTimes(1); + expect(localStorage.getItem("workspace-file-recovery:company-1:issue-1:action-1:run-1:board-1")).toContain(previous); + }); + + it("only reports recovered files after the durable preservation receipt is present", async () => { + const recovered = { ...action, evidence: { ...action.evidence, workspaceRecovery: { + ...(action.evidence.workspaceRecovery as object), status: "recovered", + }, systemRecovery: { state: "complete", preservationOperationIds: [] } } }; + api.inspect.mockResolvedValue(inspection(recovered)); await mount(); + expect(container.textContent).not.toContain("Saved files recovered"); + api.inspect.mockResolvedValue(inspection({ ...recovered, evidence: { ...recovered.evidence, + systemRecovery: { state: "complete", preservationOperationIds: ["host-preservation-operation"] }, + } })); + await act(async () => client.invalidateQueries({ queryKey: queryKeys.issues.recoveryActions("issue-1", "action-1", "run-1") })); + await settle(); + expect(container.textContent).toContain("Saved files recovered"); + expect(api.recover).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/src/components/WorkspaceExportRecovery.tsx b/ui/src/components/WorkspaceExportRecovery.tsx index a5b3dcb45e8..c009f080263 100644 --- a/ui/src/components/WorkspaceExportRecovery.tsx +++ b/ui/src/components/WorkspaceExportRecovery.tsx @@ -1,11 +1,26 @@ +import { useRef, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import type { IssueRecoveryAction } from "@paperclipai/shared"; import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared"; +import { accessApi } from "../api/access"; +import { issuesApi } from "../api/issues"; +import { useDraftOwner, useDraftOwnerReader } from "../hooks/useDraftOwner"; +import { canBoardResolveRecoveryAction, isRetainedWorkspaceExportCause } from "../lib/recovery-reconcile"; +import { queryKeys } from "../lib/queryKeys"; +import { classifySendFailure, pendingSendKeyFor, readPendingSend, writePendingSend } from "../lib/pending-send"; import { asObject, cn } from "../lib/utils"; +import { createUuid } from "../lib/uuid"; +import { Button } from "./ui/button"; -/** Infrastructure repair belongs to the harness/operator, never a task form. */ -export function WorkspaceExportRecovery({ action, className }: { - className?: string; action: IssueRecoveryAction | null; +/** Requests preservation; the existing worker owns stop, copyback and verification. */ +export function WorkspaceExportRecovery({ action, className, issueId, companyId, showStatus = true }: { + className?: string; action: IssueRecoveryAction | null; issueId?: string; companyId?: string; showStatus?: boolean; }) { + if (action && issueId && companyId && isRetainedExport(action, issueId, companyId)) { + return ; + } + if (!showStatus) return null; const ownerRecovery = action?.cause === "native_workspace_finalization_owner_unverified"; if (!action || (!ownerRecovery && !isNativeWorkspaceExportRepairCause(action.cause)) || action.ownerType !== "board" || !["active", "escalated"].includes(action.status) || typeof action.evidence.runId !== "string") return null; @@ -15,3 +30,146 @@ export function WorkspaceExportRecovery({ action, className }: { {automatic ? "Restoring saved work…" : "Workspace recovery needs operator assistance. The agent’s result is saved."}

; } + +function isRetainedExport(action: IssueRecoveryAction, issueId: string, companyId: string) { + return action.companyId === companyId && action.sourceIssueId === issueId + && action.kind === "active_run_watchdog" && ["active", "escalated"].includes(action.status) + && ["board", "system"].includes(action.ownerType) + && isRetainedWorkspaceExportCause(action.cause) + && typeof action.evidence.runId === "string" && action.evidence.runId.length > 0; +} + +type RecoveryInput = { actionId: string; runId: string }; + +function RetainedFilesRecovery({ action, issueId, companyId, className }: { + action: IssueRecoveryAction; issueId: string; companyId: string; className?: string; +}) { + const runId = action.evidence.runId as string; + const owner = useDraftOwner(); + const readOwner = useDraftOwnerReader(); + const client = useQueryClient(); + const busy = useRef(false); + const [acknowledged, setAcknowledged] = useState(false); + const [checkedUnconfirmed, setCheckedUnconfirmed] = useState(false); + const [notice, setNotice] = useState(null); + const access = useQuery({ queryKey: queryKeys.access.currentBoardAccess, + queryFn: () => accessApi.getCurrentBoardAccess(), retry: false }); + const canMutate = Boolean(owner && (access.data?.source === "local_implicit" + || owner !== "signed-out" && access.data?.userId === owner) + && canBoardResolveRecoveryAction(companyId, access.data)); + const source = useQuery({ queryKey: queryKeys.issues.recoveryActions(issueId, action.id, runId), + queryFn: () => issuesApi.recoveryActions(issueId, { runId, actionId: action.id }), enabled: canMutate, retry: false }); + const storageKey = owner ? pendingSendKeyFor(`workspace-file-recovery:${companyId}:${issueId}:${action.id}:${runId}`, owner) : null; + const matchesInput = (value: unknown): value is RecoveryInput => { + const input = asObject(value); + return input.actionId === action.id && input.runId === runId; + }; + const pending = storageKey ? readPendingSend(storageKey, matchesInput) : null; + function boundAction(data: typeof source.data) { + const current = data?.actions.find(item => item.id === action.id); + const inspected = data?.sourceInspection; + const state = asObject(current?.evidence.workspaceRecovery); + return current && isRetainedExport(current, issueId, companyId) && current.evidence.runId === runId + && state.runId === runId && state.schema === "paperclip.workspace-recovery.v1" + && inspected?.actionId === action.id && inspected.runId === runId && inspected.status === "verified" ? current : null; + } + const current = boundAction(source.data); + const recovery = asObject(current?.evidence.workspaceRecovery); + const system = asObject(current?.evidence.systemRecovery); + const serverAcknowledged = Boolean(pending && Array.isArray(recovery.requestIds) && recovery.requestIds.includes(pending.idempotencyKey)); + const restoring = ["queued", "recovering"].includes(String(recovery.status)); + const saved = recovery.status === "recovered" && system.state === "complete" + && Array.isArray(system.preservationOperationIds) && system.preservationOperationIds.length > 0 + && system.preservationOperationIds.every(id => typeof id === "string" && id.length > 0); + const retryable = recovery.status === "blocked" && recovery.retryable === true; + const invalidate = () => { + void client.invalidateQueries({ queryKey: queryKeys.issues.detail(issueId) }); + void client.invalidateQueries({ queryKey: queryKeys.issues.runs(issueId) }); + void client.invalidateQueries({ queryKey: queryKeys.runDetail(runId) }); + }; + const request = useMutation({ retry: false, mutationFn: async (restartFailedAttempt: boolean) => { + if (busy.current) return; + busy.current = true; + setNotice(null); + setCheckedUnconfirmed(false); + setAcknowledged(false); + try { + const observed = await source.refetch(); + const fresh = boundAction(observed.data); + const state = asObject(fresh?.evidence.workspaceRecovery); + if (observed.error || !fresh || state.status !== "blocked" || state.retryable !== true) + throw new Error("File recovery changed. Inspect its current status before requesting recovery."); + if (!canMutate || !owner || readOwner() !== owner || !storageKey) + throw new Error("Your account changed. Refresh before requesting file recovery."); + const previous = readPendingSend(storageKey, matchesInput); + if (restartFailedAttempt && (!previous || !Array.isArray(state.requestIds) + || !state.requestIds.includes(previous.idempotencyKey) + || asObject(fresh.evidence.systemRecovery).state !== "exhausted")) + throw new Error("The previous failed recovery is not confirmed. Check its status before trying again."); + // Unknown acceptance reuses its identity. A new bounded request requires + // an explicit operator click and a freshly confirmed exhausted attempt. + const record = (!restartFailedAttempt && previous) || { + idempotencyKey: createUuid(), payload: { actionId: action.id, runId }, createdAt: Date.now(), + }; + if (!writePendingSend(storageKey, record)) + throw new Error("The recovery request could not be saved in this browser. No request was sent."); + try { + const result = await issuesApi.recoverWorkspace(issueId, { ...record.payload, idempotencyKey: record.idempotencyKey }); + if (result.actionId !== action.id || !["queued", "recovering", "blocked", "recovered"].includes(result.status)) + throw new Error("The recovery acknowledgement could not be confirmed."); + setAcknowledged(true); + setCheckedUnconfirmed(false); + invalidate(); + await source.refetch(); + } catch (error) { + setNotice(classifySendFailure(error) === "rejected" + ? `Recovery request was not confirmed: ${error instanceof Error ? error.message : "Inspect its status."}` + : "The recovery request could not be confirmed. Check its status before trying again; the saved request will be reused."); + } + } catch (error) { + setNotice(error instanceof Error ? error.message : "Unable to inspect file recovery."); + } finally { busy.current = false; } + } }); + const check = useMutation({ retry: false, mutationFn: async () => { + const observed = await source.refetch(); + if (observed.error) throw observed.error; + const fresh = boundAction(observed.data); + if (!fresh) throw new Error("File recovery changed. Refresh the task to inspect the current recovery."); + const state = asObject(fresh.evidence.workspaceRecovery); + const ids = state.requestIds; + const unconfirmed = Boolean(pending && Array.isArray(ids) && !ids.includes(pending.idempotencyKey)); + setCheckedUnconfirmed(unconfirmed); + setNotice(unconfirmed ? "No recovery request is confirmed yet. Trying again will reuse the saved request." : null); + invalidate(); + } }); + const failedReceipt = serverAcknowledged && recovery.status === "blocked"; + const confirmedFailedAttempt = failedReceipt && retryable && system.state === "exhausted"; + const waiting = !failedReceipt && !saved && (restoring || acknowledged || serverAcknowledged); + const unconfirmed = Boolean(pending && !acknowledged && !serverAcknowledged); + return
+

{saved ? "Saved files recovered. Task recovery is still being checked." + : failedReceipt ? "File recovery has not finished. The saved source remains protected." + : restoring ? "Recovering saved files. Work remains paused until recovery is verified." + : waiting ? "File recovery is queued. Waiting for verified saved files." + : "The saved files need recovery before work can continue."}

+ {canMutate ? <> + {current && retryable && !waiting && (!pending || unconfirmed && checkedUnconfirmed || confirmedFailedAttempt) ? : null} + {waiting || unconfirmed || failedReceipt || source.isError || source.data && !current ? : null} + {source.isPending ?

Inspecting the recorded file source…

: null} + {source.isError || check.isError || notice ?

+ {notice ?? (check.error instanceof Error ? check.error.message : "File recovery could not be inspected. Check its status again.")} +

: null} + {current && recovery.status === "blocked" && (serverAcknowledged || recovery.retryable !== true) + ?

File recovery has not finished. The saved source remains protected; service assistance is needed.

: null} + {source.data && !current ?

The recorded source is unavailable or has changed. Inspect recovery before continuing.

: null} + : access.isError ?

Task access could not be checked. Refresh before requesting file recovery.

+ : access.isPending || !owner ?

Checking file recovery access…

+ :

A board member with task access can request file recovery.

} +
; +} diff --git a/ui/src/lib/recovery-reconcile.ts b/ui/src/lib/recovery-reconcile.ts index db57b8416e4..8ec331303ec 100644 --- a/ui/src/lib/recovery-reconcile.ts +++ b/ui/src/lib/recovery-reconcile.ts @@ -34,6 +34,14 @@ export function canBoardManageRuntime( return membership.membershipRole !== "viewer" && membership.membershipRole !== null; } +/** Client affordance for board task mutations; the server rechecks issue access. */ +export const canBoardResolveRecoveryAction = canBoardManageRuntime; + +/** Classifies the native export hold, independently of source or actor readiness. */ +export function isRetainedWorkspaceExportCause(cause: unknown): boolean { + return cause === "native_workspace_sync_out_retry_exhausted" || cause === "native_workspace_sync_out_unrecoverable"; +} + /** * The execution workspace a reconcile action should target. The recovery card is rendered from a * specific `workspace_validation` recovery action whose evidence pins the workspace that diverged; diff --git a/ui/src/pages/DesignGuide.tsx b/ui/src/pages/DesignGuide.tsx index b1a3250dfc8..6e479fd30bf 100644 --- a/ui/src/pages/DesignGuide.tsx +++ b/ui/src/pages/DesignGuide.tsx @@ -2315,6 +2315,12 @@ export function DesignGuide() { recovery. Recovery decisions and attempts belong in the run log; there is no execution status card or reconciliation form.

+

+ When retained-file recovery needs another authorized request, the source task and + run inspector offer Recover saved files. This queues the existing file-preservation + worker. Work remains held until the server verifies the saved files; a queued + acknowledgement does not mean that recovery finished. +

diff --git a/ui/src/pages/IssueDetail.test.tsx b/ui/src/pages/IssueDetail.test.tsx index feaa2e234f1..9923056a8b1 100644 --- a/ui/src/pages/IssueDetail.test.tsx +++ b/ui/src/pages/IssueDetail.test.tsx @@ -84,6 +84,9 @@ const mockIssuesApi = vi.hoisted(() => ({ markRead: vi.fn(), update: vi.fn(), resolveRecoveryAction: vi.fn(), + recoveryActions: vi.fn(), + recoverWorkspace: vi.fn(), + retryTask: vi.fn(), previewTreeControl: vi.fn(), getTreeControlState: vi.fn(), listTreeHolds: vi.fn(), @@ -1376,6 +1379,9 @@ describe("IssueDetail", () => { mockIssuesApi.listComments.mockResolvedValue([]); mockIssuesApi.listAttachments.mockResolvedValue([]); mockIssuesApi.listWorkProducts.mockResolvedValue([]); + mockIssuesApi.recoveryActions.mockReset(); + mockIssuesApi.recoverWorkspace.mockReset(); + mockIssuesApi.retryTask.mockReset(); mockIssuesApi.checkMonitorNow.mockReset(); mockIssuesApi.listFeedbackVotes.mockResolvedValue([]); mockIssuesApi.listInteractions.mockResolvedValue([]); @@ -3423,6 +3429,89 @@ describe("IssueDetail", () => { mockIssuesApi.update.mockReset(); }); + it.each(["verified", "unverified", "stale", "denied"] as const)("keeps native export Retry closed with %s preservation access or source", async mode => { + const action = { + id: "file-recovery", companyId: "company-1", sourceIssueId: "issue-1", + kind: "active_run_watchdog", cause: "native_workspace_sync_out_retry_exhausted", + status: "active", ownerType: "system", evidence: { runId: "stopped-run", workspaceRecovery: { + schema: "paperclip.workspace-recovery.v1", runId: "stopped-run", status: "blocked", + stage: "restoring_files", requestIds: [], retryable: true, + }, systemRecovery: { state: "exhausted", preservationOperationIds: [] } }, + } as unknown as NonNullable; + const issue = createIssue({ status: "blocked", assigneeAgentId: "agent-1", activeRecoveryAction: action, + interruption: { intentId: "held-input", responsibility: "system", state: "failed", nextRetryAt: null, + canRetry: true, userAction: null, sourceRunIds: ["stopped-run"], observedRunId: "stopped-run" }, + executionBlocker: { runId: "stopped-run", cause: "native_workspace_sync_out_retry_exhausted", canRetry: false, workspaceRepairRequired: true, + workspaceRecovery: { status: "blocked", stage: "restoring_files", canRetry: true } } as Issue["executionBlocker"], + }); + mockIssuesApi.get.mockResolvedValue(issue); + mockAuthApi.getSession.mockResolvedValue({ user: { id: "file-operator" } }); + mockAccessApi.getCurrentBoardAccess.mockResolvedValue({ userId: "file-operator", source: "session", + isInstanceAdmin: false, companyIds: ["company-1"], + memberships: [{ companyId: "company-1", status: "active", membershipRole: mode === "denied" ? "viewer" : "operator" }], + }); + mockIssuesApi.recoveryActions.mockResolvedValue({ active: action, actions: [action], sourceInspection: { + actionId: mode === "stale" ? "other-action" : action.id, runId: "stopped-run", + status: mode === "unverified" ? "blocked" : "verified", scope: "recorded_source_identity", + } }); + mockIssuesApi.recoverWorkspace.mockResolvedValue({ actionId: action.id, status: "queued" }); + await act(async () => renderIssueView()); + if (mode !== "verified") { + await waitForAssertion(() => expect(container.textContent).toContain(mode === "denied" + ? "A board member with task access" : "The recorded source is unavailable or has changed")); + expect([...container.querySelectorAll("button")].some(button => ["Recover saved files", "Retry"].includes(button.textContent ?? ""))).toBe(false); + expect(mockIssuesApi.recoverWorkspace).not.toHaveBeenCalled(); + expect(mockIssuesApi.retryTask).not.toHaveBeenCalled(); + return; + } + await waitForAssertion(() => expect([...container.querySelectorAll("button")] + .some(button => button.textContent === "Recover saved files")).toBe(true)); + expect([...container.querySelectorAll("button")].some(button => button.textContent === "Retry")).toBe(false); + const preserve = [...container.querySelectorAll("button")].find(button => button.textContent === "Recover saved files")!; + await act(async () => preserve.click()); + await waitForAssertion(() => expect(mockIssuesApi.recoverWorkspace).toHaveBeenCalledExactlyOnceWith("issue-1", { + actionId: action.id, runId: "stopped-run", idempotencyKey: expect.any(String), + })); + expect(mockIssuesApi.resolveRecoveryAction).not.toHaveBeenCalled(); + expect(mockIssuesApi.update).not.toHaveBeenCalled(); + expect(issue.executionBlocker?.workspaceRepairRequired).toBe(true); + expect(container.textContent).not.toContain("Saved files recovered"); + }); + + it("keeps the existing Retry path for exhausted legacy workspace restoration", async () => { + const interruption: NonNullable = { intentId: "legacy-input", responsibility: "system", state: "failed", nextRetryAt: null, + canRetry: true, userAction: null, sourceRunIds: ["legacy-run"], observedRunId: "legacy-run" }; + const issue = createIssue({ status: "blocked", assigneeAgentId: "agent-1", interruption, + activeRecoveryAction: { id: "legacy-recovery", companyId: "company-1", sourceIssueId: "issue-1", + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", status: "active", ownerType: "system", + evidence: { runId: "legacy-run", workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: ["legacy-lease"] }, + systemRecovery: { state: "exhausted" } }, + } as unknown as NonNullable, + executionBlocker: { runId: "legacy-run", cause: "legacy_execution_requires_reconciliation", + canRetry: true, workspaceRepairRequired: true } as Issue["executionBlocker"], + }); + mockIssuesApi.get.mockResolvedValue(issue); + mockAuthApi.getSession.mockResolvedValue({ user: { id: "legacy-operator" } }); + mockAccessApi.getCurrentBoardAccess.mockResolvedValue({ userId: "legacy-operator", source: "session", + isInstanceAdmin: false, companyIds: ["company-1"], + memberships: [{ companyId: "company-1", status: "active", membershipRole: "operator" }], + }); + mockIssuesApi.retryTask.mockResolvedValue({ requestId: "legacy-retry", accepted: true, stale: false, presentation: interruption }); + await act(async () => renderIssueView()); + await waitForAssertion(() => expect([...container.querySelectorAll("button")].some(button => button.textContent === "Retry")).toBe(true)); + expect([...container.querySelectorAll("button")].some(button => button.textContent === "Recover saved files")).toBe(false); + const retry = [...container.querySelectorAll("button")].find(button => button.textContent === "Retry")!; + await act(async () => retry.click()); + await waitForAssertion(() => expect(mockIssuesApi.retryTask).toHaveBeenCalledExactlyOnceWith(issue.id, { + idempotencyKey: expect.any(String), observedRunId: "legacy-run", + })); + expect(mockIssuesApi.recoverWorkspace).not.toHaveBeenCalled(); + expect(mockIssuesApi.resolveRecoveryAction).not.toHaveBeenCalled(); + expect(mockIssuesApi.update).not.toHaveBeenCalled(); + expect(issue.executionBlocker?.workspaceRepairRequired).toBe(true); + }); + it("moves a blocked task back to todo when the no-live-path notice tries again", async () => { const activeRecoveryAction = { id: "recovery-action-1", diff --git a/ui/src/pages/IssueDetail.tsx b/ui/src/pages/IssueDetail.tsx index 9adb8584e7c..d55fc41d6e5 100644 --- a/ui/src/pages/IssueDetail.tsx +++ b/ui/src/pages/IssueDetail.tsx @@ -85,9 +85,11 @@ import { type LiveRunForIssue, } from "../api/heartbeats"; import { instanceSettingsApi } from "../api/instanceSettings"; -import { accessApi, type CurrentBoardAccess } from "../api/access"; +import { accessApi } from "../api/access"; import { canBoardManageRuntime, + canBoardResolveRecoveryAction, + isRetainedWorkspaceExportCause, readRecoveryReconcileWorkspaceId, } from "../lib/recovery-reconcile"; import { agentsApi } from "../api/agents"; @@ -489,31 +491,11 @@ function treeControlPreviewErrorCopy(error: unknown): string { return error instanceof Error ? error.message : "Unable to load preview."; } -export function canBoardResolveRecoveryAction( - companyId: string | null | undefined, - boardAccess: CurrentBoardAccess | undefined, -) { - if (!companyId || !boardAccess) return false; - if (boardAccess.source === "local_implicit" || boardAccess.isInstanceAdmin) - return true; - if (!boardAccess.memberships || boardAccess.memberships.length === 0) { - return boardAccess.companyIds.includes(companyId); - } - - const membership = boardAccess.memberships.find( - (item) => item.companyId === companyId && item.status === "active", - ); - if (!membership) return false; - return ( - membership.membershipRole !== "viewer" && membership.membershipRole !== null - ); -} - // `canBoardManageRuntime` and `readRecoveryReconcileWorkspaceId` moved to `@/lib/recovery-reconcile` // so the run-page recovery surface can reuse them without importing this page module. Re-exported // here (from the top-of-file import) to keep existing import sites — and their tests — stable, while // the imported bindings stay usable within this module. -export { canBoardManageRuntime, readRecoveryReconcileWorkspaceId }; +export { canBoardManageRuntime, canBoardResolveRecoveryAction, readRecoveryReconcileWorkspaceId }; export function shouldScrollIssueDetailToTopOnNavigation(input: { previousIssueId: string | undefined; @@ -8144,15 +8126,18 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS issueCacheRefs={issueCacheRefs} /> - {resolvedDetailTab === "chat" && !issue.interruption && !issue.executionBlocker?.workspaceRecovery && ( + {resolvedDetailTab === "chat" && ( )} {resolvedDetailTab === "chat" && ( )}