diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 5df3d2d2b83..326a85c1687 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -235,6 +235,15 @@ Workspace readiness accepts a completed operation enclosed by a successful finalize from the same company, workspace, issue, and run. A nested snapshot that finishes before that finalize finishes does not leave a permanent wait. Unfinished operations and work after the finalize still require a later save. +Saved-work recovery ("Recover saved files") ends with its own successful +finalize for the recovered run. That finalize settles the task's finished +workspace operations from the same run that started before it, including on a +workspace that a legacy done child shares with its parent. So the dependents +of a recovered done child stop waiting, and a saved Retry starts on the next +dependency pass. Recovery finalizes already saved by earlier releases count, +with no migration. The recovery finalize never counts as a workspace-wide +save: another task that started work on a shared workspace while the recovery +waited for sandbox teardown still waits for its own save. Daytona file export can preserve an already captured workspace archive when the guarded download copy runs out of sandbox capacity. It removes partial copies, diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 565da8152da..b71f2dd7457 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -2606,6 +2606,20 @@ function pendingFinalizeBlockerCondition(companyId: string, issueId: SQL, worksp and owning_finalize.finished_at >= latest_blocker_operation.finished_at ) ) + and not ( + latest_blocker_operation.status in ('succeeded', 'failed', 'skipped') + and exists ( + select 1 from workspace_operations recovery_finalize + where recovery_finalize.company_id = ${companyId}::uuid + and recovery_finalize.issue_id = ${issueId} + and recovery_finalize.heartbeat_run_id = latest_blocker_operation.heartbeat_run_id + and recovery_finalize.phase = 'workspace_finalize' and recovery_finalize.status = 'succeeded' + and recovery_finalize.metadata->>'owningService' = 'workspace_recovery' + and (recovery_finalize.execution_workspace_id is null + or recovery_finalize.execution_workspace_id = ${workspaceId}) + and recovery_finalize.started_at >= coalesce(latest_blocker_operation.finished_at, latest_blocker_operation.started_at) + ) + ) )`; } diff --git a/server/src/services/workspace-recovery-dependency-finalize.test.ts b/server/src/services/workspace-recovery-dependency-finalize.test.ts new file mode 100644 index 00000000000..5f1210649db --- /dev/null +++ b/server/src/services/workspace-recovery-dependency-finalize.test.ts @@ -0,0 +1,83 @@ +import { and, eq } from "drizzle-orm"; +import { describe, expect, it, vi } from "vitest"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { issueRelations, issues, workspaceOperations } from "@paperclipai/db"; +import { issueService } from "./issues.js"; +import { recoverRetainedWorkspace } from "./workspace-recovery.js"; +import { readWorkspaceRecovery } from "./workspace-recovery-receipt.js"; +import { useWorkspaceRecoveryFixture, acceptedRemoteExport, action, db } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +describe("dependency readiness after a done child's work is recovered", () => { + useWorkspaceRecoveryFixture(targets); + + async function recoveredDoneChildWithParent() { + const f = await acceptedRemoteExport(); + await recoverRetainedWorkspace(f.input); + expect(readWorkspaceRecovery((await action(f.actionId)).evidence)?.status).toBe("recovered"); + const [child] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId)); + const [parent] = await db.insert(issues).values({ companyId: f.companyId, title: "Parent", + status: "blocked", priority: "medium", assigneeAgentId: child.assigneeAgentId, + executionWorkspaceId: f.workspaceId }).returning(); + await db.insert(issueRelations).values({ companyId: f.companyId, issueId: f.issueId, + relatedIssueId: parent.id, type: "blocks" }); + return { ...f, parentId: parent.id }; + } + + const recoveryFinalize = async (f: { companyId: string; runId: string }) => (await db.select().from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, f.companyId), eq(workspaceOperations.heartbeatRunId, f.runId), + eq(workspaceOperations.phase, "workspace_finalize"), eq(workspaceOperations.status, "succeeded"))))[0]!; + const readiness = (id: string) => issueService(db).getDependencyReadiness(id); + + it("releases the parent once Recover saved files has saved a done child's work", async () => { + const f = await recoveredDoneChildWithParent(); + await expect(readiness(f.parentId)).resolves.toMatchObject({ + isDependencyReady: true, pendingFinalizeBlockerIssueIds: [], unresolvedBlockerIssueIds: [] }); + const listed = (await issueService(db).list(f.companyId, { status: "blocked" }) as Array<{ id: string; + blockerAttention?: { pendingFinalizeBlockerIssueIds?: string[] } | null }>).find(issue => issue.id === f.parentId); + expect(listed?.blockerAttention?.pendingFinalizeBlockerIssueIds ?? []).toEqual([]); + }, 30_000); + + it("releases the parent when the recovery saved through a snapshot without a workspace", async () => { + const f = await recoveredDoneChildWithParent(); + const finalize = await recoveryFinalize(f); + await db.update(workspaceOperations).set({ executionWorkspaceId: null }).where(and( + eq(workspaceOperations.companyId, f.companyId), eq(workspaceOperations.heartbeatRunId, f.runId), + eq(workspaceOperations.phase, "workspace_recovery"))); + await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: f.runId, + executionWorkspaceId: f.workspaceId, phase: "workspace_finalize", status: "failed", + startedAt: new Date(finalize.startedAt.getTime() - 60_000), finishedAt: new Date(finalize.startedAt.getTime() - 59_000) }); + await expect(readiness(f.parentId)).resolves.toMatchObject({ + isDependencyReady: true, pendingFinalizeBlockerIssueIds: [] }); + }, 30_000); + + it("keeps a sibling's work that started during the recovery waiting for its own save", async () => { + const f = await recoveredDoneChildWithParent(); + const finalize = await recoveryFinalize(f); + const [sibling] = await db.insert(issues).values({ companyId: f.companyId, title: "Sibling", status: "done", + priority: "medium", executionWorkspaceId: f.workspaceId }).returning(); + await db.insert(issueRelations).values({ companyId: f.companyId, issueId: sibling.id, relatedIssueId: f.parentId, type: "blocks" }); + await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: sibling.id, + executionWorkspaceId: f.workspaceId, phase: "worktree_prepare", status: "succeeded", + startedAt: new Date(finalize.startedAt.getTime() - 1_000), finishedAt: new Date(finalize.startedAt.getTime() - 500) }); + await expect(readiness(f.parentId)).resolves.toMatchObject({ + isDependencyReady: false, pendingFinalizeBlockerIssueIds: [sibling.id] }); + }, 30_000); + + it("keeps the barrier for child workspace work started after the recovery", async () => { + const f = await recoveredDoneChildWithParent(); + const finalize = await recoveryFinalize(f); + await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, + executionWorkspaceId: f.workspaceId, phase: "worktree_prepare", status: "succeeded", + startedAt: new Date(finalize.startedAt.getTime() + 1_000), finishedAt: new Date(finalize.startedAt.getTime() + 2_000) }); + await expect(readiness(f.parentId)).resolves.toMatchObject({ + isDependencyReady: false, pendingFinalizeBlockerIssueIds: [f.issueId] }); + }, 30_000); +});