import { execFile as execFileCallback } from "node:child_process"; import { randomUUID } from "node:crypto"; import { lstat, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; import { and, eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agentWakeupRequests, agents, applyPendingMigrations, companies, createDb, environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, type Db, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js"; import { readFreshWorkspaceReadiness, verifyRetryWorkspace } from "./fresh-workspace-continuation.js"; import { describeRetainedPriorWorkspace, parsePorcelainZ } from "./fresh-workspace-seed.js"; import { recordNativeLocalProcessStop } from "./native-local-process-stop.js"; import { createSystemRecovery, failSystemRecovery, readSystemRecovery, SystemRecoveryFailure } from "./system-recovery-state.js"; import { requestTaskRetry } from "./task-retry.js"; const execFile = promisify(execFileCallback); const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RETRY_WORKSPACE_TEST_DATABASE_URL"); const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); /** R2: a Retry after recovery runs out must not drop the agent's work. */ describe.skipIf(!support.supported)("Retry workspace continuity after exhausted or blocked recovery", () => { let database: Awaited>; let db: Db; let root: string; beforeAll(async () => { root = await mkdtemp(join(tmpdir(), "paperclip-retry-workspace-")); if (externalUrl) { await applyPendingMigrations(externalUrl); database = { connectionString: externalUrl, cleanup: async () => {} } as typeof database; } else database = await startEmbeddedPostgresTestDatabase("retry-workspace-continuity-"); db = createDb(database.connectionString); }, 90_000); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); if (root) await rm(root, { recursive: true, force: true }); }); async function git(cwd: string, ...args: string[]) { return (await execFile("git", ["-C", cwd, ...args])).stdout.trim(); } type Kind = "exhausted_generation" | "blocked_recovery" | "blocked_without_generation" | "duplicate_holds"; async function seed(kind: Kind) { const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sourceRunId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Retry continuity", defaultResponsibleUserId: "board", issuePrefix: `R${companyId.replaceAll("-", "").slice(0, 8)}` }); await db.insert(agents).values({ id: agentId, companyId, name: "Native", role: "engineer", adapterType: "paperclip_runner", status: "idle", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }); const [project] = await db.insert(projects).values({ companyId, name: "Repository" }).returning(); // A real primary checkout and the task's own git worktree with uncommitted edits. const primary = join(root, `${issueId}-primary`); await execFile("git", ["init", "--initial-branch=main", primary]); await git(primary, "config", "user.email", "test@example.com"); await git(primary, "config", "user.name", "Retry test"); await writeFile(join(primary, "README.md"), "committed\n"); await git(primary, "add", "."); await git(primary, "commit", "-m", "base"); const worktree = join(root, `${issueId}-worktree`); const branchName = `TASK-${issueId.slice(0, 8)}`; await git(primary, "worktree", "add", "-b", branchName, worktree, "main"); await writeFile(join(worktree, "README.md"), "committed\nedited before the fault\n"); await writeFile(join(worktree, "NOTES.md"), "untracked pre-fault work\n"); await db.insert(issues).values({ id: issueId, companyId, projectId: project.id, title: "Edit the repository", status: "blocked", assigneeAgentId: agentId, executionWorkspacePreference: "reuse_existing", executionWorkspaceSettings: { mode: "isolated_workspace" } }); const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId: project.id, sourceIssueId: issueId, mode: "isolated_workspace", strategyType: "git_worktree", name: branchName, status: "active", cwd: worktree, providerType: "git_worktree", providerRef: worktree, baseRef: "main", branchName }).returning(); await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issueId)); const blockedKind = kind === "blocked_recovery" || kind === "blocked_without_generation"; const errorCode = blockedKind ? "native_restart_recovery_blocked" : "native_session_retry_exhausted"; const [source] = await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId, nativeIssueId: issueId, runtimeMode: "native", status: "failed", errorCode, processPid: 999_999_999, contextSnapshot: { issueId }, startedAt: new Date(Date.now() - 120_000), finishedAt: new Date(Date.now() - 60_000) }).returning(); await db.insert(nativeRunFinalizations).values({ runId: sourceRunId, companyId, issueId, phase: "terminal_failure", attempt: 3, failureCode: errorCode }); await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: sourceRunId, provider: "local", status: "expired", leasePolicy: "ephemeral", executionWorkspaceId: workspace.id, releasedAt: new Date(Date.now() - 50_000), cleanupStatus: "success" }); expect(await db.transaction(tx => recordNativeLocalProcessStop(tx as unknown as Db, source))).toBe(true); // Recovery clears the process fields after it records the host's stop observation. await db.update(heartbeatRuns).set({ processPid: null }).where(eq(heartbeatRuns.id, sourceRunId)); const generation = createSystemRecovery(sourceRunId); const systemRecovery = kind === "exhausted_generation" || kind === "duplicate_holds" ? failSystemRecovery(generation, new SystemRecoveryFailure("source_unavailable", "use_alternative_strategy")) : { ...generation, state: "complete" as const, phase: "settle_result_or_dispatch" as const }; const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", ownerType: blockedKind ? "board" : "system", returnOwnerAgentId: agentId, cause: blockedKind ? "native_restart_recovery_blocked" : "native_continuation_requires_reconciliation", fingerprint: `native-restart:${sourceRunId}`, nextAction: "Retry the task.", // A blocked restart without an admission hold records no generation. evidence: kind === "blocked_without_generation" ? { runId: sourceRunId, reason: "ownership changed" } : { runId: sourceRunId, systemRecovery } }).returning(); // Several holds can describe the same failed run; a historical one has no generation. if (kind === "duplicate_holds") await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", ownerType: "system", returnOwnerAgentId: agentId, cause: "native_session_retry_exhausted", status: "resolved", fingerprint: randomUUID(), nextAction: "Retry the task.", evidence: { runId: sourceRunId } }); return { companyId, agentId, issueId, source, workspace, worktree, primary, branchName, action }; } async function retry(f: Awaited>) { const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "board", observedRunId: f.source.id, idempotencyKey: randomUUID() }); const successorRunId = randomUUID(); const blocked: string[] = []; const continuation = await db.transaction(async tx => { await tx.select().from(issues).where(eq(issues.id, f.issueId)).for("update"); const admitted = await admitExplicitNativeContinuation({ db: tx as unknown as Db, companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, actorId: "board", actorType: "user", reason: "task_retry", commentId: null, successorRunId, taskRetryRequestId: receipt.requestId!, onBlocked: reason => blocked.push(reason) }); await tx.insert(heartbeatRuns).values({ id: successorRunId, companyId: f.companyId, agentId: f.agentId, nativeIssueId: f.issueId, runtimeMode: "native", status: "queued", contextSnapshot: { issueId: f.issueId, taskRetryRequestId: receipt.requestId } }); await tx.update(agentWakeupRequests).set({ status: "coalesced", runId: successorRunId }) .where(eq(agentWakeupRequests.id, receipt.requestId!)); return admitted; }); expect(blocked).toEqual([]); expect(continuation).toMatchObject({ previousRunId: f.source.id, taskRetryRequestId: receipt.requestId }); const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, successorRunId)); return { successor, requestId: receipt.requestId!, readiness: await readFreshWorkspaceReadiness(db, successor) }; } it.each(["exhausted_generation", "blocked_recovery", "blocked_without_generation", "duplicate_holds"] as const)( "continues a %s Retry in the task's own git worktree with its uncommitted edits", async kind => { const f = await seed(kind); const { successor, requestId, readiness } = await retry(f); expect(readiness).toEqual({ mode: "reuse_issue_workspace", retryIntentId: requestId, sourceRunId: f.source.id, workspaceId: f.workspace.id, cwd: f.worktree }); // The retained directory is the same worktree: git history, branch and pre-fault files are intact. expect((await lstat(join(readiness!.mode === "reuse_issue_workspace" ? (readiness!.cwd ?? "") : "", ".git"))).isFile()).toBe(true); expect(await git(f.worktree, "rev-parse", "--abbrev-ref", "HEAD")).toBe(f.branchName); expect(await readFile(join(f.worktree, "NOTES.md"), "utf8")).toBe("untracked pre-fault work\n"); expect(await readFile(join(f.worktree, "README.md"), "utf8")).toContain("edited before the fault"); // The task binding is unchanged and this attempt consumed the generation's alternative attempt once. const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId)); expect(task.executionWorkspaceId).toBe(f.workspace.id); const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.action.id)); if (kind !== "blocked_without_generation") expect(readSystemRecovery(action.evidence)?.freshAttemptRunId).toBe(successor.id); const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, requestId)); expect((wake.payload?.taskRetry as Record).strategy).toBe("continue"); expect(await readFreshWorkspaceReadiness(db, { ...successor, id: randomUUID() })).toBeNull(); }, 30_000); it("verifies a local checkout after fresh fallback without a plugin worker", async () => { const f = await seed("blocked_without_generation"); const { successor, requestId } = await retry(f); expect(await verifyRetryWorkspace(db, undefined, successor, f.issueId, "board", { mode: "fresh_isolated_workspace", retryIntentId: requestId, sourceRunId: f.source.id, retainedSourceIds: [], })).toEqual({ mode: "reuse_issue_workspace", retryIntentId: requestId, sourceRunId: f.source.id, workspaceId: f.workspace.id, cwd: f.worktree }); }, 30_000); it("lists renamed paths from either porcelain status column once", () => { expect(parsePorcelainZ("R new.md\0old.md\0 R moved.md\0before.md\0?? NOTES.md\0 M README.md\0")).toEqual([ "new.md", "moved.md", "NOTES.md", "README.md"]); }); it("names the retained prior worktree and its uncommitted files for a fresh attempt", async () => { const f = await seed("exhausted_generation"); const prior = await describeRetainedPriorWorkspace(db, { companyId: f.companyId, issueId: f.issueId, sourceRunId: f.source.id, successorRunId: randomUUID(), freshCwd: join(root, "fresh") }); expect(prior).toEqual({ sourceRunId: f.source.id, path: f.worktree, branchName: f.branchName, changedFilesKnown: true, changedFileCount: 2, changedFiles: expect.arrayContaining(["NOTES.md", "README.md"]) }); }, 30_000); it("ignores a workspace operation orphaned by a restart of the stopped source", async () => { const f = await seed("blocked_recovery"); // The server died during the source's preparation; no process can still run it. await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspace.id, heartbeatRunId: f.source.id, issueId: f.issueId, phase: "worktree_prepare", status: "running", startedAt: new Date(Date.now() - 24 * 60 * 60_000) }); const { readiness } = await retry(f); expect(readiness).toMatchObject({ mode: "reuse_issue_workspace", cwd: f.worktree }); }, 30_000); it.each(["live_lease", "running_operation", "corrupt_git", "missing_directory", "remote_authority", "shared_workspace", "export_pending", "repair_required", "live_source_operation"] as const)( "keeps the fresh isolated attempt when the prior workspace is unsafe: %s", async unsafe => { const f = await seed("exhausted_generation"); if (unsafe === "live_lease") await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, provider: "local", status: "active", leasePolicy: "ephemeral", executionWorkspaceId: f.workspace.id }); if (unsafe === "running_operation") await db.insert(workspaceOperations).values({ companyId: f.companyId, // A workspace-control job has no run and can predate the source settling. executionWorkspaceId: f.workspace.id, issueId: f.issueId, phase: "worktree_prepare", status: "running", startedAt: new Date(Date.now() - 24 * 60 * 60_000) }); if (unsafe === "corrupt_git") await writeFile(join(f.worktree, ".git"), "gitdir: /nonexistent/worktree\n"); if (unsafe === "missing_directory") await rm(f.worktree, { recursive: true, force: true }); if (unsafe === "remote_authority") await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: randomUUID(), source: { kind: "empty" }, location: null, writerRunId: null, dirty: false } } }) .where(eq(executionWorkspaces.id, f.workspace.id)); if (unsafe === "shared_workspace") await db.update(executionWorkspaces).set({ mode: "shared_workspace" }) .where(eq(executionWorkspaces.id, f.workspace.id)); if (unsafe === "repair_required") await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_failed" } }) .where(eq(heartbeatRuns.id, f.source.id)); // The source settled just now; an operation it started in this server process may still be running. if (unsafe === "live_source_operation") { await db.update(heartbeatRuns).set({ finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.source.id)); await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspace.id, heartbeatRunId: f.source.id, issueId: f.issueId, phase: "worktree_prepare", status: "running", startedAt: new Date(Date.now() - 1_000) }); } if (unsafe === "export_pending") await db.update(issueRecoveryActions).set({ evidence: { ...f.action.evidence, systemRecovery: { ...readSystemRecovery(f.action.evidence)!, state: "running" } } }) .where(eq(issueRecoveryActions.id, f.action.id)); const { readiness, successor } = await retry(f); expect(readiness).toMatchObject({ mode: "fresh_isolated_workspace", sourceRunId: f.source.id }); // The fresh attempt names the retained directory only when reuse would also // trust it as a source: task-owned, local, settled and with no other writer. const prior = await describeRetainedPriorWorkspace(db, { companyId: f.companyId, issueId: f.issueId, sourceRunId: f.source.id, successorRunId: successor.id, freshCwd: join(root, "fresh") }); if (unsafe === "corrupt_git") expect(prior).toEqual({ sourceRunId: f.source.id, path: f.worktree, branchName: f.branchName, changedFilesKnown: false, changedFileCount: 0, changedFiles: [] }); else expect(prior).toBeNull(); const [task] = await db.select().from(issues).where(and(eq(issues.id, f.issueId), eq(issues.companyId, f.companyId))); expect(task.executionWorkspaceId).toBe(f.workspace.id); }, 30_000); });