From ad0543ae90851da610de8df5ee0b4128ea6b1185 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:47:29 -0600 Subject: [PATCH] test(recovery): re-arm a capture-policy save whose finished coordinator names a lapsed owner (#20672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Workspace recovery re-arms a save that an older capture policy exhausted (#20569), so the files are saved without a person > - On staging, one save of this kind was not re-armed after the deploy. Its export hold also carries a workspace finalization block, and it has a resolved no-replay sibling for the same run > - #20661 found that discovery skipped a hold when the run's finished coordinator still named a lease owner whose lease had lapsed. The same discovery clause gates the capture-policy re-arm > - This pull request adds a regression test for that exact capture-policy shape. It shows that #20661 alone makes the worker re-arm and save it > - The benefit is that the capture-policy path keeps this guarantee in later changes ## Linked Issues or Issue Description Refs #20661 Refs #20569 **What happened?** An exhausted capture-policy save (`export_interrupted`, cause `provider_error`, 6 of 6 attempts, phase `preserve_files`) was not re-armed on the worker's first pass after #20569 was deployed. **What did you expect to happen?** The worker re-arms the save once under the current capture policy, saves the files, and returns the hold to the system. **Steps to reproduce** 1. Exhaust a save under capture policy 1 with `export_interrupted`. Add a workspace finalization block and a resolved no-replay sibling for the same run. 2. Leave a lapsed `native-finalizer:` lease owner on the run's `terminal_failure` coordinator. 3. Run the workspace recovery worker. Before #20661, `nextWorkspaceRecoveryAt` returns null and the hold is never selected. ## What Changed - `workspace-recovery-capture-policy.test.ts`: add "re-arms a save whose finished coordinator still names a lapsed finalizer owner". The test uses the staging record: export hold with `historicalWorkspaceSource` and `workspaceFinalizationBlock`, a resolved no-replay sibling, waiting user input, and a lapsed finalizer owner. It drives the worker's own discovery loop until the worker is idle. - No production code changes. ## Verification - Red: with the source files from before #20661 (`git checkout 686893636d^ -- `), the new test fails. `nextWorkspaceRecoveryAt` returns null, so the worker is never woken. - Green: on current master, `pnpm --filter @paperclipai/server exec vitest run src/services/workspace-recovery-capture-policy.test.ts` passes (5 tests). The file passed 4 runs in a row. - The test asserts that the save completes under policy 2, the hold returns to the system, `releasedLapsedOwner` is recorded, and the lapsed owner is released. ## Risks - Low risk. This change adds a test only. - Previous-release story: the test uses rows in the shape an older release saved, with no capture-policy marker. No migration is needed. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use (shell, code editing, browser GET reads) in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../workspace-recovery-capture-policy.test.ts | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/server/src/services/workspace-recovery-capture-policy.test.ts b/server/src/services/workspace-recovery-capture-policy.test.ts index 0cf7c81b8ad..1440c960272 100644 --- a/server/src/services/workspace-recovery-capture-policy.test.ts +++ b/server/src/services/workspace-recovery-capture-policy.test.ts @@ -3,7 +3,7 @@ import { mkdir, readFile, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; import { eq } from "drizzle-orm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, agentWakeupRequests, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { activityLog, agentWakeupRequests, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { getExecutionBlocker } from "./execution-blocker.js"; @@ -215,6 +215,41 @@ describe("saves exhausted under an older workspace capture policy", () => { expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); }, 90_000); + it("re-arms a save whose finished coordinator still names a lapsed finalizer owner", async () => { + const f = await stagingIncident(false); + const exportHold = await action(f.actionId); + const source = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!; + await db.update(issueRecoveryActions).set({ evidence: { ...exportHold.evidence, workspaceFinalizationBlock: { + schema: "paperclip.native-workspace-finalization-block.v1", runId: f.runId, agentId: source.agentId, issueId: f.issueId, + companyId: f.companyId, resultId: f.resultId, contractId: source.completionContractId, resultSha256: "5".repeat(64), + contractSha256: "0".repeat(64), statusVersion: 7, priorDecisionId: null, identityContextId: randomUUID(), + responsibleUserId: "board" } } }).where(eq(issueRecoveryActions.id, f.actionId)); + const lapsedOwner = `native-finalizer:${randomUUID()}`; + await db.update(nativeRunFinalizations).set({ leaseOwner: lapsedOwner, leaseExpiresAt: new Date(Date.now() - 8 * 60 * 60_000), + controllerBootId: randomUUID(), controllerPid: 2_147_000_000 }).where(eq(nativeRunFinalizations.runId, f.runId)); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure" }); + + expect(await nextWorkspaceRecoveryAt(db)).toBeLessThanOrEqual(Date.now()); + expect((await retainedWorkspaceCandidates(db, undefined, false, true)).map(row => row.action.id)).toContain(f.actionId); + let passes = 0; + for (; passes < 6; passes += 1) { + const due = await nextWorkspaceRecoveryAt(db); + if (due === null || due > Date.now()) break; + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => undefined, true) ?? []); + } + expect(passes).toBeLessThan(6); + + const saved = await action(f.actionId); + expect(readSystemRecovery(saved.evidence)).toMatchObject({ state: "complete", capturePolicyVersion: REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION }); + expect(readWorkspaceRecovery(saved.evidence)?.status).toBe("recovered"); + expect(saved.evidence.capturePolicyRecovery).toMatchObject({ releasedLapsedOwner: true }); + expect(saved.ownerType).toBe("system"); + expect(String(saved.nextAction)).not.toMatch(adminText); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]?.leaseOwner) + .not.toBe(lapsedOwner); + }, 90_000); + it("does not re-arm a save that the current policy already exhausted", async () => { const f = await exhaustedCaptureHold(REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION); expect((await retainedWorkspaceCandidates(db, undefined, false, true)).map(row => row.action.id)).not.toContain(f.actionId);