diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index c17babb0d32..4b4a94f468b 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -14,7 +14,10 @@ reserved for the duration of copyback and closed when it settles; even a one-con receipt. Losing the lock connection does not prove physical copyback stopped: a contender still refuses a receipt whose controller is alive. The original callback joins before its token is released, and publication checks the lock -connection and token. Graceful completion clears the receipt. A controller that +connection and token. Graceful completion clears the receipt. If that cleanup write fails after the +callback joins, only the same exact controller boot retains positive in-process +join evidence and may resume after reconnecting; an unknown token or a new boot +does not inherit that authority. A controller that has exited on the same host can be recovered automatically only when a durable successful workspace barrier proves its copyback finished. A dead parent can leave tar/Git children alive, so incomplete copyback requires operator stop diff --git a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts index fc30be3c786..84227de0049 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts @@ -10,8 +10,13 @@ import { } from "./native-restart-recovery.js"; const OWNER_KEY = "nativeWorkspaceFinalizationOwner"; +const activeOwnerTokens = new Set(); +// Positive same-boot evidence, retained only when durable cleanup could not be +// confirmed. Absence from the active set alone never proves another token joined. +const joinedOwnerTokens = new Set(); type Owner = { token: string; + controllerBootId?: string; hostname: string; pid: number; processStartedAt: string; @@ -51,7 +56,7 @@ export async function withNativeWorkspaceFinalizationOwnership( ): Promise<{ acquired: false } | { acquired: true; value: T }> { const identity = await currentNativeControllerIdentity(); const owner: Owner = { - token: randomUUID(), hostname: os.hostname(), pid: identity.pid, + token: randomUUID(), controllerBootId: identity.bootId, hostname: os.hostname(), pid: identity.pid, processStartedAt: identity.processStartedAt.toISOString(), }; const scope = and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.runtimeMode, "native")); @@ -66,8 +71,8 @@ export async function withNativeWorkspaceFinalizationOwnership( const [run] = await input.db.select({ profile: heartbeatRuns.runnerProfileJson, issueId: heartbeatRuns.nativeIssueId, agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(scope).limit(1); if (!run) throw new Error("native_workspace_finalization_binding_missing"); const rawPrior = run.profile?.[OWNER_KEY]; + const prior = readOwner(rawPrior); if (rawPrior != null) { - const prior = readOwner(rawPrior); const requireStopVerification = async () => { if (run.issueId) await issueRecoveryActionService(input.db).upsertSourceScoped({ companyId: input.companyId, sourceIssueId: run.issueId, @@ -82,27 +87,40 @@ export async function withNativeWorkspaceFinalizationOwnership( }; // A foreign host or malformed receipt cannot prove the old physical owner stopped. if (!prior || prior.hostname !== owner.hostname) return requireStopVerification(); - const takeover = await evaluateNativeControllerTakeover({ - owner: { leaseOwner: prior.token, leaseExpiresAt: new Date(0), controllerPid: prior.pid, - controllerProcessStartedAt: new Date(prior.processStartedAt) }, now: new Date(), - }); - if (!takeover.allowed) return { acquired: false } as const; - // A dead parent may have orphaned tar/Git children. Only a durable - // completed-copyback barrier proves they joined; otherwise board proof - // is required even on the same host. Both callers reuse this barrier. - const [completedCopyback] = await input.db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( - eq(workspaceOperations.companyId, input.companyId), - eq(workspaceOperations.heartbeatRunId, input.runId), - eq(workspaceOperations.issueId, run.issueId!), - eq(workspaceOperations.phase, "workspace_finalize"), - eq(workspaceOperations.status, "succeeded"), - )).limit(1); - if (!completedCopyback) return requireStopVerification(); + const knownJoinedHere = prior.controllerBootId === identity.bootId + && prior.pid === identity.pid && joinedOwnerTokens.has(prior.token) + && !activeOwnerTokens.has(prior.token); + if (!knownJoinedHere) { + const takeover = await evaluateNativeControllerTakeover({ + owner: { leaseOwner: prior.token, leaseExpiresAt: new Date(0), controllerPid: prior.pid, + controllerProcessStartedAt: new Date(prior.processStartedAt) }, now: new Date(), + }); + if (!takeover.allowed) return { acquired: false } as const; + // A dead parent may have orphaned tar/Git children. Only a durable + // completed-copyback barrier proves they joined; otherwise board proof + // is required even on the same host. Both callers reuse this barrier. + const [completedCopyback] = await input.db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, input.companyId), + eq(workspaceOperations.heartbeatRunId, input.runId), + eq(workspaceOperations.issueId, run.issueId!), + eq(workspaceOperations.phase, "workspace_finalize"), + eq(workspaceOperations.status, "succeeded"), + )).limit(1); + if (!completedCopyback) return requireStopVerification(); + } } - await input.db.update(heartbeatRuns).set({ - runnerProfileJson: sql`jsonb_set(coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb), array[${OWNER_KEY}], ${JSON.stringify(owner)}::jsonb)`, - }).where(scope); + // Claim before the potentially ambiguous write so finally can remove + // only our token even if the server committed but its response was lost. + activeOwnerTokens.add(owner.token); claimed = true; + const [claimedRun] = await input.db.update(heartbeatRuns).set({ + runnerProfileJson: sql`jsonb_set(coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb), array[${OWNER_KEY}], ${JSON.stringify(owner)}::jsonb)`, + }).where(and(scope, prior + ? sql`${heartbeatRuns.runnerProfileJson}->${OWNER_KEY}->>'token' = ${prior.token}` + : sql`${heartbeatRuns.runnerProfileJson}->>${OWNER_KEY} is null` + )).returning({ id: heartbeatRuns.id }); + if (!claimedRun) return { acquired: false } as const; + if (prior) joinedOwnerTokens.delete(prior.token); const ownership: NativeWorkspaceFinalizationOwnership = { token: owner.token, assertHeld: async () => { @@ -116,6 +134,7 @@ export async function withNativeWorkspaceFinalizationOwnership( } }, }; + await ownership.assertHeld(); actionPromise = action(ownership); const value = await actionPromise; await ownership.assertHeld(); @@ -126,9 +145,17 @@ export async function withNativeWorkspaceFinalizationOwnership( // callback settles. Join that callback before releasing its durable fence. await actionPromise?.catch(() => undefined); if (claimed) { - await input.db.update(heartbeatRuns).set({ - runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) - ${OWNER_KEY}`, - }).where(owned).catch(() => undefined); + activeOwnerTokens.delete(owner.token); + joinedOwnerTokens.add(owner.token); + try { + await input.db.update(heartbeatRuns).set({ + runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) - ${OWNER_KEY}`, + }).where(owned); + joinedOwnerTokens.delete(owner.token); + } catch { + // A reconnect in this exact controller boot can use the positive join + // receipt. A new boot still needs durable barrier or operator proof. + } } } } diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.ts index a3ba8aab9f6..8847e967d33 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts @@ -220,6 +220,30 @@ describe("native workspace finalization recovery", () => { .toEqual({ acquired: true, value: "recovered" }); }); + it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => { + const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" }); + const interruptedDb = createDb(temporary.connectionString); + let joined = false; + await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => { + await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work"); + joined = true; + await interruptedDb.$client.end({ timeout: 1 }); + })).rejects.toThrow(); + expect(joined).toBe(true); + const [beforeReconnect] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); + const receipt = beforeReconnect.runnerProfileJson?.nativeWorkspaceFinalizationOwner as Record; + expect(receipt).toBeTruthy(); + for (const unknown of [{ ...receipt, controllerBootId: randomUUID() }, { ...receipt, token: randomUUID() }]) { + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...beforeReconnect.runnerProfileJson, nativeWorkspaceFinalizationOwner: unknown } }) + .where(eq(heartbeatRuns.id, seeded.runId)); + expect(await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async () => "must not run")) + .toEqual({ acquired: false }); + } + await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId)); + expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(temporary.connectionString), companyId, runId: seeded.runId }, async () => "resumed")) + .toEqual({ acquired: true, value: "resumed" }); + }); + it("never steals pending copyback from an orphan child after its controller is killed", async () => { const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Orphan copyback child" }); const output = path.join(workspaceRoot, "orphan-copyback.txt");