diff --git a/server/src/services/execution-control-work.ts b/server/src/services/execution-control-work.ts index 69981ba1647..8bc16172fb9 100644 --- a/server/src/services/execution-control-work.ts +++ b/server/src/services/execution-control-work.ts @@ -6,7 +6,7 @@ import { type DeliveryQueue } from "./delivery-work-notifications.js"; import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "./execution-control-deadline.js"; import { obsoleteExecutionConversationHold, unresolvedExecutionDispositions } from "./execution-recovery-resolution.js"; import { heldExecutionsAwaitingFreshContinuation, nextFreshContinuationCheckAt } from "./fresh-continuation-fallback.js"; -import { hasPendingNativeReplacements } from "./native-runtime/native-safe-replacement.js"; +import { hasPendingNativeReplacements, nextNativeOwnershipProofDeadlineAt } from "./native-runtime/native-safe-replacement.js"; import { nextAbandonedNativeOwnerReleaseAt } from "./native-runtime/native-failed-run-owner-release.js"; import { hasPendingPauseResume } from "./issue-tree-resume.js"; import { hasPendingBudgetResume } from "./budget-resume-continuation.js"; @@ -89,7 +89,9 @@ export function executionControlSchedule(db: Db) { return at === null ? null : deadline([new Date(at)]); }, async replacement() { - return await hasPendingNativeReplacements(db) ? retry() : null; + if (await hasPendingNativeReplacements(db)) return retry(); + const at = await nextNativeOwnershipProofDeadlineAt(db); + return at === null ? null : deadline([new Date(at)]); }, async continuation() { const rows = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( diff --git a/server/src/services/native-runtime/native-safe-replacement.test.ts b/server/src/services/native-runtime/native-safe-replacement.test.ts index 088ae1bffc6..a0fdc9116c7 100644 --- a/server/src/services/native-runtime/native-safe-replacement.test.ts +++ b/server/src/services/native-runtime/native-safe-replacement.test.ts @@ -31,6 +31,7 @@ import { agents, companies, createDb, + environmentLeases, subscribeDatabaseWork, heartbeatRunEvents, heartbeatRuns, @@ -458,6 +459,114 @@ const support = externalDatabaseUrl const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, source.issueId)); expect(action.status).toBe("active"); }); + async function seedUnprovenOwner(finishedAgoMs: number) { + const source = await seed(); + await db.update(heartbeatRuns).set({ finishedAt: sql`now() - ${finishedAgoMs}::integer * interval '1 millisecond'`, + processPid: process.pid, processHostHash: "another-server" }).where(eq(heartbeatRuns.id, source.runId)); + invalidateNativeReplacementDiscovery(db); + return source; + } + async function freshContinuations(issueId: string) { + const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + return holds.flatMap(hold => hold.evidence.freshContinuation ? [hold.evidence.freshContinuation as Record] : []); + } + it.each([["just past the deadline", 6 * 60_000], ["stuck since before this release", 2 * 24 * 60 * 60_000]] as const)( + "continues a terminal failure whose previous provider is never proven stopped (%s)", async (_label, finishedAgoMs) => { + const source = await seedUnprovenOwner(finishedAgoMs); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + const [decision] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, source.runId)); + expect(decision!.failureDetail).toMatchObject({ replacementDenied: "provider_ownership_unverified" }); + expect(String(decision!.failureDetail?.nextAction)).not.toMatch(/[a-z]+_[a-z_]+|[0-9a-f]{8}-/); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, source.runId))).toHaveLength(0); + expect(await executionControlSchedule(db).replacement()).toBeNull(); + await settleUnrecoverableExecutions(db); + await settleUnrecoverableExecutions(db); + const continued = await freshContinuations(source.issueId); + expect(continued).toHaveLength(1); + expect(continued[0]).toMatchObject({ sourceRunId: source.runId, isolation: "fresh_isolated_workspace" }); + const [task] = await db.select().from(issues).where(eq(issues.id, source.issueId)); + expect(task!.status).not.toBe("blocked"); + }); + it("keeps waiting for ownership proof before the deadline and wakes at it", async () => { + const source = await seedUnprovenOwner(60_000); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + const [decision] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, source.runId)); + expect(decision!.failureDetail?.replacementDenied).toBeUndefined(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, source.runId)); + const wakeAt = await executionControlSchedule(db).replacement(); + expect(wakeAt).not.toBeNull(); + expect(Math.abs(wakeAt! - (run!.finishedAt!.getTime() + 5 * 60_000))).toBeLessThan(2_000); + await settleUnrecoverableExecutions(db); + expect(await freshContinuations(source.issueId)).toHaveLength(0); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, source.issueId))).toHaveLength(0); + }); + async function denial(runId: string) { + const [row] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + return row!.failureDetail?.replacementDenied; + } + it("requests a pass for a deadline that passed long before anyone asked", async () => { + const source = await seedUnprovenOwner(60_000); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + await db.update(heartbeatRuns).set({ finishedAt: sql`now() - interval '15 minutes'` }) + .where(eq(heartbeatRuns.id, source.runId)); + const wakeAt = await executionControlSchedule(db).replacement(); + expect(wakeAt).not.toBeNull(); + expect(wakeAt!).toBeLessThanOrEqual(Date.now() + 20_000); + await reconcileSafeNativeReplacements(db); + expect(await denial(source.runId)).toBe("provider_ownership_unverified"); + expect(await executionControlSchedule(db).replacement()).toBeNull(); + await settleUnrecoverableExecutions(db); + expect(await freshContinuations(source.issueId)).toMatchObject([{ isolation: "fresh_isolated_workspace" }]); + }); + it("stops waiting for an environment teardown that never finishes", async () => { + const source = await seed(); + await db.update(heartbeatRuns).set({ finishedAt: sql`now() - interval '1 minute'` }).where(eq(heartbeatRuns.id, source.runId)); + await db.insert(environmentLeases).values({ companyId: source.companyId, issueId: source.issueId, + heartbeatRunId: source.runId, status: "active" }); + invalidateNativeReplacementDiscovery(db); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + expect(await denial(source.runId)).toBeUndefined(); + expect(await executionControlSchedule(db).replacement()).not.toBeNull(); + await db.update(heartbeatRuns).set({ finishedAt: sql`now() - interval '10 minutes'` }).where(eq(heartbeatRuns.id, source.runId)); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, source.runId))).toHaveLength(0); + expect(await denial(source.runId)).toBe("provider_ownership_unverified"); + await settleUnrecoverableExecutions(db); + expect(await freshContinuations(source.issueId)).toMatchObject([{ sourceRunId: source.runId, isolation: "fresh_isolated_workspace" }]); + }); + it("does not keep waking for an overdue run the pass cannot decide", async () => { + const source = await seedUnprovenOwner(60_000); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + await db.update(heartbeatRuns).set({ processPid: null, processHostHash: null, finishedAt: sql`now() - interval '15 minutes'` }) + .where(eq(heartbeatRuns.id, source.runId)); + await issueService(db).update(source.issueId, { status: "backlog" }); + expect(await executionControlSchedule(db).replacement()).not.toBeNull(); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + expect(await denial(source.runId)).toBeUndefined(); + expect(await executionControlSchedule(db).replacement()).toBeNull(); + }); + it("decides every overdue run across successive passes", async () => { + const sources: Awaited>[] = []; + for (let index = 0; index < 26; index++) sources.push(await seedUnprovenOwner(10 * 60_000)); + await reconcileSafeNativeReplacements(db); + const decided = async () => (await Promise.all(sources.map(source => denial(source.runId)))).filter(Boolean).length; + expect(await decided()).toBe(25); + expect(await executionControlSchedule(db).replacement()).not.toBeNull(); + await reconcileSafeNativeReplacements(db); + expect(await decided()).toBe(26); + expect(await executionControlSchedule(db).replacement()).toBeNull(); + }, 120_000); + it("still starts the exact replacement when ownership is proven before the deadline", async () => { + const source = await seedUnprovenOwner(60_000); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 0 }); + await db.update(heartbeatRuns).set({ processPid: null, processHostHash: null }).where(eq(heartbeatRuns.id, source.runId)); + invalidateNativeReplacementDiscovery(db); + expect(await reconcileSafeNativeReplacements(db)).toMatchObject({ scheduled: 1 }); + const [decision] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, source.runId)); + expect(decision!.failureDetail?.replacementDenied).toBeUndefined(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, source.runId))).toHaveLength(1); + expect(await executionControlSchedule(db).replacement()).toBeNull(); + }); it.each(["closed", "reassigned", "new_execution"])("closes stale recovery after %s without changing task state or granting replay", async change => { const source = await seed(3); await reconcileSafeNativeReplacements(db); diff --git a/server/src/services/native-runtime/native-safe-replacement.ts b/server/src/services/native-runtime/native-safe-replacement.ts index 91d99925903..a84c35446b6 100644 --- a/server/src/services/native-runtime/native-safe-replacement.ts +++ b/server/src/services/native-runtime/native-safe-replacement.ts @@ -31,6 +31,7 @@ import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES } from "./native-retired-backup-sou import { completedWorkspaceRecoveryCondition } from "../workspace-recovery-receipt.js"; import { nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js"; import { verifiedWorkspaceSnapshotReuse } from "../remote-workspace-snapshot-reuse-receipt.js"; +import { RECOVERY_CONTINUATION_MAX_HOLD_MS } from "../recovery-continuation-pacing.js"; export const NATIVE_SAFE_REPLACEMENT_REASON = "native_safe_replacement"; const record = (v: unknown): Record => @@ -249,6 +250,45 @@ export function nativeReplacementPending() { ); } +export const NATIVE_OWNERSHIP_PROOF_BOUND_MS = RECOVERY_CONTINUATION_MAX_HOLD_MS; +const OWNERSHIP_UNVERIFIED = "provider_ownership_unverified"; +const ownershipProofDeadline = () => sql`(coalesce(${heartbeatRuns.finishedAt}, ${nativeRunFinalizations.updatedAt}) + + ${NATIVE_OWNERSHIP_PROOF_BOUND_MS}::integer * interval '1 millisecond')`; +const ownershipProofWaiting = () => and( + eq(nativeRunFinalizations.failureCode, "native_provider_terminal_failed"), isNull(nativeRunFinalizations.leaseOwner)); + +async function ownershipProofExpired(db: ReplacementDb, run: typeof heartbeatRuns.$inferSelect) { + const [row] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)).where(and( + eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id), nativeReplacementPending(), + ownershipProofWaiting(), sql`${ownershipProofDeadline()} <= now()`, + )).limit(1); + return Boolean(row); +} + +async function markOwnershipProofWaiting(db: ReplacementDb, run: typeof heartbeatRuns.$inferSelect) { + await db.update(nativeRunFinalizations).set({ + failureDetail: sql`coalesce(${nativeRunFinalizations.failureDetail}, '{}'::jsonb) + || jsonb_build_object('ownershipUnverifiedAt', now())`, + }).where(and(eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + ownershipProofWaiting(), sql`not (coalesce(${nativeRunFinalizations.failureDetail}, '{}'::jsonb) ? 'ownershipUnverifiedAt')`)); +} + +async function releaseOwnershipProofWake(db: ReplacementDb, run: typeof heartbeatRuns.$inferSelect) { + await db.update(nativeRunFinalizations).set({ + failureDetail: sql`${nativeRunFinalizations.failureDetail} - 'ownershipUnverifiedAt'`, + }).where(and(eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + sql`${nativeRunFinalizations.failureDetail} ? 'ownershipUnverifiedAt'`)); +} + +export async function nextNativeOwnershipProofDeadlineAt(db: Db): Promise { + const [row] = await db.select({ at: sql`(extract(epoch from min(${ownershipProofDeadline()})) * 1000)::float8` }) + .from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) + .where(and(nativeReplacementPending(), ownershipProofWaiting(), + sql`${nativeRunFinalizations.failureDetail} ? 'ownershipUnverifiedAt'`)); + return row?.at == null ? null : Math.ceil(Number(row.at)); +} + const REPLACEMENT_SCAN_LIMIT = 100; const replacementDiscovery = new WeakMap(); function replacementDiscoveryState(db: Db) { @@ -383,8 +423,14 @@ export async function reconcileSafeNativeReplacements( eq(environmentLeases.heartbeatRunId, run.id), ), ); + let overdue: boolean | undefined; + const ownershipOverdue = async () => overdue ??= await ownershipProofExpired(db, run); + const leaseHeld = leases.some((lease) => lease.releasedAt === null); // Teardown is still in progress. The next sweep rechecks its durable outcome. - if (leases.some((lease) => lease.releasedAt === null)) continue; + if (leaseHeld && !await ownershipOverdue()) { + if (coordinator.failureCode === "native_provider_terminal_failed") await markOwnershipProofWaiting(db, run); + continue; + } const historicalListener = coordinator.failureCode === "native_session_interrupted"; if (historicalListener && !await historicalListenerGuardEvidence(db, run)) continue; const preClaim = coordinator.failureCode === NATIVE_PRE_CLAIM_FAILURE_CODE; @@ -483,7 +529,10 @@ export async function reconcileSafeNativeReplacements( : []; }); const history = await replacementHistoryReady(db, run, coordinator.issueId); - if (history === false) continue; + if (history === false) { + if (await ownershipOverdue()) await releaseOwnershipProofWake(db, run); + continue; + } const historyComplete = history === true; const remoteWorkspace = await remoteWorkspaceReplacementEvidence(db, run, coordinator.issueId, leases); if (historicalListener && (!remoteWorkspace?.workspacePreserved || !remoteWorkspace.providerStopped || invocations.length)) continue; @@ -508,7 +557,7 @@ export async function reconcileSafeNativeReplacements( providerStopped: !processAlive(run.processPid) && !processAlive(run.processGroupId ? -run.processGroupId : null) && - (remoteWorkspace?.providerStopped ?? true), + (remoteWorkspace?.providerStopped ?? true) && !leaseHeld, workspacePreserved: noWorkspaceToSave || (remoteWorkspace ? remoteWorkspace.workspacePreserved : typeof workspace.cwd === "string" && (await stat(workspace.cwd).then( @@ -526,7 +575,14 @@ export async function reconcileSafeNativeReplacements( }); if (!decision.allowed) { // Lack of containment can be temporary; do not prevent the next proof. - if (decision.cause === "provider_ownership_unverified") continue; + const ownershipBound = decision.cause === OWNERSHIP_UNVERIFIED; + if (ownershipBound && !await ownershipOverdue()) { + await markOwnershipProofWaiting(db, run); + continue; + } + const nextAction = ownershipBound + ? "The previous session could not be confirmed stopped, so the task continues in a fresh run with its own workspace." + : decision.nextAction; await db.transaction(async (tx) => { await tx.execute( sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`, @@ -550,7 +606,8 @@ export async function reconcileSafeNativeReplacements( !current || current.phase !== "terminal_failure" || current.failureDetail?.successorRunId || - current.failureDetail?.replacementDenied + current.failureDetail?.replacementDenied || + (ownershipBound && !await ownershipProofExpired(tx, run)) ) return; await notifyExecutionRunWork(tx, { companyId: run.companyId }); @@ -560,7 +617,7 @@ export async function reconcileSafeNativeReplacements( failureDetail: { ...current.failureDetail, replacementDenied: decision.cause, - nextAction: decision.nextAction, + nextAction, }, updatedAt: now, }) @@ -583,7 +640,7 @@ export async function reconcileSafeNativeReplacements( cause: decision.cause, fingerprint: `native-replacement:${run.id}`, evidence: { runId: run.id, attempts: coordinator.attempt }, - nextAction: decision.nextAction, + nextAction, wakePolicy: null, maxAttempts: 3, supersedeOnIdentityChange: true, @@ -818,7 +875,7 @@ export async function reconcileSafeNativeReplacements( if (created) { options.failpoint?.("lineage_committed"); scheduled += 1; - } + } else if (await ownershipOverdue()) await releaseOwnershipProofWake(db, run); } catch (error) { if (options.failpoint) throw error; pendingFailures++;