diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 9a07565666d..65be57ed92d 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2022,6 +2022,16 @@ not grant retry authority or change the budget. When the budget is exhausted, the task is blocked and its recovery action states that provider startup failed before any provider work began. +A failed bootstrap retry cannot take back an issue lock that moved off its run. +When a newer run exists for the task, restart recovery settles the old retry at +once. When the lock was released and no newer run exists, recovery uses the +bounded admission backoff and then settles the retry. In both cases the +coordinator reaches `terminal_failure`, the board receives a reconciliation +action, and the orphaned-lease sweep can preserve the workspace and release the +sandbox lease. Before this rule, the retry re-checked the lock without a +deadline and never settled, so its sandbox lease stayed active and every later +run for the task was cancelled as `workspace_busy`. + The same limit evidence also holds new work for the agent. Before any workspace or provider work, a queued run for an agent with an active provider limit is cancelled as `provider_limit_wait`, records the expected reset and the run that diff --git a/server/src/services/native-runtime/native-remote-retry-admission.test.ts b/server/src/services/native-runtime/native-remote-retry-admission.test.ts index 69227e908e5..c717dcd5ece 100644 --- a/server/src/services/native-runtime/native-remote-retry-admission.test.ts +++ b/server/src/services/native-runtime/native-remote-retry-admission.test.ts @@ -567,6 +567,50 @@ describeEmbeddedPostgres("remote native retry execution admission", () => { } }); + it.each(["superseded", "released"] as const)("settles a %s bootstrap retry that lost its task lock instead of re-checking forever", async lock => { + const f = await failedRemoteRun("remote"); + // A bootstrap retry follows a pre-provider failure: no checkpoint, provider session or provider events. + const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const { sessionCheckpoint: _checkpoint, ...profile } = run.runnerProfileJson ?? {}; + await f.db.update(heartbeatRuns).set({ errorCode: "provider_session_open_rejected", runnerProfileJson: profile }) + .where(eq(heartbeatRuns.id, f.runId)); + await f.db.delete(nativeRunnerCheckpoints) + .where(and(eq(nativeRunnerCheckpoints.companyId, f.companyId), eq(nativeRunnerCheckpoints.runId, f.runId))); + await f.db.update(nativeRunFinalizations).set({ failureCode: "provider_session_open_rejected", failureDetail: { + recoveryMode: "bootstrap_retry", providerSessionEstablished: false, checkpointExists: false, providerEventsExist: false, + } }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (lock === "superseded") { + const [newer] = await f.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, + status: "running", createdAt: new Date(Date.now() + 1_000) }).returning({ id: heartbeatRuns.id }); + await f.db.update(issues).set({ executionRunId: newer.id }).where(eq(issues.id, f.issueId)); + } + const onClaim = vi.fn(); + const attempt = (now: Date) => claimNativeRestartRecoveries({ db: f.db, controller: environment.current.successor, + restartKind: "hard", runIds: [f.runId], now, onClaim }); + let now = new Date(); + let passes = 0; + for (; passes <= SYSTEM_RECOVERY_MAX_RETRIES + 1; passes++) { + const [disposition] = await attempt(now); + expect(disposition).toMatchObject({ kind: "blocked" }); + const [coordinator] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + if (coordinator.phase === "terminal_failure") break; + if (!("retryAt" in disposition) || !disposition.retryAt) throw new Error("a deferred bootstrap retry must persist its deadline"); + expect(coordinator.nextAttemptAt).toEqual(disposition.retryAt); + now = disposition.retryAt; + } + expect(passes).toBe(lock === "superseded" ? 0 : SYSTEM_RECOVERY_MAX_RETRIES); + expect(onClaim).not.toHaveBeenCalled(); + expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", nextAttemptAt: null, attempt: 1, failureCode: "native_restart_recovery_blocked" }); + expect(await f.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))) + .toMatchObject([{ ownerType: "board", wakePolicy: null }]); + const transaction = vi.spyOn(f.db, "transaction"); + try { + for (const elapsed of [30_000, 3_600_000]) expect(await attempt(new Date(now.getTime() + elapsed))).toEqual([]); + expect(transaction).not.toHaveBeenCalled(); + } finally { transaction.mockRestore(); } + }); + it.each(["paused", "Stop", "missing-authority"] as const)("bounds %s holds and preserves the provider and own task lock for manual reconciliation", async gate => { const f = await failedRemoteRun("remote"); if (gate === "paused") await f.db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId)); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 4eaf1ef110a..efb32887539 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -658,6 +658,11 @@ export async function claimNativeRestartRecoveries(input: { )).orderBy(desc(environmentLeases.createdAt)).limit(1); const failedRemoteRetry = row.run.status === "failed" && row.coordinator.phase === "retryable_failure" && row.coordinator.failureDetail?.recoveryMode === "exact_checkpoint_resume" && (remoteWorkspace !== null || pluginTask !== undefined); + // A same-run bootstrap retry cannot take back an issue lock that moved + // off it. Without a bounded admission it would re-check forever, never + // settle, and keep its sandbox lease from the orphan-lease sweep. + const lockLostBootstrapRetry = row.run.status === "failed" && row.coordinator.phase === "retryable_failure" + && row.coordinator.failureDetail?.recoveryMode === "bootstrap_retry" && row.issueExecutionRunId !== row.run.id; const releasedRemoteRetry = failedRemoteRetry && row.issueExecutionRunId === null; const exhaustedRemoteRetry = failedRemoteRetry && row.coordinator.attempt >= 3; const needsResumptionAdmission = releasedRemoteRetry || exhaustedRemoteRetry || @@ -706,7 +711,7 @@ export async function claimNativeRestartRecoveries(input: { if (!updated.length) throw new Error("native_retry_deferral_claim_changed"); return { kind: "blocked", runId: row.run.id, reason, retryAt, transitionUnchanged } as const; }; - if (failedRemoteRetry) { + if (failedRemoteRetry || lockLostBootstrapRetry) { const source = alias(heartbeatRuns, "native_retry_source"); const [newer] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) .innerJoin(source, and(eq(source.companyId, row.run.companyId), eq(source.id, row.run.id))) @@ -730,6 +735,10 @@ export async function claimNativeRestartRecoveries(input: { const deferred = await deferRemoteRetry("native_retry_admission_ineligible", ["done", "cancelled"].includes(issue.status)); if (deferred) return deferred; } + if (lockLostBootstrapRetry && !admission.exhausted) { + const deferred = await deferRemoteRetry("issue_execution_lock_changed"); + if (deferred) return deferred; + } if (row.issueExecutionRunId !== row.run.id && !(stopCleanup && row.issueExecutionRunId === null) && !releasedRemoteRetry && !admission.exhausted) { return { kind: "blocked",