diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index 3d52d0e11c8..13c14082d53 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -461,16 +461,17 @@ leaves its question's tool input unanswered by design. Once its Runner's sandbox lease has ended elsewhere, that committed park settles the input for retirement. -Core ends a dispatched task lease with `stop`, `status`, and `complete`. When -that sequence confirms the stop and completes the same task, Core does not also -call `onEnvironmentReleaseLease`. Core still calls it when the release did not -complete a dispatched task, and the cleanup retry calls it after any failure. +Core settles a dispatched task with `stop`, `status`, and `complete`, then +calls `onEnvironmentReleaseLease` to release the allocation. If the release +hook fails after settlement, Core retries cleanup using the recorded settlement +proof. The hook can use that proof to avoid repeating task operations. A +retained lease stays available for handoff and Core calls the release hook +when it releases the lease. + When `onEnvironmentResumeLease` returns no provider lease, Core sends `status` -first. It sends `stop` only when that `status` does not report -`executionStopped: true` and any required `preparationStopped: true` proof. -An old warm-start preparation marker does not force a repeated stop when both -proofs are present. A driver must release task-scoped resources in its -`status` or `complete` handling, not only in `onEnvironmentReleaseLease`. +first. It sends `stop` only when that `status` lacks `executionStopped: true` +or required `preparationStopped: true` proof. An old warm-start preparation +marker does not force a repeated stop when both proofs are present. Termination receipts cover all run leases that owned the retired physical task. A stopped task gets a new provider task ID for checkpoint resumption. Inspection diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 806c866f2af..1fc013bef4f 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { hasPendingRunCleanup } from "../execution-blocker.js"; import { runOwnerFenceHeldSql } from "../remote-workspace-writer-hold.js"; import { AGENT_DIRECTORY_BUSY_ERROR_CODE, AGENT_DIRECTORY_BUSY_MAX_DEFERRALS, AGENT_DIRECTORY_BUSY_RETRY_REASON, AGENT_DIRECTORY_BUSY_RETRY_WAKE_REASON, agentDirectoryBusyRetryDelayMs, type AgentDirectoryPreparationDeferral } from "./agent-directory-busy.js"; import { isInterruptedNativePreparation } from "../cancelled-native-startup.js"; @@ -832,6 +833,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen delete retryContextSnapshot.skillVersions; delete retryContextSnapshot.skillSelectionSnapshot; delete retryContextSnapshot.shadowedSkills; + delete retryContextSnapshot.retryWorkspaceRefusal; const responsibleUserId = await resolveResponsibleUserIdForRunContext( run, retryContextSnapshot, @@ -859,6 +861,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen | "issue_terminal_status" | "issue_not_in_progress" | "continuation_user_authorization_missing" + | "workspace_verification_cleanup_pending" | "issue_execution_lock_changed"; issueId: string | null; details: Record; @@ -1142,6 +1145,11 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen // A preparation that a shutdown or restart stopped before provider // work continues in the uncharged interrupted_preparation lane while // its failure budget is unspent, and in server_restart after that. + const refusal = parseObject(contextSnapshot.retryWorkspaceRefusal); + if (typeof refusal.leaseId === "string" && await hasPendingRunCleanup(tx as unknown as Db, run.companyId, run.id)) { + return { outcome: "not_scheduled", issueId, errorCode: "workspace_verification_cleanup_pending", + reason: "The previous workspace verification lease still requires cleanup.", details: {} }; + } const continuation = issueId && (retryReason === "transient_failure" || retryReason === SERVER_RESTART_RETRY_REASON || retryReason === INTERRUPTED_PREPARATION_RETRY_REASON || isPreparationWaitRetryReason(retryReason) || retryReason === AGENT_DIRECTORY_BUSY_RETRY_REASON) @@ -1157,6 +1165,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }; retryContextSnapshot.explicitUserContinuation = continuation; retryContextSnapshot.previousRunId = continuation.previousRunId; + if (continuation.taskRetryRequestId && refusal.intentId === continuation.taskRetryRequestId && + typeof refusal.workspaceId === "string") { + retryContextSnapshot.retryWorkspaceRefusal = { intentId: refusal.intentId, workspaceId: refusal.workspaceId, leaseId: null }; + } } await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); diff --git a/server/src/services/task-retry.test.ts b/server/src/services/task-retry.test.ts index 84b4232cbf1..b0d77786e87 100644 --- a/server/src/services/task-retry.test.ts +++ b/server/src/services/task-retry.test.ts @@ -322,6 +322,14 @@ describe("durable task Retry admission", () => { it.each(["admission", "preparation"] as const)( "continues a fresh-workspace Retry that a shutdown stopped before provider work (selected at %s)", async selectedAt => { const f = await seedFreshAttemptStoppedByShutdown(selectedAt); + const refusal = { intentId: f.retryRequestId, workspaceId: randomUUID(), leaseId: randomUUID() }; + if (selectedAt === "preparation") { + await db.insert(environmentLeases).values({ id: refusal.leaseId, companyId: f.input.companyId, + heartbeatRunId: f.parent.id, issueId: f.input.issueId, status: "released", + releasedAt: new Date(), cleanupStatus: "success", provider: "plugin:test:tasks" }); + await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.parent.contextSnapshot, retryWorkspaceRefusal: refusal } }) + .where(eq(heartbeatRuns.id, f.parent.id)); + } const scope = { companyId: f.input.companyId }; await Promise.all([heartbeatService(db).reconcileStrandedAssignedIssues(scope), heartbeatService(db).reconcileStrandedAssignedIssues(scope)]); @@ -334,6 +342,11 @@ describe("durable task Retry admission", () => { await expect(dispatchRun(f.input, successor)).resolves.toMatchObject({ interruptedRunId: f.parent.id }); // Preparation of the successor selects the same fresh attempt. expect(await readFreshWorkspaceReadiness(db, successor)).toEqual(f.readiness); + if (selectedAt === "preparation") { + expect(successor.contextSnapshot?.retryWorkspaceRefusal).toEqual({ ...refusal, leaseId: null }); + const [parent] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.parent.id)); + expect(parent.contextSnapshot?.retryWorkspaceRefusal).toEqual(refusal); + } // A second upgrade stops the continuation too: it carries the same attempt on. const finishedAt = new Date(Date.now() - 30_000); const [stopped] = await db.update(heartbeatRuns).set({ status: "interrupted", errorCode: "server_shutdown_interrupted", @@ -347,6 +360,28 @@ describe("durable task Retry admission", () => { await expect(dispatchRun(f.input, next[0]!)).resolves.toMatchObject({ interruptedRunId: stopped.id }); expect(await readFreshWorkspaceReadiness(db, next[0]!)).toEqual(f.readiness); }); + it("leaves the parent verification lease with cleanup before scheduling a shutdown successor", async () => { + const f = await seedFreshAttemptStoppedByShutdown("preparation"); + const refusal = { intentId: f.retryRequestId, workspaceId: randomUUID(), leaseId: randomUUID() }; + await db.insert(environmentLeases).values({ id: refusal.leaseId, companyId: f.input.companyId, + heartbeatRunId: f.parent.id, issueId: f.input.issueId, status: "pending_cleanup", + releasedAt: new Date(), cleanupStatus: "failed", provider: "plugin:test:tasks" }); + const [parent] = await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.parent.contextSnapshot, retryWorkspaceRefusal: refusal } }) + .where(eq(heartbeatRuns.id, f.parent.id)).returning(); + expect(await heartbeatService(db).scheduleBoundedRetry(parent.id, { now: new Date() })) + .toMatchObject({ outcome: "not_scheduled", errorCode: "workspace_verification_cleanup_pending" }); + expect(await successorsOf(parent.id)).toHaveLength(0); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, refusal.leaseId)); + expect(lease).toMatchObject({ status: "pending_cleanup", heartbeatRunId: parent.id }); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success" }) + .where(eq(environmentLeases.id, refusal.leaseId)); + await heartbeatService(db).reconcileStrandedAssignedIssues({ companyId: f.input.companyId }); + const successors = await successorsOf(parent.id); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot?.retryWorkspaceRefusal).toEqual({ ...refusal, leaseId: null }); + await expect(dispatchRun(f.input, successors[0])).resolves.toMatchObject({ interruptedRunId: parent.id }); + expect(await readFreshWorkspaceReadiness(db, successors[0])).toEqual(f.readiness); + }); it.each(["other_retry", "other_source", "failed_not_stopped"] as const)( "keeps a fresh-workspace Retry's own contract when it does not match: %s", async kind => { const f = await seedFreshAttemptStoppedByShutdown("admission");