From 59c7a1dae84d7640cb14a27fd79951969dbf0914 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Thu, 12 Nov 2026 21:21:34 -0800 Subject: [PATCH] Keep retry workspace cleanup scoped to its parent run Preserve the fresh workspace refusal when shutdown creates a successor, without copying the parent's lease reference. Wait for pending parent cleanup before scheduling the successor. Align the plugin cleanup guide with release after task settlement. Validated the layered server typecheck and 265 tests covering durable retries, workspace preparation, and public heartbeat preparation. Co-Authored-By: Paperclip --- doc/plugins/PLUGIN_RUNNER_EXECUTION.md | 19 +++++++------ server/src/services/heartbeat/retries.ts | 12 ++++++++ server/src/services/task-retry.test.ts | 35 ++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index 3d52d0e11c8..13c14082d53 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -461,16 +461,17 @@ leaves its question's tool input unanswered by design. Once its Runner's sandbox lease has ended elsewhere, that committed park settles the input for retirement. -Core ends a dispatched task lease with `stop`, `status`, and `complete`. When -that sequence confirms the stop and completes the same task, Core does not also -call `onEnvironmentReleaseLease`. Core still calls it when the release did not -complete a dispatched task, and the cleanup retry calls it after any failure. +Core settles a dispatched task with `stop`, `status`, and `complete`, then +calls `onEnvironmentReleaseLease` to release the allocation. If the release +hook fails after settlement, Core retries cleanup using the recorded settlement +proof. The hook can use that proof to avoid repeating task operations. A +retained lease stays available for handoff and Core calls the release hook +when it releases the lease. + When `onEnvironmentResumeLease` returns no provider lease, Core sends `status` -first. It sends `stop` only when that `status` does not report -`executionStopped: true` and any required `preparationStopped: true` proof. -An old warm-start preparation marker does not force a repeated stop when both -proofs are present. A driver must release task-scoped resources in its -`status` or `complete` handling, not only in `onEnvironmentReleaseLease`. +first. It sends `stop` only when that `status` lacks `executionStopped: true` +or required `preparationStopped: true` proof. An old warm-start preparation +marker does not force a repeated stop when both proofs are present. Termination receipts cover all run leases that owned the retired physical task. A stopped task gets a new provider task ID for checkpoint resumption. Inspection diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 806c866f2af..1fc013bef4f 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { hasPendingRunCleanup } from "../execution-blocker.js"; import { runOwnerFenceHeldSql } from "../remote-workspace-writer-hold.js"; import { AGENT_DIRECTORY_BUSY_ERROR_CODE, AGENT_DIRECTORY_BUSY_MAX_DEFERRALS, AGENT_DIRECTORY_BUSY_RETRY_REASON, AGENT_DIRECTORY_BUSY_RETRY_WAKE_REASON, agentDirectoryBusyRetryDelayMs, type AgentDirectoryPreparationDeferral } from "./agent-directory-busy.js"; import { isInterruptedNativePreparation } from "../cancelled-native-startup.js"; @@ -832,6 +833,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen delete retryContextSnapshot.skillVersions; delete retryContextSnapshot.skillSelectionSnapshot; delete retryContextSnapshot.shadowedSkills; + delete retryContextSnapshot.retryWorkspaceRefusal; const responsibleUserId = await resolveResponsibleUserIdForRunContext( run, retryContextSnapshot, @@ -859,6 +861,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen | "issue_terminal_status" | "issue_not_in_progress" | "continuation_user_authorization_missing" + | "workspace_verification_cleanup_pending" | "issue_execution_lock_changed"; issueId: string | null; details: Record; @@ -1142,6 +1145,11 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen // A preparation that a shutdown or restart stopped before provider // work continues in the uncharged interrupted_preparation lane while // its failure budget is unspent, and in server_restart after that. + const refusal = parseObject(contextSnapshot.retryWorkspaceRefusal); + if (typeof refusal.leaseId === "string" && await hasPendingRunCleanup(tx as unknown as Db, run.companyId, run.id)) { + return { outcome: "not_scheduled", issueId, errorCode: "workspace_verification_cleanup_pending", + reason: "The previous workspace verification lease still requires cleanup.", details: {} }; + } const continuation = issueId && (retryReason === "transient_failure" || retryReason === SERVER_RESTART_RETRY_REASON || retryReason === INTERRUPTED_PREPARATION_RETRY_REASON || isPreparationWaitRetryReason(retryReason) || retryReason === AGENT_DIRECTORY_BUSY_RETRY_REASON) @@ -1157,6 +1165,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }; retryContextSnapshot.explicitUserContinuation = continuation; retryContextSnapshot.previousRunId = continuation.previousRunId; + if (continuation.taskRetryRequestId && refusal.intentId === continuation.taskRetryRequestId && + typeof refusal.workspaceId === "string") { + retryContextSnapshot.retryWorkspaceRefusal = { intentId: refusal.intentId, workspaceId: refusal.workspaceId, leaseId: null }; + } } await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); diff --git a/server/src/services/task-retry.test.ts b/server/src/services/task-retry.test.ts index 84b4232cbf1..b0d77786e87 100644 --- a/server/src/services/task-retry.test.ts +++ b/server/src/services/task-retry.test.ts @@ -322,6 +322,14 @@ describe("durable task Retry admission", () => { it.each(["admission", "preparation"] as const)( "continues a fresh-workspace Retry that a shutdown stopped before provider work (selected at %s)", async selectedAt => { const f = await seedFreshAttemptStoppedByShutdown(selectedAt); + const refusal = { intentId: f.retryRequestId, workspaceId: randomUUID(), leaseId: randomUUID() }; + if (selectedAt === "preparation") { + await db.insert(environmentLeases).values({ id: refusal.leaseId, companyId: f.input.companyId, + heartbeatRunId: f.parent.id, issueId: f.input.issueId, status: "released", + releasedAt: new Date(), cleanupStatus: "success", provider: "plugin:test:tasks" }); + await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.parent.contextSnapshot, retryWorkspaceRefusal: refusal } }) + .where(eq(heartbeatRuns.id, f.parent.id)); + } const scope = { companyId: f.input.companyId }; await Promise.all([heartbeatService(db).reconcileStrandedAssignedIssues(scope), heartbeatService(db).reconcileStrandedAssignedIssues(scope)]); @@ -334,6 +342,11 @@ describe("durable task Retry admission", () => { await expect(dispatchRun(f.input, successor)).resolves.toMatchObject({ interruptedRunId: f.parent.id }); // Preparation of the successor selects the same fresh attempt. expect(await readFreshWorkspaceReadiness(db, successor)).toEqual(f.readiness); + if (selectedAt === "preparation") { + expect(successor.contextSnapshot?.retryWorkspaceRefusal).toEqual({ ...refusal, leaseId: null }); + const [parent] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.parent.id)); + expect(parent.contextSnapshot?.retryWorkspaceRefusal).toEqual(refusal); + } // A second upgrade stops the continuation too: it carries the same attempt on. const finishedAt = new Date(Date.now() - 30_000); const [stopped] = await db.update(heartbeatRuns).set({ status: "interrupted", errorCode: "server_shutdown_interrupted", @@ -347,6 +360,28 @@ describe("durable task Retry admission", () => { await expect(dispatchRun(f.input, next[0]!)).resolves.toMatchObject({ interruptedRunId: stopped.id }); expect(await readFreshWorkspaceReadiness(db, next[0]!)).toEqual(f.readiness); }); + it("leaves the parent verification lease with cleanup before scheduling a shutdown successor", async () => { + const f = await seedFreshAttemptStoppedByShutdown("preparation"); + const refusal = { intentId: f.retryRequestId, workspaceId: randomUUID(), leaseId: randomUUID() }; + await db.insert(environmentLeases).values({ id: refusal.leaseId, companyId: f.input.companyId, + heartbeatRunId: f.parent.id, issueId: f.input.issueId, status: "pending_cleanup", + releasedAt: new Date(), cleanupStatus: "failed", provider: "plugin:test:tasks" }); + const [parent] = await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.parent.contextSnapshot, retryWorkspaceRefusal: refusal } }) + .where(eq(heartbeatRuns.id, f.parent.id)).returning(); + expect(await heartbeatService(db).scheduleBoundedRetry(parent.id, { now: new Date() })) + .toMatchObject({ outcome: "not_scheduled", errorCode: "workspace_verification_cleanup_pending" }); + expect(await successorsOf(parent.id)).toHaveLength(0); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, refusal.leaseId)); + expect(lease).toMatchObject({ status: "pending_cleanup", heartbeatRunId: parent.id }); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success" }) + .where(eq(environmentLeases.id, refusal.leaseId)); + await heartbeatService(db).reconcileStrandedAssignedIssues({ companyId: f.input.companyId }); + const successors = await successorsOf(parent.id); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot?.retryWorkspaceRefusal).toEqual({ ...refusal, leaseId: null }); + await expect(dispatchRun(f.input, successors[0])).resolves.toMatchObject({ interruptedRunId: parent.id }); + expect(await readFreshWorkspaceReadiness(db, successors[0])).toEqual(f.readiness); + }); it.each(["other_retry", "other_source", "failed_not_stopped"] as const)( "keeps a fresh-workspace Retry's own contract when it does not match: %s", async kind => { const f = await seedFreshAttemptStoppedByShutdown("admission");