Keep retry workspace cleanup scoped to its parent run

Preserve the fresh workspace refusal when shutdown creates a successor, without copying the parent's lease reference. Wait for pending parent cleanup before scheduling the successor. Align the plugin cleanup guide with release after task settlement.

Validated the layered server typecheck and 265 tests covering durable retries, workspace preparation, and public heartbeat preparation.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky LeachandPaperclip committed 2026-11-12 21:21:34 -08:00
1 parent 8d9ec19e7a
commit 59c7a1dae8
3 files changed
+57 -9

No files matched your search

+10 -9
View File
@@ -461,16 +461,17 @@ leaves its question's tool input unanswered by design. Once its Runner's
sandbox lease has ended elsewhere, that committed park settles the input for
retirement.
Core ends a dispatched task lease with `stop`, `status`, and `complete`. When
that sequence confirms the stop and completes the same task, Core does not also
call `onEnvironmentReleaseLease`. Core still calls it when the release did not
complete a dispatched task, and the cleanup retry calls it after any failure.
Core settles a dispatched task with `stop`, `status`, and `complete`, then
calls `onEnvironmentReleaseLease` to release the allocation. If the release
hook fails after settlement, Core retries cleanup using the recorded settlement
proof. The hook can use that proof to avoid repeating task operations. A
retained lease stays available for handoff and Core calls the release hook
when it releases the lease.
When `onEnvironmentResumeLease` returns no provider lease, Core sends `status`
first. It sends `stop` only when that `status` does not report
`executionStopped: true` and any required `preparationStopped: true` proof.
An old warm-start preparation marker does not force a repeated stop when both
proofs are present. A driver must release task-scoped resources in its
`status` or `complete` handling, not only in `onEnvironmentReleaseLease`.
first. It sends `stop` only when that `status` lacks `executionStopped: true`
or required `preparationStopped: true` proof. An old warm-start preparation
marker does not force a repeated stop when both proofs are present.
Termination receipts cover all run leases that owned the retired physical task.
A stopped task gets a new provider task ID for checkpoint resumption. Inspection
+12
View File
@@ -1,3 +1,4 @@
import { hasPendingRunCleanup } from "../execution-blocker.js";
import { runOwnerFenceHeldSql } from "../remote-workspace-writer-hold.js";
import { AGENT_DIRECTORY_BUSY_ERROR_CODE, AGENT_DIRECTORY_BUSY_MAX_DEFERRALS, AGENT_DIRECTORY_BUSY_RETRY_REASON, AGENT_DIRECTORY_BUSY_RETRY_WAKE_REASON, agentDirectoryBusyRetryDelayMs, type AgentDirectoryPreparationDeferral } from "./agent-directory-busy.js";
import { isInterruptedNativePreparation } from "../cancelled-native-startup.js";
@@ -832,6 +833,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
delete retryContextSnapshot.skillVersions;
delete retryContextSnapshot.skillSelectionSnapshot;
delete retryContextSnapshot.shadowedSkills;
delete retryContextSnapshot.retryWorkspaceRefusal;
const responsibleUserId = await resolveResponsibleUserIdForRunContext(
run,
retryContextSnapshot,
@@ -859,6 +861,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
| "issue_terminal_status"
| "issue_not_in_progress"
| "continuation_user_authorization_missing"
| "workspace_verification_cleanup_pending"
| "issue_execution_lock_changed";
issueId: string | null;
details: Record<string, unknown>;
@@ -1142,6 +1145,11 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
// A preparation that a shutdown or restart stopped before provider
// work continues in the uncharged interrupted_preparation lane while
// its failure budget is unspent, and in server_restart after that.
const refusal = parseObject(contextSnapshot.retryWorkspaceRefusal);
if (typeof refusal.leaseId === "string" && await hasPendingRunCleanup(tx as unknown as Db, run.companyId, run.id)) {
return { outcome: "not_scheduled", issueId, errorCode: "workspace_verification_cleanup_pending",
reason: "The previous workspace verification lease still requires cleanup.", details: {} };
}
const continuation = issueId && (retryReason === "transient_failure" || retryReason === SERVER_RESTART_RETRY_REASON
|| retryReason === INTERRUPTED_PREPARATION_RETRY_REASON || isPreparationWaitRetryReason(retryReason)
|| retryReason === AGENT_DIRECTORY_BUSY_RETRY_REASON)
@@ -1157,6 +1165,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
};
retryContextSnapshot.explicitUserContinuation = continuation;
retryContextSnapshot.previousRunId = continuation.previousRunId;
if (continuation.taskRetryRequestId && refusal.intentId === continuation.taskRetryRequestId &&
typeof refusal.workspaceId === "string") {
retryContextSnapshot.retryWorkspaceRefusal = { intentId: refusal.intentId, workspaceId: refusal.workspaceId, leaseId: null };
}
}
await notifyAttentionRetentionWork(tx, { companyId: run.companyId });
+35
View File
@@ -322,6 +322,14 @@ describe("durable task Retry admission", () => {
it.each(["admission", "preparation"] as const)(
"continues a fresh-workspace Retry that a shutdown stopped before provider work (selected at %s)", async selectedAt => {
const f = await seedFreshAttemptStoppedByShutdown(selectedAt);
const refusal = { intentId: f.retryRequestId, workspaceId: randomUUID(), leaseId: randomUUID() };
if (selectedAt === "preparation") {
await db.insert(environmentLeases).values({ id: refusal.leaseId, companyId: f.input.companyId,
heartbeatRunId: f.parent.id, issueId: f.input.issueId, status: "released",
releasedAt: new Date(), cleanupStatus: "success", provider: "plugin:test:tasks" });
await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.parent.contextSnapshot, retryWorkspaceRefusal: refusal } })
.where(eq(heartbeatRuns.id, f.parent.id));
}
const scope = { companyId: f.input.companyId };
await Promise.all([heartbeatService(db).reconcileStrandedAssignedIssues(scope),
heartbeatService(db).reconcileStrandedAssignedIssues(scope)]);
@@ -334,6 +342,11 @@ describe("durable task Retry admission", () => {
await expect(dispatchRun(f.input, successor)).resolves.toMatchObject({ interruptedRunId: f.parent.id });
// Preparation of the successor selects the same fresh attempt.
expect(await readFreshWorkspaceReadiness(db, successor)).toEqual(f.readiness);
if (selectedAt === "preparation") {
expect(successor.contextSnapshot?.retryWorkspaceRefusal).toEqual({ ...refusal, leaseId: null });
const [parent] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.parent.id));
expect(parent.contextSnapshot?.retryWorkspaceRefusal).toEqual(refusal);
}
// A second upgrade stops the continuation too: it carries the same attempt on.
const finishedAt = new Date(Date.now() - 30_000);
const [stopped] = await db.update(heartbeatRuns).set({ status: "interrupted", errorCode: "server_shutdown_interrupted",
@@ -347,6 +360,28 @@ describe("durable task Retry admission", () => {
await expect(dispatchRun(f.input, next[0]!)).resolves.toMatchObject({ interruptedRunId: stopped.id });
expect(await readFreshWorkspaceReadiness(db, next[0]!)).toEqual(f.readiness);
});
it("leaves the parent verification lease with cleanup before scheduling a shutdown successor", async () => {
const f = await seedFreshAttemptStoppedByShutdown("preparation");
const refusal = { intentId: f.retryRequestId, workspaceId: randomUUID(), leaseId: randomUUID() };
await db.insert(environmentLeases).values({ id: refusal.leaseId, companyId: f.input.companyId,
heartbeatRunId: f.parent.id, issueId: f.input.issueId, status: "pending_cleanup",
releasedAt: new Date(), cleanupStatus: "failed", provider: "plugin:test:tasks" });
const [parent] = await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.parent.contextSnapshot, retryWorkspaceRefusal: refusal } })
.where(eq(heartbeatRuns.id, f.parent.id)).returning();
expect(await heartbeatService(db).scheduleBoundedRetry(parent.id, { now: new Date() }))
.toMatchObject({ outcome: "not_scheduled", errorCode: "workspace_verification_cleanup_pending" });
expect(await successorsOf(parent.id)).toHaveLength(0);
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, refusal.leaseId));
expect(lease).toMatchObject({ status: "pending_cleanup", heartbeatRunId: parent.id });
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success" })
.where(eq(environmentLeases.id, refusal.leaseId));
await heartbeatService(db).reconcileStrandedAssignedIssues({ companyId: f.input.companyId });
const successors = await successorsOf(parent.id);
expect(successors).toHaveLength(1);
expect(successors[0].contextSnapshot?.retryWorkspaceRefusal).toEqual({ ...refusal, leaseId: null });
await expect(dispatchRun(f.input, successors[0])).resolves.toMatchObject({ interruptedRunId: parent.id });
expect(await readFreshWorkspaceReadiness(db, successors[0])).toEqual(f.readiness);
});
it.each(["other_retry", "other_source", "failed_not_stopped"] as const)(
"keeps a fresh-workspace Retry's own contract when it does not match: %s", async kind => {
const f = await seedFreshAttemptStoppedByShutdown("admission");