fix(recovery): continue verified preparation after lease interruption (#18892)

Use recorded system interruption and the existing execution, controller, stopped-lease, and saved-file proof instead of a shutdown-only error label. Preserve bounded retries, user Stops, task authority, and exactly one continuation.

Keep the chat recovery fixture faithful to the preparation stage and terminal time written by the real lifecycle. Retain missing-field safety controls.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/execution-semantics.md

@@ -1841,9 +1841,10 @@ An interrupted conversation does not permanently block its task. For local conve
 
 Shutdown, process loss, and provider failure use the existing durable failure retry counter and delay. Ordinary failure recovery permits at most two automatic retries in a failure chain. Accepted-interaction infrastructure recovery retains its existing bounded policy. Repeated scheduler visits reuse the same successor; restarting the server does not reset the counter. After exhaustion, automatic attempts stop. A new explicit user message can start a fresh run and failure budget. Productive max-turn continuation and confirmed workspace waits keep their separate existing semantics.
 
-If shutdown interrupts Paperclip Runner preparation before native execution was
-selected, recovery verifies that no provider, process launch, native result or
-coordinator exists. The previous controller must have settled or lost its lease.
+If shutdown or a recorded unexpected system interruption stops Paperclip Runner
+preparation before native execution was selected, recovery verifies that no
+provider, process launch, native result or coordinator exists. The previous
+controller must have settled or lost its lease.
 Required workspace recovery must have an exact saved-file receipt and stopped
 leases before the task can continue. Once these checks pass, the server retires
 its own unchanged bootstrap-failure block and schedules one bounded retry.
@@ -1852,7 +1853,8 @@ restart. A pending Retry or saved message owns its continuation instead of
 starting a competing automatic attempt. Retry uses the same preparation proof
 and preserves the recovered workspace. Current task ownership, human blocks,
 pauses, approvals, budgets and the existing retry limit remain effective.
-Operator Stops retain their explicit continuation requirements.
+Operator Stops retain their explicit continuation requirements. Other ambiguous
+interruptions without a server system-cancellation record remain held.
 
 Graceful shutdown interrupts an owned legacy adapter through its cancellation
 control, then waits within one shared 30-second window for provider stop and

server/src/__tests__/chat-completion-delivery.test.ts

@@ -290,6 +290,9 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.
   async function interruptedBootstrap() {
     const f = await seed(); await f.finish(); const run = await f.run();
     const interrupted = await terminalizeLegacyExecution({ db, run, status: "interrupted", patch: {
+      // The real controller claim records preparation before terminal status
+      // records completion. The delivery fixture otherwise leaves both unset.
+      executionStage: "preparing", finishedAt: new Date(),
       errorCode: "server_shutdown_interrupted", runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } },
     } });
     expect(await getExecutionBlocker(db, f.companyId, f.sourceId)).toMatchObject({ runId: run.id });
@@ -338,8 +341,10 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.
     expect(await settleInterruptedNativeBootstrap(db, { run, providerDispatchStarted: false })).not.toBeNull();
     expect(await issueService(db).getById(f.sourceId)).toMatchObject({ status: "blocked" });
   });
-  it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted"])("retains an interrupted bootstrap hold for %s", async kind => {
+  it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted", "missing-stage", "unfinished"])("retains an interrupted bootstrap hold for %s", async kind => {
     const { f, run } = await interruptedBootstrap();
+    if (kind === "missing-stage") await db.update(heartbeatRuns).set({ executionStage: null }).where(eq(heartbeatRuns.id, run.id));
+    if (kind === "unfinished") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, run.id));
     if (kind === "runtime-selected") await db.update(heartbeatRuns).set({ runtimeModeResolvedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
     if (kind === "other-adapter") await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "process" } } }).where(eq(heartbeatRuns.id, run.id));
     if (kind === "restore-unsafe") await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } }).where(eq(heartbeatRuns.id, run.id));

server/src/services/cancelled-native-startup.ts

@@ -4,7 +4,7 @@ import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinaliza
 import { claimedAdapterType } from "./conversation-continuation.js";
 import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js";
 import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
-import { canContinueCancelledRun } from "./run-cancellation.js";
+import { canContinueCancelledRun, readRunCancellation } from "./run-cancellation.js";
 import { legacyControllerBootId } from "./legacy-controller-lease.js";
 import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js";
 import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js";
@@ -37,14 +37,28 @@ export async function isFailedNativePreparation(db: Db, run: Run, coordinator: C
   return !lease && !result && !await nativeStartupHasExecutionEvidence(db, run);
 }
 
+/** Candidate state only. Admission must also verify ownership, execution records,
+ * stopped leases, and any saved-file obligation below. */
+export function isInterruptedNativePreparationCandidate(run: Run): boolean {
+  const cancellation = readRunCancellation(run.resultJson);
+  // Older shutdown records predate structured cancellation. Current records
+  // identify unexpected system interruption without relying on an error label.
+  const systemInterruption = cancellation
+    ? cancellation.initiator.type === "system" && cancellation.source !== "operator" &&
+      (cancellation.source === "shutdown" || cancellation.expected === false)
+    : run.errorCode === "server_shutdown_interrupted";
+  const recovery = run.resultJson?.executionRecovery as Record<string, unknown> | undefined;
+  return Boolean(systemInterruption && run.status === "interrupted" && run.finishedAt
+    && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && run.executionStage === "preparing"
+    && claimedAdapterType(run) === "paperclip_runner" && !run.processPid && !run.processGroupId
+    && !run.processStartedAt && !run.sessionIdAfter && !run.nativeIssueId && !run.nativeSessionId
+    && run.runnerProfileJson?.nativeExecutionInput == null && recovery?.providerWorkStarted !== true);
+}
+
 export async function isInterruptedNativePreparation(db: Db, run: Run): Promise<boolean> {
   const recovery = run.resultJson?.executionRecovery as Record<string, unknown> | undefined;
   const settled = recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false;
-  if (run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || !run.finishedAt
-    || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || run.executionStage !== "preparing"
-    || claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId
-    || run.processStartedAt || run.sessionIdAfter || run.nativeIssueId || run.nativeSessionId
-    || run.runnerProfileJson?.nativeExecutionInput != null || recovery?.providerWorkStarted === true
+  if (!isInterruptedNativePreparationCandidate(run)
     || run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()
     || !settled && (!run.controllerBootId || run.controllerBootId === legacyControllerBootId)) return false;
   const [coordinator] = await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(

server/src/services/execution-recovery-resolution.ts

@@ -7,7 +7,8 @@ import { notifyExecutionRunWork } from "./execution-control-notifications.js";
 import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
 import { randomUUID } from "node:crypto";
 import { canonicalExecutionUuid } from "./execution-identity.js";
-import { claimedAdapterType, conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";
+import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";
+import { isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js";
 import { persistActivity } from "./activity-log.js";
 import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
 import { logger } from "../middleware/logger.js";
@@ -612,8 +613,7 @@ export async function settleUnrecoverableExecutions(
           if (task.status !== "blocked" && run.runtimeMode === "native") {
             nativeFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion };
           }
-          if (task.status !== "blocked" && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt &&
-              run.errorCode === "server_shutdown_interrupted" && claimedAdapterType(run) === "paperclip_runner") {
+          if (task.status !== "blocked" && isInterruptedNativePreparationCandidate(run)) {
             nativeBootstrapFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status };
           }
         }

server/src/services/heartbeat.ts

@@ -3914,7 +3914,8 @@ export function heartbeatService(
             });
           }
         }
-        if (latestRun?.status === "interrupted" && latestRun.errorCode === "server_shutdown_interrupted") {
+        if (latestRun?.status === "interrupted" && latestRun.runtimeMode === "legacy" &&
+            !latestRun.runtimeModeResolvedAt && latestRun.executionStage === "preparing") {
           latestRun = await settleInterruptedNativeBootstrap(db, { run: latestRun,
             providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted || nativeOwnershipHeld,
           }) ?? latestRun;

server/src/services/heartbeat/retries.ts

@@ -429,10 +429,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
     }
     const now = opts?.now ?? new Date();
     const preparationRecovery = parseObject(run.resultJson?.executionRecovery);
+    const interruptedPreparation = run.status === "interrupted" &&
+      preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false;
     const retryReason = opts?.retryReason ??
-      (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted" &&
-        preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false
-        ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON);
+      (interruptedPreparation ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON);
     const wakeReason =
       opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
     const maxAttempts = Math.max(
@@ -570,6 +570,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
         : baseSchedule;
 
     const requiresIssueGate =
+      interruptedPreparation ||
       (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") ||
       isTransientWorkspacePreparationCode(run.errorCode) ||
       hasConversationContinuationPolicy(run.resultJson) ||

server/src/services/heartbeat/run-control.ts

@@ -572,7 +572,8 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
   async function resumeRemoteStopComments(run: NonNullable<Awaited<ReturnType<typeof getRunExecutionWait>>>, requestId?: string) {
     if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return;
     let interruptedPreparation = false;
-    if (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") {
+    if (run.status === "interrupted" && run.runtimeMode === "legacy" &&
+        !run.runtimeModeResolvedAt && run.executionStage === "preparing") {
       const current = await getRun(run.id, { includeExecutionEvidence: true });
       if (current && await isInterruptedNativePreparation(db, current)) {
         const settled = await settleInterruptedNativeBootstrap(db, { run: current, providerDispatchStarted: false });

server/src/services/legacy-execution-recovery.ts

@@ -20,7 +20,7 @@ import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
 import { logActivity } from "./activity-log.js";
 import { isSupersededConversationRun } from "./agent-conversations.js";
 import { issueService } from "./issues.js";
-import { isInterruptedNativePreparation } from "./cancelled-native-startup.js";
+import { isInterruptedNativePreparation, isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js";
 import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js";
 
 type Run = typeof heartbeatRuns.$inferSelect;
@@ -318,10 +318,8 @@ export async function settleInterruptedNativeBootstrap(
   return db.transaction(async tx => {
     const [task] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, input.run.companyId))).for("update");
     const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId))).for("update");
-    if (!task || !run || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt ||
+    if (!task || !run || !isInterruptedNativePreparationCandidate(run) ||
         run.contextSnapshot?.issueId !== task.id ||
-        run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" ||
-        claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId ||
         executionFailureRetryCount(run) >= 2) return null;
     if (hasWorkspaceRestoreFailure(run.resultJson) && !await hasVerifiedWorkspaceRepair(tx as unknown as Db, run)) return null;
     const [native] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)).limit(1);

server/src/services/workspace-recovery.test.ts

@@ -393,16 +393,20 @@ describe("automatic retained workspace recovery", () => {
     return { ...f, source, workspaceId, resultId, git, commit };
   }
 
-  async function interruptedRemotePreparation() {
+  async function interruptedRemotePreparation(interruption: "shutdown" | "lease" = "shutdown") {
     const f = await acceptedRemoteExport();
     await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
     await db.delete(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
     const recovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] };
     await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, completionContractId: null,
-      status: "interrupted", executionStage: "preparing", errorCode: "server_shutdown_interrupted",
+      status: "interrupted", executionStage: "preparing",
+      errorCode: interruption === "lease" ? "lease_released_before_terminal" : "server_shutdown_interrupted",
       controllerBootId: randomUUID(),
-      error: "Interrupted by graceful server shutdown (SIGTERM)",
-      resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery },
+      error: interruption === "lease" ? "Environment lease released before preparation settled" : "Interrupted by graceful server shutdown (SIGTERM)",
+      resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery,
+        ...(interruption === "lease" ? { cancellation: { source: "unknown", expected: false,
+          initiator: { type: "system" }, reason: "Environment lease released before preparation settled",
+          recordedAt: new Date().toISOString() } } : {}) },
     }).where(eq(heartbeatRuns.id, f.runId));
     await db.update(issues).set({ statusVersion: 2 }).where(eq(issues.id, f.issueId));
     await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation",
@@ -416,8 +420,8 @@ describe("automatic retained workspace recovery", () => {
     return f;
   }
 
-  it("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace", async () => {
-    const f = await interruptedRemotePreparation();
+  it.each(["shutdown", "lease"] as const)("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace (%s)", async interruption => {
+    const f = await interruptedRemotePreparation(interruption);
     await recoverRetainedWorkspace(f.input);
     expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
     const intent = { observedRunId: f.runId, idempotencyKey: randomUUID() };
@@ -441,8 +445,9 @@ describe("automatic retained workspace recovery", () => {
     finally { await saved.cleanup(); }
   }, 30_000);
 
-  it.each([false, true])("automatically retries interrupted preparation after preservation, including recovered history at startup=%s", async alreadyRecovered => {
-    const f = await interruptedRemotePreparation();
+  it.each([false, true].flatMap(alreadyRecovered => (["shutdown", "lease"] as const).map(interruption => ({ alreadyRecovered, interruption }))))(
+    "automatically retries interrupted preparation after preservation ($interruption, startup=$alreadyRecovered)", async ({ alreadyRecovered, interruption }) => {
+    const f = await interruptedRemotePreparation(interruption);
     if (alreadyRecovered) await recoverRetainedWorkspace(f.input);
     const restarted = createDb(connectionString);
     try {
@@ -462,6 +467,60 @@ describe("automatic retained workspace recovery", () => {
     } finally { await restarted.$client.end(); }
   }, 30_000);
 
+  it("retries unpublished preparation from its preserved ancestor without an old failure-block receipt", async () => {
+    const f = await seed("legacy", false);
+    await requestWorkspaceRecovery({ ...f, idempotencyKey: randomUUID(), actorId: "board" });
+    await recoverRetainedWorkspace(f.input);
+    const ancestor = await run(f.runId), leaseId = randomUUID(), providerLeaseId = randomUUID();
+    await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: ancestor.agentId, status: "running" });
+    const app = boardRecoveryApp(), heartbeat = heartbeatService(db);
+    const first = (await request(app).post(`/api/issues/${f.issueId}/retry`).send({
+      observedRunId: f.runId, idempotencyKey: randomUUID(),
+    }).expect(202)).body;
+    await heartbeat.resumeTaskRetry(f.companyId, first.requestId);
+    const admitted = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
+      eq(heartbeatRuns.status, "queued")));
+    expect(admitted).toHaveLength(1);
+    const interruptedId = admitted[0].id;
+    await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "interrupted", executionStage: "preparing",
+      startedAt: new Date(Date.now() - 60_000), finishedAt: new Date(), controllerBootId: randomUUID(),
+      errorCode: "lease_released_before_terminal", error: "Lease released before preparation settled",
+      runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } },
+      resultJson: { cancellation: { source: "unknown", expected: false, initiator: { type: "system" },
+        reason: "Lease released before preparation settled", recordedAt: new Date().toISOString() } },
+    }).where(eq(heartbeatRuns.id, interruptedId));
+    await db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId));
+    const lease = { id: leaseId, companyId: f.companyId, heartbeatRunId: interruptedId,
+      provider: "daytona", providerLeaseId };
+    await db.insert(environmentLeases).values({ ...lease, issueId: f.issueId, environmentId: f.lease.environmentId,
+      status: "released", cleanupStatus: "success", releasedAt: new Date(),
+      metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId, state: "stopped" }) },
+    });
+    await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId,
+      kind: "active_run_watchdog", ownerType: "board", returnOwnerAgentId: ancestor.agentId,
+      cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${interruptedId}`,
+      status: "resolved", outcome: "blocked", nextAction: "Recorded actions require reconciliation",
+      evidence: { runId: interruptedId, originalFailureCode: "lease_released_before_terminal",
+        automaticRecovery: { policy: "preserve_without_replay_v1", runId: interruptedId, replay: "blocked", actionOutcome: "unknown" } },
+    });
+    const intent = { observedRunId: interruptedId, idempotencyKey: randomUUID() };
+    const receipt = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body;
+    await Promise.all([heartbeat.resumeTaskRetry(f.companyId, receipt.requestId),
+      heartbeat.resumeTaskRetry(f.companyId, receipt.requestId)]);
+    const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
+      eq(heartbeatRuns.status, "queued")));
+    expect(successors).toHaveLength(1);
+    expect(successors[0].contextSnapshot).toMatchObject({ previousRunId: interruptedId });
+    expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(receipt.requestId);
+    const restored = path.join(root, randomUUID()); await mkdir(restored);
+    const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId,
+      sourceRunIds: [interruptedId], cwd: restored });
+    expect(copies).toHaveLength(1);
+    expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n");
+    expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n");
+    expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
+  }, 30_000);
+
   it.each(["local", "none"])("continues verified interrupted preparation with %s leases", async provider => {
     const f = await interruptedRemotePreparation();
     await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } },
@@ -476,9 +535,10 @@ describe("automatic retained workspace recovery", () => {
   }, 30_000);
 
   it.each(["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop",
+    "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption",
     "retry_exhausted", "human_reblock", "paused", "reassigned"] as const)(
     "keeps the interrupted preparation continuation behind %s", async gate => {
-      const f = await interruptedRemotePreparation();
+      const f = await interruptedRemotePreparation("lease");
       await recoverRetainedWorkspace(f.input);
       const source = await run(f.runId);
       if (gate === "provider_started") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId,
@@ -492,6 +552,17 @@ describe("automatic retained workspace recovery", () => {
         remoteWorkspace: { ...(source.runnerProfileJson!.remoteWorkspace as Record<string, unknown>), snapshotId: randomUUID() },
       } }).where(eq(heartbeatRuns.id, f.runId));
       if (gate === "operator_stop") await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "operator_interrupted" }).where(eq(heartbeatRuns.id, f.runId));
+      if (gate === "operator_stop_metadata") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
+        cancellation: requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator" }, "Stop requested"),
+      } }).where(eq(heartbeatRuns.id, f.runId));
+      if (gate === "unrecorded_interruption") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
+        cancellation: null,
+      } }).where(eq(heartbeatRuns.id, f.runId));
+      if (gate === "missing_stop_receipt") {
+        const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
+        await db.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteExecutionTermination: null } })
+          .where(eq(environmentLeases.id, f.lease.id));
+      }
       if (gate === "retry_exhausted") await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 2 }).where(eq(heartbeatRuns.id, f.runId));
       if (gate === "human_reblock") await db.update(issues).set({ statusVersion: 3 }).where(eq(issues.id, f.issueId));
       if (gate === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, source.agentId));
@@ -500,7 +571,8 @@ describe("automatic retained workspace recovery", () => {
       await heartbeat.resumeRemoteStopComments(await run(f.runId));
       expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
         eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(0);
-      if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop"].includes(gate)) {
+      if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop",
+        "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption"].includes(gate)) {
         const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId,
           actorId: "local-board", actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() });
         await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!);

server/src/services/workspace-recovery.ts

@@ -447,7 +447,7 @@ export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedO
     and ${agentWakeupRequests.status} = 'deferred_issue_execution' and ${agentWakeupRequests.requestedByActorType} = 'user'
     and ${agentWakeupRequests.payload}->>'issueId' = ${issueRecoveryActions.sourceIssueId}::text)`;
   const interruptedPreparation = sql`(${heartbeatRuns.runtimeMode} = 'legacy'
-    and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.errorCode} = 'server_shutdown_interrupted'
+    and ${heartbeatRuns.status} = 'interrupted'
     and ${heartbeatRuns.runtimeModeResolvedAt} is null and ${heartbeatRuns.executionStage} = 'preparing'
     and ${heartbeatRuns.runnerProfileJson}#>>'{adapterDispatch,adapterType}' = 'paperclip_runner'
     and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked'