From 7b1c53f8f1cd519c2cc4fb09fe9c343acd67263d Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:54:16 -0600 Subject: [PATCH] fix(recovery): continue verified preparation after lease interruption (#18892) Use recorded system interruption and the existing execution, controller, stopped-lease, and saved-file proof instead of a shutdown-only error label. Preserve bounded retries, user Stops, task authority, and exactly one continuation. Keep the chat recovery fixture faithful to the preparation stage and terminal time written by the real lifecycle. Retain missing-field safety controls. Co-Authored-By: Paperclip --- doc/execution-semantics.md | 10 +- .../chat-completion-delivery.test.ts | 7 +- .../src/services/cancelled-native-startup.ts | 26 ++++-- .../services/execution-recovery-resolution.ts | 6 +- server/src/services/heartbeat.ts | 3 +- server/src/services/heartbeat/retries.ts | 7 +- server/src/services/heartbeat/run-control.ts | 3 +- .../src/services/legacy-execution-recovery.ts | 6 +- .../src/services/workspace-recovery.test.ts | 92 +++++++++++++++++-- server/src/services/workspace-recovery.ts | 2 +- 10 files changed, 128 insertions(+), 34 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index f18aa7128a7..336687a07a6 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1841,9 +1841,10 @@ An interrupted conversation does not permanently block its task. For local conve Shutdown, process loss, and provider failure use the existing durable failure retry counter and delay. Ordinary failure recovery permits at most two automatic retries in a failure chain. Accepted-interaction infrastructure recovery retains its existing bounded policy. Repeated scheduler visits reuse the same successor; restarting the server does not reset the counter. After exhaustion, automatic attempts stop. A new explicit user message can start a fresh run and failure budget. Productive max-turn continuation and confirmed workspace waits keep their separate existing semantics. -If shutdown interrupts Paperclip Runner preparation before native execution was -selected, recovery verifies that no provider, process launch, native result or -coordinator exists. The previous controller must have settled or lost its lease. +If shutdown or a recorded unexpected system interruption stops Paperclip Runner +preparation before native execution was selected, recovery verifies that no +provider, process launch, native result or coordinator exists. The previous +controller must have settled or lost its lease. Required workspace recovery must have an exact saved-file receipt and stopped leases before the task can continue. Once these checks pass, the server retires its own unchanged bootstrap-failure block and schedules one bounded retry. @@ -1852,7 +1853,8 @@ restart. A pending Retry or saved message owns its continuation instead of starting a competing automatic attempt. Retry uses the same preparation proof and preserves the recovered workspace. Current task ownership, human blocks, pauses, approvals, budgets and the existing retry limit remain effective. -Operator Stops retain their explicit continuation requirements. +Operator Stops retain their explicit continuation requirements. Other ambiguous +interruptions without a server system-cancellation record remain held. Graceful shutdown interrupts an owned legacy adapter through its cancellation control, then waits within one shared 30-second window for provider stop and diff --git a/server/src/__tests__/chat-completion-delivery.test.ts b/server/src/__tests__/chat-completion-delivery.test.ts index 2dfd4290972..df769c99372 100644 --- a/server/src/__tests__/chat-completion-delivery.test.ts +++ b/server/src/__tests__/chat-completion-delivery.test.ts @@ -290,6 +290,9 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue. async function interruptedBootstrap() { const f = await seed(); await f.finish(); const run = await f.run(); const interrupted = await terminalizeLegacyExecution({ db, run, status: "interrupted", patch: { + // The real controller claim records preparation before terminal status + // records completion. The delivery fixture otherwise leaves both unset. + executionStage: "preparing", finishedAt: new Date(), errorCode: "server_shutdown_interrupted", runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, } }); expect(await getExecutionBlocker(db, f.companyId, f.sourceId)).toMatchObject({ runId: run.id }); @@ -338,8 +341,10 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue. expect(await settleInterruptedNativeBootstrap(db, { run, providerDispatchStarted: false })).not.toBeNull(); expect(await issueService(db).getById(f.sourceId)).toMatchObject({ status: "blocked" }); }); - it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted"])("retains an interrupted bootstrap hold for %s", async kind => { + it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted", "missing-stage", "unfinished"])("retains an interrupted bootstrap hold for %s", async kind => { const { f, run } = await interruptedBootstrap(); + if (kind === "missing-stage") await db.update(heartbeatRuns).set({ executionStage: null }).where(eq(heartbeatRuns.id, run.id)); + if (kind === "unfinished") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, run.id)); if (kind === "runtime-selected") await db.update(heartbeatRuns).set({ runtimeModeResolvedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); if (kind === "other-adapter") await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "process" } } }).where(eq(heartbeatRuns.id, run.id)); if (kind === "restore-unsafe") await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } }).where(eq(heartbeatRuns.id, run.id)); diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index e742a2133e6..88bb022617a 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -4,7 +4,7 @@ import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinaliza import { claimedAdapterType } from "./conversation-continuation.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; -import { canContinueCancelledRun } from "./run-cancellation.js"; +import { canContinueCancelledRun, readRunCancellation } from "./run-cancellation.js"; import { legacyControllerBootId } from "./legacy-controller-lease.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; @@ -37,14 +37,28 @@ export async function isFailedNativePreparation(db: Db, run: Run, coordinator: C return !lease && !result && !await nativeStartupHasExecutionEvidence(db, run); } +/** Candidate state only. Admission must also verify ownership, execution records, + * stopped leases, and any saved-file obligation below. */ +export function isInterruptedNativePreparationCandidate(run: Run): boolean { + const cancellation = readRunCancellation(run.resultJson); + // Older shutdown records predate structured cancellation. Current records + // identify unexpected system interruption without relying on an error label. + const systemInterruption = cancellation + ? cancellation.initiator.type === "system" && cancellation.source !== "operator" && + (cancellation.source === "shutdown" || cancellation.expected === false) + : run.errorCode === "server_shutdown_interrupted"; + const recovery = run.resultJson?.executionRecovery as Record | undefined; + return Boolean(systemInterruption && run.status === "interrupted" && run.finishedAt + && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && run.executionStage === "preparing" + && claimedAdapterType(run) === "paperclip_runner" && !run.processPid && !run.processGroupId + && !run.processStartedAt && !run.sessionIdAfter && !run.nativeIssueId && !run.nativeSessionId + && run.runnerProfileJson?.nativeExecutionInput == null && recovery?.providerWorkStarted !== true); +} + export async function isInterruptedNativePreparation(db: Db, run: Run): Promise { const recovery = run.resultJson?.executionRecovery as Record | undefined; const settled = recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; - if (run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || !run.finishedAt - || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || run.executionStage !== "preparing" - || claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId - || run.processStartedAt || run.sessionIdAfter || run.nativeIssueId || run.nativeSessionId - || run.runnerProfileJson?.nativeExecutionInput != null || recovery?.providerWorkStarted === true + if (!isInterruptedNativePreparationCandidate(run) || run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date() || !settled && (!run.controllerBootId || run.controllerBootId === legacyControllerBootId)) return false; const [coordinator] = await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and( diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index a6f80f26ba0..5d9d9e3b815 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -7,7 +7,8 @@ import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { randomUUID } from "node:crypto"; import { canonicalExecutionUuid } from "./execution-identity.js"; -import { claimedAdapterType, conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; +import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; +import { isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { persistActivity } from "./activity-log.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { logger } from "../middleware/logger.js"; @@ -612,8 +613,7 @@ export async function settleUnrecoverableExecutions( if (task.status !== "blocked" && run.runtimeMode === "native") { nativeFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion }; } - if (task.status !== "blocked" && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && - run.errorCode === "server_shutdown_interrupted" && claimedAdapterType(run) === "paperclip_runner") { + if (task.status !== "blocked" && isInterruptedNativePreparationCandidate(run)) { nativeBootstrapFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status }; } } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8215d222fc8..8bde92adcda 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3914,7 +3914,8 @@ export function heartbeatService( }); } } - if (latestRun?.status === "interrupted" && latestRun.errorCode === "server_shutdown_interrupted") { + if (latestRun?.status === "interrupted" && latestRun.runtimeMode === "legacy" && + !latestRun.runtimeModeResolvedAt && latestRun.executionStage === "preparing") { latestRun = await settleInterruptedNativeBootstrap(db, { run: latestRun, providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted || nativeOwnershipHeld, }) ?? latestRun; diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 2ea6de7c323..07bf4659789 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -429,10 +429,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } const now = opts?.now ?? new Date(); const preparationRecovery = parseObject(run.resultJson?.executionRecovery); + const interruptedPreparation = run.status === "interrupted" && + preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false; const retryReason = opts?.retryReason ?? - (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted" && - preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false - ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON); + (interruptedPreparation ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON); const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; const maxAttempts = Math.max( @@ -570,6 +570,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen : baseSchedule; const requiresIssueGate = + interruptedPreparation || (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") || isTransientWorkspacePreparationCode(run.errorCode) || hasConversationContinuationPolicy(run.resultJson) || diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index ead808b6623..461fcffb616 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -572,7 +572,8 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont async function resumeRemoteStopComments(run: NonNullable>>, requestId?: string) { if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; let interruptedPreparation = false; - if (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") { + if (run.status === "interrupted" && run.runtimeMode === "legacy" && + !run.runtimeModeResolvedAt && run.executionStage === "preparing") { const current = await getRun(run.id, { includeExecutionEvidence: true }); if (current && await isInterruptedNativePreparation(db, current)) { const settled = await settleInterruptedNativeBootstrap(db, { run: current, providerDispatchStarted: false }); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 68138666771..44bdebd1004 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -20,7 +20,7 @@ import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { logActivity } from "./activity-log.js"; import { isSupersededConversationRun } from "./agent-conversations.js"; import { issueService } from "./issues.js"; -import { isInterruptedNativePreparation } from "./cancelled-native-startup.js"; +import { isInterruptedNativePreparation, isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -318,10 +318,8 @@ export async function settleInterruptedNativeBootstrap( return db.transaction(async tx => { const [task] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, input.run.companyId))).for("update"); const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId))).for("update"); - if (!task || !run || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || + if (!task || !run || !isInterruptedNativePreparationCandidate(run) || run.contextSnapshot?.issueId !== task.id || - run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || - claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId || executionFailureRetryCount(run) >= 2) return null; if (hasWorkspaceRestoreFailure(run.resultJson) && !await hasVerifiedWorkspaceRepair(tx as unknown as Db, run)) return null; const [native] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)).limit(1); diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 7fa4babe6b5..b2d4f1c9155 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -393,16 +393,20 @@ describe("automatic retained workspace recovery", () => { return { ...f, source, workspaceId, resultId, git, commit }; } - async function interruptedRemotePreparation() { + async function interruptedRemotePreparation(interruption: "shutdown" | "lease" = "shutdown") { const f = await acceptedRemoteExport(); await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); await db.delete(nativeRunResults).where(eq(nativeRunResults.runId, f.runId)); const recovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] }; await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, completionContractId: null, - status: "interrupted", executionStage: "preparing", errorCode: "server_shutdown_interrupted", + status: "interrupted", executionStage: "preparing", + errorCode: interruption === "lease" ? "lease_released_before_terminal" : "server_shutdown_interrupted", controllerBootId: randomUUID(), - error: "Interrupted by graceful server shutdown (SIGTERM)", - resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery }, + error: interruption === "lease" ? "Environment lease released before preparation settled" : "Interrupted by graceful server shutdown (SIGTERM)", + resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery, + ...(interruption === "lease" ? { cancellation: { source: "unknown", expected: false, + initiator: { type: "system" }, reason: "Environment lease released before preparation settled", + recordedAt: new Date().toISOString() } } : {}) }, }).where(eq(heartbeatRuns.id, f.runId)); await db.update(issues).set({ statusVersion: 2 }).where(eq(issues.id, f.issueId)); await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation", @@ -416,8 +420,8 @@ describe("automatic retained workspace recovery", () => { return f; } - it("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace", async () => { - const f = await interruptedRemotePreparation(); + it.each(["shutdown", "lease"] as const)("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace (%s)", async interruption => { + const f = await interruptedRemotePreparation(interruption); await recoverRetainedWorkspace(f.input); expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true); const intent = { observedRunId: f.runId, idempotencyKey: randomUUID() }; @@ -441,8 +445,9 @@ describe("automatic retained workspace recovery", () => { finally { await saved.cleanup(); } }, 30_000); - it.each([false, true])("automatically retries interrupted preparation after preservation, including recovered history at startup=%s", async alreadyRecovered => { - const f = await interruptedRemotePreparation(); + it.each([false, true].flatMap(alreadyRecovered => (["shutdown", "lease"] as const).map(interruption => ({ alreadyRecovered, interruption }))))( + "automatically retries interrupted preparation after preservation ($interruption, startup=$alreadyRecovered)", async ({ alreadyRecovered, interruption }) => { + const f = await interruptedRemotePreparation(interruption); if (alreadyRecovered) await recoverRetainedWorkspace(f.input); const restarted = createDb(connectionString); try { @@ -462,6 +467,60 @@ describe("automatic retained workspace recovery", () => { } finally { await restarted.$client.end(); } }, 30_000); + it("retries unpublished preparation from its preserved ancestor without an old failure-block receipt", async () => { + const f = await seed("legacy", false); + await requestWorkspaceRecovery({ ...f, idempotencyKey: randomUUID(), actorId: "board" }); + await recoverRetainedWorkspace(f.input); + const ancestor = await run(f.runId), leaseId = randomUUID(), providerLeaseId = randomUUID(); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: ancestor.agentId, status: "running" }); + const app = boardRecoveryApp(), heartbeat = heartbeatService(db); + const first = (await request(app).post(`/api/issues/${f.issueId}/retry`).send({ + observedRunId: f.runId, idempotencyKey: randomUUID(), + }).expect(202)).body; + await heartbeat.resumeTaskRetry(f.companyId, first.requestId); + const admitted = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.status, "queued"))); + expect(admitted).toHaveLength(1); + const interruptedId = admitted[0].id; + await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "interrupted", executionStage: "preparing", + startedAt: new Date(Date.now() - 60_000), finishedAt: new Date(), controllerBootId: randomUUID(), + errorCode: "lease_released_before_terminal", error: "Lease released before preparation settled", + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: { cancellation: { source: "unknown", expected: false, initiator: { type: "system" }, + reason: "Lease released before preparation settled", recordedAt: new Date().toISOString() } }, + }).where(eq(heartbeatRuns.id, interruptedId)); + await db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId)); + const lease = { id: leaseId, companyId: f.companyId, heartbeatRunId: interruptedId, + provider: "daytona", providerLeaseId }; + await db.insert(environmentLeases).values({ ...lease, issueId: f.issueId, environmentId: f.lease.environmentId, + status: "released", cleanupStatus: "success", releasedAt: new Date(), + metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId, state: "stopped" }) }, + }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", ownerType: "board", returnOwnerAgentId: ancestor.agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${interruptedId}`, + status: "resolved", outcome: "blocked", nextAction: "Recorded actions require reconciliation", + evidence: { runId: interruptedId, originalFailureCode: "lease_released_before_terminal", + automaticRecovery: { policy: "preserve_without_replay_v1", runId: interruptedId, replay: "blocked", actionOutcome: "unknown" } }, + }); + const intent = { observedRunId: interruptedId, idempotencyKey: randomUUID() }; + const receipt = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body; + await Promise.all([heartbeat.resumeTaskRetry(f.companyId, receipt.requestId), + heartbeat.resumeTaskRetry(f.companyId, receipt.requestId)]); + const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.status, "queued"))); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot).toMatchObject({ previousRunId: interruptedId }); + expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(receipt.requestId); + const restored = path.join(root, randomUUID()); await mkdir(restored); + const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId, + sourceRunIds: [interruptedId], cwd: restored }); + expect(copies).toHaveLength(1); + expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + }, 30_000); + it.each(["local", "none"])("continues verified interrupted preparation with %s leases", async provider => { const f = await interruptedRemotePreparation(); await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, @@ -476,9 +535,10 @@ describe("automatic retained workspace recovery", () => { }, 30_000); it.each(["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop", + "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption", "retry_exhausted", "human_reblock", "paused", "reassigned"] as const)( "keeps the interrupted preparation continuation behind %s", async gate => { - const f = await interruptedRemotePreparation(); + const f = await interruptedRemotePreparation("lease"); await recoverRetainedWorkspace(f.input); const source = await run(f.runId); if (gate === "provider_started") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, @@ -492,6 +552,17 @@ describe("automatic retained workspace recovery", () => { remoteWorkspace: { ...(source.runnerProfileJson!.remoteWorkspace as Record), snapshotId: randomUUID() }, } }).where(eq(heartbeatRuns.id, f.runId)); if (gate === "operator_stop") await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "operator_interrupted" }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "operator_stop_metadata") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + cancellation: requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator" }, "Stop requested"), + } }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "unrecorded_interruption") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + cancellation: null, + } }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "missing_stop_receipt") { + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteExecutionTermination: null } }) + .where(eq(environmentLeases.id, f.lease.id)); + } if (gate === "retry_exhausted") await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 2 }).where(eq(heartbeatRuns.id, f.runId)); if (gate === "human_reblock") await db.update(issues).set({ statusVersion: 3 }).where(eq(issues.id, f.issueId)); if (gate === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, source.agentId)); @@ -500,7 +571,8 @@ describe("automatic retained workspace recovery", () => { await heartbeat.resumeRemoteStopComments(await run(f.runId)); expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(0); - if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop"].includes(gate)) { + if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop", + "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption"].includes(gate)) { const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() }); await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index ad1d8ab4f6c..dc8de2e9f1a 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -447,7 +447,7 @@ export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedO and ${agentWakeupRequests.status} = 'deferred_issue_execution' and ${agentWakeupRequests.requestedByActorType} = 'user' and ${agentWakeupRequests.payload}->>'issueId' = ${issueRecoveryActions.sourceIssueId}::text)`; const interruptedPreparation = sql`(${heartbeatRuns.runtimeMode} = 'legacy' - and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.errorCode} = 'server_shutdown_interrupted' + and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.runtimeModeResolvedAt} is null and ${heartbeatRuns.executionStage} = 'preparing' and ${heartbeatRuns.runnerProfileJson}#>>'{adapterDispatch,adapterType}' = 'paperclip_runner' and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked'