diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index d027cdbfaea..b400dc6bd3a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2158,7 +2158,7 @@ A Stop or reassignment can reach a native run while its provider session is still opening. The startup owner then fences the late handle before any prompt, and the Stop acknowledgement records `nativeCancellation.beforeProviderTurn`. A reattached provider from restart recovery never gets this proof. When the run -also has no runner event outside the startup lifecycle (harness, session, workspace, +also has no runner event outside the exact startup lifecycle events (harness, session, workspace ready, runtime phase, credential refresh), no turn or provider event, and no native result, no provider outcome is unknown. The completed copyback receipt then releases the hold for the stopped @@ -2171,6 +2171,14 @@ complete, so recovery and that turn cannot both wait for each other. Running turns, and queued turns for other tasks that share the workspace, still block recovery. +After recovery saves the files of a run whose sandbox is destroyed after each +turn (an ephemeral lease), it re-stops the sandbox and then queues the destroy. +The lease carries the verified stop forward as +`workspaceRecoveryCleanup.stopReceipt`. The saved message, Retry and the next +remote workspace writer wait for that stop, not for the destroy. So they continue +right after recovery, instead of after the cleanup retry. A lease without a +matching stop receipt still blocks them. + The deadline is a backstop, not the normal failure path. When native task preparation fails after a fresh controller claim and before any provider turn, the controller stops renewal and releases only that exact claim. Ordinary diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index b67d5c16f2b..ed1630f6f07 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -11,7 +11,7 @@ import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js import { EXECUTION_RECONCILIATION_CAUSES, type ExecutionBlocker } from "@paperclipai/shared"; import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA, nativeRunHasNoWorkspaceAccess } from "./workspace-restore-recovery-state.js"; import { unreconciledNativeExportPredicate } from "./native-runtime/native-workspace-reconstruction.js"; -import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; import { completedWorkspaceRecoveryCondition, hasVerifiedWorkspaceRepair, workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; import { nativeManualAdmissionHoldsExecution } from "./native-runtime/native-runner-ownership.js"; @@ -69,10 +69,17 @@ const blockerRunFields = { export async function hasPendingRunCleanup(db: Db, companyId: string, runId: string): Promise { if (await nativeManualAdmissionHoldsExecution(db, { companyId, runId })) return true; + // Recovery saved the files and re-stopped this sandbox before queueing its + // destroy. That destroy is resource cleanup, not a live owner of the run. + const recoveredStopped = (await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, runId), + sql`${environmentLeases.metadata}->'workspaceRecoveryCleanup' is not null`, + ))).filter(hasRecoveredCleanupStopReceipt).map(lease => lease.id); const [state] = await db.execute<{ pending: boolean }>(sql`SELECT ${heartbeatRuns.runnerProfileJson}->>'nativeWorkspaceFinalizationOwner' IS NOT NULL OR EXISTS (SELECT 1 FROM ${environmentLeases} WHERE ${environmentLeases.companyId} = ${companyId} AND ${environmentLeases.heartbeatRunId} = ${runId} + ${recoveredStopped.length ? sql`AND ${environmentLeases.id}::text NOT IN (${sql.join(recoveredStopped.map(id => sql`${id}`), sql`, `)})` : sql``} AND (( ${environmentLeases.releasedAt} IS NULL AND NOT COALESCE( ${environmentLeases.status} = 'retained' AND ${environmentLeases.leasePolicy} = 'reuse_by_environment' AND ${environmentLeases.cleanupStatus} = 'success', false)) @@ -126,14 +133,18 @@ export function nativeExecutionRecoveryActionPredicate() { and ${heartbeatRuns.runtimeMode} = 'native')`; } -// Runner startup lifecycle families: they open no turn. Every other event the -// runner sourced counts as provider work, so a new or unlisted family keeps -// the hold (fail closed). Server-side records without a source event (spans, -// process identity, lifecycle) count only when they name turn or provider work. -export const PROVIDER_STARTUP_EVENT_PREFIXES = ["harness.", "session.", "workspace.", "runtime.phase.", - "runtime.activity.", "provider.notice.", "auth.credentials.", "mcp_app.discovered", "mcp_app.initializing", "mcp_app.ready", - "mcp_app.resource.", "mcp_app.host_context.", "mcp_app.teardown", "mcp_app.failed"] as const; -export const PROVIDER_STARTUP_EVENT_TYPES = ["run.attached", "run.detached"] as const; +// Exact runner startup lifecycle events (event-v3 schema): they open no turn. +// Every other runner-sourced event counts as provider work, so a new, unlisted +// or turn-scoped event (for example workspace.diff.recorded or a session goal) +// keeps the hold (fail closed). Server-side records without a source event +// (spans, process identity, lifecycle) count only when they name turn or +// provider work. +export const PROVIDER_STARTUP_EVENT_TYPES = ["harness.starting", "harness.ready", "harness.diagnostic", "harness.exited", + "session.starting", "session.started", "session.resuming", "session.resumed", "session.reconciled", "session.updated", + "session.capabilities.updated", "session.closed", "session.failed", "workspace.ready", "runtime.phase.changed", + "runtime.activity.recorded", "provider.notice.recorded", "auth.credentials.updated", "run.attached", "run.detached", + "mcp_app.discovered", "mcp_app.initializing", "mcp_app.ready", "mcp_app.resource.resolved", + "mcp_app.host_context.changed", "mcp_app.teardown", "mcp_app.failed"] as const; const SERVER_TURN_WORK_EVENT_PREFIXES = ["turn.", "item.", "tool.", "semantic_tool.", "run.result."] as const; const SERVER_TURN_WORK_EVENT_TYPES = ["provider.event", "provider.rpc_result"] as const; @@ -144,8 +155,7 @@ const anyType = (eventType: SQL, types: readonly string[]) => /** SQL truth for an event row (alias columns) that may prove provider turn work. */ export function providerTurnWorkEventCondition(eventType: SQL, sourceEventId: SQL) { - return sql`((${sourceEventId} is not null and not (${anyType(eventType, PROVIDER_STARTUP_EVENT_TYPES)} - or ${anyPrefix(eventType, PROVIDER_STARTUP_EVENT_PREFIXES)})) + return sql`((${sourceEventId} is not null and not ${anyType(eventType, PROVIDER_STARTUP_EVENT_TYPES)}) or ${anyType(eventType, SERVER_TURN_WORK_EVENT_TYPES)} or ${anyPrefix(eventType, SERVER_TURN_WORK_EVENT_PREFIXES)})`; } diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 7593411ffb5..eec6f8c3dde 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -17,7 +17,7 @@ import { readQueuedInteractionResponse } from "./queued-interaction-response.js" import { isCancelledNativeStartup, isFailedNativePreparation, isFailedNativePreparationRetryAncestor } from "./cancelled-native-startup.js"; import { hasNativeLocalProcessStop, hasHistoricalSuspendedNativeSession, recordedLocalExecutionStopped } from "./native-local-process-stop.js"; import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js"; -import { hasRemoteTerminationReceipt, remoteLeaseCleanupScope } from "./remote-execution-termination.js"; +import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt, remoteLeaseCleanupScope } from "./remote-execution-termination.js"; import { z } from "zod"; import { and, desc, eq, gt, inArray, isNull, ne, not, or, sql } from "drizzle-orm"; import { @@ -435,7 +435,7 @@ export async function admitExplicitNativeContinuation(input: { const remote = leases.some(lease => lease.provider !== "local"); if (remote) { // Never interpret remote PIDs using the control-plane host's process table. - if (!leases.every(hasRemoteTerminationReceipt)) return blocked("remote_cleanup", "Waiting for the previous environment to stop. Your message will start automatically."); + if (!leases.every(lease => hasRemoteTerminationReceipt(lease) || hasRecoveredCleanupStopReceipt(lease))) return blocked("remote_cleanup", "Waiting for the previous environment to stop. Your message will start automatically."); if (run.runtimeMode === "native" && !input.dryRun && !leases.every(lease => completeTerminatedRemoteNativeSessionCleanup({ companyId, runId: run.id, remoteCleanupScope: remoteLeaseCleanupScope(lease)!, }))) return null; diff --git a/server/src/services/remote-execution-termination.ts b/server/src/services/remote-execution-termination.ts index d30b60ae697..e18a6fdaae9 100644 --- a/server/src/services/remote-execution-termination.ts +++ b/server/src/services/remote-execution-termination.ts @@ -37,6 +37,19 @@ export function hasRemoteTerminationReceipt(lease: LeaseIdentity & { remoteTerminationReceipt(lease, receipt)); } +/** Workspace recovery saved this run's files, re-stopped its sandbox, and then + * queued the stopped sandbox for destroy. The destroy is resource cleanup: the + * stop receipt it carried forward still proves no provider can run there. */ +export function hasRecoveredCleanupStopReceipt(lease: LeaseIdentity & { + releasedAt: unknown; status: string; metadata: Record | null; +}): boolean { + const cleanup = lease.metadata?.workspaceRecoveryCleanup as Record | undefined; + if (!cleanup || cleanup.runId !== lease.heartbeatRunId || !lease.releasedAt || + !["pending_cleanup", "released", "expired", "failed"].includes(lease.status)) return false; + return hasRemoteTerminationReceipt({ ...lease, status: "released", cleanupStatus: "success", + metadata: { remoteExecutionTermination: cleanup.stopReceipt } }); +} + export function remoteLeaseCleanupScope(lease: Pick) { return lease.provider && lease.provider !== "local" && lease.providerLeaseId ? JSON.stringify([lease.provider, lease.providerLeaseId]) : undefined; diff --git a/server/src/services/remote-workspace-reservation.test.ts b/server/src/services/remote-workspace-reservation.test.ts index 76b7d0d5eed..b1980b44041 100644 --- a/server/src/services/remote-workspace-reservation.test.ts +++ b/server/src/services/remote-workspace-reservation.test.ts @@ -7,7 +7,7 @@ import request from "supertest"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { agents, companies, companyMemberships, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, - executionWorkspaces, heartbeatRuns, issues, projects, projectWorkspaces } from "@paperclipai/db"; + environmentLeases, executionWorkspaces, heartbeatRuns, issues, projects, projectWorkspaces } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; @@ -15,6 +15,7 @@ import { createStorageService } from "../storage/service.js"; import { instanceSettingsService } from "./instance-settings.js"; import { issueService } from "./issues.js"; import { reserveRemoteWorkspace } from "./remote-workspaces.js"; +import { remoteTerminationReceipt } from "./remote-execution-termination.js"; import { executionWorkspaceService } from "./execution-workspaces.js"; import { accessService } from "./access.js"; import { taskWorkspaceRoutes } from "../routes/task-workspaces.js"; @@ -152,6 +153,31 @@ describe.skipIf(!support.supported)("remote workspace reservation authority", () await expect(reserveRemoteWorkspace(f.input)).resolves.toMatchObject({ id: f.workspace.id }); }); + it.each(["verified-stop", "no-stop-receipt", "foreign-receipt"] as const) + ("hands over from an ended writer whose recovered sandbox awaits destroy (%s)", async kind => { + const f = await fixture(); + const [writer] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, + issueId: f.parent.id, status: "cancelled", finishedAt: new Date() }).returning(); + const providerLeaseId = randomUUID(); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.company.id, environmentId: f.input.environmentId, + heartbeatRunId: writer.id, issueId: f.parent.id, provider: "daytona", providerLeaseId, + status: "pending_cleanup", cleanupStatus: "failed", leasePolicy: "ephemeral", releasedAt: new Date() }).returning(); + const receipt = remoteTerminationReceipt(lease, { providerLeaseId, state: "stopped" })!; + const stopReceipt = kind === "verified-stop" ? receipt : kind === "foreign-receipt" ? { ...receipt, runId: randomUUID() } : undefined; + await db.update(environmentLeases).set({ metadata: { workspaceRecoveryCleanup: { runId: writer.id, stopReceipt } } }) + .where(eq(environmentLeases.id, lease.id)); + // Recovery saved the files, so the writer is clean but not yet destroyed. + const ended = { ...f.authority, writerRunId: writer.id, dirty: false }; + await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: ended } }).where(eq(executionWorkspaces.id, f.workspace.id)); + if (kind === "verified-stop") { + await expect(reserveRemoteWorkspace(f.input)).resolves.toMatchObject({ id: f.workspace.id }); + expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)).toMatchObject({ writerRunId: f.run.id, epoch: 4 }); + } else { + await expect(reserveRemoteWorkspace(f.input)).rejects.toThrow(); + expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)).toEqual(ended); + } + }); + it("grants only one writer when two bound children reserve concurrently", async () => { const f = await fixture(); const sibling = await issueService(db).create(f.company.id, { parentId: f.parent.id, diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index f75f2ce5503..298e1eed7ec 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -22,7 +22,7 @@ import { logger } from "../middleware/logger.js"; import { createGitRemoteAuthProvider } from "./git-credentials.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { assertExecutionWorkspaceRecordAccess, assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js"; -import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { canonicalExecutionUuid } from "./execution-identity.js"; import { persistActivity } from "./activity-log.js"; import { @@ -240,7 +240,7 @@ export async function reserveRemoteWorkspace(input: { && prior.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE && bootstrap.kind === "bootstrap" && bootstrap.providerWorkStarted === false; const unusedReservation = !authority.dirty && prior && ["failed", "cancelled", "interrupted", "timed_out", "succeeded"].includes(prior.status) - && leases.every(lease => hasRemoteTerminationReceipt(lease) || (pristinePreparation + && leases.every(lease => hasRemoteTerminationReceipt(lease) || hasRecoveredCleanupStopReceipt(lease) || (pristinePreparation && lease.releasedAt && lease.cleanupStatus === "success" && ["released", "expired", "failed"].includes(lease.status))); if (!unusedReservation) { // An ended writer still holds the workspace until its save and sandbox diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index feec7f5a4b3..635091541d0 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -84,7 +84,7 @@ import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/nati import { canonicalProviderPackJson } from "./native-runtime/remote-provider-pack-verification.js"; import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "./remote-workspace-transport.js"; import { withDedicatedDbConnection } from "@paperclipai/db"; -import { getExecutionBlocker } from "./execution-blocker.js"; +import { getExecutionBlocker, hasPendingRunCleanup } from "./execution-blocker.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; import { createHeartbeatRunOutput } from "./heartbeat/run-output.js"; import { objectLogFixture } from "./heartbeat/__tests__/object-log-fixture.js"; @@ -2496,11 +2496,9 @@ describe("automatic retained workspace recovery", () => { await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", actorSource: "local_implicit", observedRunId: source.id, idempotencyKey: randomUUID() }); - await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); - expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0]) - .toMatchObject({ status: "deferred_issue_execution", payload: { executionWait: { reason: "remote_cleanup" } } }); - await environmentService(db).releaseLease(settledLease.id, "released", { cleanupStatus: "success", - remoteExecutionTermination: remoteTerminationReceipt(settledLease, { providerLeaseId: settledLease.providerLeaseId, state: "destroyed" }) }); + // The saved, re-stopped sandbox only awaits destroy. That cleanup does not + // hold the successor: it continues now instead of on the cleanup retry. + expect(settledLease.metadata?.workspaceRecoveryCleanup).toMatchObject({ stopReceipt: { runId: f.runId, leaseId: settledLease.id } }); await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued"))); expect(successor).toBeDefined(); @@ -3688,7 +3686,7 @@ describe("automatic retained workspace recovery", () => { expect((await pendingTurns(f)).map(item => item.id)).toEqual([handoff.id]); }, 30_000); - it.each(["no-proof", "turn-event", "provider-event", "unlisted-provider-event", "running-writer", "other-task-writer", + it.each(["no-proof", "turn-event", "provider-event", "unlisted-provider-event", "workspace-diff-event", "running-writer", "other-task-writer", "unscoped-queued-writer", "non-gating-hold"] as const) ("keeps the stopped agent held when provider work cannot be excluded (%s)", async kind => { const f = await seed(); @@ -3698,10 +3696,11 @@ describe("automatic retained workspace recovery", () => { eventType: "turn.started", stream: "system", level: "info", message: "turn started" }); if (kind === "provider-event") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.runId, agentId, seq: 9_999, eventType: "item.completed", stream: "system", level: "info", message: "provider item", sourceEventId: randomUUID() }); - // A runner family outside the startup lifecycle counts as work even when unlisted. - if (kind === "unlisted-provider-event") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.runId, - agentId, seq: 9_999, eventType: "delegation.completed", stream: "system", level: "info", message: "delegated", - sourceEventId: randomUUID() }); + // A runner event outside the startup lifecycle counts as work even when unlisted, + // including turn-scoped events that share a startup family name. + if (kind === "unlisted-provider-event" || kind === "workspace-diff-event") await db.insert(heartbeatRunEvents).values({ + companyId: f.companyId, runId: f.runId, agentId, seq: 9_999, stream: "system", level: "info", message: kind, + eventType: kind === "unlisted-provider-event" ? "delegation.completed" : "workspace.diff.recorded", sourceEventId: randomUUID() }); if (kind === "running-writer" || kind === "other-task-writer" || kind === "unscoped-queued-writer") { // A started turn, or a queued one that this task's dispatch gate does not cover, is a real writer. await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId, runtimeMode: "legacy", @@ -3729,6 +3728,59 @@ describe("automatic retained workspace recovery", () => { expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ recoveryActionId: f.actionId, runId: f.runId }); }, 30_000); + it.each(["verified-stop", "no-stop-receipt", "foreign-receipt"] as const) + ("admits the interrupt redirect once recovery saved and stopped the sandbox, before its destroy finishes (%s)", async kind => { + // An interrupt comment after the session opened: the run did provider work, + // its files were recovered, and the stopped sandbox was queued for destroy. + const f = await seed(); + const source = await run(f.runId); + const finishedAt = new Date(Date.now() - 60_000); + await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "operator_interrupted", finishedAt, + resultJson: { ...source.resultJson, operatorInterrupted: true, interruptionSource: "issue_comment_interrupt", + nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId: f.runId, companyId: f.companyId, + issueId: f.issueId, scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", + dispatched: true, intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() } }, + }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issueId)); + await recoverRetainedWorkspace(f.input); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "complete" }); + await scheduleRecoveredWorkspaceCleanup(db, await run(f.runId)); + const [queued] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(queued).toMatchObject({ status: "pending_cleanup", metadata: { workspaceRecoveryCleanup: { runId: f.runId } } }); + if (kind !== "verified-stop") { + const cleanup = queued.metadata!.workspaceRecoveryCleanup as Record; + const stopReceipt = kind === "foreign-receipt" + ? { ...(cleanup.stopReceipt as Record), runId: randomUUID() } : undefined; + await db.update(environmentLeases).set({ metadata: { ...queued.metadata, + workspaceRecoveryCleanup: { ...cleanup, stopReceipt } } }).where(eq(environmentLeases.id, f.lease.id)); + } + const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "operator", body: "Use the other approach instead." }).returning(); + // The same input as a plain message and as a saved native interrupt send. + const queueId = randomUUID(); + await db.insert(agentWakeupRequests).values({ id: queueId, companyId: f.companyId, agentId: source.agentId, + source: "automation", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "system", requestedByActorId: "heartbeat", + payload: { issueId: f.issueId, _paperclipWakeContext: { wakeCommentIds: [comment.id] }, + queuedCommentInterrupt: { actorId: "operator", requestedAt: new Date().toISOString() } } }); + expect(await hasPendingRunCleanup(db, f.companyId, f.runId)).toBe(kind !== "verified-stop"); + for (const interrupt of [false, true]) { + const reasons: string[] = []; + const admitted = await admitExplicitNativeContinuation({ db, companyId: f.companyId, issueId: f.issueId, + agentId: source.agentId, actorType: "user", actorId: "operator", reason: "issue_commented", + commentId: comment.id, successorRunId: randomUUID(), dryRun: true, onBlocked: reason => reasons.push(reason), + ...(interrupt ? { queuedCommentInterruptId: queueId } : {}) }); + if (kind === "verified-stop") { + // Destroying a stopped sandbox whose files are saved is cleanup, not ownership. + expect(reasons, JSON.stringify({ interrupt, reasons })).toEqual([]); + expect(admitted).toMatchObject({ previousRunId: f.runId, commentId: comment.id }); + } else { + expect(admitted).toBeNull(); + expect(reasons).toContain("remote_cleanup"); + } + } + }, 30_000); + it("keeps a snapshot-only recovery held for a new assignee because the task workspace lacks the saved files", async () => { const f = await seed("native", false); await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "issue_reassigned" }).where(eq(heartbeatRuns.id, f.runId)); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 9372a62c97d..955a20a222b 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -461,7 +461,9 @@ export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof hear if (other) return; await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", leasePolicy: "ephemeral", updatedAt: new Date(), metadata: sql`(${environmentLeases.metadata} - 'workspaceRestoreRecovery' - 'remoteExecutionTermination') || - ${JSON.stringify({ workspaceRecoveryCleanup: { runId: run.id, ...(legacy ? { descriptorSha256: legacy.descriptorSha256 } + ${JSON.stringify({ workspaceRecoveryCleanup: { runId: run.id, + // Keep the verified stop: successors wait on the stop, not on the destroy. + stopReceipt: lease.metadata?.remoteExecutionTermination, ...(legacy ? { descriptorSha256: legacy.descriptorSha256 } : { snapshotId: remote!.snapshotId, epoch: remote!.epoch }) }, pendingCleanupInFlight: false, pendingCleanupRetryAfterMs: 0, pendingCleanupRetryAttempts: 0, pendingCleanupLeaseExpiresAtMs: 0 })}::jsonb`, }).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, ref.leaseId),