diff --git a/server/src/__tests__/environment-retained-orphan-sandbox.test.ts b/server/src/__tests__/environment-retained-orphan-sandbox.test.ts index 66e4d2926d2..8c7b329bf6f 100644 --- a/server/src/__tests__/environment-retained-orphan-sandbox.test.ts +++ b/server/src/__tests__/environment-retained-orphan-sandbox.test.ts @@ -21,7 +21,7 @@ import { heartbeatService, type HeartbeatEnvironmentRuntime } from "../services/ import { hasRecoveredCleanupStopReceipt } from "../services/remote-execution-termination.js"; import { taskChatRetryExecutionStopped } from "../services/chat-channels/task-retry-authority.js"; import { createOwnershipReportState, runOwnershipReconciliationPass } from "../services/recovery/ownership-invariant-report.js"; -import { ORPHANED_SANDBOX_RECOVERY_KEY } from "../services/sandbox-stop-and-retain.js"; +import { ORPHANED_SANDBOX_RECOVERY_KEY, settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js"; const support = await getEmbeddedPostgresTestSupport(); const suite = support.supported ? describe : describe.skip; @@ -111,6 +111,37 @@ suite("retained orphan sandbox on the shared ownership path", () => { recoveredOrphanedRunLease: true, ownerProof: "pid_dead", runId }); }); + it("keeps the orphan marker for a stop intent saved before the top-level marker existed", async () => { + const runId = await run("failed"); + const id = randomUUID(), providerLeaseId = `sandbox://fake/${id}`, attemptId = randomUUID(); + const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId: attemptId, companyId, runId, leaseId: id, + provider: "fake", providerLeaseId, builtinProvider: "fake", recoveredOrphanedRunLease: true, ownerProof: "pid_dead" }; + await db.insert(environmentLeases).values({ id, companyId, environmentId, issueId, heartbeatRunId: runId, + status: "pending_cleanup", cleanupStatus: "failed", failureReason: "sandbox_stop_pending", leasePolicy: "ephemeral", + provider: "fake", providerLeaseId, metadata: { sandboxStopAndRetain: intent, pendingCleanupAttemptId: attemptId, + pendingCleanupInFlight: true } }); + const settled = await settleStopOnlyCleanup(db, await row(id), { attemptId, receipt: { providerLeaseId, state: "stopped" } }); + expect(settled).toMatchObject({ status: "released", cleanupStatus: "success" }); + expect(settled?.metadata?.[ORPHANED_SANDBOX_RECOVERY_KEY]).toMatchObject({ recoveredOrphanedRunLease: true, + ownerProof: "pid_dead", runId, fromStopIntent: true }); + const { kind } = await pass(); + expect(kind).toMatchObject({ settled: 1 }); + expect((await row(id))).toMatchObject({ status: "pending_cleanup", failureReason: "retained_orphan_sandbox_released" }); + }); + + it("does not mark an ordinary stop-and-retain as an orphan", async () => { + const runId = await run("failed"); + const id = randomUUID(), providerLeaseId = `sandbox://fake/${id}`, attemptId = randomUUID(); + const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId: attemptId, companyId, runId, leaseId: id, + provider: "fake", providerLeaseId, builtinProvider: "fake" }; + await db.insert(environmentLeases).values({ id, companyId, environmentId, issueId, heartbeatRunId: runId, + status: "pending_cleanup", cleanupStatus: "failed", leasePolicy: "ephemeral", provider: "fake", providerLeaseId, + metadata: { sandboxStopAndRetain: intent, pendingCleanupAttemptId: attemptId, pendingCleanupInFlight: true } }); + const settled = await settleStopOnlyCleanup(db, await row(id), { attemptId, receipt: { providerLeaseId, state: "stopped" } }); + expect(settled?.metadata?.[ORPHANED_SANDBOX_RECOVERY_KEY]).toBeUndefined(); + expect((await pass()).kind).toMatchObject({ settled: 0 }); + }); + it("destroys a retained orphan that nothing references, through the cleanup sweep", async () => { const leaseId = await retainedOrphan(await run("failed")); const { kind } = await pass(); diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index a1ccefc1d6d..6269ed3b59d 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -184,6 +184,10 @@ export async function settleStopOnlyCleanup(db: Db, lease: Lease, options: { att const receipt = remoteTerminationReceipt(lease, options.receipt), stopped = receipt?.state === "stopped"; const now = new Date(); const removeIntent: SQL = stopped ? sql`- 'sandboxStopAndRetain'` : sql``; + const recordedIntent = lease.metadata?.[SANDBOX_STOP_AND_RETAIN_KEY] as Record | undefined; + const legacyOrphan = stopped && recordedIntent?.recoveredOrphanedRunLease === true && !lease.metadata?.[ORPHANED_SANDBOX_RECOVERY_KEY] + ? { [ORPHANED_SANDBOX_RECOVERY_KEY]: { recoveredOrphanedRunLease: true, ownerProof: recordedIntent.ownerProof ?? null, + runId: lease.heartbeatRunId, recoveredAt: now.toISOString(), fromStopIntent: true } } : {}; const [updated] = await writeEnvironmentCleanup(db, async tx => { if (stopped) await notifyRetainedNativeSessionCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId! }); return tx.update(environmentLeases).set({ @@ -199,6 +203,7 @@ export async function settleStopOnlyCleanup(db: Db, lease: Lease, options: { att : { builtinProvider: lease.provider, method: "builtin.stopLease" }), confirmedAt: receipt.confirmedAt, } } : {}), + ...legacyOrphan, pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0, })}::jsonb`, }).where(stopOnlyAttempt(lease, options.attemptId)).returning();