diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 83f248ee413..a3913dac9c2 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1472,7 +1472,21 @@ so the cleanup sweep destroys the old sandbox later and no provider delete delay the first turn. It creates a new sandbox, records the acquisition as `replacement` with reason `resume_failed`, and workspace preparation restores the saved snapshot into it. Leases saved by older releases have the same fields and -follow the same rule; one without a verified backup stays strict. +follow the same rule. + +A native runner session without a verified harness backup is replaced only when +the provider has proven it cannot start the sandbox: the lease is idle and every +resume call timed out (no rate limit, reset, or other failure in between) through +the whole extended start deadline (for example a +Daytona sandbox stuck in `archiving`). The other conditions above still apply, so +the task's files come from the committed saved snapshot. The new sandbox starts a +fresh runner session. The old lease keeps its status and its sandbox, and is never +destroyed: one guarded write marks it `reusableSandboxKeptAfterReplacement`, and +later runs no longer offer it for resume. Every automatic teardown path skips a +marked lease: obsolete-lease cleanup, run and workspace teardown, expiry, +retained-orphan settlement, recovered-workspace cleanup, and abandoned-file +release. Only an operator-consented environment delete destroys it. Without that proof, a lease without a +verified backup stays strict. Otherwise (unsaved work, a live owner, a workspace with no committed saved version, an access refusal, or an unavailable worker) the refusal keeps the strict diff --git a/server/src/__tests__/environment-retained-orphan-sandbox.test.ts b/server/src/__tests__/environment-retained-orphan-sandbox.test.ts index 86fcc370bbe..a753a124c8a 100644 --- a/server/src/__tests__/environment-retained-orphan-sandbox.test.ts +++ b/server/src/__tests__/environment-retained-orphan-sandbox.test.ts @@ -308,7 +308,7 @@ suite("retained orphan sandbox on the shared ownership path", () => { * is the exact row shape releases before this change wrote; it carries no * new marker. */ async function warmLease(runId: string, input: { status?: string; workspaceId: string | null; receipt?: boolean; - providerLeaseId?: string; createdAt?: Date }) { + providerLeaseId?: string; createdAt?: Date; metadata?: Record }) { const id = randomUUID(), providerLeaseId = input.providerLeaseId ?? `sandbox://fake/${id}`; const now = input.createdAt ?? new Date(); await db.insert(environmentLeases).values({ id, companyId, environmentId, issueId, heartbeatRunId: runId, @@ -317,7 +317,7 @@ suite("retained orphan sandbox on the shared ownership path", () => { createdAt: now, updatedAt: now, metadata: { agentId, reuseLease: true, ...(input.receipt === false ? {} : { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId, runId, leaseId: id, provider: "fake", providerLeaseId, - state: "stopped", confirmedAt: now.toISOString() } }) } }); + state: "stopped", confirmedAt: now.toISOString() } }), ...input.metadata } }); return id; } @@ -374,6 +374,13 @@ suite("retained orphan sandbox on the shared ownership path", () => { await db.insert(issues).values({ companyId, title: "Sibling on the shared workspace", executionWorkspaceId: shared }); return [await warmLease(await run("failed"), { workspaceId: shared })]; }], + ["a replacement kept it because it may hold the only runner session", async () => { + const [oldWorkspace, newWorkspace] = [await workspace(), await workspace()]; + await db.update(issues).set({ executionWorkspaceId: newWorkspace }).where(eq(issues.id, issueId)); + return [await warmLease(await run("failed"), { workspaceId: oldWorkspace, metadata: { + reusableSandboxKeptAfterReplacement: { runId: randomUUID(), replacementLeaseId: randomUUID(), recordedAt: new Date().toISOString() }, + } })]; + }], ["the task still has a run in flight", async () => { const [oldWorkspace, newWorkspace] = [await workspace(), await workspace()]; await db.update(issues).set({ executionWorkspaceId: newWorkspace }).where(eq(issues.id, issueId)); diff --git a/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts b/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts index 2a0b7920c82..794a9556d1a 100644 --- a/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts +++ b/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts @@ -394,6 +394,119 @@ describeEmbeddedPostgres("reusable sandbox resume failure replacement", () => { }); }); + it("continues on a new sandbox and keeps the old one when an idle sandbox never starts and has no harness backup (staging shape)", async () => { + const seeded = await seed({ leaseStatus: "released", backupStamp: false }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (_attempt, timeoutMs) => { throw clock.timeOut(timeoutMs); }); + + const acquired = await acquire(seeded, runtime); + + expect(acquired.lease.providerLeaseId).toBe("replacement-sandbox"); + expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "replacement", reason: "resume_failed" }); + expect(acquired.lease.metadata?.nativeHarnessBackup).toBeUndefined(); + expect(call.mock.calls.filter(([, method]) => method === "environmentResumeLease").length).toBeGreaterThan(1); + expect(clock.elapsed()).toBeLessThanOrEqual(330_000); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything()); + const kept = await environmentService(db).getLeaseById(seeded.lease.id); + expect(kept).toMatchObject({ status: "released", providerLeaseId: seeded.providerLeaseId, cleanupStatus: null }); + expect(kept?.metadata?.reusableSandboxKeptAfterReplacement).toMatchObject({ + runId: seeded.runId, replacementLeaseId: acquired.lease.id, + }); + + const nextRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, + invocationSource: "assignment", status: "running" }); + await environmentService(db).releaseLease(acquired.lease.id, "released"); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, seeded.runId)); + const next = worker(seeded, async () => ({ providerLeaseId: "replacement-sandbox", + metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } })); + const resumed = await acquire(seeded, next.runtime, nextRunId); + expect(resumed.lease.providerLeaseId).toBe("replacement-sandbox"); + const nextResumes = next.call.mock.calls.filter(([, method]) => method === "environmentResumeLease"); + expect(nextResumes).toHaveLength(1); + expect(nextResumes[0]![2]).toMatchObject({ providerLeaseId: "replacement-sandbox" }); + await expect(environmentService(db).getLeaseById(seeded.lease.id)).resolves.toMatchObject({ + status: "released", providerLeaseId: seeded.providerLeaseId, + }); + }); + + it("never destroys a sandbox kept after replacement through the run or workspace teardown paths", async () => { + const seeded = await seed({ leaseStatus: "released", backupStamp: false }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (_attempt, timeoutMs) => { throw clock.timeOut(timeoutMs); }); + const acquired = await acquire(seeded, runtime); + await environmentService(db).releaseLease(acquired.lease.id, "released"); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, seeded.runId)); + const kept = (await environmentService(db).getLeaseById(seeded.lease.id))!; + + await expect(runtime.destroyRunLease({ environment: seeded.environment, lease: kept })).resolves.toBeNull(); + const destroyed = await runtime.destroyReusableSandboxLeases({ companyId: seeded.companyId, + executionWorkspaceId: seeded.executionWorkspaceId }); + + expect(destroyed.map(record => record.lease.id)).toEqual([acquired.lease.id]); + expect(call.mock.calls.filter(([, method, params]) => method === "environmentDestroyLease" + && (params as { providerLeaseId?: string }).providerLeaseId === seeded.providerLeaseId)).toHaveLength(0); + await expect(environmentService(db).getLeaseById(seeded.lease.id)).resolves.toMatchObject({ + status: "released", providerLeaseId: seeded.providerLeaseId, + }); + }); + + it("destroys a sandbox kept after replacement when an operator approves the environment delete", async () => { + const seeded = await seed({ leaseStatus: "released", backupStamp: false }); + const clock = rpcTimeoutClock(); + const { runtime } = worker(seeded, async (_attempt, timeoutMs) => { throw clock.timeOut(timeoutMs); }); + const acquired = await acquire(seeded, runtime); + await environmentService(db).releaseLease(acquired.lease.id, "released"); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, seeded.runId)); + + const teardown = worker(seeded, async () => { throw new Error("no resume during delete"); }); + teardown.call.mockImplementation(async (_pluginId: string, method: string, params: unknown) => { + if (method === "environmentDestroyLease") return { providerLeaseId: (params as { providerLeaseId: string }).providerLeaseId }; + throw new Error(`Unexpected plugin method: ${method}`); + }); + const result = await teardown.runtime.destroyReusableSandboxLeasesForEnvironment({ + environmentId: seeded.environment.id, consentedBy: "operator" }); + + expect(result).toMatchObject({ destroyed: 2, failed: 0 }); + expect(teardown.call.mock.calls.filter(([, method, params]) => method === "environmentDestroyLease" + && (params as { providerLeaseId?: string }).providerLeaseId === seeded.providerLeaseId)).toHaveLength(1); + const deleted = await environmentService(db).getLeaseById(seeded.lease.id); + expect(deleted?.status).not.toBe("pending_cleanup"); + expect(deleted?.metadata?.reusableSandboxKeptAfterReplacement).toBeUndefined(); + }); + + it("keeps an idle sandbox with no harness backup when some resume calls failed for another reason", async () => { + const seeded = await seed({ leaseStatus: "released", backupStamp: false }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (attempt, timeoutMs) => { + if (attempt === 1) throw clock.timeOut(timeoutMs); + throw Object.assign(clock.timeOut(timeoutMs), { message: "connection reset by peer (ECONNRESET)" }); + }); + + await expect(acquire(seeded, runtime)).rejects.toThrow(/the lease was preserved and no replacement was created\.$/); + expect(call.mock.calls.filter(([, method]) => method === "environmentResumeLease").length).toBeGreaterThan(1); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything()); + const kept = await environmentService(db).getLeaseById(seeded.lease.id); + expect(kept?.metadata?.reusableSandboxKeptAfterReplacement).toBeUndefined(); + }); + + it.each([ + ["unsaved work on the sandbox", { dirty: true }], + ["no saved workspace version", { snapshot: null }], + ] as Array<[string, Shape]>)("keeps an idle sandbox with no harness backup that never starts and fails strictly with %s", async (_label, shape) => { + const seeded = await seed({ leaseStatus: "released", backupStamp: false, ...shape }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (_attempt, timeoutMs) => { throw clock.timeOut(timeoutMs); }); + + await expect(acquire(seeded, runtime)).rejects.toThrow( + /could not be resumed \(cause=transport_timeout\); the lease was preserved and no replacement was created\.$/, + ); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything()); + const kept = await environmentService(db).getLeaseById(seeded.lease.id); + expect(kept).toMatchObject({ status: "released", providerLeaseId: seeded.providerLeaseId }); + expect(kept?.metadata?.reusableSandboxKeptAfterReplacement).toBeUndefined(); + }); + it("does not extend the wait for a sandbox that a live run still holds", async () => { const seeded = await seed({ previousRunStatus: "running", dirty: true }); const clock = rpcTimeoutClock(); diff --git a/server/src/services/environment-retained-orphan-settlement.ts b/server/src/services/environment-retained-orphan-settlement.ts index f9ba599d920..53fa0be16bc 100644 --- a/server/src/services/environment-retained-orphan-settlement.ts +++ b/server/src/services/environment-retained-orphan-settlement.ts @@ -7,6 +7,7 @@ import type { OwnerProof } from "./owner-proof.js"; import type { InProgressRecord, SettleOutcome } from "./recovery/ownership-registry.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { ORPHANED_SANDBOX_RECOVERY_KEY, prepareSupersededWarmSandboxStop } from "./sandbox-stop-and-retain.js"; +import { REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY } from "./reusable-sandbox-kept-marker.js"; /** Failure reason on a retained orphan sandbox released for destruction. */ export const RETAINED_ORPHAN_RELEASED_REASON = "retained_orphan_sandbox_released"; @@ -111,7 +112,7 @@ function softlyHeldSql(now?: Date) { function alwaysHeldSql() { return sql`(${environmentLeases.metadata} ?| array['nativeWorkspaceExportResume', 'sandboxStopAndRetain', - 'workspaceRestoreRecovery', 'workspaceRecoveryCleanup', 'runnerTaskSourceConsumer'] + 'workspaceRestoreRecovery', 'workspaceRecoveryCleanup', 'runnerTaskSourceConsumer', ${REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY}] or ${abandonedWorkspaceFilesHeldOnLeaseRowSql()} or coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'true' or coalesce(${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,state}' = 'pending', false) @@ -241,6 +242,7 @@ async function heldBy(tx: Db, lease: LeaseRow, runId: string, now: Date): Promis if (metadata.workspaceRestoreRecovery !== undefined) return "workspace_restore_recovery"; if (metadata.workspaceRecoveryCleanup !== undefined) return "workspace_recovery_cleanup"; if (metadata.runnerTaskSourceConsumer !== undefined) return "source_recovery_consumer"; + if (metadata[REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY] !== undefined) return "kept_after_replacement"; const shutdown = metadata.legacyShutdownWorkspaceSettlement as Record | undefined; if (shutdown?.state === "pending") return "legacy_shutdown_fence"; const [other] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 78ea669145a..a0b079c8f85 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -7,10 +7,15 @@ import { sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; import { NATIVE_PREDECESSOR_RETIREMENT_UNPROVEN_CODE, readReplaceBeforeReuseMarker } from "./native-runtime/predecessor-retirement.js"; import { REUSABLE_SANDBOX_IDLE_RESUME_BUDGET_MS, + REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY, REUSABLE_SANDBOX_RESUME_BUDGET_MS, + keepReplacedReusableSandboxLease, retireReplacedReusableSandboxLease, + reusableSandboxKeptAfterReplacement, reusableSandboxLeaseIsIdle, - reusableSandboxReplacementIsSafe, + reusableSandboxNotKeptAfterReplacementSql, + reusableSandboxReplacementPlan, + type ReusableSandboxReplacementPlan, sandboxStateTransitionFailure, } from "./reusable-sandbox-resume-replacement.js"; import { PluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js"; @@ -1808,6 +1813,7 @@ function createSandboxEnvironmentDriver( ? (await environmentsSvc.listLeases(input.environment.id)) .filter((lease) => lease.leasePolicy === "reuse_by_environment" && + !reusableSandboxKeptAfterReplacement(lease) && reusableLeaseCanBeResumed({ lease, heartbeatRunId: input.heartbeatRunId }) && lease.executionWorkspaceId === input.executionWorkspaceId && (input.executionWorkspaceId !== null || lease.issueId === input.issueId) && @@ -1859,6 +1865,7 @@ function createSandboxEnvironmentDriver( let providerLease: PluginEnvironmentLease | null = null; let replacingUnretiredOwner = false; + let replacementPlan: ReusableSandboxReplacementPlan | null = null; let replacementReason: | "not_found" | "expired" @@ -1888,13 +1895,16 @@ function createSandboxEnvironmentDriver( // sandbox asked for a fresh one. Replace it under the same safety // rule as a failed resume; otherwise resume it as before. replacingUnretiredOwner = readReplaceBeforeReuseMarker(reusableLease.metadata) !== null - && await reusableSandboxReplacementIsSafe({ + && await reusableSandboxReplacementPlan({ db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, executionWorkspaceId: input.executionWorkspaceId, adapterType: input.adapterType, cause: new Error(NATIVE_PREDECESSOR_RETIREMENT_UNPROVEN_CODE), - }); - if (replacingUnretiredOwner) replacementReason = "resume_failed"; - else try { + }) === "retire"; + let idleStartWaitSpent = false; + if (replacingUnretiredOwner) { + replacementReason = "resume_failed"; + replacementPlan = "retire"; + } else try { // One run spends a bounded budget resuming, retries included; the // provider's own start deadline (five minutes on Daytona) is not it. const providerStartTimeoutMs = resolvePluginSandboxRpcTimeoutMs( @@ -1909,6 +1919,8 @@ function createSandboxEnvironmentDriver( Math.min(providerStartTimeoutMs, REUSABLE_SANDBOX_IDLE_RESUME_BUDGET_MS), ); let retryDelayMs = 250; + let idleStartWaitExtended = false; + let everyResumeTimedOut = true; let resumed: PluginEnvironmentLease; while (true) { try { @@ -1932,6 +1944,7 @@ function createSandboxEnvironmentDriver( break; } catch (error) { if (!transientSandboxResumeFailure(error)) throw error; + if (sandboxProviderFailureLabel(error).label !== "transport_timeout") everyResumeTimedOut = false; const retryAt = Date.now() + retryDelayMs * 1.25; if (retryAt >= resumeDeadline) { const waitForIdleStart = resumeDeadline < idleResumeDeadline && retryAt < idleResumeDeadline @@ -1939,8 +1952,12 @@ function createSandboxEnvironmentDriver( && await reusableSandboxLeaseIsIdle({ db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, }); - if (!waitForIdleStart) throw error; + if (!waitForIdleStart) { + idleStartWaitSpent = idleStartWaitExtended && everyResumeTimedOut; + throw error; + } resumeDeadline = idleResumeDeadline; + idleStartWaitExtended = true; } const jitteredDelayMs = Math.max( 1, @@ -1967,10 +1984,12 @@ function createSandboxEnvironmentDriver( } catch (error) { // A refusal is not proof the sandbox is lost. Replace it only when // nothing lives solely on it; otherwise preserve it for the retry. - if (!await reusableSandboxReplacementIsSafe({ + replacementPlan = await reusableSandboxReplacementPlan({ db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, executionWorkspaceId: input.executionWorkspaceId, adapterType: input.adapterType, cause: error, - })) { + providerCannotStart: idleStartWaitSpent, + }); + if (!replacementPlan) { throw new ReusableSandboxResumeError({ provider: parsed.config.provider, providerLeaseId: reusableLease.providerLeaseId, @@ -2142,9 +2161,16 @@ function createSandboxEnvironmentDriver( // carries the harness backup, is recorded. A failed creation or a // restart before this point leaves it eligible for the next attempt. if (replacementReason === "resume_failed" && reusableLease && input.heartbeatRunId) { - await retireReplacedReusableSandboxLease({ - db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, - }); + if (replacementPlan === "keep") { + await keepReplacedReusableSandboxLease({ + db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, + replacementLeaseId: recordedLease.id, + }); + } else { + await retireReplacedReusableSandboxLease({ + db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, + }); + } } return recordedLease; } @@ -2195,6 +2221,7 @@ function createSandboxEnvironmentDriver( ? (await environmentsSvc.listLeases(input.environment.id)) .filter((lease) => lease.leasePolicy === "reuse_by_environment" && + !reusableSandboxKeptAfterReplacement(lease) && reusableLeaseCanBeResumed({ lease, heartbeatRunId: input.heartbeatRunId }) && lease.executionWorkspaceId === input.executionWorkspaceId && (input.executionWorkspaceId !== null || lease.issueId === input.issueId) && @@ -3216,6 +3243,7 @@ function createSandboxEnvironmentDriver( lease: EnvironmentLease; failureReason: string; }): Promise { + if (reusableSandboxKeptAfterReplacement(input.lease)) return null; if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; if (await abandonedWorkspaceFilesHeldOnLease(db, input.lease)) { await parkAbandonedWorkspaceFilesLease(db, input.lease, ["released", "retained", "failed", "pending_cleanup"]); @@ -3932,6 +3960,7 @@ export function environmentRuntimeService( runId?: string; failureReason: string; anyLeasePolicy?: boolean; + releasesKeptSandbox?: boolean; }): Promise { if (await nativeWorkspaceExportPendingForLease(db, input.leaseRow)) return null; if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.leaseRow))) return null; @@ -3952,7 +3981,8 @@ export function environmentRuntimeService( lastUsedAt: now, updatedAt: now, metadata: sql`(case when jsonb_typeof(${environmentLeases.metadata}) = 'object' - then ${environmentLeases.metadata} else '{}'::jsonb end - 'remoteExecutionTermination') + then ${environmentLeases.metadata} else '{}'::jsonb end - 'remoteExecutionTermination' + - ${input.releasesKeptSandbox ? REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY : ""}::text) || ${JSON.stringify({ pendingCleanupAttemptId: attemptId, pendingCleanupInFlight: true, pendingCleanupLeaseExpiresAtMs: Date.now() + 15 * 60_000, pendingCleanupController: controller })}::jsonb`, }).where(and( @@ -3963,6 +3993,7 @@ export function environmentRuntimeService( : input.anyLeasePolicy ? undefined : eq(environmentLeases.leasePolicy, "reuse_by_environment"), eq(environmentLeases.status, input.leaseRow.status), input.scopeCondition, + input.releasesKeptSandbox ? undefined : reusableSandboxNotKeptAfterReplacementSql(), sql`coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') != 'true'`, @@ -4602,6 +4633,7 @@ export function environmentRuntimeService( continue; } const lease = await destroyClaimedSandboxLease({ environment, leaseRow, failureReason, anyLeasePolicy: true, + releasesKeptSandbox: true, scopeCondition: inArray(environmentLeases.leasePolicy, [...KEPT_SANDBOX_LEASE_POLICIES]) }); if (!lease) { // Lost to a racing resume or a concurrent terminal transition — the diff --git a/server/src/services/remote-workspace-abandoned-files-hold.ts b/server/src/services/remote-workspace-abandoned-files-hold.ts index e9efa9373fd..3183d002bf6 100644 --- a/server/src/services/remote-workspace-abandoned-files-hold.ts +++ b/server/src/services/remote-workspace-abandoned-files-hold.ts @@ -3,6 +3,7 @@ import { companies, environmentLeases, executionWorkspaces, heartbeatRuns, issue import { logger } from "../middleware/logger.js"; import { logActivity } from "./activity-log.js"; import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; +import { reusableSandboxNotKeptAfterReplacementSql } from "./reusable-sandbox-kept-marker.js"; import { ABANDONED_WORKSPACE_FILES_RETENTION_DAYS, ABANDONED_WORKSPACE_FILES_RETENTION_MS, REMOTE_WORKSPACE_ABANDONED_EPOCH_SCHEMA, } from "./remote-workspace-epoch-abandonment.js"; @@ -129,7 +130,7 @@ export async function releaseAbandonedWorkspaceFiles(db: Db, input: { || ${JSON.stringify({ [ABANDONED_WORKSPACE_FILES_RELEASE_KEY]: marker })}::jsonb`, }).where(and(eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.providerLeaseId, providerLeaseId), eq(environmentLeases.status, "retained"), sql`${environmentLeases.metadata} ? ${ABANDONED_WORKSPACE_FILES_PARKED_KEY}`, - sql`not ${abandonedWorkspaceFilesHeldOnLeaseRowSql()}`)).returning({ id: environmentLeases.id }), + sql`not ${abandonedWorkspaceFilesHeldOnLeaseRowSql()}`, reusableSandboxNotKeptAfterReplacementSql())).returning({ id: environmentLeases.id }), { companyId: input.companyId }); const scheduled = unparked.length > 0; const summary = input.reason === "retention_expired" diff --git a/server/src/services/reusable-sandbox-kept-marker.ts b/server/src/services/reusable-sandbox-kept-marker.ts new file mode 100644 index 00000000000..fca2121cafa --- /dev/null +++ b/server/src/services/reusable-sandbox-kept-marker.ts @@ -0,0 +1,14 @@ +import { sql } from "drizzle-orm"; +import { environmentLeases } from "@paperclipai/db"; +import type { EnvironmentLease } from "@paperclipai/shared"; + +export const REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY = "reusableSandboxKeptAfterReplacement"; + +export function reusableSandboxKeptAfterReplacement(lease: Pick): boolean { + const marker = lease.metadata?.[REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY]; + return typeof marker === "object" && marker !== null && !Array.isArray(marker); +} + +export function reusableSandboxNotKeptAfterReplacementSql() { + return sql`not coalesce(${environmentLeases.metadata} ? ${REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY}, false)`; +} diff --git a/server/src/services/reusable-sandbox-resume-replacement.ts b/server/src/services/reusable-sandbox-resume-replacement.ts index d323a93a9cb..401355a013f 100644 --- a/server/src/services/reusable-sandbox-resume-replacement.ts +++ b/server/src/services/reusable-sandbox-resume-replacement.ts @@ -9,6 +9,13 @@ import { verifyRecoverableNativeHarnessBackup } from "./native-runtime/native-ha import { findPluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js"; import { abandonedWorkspaceFilesHeldOnLeaseRowSql, ABANDONED_WORKSPACE_FILES_PARKED_KEY, ABANDONED_WORKSPACE_FILES_PARKED_REASON } from "./remote-workspace-abandoned-files-hold.js"; import { sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; +import { REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY } from "./reusable-sandbox-kept-marker.js"; + +export { + REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY, + reusableSandboxKeptAfterReplacement, + reusableSandboxNotKeptAfterReplacementSql, +} from "./reusable-sandbox-kept-marker.js"; /** * Total time one run spends resuming a preserved reusable sandbox, including @@ -23,6 +30,10 @@ export const REUSABLE_SANDBOX_IDLE_RESUME_BUDGET_MS = 330_000; /** Failure reason recorded on a preserved lease retired in favor of a replacement. */ export const REUSABLE_SANDBOX_RESUME_REPLACED_REASON = "resume_failed"; + +export type ReusableSandboxReplacementPlan = "retire" | "keep"; + + /** * A provider refused to start a sandbox because it is still changing state, * for example auto-stopping or archiving after an idle period. Waiting can @@ -55,21 +66,23 @@ export async function reusableSandboxLeaseIsIdle(input: { /** * Decide whether a preserved reusable sandbox that could not be resumed may be - * replaced. + * replaced, and what happens to it. * * Replacement is safe only when nothing lives solely on that sandbox: * - its last run has ended (released or retained, and no live run holds it); * - the task workspace is clean and its saved version is a committed snapshot, * so a new sandbox restores exactly the same files; - * - a native runner session has a verified, lease-bound harness backup. + * - a native runner session has a verified, lease-bound harness backup, or the + * provider proved it cannot start the sandbox, which is then kept, not retired. * Anything else (unsaved work, a live owner, a legacy workspace with no saved * version, an access refusal) keeps the strict preserve-and-retry behavior. * * This only reads. The caller creates and records the replacement first, then - * calls `retireReplacedReusableSandboxLease`, so a failed creation or a restart - * in between leaves the preserved lease eligible for the next attempt. + * calls `retireReplacedReusableSandboxLease` or `keepReplacedReusableSandboxLease`, + * so a failed creation or a restart in between leaves the preserved lease + * eligible for the next attempt. */ -export async function reusableSandboxReplacementIsSafe(input: { +export async function reusableSandboxReplacementPlan(input: { db: Db; lease: EnvironmentLease; companyId: string; @@ -77,17 +90,18 @@ export async function reusableSandboxReplacementIsSafe(input: { executionWorkspaceId: string | null; adapterType: string | null | undefined; cause: unknown; -}): Promise { + providerCannotStart?: boolean; +}): Promise { const { db, lease } = input; - if (!lease.providerLeaseId || !await reusableSandboxLeaseIsIdle(input)) return false; - if (!input.executionWorkspaceId || lease.executionWorkspaceId !== input.executionWorkspaceId) return false; + if (!lease.providerLeaseId || !await reusableSandboxLeaseIsIdle(input)) return null; + if (!input.executionWorkspaceId || lease.executionWorkspaceId !== input.executionWorkspaceId) return null; // A replacement cannot succeed where the resume was refused for access, and // an unavailable worker cannot create one either. - if (findPluginWorkerUnavailableError(input.cause)) return false; + if (findPluginWorkerUnavailableError(input.cause)) return null; const failure = sandboxProviderFailureLabel(input.cause); const status = failure.status ?? readEnvironmentResumeErrorDiagnostic(input.cause)?.httpStatus ?? null; if (failure.label === "provider_access_denied" || failure.label === "permission_denied" - || status === 401 || status === 403) return false; + || status === 401 || status === 403) return null; const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces) .where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.executionWorkspaceId))); @@ -95,25 +109,27 @@ export async function reusableSandboxReplacementIsSafe(input: { try { authority = readRemoteWorkspaceAuthority(workspace?.metadata); } catch { - return false; + return null; } - if (!authority || authority.dirty || !authority.snapshotId) return false; - if (authority.writerRunId !== null && authority.writerRunId !== input.heartbeatRunId) return false; - if (authority.location && authority.location.providerLeaseId !== lease.providerLeaseId) return false; + if (!authority || authority.dirty || !authority.snapshotId) return null; + if (authority.writerRunId !== null && authority.writerRunId !== input.heartbeatRunId) return null; + if (authority.location && authority.location.providerLeaseId !== lease.providerLeaseId) return null; const [snapshot] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and( eq(executionWorkspaceSnapshots.companyId, input.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, input.executionWorkspaceId), eq(executionWorkspaceSnapshots.id, authority.snapshotId), eq(executionWorkspaceSnapshots.status, "committed"), )); - if (!snapshot) return false; + if (!snapshot) return null; const scope = lease.metadata?.reusableSandboxLease as { adapterType?: unknown } | undefined; if ((input.adapterType === "paperclip_runner" || scope?.adapterType === "paperclip_runner") && !await verifyRecoverableNativeHarnessBackup({ db, companyId: lease.companyId, - stamp: lease.metadata?.nativeHarnessBackup, providerLeaseId: lease.providerLeaseId })) return false; + stamp: lease.metadata?.nativeHarnessBackup, providerLeaseId: lease.providerLeaseId })) { + return input.providerCannotStart === true ? "keep" : null; + } - return true; + return "retire"; } /** @@ -158,3 +174,34 @@ export async function retireReplacedReusableSandboxLease(input: { )).returning({ id: environmentLeases.id }), { companyId: input.companyId, runId: lease.heartbeatRunId ?? undefined }); return Boolean(claimed); } + +export async function keepReplacedReusableSandboxLease(input: { + db: Db; + lease: EnvironmentLease; + companyId: string; + heartbeatRunId: string; + replacementLeaseId: string; +}): Promise { + const { db, lease } = input; + if (!lease.providerLeaseId) return false; + const now = new Date(); + const marker = { [REUSABLE_SANDBOX_KEPT_AFTER_REPLACEMENT_KEY]: { + runId: input.heartbeatRunId, replacementLeaseId: input.replacementLeaseId, recordedAt: now.toISOString(), + } }; + const [kept] = await db.update(environmentLeases).set({ + metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify(marker)}::jsonb`, + updatedAt: now, + }).where(and( + eq(environmentLeases.id, lease.id), + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + inArray(environmentLeases.status, ["released", "retained"]), + sql`NOT EXISTS ( + SELECT 1 FROM ${heartbeatRuns} + WHERE ${heartbeatRuns.id} = ${environmentLeases.heartbeatRunId} + AND ${heartbeatRuns.id} <> ${input.heartbeatRunId} + AND ${heartbeatRuns.status} IN ('queued', 'scheduled_retry', 'running') + )`, + )).returning({ id: environmentLeases.id }); + return Boolean(kept); +} diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index f95b57d7b94..57fcb89aa20 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -40,6 +40,7 @@ import { capturePolicyRearmDueCondition, retainedSourceFallbackDueCondition, sto import { rearmExhaustedCapturePolicy } from "./workspace-recovery-capture-policy.js"; import { rearmExhaustedStopProof } from "./workspace-recovery-stop-proof.js"; import { settleRecoveredWorkspaceRestoreHolds } from "./workspace-restore-hold-settlement.js"; +import { reusableSandboxNotKeptAfterReplacementSql } from "./reusable-sandbox-kept-marker.js"; const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"]; const terminal = ["failed", "interrupted", "timed_out", "cancelled", "succeeded"]; @@ -600,7 +601,8 @@ export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof hear eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.providerLeaseId, ref.providerLeaseId), eq(environmentLeases.status, "released"), eq(environmentLeases.cleanupStatus, "success"), sql`${environmentLeases.metadata} = ${JSON.stringify(lease.metadata)}::jsonb`, - sql`${environmentLeases.metadata}->>'workspaceRecoveryCleanup' is null`)), { companyId: run.companyId, runId: run.id }); + sql`${environmentLeases.metadata}->>'workspaceRecoveryCleanup' is null`, reusableSandboxNotKeptAfterReplacementSql())), + { companyId: run.companyId, runId: run.id }); } /** `dueOnly` narrows the obligations to exhausted holds whose re-arm or * fallback is due now. Cleanup, interrupted preparation and saved input that