diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 078c3598b23..2c5937f9dc2 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1444,6 +1444,15 @@ progress", or still stopping or archiving at the provider's start deadline). A common cause is a follow-up that arrives while the provider is auto-stopping or archiving an idle sandbox. +When a resume call only timed out and the preserved lease is idle (`released` or +`retained`, with no live run holding it), the provider is usually still starting +the sandbox. The run then keeps retrying the same sandbox up to the provider's +start deadline (at most 330 seconds from the first attempt) instead of failing +at 90 seconds. Each call still has at most 90 seconds. A retry that finds the +stopped sandbox still settling the earlier, timed-out call is retried too. This applies whether or +not the sandbox could be replaced, so a sandbox with unsaved work resumes in +the same run instead of waiting behind a Retry. + When the resume still fails, the run replaces the sandbox only when nothing lives solely on it: the lease is `released` or `retained` and no live run holds it, the task workspace is clean with a committed saved snapshot, and a native runner diff --git a/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts b/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts index 08e89747566..2a0b7920c82 100644 --- a/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts +++ b/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts @@ -153,11 +153,11 @@ describeEmbeddedPostgres("reusable sandbox resume failure replacement", () => { return { companyId, agentId, environment, pluginId, runId, executionWorkspaceId, lease, providerLeaseId }; } - function worker(seeded: Awaited>, resume: (attempt: number) => Promise, + function worker(seeded: Awaited>, resume: (attempt: number, timeoutMs: number) => Promise, options: { acquireFails?: boolean } = {}) { let attempts = 0; - const call = vi.fn(async (_pluginId: string, method: string, _params: unknown, _timeoutMs?: number) => { - if (method === "environmentResumeLease") return await resume(++attempts); + const call = vi.fn(async (_pluginId: string, method: string, _params: unknown, timeoutMs?: number) => { + if (method === "environmentResumeLease") return await resume(++attempts, timeoutMs ?? 0); if (method === "environmentAcquireLease") { if (options.acquireFails) throw new JsonRpcCallError({ code: -32000, message: "create refused" }); return { providerLeaseId: "replacement-sandbox", metadata: { provider: "fake-plugin", image: "fake:test", @@ -324,4 +324,84 @@ describeEmbeddedPostgres("reusable sandbox resume failure replacement", () => { status: shape.leaseStatus ?? "retained", providerLeaseId: seeded.providerLeaseId, }); }); + + function rpcTimeoutClock() { + let clock = Date.now(); + const startedAt = clock; + vi.spyOn(Date, "now").mockImplementation(() => clock); + return { + elapsed: () => clock - startedAt, + timeOut: (timeoutMs: number) => { + clock += timeoutMs; + return new JsonRpcCallError({ code: -32003, + message: `RPC call "environmentResumeLease" timed out after ${timeoutMs}ms` }); + }, + }; + } + + it("keeps waiting for an idle sandbox whose resume timed out and resumes it in the same run (staging shape)", async () => { + const seeded = await seed({ leaseStatus: "released", dirty: true }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (attempt, timeoutMs) => { + if (attempt === 1) throw clock.timeOut(timeoutMs); + return { providerLeaseId: seeded.providerLeaseId, metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }; + }); + + const acquired = await acquire(seeded, runtime); + + expect(acquired.lease.providerLeaseId).toBe(seeded.providerLeaseId); + expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" }); + const resumes = call.mock.calls.filter(([, method]) => method === "environmentResumeLease"); + expect(resumes).toHaveLength(2); + for (const [, , , timeoutMs] of resumes) expect(timeoutMs).toBeLessThanOrEqual(90_000); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything()); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything()); + }); + + it("waits out the in-flight resume when a retry finds the stopped sandbox still settling", async () => { + const seeded = await seed({ leaseStatus: "released", dirty: true }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (attempt, timeoutMs) => { + if (attempt === 1) throw clock.timeOut(timeoutMs); + if (attempt === 2) throw new JsonRpcCallError({ code: -32000, + message: "The stopped Daytona sandbox is still settling cancelled work. Retry shortly." }); + return { providerLeaseId: seeded.providerLeaseId, metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }; + }); + + const acquired = await acquire(seeded, runtime); + + expect(acquired.lease.providerLeaseId).toBe(seeded.providerLeaseId); + expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" }); + expect(call.mock.calls.filter(([, method]) => method === "environmentResumeLease")).toHaveLength(3); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything()); + }); + + it("stops waiting at the provider start deadline and keeps the sandbox when every resume times out", async () => { + const seeded = await seed({ leaseStatus: "released", dirty: true }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (_attempt, timeoutMs) => { throw clock.timeOut(timeoutMs); }); + + await expect(acquire(seeded, runtime)).rejects.toThrow( + /could not be resumed \(cause=transport_timeout\); the lease was preserved and no replacement was created\.$/, + ); + + expect(call.mock.calls.filter(([, method]) => method === "environmentResumeLease").length).toBeGreaterThan(1); + expect(clock.elapsed()).toBeLessThanOrEqual(330_000); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything()); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything()); + await expect(environmentService(db).getLeaseById(seeded.lease.id)).resolves.toMatchObject({ + status: "released", providerLeaseId: seeded.providerLeaseId, + }); + }); + + it("does not extend the wait for a sandbox that a live run still holds", async () => { + const seeded = await seed({ previousRunStatus: "running", dirty: true }); + const clock = rpcTimeoutClock(); + const { call, runtime } = worker(seeded, async (_attempt, timeoutMs) => { throw clock.timeOut(timeoutMs); }); + + await expect(acquire(seeded, runtime)).rejects.toThrow(/could not be resumed \(cause=transport_timeout\)/); + + expect(call.mock.calls.filter(([, method]) => method === "environmentResumeLease")).toHaveLength(1); + expect(clock.elapsed()).toBeLessThanOrEqual(90_000); + }); }); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 62294857264..bec487ba3e4 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -3,10 +3,13 @@ import { ENVIRONMENT_LEASE_CONTROLLER_KEY } from "./environment-lease-controller import { RetainedSandboxAbsentError, resumeReplyConfirmsAbsence } from "./retained-sandbox-absent-error.js"; import { readNativeWorkspaceMigration } from "./native-runtime/native-workspace-migration.js"; import { ReusableSandboxResumeError } from "./reusable-sandbox-resume-error.js"; +import { sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; import { NATIVE_PREDECESSOR_RETIREMENT_UNPROVEN_CODE, readReplaceBeforeReuseMarker } from "./native-runtime/predecessor-retirement.js"; import { + REUSABLE_SANDBOX_IDLE_RESUME_BUDGET_MS, REUSABLE_SANDBOX_RESUME_BUDGET_MS, retireReplacedReusableSandboxLease, + reusableSandboxLeaseIsIdle, reusableSandboxReplacementIsSafe, sandboxStateTransitionFailure, } from "./reusable-sandbox-resume-replacement.js"; @@ -1890,14 +1893,17 @@ function createSandboxEnvironmentDriver( else try { // One run spends a bounded budget resuming, retries included; the // provider's own start deadline (five minutes on Daytona) is not it. - const configuredResumeTimeoutMs = Math.min( - resolvePluginSandboxRpcTimeoutMs( - workerConfig, - pluginProvider.resolved.driver.defaultAcquireTimeoutMs, - ) ?? 60_000, - REUSABLE_SANDBOX_RESUME_BUDGET_MS, + const providerStartTimeoutMs = resolvePluginSandboxRpcTimeoutMs( + workerConfig, + pluginProvider.resolved.driver.defaultAcquireTimeoutMs, + ) ?? 60_000; + const configuredResumeTimeoutMs = Math.min(providerStartTimeoutMs, REUSABLE_SANDBOX_RESUME_BUDGET_MS); + const resumeStartedAt = Date.now(); + let resumeDeadline = resumeStartedAt + configuredResumeTimeoutMs; + const idleResumeDeadline = resumeStartedAt + Math.max( + configuredResumeTimeoutMs, + Math.min(providerStartTimeoutMs, REUSABLE_SANDBOX_IDLE_RESUME_BUDGET_MS), ); - const resumeDeadline = Date.now() + configuredResumeTimeoutMs; let retryDelayMs = 250; let resumed: PluginEnvironmentLease; while (true) { @@ -1921,10 +1927,17 @@ function createSandboxEnvironmentDriver( ); break; } catch (error) { - if ( - !transientSandboxResumeFailure(error) || - Date.now() + retryDelayMs * 1.25 >= resumeDeadline - ) throw error; + if (!transientSandboxResumeFailure(error)) throw error; + const retryAt = Date.now() + retryDelayMs * 1.25; + if (retryAt >= resumeDeadline) { + const waitForIdleStart = resumeDeadline < idleResumeDeadline && retryAt < idleResumeDeadline + && sandboxProviderFailureLabel(error).label === "transport_timeout" + && await reusableSandboxLeaseIsIdle({ + db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, + }); + if (!waitForIdleStart) throw error; + resumeDeadline = idleResumeDeadline; + } const jitteredDelayMs = Math.max( 1, Math.round(retryDelayMs * (0.75 + Math.random() * 0.5)), diff --git a/server/src/services/reusable-sandbox-resume-replacement.ts b/server/src/services/reusable-sandbox-resume-replacement.ts index 466a9cfc34c..c25befc9837 100644 --- a/server/src/services/reusable-sandbox-resume-replacement.ts +++ b/server/src/services/reusable-sandbox-resume-replacement.ts @@ -17,6 +17,8 @@ import { sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; */ export const REUSABLE_SANDBOX_RESUME_BUDGET_MS = 90_000; +export const REUSABLE_SANDBOX_IDLE_RESUME_BUDGET_MS = 330_000; + /** Failure reason recorded on a preserved lease retired in favor of a replacement. */ export const REUSABLE_SANDBOX_RESUME_REPLACED_REASON = "resume_failed"; @@ -28,11 +30,28 @@ export const REUSABLE_SANDBOX_RESUME_REPLACED_REASON = "resume_failed"; export function sandboxStateTransitionFailure(error: unknown): boolean { for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { const message = cause instanceof Error ? cause.message : typeof cause === "string" ? cause : ""; - if (/state change in progress|\bis still [a-z_]+ at its start deadline\b/i.test(message)) return true; + if (/state change in progress|\bis still [a-z_]+ at its start deadline\b|\bis still settling cancelled work\b/i.test(message)) return true; } return false; } +export async function reusableSandboxLeaseIsIdle(input: { + db: Db; + lease: EnvironmentLease; + companyId: string; + heartbeatRunId: string | null; +}): Promise { + const { db, lease } = input; + if (!input.heartbeatRunId || !lease.providerLeaseId || lease.companyId !== input.companyId) return false; + if (lease.status !== "released" && lease.status !== "retained") return false; + if (lease.heartbeatRunId && lease.heartbeatRunId !== input.heartbeatRunId) { + const [owner] = await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId))); + if (owner && ["queued", "scheduled_retry", "running"].includes(owner.status)) return false; + } + return true; +} + /** * Decide whether a preserved reusable sandbox that could not be resumed may be * replaced. @@ -59,8 +78,7 @@ export async function reusableSandboxReplacementIsSafe(input: { cause: unknown; }): Promise { const { db, lease } = input; - if (!input.heartbeatRunId || !lease.providerLeaseId || lease.companyId !== input.companyId) return false; - if (lease.status !== "released" && lease.status !== "retained") return false; + if (!lease.providerLeaseId || !await reusableSandboxLeaseIsIdle(input)) return false; if (!input.executionWorkspaceId || lease.executionWorkspaceId !== input.executionWorkspaceId) return false; // A replacement cannot succeed where the resume was refused for access, and // an unavailable worker cannot create one either. @@ -69,12 +87,6 @@ export async function reusableSandboxReplacementIsSafe(input: { const status = failure.status ?? readEnvironmentResumeErrorDiagnostic(input.cause)?.httpStatus ?? null; if (failure.label === "provider_access_denied" || failure.label === "permission_denied" || status === 401 || status === 403) return false; - // A run that still holds the sandbox may have work on it. - if (lease.heartbeatRunId && lease.heartbeatRunId !== input.heartbeatRunId) { - const [owner] = await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId))); - if (owner && ["queued", "scheduled_retry", "running"].includes(owner.status)) return false; - } const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces) .where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.executionWorkspaceId)));