diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index fc3720f0edc..625129be469 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1767,6 +1767,15 @@ agent, workspace, retained session, and termination of prior execution. The checkpoint must match the exact Runner, lease, run, session, turn, and item. A missing restart claim does not remove these checks or authorize a fresh session. +Reattachment readies the exact retained sandbox lease through the provider's +bounded lifecycle resume before running short preparation commands such as the +managed project-authentication check. The resume receipt must confirm the same +provider allocation without reporting expiration. Missing or mismatched receipts +hold recovery; this path never acquires a replacement sandbox. The validated +lease retains its recorded working directory, since optional provider metadata +can describe the allocation root. A resume setup failure preserves the retained +sandbox and saved work. Plugin task reattachment remains with its task owner. + After a remote sandbox session passes provider recovery and durable run admission, its local suspension receipt is removed. This applies to built-in sandbox runners and plugin tasks. The running session then owns authority; diff --git a/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts b/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts index 48c095ba586..851807e98b1 100644 --- a/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts @@ -240,10 +240,13 @@ describe("E2B sandbox provider plugin", () => { expect(mockCreate).toHaveBeenCalledWith("base", expect.objectContaining({ apiKey: "host-key" })); }); - it("kills the sandbox if resume setup fails after reconnect", async () => { + it.each(([true, false] as const).flatMap(reuseLease => + (["timeout", "workspace"] as const).map(phase => ({ reuseLease, phase }))))( + "preserves the retained sandbox after resume $phase failure (reuse=$reuseLease)", async ({ reuseLease, phase }) => { const sandbox = createMockSandbox(); - const failure = new Error("set-timeout failed"); - sandbox.setTimeout.mockRejectedValueOnce(failure); + const failure = new Error("retained sandbox readiness failed"); + if (phase === "timeout") sandbox.setTimeout.mockRejectedValueOnce(failure); + else sandbox.commands.run.mockRejectedValueOnce(failure); mockConnect.mockResolvedValue(sandbox); await expect(plugin.definition.onEnvironmentResumeLease?.({ @@ -256,11 +259,13 @@ describe("E2B sandbox provider plugin", () => { template: "base", apiKey: "resolved-key", timeoutMs: 300000, - reuseLease: false, + reuseLease, }, - })).rejects.toThrow("set-timeout failed"); + })).rejects.toBe(failure); - expect(sandbox.kill).toHaveBeenCalled(); + expect(sandbox.kill).not.toHaveBeenCalled(); + expect(sandbox.files.remove).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); }); it("executes commands through a connected sandbox when stdin is provided", async () => { diff --git a/packages/plugins/sandbox-providers/e2b/src/plugin.ts b/packages/plugins/sandbox-providers/e2b/src/plugin.ts index 83bd930687c..21acbc88f55 100644 --- a/packages/plugins/sandbox-providers/e2b/src/plugin.ts +++ b/packages/plugins/sandbox-providers/e2b/src/plugin.ts @@ -310,18 +310,15 @@ const plugin = definePlugin({ const config = parseDriverConfig(params.config); try { const sandbox = await connectSandbox(config, params.providerLeaseId); - try { - await sandbox.setTimeout(config.timeoutMs); - const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); + // A reconnect failure must preserve the retained allocation and its work. + // Explicit release or destroy owns cleanup of an existing sandbox. + await sandbox.setTimeout(config.timeoutMs); + const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); - return { - providerLeaseId: sandbox.sandboxId, - metadata: leaseMetadata({ config, sandbox, remoteCwd, resumedLease: true }), - }; - } catch (error) { - await sandbox.kill().catch(() => undefined); - throw error; - } + return { + providerLeaseId: sandbox.sandboxId, + metadata: leaseMetadata({ config, sandbox, remoteCwd, resumedLease: true }), + }; } catch (error) { if (error instanceof SandboxNotFoundError) { return { providerLeaseId: null, metadata: { expired: true } }; diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index c33d65a1ca9..88743ab99e1 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -192,6 +192,10 @@ function makeRealizeInput(overrides: { function makeMockRuntime(overrides: Partial = {}): EnvironmentRuntimeService { return { acquireRunLease: vi.fn(), + resumeRunLease: vi.fn(async ({ lease, requestedCwd }: Parameters[0]) => ({ + providerLeaseId: lease.providerLeaseId, + metadata: { ...lease.metadata, remoteCwd: requestedCwd ?? lease.metadata?.remoteCwd }, + })), releaseRunLeases: vi.fn(), execute: vi.fn().mockResolvedValue({ exitCode: 0, @@ -906,17 +910,82 @@ describe("live remote runner lease reattachment", () => { mockGetEnvironment.mockResolvedValue({ ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } }); mockGetLease.mockResolvedValue(originalLease()); }); - it("reattaches the original active ephemeral lease without acquiring or resuming a sandbox", async () => { + it("readies the original active ephemeral lease without acquiring a replacement", async () => { const savedLease = originalLease(); mockGetLease.mockResolvedValue(savedLease); const runtime = makeMockRuntime(); const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input); expect(mockGetLease).toHaveBeenCalledWith("lease-1"); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(runtime.resumeRunLease).toHaveBeenCalledWith({ + environment: result.environment, lease: savedLease, requestedCwd: "/remote/workspace", + }); expect(result.lease).toEqual(savedLease); expect(result.leaseContext.executionWorkspaceId).toBe("ew-1"); expect(result.environment.config.reuseLease).toBe(false); }); + it("does not return the retained sandbox to preparation while its lifecycle resume is pending", async () => { + let ready = false; + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const savedLease = originalLease(); + const before = structuredClone(savedLease); + mockGetLease.mockResolvedValue(savedLease); + const runtime = makeMockRuntime({ resumeRunLease: vi.fn(async () => { + await gate; + ready = true; + return { providerLeaseId: "original-sandbox", metadata: { remoteCwd: "/remote/workspace" } }; + }) }); + const pending = environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input); + try { + const state = await Promise.race([ + pending.then(() => "returned"), + new Promise(resolve => setImmediate(() => resolve("pending"))), + ]); + expect(state).toBe("pending"); + expect(ready).toBe(false); + } finally { release(); } + expect((await pending).lease).toEqual(before); + expect(ready).toBe(true); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(runtime.execute).not.toHaveBeenCalled(); + expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled(); + }); + it.each([ + null, + { providerLeaseId: "replacement", metadata: { remoteCwd: "/remote/workspace" } }, + { providerLeaseId: "original-sandbox", metadata: { expired: true } }, + ])("holds recovery when the retained lease resume cannot confirm its original allocation (%j)", async receipt => { + const runtime = makeMockRuntime({ resumeRunLease: vi.fn(async () => receipt) }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) + .rejects.toThrow("native_remote_recovery_lease_mismatch"); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(runtime.execute).not.toHaveBeenCalled(); + expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled(); + }); + it.each([undefined, { remoteCwd: "/allocation/root" }])( + "keeps the recorded workspace when resume returns allocation metadata (%j)", async metadata => { + const savedLease = originalLease(); + mockGetLease.mockResolvedValue(savedLease); + const runtime = makeMockRuntime({ resumeRunLease: vi.fn(async () => ({ + providerLeaseId: "original-sandbox", metadata, + })) }); + const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input); + expect(result.lease).toBe(savedLease); + expect(result.lease.metadata?.remoteCwd).toBe("/remote/workspace"); + expect(runtime.resumeRunLease).toHaveBeenCalledOnce(); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled(); + }, + ); + it("preserves a retained sandbox startup failure without acquiring a replacement", async () => { + const error = new Error("retained sandbox startup timed out"); + const runtime = makeMockRuntime({ resumeRunLease: vi.fn(async () => { throw error; }) }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) + .rejects.toBe(error); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(runtime.execute).not.toHaveBeenCalled(); + }); it.each(["active", "foreign-company", "revoked", "undispatched", "stopped"])("inspects the original plugin task lease (%s)", async state => { mockGetEnvironment.mockResolvedValue(makeEnvironment("plugin")); const lease = originalLease(); @@ -932,6 +1001,7 @@ describe("live remote runner lease reattachment", () => { if (state === "active") await expect(acquired).resolves.toMatchObject({ lease }); else await expect(acquired).rejects.toThrow("native_remote_recovery_lease_mismatch"); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(runtime.resumeRunLease).not.toHaveBeenCalled(); }); it.each([ { companyId: "foreign" }, { environmentId: "foreign" }, { heartbeatRunId: "another-run" }, @@ -946,6 +1016,7 @@ describe("live remote runner lease reattachment", () => { await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) .rejects.toThrow("native_remote_recovery_lease_mismatch"); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(runtime.resumeRunLease).not.toHaveBeenCalled(); }); it("never replaces a missing original lease", async () => { mockGetLease.mockResolvedValue(null); diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 943d01f2e58..957531683f2 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -327,9 +327,8 @@ export function environmentRunOrchestrator( }; let leaseRecord: EnvironmentRuntimeLeaseRecord; if (input.reattachRemoteLease) { - // Reattachment is inspection of an already running sandbox, never a new - // acquisition or a lifecycle resume. Those paths can create a replacement - // when the admitted per-turn policy deliberately disables reusable leases. + // Reattachment keeps the original lease, including ephemeral leases. + // Never reacquire: acquisition can replace a non-reusable sandbox. const expected = input.reattachRemoteLease; const lease = await environmentsSvc.getLeaseById(expected.leaseId); if (!lease || @@ -347,6 +346,21 @@ export function environmentRunOrchestrator( || lease.providerLeaseId !== expected.providerLeaseId || lease.metadata?.remoteCwd !== expected.remoteCwd)) { throw new Error("native_remote_recovery_lease_mismatch"); } + if (expected.kind !== "plugin_task") { + // The provider may have archived this retained sandbox while its + // controller was offline. Ready the exact lease under its lifecycle + // budget before short preparation commands (such as project-auth + // checks) can run. This API cannot acquire a replacement. + const resumed = await environmentRuntime.resumeRunLease({ + environment, lease, requestedCwd: expected.remoteCwd, + }); + if (!resumed || resumed.providerLeaseId !== expected.providerLeaseId || + resumed.metadata?.expired === true) { + throw new Error("native_remote_recovery_lease_mismatch"); + } + // Resume confirms allocation readiness. Its optional metadata may + // describe the allocation root; keep the validated lease's workspace. + } leaseRecord = { environment, lease, leaseContext: buildEnvironmentLeaseContext(input) }; } else { leaseRecord = await acquireLease(acquisitionInput); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 77c6dff225d..31983fa5ac7 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -4307,7 +4307,7 @@ export function environmentRuntimeService( return { destroyed, failed, skippedLiveRun }; }, - async resumeRunLease(input: EnvironmentDriverLeaseInput): Promise { + async resumeRunLease(input: EnvironmentDriverResumeInput): Promise { if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.resumeRunLease) { diff --git a/server/src/services/native-runtime/native-stop-accounting.integration.test.ts b/server/src/services/native-runtime/native-stop-accounting.integration.test.ts index d2ef4378083..33ea4d80323 100644 --- a/server/src/services/native-runtime/native-stop-accounting.integration.test.ts +++ b/server/src/services/native-runtime/native-stop-accounting.integration.test.ts @@ -222,7 +222,9 @@ describe("native Stop accounting after the last usage receipt", () => { const sync = vi.spyOn(workspaceSync, "prepareNativeWorkspaceSync").mockResolvedValue(null); const stage = vi.spyOn(attachmentStaging, "stageNativeRunnerWakeAttachments").mockRejectedValue(new Error("Stop cleanup must not stage attachments")); const acquire = vi.fn(async () => { throw new Error("Stop cleanup cannot acquire a new sandbox"); }); + const resume = vi.fn(async () => ({ providerLeaseId: "existing-sandbox" })); const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", acquireRunLease: acquire, + resumeRunLease: resume, resolveCapabilities: async () => classifyEnvironmentCapabilities({}), realizeWorkspace: async () => ({ cwd: home.home }), retryPendingSandboxTeardown: async ({ lease }) => { @@ -244,6 +246,11 @@ describe("native Stop accounting after the last usage receipt", () => { const holds = await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)); expect(await readClaimedNativeStopRecovery(db, current, owner.leaseOwner ?? undefined)).toBe(true); expect(holds).toMatchObject([{ id: hold.id, state: "held" }]); + expect(resume).toHaveBeenCalledOnce(); + expect(resume).toHaveBeenCalledWith(expect.objectContaining({ + lease: expect.objectContaining({ id: native.environmentLeaseId, providerLeaseId: "existing-sandbox" }), + requestedCwd: "/workspace", + })); reachedSelection = true; } }); try {