diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index b9322b31010..7b2c9ce59da 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1244,6 +1244,19 @@ the retained state continues to block unverified reuse. ### Warm sandbox continuity +For the native runner, warm mode requests a reusable sandbox lease **before** +lease acquisition. The environment's explicit runner lifecycle overrides the +agent default; an inherited lifecycle uses the current agent setting on each +new turn. This run-scoped configuration does not modify the shared environment +or replace the task's workspace. Switching an existing task from per-turn to +warm therefore acquires a reusable lease through the normal provider path, +instead of starting warm execution on an ephemeral lease. Provider capability, +ownership, cleanup, and verified restore checks still apply; an unsupported +provider must not be treated as reusable. Existing active runs retain their +admitted lifecycle, including when recovery acquires a lease after the agent or +environment lifecycle setting changes. Recovery uses the persisted execution +input for both lease acquisition and lifecycle validation. + A warm sandbox's shared workspace binding persists independently of the experimental isolated-workspaces UI. Ordinary workspace updates remain gated; the runtime can bind only a validated shared workspace in the issue's company diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index f3d1fc659c9..5cf2c2f9fd8 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -11,6 +11,7 @@ const mockUpdateLeaseMetadata = vi.hoisted(() => vi.fn()); const mockUpdateExecutionWorkspace = vi.hoisted(() => vi.fn()); const mockLogActivity = vi.hoisted(() => vi.fn()); const mockLoggerInfo = vi.hoisted(() => vi.fn()); +const mockGetEnvironment = vi.hoisted(() => vi.fn()); vi.mock("../services/environment-execution-target.js", () => ({ resolveEnvironmentExecutionTarget: mockResolveEnvironmentExecutionTarget, @@ -28,7 +29,7 @@ vi.mock("../services/workspace-realization.js", () => ({ vi.mock("../services/environments.js", () => ({ environmentService: vi.fn(() => ({ ensureLocalEnvironment: vi.fn(), - getById: vi.fn(), + getById: mockGetEnvironment, acquireLease: vi.fn(), releaseLease: vi.fn(), updateLeaseMetadata: mockUpdateLeaseMetadata, @@ -743,3 +744,74 @@ describe("environmentRunOrchestrator — realizeForRun", () => { expect(mockResolveEnvironmentExecutionTarget).not.toHaveBeenCalled(); }); }); + + +describe("native runner lifecycle changes before lease acquisition", () => { + it("requests a reusable lease when an existing task switches from per-turn to warm", async () => { + const environment = { ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } }; + mockGetEnvironment.mockResolvedValue(environment); + const acquireRunLease = vi.fn(async (input) => ({ + environment: input.environment, + lease: makeLease({ leasePolicy: input.environment.config.reuseLease ? "reuse_by_environment" : "ephemeral" }), + leaseContext: { executionWorkspaceId: "existing-workspace", executionWorkspaceMode: "shared_workspace" }, + })); + const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: makeMockRuntime({ acquireRunLease }) }); + const input = { + companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local", + adapterType: "paperclip_runner", issueId: "existing-task", heartbeatRunId: "run-1", agentId: "agent-1", + persistedExecutionWorkspace: { id: "existing-workspace", mode: "shared_workspace" as const }, + executionWorkspaceSettings: null, + }; + const cold = await orchestrator.acquireForRun({ ...input, adapterConfig: { lifecycleMode: "per_turn" } }); + expect(cold.lease.leasePolicy).toBe("ephemeral"); + const warm = await orchestrator.acquireForRun({ ...input, heartbeatRunId: "run-2", adapterConfig: { lifecycleMode: "warm", idleTimeoutMs: 60_000 } }); + expect(warm.lease.leasePolicy).toBe("reuse_by_environment"); + expect(warm.environment.config).toEqual({ provider: "daytona", reuseLease: true }); + expect(acquireRunLease.mock.calls[1][0].persistedExecutionWorkspace).toEqual(input.persistedExecutionWorkspace); + // Run-scoped adaptation must not change the shared environment or other agents. + expect(environment.config).toEqual({ provider: "daytona", reuseLease: false }); + const coldAgain = await orchestrator.acquireForRun({ ...input, heartbeatRunId: "run-3", adapterConfig: { lifecycleMode: "per_turn" } }); + expect(coldAgain.lease.leasePolicy).toBe("ephemeral"); + }); + + it.each([ + ["paperclip_runner", undefined, undefined, false], + ["paperclip_runner", "inherit", "per_turn", false], + ["paperclip_runner", "per_turn", "warm", false], + ["paperclip_runner", "warm", "per_turn", true], + ["paperclip_runner", "inherit", "warm", true], + ["codex_local", "inherit", "warm", false], + ])("respects environment precedence and adapter scope (%s, %s, %s)", async (adapterType, runnerLifecycleMode, lifecycleMode, expectedReuse) => { + const environment = { ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false, runnerLifecycleMode } }; + mockGetEnvironment.mockResolvedValue(environment); + const acquireRunLease = vi.fn(async (input) => ({ environment: input.environment, lease: makeLease(), leaseContext: {} })); + const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: makeMockRuntime({ acquireRunLease }) }); + await orchestrator.acquireForRun({ + companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local", adapterType, + adapterConfig: { lifecycleMode }, issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1", + persistedExecutionWorkspace: null, executionWorkspaceSettings: null, + }); + expect(acquireRunLease.mock.calls[0][0].environment.config.reuseLease).toBe(expectedReuse); + }); +}); + + +describe("admitted native lifecycle recovery", () => { + it.each([ + ["warm", "per_turn", "per_turn", true], + ["per_turn", "warm", "warm", false], + ] as const)("preserves admitted %s over changed agent/environment settings", async (admittedLifecycleMode, lifecycleMode, runnerLifecycleMode, expectedReuse) => { + const environment = { ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false, runnerLifecycleMode } }; + mockGetEnvironment.mockResolvedValue(environment); + const acquireRunLease = vi.fn(async (input) => ({ environment: input.environment, lease: makeLease(), leaseContext: {} })); + const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: makeMockRuntime({ acquireRunLease }) }); + await orchestrator.acquireForRun({ + companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local", + adapterType: "paperclip_runner", adapterConfig: { lifecycleMode }, admittedLifecycleMode, + issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1", + persistedExecutionWorkspace: null, executionWorkspaceSettings: null, + }); + expect(acquireRunLease.mock.calls[0][0].environment.config.reuseLease).toBe(expectedReuse); + expect(environment.config.reuseLease).toBe(false); + }); +}); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index bffb022301a..8ec03b56d3a 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -39,6 +39,7 @@ import { import * as sandboxProviderRuntime from "../services/sandbox-provider-runtime.ts"; import * as environmentsModule from "../services/environments.ts"; import { logger } from "../middleware/logger.ts"; +import { resolveRunnerEnvironmentForRun } from "../services/runner-environment-lifecycle.js"; import { environmentService } from "../services/environments.ts"; import { remoteExecutionHasStopped } from "../services/remote-execution-termination.ts"; import { heartbeatService } from "../services/heartbeat.ts"; @@ -466,6 +467,55 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease }; } + it("acquires and resumes a reusable lease after a task changes from per-turn to warm", async () => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); + await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); + const environment = { ...seeded.environment, config: { ...seeded.environment.config, reuseLease: false } }; + await environmentService(db).update(environment.id, { config: environment.config }); + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Lifecycle switch", status: "in_progress", assigneeAgentId: seeded.agentId }); + let acquisitions = 0; + let warmProviderLeaseId = ""; + const call = vi.fn(async (_pluginId: string, method: string, input: any) => { + if (method === "environmentAcquireLease") return { + providerLeaseId: `lifecycle-${++acquisitions}`, metadata: { remoteCwd: "/workspace" }, + }; + if (method === "environmentResumeLease") return { + providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace" }, + }; + if (method === "environmentDestroyLease" || method === "environmentReleaseLease") return { + providerLeaseId: input.providerLeaseId, state: method === "environmentDestroyLease" ? "destroyed" : "stopped", + }; + throw new Error(`Unexpected lifecycle method: ${method}`); + }); + const runtime = environmentRuntimeService(db, { pluginWorkerManager: { + isRunning: () => true, call, + getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), + } as unknown as PluginWorkerManager }); + const input = { + companyId: seeded.companyId, agentId: seeded.agentId, issueId, + adapterType: "paperclip_runner", persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" as const }, + }; + const cold = await runtime.acquireRunLease({ ...input, environment, heartbeatRunId: seeded.runId }); + expect(cold.lease.leasePolicy).toBe("ephemeral"); + await runtime.releaseRunLeases(seeded.runId, "released"); + const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" }); + const warmRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: warmRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); + const warm = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: warmRunId }); + warmProviderLeaseId = warm.lease.providerLeaseId!; + expect(warm.lease.leasePolicy).toBe("reuse_by_environment"); + expect(warm.lease.executionWorkspaceId).toBe(seeded.executionWorkspaceId); + expect((await runtime.resolveCapabilities({ environment: warmEnvironment, lease: warm.lease })).reusableLeases).toBe(true); + await runtime.releaseRunLeases(warmRunId, "released", undefined, "stop_and_retain"); + const nextRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); + const next = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: nextRunId }); + expect(next.lease.providerLeaseId).toBe(warmProviderLeaseId); + expect(acquisitions).toBe(2); + expect((await environmentService(db).getById(environment.id))?.config.reuseLease).toBe(false); + }); + it.each(["stopped", "missing", "wrong-lease", "error"])( "startup cancellation is run-scoped and needs a matching receipt: %s", async (outcome) => { diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 97cca667e6a..91f887ac776 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -25,6 +25,7 @@ import type { ExecutionWorkspaceConfig, IssueExecutionWorkspaceSettings, } from "@paperclipai/shared"; +import { resolveRunnerEnvironmentForRun } from "./runner-environment-lifecycle.js"; import { environmentService } from "./environments.js"; import { environmentRuntimeService, @@ -265,6 +266,8 @@ export function environmentRunOrchestrator( selectedEnvironmentId: string; localEnvironmentId: string; adapterType: string; + adapterConfig?: Record; + admittedLifecycleMode?: "warm" | "per_turn"; issueId: string | null; heartbeatRunId: string; agentId: string; @@ -272,12 +275,16 @@ export function environmentRunOrchestrator( executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; }): Promise { // Step 1: Resolve environment - const environment = await resolveEnvironment({ + const selectedEnvironment = await resolveEnvironment({ companyId: input.companyId, selectedEnvironmentId: input.selectedEnvironmentId, localEnvironmentId: input.localEnvironmentId, }); + const environment = resolveRunnerEnvironmentForRun( + selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, + ); + // Step 2: Acquire lease const leaseRecord = await acquireLease({ companyId: input.companyId, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8c2979dd13c..d20bbd95a27 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -22127,6 +22127,8 @@ export function heartbeatService( selectedEnvironmentId, localEnvironmentId: localEnvironment.id, adapterType: agent.adapterType, + adapterConfig: parseObject(agent.adapterConfig), + admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode, issueId: issueId ?? null, heartbeatRunId: run.id, agentId: agent.id, @@ -23290,9 +23292,10 @@ export function heartbeatService( // Other managed harnesses still require per-turn credential cleanup. const supportsManagedWarmSession = agent.adapterType === "paperclip_runner" && nativeRuntimeResolution.profile.backend === "codex_app_server"; - const effectiveLifecyclePolicy = managedAiRuntime && !supportsManagedWarmSession - ? { mode: "per_turn" as const, idleTimeoutMs: null } - : environmentLifecyclePolicy ?? agentLifecyclePolicy; + const effectiveLifecyclePolicy = persistedNativeExecutionInput?.session.lifecyclePolicy ?? + (managedAiRuntime && !supportsManagedWarmSession + ? { mode: "per_turn" as const, idleTimeoutMs: null } + : environmentLifecyclePolicy ?? agentLifecyclePolicy); if ( effectiveLifecyclePolicy.mode === "warm" && executionTarget?.kind === "remote" && diff --git a/server/src/services/runner-environment-lifecycle.ts b/server/src/services/runner-environment-lifecycle.ts new file mode 100644 index 00000000000..fcb898bae20 --- /dev/null +++ b/server/src/services/runner-environment-lifecycle.ts @@ -0,0 +1,26 @@ +import type { Environment } from "@paperclipai/shared"; + +/** Resolve runner retention before acquiring a sandbox, not after it is ephemeral. + * This is a run-scoped view; never persist it over a shared environment's config. + * The driver still verifies provider support and chooses the actual lease policy. + */ +export function resolveRunnerEnvironmentForRun>( + environment: T, + adapterType: string, + adapterConfig: Record = {}, + admittedLifecycleMode?: "warm" | "per_turn", +): T { + if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") return environment; + const config = environment.config ?? {}; + const lifecycleMode = admittedLifecycleMode ?? (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn" + ? config.runnerLifecycleMode + : adapterConfig.lifecycleMode); + if (lifecycleMode !== "warm") return environment; + return { + ...environment, + config: { + ...config, + reuseLease: true, + }, + }; +}