diff --git a/doc/plans/2026-11-08-large-workspace-speed.md b/doc/plans/2026-11-08-large-workspace-speed.md new file mode 100644 index 00000000000..bbab3aae9a5 --- /dev/null +++ b/doc/plans/2026-11-08-large-workspace-speed.md @@ -0,0 +1,59 @@ +# Speed with a paperclip-sized workspace + +Status: measured 2026-11-08. Follows [`2026-11-07-warm-follow-up-ttft.md`](2026-11-07-warm-follow-up-ttft.md). + +## Summary + +- **Locally, follow-ups are fast and first turns are not.** A first turn takes 31–34 s to first token in a fresh paperclip clone, mostly the repository's own `provision-worktree.sh` (about 20 s). Follow-ups take 7 s. +- **On Daytona, every follow-up turn re-uploaded the whole workspace.** With a default environment a follow-up took 30 s, because each turn got a new sandbox and synced the repository into it again (13–22 s), then started a new runner (8–9 s). +- **A reusable warm sandbox removes that wait.** The same follow-up takes 8.1 s (Codex) and 10.3 s (Claude). An agent that never chose a lifecycle, in an environment that inherits it, now runs Codex, OpenCode and Claude warm on a sandbox. +- **The wait was invisible in the run log.** It sat inside one opaque `heartbeat.prepare_after_environment` span. Named preparation phases are now run-log spans too. + +## Method + +- Driver: [`tests/restart-chaos/ttft-baseline.ts`](../../tests/restart-chaos/ttft-baseline.ts). + - `--workspace-repo` gives a fresh clone of paperclipai/paperclip at origin/master (14.8k files, 484 MB, no `node_modules`) as the project's primary workspace, with git_worktree isolation. The repository's `scripts/provision-worktree.sh` runs automatically, as for a real user. + - The `postfault` phase is the first follow-up after an idle K1 restart. + - `--env daytona` (and `--daytona-warm` for the reusable warm fixture) uses the signed Runner image built from master c8822ffd4b. +- One sample per cell, Codex and Claude. The figures are comment-to-first-token (`task.time_to_first_token`) from the run log only. +- Every Daytona sandbox the runs created was cleaned up. + +## Results (time to first token) + +| | Local | Daytona, per-turn sandbox | Daytona, warm reusable sandbox | +|---|---|---|---| +| First turn | 31–34 s | 58–63 s | 45–51 s | +| Follow-up | 7 s | 30 s | 8.1 s (Codex), 10.3 s (Claude) | +| Follow-up after an idle K1 | 6.8–8.2 s | 38–46 s | 20.8 s (Claude) | + +## Where a remote follow-up spent its time (Codex, per-turn sandbox) + +| Step | Time | +|---|---| +| `workspace_sync.transfer`: the whole workspace into the new sandbox | 13.3 s | +| `runner.session.resume`: a new runner for the turn | 8–9 s | +| Environment acquire, workspace binding, instructions | about 4 s | +| `workspace_finalize` after the turn (copyback; not before the first token) | 8–10 s | + +Run-log reuse records show `environment: fresh/created` on every follow-up. + +## Largest waits, ranked + +1. **A full workspace upload on every remote turn** (13–22 s). Fixed: warm keeps the sandbox, and the workspace is adopted after a stamp check. +2. **A new runner on every remote turn** (8–9 s). Fixed by the same change. +3. **First-turn provision on a large repository** (about 20 s locally). The repository's own script; next step: a pre-provisioned worktree pool keyed by base commit and lockfile hash. +4. **Copyback after each remote turn** (8–10 s). It is after the reply, not before the first token. +5. **First-turn seed build** (git clone and pack 4.5 s, seed verify and publish 3.7 s). + +## The default change + +- **Scope.** A sandbox environment that inherits the lifecycle (`runnerLifecycleMode` unset), and an agent with no stored lifecycle. Covers Codex and OpenCode, as locally, plus Claude: on a sandbox the upload, not the provider relaunch, dominates its follow-up. +- **Gate.** One rule (`defaultsRemoteSandboxRunToWarm`) decides it for both the lease and the runner. It applies only when the sandbox provider keeps reusable leases (Daytona does; E2B, Modal and the built-in `fake` provider do not), and with a managed AI account only when that harness supports a managed warm session (Codex). Otherwise the run is per-turn, exactly as before. The lease side asks the provider before acquisition; the runner side reads the acquired lease, so a default never produces a refused warm run. +- **Mechanics.** Warm on a sandbox turns on `reuseLease` for the run. The follow-up keeps the live sandbox and runner and adopts the workspace. +- **What does not change.** Explicit choices, environments that pin a lifecycle, plugin-task environments and admitted sessions keep their policy. +- **Cost.** An idle sandbox stays up for the warm idle window (the agent's `idleTimeoutMs`, 5 minutes by default). After that the controller suspends the runner, and the provider auto-stop follows the same window. + +## Gaps + +- One sample per cell on a shared host. Daytona's warm path still needs a K1, K2 and K6 chaos wave before the default ships widely. +- In the warm run, the Codex post-restart sample finished but produced no report row. diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 08a942542a0..478a058e55e 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -209,6 +209,12 @@ export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWor | { mode: "per_turn"; idleTimeoutMs: null } | { mode: "warm"; idleTimeoutMs: number } | null; + /** + * The environment's configured warm idle window, present even when the + * environment inherits the lifecycle. A defaulted warm runner uses it first, + * as the provider's auto-stop does, so both keep the same window. + */ + readonly runnerIdleTimeoutMs?: number; /** Whether this environment is configured to reuse its provider lease. */ readonly reusableLeaseConfigured?: boolean; /** Host-observed provenance for this exact sandbox acquisition. */ diff --git a/server/src/__tests__/environment-execution-target-capabilities.test.ts b/server/src/__tests__/environment-execution-target-capabilities.test.ts index 677223491a5..23677af4f40 100644 --- a/server/src/__tests__/environment-execution-target-capabilities.test.ts +++ b/server/src/__tests__/environment-execution-target-capabilities.test.ts @@ -218,6 +218,16 @@ describe("resolveEnvironmentExecutionTarget effective capability snapshot", () = expect(target.reusableLeaseConfigured).toBe(true); }); + it("carries an inheriting environment's idle window for the remote warm default", async () => { + const { target } = await buildSandboxTarget({ + snapshot: FULL_GRANT, + supportsSync: false, + config: { reuseLease: true, runnerIdleTimeoutMs: 900_000 }, + }); + expect(target.runnerLifecyclePolicy).toBeNull(); + expect(target.runnerIdleTimeoutMs).toBe(900_000); + }); + it("carries host-owned sandbox acquisition provenance without persisting provider ids in metadata", async () => { mockResolveEnvironmentDriverConfigForRuntime.mockResolvedValue({ driver: "sandbox", diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index c922db7ef04..5b7629cd36e 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -851,6 +851,34 @@ describe("native runner lifecycle changes before lease acquisition", () => { expect(coldAgain.lease.leasePolicy).toBe("ephemeral"); }); + // The remote-sandbox warm default asks the provider before it requests a + // reusable lease, and asks only for a run whose lifecycle nobody chose. + it.each([ + ["a provider with reusable leases", {}, true, true, true], + ["a provider without reusable leases (E2B, Modal, fake)", {}, false, true, false], + ["an explicit per-turn agent", { lifecycleMode: "per_turn" }, true, false, false], + ["a managed Claude account", { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6", managed: true }, true, false, false], + ] as const)("defaults an inheriting sandbox run to warm only on %s", async (_name, agent, supported, probed, warm) => { + const environment = { ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } }; + mockGetEnvironment.mockResolvedValue(environment); + const acquireRunLease = vi.fn(async (input) => ({ environment: input.environment, lease: makeLease(), leaseContext: { executionWorkspaceId: null, executionWorkspaceMode: null } })); + const providerSupportsReusableLeases = vi.fn(async () => supported); + const orchestrator = environmentRunOrchestrator({} as never, { + environmentRuntime: { ...makeMockRuntime({ acquireRunLease }), providerSupportsReusableLeases } as never, + }); + const { managed, ...adapterConfig } = { managed: false, ...agent }; + await orchestrator.acquireForRun({ + companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local", adapterType: "paperclip_runner", + adapterConfig, managedAiCredentialIdentity: managed ? "account:generation" : undefined, + issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1", + persistedExecutionWorkspace: null, executionWorkspaceSettings: null, + }); + expect(providerSupportsReusableLeases).toHaveBeenCalledTimes(probed ? 1 : 0); + expect(acquireRunLease.mock.calls[0][0].environment.config.reuseLease).toBe(warm); + expect(acquireRunLease.mock.calls[0][0].warmRunnerIdleTimeoutMs).toBe(warm ? 300_000 : undefined); + expect(environment.config.reuseLease).toBe(false); + }); + it.each([ ["paperclip_runner", undefined, undefined, false], ["paperclip_runner", "inherit", "per_turn", false], diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts index 51313404fd8..6d78e7f00a9 100644 --- a/server/src/services/environment-execution-target.ts +++ b/server/src/services/environment-execution-target.ts @@ -416,6 +416,7 @@ export async function resolveEnvironmentExecutionTarget(input: { : parsed.config.runnerLifecycleMode === "per_turn" ? { mode: "per_turn", idleTimeoutMs: null } : null, + ...(typeof parsed.config.runnerIdleTimeoutMs === "number" ? { runnerIdleTimeoutMs: parsed.config.runnerIdleTimeoutMs } : {}), reusableLeaseConfigured: parsed.config.reuseLease === true, sandboxLeaseAcquisition: sandboxLeaseAcquisitionFromMetadata( input.lease?.metadata?.sandboxLeaseAcquisition, diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 2cc5c05d702..62a8310a482 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -37,6 +37,7 @@ import type { import { resolveRunnerEnvironmentForRun, resolveRunnerReconnectGraceMsForRun, + runnerLifecycleDefaultNeedsLeaseProbe, resolveWarmRunnerIdleTimeoutMsForRun, } from "./runner-environment-lifecycle.js"; import { environmentService } from "./environments.js"; @@ -313,8 +314,17 @@ export function environmentRunOrchestrator( localEnvironmentId: input.localEnvironmentId, }); + // The remote-sandbox warm default needs a provider that keeps reusable + // leases. Probe it only for a run the default can apply to; every other + // run resolves without it, exactly as before. + const reusableLeaseSupported = runnerLifecycleDefaultNeedsLeaseProbe( + selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, input.managedAiCredentialIdentity, + ) && await Promise.resolve(environmentRuntime.providerSupportsReusableLeases?.(selectedEnvironment)).then( + (supported) => supported === true, () => false); + const lifecycleDefaultContext = { reusableLeaseSupported, managedAiCredentialIdentity: input.managedAiCredentialIdentity }; const environment = resolveRunnerEnvironmentForRun( selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, input.managedAiCredentialIdentity, + lifecycleDefaultContext, ); if (input.recoverPluginTask && environment.driver === "plugin" && input.adapterType === "paperclip_runner") { @@ -333,7 +343,7 @@ export function environmentRunOrchestrator( // Step 2: Acquire lease const warmRunnerIdleTimeoutMs = resolveWarmRunnerIdleTimeoutMsForRun( - selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, + selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, lifecycleDefaultContext, ); const runnerReconnectGraceMs = input.adapterType ? resolveRunnerReconnectGraceMsForRun(selectedEnvironment, input.adapterType) diff --git a/server/src/services/environment-runtime-reusable-lease-probe.test.ts b/server/src/services/environment-runtime-reusable-lease-probe.test.ts new file mode 100644 index 00000000000..f8c5827c46b --- /dev/null +++ b/server/src/services/environment-runtime-reusable-lease-probe.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const resolveByKey = vi.hoisted(() => vi.fn()); +vi.mock("./plugin-environment-driver.js", async (importOriginal) => ({ + ...await importOriginal(), + resolvePluginSandboxProviderDriverByKey: resolveByKey, +})); + +const { environmentRuntimeService } = await import("./environment-runtime.js"); +const { REUSABLE_LEASE_WORKER_METHODS } = await import("./plugin-environment-driver.js"); + +function workerManager(supportedMethods: readonly string[]) { + return { getWorker: () => ({ supportedMethods }), isRunning: () => true } as never; +} +const sandbox = (provider: string) => ({ driver: "sandbox" as const, config: { provider } }); +const plugin = (driver: Record) => ({ plugin: { id: "plugin-1", status: "ready" }, driver: { driverKey: "daytona", kind: "sandbox_provider", ...driver } }); + +// The remote-sandbox warm default asks this before acquisition. It must match +// the acquisition rule, so the default never requests a lease it cannot reuse. +describe("sandbox provider reusable-lease probe", () => { + beforeEach(() => resolveByKey.mockReset()); + + it("reads false for the built-in fake provider, which has no reusable leases", async () => { + const runtime = environmentRuntimeService({} as never, { pluginWorkerManager: workerManager([]) }); + await expect(runtime.providerSupportsReusableLeases(sandbox("fake"))).resolves.toBe(false); + expect(resolveByKey).not.toHaveBeenCalled(); + }); + + it("reads true only for a running plugin that declares and verifies reusable leases", async () => { + const verified = environmentRuntimeService({} as never, { pluginWorkerManager: workerManager(REUSABLE_LEASE_WORKER_METHODS) }); + resolveByKey.mockResolvedValue(plugin({ sandboxCapabilities: { reusableLeases: true } })); + await expect(verified.providerSupportsReusableLeases(sandbox("daytona"))).resolves.toBe(true); + expect(resolveByKey).toHaveBeenCalledWith(expect.objectContaining({ driverKey: "daytona", requireRunning: true })); + + // A provider that does not declare reusable leases (for example E2B or Modal). + resolveByKey.mockResolvedValue(plugin({})); + await expect(verified.providerSupportsReusableLeases(sandbox("e2b"))).resolves.toBe(false); + + // Declared, but the worker does not verify the reuse methods: acquisition would use an ephemeral lease. + const unverified = environmentRuntimeService({} as never, { pluginWorkerManager: workerManager(["environmentExecute"]) }); + resolveByKey.mockResolvedValue(plugin({ sandboxCapabilities: { reusableLeases: true } })); + await expect(unverified.providerSupportsReusableLeases(sandbox("daytona"))).resolves.toBe(false); + + // No running worker yet: no default, and no wait for one. + resolveByKey.mockResolvedValue(null); + await expect(verified.providerSupportsReusableLeases(sandbox("daytona"))).resolves.toBe(false); + }); + + it("reads false for a non-sandbox environment", async () => { + const runtime = environmentRuntimeService({} as never, { pluginWorkerManager: workerManager(REUSABLE_LEASE_WORKER_METHODS) }); + await expect(runtime.providerSupportsReusableLeases({ driver: "local", config: {} })).resolves.toBe(false); + }); +}); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index c401e326606..fa4b8e0cf29 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -674,6 +674,13 @@ export interface EnvironmentRuntimeDriver { ): Promise; /** True when the lease's plugin worker advertises both sync verbs. */ supportsSync?(input: EnvironmentDriverLeaseInput): boolean; + /** + * True when a new lease for this environment can be reusable: the provider + * declares reusable leases and, for a plugin, its running worker verifies the + * reuse methods. It uses the acquisition path's rule without waiting for a + * worker; a worker that is not running yet reads false. + */ + providerSupportsReusableLeases?(environment: Pick): Promise; /** * Resolve the effective eight-field capability snapshot for this driver and * lease. Every driver implements this general method, so a caller never @@ -2706,6 +2713,31 @@ function createSandboxEnvironmentDriver( throw new Error("Sandbox driver does not support direct command execution for built-in providers."); }, + async providerSupportsReusableLeases(environment) { + let parsed: ReturnType; + try { + parsed = parseEnvironmentDriverConfig(environment as Environment); + } catch { + return false; // Acquisition reports the invalid config; the default stays off. + } + if (parsed.driver !== "sandbox") return false; + const provider = parsed.config.provider; + if (isBuiltinSandboxProvider(provider)) { + return resolveDeclaredSandboxCapabilities({ + supportsReusableLeases: getBuiltinSandboxProvider(provider)?.supportsReusableLeases, + }).reusableLeases === true; + } + if (!pluginWorkerManager) return false; + // Same rule as acquisition, without its worker-ready wait. + const running = await resolvePluginSandboxProviderDriverByKey({ + db, driverKey: provider, workerManager: pluginWorkerManager, requireRunning: true, + }); + if (!running) return false; + return resolveDeclaredSandboxCapabilities(running.driver).reusableLeases === true + && capabilityIsVerified("reusableLeases", + new Set(pluginWorkerManager.getWorker(running.plugin.id)?.supportedMethods ?? [])); + }, + supportsSync(input) { if (!input.lease.metadata?.sandboxProviderPlugin || !pluginWorkerManager) return false; const pluginId = readString(input.lease.metadata?.pluginId); @@ -4478,6 +4510,11 @@ export function environmentRuntimeService( return driver?.supportsSync?.(input) ?? false; }, + /** See {@link EnvironmentRuntimeDriver.providerSupportsReusableLeases}; false for a driver without it. */ + async providerSupportsReusableLeases(environment: Pick): Promise { + return await getDriver(environment.driver)?.providerSupportsReusableLeases?.(environment) ?? false; + }, + /** * Resolve the general per-lease capability snapshot through the driver's * {@link EnvironmentRuntimeDriver.resolveCapabilities}. Every registered diff --git a/server/src/services/heartbeat/native-lifecycle.test.ts b/server/src/services/heartbeat/native-lifecycle.test.ts index 412052c5747..86bc6ca244d 100644 --- a/server/src/services/heartbeat/native-lifecycle.test.ts +++ b/server/src/services/heartbeat/native-lifecycle.test.ts @@ -36,6 +36,8 @@ describe("admitted native lifecycle policy", () => { describe("default lifecycle for an agent that never chose one", () => { const defaultWarm: NativeLifecyclePolicy = { mode: "warm", idleTimeoutMs: 300_000 }; const unset: Input = { ...base, agentMode: undefined, agentIdleTimeoutMs: undefined, localTarget: true }; + const reusable = { reusableLease: true }; + const ephemeral = { reusableLease: false }; it.each<{ name: string; change: Partial; expected: NativeLifecyclePolicy }>([ { name: "local Codex keeps a warm runner for the default idle window", change: {}, expected: defaultWarm }, { name: "local OpenCode is warm", change: { backend: "opencode_server" }, expected: defaultWarm }, @@ -44,6 +46,26 @@ describe("default lifecycle for an agent that never chose one", () => { { name: "an explicit per-turn choice is honored", change: { agentMode: "per_turn" }, expected: perTurn }, { name: "an explicit warm idle window is honored", change: { agentMode: "warm", agentIdleTimeoutMs: 900_000 }, expected: warm }, { name: "a remote target stays per-turn", change: { localTarget: false }, expected: perTurn }, + // A remote sandbox follow-up otherwise gets a new sandbox and uploads the whole workspace. + { name: "remote-sandbox Codex on a reusable lease is warm", change: { localTarget: false, remoteSandboxTarget: reusable }, expected: defaultWarm }, + { name: "remote-sandbox OpenCode on a reusable lease is warm", change: { localTarget: false, remoteSandboxTarget: reusable, backend: "opencode_server" }, expected: defaultWarm }, + { name: "remote-sandbox Claude on a reusable lease is warm", change: { localTarget: false, remoteSandboxTarget: reusable, backend: "acpx_runtime", acpxAgent: "claude" }, expected: defaultWarm }, + // Rule 1: without a reusable lease (E2B, Modal, fake, or reuse unavailable) the run stays per-turn, as on master. + { name: "remote-sandbox Codex without a reusable lease stays per-turn", change: { localTarget: false, remoteSandboxTarget: ephemeral }, expected: perTurn }, + { name: "remote-sandbox Claude without a reusable lease stays per-turn", change: { localTarget: false, remoteSandboxTarget: ephemeral, backend: "acpx_runtime", acpxAgent: "claude" }, expected: perTurn }, + { name: "a remote-sandbox unmeasured ACP harness stays per-turn", change: { localTarget: false, remoteSandboxTarget: reusable, backend: "acpx_runtime", acpxAgent: "grok" }, expected: perTurn }, + // Rule 2: explicit configuration wins. + { name: "a remote-sandbox explicit per-turn choice is honored", change: { localTarget: false, remoteSandboxTarget: reusable, agentMode: "per_turn" }, expected: perTurn }, + { name: "a remote-sandbox default honors the agent's idle window", change: { localTarget: false, remoteSandboxTarget: reusable, agentIdleTimeoutMs: 900_000 }, expected: warm }, + // The same window the environment view forwards to the provider: the environment's, then the agent's. + { name: "a remote-sandbox default honors an inheriting environment's idle window", change: { localTarget: false, + remoteSandboxTarget: { reusableLease: true, idleTimeoutMs: 900_000 }, agentIdleTimeoutMs: 600_000 }, expected: warm }, + { name: "an explicit warm choice keeps the agent's idle window", change: { localTarget: false, agentMode: "warm", + remoteSandboxTarget: { reusableLease: true, idleTimeoutMs: 600_000 }, agentIdleTimeoutMs: 900_000 }, expected: warm }, + { name: "a remote-sandbox environment that pins per-turn wins", change: { localTarget: false, remoteSandboxTarget: reusable, environmentPolicy: perTurn }, expected: perTurn }, + { name: "a remote-sandbox session admitted per-turn stays per-turn", change: { localTarget: false, remoteSandboxTarget: reusable, persistedPolicy: perTurn }, expected: perTurn }, + { name: "remote-sandbox managed Claude stays per-turn", change: { localTarget: false, remoteSandboxTarget: reusable, backend: "acpx_runtime", acpxAgent: "claude", managedAiSelected: true }, expected: perTurn }, + { name: "remote-sandbox managed Codex, which supports a managed warm session, is warm", change: { localTarget: false, remoteSandboxTarget: reusable, managedAiSelected: true }, expected: defaultWarm }, { name: "an unknown target stays per-turn", change: { localTarget: undefined }, expected: perTurn }, { name: "an unmeasured ACP harness stays per-turn", change: { backend: "acpx_runtime", acpxAgent: "grok" }, expected: perTurn }, { name: "managed Claude without warm support stays per-turn", change: { backend: "acpx_runtime", acpxAgent: "claude", managedAiSelected: true }, expected: perTurn }, diff --git a/server/src/services/heartbeat/native-lifecycle.ts b/server/src/services/heartbeat/native-lifecycle.ts index 80d0222354e..21789901d25 100644 --- a/server/src/services/heartbeat/native-lifecycle.ts +++ b/server/src/services/heartbeat/native-lifecycle.ts @@ -1,20 +1,11 @@ import { PAPERCLIP_RUNNER_IDLE_TIMEOUT_DEFAULT_MS, resolvePaperclipRunnerIdleTimeoutMs } from "@paperclipai/adapter-utils"; import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; -import { supportsManagedNativeWarmSession } from "../native-runtime/native-warm-capability.js"; +import { + defaultsRemoteSandboxRunToWarm, defaultsToWarmFollowUp, supportsManagedNativeWarmSession, +} from "../native-runtime/native-warm-capability.js"; export type NativeLifecyclePolicy = NonNullable; -/** - * Harnesses measured to start a follow-up turn faster on a live runner - * (doc/plans/2026-11-07-warm-follow-up-ttft.md). An agent that never chose a - * lifecycle gets warm for these on a local target. ACPX harnesses (Claude) - * relaunch their provider process on every follow-up turn by design, so warm - * does not shorten their follow-up and they stay per-turn. - */ -function defaultsToWarmLocally(backend: string): boolean { - return backend === "codex_app_server" || backend === "opencode_server"; -} - export function resolveHeartbeatNativeLifecyclePolicy(input: { adapterType: string; backend: Parameters[0]["backend"]; @@ -28,19 +19,38 @@ export function resolveHeartbeatNativeLifecyclePolicy(input: { taskOwned: boolean; /** The run executes on this controller's host (no remote transport). */ localTarget?: boolean; + /** + * The run executes in a remote sandbox (not a plugin task). `reusableLease` + * is true only when the acquired lease is reusable: configured, and + * supported by the sandbox provider. `idleTimeoutMs` is the environment's + * configured window, which wins over the agent's for the default, as in the + * window the environment view forwards to the provider. + */ + remoteSandboxTarget?: { reusableLease: boolean; idleTimeoutMs?: number }; }): NativeLifecyclePolicy { - // An explicit choice wins. With none, a local measured harness keeps its - // runner and provider session for the default idle window so a follow-up - // turn skips the launch and session load. - const agentMode = input.agentMode === "warm" || input.agentMode === "per_turn" ? input.agentMode - : input.adapterType === "paperclip_runner" && input.localTarget === true && defaultsToWarmLocally(input.backend) - ? "warm" : "per_turn"; + // An explicit choice wins. With none, a measured harness keeps its runner and + // provider session between turns: on a local target, and on a remote sandbox + // whose provider keeps the lease (the rule the run's environment view uses). + const profile = { backend: input.backend, acpxAgent: input.acpxAgent }; + const explicit = input.agentMode === "warm" || input.agentMode === "per_turn"; + const localDefault = input.localTarget === true && defaultsToWarmFollowUp(profile, "local"); + const remoteDefault = input.remoteSandboxTarget !== undefined && defaultsRemoteSandboxRunToWarm({ + profile, + reusableLeaseSupported: input.remoteSandboxTarget.reusableLease, + managedAi: input.managedAiSelected ? { attribution: input.managedAiAttribution } : null, + }); + const agentMode = explicit ? input.agentMode + : input.adapterType === "paperclip_runner" && (localDefault || remoteDefault) ? "warm" : "per_turn"; + // An explicit warm choice honors the agent's idle window. The remote default + // uses the window the environment view forwards to the provider: the + // environment's, then the agent's. const agentPolicy: NativeLifecyclePolicy = agentMode === "warm" - ? { mode: "warm", idleTimeoutMs: input.agentMode === "warm" - ? resolvePaperclipRunnerIdleTimeoutMs(input.agentIdleTimeoutMs) : PAPERCLIP_RUNNER_IDLE_TIMEOUT_DEFAULT_MS } + ? { mode: "warm", idleTimeoutMs: explicit ? resolvePaperclipRunnerIdleTimeoutMs(input.agentIdleTimeoutMs) + : remoteDefault ? resolvePaperclipRunnerIdleTimeoutMs(input.remoteSandboxTarget?.idleTimeoutMs ?? input.agentIdleTimeoutMs) + : PAPERCLIP_RUNNER_IDLE_TIMEOUT_DEFAULT_MS } : { mode: "per_turn", idleTimeoutMs: null }; const supportsManagedWarm = input.adapterType === "paperclip_runner" && (input.taskOwned || - supportsManagedNativeWarmSession({ backend: input.backend, acpxAgent: input.acpxAgent }, input.managedAiAttribution)); + supportsManagedNativeWarmSession(profile, input.managedAiAttribution)); return input.persistedPolicy ?? (["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(input.backend) || input.managedAiSelected && !supportsManagedWarm ? { mode: "per_turn", idleTimeoutMs: null } diff --git a/server/src/services/heartbeat/runtime-selection.test.ts b/server/src/services/heartbeat/runtime-selection.test.ts index 381c4c329ab..0d8239399ec 100644 --- a/server/src/services/heartbeat/runtime-selection.test.ts +++ b/server/src/services/heartbeat/runtime-selection.test.ts @@ -26,6 +26,7 @@ import { resolveHeartbeatNativeRuntimeMode } from "../native-runtime/runtime-mod import { claimNativeRestartRecoveries } from "../native-runtime/native-restart-recovery.js"; import { buildNativeModelEnvelope, canonicalNativeRuntimeContextDigest } from "../../vendor/paperclip-runner/index.js"; import { prepareAndSelectHeartbeatRuntime, selectHeartbeatRuntime, type HeartbeatRuntimeSelectionInput } from "./runtime-selection.js"; +import { resolveRunnerEnvironmentForRun, resolveWarmRunnerIdleTimeoutMsForRun } from "../runner-environment-lifecycle.js"; import { reserveRunBudget } from "../budget-reservations.js"; import { legacyControllerClaim } from "../legacy-controller-lease.js"; import { agentInstructionWorkingCopyService } from "../agent-instruction-working-copies.js"; @@ -765,6 +766,67 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = expect(await coordinators(run.id)).toHaveLength(1); }); + // The remote-sandbox warm default follows the acquired lease: it never turns a + // run that works per-turn into a refused warm run. + it.each([ + ["a reusable lease", { reusableLeaseConfigured: true, effectiveCapabilities: { reusableLeases: true } }, "warm", "keep_running"], + ["reuse configured on a provider without reusable leases", { reusableLeaseConfigured: true, effectiveCapabilities: { reusableLeases: false } }, "per_turn", "destroy_after_turn"], + ["an ephemeral lease", {}, "per_turn", "destroy_after_turn"], + ] as const)("defaults an agent without a lifecycle on %s", async (_name, lease, lifecycleMode, sandboxResource) => { + const { input } = await fixture(); + const { lifecycleMode: _unset, ...adapterConfig } = (input.agent.adapterConfig ?? {}) as Record; + input.agent.adapterConfig = adapterConfig; + input.workspace.executionTarget = { ...sandbox, ...lease } as never; + input.beforeNativeRuntimeSelection = async () => { throw new Error("selection interrupted"); }; + await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("selection interrupted"); + expect(input.onNativeLifecycleSelected).toHaveBeenCalledWith({ provider: "codex", harness: "codex_app_server", lifecycleMode, sandboxResource }); + }); + + // The environment view (lease) and the native policy (runner) must make the + // same remote-sandbox decision for every harness, including a provider-less + // Codex agent and ACPX without an agent (Claude). + it.each([ + [{ provider: "codex", model: "gpt-5.6-luna" }, "warm"], + [{ model: "gpt-5.6-luna" }, "warm"], + [{ provider: "opencode", model: "openai/gpt-5" }, "warm"], + [{ provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6" }, "warm"], + [{ provider: "acpx", model: "claude-sonnet-4-6" }, "warm"], + [{ provider: "acpx", acpxAgent: "copilot", model: "gpt-5" }, "per_turn"], + ] as const)("agrees with the environment view's lease for %j", async (adapterConfig, mode) => { + const { input, run, agent, issue } = await fixture(); + input.agent.adapterConfig = adapterConfig; + input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, runtimeConfig: {}, adapterConfig, agent, issue, target: null, workspaceId: null }); + // An inheriting environment with its own idle window: the provider and the runner must use the same one. + const environment = { driver: "sandbox" as const, config: { provider: "daytona", runnerIdleTimeoutMs: 900_000 } as Record }; + const view = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", adapterConfig, undefined, undefined, { reusableLeaseSupported: true }); + expect(view.config.reuseLease === true).toBe(mode === "warm"); + const forwardedIdleMs = resolveWarmRunnerIdleTimeoutMsForRun(environment, "paperclip_runner", adapterConfig, undefined, { reusableLeaseSupported: true }); + input.workspace.executionTarget = { ...sandbox, reusableLeaseConfigured: view.config.reuseLease === true, effectiveCapabilities: { reusableLeases: true }, + runnerIdleTimeoutMs: 900_000 } as never; + const stop = new Error("policy captured"); + const policies: unknown[] = []; + await expect(prepareAndSelectHeartbeatRuntime(db, input, { + measure: async (_phase, work) => work(), + prepareInstructions: async (policy) => { policies.push(policy); throw stop; }, + })).rejects.toBe(stop); + expect(policies).toEqual([mode === "warm" ? { mode, idleTimeoutMs: forwardedIdleMs } : { mode, idleTimeoutMs: null }]); + expect(forwardedIdleMs).toBe(mode === "warm" ? 900_000 : undefined); + }); + + it("keeps a plugin task's sandbox target out of the remote default", async () => { + const { input } = await fixture(); + const { lifecycleMode: _unset, ...adapterConfig } = (input.agent.adapterConfig ?? {}) as Record; + input.agent.adapterConfig = adapterConfig; + input.workspace.executionTarget = { ...sandbox, reusableLeaseConfigured: true, effectiveCapabilities: { reusableLeases: true }, runnerTask: vi.fn() } as never; + const stop = new Error("policy captured"); + const policies: unknown[] = []; + await expect(prepareAndSelectHeartbeatRuntime(db, input, { + measure: async (_phase, work) => work(), + prepareInstructions: async (policy) => { policies.push(policy?.mode); throw stop; }, + })).rejects.toBe(stop); + expect(policies).toEqual(["per_turn"]); + }); + it("rejects a warm native sandbox that cannot reuse its lease", async () => { const { input, run } = await fixture(); input.workspace.executionTarget = { ...sandbox, runnerLifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 } }; diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index 6af0b2490f6..3e6655b2077 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -17,6 +17,7 @@ import { isConversation } from "../agent-conversations.js"; import { renewLegacyControllerLease } from "../legacy-controller-lease.js"; import type { prepareManagedAiRuntime } from "../ai-connection-runtime.js"; import { resolveHeartbeatNativeLifecyclePolicy, type NativeLifecyclePolicy } from "./native-lifecycle.js"; +import { resolveWarmProviderSelection } from "../native-runtime/native-warm-capability.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext, @@ -170,6 +171,14 @@ export interface NativeSandboxLifecycle { failoverBackup: "verified"; } +/** The acquired sandbox lease is reusable: configured, and supported by the provider. */ +function sandboxTargetHasReusableLease(target: { + reusableLeaseConfigured?: boolean; + effectiveCapabilities?: { reusableLeases: boolean } | null; +}): boolean { + return target.reusableLeaseConfigured === true && target.effectiveCapabilities?.reusableLeases === true; +} + export function resolveReusableSandboxLifecycle(input: { lifecyclePolicy: | { mode: "per_turn"; idleTimeoutMs: null } @@ -184,9 +193,7 @@ export function resolveReusableSandboxLifecycle(input: { if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") { return null; } - const reusableLease = - input.target.reusableLeaseConfigured === true && - input.target.effectiveCapabilities?.reusableLeases === true; + const reusableLease = sandboxTargetHasReusableLease(input.target); if (input.lifecyclePolicy.mode === "warm" && !reusableLease) { throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease"); } @@ -262,15 +269,26 @@ function readNonEmptyString(value: unknown): string | null { function resolveRuntimeLifecyclePolicy(input: HeartbeatRuntimeSelectionInput): NativeLifecyclePolicy | null { if (input.nativeRuntimeResolution.kind !== "native") return null; + const target = input.workspace.executionTarget; + const sandboxTarget = target?.kind === "remote" && target.transport === "sandbox" ? target : null; + const backend = input.nativeRuntimeResolution.profile.backend; + const rawAcpxAgent = parseObject(input.agent.adapterConfig).acpxAgent; + // The environment view resolves the harness through the runtime's resolver + // (ACPX without an agent is Claude); use the same harness here so the lease + // and the runner decisions agree. + const resolved = backend === "acpx_runtime" && rawAcpxAgent === undefined + ? resolveWarmProviderSelection(input.agent.adapterConfig, input.agent.companyId) : null; return resolveHeartbeatNativeLifecyclePolicy({ - adapterType: input.agent.adapterType, backend: input.nativeRuntimeResolution.profile.backend, - acpxAgent: parseObject(input.agent.adapterConfig).acpxAgent, + adapterType: input.agent.adapterType, backend, + acpxAgent: resolved?.backend === backend ? resolved.acpxAgent : rawAcpxAgent, agentMode: parseObject(input.agent.adapterConfig).lifecycleMode, agentIdleTimeoutMs: parseObject(input.agent.adapterConfig).idleTimeoutMs, - environmentPolicy: input.workspace.executionTarget?.kind === "remote" && input.workspace.executionTarget.transport === "sandbox" ? input.workspace.executionTarget.runnerLifecyclePolicy : null, + environmentPolicy: sandboxTarget ? sandboxTarget.runnerLifecyclePolicy : null, persistedPolicy: input.session.persistedNativeExecutionInput?.session.lifecyclePolicy, managedAiSelected: Boolean(input.config.managedAiRuntime), managedAiAttribution: input.config.managedAiRuntime?.attribution, - taskOwned: Boolean(input.workspace.executionTarget?.kind === "remote" && input.workspace.executionTarget.transport === "sandbox" && input.workspace.executionTarget.runnerTask), - localTarget: input.workspace.executionTarget?.kind !== "remote", + taskOwned: Boolean(sandboxTarget?.runnerTask), + localTarget: target?.kind !== "remote", + remoteSandboxTarget: sandboxTarget && !sandboxTarget.runnerTask + ? { reusableLease: sandboxTargetHasReusableLease(sandboxTarget), idleTimeoutMs: sandboxTarget.runnerIdleTimeoutMs } : undefined, }); } diff --git a/server/src/services/native-runtime/native-warm-capability.ts b/server/src/services/native-runtime/native-warm-capability.ts index 48d015698ca..9c1408976af 100644 --- a/server/src/services/native-runtime/native-warm-capability.ts +++ b/server/src/services/native-runtime/native-warm-capability.ts @@ -1,7 +1,21 @@ -import type { PaperclipRunnerProviderProfile } from "./provider-profile.js"; +import { resolvePaperclipRunnerProviderProfile, type PaperclipRunnerProviderProfile } from "./provider-profile.js"; type WarmProviderSelection = { backend: PaperclipRunnerProviderProfile["backend"]; acpxAgent?: unknown }; +/** + * The harness an agent config selects, resolved exactly as the runtime resolves + * it (an omitted provider is Codex; ACPX without an agent is Claude). Null when + * the runtime would refuse the config. + */ +export function resolveWarmProviderSelection(adapterConfig: unknown, companyId?: string): WarmProviderSelection | null { + try { + const profile = resolvePaperclipRunnerProviderProfile(adapterConfig, { companyId }); + return { backend: profile.backend, acpxAgent: profile.provider === "acpx" ? profile.acpxAgent : undefined }; + } catch { + return null; + } +} + /** Providers with a session-owned home and the current instruction-grant handoff. */ export function supportsNativeWarmFiles(profile: WarmProviderSelection): boolean { return profile.backend === "codex_app_server" || @@ -18,3 +32,35 @@ export function supportsManagedNativeWarmSession( (profile.backend === "acpx_runtime" && profile.acpxAgent === "copilot" && attribution?.provider === "github" && attribution.method === "api_key"); } + +/** + * Harnesses measured to start a follow-up turn faster on a live runner + * (doc/plans/2026-11-07-warm-follow-up-ttft.md, doc/plans/2026-11-08-large-workspace-speed.md). + * Locally, ACPX harnesses (Claude) relaunch their provider process on every + * follow-up turn, so warm does not shorten their follow-up. On a remote sandbox + * the dominant cost is a new sandbox and a full workspace upload, so Claude + * gains from warm there too. + */ +export function defaultsToWarmFollowUp(profile: WarmProviderSelection, target: "local" | "remote_sandbox"): boolean { + if (profile.backend === "codex_app_server" || profile.backend === "opencode_server") return true; + return target === "remote_sandbox" && profile.backend === "acpx_runtime" && profile.acpxAgent === "claude"; +} + +/** + * The one rule for the remote-sandbox warm default. The run-scoped environment + * view (lease retention) and the native session policy (runner retention) both + * call it, and apply it only when no lifecycle was chosen anywhere (admitted + * session, environment, agent). It holds only when the sandbox provider keeps + * reusable leases, so a default never turns a run that works per-turn into a + * refused warm run, and only when a selected managed AI account supports a + * warm session. + */ +export function defaultsRemoteSandboxRunToWarm(input: { + profile: WarmProviderSelection; + reusableLeaseSupported: boolean; + managedAi: { attribution?: { provider: string; method: string } } | null; +}): boolean { + return input.reusableLeaseSupported + && defaultsToWarmFollowUp(input.profile, "remote_sandbox") + && (input.managedAi === null || supportsManagedNativeWarmSession(input.profile, input.managedAi.attribution)); +} diff --git a/server/src/services/runner-environment-lifecycle.test.ts b/server/src/services/runner-environment-lifecycle.test.ts index 14836ad64d9..aa8b011a731 100644 --- a/server/src/services/runner-environment-lifecycle.test.ts +++ b/server/src/services/runner-environment-lifecycle.test.ts @@ -1,5 +1,5 @@ -import {expect,it} from "vitest"; -import {resolveRunnerEnvironmentForRun,resolveWarmRunnerIdleTimeoutMsForRun} from "./runner-environment-lifecycle.js"; +import {describe,expect,it} from "vitest"; +import {resolveRunnerEnvironmentForRun,resolveWarmRunnerIdleTimeoutMsForRun,runnerLifecycleDefaultNeedsLeaseProbe} from "./runner-environment-lifecycle.js"; it.each([undefined,"codex","acpx","opencode"])("defaults plugin Runner execution to warm without provider gates: %s",provider=>{ const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{}}}; @@ -30,3 +30,66 @@ it("forwards the run's warm idle window for a warm sandbox-provider run only",() expect(resolveWarmRunnerIdleTimeoutMsForRun(sandbox,"claude_local",{lifecycleMode:"warm"})).toBeUndefined(); expect(resolveWarmRunnerIdleTimeoutMsForRun({driver:"plugin" as const,config:{}},"paperclip_runner",{lifecycleMode:"warm"})).toBeUndefined(); }); + +describe("remote-sandbox warm default", () => { + const inheriting = { driver: "sandbox" as const, config: { provider: "daytona", reuseLease: false } }; + const supported = { reusableLeaseSupported: true }; + const opencode = { provider: "opencode", model: "openai/gpt-5" }; + const claude = { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6" }; + + it.each([ + ["Codex", { provider: "codex" }, true], + // Rule 3: an omitted provider resolves to Codex, as the runtime resolves it. + ["an omitted provider (Codex)", {}, true], + ["OpenCode", opencode, true], + ["ACPX Claude", claude, true], + ["an unmeasured ACPX harness", { provider: "acpx", acpxAgent: "grok", model: "grok-4" }, false], + ["a config the runtime refuses", { provider: "opencode" }, false], + ] as const)("%s on a provider with reusable leases: warm=%s", (_name, adapterConfig, warm) => { + const resolved = resolveRunnerEnvironmentForRun(inheriting, "paperclip_runner", adapterConfig, undefined, undefined, supported); + expect(resolved.config.reuseLease).toBe(warm); + expect(resolveWarmRunnerIdleTimeoutMsForRun(inheriting, "paperclip_runner", adapterConfig, undefined, supported)) + .toBe(warm ? 300_000 : undefined); + expect(runnerLifecycleDefaultNeedsLeaseProbe(inheriting, "paperclip_runner", adapterConfig)).toBe(warm); + // The shared environment itself is never rewritten. + expect(inheriting.config.reuseLease).toBe(false); + }); + + // Rule 1: a provider without reusable leases (E2B, Modal, fake) keeps master's per-turn behavior. + it.each([{ provider: "codex" }, {}, claude])("keeps %j per-turn on a provider without reusable leases", (adapterConfig) => { + for (const context of [{}, { reusableLeaseSupported: false }]) { + expect(resolveRunnerEnvironmentForRun(inheriting, "paperclip_runner", adapterConfig, undefined, undefined, context)).toBe(inheriting); + expect(resolveWarmRunnerIdleTimeoutMsForRun(inheriting, "paperclip_runner", adapterConfig, undefined, context)).toBeUndefined(); + } + }); + + // Rule 2: explicit configuration wins and needs no probe. + it("honors explicit lifecycles and the configured idle window", () => { + const pinned = { driver: "sandbox" as const, config: { provider: "daytona", runnerLifecycleMode: "per_turn" } }; + expect(resolveRunnerEnvironmentForRun(pinned, "paperclip_runner", { provider: "codex" }, undefined, undefined, supported)).toBe(pinned); + expect(runnerLifecycleDefaultNeedsLeaseProbe(pinned, "paperclip_runner", { provider: "codex" })).toBe(false); + expect(resolveRunnerEnvironmentForRun(inheriting, "paperclip_runner", { provider: "codex", lifecycleMode: "per_turn" }, undefined, undefined, supported)).toBe(inheriting); + expect(runnerLifecycleDefaultNeedsLeaseProbe(inheriting, "paperclip_runner", { lifecycleMode: "per_turn" })).toBe(false); + expect(resolveRunnerEnvironmentForRun(inheriting, "paperclip_runner", { provider: "codex" }, "per_turn", undefined, supported)).toBe(inheriting); + expect(runnerLifecycleDefaultNeedsLeaseProbe(inheriting, "paperclip_runner", {}, "per_turn")).toBe(false); + expect(resolveWarmRunnerIdleTimeoutMsForRun(inheriting, "paperclip_runner", { idleTimeoutMs: 900_000 }, undefined, supported)).toBe(900_000); + expect(resolveWarmRunnerIdleTimeoutMsForRun({ ...inheriting, config: { ...inheriting.config, runnerIdleTimeoutMs: 600_000 } }, + "paperclip_runner", { idleTimeoutMs: 900_000 }, undefined, supported)).toBe(600_000); + }); + + // Rule 2: a managed AI account without a warm session gets no default; managed Codex supports one. + it("defaults managed AI runs only when the harness supports a managed warm session", () => { + expect(resolveRunnerEnvironmentForRun(inheriting, "paperclip_runner", claude, undefined, "account:generation", supported)).toBe(inheriting); + expect(runnerLifecycleDefaultNeedsLeaseProbe(inheriting, "paperclip_runner", claude, undefined, "account:generation")).toBe(false); + expect(resolveWarmRunnerIdleTimeoutMsForRun(inheriting, "paperclip_runner", claude, undefined, + { ...supported, managedAiCredentialIdentity: "account:generation" })).toBeUndefined(); + expect(resolveRunnerEnvironmentForRun(inheriting, "paperclip_runner", {}, undefined, "account:generation", supported).config.reuseLease).toBe(true); + }); + + it("never applies to plugin tasks or other adapters", () => { + expect(runnerLifecycleDefaultNeedsLeaseProbe({ driver: "plugin", config: {} }, "paperclip_runner", {})).toBe(false); + expect(runnerLifecycleDefaultNeedsLeaseProbe(inheriting, "codex_local", {})).toBe(false); + expect(resolveRunnerEnvironmentForRun(inheriting, "codex_local", {}, undefined, undefined, supported)).toBe(inheriting); + }); +}); + diff --git a/server/src/services/runner-environment-lifecycle.ts b/server/src/services/runner-environment-lifecycle.ts index 16b174d8afe..30b63b9cb48 100644 --- a/server/src/services/runner-environment-lifecycle.ts +++ b/server/src/services/runner-environment-lifecycle.ts @@ -1,15 +1,75 @@ import { resolvePaperclipRunnerIdleTimeoutMs } from "@paperclipai/adapter-utils"; import type { Environment } from "@paperclipai/shared"; import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "./native-runtime/native-runner-recovery-contract.js"; +import { defaultsRemoteSandboxRunToWarm, resolveWarmProviderSelection } from "./native-runtime/native-warm-capability.js"; -function resolveRunnerLifecycleModeForRun( +/** Facts the remote-sandbox warm default needs beyond the configuration. */ +export interface RunnerLifecycleDefaultContext { + /** The sandbox provider keeps reusable leases (probed before acquisition). */ + reusableLeaseSupported?: boolean; + /** The run uses a managed AI account. */ + managedAiCredentialIdentity?: string; +} + +function explicitLifecycleMode( config: Record, adapterConfig: Record, admittedLifecycleMode?: "warm" | "per_turn", +): "warm" | "per_turn" | undefined { + if (admittedLifecycleMode) return admittedLifecycleMode; + if (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn") return config.runnerLifecycleMode; + if (adapterConfig.lifecycleMode === "warm" || adapterConfig.lifecycleMode === "per_turn") return adapterConfig.lifecycleMode; + return undefined; +} + +/** + * The remote-sandbox warm default for a run whose lifecycle nobody chose, + * through the same rule as the native session policy. The harness resolves + * exactly as the runtime resolves it (an omitted provider is Codex); a config + * the runtime would refuse gets no default here. + */ +function sandboxRunDefaultsToWarm( + adapterConfig: Record, + context: RunnerLifecycleDefaultContext, +): boolean { + const profile = resolveWarmProviderSelection(adapterConfig); + if (!profile) return false; + return defaultsRemoteSandboxRunToWarm({ + profile, + reusableLeaseSupported: context.reusableLeaseSupported === true, + // The attribution is not known here; a harness that needs it for a managed + // warm session (Copilot) is not a measured default, so this is exact. + managedAi: context.managedAiCredentialIdentity ? {} : null, + }); +} + +function resolveRunnerLifecycleModeForRun( + environment: Pick, + adapterConfig: Record, + admittedLifecycleMode: "warm" | "per_turn" | undefined, + context: RunnerLifecycleDefaultContext, ): unknown { - return admittedLifecycleMode ?? (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn" - ? config.runnerLifecycleMode - : adapterConfig.lifecycleMode); + const config = environment.config ?? {}; + const explicit = explicitLifecycleMode(config, adapterConfig, admittedLifecycleMode); + if (explicit) return explicit; + if (environment.driver === "sandbox" && sandboxRunDefaultsToWarm(adapterConfig, context)) return "warm"; + return adapterConfig.lifecycleMode; +} + +/** + * True when only the provider's reusable-lease support is left to decide this + * run's sandbox lifecycle, so the caller probes it only for these runs. + */ +export function runnerLifecycleDefaultNeedsLeaseProbe( + environment: Pick, + adapterType: string, + adapterConfig: Record = {}, + admittedLifecycleMode?: "warm" | "per_turn", + managedAiCredentialIdentity?: string, +): boolean { + if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") return false; + if (explicitLifecycleMode(environment.config ?? {}, adapterConfig, admittedLifecycleMode)) return false; + return sandboxRunDefaultsToWarm(adapterConfig, { reusableLeaseSupported: true, managedAiCredentialIdentity }); } /** Resolve runner retention before acquiring a sandbox, not after it is ephemeral. @@ -22,10 +82,12 @@ export function resolveRunnerEnvironmentForRun = {}, admittedLifecycleMode?: "warm" | "per_turn", managedAiCredentialIdentity?: string, + defaultContext: Pick = {}, ): T { if (adapterType !== "paperclip_runner" || !["sandbox", "plugin"].includes(environment.driver)) return environment; const config = environment.config ?? {}; - const lifecycleMode = resolveRunnerLifecycleModeForRun(config, adapterConfig, admittedLifecycleMode); + const lifecycleMode = resolveRunnerLifecycleModeForRun(environment, adapterConfig, admittedLifecycleMode, + { ...defaultContext, managedAiCredentialIdentity }); // Warm tasks need a managed credential identity; recovery keeps its admitted policy. if (environment.driver === "plugin") return { ...environment, config: { ...config, runnerLifecycleMode: admittedLifecycleMode ?? (managedAiCredentialIdentity && lifecycleMode !== "per_turn" ? "warm" : "per_turn"), @@ -53,10 +115,11 @@ export function resolveWarmRunnerIdleTimeoutMsForRun( adapterType: string, adapterConfig: Record = {}, admittedLifecycleMode?: "warm" | "per_turn", + defaultContext: RunnerLifecycleDefaultContext = {}, ): number | undefined { if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") return undefined; const config = environment.config ?? {}; - if (resolveRunnerLifecycleModeForRun(config, adapterConfig, admittedLifecycleMode) !== "warm") return undefined; + if (resolveRunnerLifecycleModeForRun(environment, adapterConfig, admittedLifecycleMode, defaultContext) !== "warm") return undefined; return resolvePaperclipRunnerIdleTimeoutMs(config.runnerIdleTimeoutMs ?? adapterConfig.idleTimeoutMs); } diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index b55c309adde..5cdcb40f39b 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -265,8 +265,8 @@ describe("Daytona options", () => { const wave = (...extra: string[]) => parseCommand(["--env", "daytona", "--daytona-image", image, "--tasks", "20", "--concurrency", "4", ...extra]); expect(() => wave("--daytona-warm")).toThrow(/--daytona-warm keeps each task's sandbox.*20 tasks/); expect(() => wave("--daytona-inherit", "--lifecycle", "warm")).toThrow(/--daytona-inherit with --lifecycle warm keeps each task's sandbox/); - // --lifecycle default leaves the lifecycle unset everywhere, and a remote run then stays per-turn. - expect(wave("--daytona-inherit", "--lifecycle", "default")).toMatchObject({ mode: "wave" }); + // --lifecycle default leaves the lifecycle unset everywhere; Daytona keeps reusable leases, so the run defaults to warm. + expect(() => wave("--daytona-inherit", "--lifecycle", "default")).toThrow(/--daytona-inherit with --lifecycle default keeps each task's sandbox/); // The catalog agents pin per-turn, so an inheriting environment alone releases each sandbox after its turn. expect(wave("--daytona-inherit")).toMatchObject({ mode: "wave", options: { daytonaInherit: true, tasks: 20 } }); expect(wave("--daytona-inherit", "--lifecycle", "per_turn")).toMatchObject({ mode: "wave" }); diff --git a/tests/restart-chaos/ttft-baseline-stats.test.ts b/tests/restart-chaos/ttft-baseline-stats.test.ts index c999d46b833..e2d7d7bdbc2 100644 --- a/tests/restart-chaos/ttft-baseline-stats.test.ts +++ b/tests/restart-chaos/ttft-baseline-stats.test.ts @@ -99,7 +99,7 @@ describe("ttft baseline statistics", () => { expect(daytonaRetainsSandboxes({ warmEnvironment: true, inheritEnvironment: false, lifecycle: undefined })).toBe(true); expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: undefined })).toBe(false); expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "per_turn" })).toBe(false); - expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "default" })).toBe(false); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "default" })).toBe(true); expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "warm" })).toBe(true); expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: false, lifecycle: "warm" })).toBe(false); }); diff --git a/tests/restart-chaos/ttft-baseline-stats.ts b/tests/restart-chaos/ttft-baseline-stats.ts index df6a3e76f0d..335f4879f99 100644 --- a/tests/restart-chaos/ttft-baseline-stats.ts +++ b/tests/restart-chaos/ttft-baseline-stats.ts @@ -314,11 +314,12 @@ export function openShellExecutions(events: readonly BaselineEvent[], marker?: s * Whether finished tasks keep their Daytona sandbox for the idle window. The warm * fixture always does. An inheriting environment defers to the agent, whose catalog * profile pins per_turn unless --lifecycle warm replaces it. --lifecycle default - * leaves the agent unset, and a remote run with no lifecycle anywhere is per-turn. + * leaves the agent unset; on Daytona, which keeps reusable leases, a measured + * harness with no lifecycle anywhere then defaults to warm. */ export function daytonaRetainsSandboxes(input: { warmEnvironment: boolean; inheritEnvironment: boolean; lifecycle: string | undefined }): boolean { if (input.warmEnvironment) return true; - return input.inheritEnvironment && input.lifecycle === "warm"; + return input.inheritEnvironment && (input.lifecycle === "warm" || input.lifecycle === "default"); } /** Daytona sandboxes a TTFT baseline can hold at once, for the vCPU budget check. */