diff --git a/doc/plans/2026-11-08-large-workspace-speed.md b/doc/plans/2026-11-08-large-workspace-speed.md new file mode 100644 index 00000000000..53cbac3377d --- /dev/null +++ b/doc/plans/2026-11-08-large-workspace-speed.md @@ -0,0 +1,58 @@ +# Speed with a paperclip-sized workspace + +Status: measured 2026-11-08. Follows [`2026-11-07-warm-follow-up-ttft.md`](2026-11-07-warm-follow-up-ttft.md). + +## Summary + +- **Locally, follow-ups are fast and first turns are not.** A first turn takes 31–34 s to first token in a fresh paperclip clone, mostly the repository's own `provision-worktree.sh` (about 20 s). Follow-ups take 7 s. +- **On Daytona, every follow-up turn re-uploaded the whole workspace.** With a default environment a follow-up took 30 s, because each turn got a new sandbox and synced the repository into it again (13–22 s), then started a new runner (8–9 s). +- **A reusable warm sandbox removes that wait.** The same follow-up takes 8.1 s (Codex) and 10.3 s (Claude). An agent that never chose a lifecycle, in an environment that inherits it, now runs Codex, OpenCode and Claude warm on a sandbox. +- **The wait was invisible in the run log.** It sat inside one opaque `heartbeat.prepare_after_environment` span. Named preparation phases are now run-log spans too. + +## Method + +- Driver: [`tests/restart-chaos/ttft-baseline.ts`](../../tests/restart-chaos/ttft-baseline.ts). + - `--workspace-repo` gives a fresh clone of paperclipai/paperclip at origin/master (14.8k files, 484 MB, no `node_modules`) as the project's primary workspace, with git_worktree isolation. The repository's `scripts/provision-worktree.sh` runs automatically, as for a real user. + - The `postfault` phase is the first follow-up after an idle K1 restart. + - `--env daytona` (and `--daytona-warm` for the reusable warm fixture) uses the signed Runner image built from master c8822ffd4b. +- One sample per cell, Codex and Claude. The figures are comment-to-first-token (`task.time_to_first_token`) from the run log only. +- Every Daytona sandbox the runs created was cleaned up. + +## Results (time to first token) + +| | Local | Daytona, per-turn sandbox | Daytona, warm reusable sandbox | +|---|---|---|---| +| First turn | 31–34 s | 58–63 s | 45–51 s | +| Follow-up | 7 s | 30 s | 8.1 s (Codex), 10.3 s (Claude) | +| Follow-up after an idle K1 | 6.8–8.2 s | 38–46 s | 20.8 s (Claude) | + +## Where a remote follow-up spent its time (Codex, per-turn sandbox) + +| Step | Time | +|---|---| +| `workspace_sync.transfer`: the whole workspace into the new sandbox | 13.3 s | +| `runner.session.resume`: a new runner for the turn | 8–9 s | +| Environment acquire, workspace binding, instructions | about 4 s | +| `workspace_finalize` after the turn (copyback; not before the first token) | 8–10 s | + +Run-log reuse records show `environment: fresh/created` on every follow-up. + +## Largest waits, ranked + +1. **A full workspace upload on every remote turn** (13–22 s). Fixed: warm keeps the sandbox, and the workspace is adopted after a stamp check. +2. **A new runner on every remote turn** (8–9 s). Fixed by the same change. +3. **First-turn provision on a large repository** (about 20 s locally). The repository's own script; next step: a pre-provisioned worktree pool keyed by base commit and lockfile hash. +4. **Copyback after each remote turn** (8–10 s). It is after the reply, not before the first token. +5. **First-turn seed build** (git clone and pack 4.5 s, seed verify and publish 3.7 s). + +## The default change + +- **Scope.** A sandbox environment that inherits the lifecycle (`runnerLifecycleMode` unset), and an agent with no stored lifecycle. Covers Codex and OpenCode, as locally, plus Claude: on a sandbox the upload, not the provider relaunch, dominates its follow-up. +- **Mechanics.** Warm on a sandbox already turns on `reuseLease`. The follow-up keeps the live sandbox and runner and adopts the workspace. +- **What does not change.** Explicit choices, environments that pin a lifecycle, plugin-task environments and admitted sessions keep their policy. +- **Cost.** An idle sandbox stays up for the warm idle window (5 minutes). After that the controller suspends the runner, and the provider auto-stop follows the same window. + +## Gaps + +- One sample per cell on a shared host. Daytona's warm path still needs a K1, K2 and K6 chaos wave before the default ships widely. +- In the warm run, the Codex post-restart sample finished but produced no report row. diff --git a/server/src/services/heartbeat/native-lifecycle.test.ts b/server/src/services/heartbeat/native-lifecycle.test.ts index 412052c5747..f8b0b28b1ff 100644 --- a/server/src/services/heartbeat/native-lifecycle.test.ts +++ b/server/src/services/heartbeat/native-lifecycle.test.ts @@ -44,6 +44,14 @@ describe("default lifecycle for an agent that never chose one", () => { { name: "an explicit per-turn choice is honored", change: { agentMode: "per_turn" }, expected: perTurn }, { name: "an explicit warm idle window is honored", change: { agentMode: "warm", agentIdleTimeoutMs: 900_000 }, expected: warm }, { name: "a remote target stays per-turn", change: { localTarget: false }, expected: perTurn }, + // A remote sandbox follow-up otherwise gets a new sandbox and uploads the whole workspace. + { name: "remote-sandbox Codex is warm", change: { localTarget: false, remoteSandboxTarget: true }, expected: defaultWarm }, + { name: "remote-sandbox OpenCode is warm", change: { localTarget: false, remoteSandboxTarget: true, backend: "opencode_server" }, expected: defaultWarm }, + { name: "remote-sandbox Claude is warm", change: { localTarget: false, remoteSandboxTarget: true, backend: "acpx_runtime", acpxAgent: "claude" }, expected: defaultWarm }, + { name: "a remote-sandbox unmeasured ACP harness stays per-turn", change: { localTarget: false, remoteSandboxTarget: true, backend: "acpx_runtime", acpxAgent: "grok" }, expected: perTurn }, + { name: "a remote-sandbox explicit per-turn choice is honored", change: { localTarget: false, remoteSandboxTarget: true, agentMode: "per_turn" }, expected: perTurn }, + { name: "a remote-sandbox environment that pins per-turn wins", change: { localTarget: false, remoteSandboxTarget: true, environmentPolicy: perTurn }, expected: perTurn }, + { name: "a remote-sandbox session admitted per-turn stays per-turn", change: { localTarget: false, remoteSandboxTarget: true, persistedPolicy: perTurn }, expected: perTurn }, { name: "an unknown target stays per-turn", change: { localTarget: undefined }, expected: perTurn }, { name: "an unmeasured ACP harness stays per-turn", change: { backend: "acpx_runtime", acpxAgent: "grok" }, expected: perTurn }, { name: "managed Claude without warm support stays per-turn", change: { backend: "acpx_runtime", acpxAgent: "claude", managedAiSelected: true }, expected: perTurn }, diff --git a/server/src/services/heartbeat/native-lifecycle.ts b/server/src/services/heartbeat/native-lifecycle.ts index 80d0222354e..0610c9111b6 100644 --- a/server/src/services/heartbeat/native-lifecycle.ts +++ b/server/src/services/heartbeat/native-lifecycle.ts @@ -15,6 +15,15 @@ function defaultsToWarmLocally(backend: string): boolean { return backend === "codex_app_server" || backend === "opencode_server"; } +/** + * On a remote sandbox the follow-up's dominant cost is a new sandbox and a full + * workspace upload (doc/plans/2026-11-08-large-workspace-speed.md), so Claude + * gains from warm there too: its provider relaunch is small next to the upload. + */ +export function defaultsToWarmOnRemoteSandbox(backend: string, acpxAgent: unknown): boolean { + return defaultsToWarmLocally(backend) || (backend === "acpx_runtime" && acpxAgent === "claude"); +} + export function resolveHeartbeatNativeLifecyclePolicy(input: { adapterType: string; backend: Parameters[0]["backend"]; @@ -28,12 +37,15 @@ export function resolveHeartbeatNativeLifecyclePolicy(input: { taskOwned: boolean; /** The run executes on this controller's host (no remote transport). */ localTarget?: boolean; + /** The run executes in a remote sandbox (not a plugin task). */ + remoteSandboxTarget?: boolean; }): NativeLifecyclePolicy { // An explicit choice wins. With none, a local measured harness keeps its // runner and provider session for the default idle window so a follow-up // turn skips the launch and session load. const agentMode = input.agentMode === "warm" || input.agentMode === "per_turn" ? input.agentMode - : input.adapterType === "paperclip_runner" && input.localTarget === true && defaultsToWarmLocally(input.backend) + : input.adapterType === "paperclip_runner" && ((input.localTarget === true && defaultsToWarmLocally(input.backend)) + || (input.remoteSandboxTarget === true && defaultsToWarmOnRemoteSandbox(input.backend, input.acpxAgent))) ? "warm" : "per_turn"; const agentPolicy: NativeLifecyclePolicy = agentMode === "warm" ? { mode: "warm", idleTimeoutMs: input.agentMode === "warm" diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index da5dc8269b9..d562140fd64 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -271,6 +271,8 @@ function resolveRuntimeLifecyclePolicy(input: HeartbeatRuntimeSelectionInput): N managedAiSelected: Boolean(input.config.managedAiRuntime), managedAiAttribution: input.config.managedAiRuntime?.attribution, taskOwned: Boolean(input.workspace.executionTarget?.kind === "remote" && input.workspace.executionTarget.transport === "sandbox" && input.workspace.executionTarget.runnerTask), localTarget: input.workspace.executionTarget?.kind !== "remote", + remoteSandboxTarget: input.workspace.executionTarget?.kind === "remote" && input.workspace.executionTarget.transport === "sandbox" + && !input.workspace.executionTarget.runnerTask, }); } diff --git a/server/src/services/runner-environment-lifecycle.test.ts b/server/src/services/runner-environment-lifecycle.test.ts index 14836ad64d9..eb6dbe0e396 100644 --- a/server/src/services/runner-environment-lifecycle.test.ts +++ b/server/src/services/runner-environment-lifecycle.test.ts @@ -30,3 +30,21 @@ it("forwards the run's warm idle window for a warm sandbox-provider run only",() expect(resolveWarmRunnerIdleTimeoutMsForRun(sandbox,"claude_local",{lifecycleMode:"warm"})).toBeUndefined(); expect(resolveWarmRunnerIdleTimeoutMsForRun({driver:"plugin" as const,config:{}},"paperclip_runner",{lifecycleMode:"warm"})).toBeUndefined(); }); + +it.each([ + [{ provider: "codex" }, true], [{ provider: "opencode" }, true], [{ provider: "acpx", acpxAgent: "claude" }, true], + [{ provider: "acpx", acpxAgent: "grok" }, false], [{ provider: "codex", lifecycleMode: "per_turn" }, false], [{}, false], +] as const)("defaults a measured harness on an inheriting sandbox environment to a reusable warm sandbox: %j", (adapterConfig, warm) => { + const sandbox = { driver: "sandbox" as const, config: { provider: "daytona", reuseLease: false } }; + const resolved = resolveRunnerEnvironmentForRun(sandbox, "paperclip_runner", adapterConfig); + expect(resolved.config.reuseLease).toBe(warm); + expect(resolveWarmRunnerIdleTimeoutMsForRun(sandbox, "paperclip_runner", adapterConfig)).toBe(warm ? 300_000 : undefined); + // The shared environment itself is never rewritten. + expect(sandbox.config.reuseLease).toBe(false); +}); +it("keeps an environment that pins per-turn, and an admitted per-turn recovery, per-turn", () => { + const pinned = { driver: "sandbox" as const, config: { provider: "daytona", runnerLifecycleMode: "per_turn" } }; + expect(resolveRunnerEnvironmentForRun(pinned, "paperclip_runner", { provider: "codex" })).toBe(pinned); + const inheriting = { driver: "sandbox" as const, config: { provider: "daytona" } }; + expect(resolveRunnerEnvironmentForRun(inheriting, "paperclip_runner", { provider: "codex" }, "per_turn")).toBe(inheriting); +}); diff --git a/server/src/services/runner-environment-lifecycle.ts b/server/src/services/runner-environment-lifecycle.ts index 16b174d8afe..112d5fa822d 100644 --- a/server/src/services/runner-environment-lifecycle.ts +++ b/server/src/services/runner-environment-lifecycle.ts @@ -2,14 +2,27 @@ import { resolvePaperclipRunnerIdleTimeoutMs } from "@paperclipai/adapter-utils" import type { Environment } from "@paperclipai/shared"; import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "./native-runtime/native-runner-recovery-contract.js"; +/** + * The same default as the native session policy: an agent that never chose a + * lifecycle, in an environment that inherits it, runs a measured harness warm + * on a sandbox, so its follow-up keeps the sandbox and workspace instead of + * uploading the workspace into a new one. + */ +function defaultsToWarmOnSandbox(adapterConfig: Record): boolean { + const provider = adapterConfig.provider; + return provider === "codex" || provider === "opencode" || (provider === "acpx" && adapterConfig.acpxAgent === "claude"); +} + function resolveRunnerLifecycleModeForRun( config: Record, adapterConfig: Record, admittedLifecycleMode?: "warm" | "per_turn", + driver?: string, ): unknown { - return admittedLifecycleMode ?? (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn" - ? config.runnerLifecycleMode - : adapterConfig.lifecycleMode); + if (admittedLifecycleMode) return admittedLifecycleMode; + if (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn") return config.runnerLifecycleMode; + if (adapterConfig.lifecycleMode === "warm" || adapterConfig.lifecycleMode === "per_turn") return adapterConfig.lifecycleMode; + return driver === "sandbox" && defaultsToWarmOnSandbox(adapterConfig) ? "warm" : adapterConfig.lifecycleMode; } /** Resolve runner retention before acquiring a sandbox, not after it is ephemeral. @@ -25,7 +38,7 @@ export function resolveRunnerEnvironmentForRun