From ab5bb8552a4fa2dbaa6006167db54a9d3da4e06c Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 11:05:52 -0500 Subject: [PATCH] fix(boat): bound provider-pack cold bootstrap separately Allow thirty minutes for computer ACPX pack staging without extending turn, recovery, finalization or warm-idle deadlines. Co-Authored-By: Paperclip --- .../src/native-session-runtime.test.ts | 14 +++++++++++--- .../src/native-session-runtime.ts | 8 +++++--- .../native-session-executor.test.ts | 16 ++++++++++++++++ .../native-runtime/native-session-executor.ts | 14 ++++++++++++++ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 610813ef9c..83ecfe439d 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -326,7 +326,7 @@ describe("executeNativeSession recovery", () => { const completeRun = vi.fn(async () => {}); try { const execution = executeNativeSession({ - input, turnTimeoutMs, + input, turnTimeoutMs, bootstrapTimeoutMs: 5, backend: { async descriptor() { return { kind: "mock", name: "long-lived", version: "1", capabilities }; }, async openSession() { return session; }, @@ -2256,15 +2256,18 @@ describe("executeNativeSession recovery", () => { runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 5, + bootstrapTimeoutMs: 10, }); const execution = execute(); const rejection = expect(execution).rejects.toThrow( - "native session bootstrap timed out after 5ms", + "native session bootstrap timed out after 10ms", ); await bootstrapStarted; expect(bootstrapSignal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(5); + expect(bootstrapSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); await rejection; expect(bootstrapSignal?.aborted).toBe(true); expect(openRun).not.toHaveBeenCalled(); @@ -2659,6 +2662,7 @@ describe("executeNativeSession recovery", () => { runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 10, + bootstrapTimeoutMs: 100, keepSessionOpen: true, onSession: (current) => retainedSessions.push(current), }); @@ -5513,6 +5517,7 @@ describe("executeNativeSession recovery", () => { controlPlaneInstanceId: "control-recovery", persistedSession: structuredClone(checkpoint), timeoutMs: 5, + bootstrapTimeoutMs: 100, onSession, }); const rejection = expect(execution).rejects.toThrow( @@ -5648,15 +5653,18 @@ describe("executeNativeSession recovery", () => { controlPlaneInstanceId: "control-recovery", persistedSession: structuredClone(checkpoint), timeoutMs: 5, + bootstrapTimeoutMs: 10, onSession, }); const rejection = expect(execution).rejects.toThrow( - "native session replacement bootstrap timed out after 5ms", + "native session replacement bootstrap timed out after 10ms", ); await replacementStarted; expect(replacementSignal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(5); + expect(replacementSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); await rejection; expect(replacementSignal?.aborted).toBe(true); expect(openRun).not.toHaveBeenCalled(); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index e01dd745e2..51b43052d4 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -207,6 +207,8 @@ export interface ExecuteNativeSessionOptions { remoteCleanupScope?: string; /** Operation bound; explicit values also preserve the legacy turn bound. */ timeoutMs?: number; + /** Initial/replacement provider creation only; leaves recovery, turns and finalization unchanged. */ + bootstrapTimeoutMs?: number; /** Total turn duration. Zero or no configured bound allows long-running work. */ turnTimeoutMs?: number; /** Abort admission while waiting for prior cleanup in the same domain. */ @@ -2094,8 +2096,8 @@ export async function executeNativeSession( workingDirectory: input.workspace.cwd ?? undefined, }; session = await runAbortableOperationWithin({ - timeoutMs: recoveryTimeoutMs, - timeoutMessage: `native session replacement bootstrap timed out after ${recoveryTimeoutMs}ms`, + timeoutMs: options.bootstrapTimeoutMs ?? recoveryTimeoutMs, + timeoutMessage: `native session replacement bootstrap timed out after ${options.bootstrapTimeoutMs ?? recoveryTimeoutMs}ms`, operation: (signal) => { const abortableReplacementInput = { ...replacementInput, signal }; return options.backend.openReplacementSession @@ -2121,7 +2123,7 @@ export async function executeNativeSession( // A fresh provider session is part of admission. Prove that it exists // before opening durable run state because ControlPlanePort intentionally // exposes no rollback for an admitted run. - const bootstrapTimeoutMs = options.timeoutMs ?? 900_000; + const bootstrapTimeoutMs = options.bootstrapTimeoutMs ?? options.timeoutMs ?? 900_000; const bootstrapInput = { identity, workingDirectory: input.workspace.cwd ?? undefined, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4bd3472ee3..3a7fd4f14d 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,3 +1,4 @@ +import { nativeSessionBootstrapTimeoutMs } from "./native-session-executor.js"; import { prepareHeartbeatGitHubLaunchers } from "../heartbeat-github-launchers.js"; import { gunzipSync } from "node:zlib"; import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; @@ -13766,3 +13767,18 @@ it.each([ ), ).toBe("native_event_replay_conflict"); }); + + +describe("computer provider-pack bootstrap policy", () => { + it.each([ + ["acpx", "computer", 1_800_000], + ["codex", "computer", undefined], + ["opencode", "computer", undefined], + ["acpx", "sandbox", undefined], + ["acpx", "ssh", undefined], + ["acpx", "local", undefined], + ] as const)("bounds only %s on %s", (provider, transport, expected) => { + const target = transport === "local" ? undefined : { kind: "remote" as const, transport }; + expect(nativeSessionBootstrapTimeoutMs(provider, target)).toBe(expected); + }); +}); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index d411b71e70..019da7ce56 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7497,6 +7497,19 @@ export async function renewNativeSessionExecutionLease(input: { if (!updated) throw new Error("native_session_lease_lost"); } +/** Computer ACPX cold starts stage the complete qualified Linux provider pack. + * Measured 1.1 GiB transfers plus compression/verification need about 24 minutes; + * this finite admission budget does not extend warm leases or turn execution. + */ +export function nativeSessionBootstrapTimeoutMs( + providerKind: NativeExecutionInput["provider"]["kind"], + target?: Pick, "kind" | "transport"> | { kind: "local" } | null, +): number | undefined { + return providerKind === "acpx" && target?.kind === "remote" && target.transport === "computer" + ? 30 * 60 * 1_000 + : undefined; +} + function startNativeSessionExecutionLeaseRenewal(input: { db: Db; runId: string; @@ -8802,6 +8815,7 @@ async function executePaperclipNativeSessionWithinScope( input: runnerExecution, remoteCleanupScope: remoteCleanupLease ? remoteLeaseCleanupScope(remoteCleanupLease) : undefined, turnTimeoutMs: input.turnTimeoutMs, + bootstrapTimeoutMs: nativeSessionBootstrapTimeoutMs(input.execution.provider.kind, input.runnerExecutionTarget), backend: input.backend ?? runnerdBackend ??