diff --git a/doc/opencode-cold-recovery.md b/doc/opencode-cold-recovery.md index cb3cc61c9cf..8667b10a734 100644 --- a/doc/opencode-cold-recovery.md +++ b/doc/opencode-cold-recovery.md @@ -43,14 +43,28 @@ sandbox, and nothing is encrypted, uploaded or downloaded. A durable artifact therefore still exists before the first turn. Concurrent first admissions on one server with the same company and manifest share one in-flight capture. The other admissions wait for it, then reuse its ready object under their own -ownership fence. If the shared capture fails, each waiting admission captures on -its own. Admissions on different servers may still upload the same closure +ownership fence. If the shared capture fails, one waiting admission starts the +next capture and the others keep waiting for it. When the running capture has +not finished after 90 seconds, one more waiting admission starts a capture from +its own sandbox, because the copy speed out of a sandbox differs from one +sandbox to the next. At most two captures run at once for one company and +manifest; a capture counts until its copy out of the sandbox ends. A run that +finds a saved object it cannot reuse starts its own capture as soon as a slot +is free, without waiting for the 90 seconds. The first capture that saves a ready object ends the wait +for every admission: an admission whose own capture is still copying reuses that +object and abandons its capture, and an admission whose own capture fails +reuses it too. An abandoned capture never uploads or publishes. Its copy out of +the sandbox is not interrupted: that provider call runs until it ends or reaches +its own timeout, and its local files are then removed. Admissions on different servers may still upload the same closure independently. Each capture has the 512 MiB limit; a run can retain at most three pending uploads. There is no single deadline for the whole capture. Each network step has its own bound instead: each export command has 120 seconds, and the sandbox -transfer keeps its provider's command timeouts. The upload and the download that +transfer keeps its provider's command timeouts. When the copy out of the +sandbox fails, the failure is `native_provider_artifacts_export_timeout` if the +provider call timed out, and `native_provider_artifacts_export_failed` +otherwise. The upload and the download that verifies it each have the larger of 120 seconds and the time for the object at 256 KiB per second. When an upload or download step exceeds its bound, the failure is `native_provider_artifacts_transfer_timeout` and names the step. The diff --git a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts index 0a6c76312d5..9e6953fadee 100644 --- a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts +++ b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts @@ -11,6 +11,7 @@ import type { NativeRunIdentity, NativeSession, NativeSessionBackend, PrpEvent } import type { ExecutionContinuationEnvelope } from "@paperclipai/shared"; import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; import { describe, expect, it, vi } from "vitest"; +import { logger } from "../middleware/logger.js"; import { heartbeatService, type HeartbeatServiceOptions } from "../services/heartbeat.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { closeIdleWarmNativeSessionsForRestart, scopedRunnerdStateRoot } from "../services/native-runtime/native-session-executor.js"; @@ -378,9 +379,12 @@ describe.skipIf(!support.supported)("heartbeat native pre-dispatch failure retry const failure: AdmissionFailure = { remaining: 1, turnNeverIssued: true, environmentId: environment!.id, provider, providerLeaseId: `sandbox-${provider}`, leasePolicy, failedRuns: [], sessionRoots: [] }; const heartbeat = heartbeatService(db, { nativeSessionBackendFactory: nativeBackend(db, { admissionFailure: failure, turnMessages }) }); + const warn = vi.spyOn(logger, "warn"); try { const { issueId, runId } = await startTask(db, heartbeat); expect(failure.failedRuns).toEqual([runId]); + expect(warn.mock.calls.filter(([, message]) => typeof message === "string" && message.includes("could not be marked workspace_failed"))) + .toEqual([]); expect(turnMessages.has(runId)).toBe(false); expect(existsSync(failure.sessionRoots[0]!)).toBe(false); const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); @@ -416,6 +420,7 @@ describe.skipIf(!support.supported)("heartbeat native pre-dispatch failure retry const [settled] = await db.select().from(issues).where(eq(issues.id, issueId)); expect(await taskInterruptionPresentation(db, settled!)).toMatchObject({ state: "resolved", canRetry: false }); } finally { + warn.mockRestore(); await heartbeat.drainActiveRunExecutions(); await closeIdleWarmNativeSessionsForRestart(); await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and( diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 1c3ed151c07..e8f19f6b5c8 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1315,20 +1315,22 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe } return { dispatched: true, adapterResult: acceptedResult }; } - try { - await measureRunPhase("execution.native_finalize", () => finalizeNativeRun({ - db, - runId: run.id, - workspaceFinalizeStatus: "failed", - })); - await dispatchPendingNativeStatusWakeups({ - companyId: run.companyId, - }); - } catch (finalizeErr) { - logger.warn( - { err: finalizeErr, runId: run.id }, - "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results", - ); + if (proposedResult) { + try { + await measureRunPhase("execution.native_finalize", () => finalizeNativeRun({ + db, + runId: run.id, + workspaceFinalizeStatus: "failed", + })); + await dispatchPendingNativeStatusWakeups({ + companyId: run.companyId, + }); + } catch (finalizeErr) { + logger.warn( + { err: finalizeErr, runId: run.id }, + "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results", + ); + } } } throw adapterErr; diff --git a/server/src/services/native-runtime/failed-before-provider-turn.ts b/server/src/services/native-runtime/failed-before-provider-turn.ts index ca8ea9701c5..fd6b79dd01f 100644 --- a/server/src/services/native-runtime/failed-before-provider-turn.ts +++ b/server/src/services/native-runtime/failed-before-provider-turn.ts @@ -10,7 +10,7 @@ export type FailedBeforeProviderTurnRecord = { proof: FailedBeforeProviderTurnProof; technicalCause?: string; }; -const TRANSFER_TIMEOUT_CODE = "native_provider_artifacts_transfer_timeout"; +const SLOW_SETUP_CODES = new Set(["native_provider_artifacts_transfer_timeout", "native_provider_artifacts_export_timeout"]); function technicalCauseCode(cause: unknown): string { const code = cause && typeof cause === "object" ? (cause as { code?: unknown }).code : undefined; @@ -35,7 +35,7 @@ export class NativeFailedBeforeProviderTurnError extends Error { readonly technicalCause: string; constructor(readonly providerLeaseId: string | null, cause: unknown) { const technicalCause = technicalCauseCode(cause); - super(technicalCause === TRANSFER_TIMEOUT_CODE + super(SLOW_SETUP_CODES.has(technicalCause) ? "Setting up the computer took too long, so the task is continuing on a fresh one." : "The computer could not be set up before the agent started, so the task is continuing on a fresh one.", { cause }); this.name = "NativeFailedBeforeProviderTurnError"; diff --git a/server/src/services/native-runtime/native-provider-artifact-capture.ts b/server/src/services/native-runtime/native-provider-artifact-capture.ts index 8f82e5cf4a9..43fdaaa4481 100644 --- a/server/src/services/native-runtime/native-provider-artifact-capture.ts +++ b/server/src/services/native-runtime/native-provider-artifact-capture.ts @@ -9,39 +9,165 @@ import type { OpenCodeLaunchArtifacts } from "./remote-opencode-artifacts.js"; import type { NativeExecutionContext } from "./native-execution-context.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { NativeProviderArtifactTransferTimeoutError } from "./native-provider-artifacts.js"; +import { isPluginRpcTimeout } from "./remote-runner-recovery-probe.js"; const names = ["nodeCommand", "opencodeProxy", "opencodeExecutable"] as const; export const NATIVE_PROVIDER_ARTIFACT_PUBLISH_ATTEMPTS = 3; -const sharedCaptures = new Map>(); +export const NATIVE_PROVIDER_ARTIFACT_CAPTURE_HEDGE_MS = 90_000; -export async function shareNativeProviderArtifactCapture(input: { +type Latch = { promise: Promise; resolve: () => void }; + +function latch(): Latch { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +export type ShareNativeProviderArtifactCaptureInput = { key: string; signal: AbortSignal; reuse: () => Promise; - capture: () => Promise; -}): Promise { - const inFlight = sharedCaptures.get(input.key); - if (inFlight) { - let stop!: () => void; - const stopped = new Promise((_resolve, reject) => { - stop = () => reject(input.signal.reason); - input.signal.addEventListener("abort", stop, { once: true }); - if (input.signal.aborted) stop(); - }); + capture: (signal: AbortSignal, hold: (work: Promise) => void) => Promise; + hedgeAfterMs?: number; +}; + +const MAX_CONCURRENT_CAPTURES = 2; + +class SharedArtifactCapture { + private readonly state = { saved: false, running: 0, lastStartedAtMs: 0 }; + private changed = latch(); + + constructor(private readonly key: string) {} + + get saved() { + return this.state.saved; + } + + tryStart(hedgeAfterMs: number, options: { afterSavedMiss: boolean }): (() => void) | null { + if (this.state.running >= MAX_CONCURRENT_CAPTURES) return null; + if (!options.afterSavedMiss && this.state.running > 0 && this.state.lastStartedAtMs + hedgeAfterMs > Date.now()) return null; + this.state.running += 1; + this.state.lastStartedAtMs = Date.now(); + let released = false; + return () => { + if (released) return; + released = true; + this.state.running -= 1; + this.notify(); + if (this.state.running === 0 && sharedCaptures.get(this.key) === this) sharedCaptures.delete(this.key); + }; + } + + async waitForChange(hedgeAfterMs: number, stopped: Promise, options: { afterSavedMiss: boolean }) { + let timer: ReturnType | undefined; + const waits: Promise[] = [this.changed.promise, stopped]; + const hedgeInMs = this.state.lastStartedAtMs + hedgeAfterMs - Date.now(); + if (!options.afterSavedMiss && this.state.running < MAX_CONCURRENT_CAPTURES && hedgeInMs > 0) { + waits.push(new Promise(resolve => { timer = setTimeout(resolve, hedgeInMs); })); + } try { - await Promise.race([inFlight.catch(() => undefined), stopped]); + await Promise.race(waits); } finally { - input.signal.removeEventListener("abort", stop); + clearTimeout(timer); + } + } + + async savedCopy() { + while (!this.state.saved) await this.changed.promise; + } + + markSaved() { + this.state.saved = true; + this.notify(); + } + + private notify() { + const changed = this.changed; + this.changed = latch(); + changed.resolve(); + } +} + +const sharedCaptures = new Map(); + +function sharedCaptureFor(key: string) { + let shared = sharedCaptures.get(key); + if (!shared) { + shared = new SharedArtifactCapture(key); + sharedCaptures.set(key, shared); + } + return shared; +} + +function stopWith(signal: AbortSignal) { + let stop!: () => void; + const promise = new Promise((_resolve, reject) => { + stop = () => reject(signal.reason); + signal.addEventListener("abort", stop, { once: true }); + if (signal.aborted) stop(); + }); + promise.catch(() => undefined); + return { promise, dispose: () => signal.removeEventListener("abort", stop) }; +} + +export async function shareNativeProviderArtifactCapture(input: ShareNativeProviderArtifactCaptureInput): Promise { + const hedgeAfterMs = input.hedgeAfterMs ?? NATIVE_PROVIDER_ARTIFACT_CAPTURE_HEDGE_MS; + const stopped = stopWith(input.signal); + try { + let afterSavedMiss = false; + for (;;) { + const shared = sharedCaptureFor(input.key); + const release = shared.tryStart(hedgeAfterMs, { afterSavedMiss }); + if (release) return await captureOrReuse(input, shared, release, stopped.promise); + if (afterSavedMiss || !shared.saved) await shared.waitForChange(hedgeAfterMs, stopped.promise, { afterSavedMiss }); + const reused = await input.reuse(); + if (reused) { + shared.markSaved(); + return reused; + } + afterSavedMiss ||= shared.saved; + } + } finally { + stopped.dispose(); + } +} + +async function captureOrReuse( + input: ShareNativeProviderArtifactCaptureInput, + shared: SharedArtifactCapture, + release: () => void, + stopped: Promise, +): Promise { + const superseded = new AbortController(); + const held: Promise[] = []; + const own = Promise.resolve().then(() => input.capture(AbortSignal.any([input.signal, superseded.signal]), work => held.push(work))); + void own.then(() => undefined, () => undefined).then(() => Promise.allSettled(held)).then(release); + try { + const first = await Promise.race([own.then(value => ({ value })), shared.savedCopy().then(() => null), stopped]); + if (first) { + shared.markSaved(); + return first.value; } const reused = await input.reuse(); - if (reused) return reused; + if (!reused) return await own; + superseded.abort(new Error("native_provider_artifacts_capture_superseded")); + return reused; + } catch (error) { + if (input.signal.aborted) throw error; + const reused = await input.reuse().catch(() => null); + if (!reused) throw error; + shared.markSaved(); + return reused; } - const own = input.capture(); - sharedCaptures.set(input.key, own); - try { - return await own; - } finally { - if (sharedCaptures.get(input.key) === own) sharedCaptures.delete(input.key); +} + +export class NativeProviderArtifactExportError extends Error { + readonly code: "native_provider_artifacts_export_timeout" | "native_provider_artifacts_export_failed"; + constructor(cause: unknown) { + const code = isPluginRpcTimeout(cause) ? "native_provider_artifacts_export_timeout" : "native_provider_artifacts_export_failed"; + super(`${code}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }); + this.name = "NativeProviderArtifactExportError"; + this.code = code; } } @@ -87,6 +213,7 @@ export async function captureRemoteNativeProviderArtifacts( manifest: NativeProviderArtifactSource["manifest"]; signal: AbortSignal; context?: NativeExecutionContext; + hold?: (work: Promise) => void; }, publish: (source: NativeProviderArtifactSource) => Promise, ): Promise { @@ -134,7 +261,9 @@ export async function captureRemoteNativeProviderArtifacts( input.context?.assertOpen(); await input.runner.syncOut!([ { operationId: `provider-artifacts-${randomUUID()}`, files }, - ]); + ]).catch((error: unknown) => { + throw new NativeProviderArtifactExportError(error); + }); } finally { // The export copy is scratch under the excluded runtime dir. A failed // removal must not replace the transfer outcome; it is logged instead. @@ -172,6 +301,7 @@ export async function captureRemoteNativeProviderArtifacts( } }); void operation.catch(() => undefined); + input.hold?.(operation); let abort!: () => void; const stopped = new Promise((_resolve, reject) => { abort = () => reject(input.signal.reason); diff --git a/server/src/services/native-runtime/native-provider-artifact-gate.test.ts b/server/src/services/native-runtime/native-provider-artifact-gate.test.ts index fdda6a4ddd5..e1a27122930 100644 --- a/server/src/services/native-runtime/native-provider-artifact-gate.test.ts +++ b/server/src/services/native-runtime/native-provider-artifact-gate.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import type { NativeProviderArtifactSource } from "./native-provider-artifacts.js"; +import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES } from "@paperclipai/plugin-sdk"; import { NativeProviderArtifactTransferTimeoutError } from "./native-provider-artifacts.js"; +import { NativeFailedBeforeProviderTurnError } from "./failed-before-provider-turn.js"; import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture, @@ -52,6 +54,203 @@ describe("shared provider artifact capture", () => { expect(capture).toHaveBeenCalledTimes(2); }); + it("starts a run's turn from another run's saved artifact while its own capture is still copying", async () => { + const key = `company:${Math.random()}`; + const hung = deferred(); + const fast = deferred(); + const signals: AbortSignal[] = []; + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn() + .mockImplementationOnce((signal: AbortSignal) => { signals.push(signal); return hung.promise; }) + .mockImplementationOnce((signal: AbortSignal) => { signals.push(signal); return fast.promise; }); + const slow = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + const other = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + await vi.waitFor(() => expect(capture).toHaveBeenCalledTimes(2), { interval: 1 }); + ready = true; + fast.resolve("captured"); + await expect(other).resolves.toBe("captured"); + await expect(slow).resolves.toBe("reused"); + expect(signals[0]!.aborted).toBe(true); + expect(signals[1]!.aborted).toBe(false); + }); + + it("uses a saved artifact when the run's own capture fails after another run saved one", async () => { + const key = `company:${Math.random()}`; + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn(async () => { + ready = true; + throw Object.assign(new Error('RPC call "environmentSyncOut" timed out after 330000ms'), { code: -32003 }); + }); + await expect(shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture })) + .resolves.toBe("reused"); + }); + + it("ends a still-running capture when another run's failed capture finds a saved copy", async () => { + const key = `company:${Math.random()}`; + const hung = deferred(); + const signals: AbortSignal[] = []; + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn() + .mockImplementationOnce((signal: AbortSignal) => { signals.push(signal); return hung.promise; }) + .mockImplementationOnce(async () => { + ready = true; + throw new Error("copy failed after another server saved the files"); + }); + const slow = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + const failing = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + await expect(failing).resolves.toBe("reused"); + await expect(slow).resolves.toBe("reused"); + expect(signals[0]!.aborted).toBe(true); + }); + + it("ends the slow capture when a waiting run finds a copy another server saved", async () => { + const key = `company:${Math.random()}`; + const hung = deferred(); + const signals: AbortSignal[] = []; + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn().mockImplementationOnce((signal: AbortSignal) => { signals.push(signal); return hung.promise; }); + const slow = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + ready = true; + const waiting = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + await expect(waiting).resolves.toBe("reused"); + await expect(slow).resolves.toBe("reused"); + expect(signals[0]!.aborted).toBe(true); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("starts its own capture once instead of asking again when a saved copy cannot be reused", async () => { + const key = `company:${Math.random()}`; + const hung = deferred(); + const reuse = vi.fn(async () => null); + const capture = vi.fn() + .mockImplementationOnce(() => hung.promise) + .mockImplementationOnce(async () => "captured") + .mockImplementationOnce(async () => "own"); + const slow = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + const hedge = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + await expect(hedge).resolves.toBe("captured"); + const late = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + await expect(late).resolves.toBe("own"); + expect(reuse.mock.calls.length).toBeLessThan(6); + hung.resolve("late"); + await slow; + }); + + it("never runs more than two captures at once however many hedge windows pass", async () => { + const key = `company:${Math.random()}`; + const hung = deferred(); + const hedge = deferred(); + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn().mockImplementationOnce(() => hung.promise).mockImplementationOnce(() => hedge.promise); + const runs = Array.from({ length: 6 }, () => shareNativeProviderArtifactCapture({ + key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 })); + await new Promise(resolve => setTimeout(resolve, 80)); + expect(capture).toHaveBeenCalledTimes(2); + expect(reuse.mock.calls.length).toBeLessThan(20); + ready = true; + hedge.resolve("captured"); + expect((await Promise.all(runs)).sort()).toEqual(["captured", ...Array(5).fill("reused")]); + expect(capture).toHaveBeenCalledTimes(2); + }); + + it("keeps a stopped capture's slot until its copy out of the sandbox ends", async () => { + const key = `company:${Math.random()}`; + const transfer = deferred(); + const stop = new AbortController(); + const reuse = vi.fn(async () => null); + const pending = () => new Promise(() => undefined); + const capture = vi.fn() + .mockImplementationOnce((signal: AbortSignal, hold: (work: Promise) => void) => { + hold(transfer.promise); + return new Promise((_resolve, reject) => signal.addEventListener("abort", () => reject(signal.reason))); + }) + .mockImplementation(pending); + const stopped = shareNativeProviderArtifactCapture({ key, signal: stop.signal, reuse, capture, hedgeAfterMs: 0 }); + await vi.waitFor(() => expect(capture).toHaveBeenCalledTimes(1), { interval: 1 }); + stop.abort(new Error("controlled Stop")); + await expect(stopped).rejects.toThrow("controlled Stop"); + void shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 0 }); + void shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 0 }); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(capture).toHaveBeenCalledTimes(2); + transfer.resolve(); + await vi.waitFor(() => expect(capture).toHaveBeenCalledTimes(3), { interval: 1 }); + }); + + it("keeps the two-capture cap for runs that cannot reuse the saved copy", async () => { + const key = `company:${Math.random()}`; + const releases: Array<() => void> = []; + let active = 0, peak = 0; + const reuse = vi.fn(async () => null); + const capture = vi.fn((_signal: AbortSignal, hold: (work: Promise) => void) => { + const transfer = deferred(); + active += 1; + peak = Math.max(peak, active); + hold(transfer.promise.finally(() => { active -= 1; })); + releases.push(() => transfer.resolve()); + return capture.mock.calls.length === 2 ? Promise.resolve("captured") : new Promise(() => undefined); + }); + void shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 }); + await expect(shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 })) + .resolves.toBe("captured"); + const misses = [0, 1].map(() => shareNativeProviderArtifactCapture({ + key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 10 })); + await new Promise(resolve => setTimeout(resolve, 40)); + expect(capture).toHaveBeenCalledTimes(2); + expect(peak).toBe(2); + releases[1]!(); + await vi.waitFor(() => expect(capture).toHaveBeenCalledTimes(3), { interval: 1 }); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(capture).toHaveBeenCalledTimes(3); + expect(peak).toBe(2); + expect(reuse.mock.calls.length).toBeLessThan(12); + releases.forEach(release => release()); + void misses; + }); + + it("starts one more capture when the shared capture runs past the hedge delay, not one per waiting run", async () => { + const key = `company:${Math.random()}`; + const hung = deferred(); + const hedge = deferred(); + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn().mockImplementationOnce(() => hung.promise).mockImplementationOnce(() => hedge.promise); + const runs = Array.from({ length: 8 }, () => shareNativeProviderArtifactCapture({ + key, signal: new AbortController().signal, reuse, capture, hedgeAfterMs: 50 })); + await vi.waitFor(() => expect(capture).toHaveBeenCalledTimes(2), { interval: 1 }); + await new Promise(resolve => setTimeout(resolve, 5)); + expect(capture).toHaveBeenCalledTimes(2); + ready = true; + hedge.resolve("captured"); + expect((await Promise.all(runs)).sort()).toEqual(["captured", ...Array(7).fill("reused")]); + expect(capture).toHaveBeenCalledTimes(2); + }); + + it("does not start a capture per waiting run when the shared capture fails", async () => { + const key = `company:${Math.random()}`; + const leader = deferred(); + const replacement = deferred(); + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn().mockImplementationOnce(() => leader.promise).mockImplementationOnce(() => replacement.promise); + const runs = Promise.allSettled(Array.from({ length: 6 }, () => shareNativeProviderArtifactCapture({ + key, signal: new AbortController().signal, reuse, capture }))); + leader.reject(new Error("leader copy failed")); + await vi.waitFor(() => expect(capture).toHaveBeenCalledTimes(2), { interval: 1 }); + await new Promise(resolve => setTimeout(resolve, 5)); + expect(capture).toHaveBeenCalledTimes(2); + ready = true; + replacement.resolve("captured"); + const settled = await runs; + expect(settled.filter(outcome => outcome.status === "rejected")).toHaveLength(1); + expect(capture).toHaveBeenCalledTimes(2); + }); + it("lets Stop end a wait for another run's capture", async () => { const key = `company:${Math.random()}`; const leader = deferred(); @@ -101,6 +300,41 @@ describe("provider artifact transfer step", () => { expect(publish).toHaveBeenCalledTimes(3); }); + it("names a stalled copy out of the sandbox instead of an unknown cause", async () => { + vi.mocked(runner.syncOut!).mockRejectedValueOnce(new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.TIMEOUT, + message: 'RPC call "environmentSyncOut" timed out after 330000ms' })); + const publish = vi.fn(); + const failure = await capture(publish).catch((error: unknown) => error); + expect(failure).toMatchObject({ code: "native_provider_artifacts_export_timeout" }); + expect(new NativeFailedBeforeProviderTurnError(null, failure)).toMatchObject({ + technicalCause: "native_provider_artifacts_export_timeout", + message: "Setting up the computer took too long, so the task is continuing on a fresh one.", + }); + expect(publish).not.toHaveBeenCalled(); + }); + + it("reports the copy out of the sandbox as held work that outlives a Stop", async () => { + const transfer = deferred<{ operations: [] }>(); + vi.mocked(runner.syncOut!).mockImplementationOnce(() => transfer.promise); + const stop = new AbortController(); + const held: Promise[] = []; + const capturing = captureRemoteNativeProviderArtifacts({ + runner, remoteCwd: "/workspace", manifest, signal: stop.signal, hold: work => held.push(work), + launch: { root: "/owned/qualified", accountUid: 1000, + paths: { nodeCommand: "/qualified/node", opencodeProxy: "/qualified/proxy", opencodeExecutable: "/qualified/opencode" } }, + }, vi.fn()); + await vi.waitFor(() => expect(runner.syncOut).toHaveBeenCalled(), { interval: 1 }); + stop.abort(new Error("controlled Stop")); + await expect(capturing).rejects.toThrow("controlled Stop"); + expect(held).toHaveLength(1); + let settled = false; + void held[0]!.then(() => { settled = true; }, () => { settled = true; }); + await new Promise(resolve => setTimeout(resolve, 10)); + expect(settled).toBe(false); + transfer.resolve({ operations: [] }); + await vi.waitFor(() => expect(settled).toBe(true), { interval: 1 }); + }); + it("does not retry a failure that is not a transfer timeout", async () => { const publish = vi.fn().mockRejectedValue(new Error("native_provider_artifacts_owner_changed")); await expect(capture(publish)).rejects.toThrow("native_provider_artifacts_owner_changed"); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 719559c3e2d..f8bc84eb968 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -19196,9 +19196,10 @@ async function createRunnerdBackendWithinSessionClaim( launchRoot: launch.root, accountUid: launch.accountUid }); providerArtifacts = await reuse() ?? await shareNativeProviderArtifactCapture({ key: `${input.execution.binding.companyId}:${manifest.digest}`, signal, reuse, - capture: () => captureRemoteNativeProviderArtifacts({ runner: remoteCommandRunner, - remoteCwd: remoteTarget.remoteCwd, launch, manifest, signal, context: input.executionContext }, - source => retainNativeProviderArtifacts({ db: input.db, storage: artifactStorage, owner: backupOwner, source, signal })), + capture: (captureSignal, hold) => captureRemoteNativeProviderArtifacts({ runner: remoteCommandRunner, + remoteCwd: remoteTarget.remoteCwd, launch, manifest, signal: captureSignal, context: input.executionContext, hold }, + source => retainNativeProviderArtifacts({ db: input.db, storage: artifactStorage, owner: backupOwner, source, + signal: captureSignal })), }); } : undefined, bindDiagnosticSession: session => {