diff --git a/doc/opencode-cold-recovery.md b/doc/opencode-cold-recovery.md index dd26b43c0ad..cb3cc61c9cf 100644 --- a/doc/opencode-cold-recovery.md +++ b/doc/opencode-cold-recovery.md @@ -33,11 +33,30 @@ files are excluded from each later harness archive. A pending upload records its exact encrypted object reference before upload, so an interrupted reply can be retired without guessing an object key. -A ready company closure is reused. Concurrent first admissions may independently -upload that closure: each admission retains its own cancellation, ownership fence, -and pending receipt rather than depending on another controller's live transfer. -Each capture has the 512 MiB limit and five-minute deadline; a run can retain at -most three pending uploads. Publication selects one verified ready object under +A ready company closure is reused. Before it copies anything, admission looks +for a ready, verified company object whose descriptor has the same file paths +and SHA-256 digests as the selected manifest. The database matches the target, +paths and digests, so the lookup reads only matching objects, not every +version the company keeps. When one exists, admission pins it +for the run and records the run receipt in one transaction. No file leaves the +sandbox, and nothing is encrypted, uploaded or downloaded. A durable artifact +therefore still exists before the first turn. Concurrent first admissions on +one server with the same company and manifest share one in-flight capture. The +other admissions wait for it, then reuse its ready object under their own +ownership fence. If the shared capture fails, each waiting admission captures on +its own. Admissions on different servers may still upload the same closure +independently. + +Each capture has the 512 MiB limit; a run can retain at most three pending +uploads. There is no single deadline for the whole capture. Each network step +has its own bound instead: each export command has 120 seconds, and the sandbox +transfer keeps its provider's command timeouts. The upload and the download that +verifies it each have the larger of 120 seconds and the time for the object at +256 KiB per second. When an upload or download step exceeds its bound, the +failure is `native_provider_artifacts_transfer_timeout` and names the step. The +step is then retried from the local copy, up to three attempts in all, without +copying the files out of the sandbox again. Stop interrupts the capture and the +wait for a shared capture. Publication selects one verified ready object under the closure lock. A loser pins that winner before deleting only its own object. This bounds each admitted run, not total simultaneous company egress. A burst can therefore transfer duplicate bytes; distributed upload election is not included. diff --git a/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts index fcc6d5b5709..d3d6a731189 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts @@ -19,7 +19,8 @@ import { nativeHarnessBackupExpirationService } from "./native-harness-backup-ex import { legacyControllerBootId } from "../legacy-controller-lease.js"; import { DurablePrpControlPlane, captureLegacyDurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js"; import { loadNativeRunnerCheckpoint, nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; -import { restoreNativeProviderArtifacts } from "./native-provider-artifacts.js"; +import { NativeProviderArtifactTransferTimeoutError, nativeProviderArtifactTransferBounds, restoreNativeProviderArtifacts, + readyNativeProviderArtifactCandidates, retainNativeProviderArtifacts, retainReadyNativeProviderArtifacts } from "./native-provider-artifacts.js"; import * as backupArchive from "./native-harness-backup-archive.js"; import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; @@ -284,6 +285,79 @@ describeEmbeddedPostgres("durable native provider backups", () => { expect(await first.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, artifact.id))).toHaveLength(0); expect(await first.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.artifactId, artifact.id))).toHaveLength(0); }); + it("admits a later run from the company's ready launch artifact with a lookup and no transfer", async () => { + const version = randomUUID(), first = await artifactFixture(version), other = await artifactFixture(version); + const second = { ...other, storage: first.storage }; + const stored = await retainNativeProviderArtifacts({ ...first, source: first.providerArtifacts }); + const put = vi.spyOn(first.storage, "putFile"), get = vi.spyOn(first.storage, "getObject"); + let receipt: Awaited> = null; + try { + const startedAt = performance.now(); + receipt = await retainReadyNativeProviderArtifacts({ db: second.db, owner: second.owner, manifest: second.manifest, + launchRoot: second.launchRoot, accountUid: 1000 }); + const elapsedMs = performance.now() - startedAt; + console.info(`[provider-artifact-gate] ready artifact admission took ${elapsedMs.toFixed(1)} ms`); + expect(elapsedMs).toBeLessThan(1_000); + expect(receipt).toMatchObject({ artifactId: stored.artifactId, closureDigest: stored.closureDigest, + providerPackDigest: second.manifest.digest, launchRoot: second.launchRoot, accountUid: 1000 }); + expect(put).not.toHaveBeenCalled(); + expect(get).not.toHaveBeenCalled(); + } finally { put.mockRestore(); get.mockRestore(); } + const [run] = await second.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, second.runId)); + expect(run.runnerProfileJson?.nativeProviderArtifacts).toMatchObject({ artifactId: stored.artifactId }); + const pins = await second.db.select().from(nativeProviderArtifactReferences).where(and( + eq(nativeProviderArtifactReferences.artifactId, stored.artifactId), eq(nativeProviderArtifactReferences.runId, second.runId))); + expect(pins).toHaveLength(1); + const replacement = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(replacement); await mkdir(replacement); + await restoreNativeProviderArtifacts({ ...second, companyId: second.companyId, receipt: receipt!, targetRoot: join(replacement, "provider-artifacts") }); + expect(await readFile(join(replacement, "provider-artifacts/opencodeProxy"), "utf8")).toBe(second.original.opencodeProxy); + }); + it("loads only the ready artifact whose launch files match, however many other versions the company keeps", async () => { + const version = randomUUID(), first = await artifactFixture(version), other = await artifactFixture(version); + const second = { ...other, storage: first.storage }; + const stored = await retainNativeProviderArtifacts({ ...first, source: first.providerArtifacts }); + const [kept] = await first.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, stored.artifactId)); + const descriptor = kept.descriptor as { entries: { name: string; sha256: string }[] }; + await first.db.insert(nativeProviderArtifacts).values(Array.from({ length: 200 }, (_, index) => ({ + companyId: second.companyId, status: "ready", reference: { decoy: index }, + closureDigest: `sha256:${createHash("sha256").update(`decoy ${version} ${index}`).digest("hex")}`, + descriptor: { ...descriptor, entries: descriptor.entries.map(entry => entry.name === "opencodeProxy" + ? { ...entry, sha256: `sha256:${createHash("sha256").update(`older proxy ${index}`).digest("hex")}` } : entry) }, + }))); + const candidates = await readyNativeProviderArtifactCandidates(second.db, second.companyId, second.manifest); + expect(candidates.map(row => row.id)).toEqual([stored.artifactId]); + await expect(retainReadyNativeProviderArtifacts({ db: second.db, owner: second.owner, manifest: second.manifest, + launchRoot: second.launchRoot, accountUid: 1000 })).resolves.toMatchObject({ artifactId: stored.artifactId }); + }); + it("captures a launch artifact whose bytes differ from every ready artifact", async () => { + const first = await artifactFixture(randomUUID()), second = await artifactFixture(randomUUID()); + await retainNativeProviderArtifacts({ ...first, source: first.providerArtifacts }); + await expect(retainReadyNativeProviderArtifacts({ db: second.db, owner: second.owner, manifest: second.manifest, + launchRoot: second.launchRoot, accountUid: 1000 })).resolves.toBeNull(); + const [run] = await second.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, second.runId)); + expect(run.runnerProfileJson?.nativeProviderArtifacts).toBeUndefined(); + }); + it.each(["upload", "download"] as const)("bounds a stalled %s with its own timeout and real cause", async step => { + const f = await artifactFixture(); + const bounds = { ...nativeProviderArtifactTransferBounds }; + Object.assign(nativeProviderArtifactTransferBounds, { minimumStepMs: 50, minimumBytesPerSecond: Number.MAX_SAFE_INTEGER }); + const stalled = (signal?: AbortSignal) => new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + const stall = step === "upload" + ? vi.spyOn(f.storage, "putFile").mockImplementation(async input => stalled(input.signal)) + : vi.spyOn(f.storage, "getObject").mockImplementation(async (_companyId, _key, options) => stalled(options?.signal)); + try { + const failure = retainNativeProviderArtifacts({ ...f, source: f.providerArtifacts }); + await expect(failure).rejects.toBeInstanceOf(NativeProviderArtifactTransferTimeoutError); + await expect(failure).rejects.toMatchObject({ step, + message: expect.stringMatching(new RegExp(`^native_provider_artifacts_transfer_timeout: the ${step} of \\d+ bytes did not finish within 1 s$`)), + }); + } finally { + stall.mockRestore(); + Object.assign(nativeProviderArtifactTransferBounds, bounds); + } + }); it("keeps independent overlapping uploads bounded and retains only their verified winner", async () => { const version = randomUUID(), first = await artifactFixture(version), other = await artifactFixture(version); const second = { ...other, storage: first.storage }, put = first.storage.putFile.bind(first.storage); diff --git a/server/src/services/native-runtime/native-provider-artifact-capture.ts b/server/src/services/native-runtime/native-provider-artifact-capture.ts index 4ea9fe30a84..8f82e5cf4a9 100644 --- a/server/src/services/native-runtime/native-provider-artifact-capture.ts +++ b/server/src/services/native-runtime/native-provider-artifact-capture.ts @@ -8,8 +8,42 @@ import type { NativeProviderArtifactSource } from "./native-provider-artifacts.j import type { OpenCodeLaunchArtifacts } from "./remote-opencode-artifacts.js"; import type { NativeExecutionContext } from "./native-execution-context.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; +import { NativeProviderArtifactTransferTimeoutError } from "./native-provider-artifacts.js"; const names = ["nodeCommand", "opencodeProxy", "opencodeExecutable"] as const; +export const NATIVE_PROVIDER_ARTIFACT_PUBLISH_ATTEMPTS = 3; +const sharedCaptures = new Map>(); + +export async function shareNativeProviderArtifactCapture(input: { + key: string; + signal: AbortSignal; + reuse: () => Promise; + capture: () => Promise; +}): Promise { + const inFlight = sharedCaptures.get(input.key); + if (inFlight) { + let stop!: () => void; + const stopped = new Promise((_resolve, reject) => { + stop = () => reject(input.signal.reason); + input.signal.addEventListener("abort", stop, { once: true }); + if (input.signal.aborted) stop(); + }); + try { + await Promise.race([inFlight.catch(() => undefined), stopped]); + } finally { + input.signal.removeEventListener("abort", stop); + } + const reused = await input.reuse(); + if (reused) return reused; + } + const own = input.capture(); + sharedCaptures.set(input.key, own); + try { + return await own; + } finally { + if (sharedCaptures.get(input.key) === own) sharedCaptures.delete(input.key); + } +} // Copies the qualified launch files into a fresh export directory. The launch // root sits under the sandbox account's home, outside the workspace remote dir, @@ -108,14 +142,21 @@ export async function captureRemoteNativeProviderArtifacts( logger.warn({ err }, "native provider artifact export cleanup failed"); }); } - input.signal.throwIfAborted(); - input.context?.assertOpen(); - return publish({ - sourceRoot: root, - manifest: input.manifest, - launchRoot: input.launch.root, - accountUid: input.launch.accountUid, - }); + for (let attempt = 1; ; attempt++) { + input.signal.throwIfAborted(); + input.context?.assertOpen(); + try { + return await publish({ + sourceRoot: root, + manifest: input.manifest, + launchRoot: input.launch.root, + accountUid: input.launch.accountUid, + }); + } catch (error) { + if (!(error instanceof NativeProviderArtifactTransferTimeoutError) || attempt >= NATIVE_PROVIDER_ARTIFACT_PUBLISH_ATTEMPTS) throw error; + logger.warn({ err: error, attempt }, "native provider artifact transfer timed out; retrying the transfer"); + } + } }) .finally(async () => { try { diff --git a/server/src/services/native-runtime/native-provider-artifact-gate.test.ts b/server/src/services/native-runtime/native-provider-artifact-gate.test.ts new file mode 100644 index 00000000000..fdda6a4ddd5 --- /dev/null +++ b/server/src/services/native-runtime/native-provider-artifact-gate.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeProviderArtifactSource } from "./native-provider-artifacts.js"; +import { NativeProviderArtifactTransferTimeoutError } from "./native-provider-artifacts.js"; +import { + captureRemoteNativeProviderArtifacts, + shareNativeProviderArtifactCapture, +} from "./native-provider-artifact-capture.js"; + +function deferred() { + let resolve!: (value: T) => void, reject!: (error: unknown) => void; + const promise = new Promise((done, fail) => { resolve = done; reject = fail; }); + return { promise, resolve, reject }; +} + +describe("shared provider artifact capture", () => { + it("starts a capture at once when no capture for the same closure is in flight", async () => { + const reuse = vi.fn(async () => "ready"); + const capture = vi.fn(async () => "captured"); + await expect(shareNativeProviderArtifactCapture({ key: `company:${Math.random()}`, + signal: new AbortController().signal, reuse, capture })).resolves.toBe("captured"); + expect(capture).toHaveBeenCalledOnce(); + expect(reuse).not.toHaveBeenCalled(); + }); + + it("lets concurrent first runs of one company share a single in-flight capture", async () => { + const key = `company:${Math.random()}`; + const leader = deferred(); + let ready = false; + const reuse = vi.fn(async () => ready ? "reused" : null); + const capture = vi.fn(() => leader.promise); + const runs = Array.from({ length: 8 }, () => shareNativeProviderArtifactCapture({ + key, signal: new AbortController().signal, reuse, capture })); + await new Promise(resolve => setImmediate(resolve)); + expect(capture).toHaveBeenCalledOnce(); + ready = true; + leader.resolve("captured"); + expect(await Promise.all(runs)).toEqual(["captured", ...Array(7).fill("reused")]); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("captures again when the shared capture failed", async () => { + const key = `company:${Math.random()}`; + const leader = deferred(); + const capture = vi.fn().mockImplementationOnce(() => leader.promise).mockResolvedValue("own"); + const reuse = vi.fn(async () => null); + const first = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture }); + const second = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture }); + leader.reject(new Error("leader upload failed")); + await expect(first).rejects.toThrow("leader upload failed"); + await expect(second).resolves.toBe("own"); + expect(capture).toHaveBeenCalledTimes(2); + }); + + it("lets Stop end a wait for another run's capture", async () => { + const key = `company:${Math.random()}`; + const leader = deferred(); + const stop = new AbortController(); + const capture = vi.fn(() => leader.promise); + const reuse = vi.fn(async () => null); + const first = shareNativeProviderArtifactCapture({ key, signal: new AbortController().signal, reuse, capture }); + const waiting = shareNativeProviderArtifactCapture({ key, signal: stop.signal, reuse, capture }); + stop.abort(new Error("controlled Stop")); + await expect(waiting).rejects.toThrow("controlled Stop"); + leader.resolve("captured"); + await expect(first).resolves.toBe("captured"); + expect(capture).toHaveBeenCalledOnce(); + }); +}); + +describe("provider artifact transfer step", () => { + const runner = { + execute: vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null })), + syncOut: vi.fn(async () => ({ operations: [] })), + } as unknown as CommandManagedRuntimeRunner; + const manifest = { payload: { artifacts: Object.fromEntries(["nodeCommand", "opencodeProxy", "opencodeExecutable"] + .map(name => [name, { path: name }])) } } as NativeProviderArtifactSource["manifest"]; + const capture = (publish: (source: NativeProviderArtifactSource) => Promise) => captureRemoteNativeProviderArtifacts({ + runner, remoteCwd: "/workspace", manifest, signal: new AbortController().signal, + launch: { root: "/owned/qualified", accountUid: 1000, + paths: { nodeCommand: "/qualified/node", opencodeProxy: "/qualified/proxy", opencodeExecutable: "/qualified/opencode" } }, + }, publish); + + it("retries a timed-out transfer without copying the launch files out of the sandbox again", async () => { + const publish = vi.fn() + .mockRejectedValueOnce(new NativeProviderArtifactTransferTimeoutError("upload", 120_000, 1024)) + .mockResolvedValue("receipt"); + vi.mocked(runner.syncOut!).mockClear(); + await expect(capture(publish)).resolves.toBe("receipt"); + expect(publish).toHaveBeenCalledTimes(2); + expect(runner.syncOut).toHaveBeenCalledOnce(); + }); + + it("fails with the transfer's own cause after the bounded attempts", async () => { + const publish = vi.fn().mockRejectedValue(new NativeProviderArtifactTransferTimeoutError("download", 120_000, 1024)); + await expect(capture(publish)).rejects.toMatchObject({ + name: "NativeProviderArtifactTransferTimeoutError", + code: "native_provider_artifacts_transfer_timeout", + message: "native_provider_artifacts_transfer_timeout: the download of 1024 bytes did not finish within 120 s", + }); + expect(publish).toHaveBeenCalledTimes(3); + }); + + it("does not retry a failure that is not a transfer timeout", async () => { + const publish = vi.fn().mockRejectedValue(new Error("native_provider_artifacts_owner_changed")); + await expect(capture(publish)).rejects.toThrow("native_provider_artifacts_owner_changed"); + expect(publish).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/services/native-runtime/native-provider-artifacts.ts b/server/src/services/native-runtime/native-provider-artifacts.ts index 4d472794f41..c65db0586b5 100644 --- a/server/src/services/native-runtime/native-provider-artifacts.ts +++ b/server/src/services/native-runtime/native-provider-artifacts.ts @@ -37,7 +37,7 @@ import { cp, mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pipeline } from "node:stream/promises"; -import { and, eq, inArray, not, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, not, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, @@ -63,6 +63,49 @@ import { } from "./remote-provider-pack-verification.js"; import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; +export const nativeProviderArtifactTransferBounds = { + minimumBytesPerSecond: 256 * 1024, + minimumStepMs: 120_000, +}; + +export class NativeProviderArtifactTransferTimeoutError extends Error { + readonly code = "native_provider_artifacts_transfer_timeout"; + constructor(readonly step: "upload" | "download", readonly limitMs: number, readonly byteSize: number) { + super(`native_provider_artifacts_transfer_timeout: the ${step} of ${byteSize} bytes did not finish within ${Math.ceil(limitMs / 1000)} s`); + this.name = "NativeProviderArtifactTransferTimeoutError"; + } +} + +export function nativeProviderArtifactTransferLimitMs(byteSize: number) { + return Math.max( + nativeProviderArtifactTransferBounds.minimumStepMs, + Math.ceil((byteSize / nativeProviderArtifactTransferBounds.minimumBytesPerSecond) * 1000), + ); +} + +async function boundedTransfer( + step: "upload" | "download", + byteSize: number, + signal: AbortSignal | undefined, + transfer: (signal: AbortSignal) => Promise, +): Promise { + const limitMs = nativeProviderArtifactTransferLimitMs(byteSize); + const expired = new AbortController(); + const timer = setTimeout( + () => expired.abort(new NativeProviderArtifactTransferTimeoutError(step, limitMs, byteSize)), + limitMs, + ); + timer.unref?.(); + try { + return await transfer(signal ? AbortSignal.any([signal, expired.signal]) : expired.signal); + } catch (error) { + if (expired.signal.aborted && !signal?.aborted) throw expired.signal.reason; + throw error; + } finally { + clearTimeout(timer); + } +} + async function recordRunReceipt( tx: Parameters[0]>[0], owner: NativeRunnerCheckpointFenceOwner, @@ -291,7 +334,7 @@ export async function retainNativeProviderArtifacts(input: { .returning({ id: nativeProviderArtifacts.id }); if (!owned) fail("owner_changed"); }); - await input.storage.putFile({ + await boundedTransfer("upload", reference.byteSize, input.signal, signal => input.storage.putFile({ companyId: input.owner.companyId, namespace: "native-provider-artifacts", objectKey: reference.objectKey, @@ -302,14 +345,14 @@ export async function retainNativeProviderArtifacts(input: { sha256: reference.sha256.slice(7), // A file source can be resent after a request-rate limit; a stream cannot. sourceFile: encrypted, - signal: input.signal, - }); - await receive( + signal, + })); + await boundedTransfer("download", reference.byteSize, input.signal, signal => receive( input.storage, reference, join(scratch, "verified"), - input.signal, - ); + signal, + )); const published = await input.db.transaction(async (tx) => { await artifactOwnerFence(tx, input.owner, input.source.manifest); await tx.execute( @@ -413,6 +456,82 @@ export async function retainNativeProviderArtifacts(input: { } } +export const READY_NATIVE_PROVIDER_ARTIFACT_CANDIDATE_LIMIT = 4; + +export async function readyNativeProviderArtifactCandidates( + tx: Pick, + companyId: string, + manifest: ProviderPackIdentity, +) { + const entries = names.map((name) => { + const artifact = manifest.payload.artifacts[name]; + return artifact ? { name, path: artifact.path, sha256: artifact.sha256 } : null; + }); + if (entries.some((entry) => entry === null)) return []; + const probe = JSON.stringify({ schema, target: manifest.payload.target, entries }); + return tx + .select({ + id: nativeProviderArtifacts.id, + closureDigest: nativeProviderArtifacts.closureDigest, + descriptor: nativeProviderArtifacts.descriptor, + }) + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, companyId), + eq(nativeProviderArtifacts.status, "ready"), + sql`${nativeProviderArtifacts.reference} is not null`, + sql`${nativeProviderArtifacts.descriptor} @> ${probe}::jsonb`, + ), + ) + .orderBy(desc(nativeProviderArtifacts.updatedAt)) + .limit(READY_NATIVE_PROVIDER_ARTIFACT_CANDIDATE_LIMIT); +} + +export async function retainReadyNativeProviderArtifacts(input: { + db: Db; + owner: NativeRunnerCheckpointFenceOwner; + manifest: ProviderPackIdentity; + launchRoot: string; + accountUid: number; +}): Promise { + manifestIdentity(input.manifest); + launchRootIdentity(input.launchRoot, input.manifest, input.accountUid); + const source = { sourceRoot: "", manifest: input.manifest, launchRoot: input.launchRoot, accountUid: input.accountUid }; + return input.db.transaction(async (tx) => { + await artifactOwnerFence(tx, input.owner, input.manifest); + const rows = await readyNativeProviderArtifactCandidates(tx, input.owner.companyId, input.manifest); + for (const row of rows) { + const receipt = receiptFor(row.id, row.closureDigest, source); + try { + if (digest(canonicalProviderPackJson(row.descriptor)) !== row.closureDigest) continue; + descriptorIdentity(row.descriptor as Descriptor, receipt); + } catch { + continue; + } + const [locked] = await tx + .select({ id: nativeProviderArtifacts.id }) + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.id, row.id), + eq(nativeProviderArtifacts.status, "ready"), + ), + ) + .for("update"); + if (!locked) continue; + await tx + .insert(nativeProviderArtifactReferences) + .values({ companyId: input.owner.companyId, artifactId: row.id, runId: input.owner.runId }) + .onConflictDoNothing(); + await recordRunReceipt(tx, input.owner, receipt); + return receipt; + } + return null; + }); +} + export async function retainExistingNativeProviderArtifacts(input: { db: Db; owner: NativeRunnerCheckpointFenceOwner; @@ -528,7 +647,8 @@ export async function restoreNativeProviderArtifacts(input: { const encrypted = join(scratch, "encrypted"), archive = join(scratch, "archive"), restored = join(scratch, "restored"); - await receive(input.storage, reference, encrypted, input.signal); + await boundedTransfer("download", reference.byteSize, input.signal, signal => + receive(input.storage, reference, encrypted, signal)); const decipher = createDecipheriv( "aes-256-gcm", key, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 36b0662d4ba..1f05f6623b9 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -2868,7 +2868,7 @@ describe("remote provider pack manifest", () => { return JSON.stringify(value); }; - it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-new-durable-store-failed", "opencode-new-durable-publish-failed", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { + it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-new-durable-store-failed", "opencode-new-durable-publish-failed", "opencode-new-durable-slow-capture", "opencode-new-durable-ready", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { const firstDurable = provider.startsWith("opencode-new-durable"); // A sandbox connection drops once while the settled state is read out. let checkpointReadResets = provider === "opencode-new-durable-reset" ? 1 : 0; @@ -2961,6 +2961,8 @@ describe("remote provider pack manifest", () => { const owner = vi.spyOn(harnessStorage, "nativeHarnessBackupOwner").mockResolvedValue({ runId: remoteExecution.binding.runId } as never); const validate = vi.spyOn(harnessStorage, "assertNativeHarnessBackupOwner").mockResolvedValue(undefined); const retain = vi.spyOn(providerArtifactStorage, "retainNativeProviderArtifacts").mockResolvedValue(newReceipt); + const ready = vi.spyOn(providerArtifactStorage, "retainReadyNativeProviderArtifacts") + .mockResolvedValue(provider === "opencode-new-durable-ready" ? newReceipt : null); const capture = vi.spyOn(providerArtifactCapture, "captureRemoteNativeProviderArtifacts"); const backup = vi.spyOn(harnessStorage, "persistNativeHarnessBackup").mockImplementation(async input => { expect(JSON.parse(await readFile(join(input.root, "manifest.json"), "utf8")).providerArtifacts).toEqual(newReceipt); @@ -2970,7 +2972,7 @@ describe("remote provider pack manifest", () => { if (provider === "opencode-new-durable-publish-failed") await rm(input.root, { recursive: true, force: true }); return undefined as never; }); - onTestFinished(() => { storage.mockRestore(); owner.mockRestore(); validate.mockRestore(); retain.mockRestore(); capture.mockRestore(); backup.mockRestore(); }); + onTestFinished(() => { storage.mockRestore(); owner.mockRestore(); validate.mockRestore(); retain.mockRestore(); ready.mockRestore(); capture.mockRestore(); backup.mockRestore(); }); } const execute = vi.fn(async (command: { command: string; args?: string[] }) => { const script = command.args?.[1] ?? ""; @@ -3058,7 +3060,21 @@ describe("remote provider pack manifest", () => { expect(execute.mock.calls.filter(([call]) => call.args?.[0] === "--build-metadata")).toHaveLength(1); if (firstDurable) { expect(managedBackend.providerAdmissionRequired?.()).toBe(true); - await managedBackend.providerAdmissionPersisted!(); + const fiveMinutesElapsed = provider === "opencode-new-durable-slow-capture" + ? vi.spyOn(AbortSignal, "timeout").mockImplementation(() => + AbortSignal.abort(new DOMException("The operation was aborted due to timeout", "TimeoutError"))) + : null; + const admissionStartedAt = performance.now(); + try { await managedBackend.providerAdmissionPersisted!(); } finally { fiveMinutesElapsed?.mockRestore(); } + expect(providerArtifactStorage.retainReadyNativeProviderArtifacts).toHaveBeenCalledWith(expect.objectContaining({ + manifest: installed, launchRoot, accountUid: 1000 })); + if (provider === "opencode-new-durable-ready") { + console.info(`[provider-artifact-gate] admission gate with a ready artifact took ${(performance.now() - admissionStartedAt).toFixed(1)} ms`); + expect(providerArtifactCapture.captureRemoteNativeProviderArtifacts).not.toHaveBeenCalled(); + expect(providerArtifactStorage.retainNativeProviderArtifacts).not.toHaveBeenCalled(); + expect(syncOut).not.toHaveBeenCalled(); + return; + } expect(providerArtifactCapture.captureRemoteNativeProviderArtifacts).toHaveBeenCalledTimes(1); expect(providerArtifactCapture.captureRemoteNativeProviderArtifacts).toHaveBeenCalledWith( expect.objectContaining({ remoteCwd: "/workspace" }), expect.any(Function)); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 9fd14ac5af6..176282bf810 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -2,7 +2,7 @@ import { currentControllerProcess } from "../controller-process-identity.js"; import { remoteRecoveryLeaseMismatch } from "./remote-recovery-lease-mismatch.js"; import { isGovernancePauseStop } from "../governance-pause-stop.js"; import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token-lease-settlement.js"; -import { captureRemoteNativeProviderArtifacts } from "./native-provider-artifact-capture.js"; +import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js"; import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js"; @@ -23,7 +23,7 @@ import { getStartupRecoveryState } from "../../startup-recovery-state.js"; import { assertNativeHarnessBackupOwner, durableNativeHarnessStorage, NATIVE_HARNESS_DOWNLOAD_IDLE_TIMEOUT_MS, nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import { restoreNativeProviderArtifacts, retainNativeProviderArtifacts, retainExistingNativeProviderArtifacts, - type NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; + retainReadyNativeProviderArtifacts, type NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; import { prepareRemoteOpenCodeArtifacts, type OpenCodeLaunchArtifacts } from "./remote-opencode-artifacts.js"; import { prepareRuntimeContextSource, rehydratePinnedInstructionAsset } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; @@ -364,6 +364,7 @@ import { type ActiveNativeSession = { session: NativeSession; cancelRequested: boolean; + admissionAbort?: AbortController; cancellationSettled?: Promise; restartDetach?: Promise; }; @@ -9999,6 +10000,7 @@ async function performNativeCancellation( dispatched = true; if (!active.cancelRequested) { active.cancelRequested = true; + active.admissionAbort?.abort(new NativeCancellationPendingRecoveryError()); active.cancellationSettled = new Promise(resolve => { settleActiveCancellation = resolve; }); try { if (active.session.cancel) { @@ -12557,7 +12559,7 @@ async function executePaperclipNativeSessionAttempt( } } if (session) { - const active: ActiveNativeSession = { session, cancelRequested: false }; + const active: ActiveNativeSession = { session, cancelRequested: false, admissionAbort: new AbortController() }; activeNativeSessions.set(input.execution.binding.runId, active); const startup = nativeSessionStartups.get(input.execution.binding.runId); const stopBeforeProviderTurn = startup?.stopRequested === true && startup.sessionPublished !== true @@ -12655,7 +12657,7 @@ async function executePaperclipNativeSessionAttempt( ?? (error instanceof Error && /^native_[a-z_]+$/.test(error.message) ? error.message : "unknown"); try { await input.onLog?.("stderr", `[paperclip-runner] native provider-pack admission receipt could not be saved (cause=${cause})\n`); } catch {} } - await runnerdBackend?.providerAdmissionPersisted?.(); + await runnerdBackend?.providerAdmissionPersisted?.(active.admissionAbort!.signal); if (session.resolveRuntimeRequest) await liveQuestions.attach(); if (active.cancelRequested) { await active.cancellationSettled; @@ -16064,7 +16066,7 @@ type ManagedRunnerdBackend = NativeSessionBackend & { bindManagedSession(session: NativeSession, options?: { releaseResumeOnly: true }): NativeSession; bindDiagnosticSession?(session: NativeSession): void; sessionCheckpointPersisted?(snapshot: PersistedNativeSession): void; - providerAdmissionPersisted?(): Promise; + providerAdmissionPersisted?(signal?: AbortSignal): Promise; providerAdmissionRequired?(): boolean; }; @@ -16555,8 +16557,10 @@ async function createRunnerdBackendWithinSessionClaim( input.execution.provider.kind === "acpx"); const artifactStorage = remoteTarget?.transport === "sandbox" && input.controllerPersistence && input.execution.provider.kind === "opencode" ? durableNativeHarnessStorage() : null; - const providerArtifactsSignal = () => input.executionContext?.signal - ? AbortSignal.any([input.executionContext.signal, AbortSignal.timeout(300_000)]) : AbortSignal.timeout(300_000); + const providerArtifactsSignal = (stop?: AbortSignal) => { + const signals = [input.executionContext?.signal, stop].filter((signal): signal is AbortSignal => signal !== undefined); + return signals.length ? AbortSignal.any(signals) : new AbortController().signal; + }; const liveProviderRecovery = input.restartRecovery?.kind === "reattach_remote_runner" || input.remoteWarmRecovery !== undefined; const artifactBackup = artifactStorage ? await verifyNativeHarnessBackupStreamed({ root, execution: input.execution, runnerInstanceId: input.runnerInstanceId }) : null; @@ -18547,8 +18551,8 @@ async function createRunnerdBackendWithinSessionClaim( providerAdmissionRequired: () => replacementBackend?.providerAdmissionRequired ? replacementBackend.providerAdmissionRequired() : Boolean(artifactStorage && (providerArtifacts || !legacyProviderContinuity && !liveProviderRecovery)), - providerAdmissionPersisted: artifactStorage ? async () => { - if (replacementBackend) return replacementBackend.providerAdmissionPersisted?.(); + providerAdmissionPersisted: artifactStorage ? async (stop?: AbortSignal) => { + if (replacementBackend) return replacementBackend.providerAdmissionPersisted?.(stop); const backupOwner = sessionHarnessBackupOwners.get(authorityEpoch); if ((liveProviderRecovery || legacyProviderContinuity) && !providerArtifacts) return; if (!backupOwner || !expectedProviderPackManifest || !remoteCommandRunner) { @@ -18559,11 +18563,17 @@ async function createRunnerdBackendWithinSessionClaim( return; } if (!openCodeLaunchArtifacts) throw new Error("native_provider_artifacts_admission_missing"); - const signal = providerArtifactsSignal(); + const signal = providerArtifactsSignal(stop); if (!remoteTarget) throw new Error("native_provider_artifacts_admission_missing"); - providerArtifacts = await captureRemoteNativeProviderArtifacts({ runner: remoteCommandRunner, - remoteCwd: remoteTarget.remoteCwd, launch: openCodeLaunchArtifacts, manifest: expectedProviderPackManifest, signal, context: input.executionContext }, - source => retainNativeProviderArtifacts({ db: input.db, storage: artifactStorage, owner: backupOwner, source, signal })); + const launch = openCodeLaunchArtifacts, manifest = expectedProviderPackManifest; + const reuse = () => retainReadyNativeProviderArtifacts({ db: input.db, owner: backupOwner, manifest, + launchRoot: launch.root, accountUid: launch.accountUid }); + providerArtifacts = await reuse() ?? await shareNativeProviderArtifactCapture({ + key: `${input.execution.binding.companyId}:${manifest.digest}`, signal, reuse, + capture: () => captureRemoteNativeProviderArtifacts({ runner: remoteCommandRunner, + remoteCwd: remoteTarget.remoteCwd, launch, manifest, signal, context: input.executionContext }, + source => retainNativeProviderArtifacts({ db: input.db, storage: artifactStorage, owner: backupOwner, source, signal })), + }); } : undefined, bindDiagnosticSession: session => { if (replacementBackend) return replacementBackend.bindDiagnosticSession?.(session);