diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 5e208871f66..cda6f9ab3ac 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1874,6 +1874,14 @@ versions across directories. With these artifacts preinstalled, startup links and verifies them without uploading a binary or installing packages. Deploy the updated sandbox image with the matching runner qualification changes. +When the provider allows concurrent sync operations, native startup prepares the +runner and the default image Codex harness together, with at most two operations +in flight. It still verifies the runner contract, selected transport, executable +hash, and final launch paths. Both preparations must settle before startup can +return a failure. Providers without this capability, explicit harness uploads, +and npm installs keep the serial path. See the +[native artifact controls](../server/src/services/native-runtime/native-artifact-preparation.test.ts). + ### Remote GitHub launcher staging Managed GitHub launchers stage their nine text files in one bounded control diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index f49bba35587..359b6b0e64f 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -96,6 +96,15 @@ or Legacy CLI check cannot replace that runtime evidence. Missing dependencies, credentials, invalid models and incompatible settings fail without silently changing runners. Account login remains board/owner-bound. +Scoped artifact-preparation source evidence: [native preparation](../../server/src/services/native-runtime/native-session-executor.ts) +can overlap runner and default image-harness work only when the selected command +runner declares concurrent sync support. Current executable hash, metadata, +transport, and final-path checks still precede dispatch. The caller waits for +both operations on failure. Explicit harness uploads and npm installs remain +serial. This changes no API or agent grant. The +[real-shell controls](../../server/src/services/native-runtime/native-artifact-preparation.test.ts) +do not qualify the whole validation capability or a live response-time target. + For a linked Codex subscription with server-resolved managed credential provenance, setup checks the resolved owner and stages a private probe copy before local or remote isolation. Without a selected API key, an unbound linked `CODEX_HOME` is diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index adc306c23d2..1e3503dfd5a 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -66,7 +66,13 @@ recipe first. Discovery/creation read company-bound immutable-image evidence without allocating a target. A mutable image or snapshot name does not establish native compatibility. Then test the actual selected runtime with its selected account/model: a legacy CLI or authentication probe alone cannot establish native -readiness. Missing dependencies, failed authentication, invalid models and +readiness. For an image-provided Codex harness, verify both final executables +after concurrent artifact preparation on an eligible provider. Repeat with a +failed runner probe and confirm the harness work has settled before failure +returns. Providers without concurrent sync and explicit harness installs remain +serial. The [real-shell artifact controls](../../server/src/services/native-runtime/native-artifact-preparation.test.ts) +cover these source contracts; they do not establish live reply latency. +Missing dependencies, failed authentication, invalid models and incompatible custom settings must remain actionable errors without silent runner switching. Saved unchanged execution can still be tested without applying new selection policy, but runtime/account validation must still run. diff --git a/server/src/services/native-runtime/native-artifact-preparation.test.ts b/server/src/services/native-runtime/native-artifact-preparation.test.ts new file mode 100644 index 00000000000..5e5186fe2a0 --- /dev/null +++ b/server/src/services/native-runtime/native-artifact-preparation.test.ts @@ -0,0 +1,155 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setImmediate } from "node:timers/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { createRemoteNativeArtifactPreparation } from "./native-session-executor.js"; + +type Request = Parameters[0]; +const roots: string[] = []; +const metadata = { + schema: "paperclip-runner/runnerd-build-metadata/v1", + binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", + binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], + prpTransportModes: ["dial_ws_loopback", "dial_wss", "listen_ws"], +}; + +async function fixture(input: { + concurrent?: boolean; + before?: (request: Request) => Promise; +} = {}) { + const root = await mkdtemp(join(tmpdir(), "native-artifact-preparation-")); + roots.push(root); + const home = join(root, "home"); + const bin = join(home, ".local", "bin"); + const cwd = join(root, "workspace"); + await mkdir(bin, { recursive: true }); + await mkdir(cwd); + const runnerSource = join(bin, "paperclip-runnerd"); + const codexSource = join(bin, "codex"); + const runnerScript = `#!/bin/sh\n[ "$1" = --build-metadata ] || exit 17\nprintf '%s\\n' '${JSON.stringify(metadata)}'\n`; + await writeFile(runnerSource, runnerScript, { mode: 0o700 }); + await writeFile(codexSource, "#!/bin/sh\nprintf '%s\\n' 'codex-cli 0.160.0'\n", { mode: 0o700 }); + let active = 0; + let maxActive = 0; + const requests: Request[] = []; + const runner: CommandManagedRuntimeRunner = { + allowConcurrentSyncOperations: input.concurrent ?? true, + async execute(request) { + requests.push(request); + active++; + maxActive = Math.max(maxActive, active); + try { + await input.before?.(request); + return await new Promise((resolve, reject) => { + const startedAt = new Date().toISOString(); + const child = execFile(request.command, request.args ?? [], { + cwd: request.cwd ?? cwd, + env: { HOME: home, PATH: `${bin}:${process.env.PATH ?? "/usr/bin:/bin"}` }, + timeout: 10_000, + }, (error, stdout, stderr) => { + if (error && typeof error.code !== "number") return reject(error); + resolve({ exitCode: typeof error?.code === "number" ? error.code : 0, + signal: error?.signal ?? null, pid: child.pid ?? null, startedAt, + timedOut: error?.killed === true, stdout, stderr }); + }); + child.stdin?.end(request.stdin); + }); + } finally { + active--; + } + }, + }; + const runtimeRoot = join(cwd, ".paperclip-runtime"); + const runnerTarget = join(runtimeRoot, "bin", "paperclip-runnerd"); + const codexTarget = join(runtimeRoot, "bin", "codex"); + const options: Parameters[0] = { + target: { kind: "remote", transport: "sandbox", providerKey: "daytona", remoteCwd: cwd, runner }, + runner, + remoteBinary: runnerTarget, + controllerRunnerBinary: runnerSource, + remoteRuntimeRoot: runtimeRoot, + remoteCodexBinary: codexTarget, + targetPlatformArtifactVerified: true, + }; + return { options, requests, runnerSource, runnerTarget, codexSource, codexTarget, + runnerScript, active: () => active, maxActive: () => maxActive }; +} + +const discovers = (request: Request, name: string) => request.command === "sh" + && request.args?.some(value => value.includes(`for candidate in /opt/paperclip-runner/bin/${name} `)); + +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +describe("remote native artifact preparation", () => { + it("overlaps the default runner and harness while retaining both executable checks", async () => { + const f = await fixture(); + await createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws"); + expect(f.maxActive()).toBe(2); + expect(f.active()).toBe(0); + expect(await readFile(f.runnerTarget, "utf8")).toBe(f.runnerScript); + expect(await readFile(f.runnerSource, "utf8")).toBe(f.runnerScript); + expect(f.requests.filter(request => request.command === f.runnerTarget && request.args?.[0] === "--build-metadata")).toHaveLength(1); + expect(f.requests.filter(request => request.command === f.codexTarget && request.args?.[0] === "--version")).toHaveLength(1); + expect(f.requests.some(request => request.command === "npm")).toBe(false); + }); + + it("keeps providers without concurrent sync support serial", async () => { + const f = await fixture({ concurrent: false }); + await createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws"); + expect(f.maxActive()).toBe(1); + }); + + it("keeps an explicit harness install after runner verification", async () => { + const f = await fixture(); + await createRemoteNativeArtifactPreparation({ ...f.options, runnerRemoteCodexPath: f.codexSource }).prepare("listen_ws"); + expect(f.maxActive()).toBe(1); + const verified = f.requests.findIndex(request => request.command === f.runnerTarget); + const harness = f.requests.findIndex(request => request.command === f.codexTarget); + expect(verified).toBeGreaterThan(-1); + expect(harness).toBeGreaterThan(verified); + }); + + it("waits for an admitted harness operation before returning the runner failure", async () => { + const failure = new Error("runner discovery unavailable"); + let releaseHarness!: () => void; + const harnessGate = new Promise(resolve => { releaseHarness = resolve; }); + let harnessStarted = false; + const f = await fixture({ before: async request => { + if (discovers(request, "paperclip-runnerd")) throw failure; + if (discovers(request, "codex")) { harnessStarted = true; await harnessGate; } + } }); + let rejected = false; + const work = createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws").catch(error => { + rejected = true; + return error; + }); + try { + await setImmediate(); + expect(harnessStarted).toBe(true); + expect(rejected).toBe(false); + } finally { + releaseHarness(); + expect(await work).toBe(failure); + } + expect(f.active()).toBe(0); + expect(f.requests.some(request => request.command === f.codexTarget)).toBe(true); + }); + + it("preserves the runner failure when both concurrent preparations fail", async () => { + const runnerFailure = new Error("runner discovery unavailable"); + const harnessFailure = new Error("harness discovery unavailable"); + const f = await fixture({ before: async request => { + if (discovers(request, "paperclip-runnerd")) throw runnerFailure; + if (discovers(request, "codex")) throw harnessFailure; + } }); + await expect(createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws")).rejects.toBe(runnerFailure); + expect(f.active()).toBe(0); + }); +}); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 51ff8ed5b4f..13aed940033 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -13372,203 +13372,215 @@ export function createRemoteNativeArtifactPreparation(input: { return false; } }; - let runnerArtifactPrepared = false; - if ( - input.reuseRetainedRunner === true && - existsSync(controllerRunnerBinary) - ) { - runnerArtifactPrepared = await measureNativeRunnerSpan( - input.trace, - "runner.artifact.verify_retained", - () => matchesControllerRunnerArtifact(remoteBinary), - ); - } - const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null; - if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) { - const preinstalledRunner = await measureNativeRunnerSpan( - input.trace, - "runner.artifact.discover", - () => discoverPreinstalledExecutable("paperclip-runnerd"), - ); - if (preinstalledRunner) { - try { - await measureNativeRunnerSpan( - input.trace, - "runner.artifact.verify_preinstalled", - () => verifyRemoteRunner(requiredMode, preinstalledRunner), - ); - if (existsSync(controllerRunnerBinary) && - !await matchesControllerRunnerArtifact(preinstalledRunner)) { - throw new Error("runner_remote_preinstalled_artifact_mismatch"); + const prepareRunner = async () => { + let runnerArtifactPrepared = false; + if ( + input.reuseRetainedRunner === true && + existsSync(controllerRunnerBinary) + ) { + runnerArtifactPrepared = await measureNativeRunnerSpan( + input.trace, + "runner.artifact.verify_retained", + () => matchesControllerRunnerArtifact(remoteBinary), + ); + } + const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null; + if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) { + const preinstalledRunner = await measureNativeRunnerSpan( + input.trace, + "runner.artifact.discover", + () => discoverPreinstalledExecutable("paperclip-runnerd"), + ); + if (preinstalledRunner) { + try { + await measureNativeRunnerSpan( + input.trace, + "runner.artifact.verify_preinstalled", + () => verifyRemoteRunner(requiredMode, preinstalledRunner), + ); + if (existsSync(controllerRunnerBinary) && + !await matchesControllerRunnerArtifact(preinstalledRunner)) { + throw new Error("runner_remote_preinstalled_artifact_mismatch"); + } + await measureNativeRunnerSpan( + input.trace, + "runner.artifact.link", + () => linkPreinstalledExecutable(preinstalledRunner, remoteBinary), + ); + runnerArtifactPrepared = true; + await input.onLog?.( + "stderr", + "[paperclip-runner] using preinstalled runnerd from the sandbox image\n", + ); + } catch { + runnerArtifactPrepared = false; } - await measureNativeRunnerSpan( - input.trace, - "runner.artifact.link", - () => linkPreinstalledExecutable(preinstalledRunner, remoteBinary), - ); - runnerArtifactPrepared = true; - await input.onLog?.( - "stderr", - "[paperclip-runner] using preinstalled runnerd from the sandbox image\n", - ); - } catch { - runnerArtifactPrepared = false; } } - } - if (!runnerArtifactPrepared) { - const sourceBinary = controllerRunnerBinary; - if (!existsSync(sourceBinary)) { - throw new Error("runner_remote_artifact_unavailable"); - } - if (!explicitRemoteBinary && !input.targetPlatformArtifactVerified) { - const platform = await remoteCommandRunner.execute({ - command: "sh", - args: ["-c", "uname -s; uname -m"], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 10_000, + if (!runnerArtifactPrepared) { + const sourceBinary = controllerRunnerBinary; + if (!existsSync(sourceBinary)) { + throw new Error("runner_remote_artifact_unavailable"); + } + if (!explicitRemoteBinary && !input.targetPlatformArtifactVerified) { + const platform = await remoteCommandRunner.execute({ + command: "sh", + args: ["-c", "uname -s; uname -m"], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 10_000, + }); + const [remoteOs = "", remoteArch = ""] = platform.stdout + .trim() + .split(/\r?\n/); + const localOs = + process.platform === "darwin" + ? "Darwin" + : process.platform === "linux" + ? "Linux" + : process.platform; + const localArch = + process.arch === "x64" + ? "x86_64" + : process.arch === "arm64" + ? "aarch64" + : process.arch; + const archMatches = + remoteArch === localArch || + (localArch === "aarch64" && remoteArch === "arm64"); + if ( + platform.exitCode !== 0 || + platform.timedOut || + remoteOs !== localOs || + !archMatches + ) { + throw new Error( + "runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture", + ); + } + } + await stageRemoteRunnerFile({ + target: remoteTarget, + runner: remoteCommandRunner, + sourcePath: sourceBinary, + targetPath: remoteBinary, + mode: 0o700, }); - const [remoteOs = "", remoteArch = ""] = platform.stdout - .trim() - .split(/\r?\n/); - const localOs = - process.platform === "darwin" - ? "Darwin" - : process.platform === "linux" - ? "Linux" - : process.platform; - const localArch = - process.arch === "x64" - ? "x86_64" - : process.arch === "arm64" - ? "aarch64" - : process.arch; - const archMatches = - remoteArch === localArch || - (localArch === "aarch64" && remoteArch === "arm64"); - if ( - platform.exitCode !== 0 || - platform.timedOut || - remoteOs !== localOs || - !archMatches - ) { + } + await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () => + verifyRemoteRunner(requiredMode), + ); + }; + const prepareHarness = async () => { + if (remoteCodexBinary && explicitRemoteCodex) { + if (!existsSync(explicitRemoteCodex)) { + throw new Error("runner_remote_codex_artifact_unavailable"); + } + await stageRemoteRunnerFile({ + target: remoteTarget, + runner: remoteCommandRunner, + sourcePath: explicitRemoteCodex, + targetPath: remoteCodexBinary, + mode: 0o700, + }); + await verifyRemoteCodex(); + } + if (remoteCodexBinary && remoteCodexNpmSpec) { + let usedPreinstalledCodex = false; + const preinstalledCodex = await measureNativeRunnerSpan( + input.trace, + "harness.artifact.discover", + () => discoverPreinstalledExecutable("codex"), + ); + if (preinstalledCodex) { + try { + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify_preinstalled", + () => verifyRemoteCodex(preinstalledCodex), + ); + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.link", + () => + linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), + ); + usedPreinstalledCodex = true; + await input.onLog?.( + "stderr", + "[paperclip-runner] using preinstalled Codex from the sandbox image\n", + ); + } catch { + usedPreinstalledCodex = false; + } + } + if (!usedPreinstalledCodex) { + const installRoot = posix.join( + remoteRuntimeRoot!, + "harnesses", + "codex", + ); + const installResult = await remoteCommandRunner.execute({ + command: "npm", + args: [ + "install", + "--prefix", + installRoot, + "--no-audit", + "--no-fund", + remoteCodexNpmSpec, + ], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 180_000, + }); + if (installResult.exitCode !== 0 || installResult.timedOut) { + throw new Error("runner_remote_codex_install_failed"); + } + } + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify", + () => verifyRemoteCodex(), + ); + } + if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) { + const preinstalledCodex = await measureNativeRunnerSpan( + input.trace, + "harness.artifact.discover", + () => discoverPreinstalledExecutable("codex"), + ); + if (!preinstalledCodex) { throw new Error( - "runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture", + "runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC", ); } - } - await stageRemoteRunnerFile({ - target: remoteTarget, - runner: remoteCommandRunner, - sourcePath: sourceBinary, - targetPath: remoteBinary, - mode: 0o700, - }); - } - await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () => - verifyRemoteRunner(requiredMode), - ); - if (remoteCodexBinary && explicitRemoteCodex) { - if (!existsSync(explicitRemoteCodex)) { - throw new Error("runner_remote_codex_artifact_unavailable"); - } - await stageRemoteRunnerFile({ - target: remoteTarget, - runner: remoteCommandRunner, - sourcePath: explicitRemoteCodex, - targetPath: remoteCodexBinary, - mode: 0o700, - }); - await verifyRemoteCodex(); - } - if (remoteCodexBinary && remoteCodexNpmSpec) { - let usedPreinstalledCodex = false; - const preinstalledCodex = await measureNativeRunnerSpan( - input.trace, - "harness.artifact.discover", - () => discoverPreinstalledExecutable("codex"), - ); - if (preinstalledCodex) { - try { - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify_preinstalled", - () => verifyRemoteCodex(preinstalledCodex), - ); - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.link", - () => - linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), - ); - usedPreinstalledCodex = true; - await input.onLog?.( - "stderr", - "[paperclip-runner] using preinstalled Codex from the sandbox image\n", - ); - } catch { - usedPreinstalledCodex = false; - } - } - if (!usedPreinstalledCodex) { - const installRoot = posix.join( - remoteRuntimeRoot!, - "harnesses", - "codex", + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify_preinstalled", + () => verifyRemoteCodex(preinstalledCodex), ); - const installResult = await remoteCommandRunner.execute({ - command: "npm", - args: [ - "install", - "--prefix", - installRoot, - "--no-audit", - "--no-fund", - remoteCodexNpmSpec, - ], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 180_000, - }); - if (installResult.exitCode !== 0 || installResult.timedOut) { - throw new Error("runner_remote_codex_install_failed"); - } - } - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify", - () => verifyRemoteCodex(), - ); - } - if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) { - const preinstalledCodex = await measureNativeRunnerSpan( - input.trace, - "harness.artifact.discover", - () => discoverPreinstalledExecutable("codex"), - ); - if (!preinstalledCodex) { - throw new Error( - "runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC", + await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () => + linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), + ); + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify", + () => verifyRemoteCodex(), + ); + await input.onLog?.( + "stderr", + "[paperclip-runner] using preinstalled Codex from the sandbox image\n", ); } - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify_preinstalled", - () => verifyRemoteCodex(preinstalledCodex), - ); - await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () => - linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), - ); - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify", - () => verifyRemoteCodex(), - ); - await input.onLog?.( - "stderr", - "[paperclip-runner] using preinstalled Codex from the sandbox image\n", - ); + }; + if (remoteCommandRunner.allowConcurrentSyncOperations === true && + remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) { + const settled = await Promise.allSettled([prepareRunner(), prepareHarness()]); + for (const result of settled) if (result.status === "rejected") throw result.reason; + } else { + await prepareRunner(); + await prepareHarness(); } }; return { prepare, verifyRemoteRunner, verifyRemoteCodex };