perf(runner): overlap default remote artifact preparation (#16608)

Prepare the native runner and default image Codex harness together on providers that declare concurrent sync support. Preserve all executable checks, wait for both operations on failure, and retain serial explicit harness installs.

Verified with 881 affected cases, full typecheck/build, exact-head full CI and Greptile Base 5/5. The real Daytona component median fell from 839ms to 432ms; full deployed reply latency remains unproven.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-10-16 17:29:27 -05:00
1 parent c0ac2772b0
commit 9a3290fe10
5 files changed
+378 -188

No files matched your search

+8
View File
@@ -1874,6 +1874,14 @@ versions across directories. With these artifacts preinstalled, startup links
and verifies them without uploading a binary or installing packages. Deploy
the updated sandbox image with the matching runner qualification changes.
When the provider allows concurrent sync operations, native startup prepares the
runner and the default image Codex harness together, with at most two operations
in flight. It still verifies the runner contract, selected transport, executable
hash, and final launch paths. Both preparations must settle before startup can
return a failure. Providers without this capability, explicit harness uploads,
and npm installs keep the serial path. See the
[native artifact controls](../server/src/services/native-runtime/native-artifact-preparation.test.ts).
### Remote GitHub launcher staging
Managed GitHub launchers stage their nine text files in one bounded control
+9
View File
@@ -96,6 +96,15 @@ or Legacy CLI check cannot replace that runtime evidence. Missing dependencies,
credentials, invalid models and incompatible settings fail without silently
changing runners. Account login remains board/owner-bound.
Scoped artifact-preparation source evidence: [native preparation](../../server/src/services/native-runtime/native-session-executor.ts)
can overlap runner and default image-harness work only when the selected command
runner declares concurrent sync support. Current executable hash, metadata,
transport, and final-path checks still precede dispatch. The caller waits for
both operations on failure. Explicit harness uploads and npm installs remain
serial. This changes no API or agent grant. The
[real-shell controls](../../server/src/services/native-runtime/native-artifact-preparation.test.ts)
do not qualify the whole validation capability or a live response-time target.
For a linked Codex subscription with server-resolved managed credential provenance,
setup checks the resolved owner and stages a private probe copy before local or
remote isolation. Without a selected API key, an unbound linked `CODEX_HOME` is
+7 -1
View File
@@ -66,7 +66,13 @@ recipe first. Discovery/creation read company-bound immutable-image evidence
without allocating a target. A mutable image or snapshot name does not establish
native compatibility. Then test the actual selected runtime with its selected
account/model: a legacy CLI or authentication probe alone cannot establish native
readiness. Missing dependencies, failed authentication, invalid models and
readiness. For an image-provided Codex harness, verify both final executables
after concurrent artifact preparation on an eligible provider. Repeat with a
failed runner probe and confirm the harness work has settled before failure
returns. Providers without concurrent sync and explicit harness installs remain
serial. The [real-shell artifact controls](../../server/src/services/native-runtime/native-artifact-preparation.test.ts)
cover these source contracts; they do not establish live reply latency.
Missing dependencies, failed authentication, invalid models and
incompatible custom settings must remain actionable errors without silent runner
switching. Saved unchanged execution can still be tested without applying new
selection policy, but runtime/account validation must still run.
@@ -0,0 +1,155 @@
import { execFile } from "node:child_process";
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { setImmediate } from "node:timers/promises";
import { afterEach, describe, expect, it } from "vitest";
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
import { createRemoteNativeArtifactPreparation } from "./native-session-executor.js";
type Request = Parameters<CommandManagedRuntimeRunner["execute"]>[0];
const roots: string[] = [];
const metadata = {
schema: "paperclip-runner/runnerd-build-metadata/v1",
binaryName: "paperclip-runnerd",
packageName: "@paperclipai/paperclip-runner",
binaryContractVersion: 2,
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"],
prpTransportModes: ["dial_ws_loopback", "dial_wss", "listen_ws"],
};
async function fixture(input: {
concurrent?: boolean;
before?: (request: Request) => Promise<void>;
} = {}) {
const root = await mkdtemp(join(tmpdir(), "native-artifact-preparation-"));
roots.push(root);
const home = join(root, "home");
const bin = join(home, ".local", "bin");
const cwd = join(root, "workspace");
await mkdir(bin, { recursive: true });
await mkdir(cwd);
const runnerSource = join(bin, "paperclip-runnerd");
const codexSource = join(bin, "codex");
const runnerScript = `#!/bin/sh\n[ "$1" = --build-metadata ] || exit 17\nprintf '%s\\n' '${JSON.stringify(metadata)}'\n`;
await writeFile(runnerSource, runnerScript, { mode: 0o700 });
await writeFile(codexSource, "#!/bin/sh\nprintf '%s\\n' 'codex-cli 0.160.0'\n", { mode: 0o700 });
let active = 0;
let maxActive = 0;
const requests: Request[] = [];
const runner: CommandManagedRuntimeRunner = {
allowConcurrentSyncOperations: input.concurrent ?? true,
async execute(request) {
requests.push(request);
active++;
maxActive = Math.max(maxActive, active);
try {
await input.before?.(request);
return await new Promise((resolve, reject) => {
const startedAt = new Date().toISOString();
const child = execFile(request.command, request.args ?? [], {
cwd: request.cwd ?? cwd,
env: { HOME: home, PATH: `${bin}:${process.env.PATH ?? "/usr/bin:/bin"}` },
timeout: 10_000,
}, (error, stdout, stderr) => {
if (error && typeof error.code !== "number") return reject(error);
resolve({ exitCode: typeof error?.code === "number" ? error.code : 0,
signal: error?.signal ?? null, pid: child.pid ?? null, startedAt,
timedOut: error?.killed === true, stdout, stderr });
});
child.stdin?.end(request.stdin);
});
} finally {
active--;
}
},
};
const runtimeRoot = join(cwd, ".paperclip-runtime");
const runnerTarget = join(runtimeRoot, "bin", "paperclip-runnerd");
const codexTarget = join(runtimeRoot, "bin", "codex");
const options: Parameters<typeof createRemoteNativeArtifactPreparation>[0] = {
target: { kind: "remote", transport: "sandbox", providerKey: "daytona", remoteCwd: cwd, runner },
runner,
remoteBinary: runnerTarget,
controllerRunnerBinary: runnerSource,
remoteRuntimeRoot: runtimeRoot,
remoteCodexBinary: codexTarget,
targetPlatformArtifactVerified: true,
};
return { options, requests, runnerSource, runnerTarget, codexSource, codexTarget,
runnerScript, active: () => active, maxActive: () => maxActive };
}
const discovers = (request: Request, name: string) => request.command === "sh"
&& request.args?.some(value => value.includes(`for candidate in /opt/paperclip-runner/bin/${name} `));
afterEach(async () => {
await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true })));
});
describe("remote native artifact preparation", () => {
it("overlaps the default runner and harness while retaining both executable checks", async () => {
const f = await fixture();
await createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws");
expect(f.maxActive()).toBe(2);
expect(f.active()).toBe(0);
expect(await readFile(f.runnerTarget, "utf8")).toBe(f.runnerScript);
expect(await readFile(f.runnerSource, "utf8")).toBe(f.runnerScript);
expect(f.requests.filter(request => request.command === f.runnerTarget && request.args?.[0] === "--build-metadata")).toHaveLength(1);
expect(f.requests.filter(request => request.command === f.codexTarget && request.args?.[0] === "--version")).toHaveLength(1);
expect(f.requests.some(request => request.command === "npm")).toBe(false);
});
it("keeps providers without concurrent sync support serial", async () => {
const f = await fixture({ concurrent: false });
await createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws");
expect(f.maxActive()).toBe(1);
});
it("keeps an explicit harness install after runner verification", async () => {
const f = await fixture();
await createRemoteNativeArtifactPreparation({ ...f.options, runnerRemoteCodexPath: f.codexSource }).prepare("listen_ws");
expect(f.maxActive()).toBe(1);
const verified = f.requests.findIndex(request => request.command === f.runnerTarget);
const harness = f.requests.findIndex(request => request.command === f.codexTarget);
expect(verified).toBeGreaterThan(-1);
expect(harness).toBeGreaterThan(verified);
});
it("waits for an admitted harness operation before returning the runner failure", async () => {
const failure = new Error("runner discovery unavailable");
let releaseHarness!: () => void;
const harnessGate = new Promise<void>(resolve => { releaseHarness = resolve; });
let harnessStarted = false;
const f = await fixture({ before: async request => {
if (discovers(request, "paperclip-runnerd")) throw failure;
if (discovers(request, "codex")) { harnessStarted = true; await harnessGate; }
} });
let rejected = false;
const work = createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws").catch(error => {
rejected = true;
return error;
});
try {
await setImmediate();
expect(harnessStarted).toBe(true);
expect(rejected).toBe(false);
} finally {
releaseHarness();
expect(await work).toBe(failure);
}
expect(f.active()).toBe(0);
expect(f.requests.some(request => request.command === f.codexTarget)).toBe(true);
});
it("preserves the runner failure when both concurrent preparations fail", async () => {
const runnerFailure = new Error("runner discovery unavailable");
const harnessFailure = new Error("harness discovery unavailable");
const f = await fixture({ before: async request => {
if (discovers(request, "paperclip-runnerd")) throw runnerFailure;
if (discovers(request, "codex")) throw harnessFailure;
} });
await expect(createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws")).rejects.toBe(runnerFailure);
expect(f.active()).toBe(0);
});
});
@@ -13372,203 +13372,215 @@ export function createRemoteNativeArtifactPreparation(input: {
return false;
}
};
let runnerArtifactPrepared = false;
if (
input.reuseRetainedRunner === true &&
existsSync(controllerRunnerBinary)
) {
runnerArtifactPrepared = await measureNativeRunnerSpan(
input.trace,
"runner.artifact.verify_retained",
() => matchesControllerRunnerArtifact(remoteBinary),
);
}
const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null;
if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) {
const preinstalledRunner = await measureNativeRunnerSpan(
input.trace,
"runner.artifact.discover",
() => discoverPreinstalledExecutable("paperclip-runnerd"),
);
if (preinstalledRunner) {
try {
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.verify_preinstalled",
() => verifyRemoteRunner(requiredMode, preinstalledRunner),
);
if (existsSync(controllerRunnerBinary) &&
!await matchesControllerRunnerArtifact(preinstalledRunner)) {
throw new Error("runner_remote_preinstalled_artifact_mismatch");
const prepareRunner = async () => {
let runnerArtifactPrepared = false;
if (
input.reuseRetainedRunner === true &&
existsSync(controllerRunnerBinary)
) {
runnerArtifactPrepared = await measureNativeRunnerSpan(
input.trace,
"runner.artifact.verify_retained",
() => matchesControllerRunnerArtifact(remoteBinary),
);
}
const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null;
if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) {
const preinstalledRunner = await measureNativeRunnerSpan(
input.trace,
"runner.artifact.discover",
() => discoverPreinstalledExecutable("paperclip-runnerd"),
);
if (preinstalledRunner) {
try {
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.verify_preinstalled",
() => verifyRemoteRunner(requiredMode, preinstalledRunner),
);
if (existsSync(controllerRunnerBinary) &&
!await matchesControllerRunnerArtifact(preinstalledRunner)) {
throw new Error("runner_remote_preinstalled_artifact_mismatch");
}
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.link",
() => linkPreinstalledExecutable(preinstalledRunner, remoteBinary),
);
runnerArtifactPrepared = true;
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled runnerd from the sandbox image\n",
);
} catch {
runnerArtifactPrepared = false;
}
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.link",
() => linkPreinstalledExecutable(preinstalledRunner, remoteBinary),
);
runnerArtifactPrepared = true;
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled runnerd from the sandbox image\n",
);
} catch {
runnerArtifactPrepared = false;
}
}
}
if (!runnerArtifactPrepared) {
const sourceBinary = controllerRunnerBinary;
if (!existsSync(sourceBinary)) {
throw new Error("runner_remote_artifact_unavailable");
}
if (!explicitRemoteBinary && !input.targetPlatformArtifactVerified) {
const platform = await remoteCommandRunner.execute({
command: "sh",
args: ["-c", "uname -s; uname -m"],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
if (!runnerArtifactPrepared) {
const sourceBinary = controllerRunnerBinary;
if (!existsSync(sourceBinary)) {
throw new Error("runner_remote_artifact_unavailable");
}
if (!explicitRemoteBinary && !input.targetPlatformArtifactVerified) {
const platform = await remoteCommandRunner.execute({
command: "sh",
args: ["-c", "uname -s; uname -m"],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
});
const [remoteOs = "", remoteArch = ""] = platform.stdout
.trim()
.split(/\r?\n/);
const localOs =
process.platform === "darwin"
? "Darwin"
: process.platform === "linux"
? "Linux"
: process.platform;
const localArch =
process.arch === "x64"
? "x86_64"
: process.arch === "arm64"
? "aarch64"
: process.arch;
const archMatches =
remoteArch === localArch ||
(localArch === "aarch64" && remoteArch === "arm64");
if (
platform.exitCode !== 0 ||
platform.timedOut ||
remoteOs !== localOs ||
!archMatches
) {
throw new Error(
"runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture",
);
}
}
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: sourceBinary,
targetPath: remoteBinary,
mode: 0o700,
});
const [remoteOs = "", remoteArch = ""] = platform.stdout
.trim()
.split(/\r?\n/);
const localOs =
process.platform === "darwin"
? "Darwin"
: process.platform === "linux"
? "Linux"
: process.platform;
const localArch =
process.arch === "x64"
? "x86_64"
: process.arch === "arm64"
? "aarch64"
: process.arch;
const archMatches =
remoteArch === localArch ||
(localArch === "aarch64" && remoteArch === "arm64");
if (
platform.exitCode !== 0 ||
platform.timedOut ||
remoteOs !== localOs ||
!archMatches
) {
}
await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () =>
verifyRemoteRunner(requiredMode),
);
};
const prepareHarness = async () => {
if (remoteCodexBinary && explicitRemoteCodex) {
if (!existsSync(explicitRemoteCodex)) {
throw new Error("runner_remote_codex_artifact_unavailable");
}
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: explicitRemoteCodex,
targetPath: remoteCodexBinary,
mode: 0o700,
});
await verifyRemoteCodex();
}
if (remoteCodexBinary && remoteCodexNpmSpec) {
let usedPreinstalledCodex = false;
const preinstalledCodex = await measureNativeRunnerSpan(
input.trace,
"harness.artifact.discover",
() => discoverPreinstalledExecutable("codex"),
);
if (preinstalledCodex) {
try {
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify_preinstalled",
() => verifyRemoteCodex(preinstalledCodex),
);
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.link",
() =>
linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary),
);
usedPreinstalledCodex = true;
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled Codex from the sandbox image\n",
);
} catch {
usedPreinstalledCodex = false;
}
}
if (!usedPreinstalledCodex) {
const installRoot = posix.join(
remoteRuntimeRoot!,
"harnesses",
"codex",
);
const installResult = await remoteCommandRunner.execute({
command: "npm",
args: [
"install",
"--prefix",
installRoot,
"--no-audit",
"--no-fund",
remoteCodexNpmSpec,
],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 180_000,
});
if (installResult.exitCode !== 0 || installResult.timedOut) {
throw new Error("runner_remote_codex_install_failed");
}
}
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify",
() => verifyRemoteCodex(),
);
}
if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) {
const preinstalledCodex = await measureNativeRunnerSpan(
input.trace,
"harness.artifact.discover",
() => discoverPreinstalledExecutable("codex"),
);
if (!preinstalledCodex) {
throw new Error(
"runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture",
"runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC",
);
}
}
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: sourceBinary,
targetPath: remoteBinary,
mode: 0o700,
});
}
await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () =>
verifyRemoteRunner(requiredMode),
);
if (remoteCodexBinary && explicitRemoteCodex) {
if (!existsSync(explicitRemoteCodex)) {
throw new Error("runner_remote_codex_artifact_unavailable");
}
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: explicitRemoteCodex,
targetPath: remoteCodexBinary,
mode: 0o700,
});
await verifyRemoteCodex();
}
if (remoteCodexBinary && remoteCodexNpmSpec) {
let usedPreinstalledCodex = false;
const preinstalledCodex = await measureNativeRunnerSpan(
input.trace,
"harness.artifact.discover",
() => discoverPreinstalledExecutable("codex"),
);
if (preinstalledCodex) {
try {
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify_preinstalled",
() => verifyRemoteCodex(preinstalledCodex),
);
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.link",
() =>
linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary),
);
usedPreinstalledCodex = true;
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled Codex from the sandbox image\n",
);
} catch {
usedPreinstalledCodex = false;
}
}
if (!usedPreinstalledCodex) {
const installRoot = posix.join(
remoteRuntimeRoot!,
"harnesses",
"codex",
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify_preinstalled",
() => verifyRemoteCodex(preinstalledCodex),
);
const installResult = await remoteCommandRunner.execute({
command: "npm",
args: [
"install",
"--prefix",
installRoot,
"--no-audit",
"--no-fund",
remoteCodexNpmSpec,
],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 180_000,
});
if (installResult.exitCode !== 0 || installResult.timedOut) {
throw new Error("runner_remote_codex_install_failed");
}
}
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify",
() => verifyRemoteCodex(),
);
}
if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) {
const preinstalledCodex = await measureNativeRunnerSpan(
input.trace,
"harness.artifact.discover",
() => discoverPreinstalledExecutable("codex"),
);
if (!preinstalledCodex) {
throw new Error(
"runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC",
await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () =>
linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary),
);
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify",
() => verifyRemoteCodex(),
);
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled Codex from the sandbox image\n",
);
}
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify_preinstalled",
() => verifyRemoteCodex(preinstalledCodex),
);
await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () =>
linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary),
);
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify",
() => verifyRemoteCodex(),
);
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled Codex from the sandbox image\n",
);
};
if (remoteCommandRunner.allowConcurrentSyncOperations === true &&
remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) {
const settled = await Promise.allSettled([prepareRunner(), prepareHarness()]);
for (const result of settled) if (result.status === "rejected") throw result.reason;
} else {
await prepareRunner();
await prepareHarness();
}
};
return { prepare, verifyRemoteRunner, verifyRemoteCodex };