From d385c430bfdcf53bf369fd179f4fd2a912a116fc Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 1 Oct 2026 22:58:39 -0500 Subject: [PATCH] test(runner): qualify Pi through installed public CLI Verify installed CLI/server pins and default runtime resolution for explicit Pi Product cells. Keep Cursor and Copilot pending while qualified Pi runs without the candidate override. Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 23 +++++++ tests/runner-e2e/extended-harnesses.test.ts | 8 ++- tests/runner-e2e/harness-env.ts | 8 ++- tests/runner-e2e/installed-cli.test.ts | 69 +++++++++++++++++++ tests/runner-e2e/installed-cli.ts | 52 ++++++++++++++ tests/runner-e2e/launch.ts | 8 ++- tests/runner-e2e/pi-controls-evidence.test.ts | 2 +- tests/runner-e2e/pi-native-cases.test.ts | 4 +- tests/runner-e2e/server.ts | 12 +++- 9 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 tests/runner-e2e/installed-cli.test.ts create mode 100644 tests/runner-e2e/installed-cli.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 4bb5da377b7..57114f5329a 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1797,3 +1797,26 @@ the evaluated checkout byte for byte. The skill snapshot and provider run eviden are retained privately alongside the grading checkpoints for failure diagnosis. Claude receives a fresh provider home and config directory inside the disposable workspace so a user's installed skill cannot shadow the managed skill under test. + +### Published-install Pi lane + +The 26 explicit Pi cells can run against an independently installed public CLI +and server instead of the source CLI. Supply all four reviewed pins: +`PAPERCLIP_RUNNER_E2E_INSTALLED_CLI` (canonical `paperclipai/dist/index.js`), +`PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256` (bare SHA-256), +`PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT` (canonical resolved public server +package root), and `PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256` (its +`dist/index.js` SHA-256). The CLI must resolve that exact server dependency, with +matching public package versions. Build/install provenance and the complete +installed dependency/assets inventory remain separate required evidence; the +entrypoint checks alone do not prove that closure. + +Run the explicit `paperclipai runtime setup pi` for that installation first. +This lane rejects candidate flags, local/remote daemon overrides, provider asset +or pack overrides, and Node injection. It launches the installed JavaScript CLI +without a TypeScript loader, rechecks its pins on controller restart, and records +`installed-cli-admission.json` in private attempt evidence. It supports only the +explicit Pi extended, native, controls and warm suites, which require none of the +source-only response barriers. Cursor/Copilot qualification gates stay intact. +Pi's historical `qualificationCandidate` fixture selector remains a roster key; +it no longer grants an opt-in when the source Pi declaration admits normal use. diff --git a/tests/runner-e2e/extended-harnesses.test.ts b/tests/runner-e2e/extended-harnesses.test.ts index f797112e078..74d8e7b551b 100644 --- a/tests/runner-e2e/extended-harnesses.test.ts +++ b/tests/runner-e2e/extended-harnesses.test.ts @@ -47,7 +47,9 @@ describe("extended ACP harness qualification", () => { expect(cell.task.buildMatchers("nonce", cell)).toContainEqual({ kind: "environment", expected: cell.environment.id }); const config = cell.profile.buildAgent({ executionId: "warm", environmentId: "env", environmentFixtureId: cell.environment.id, workspacePath: "/tmp/workspace", secretRefs: { [cell.profile.credential]: { type: "secret_ref", secretId: "11111111-1111-4111-8111-111111111111", version: "latest" } } }).adapterConfig; expect(config).toMatchObject({ lifecycleMode: "warm", idleTimeoutMs: 300_000, timeoutSec: 120 }); - expect(JSON.parse(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: cell.profile.qualificationCandidate, model: cell.profile.model }]); + const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION; + if (cell.profile.qualificationCandidate === "pi") expect(admission).toBeUndefined(); + else expect(JSON.parse(admission!)).toEqual([{ agent: cell.profile.qualificationCandidate, model: cell.profile.model }]); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit"); } expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => cell.suite.id === "rich-acp-warm-continuity")).toBe(false); @@ -57,7 +59,9 @@ describe("extended ACP harness qualification", () => { const cells = runnerMatrix.filter(cell => cell.suite.id === suiteId); expect(new Set(cells.map(cell => cell.environment.id))).toEqual(new Set(["local", "daytona"])); for (const cell of cells) { - expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeDefined(); + const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION; + if (cell.profile.qualificationCandidate === "pi") expect(admission).toBeUndefined(); + else expect(admission).toBeDefined(); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, profile: { ...cell.profile, qualificationCandidate: cell.profile.qualificationCandidate === "pi" ? "cursor" : "pi" } }])).toThrow("explicit"); } } diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 0748d6ec6f0..511cb37b34d 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -1,3 +1,5 @@ +import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; +import { installedCliKeys } from "./installed-cli.js"; import path from "node:path"; import { chatNeedsApiTools, isManagedHiringCase } from "./chat-cases.js"; import { CREDENTIAL_NAMES } from "./types.js"; @@ -125,6 +127,10 @@ export function buildRunnerE2EProcessEnvironment( if (!admittedSuite || !execution.suite.manualOnly) { throw new Error("Candidate qualification requires an explicit provider qualification suite"); } + if (agent === "pi" && QUALIFIED_ACPX_PROFILES.pi.qualificationStatus !== "pending") { + if (execution.profile.model !== QUALIFIED_ACPX_PROFILES.pi.qualificationModel) throw new Error("Qualified Pi requires its exact declared model"); + continue; + } const prior = candidates.get(agent); if (prior !== undefined && prior !== execution.profile.model) throw new Error("Conflicting candidate models"); candidates.set(agent, execution.profile.model); @@ -174,7 +180,7 @@ export function buildPaperclipServerEnvironment( ]) { delete result[key]; } - for (const key of GENERATED_SERVER_SECRET_KEYS) delete result[key]; + for (const key of [...GENERATED_SERVER_SECRET_KEYS, ...installedCliKeys]) delete result[key]; Object.assign(result, overrides); return result; } diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.ts new file mode 100644 index 00000000000..a351afebe52 --- /dev/null +++ b/tests/runner-e2e/installed-cli.test.ts @@ -0,0 +1,69 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { assertInstalledCliSelection, installedCliKeys, verifyInstalledCli } from "./installed-cli.js"; +import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js"; +const cells = runnerMatrix.filter(e => e.profile.id === "runner-acpx-pi"); +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const hash = (value: string) => createHash("sha256").update(value).digest("hex"); +async function fixture() { + const root = await realpath(await mkdtemp(join(tmpdir(), "e2e-installed-cli-"))); roots.push(root); + const cli = join(root, "node_modules/paperclipai"); const server = join(root, "node_modules/@paperclipai/server"); + await mkdir(join(cli, "dist"), { recursive: true }); await mkdir(join(server, "dist"), { recursive: true }); + await writeFile(join(cli, "package.json"), JSON.stringify({ name: "paperclipai", version: "1.2.3", type: "module", bin: { paperclipai: "./dist/index.js" } })); + await writeFile(join(server, "package.json"), JSON.stringify({ name: "@paperclipai/server", version: "1.2.3", type: "module", exports: { ".": { import: "./dist/index.js" } } })); + await writeFile(join(cli, "dist/index.js"), "// installed public CLI"); await writeFile(join(server, "dist/index.js"), "// installed public server"); + const env = { PAPERCLIP_RUNNER_E2E_INSTALLED_CLI: join(cli, "dist/index.js"), PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256: hash("// installed public CLI"), PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT: server, PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256: hash("// installed public server") }; + return { root, cli, server, env }; +} +it("admits all26 current Pi cases through pinned installed public CLI/server with no overrides", async () => { + const { env, server } = await fixture(); expect(cells).toHaveLength(26); + for (const cell of cells) expect(await verifyInstalledCli(env, [cell])).toMatchObject({ serverRoot: server, defaultRuntimeResolution: true, qualificationOverride: false }); +}); +it("qualified Pi always drops ambient candidate admission while pending C/C retain exact gates", () => { + for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); + for (const agent of ["cursor", "copilot"]) { + const cell = runnerMatrix.find(e => e.profile.qualificationCandidate === agent)!; + expect(JSON.parse(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent, model: cell.profile.model }]); + } + const changed = { ...cells[0]!, profile: { ...cells[0]!.profile, model: "wrong-model" } }; + expect(() => buildRunnerE2EProcessEnvironment({}, [changed])).toThrow("exact declared model"); +}); +it("rejects partial authority, unsupported cells and every runtime/path override", async () => { + const { env } = await fixture(); + for (const key of installedCliKeys) { const changed = { ...env, [key]: undefined }; await expect(verifyInstalledCli(changed, [cells[0]!])).rejects.toThrow("complete pins"); } + expect(() => assertInstalledCliSelection(env, [])).toThrow("explicit supported Pi"); + expect(() => assertInstalledCliSelection(env, [runnerMatrix.find(e => e.profile.qualificationCandidate === "cursor")!])).toThrow("explicit supported Pi"); + for (const key of ["PAPERCLIP_RUNNER_BINARY", "PAPERCLIP_RUNNER_REMOTE_BINARY_PATH", "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "PAPERCLIP_RUNNER_ACPX_QUALIFICATION", "PAPERCLIP_ACPX_BUILTIN_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST", "NODE_OPTIONS", "NODE_PATH"]) await expect(verifyInstalledCli({ ...env, [key]: "foreign" }, [cells[0]!])).rejects.toThrow("forbids"); +}); +it("rejects stale bytes and another server root instead of falling back to source", async () => { + const f = await fixture(); const other = await fixture(); + await expect(verifyInstalledCli({ ...f.env, PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT: other.server }, [cells[0]!])).rejects.toThrow("foreign server"); + await expect(verifyInstalledCli({ ...f.env, PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256: "0".repeat(64) }, [cells[0]!])).rejects.toThrow("reviewed pins"); + await writeFile(join(f.server, "dist/index.js"), "changed"); + await expect(verifyInstalledCli(f.env, [cells[0]!])).rejects.toThrow("reviewed pins"); +}); +it("rejects linked entrypoints and source-export package identities", async () => { + const f = await fixture(); const other = await fixture(); + await rm(join(f.cli, "dist/index.js")); await symlink(join(other.cli, "dist/index.js"), join(f.cli, "dist/index.js")); + await expect(verifyInstalledCli(f.env, [cells[0]!])).rejects.toThrow("canonical and unlinked"); + await rm(join(f.cli, "dist/index.js")); await writeFile(join(f.cli, "dist/index.js"), "// installed public CLI"); + await writeFile(join(f.server, "package.json"), JSON.stringify({ name: "@paperclipai/server", version: "1.2.3", exports: { ".": "./src/index.ts" } })); + await expect(verifyInstalledCli(f.env, [cells[0]!])).rejects.toThrow("public package identity"); +}); +it("keeps installed proof inputs out of the production server environment", async () => { + const { env } = await fixture(); const server = buildPaperclipServerEnvironment(env); + for (const key of installedCliKeys) expect(server[key]).toBeUndefined(); + await expect(verifyInstalledCli({}, [])).resolves.toBeUndefined(); +}); + +it.skipIf(process.platform === "win32")("rejects a FIFO entrypoint without waiting for a writer", async () => { + const f = await fixture(); await rm(f.env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI); + execFileSync("/usr/bin/mkfifo", [f.env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI], { timeout: 5000, env: {} }); + await expect(verifyInstalledCli(f.env, [cells[0]!])).rejects.toThrow("bounded regular file"); +}); diff --git a/tests/runner-e2e/installed-cli.ts b/tests/runner-e2e/installed-cli.ts new file mode 100644 index 00000000000..b24ddf26166 --- /dev/null +++ b/tests/runner-e2e/installed-cli.ts @@ -0,0 +1,52 @@ +/** Optional published-install lane. No production hook or admission override. */ +import { constants } from "node:fs"; +import { lstat, open, realpath } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { findPackageJSON } from "node:module"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import type { MatrixExecution } from "./types.js"; + +export const installedCliKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_CLI", "PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256"] as const; +const overrideKeys = ["PAPERCLIP_RUNNER_BINARY", "PAPERCLIP_RUNNER_REMOTE_BINARY_PATH", "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "PAPERCLIP_RUNNER_ACPX_QUALIFICATION", "PAPERCLIP_ACPX_BUILTIN_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST"] as const; +const suites = new Set(["extended-harnesses", "pi-native", "pi-controls", "rich-acp-warm-continuity"]); +export function usesInstalledCli(env: NodeJS.ProcessEnv): boolean { + return installedCliKeys.some(key => env[key] !== undefined); +} +export function assertInstalledCliSelection(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[]): void { + if (!usesInstalledCli(env)) return; + if (!installedCliKeys.every(key => Boolean(env[key])) || executions.length === 0 || executions.some(e => e.profile.id !== "runner-acpx-pi" || e.profile.qualificationCandidate !== "pi" || !e.suite.manualOnly || !suites.has(e.suite.id))) throw new Error("Installed CLI proof requires complete pins and explicit supported Pi cases"); + for (const key of overrideKeys) if (env[key] !== undefined) throw new Error(`Installed CLI proof forbids ${key}`); + if (env.NODE_OPTIONS !== undefined || env.NODE_PATH !== undefined) throw new Error("Installed CLI proof forbids ambient Node injection"); +} +async function readOwned(path: string, maximum: number): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = await handle.stat({ bigint: true }); + if (!before.isFile() || before.size < 1n || before.size > BigInt(maximum)) throw new Error("Installed CLI proof requires a bounded regular file"); + const bytes = await handle.readFile(); const after = await handle.stat({ bigint: true }); const named = await lstat(path, { bigint: true }); + if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs || named.dev !== before.dev || named.ino !== before.ino || named.isSymbolicLink() || bytes.length !== Number(before.size)) throw new Error("Installed CLI proof file changed during admission"); + return bytes; + } finally { await handle.close(); } +} +export async function verifyInstalledCli(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[]) { + if (!usesInstalledCli(env)) return undefined; + assertInstalledCliSelection(env, executions); + const entry = env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI!; + const serverRoot = env.PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT!; + for (const path of [entry, serverRoot]) if (!isAbsolute(path) || resolve(path) !== path || await realpath(path) !== path) throw new Error("Installed CLI proof paths must be canonical and unlinked"); + if (!entry.endsWith("/dist/index.js")) throw new Error("Installed CLI proof requires published dist/index.js"); + const cliRoot = resolve(dirname(entry), ".."); + const cli = JSON.parse((await readOwned(join(cliRoot, "package.json"), 65536)).toString()); + const manifest = findPackageJSON("@paperclipai/server", pathToFileURL(entry)); + if (!manifest || await realpath(manifest) !== join(serverRoot, "package.json")) throw new Error("Installed CLI resolved a foreign server dependency"); + const server = JSON.parse((await readOwned(join(serverRoot, "package.json"), 65536)).toString()); + if (cli.name !== "paperclipai" || cli.bin?.paperclipai !== "./dist/index.js" || server.name !== "@paperclipai/server" || typeof cli.version !== "string" || cli.version !== server.version || server.exports?.["."]?.import !== "./dist/index.js") throw new Error("Installed CLI/server public package identity differs"); + const serverEntry = join(serverRoot, "dist/index.js"); + if (await realpath(serverEntry) !== serverEntry) throw new Error("Installed server entrypoint escapes its package"); + const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + const cliSha256 = digest(await readOwned(entry, 64 * 1024 * 1024)); + const serverSha256 = digest(await readOwned(serverEntry, 16 * 1024 * 1024)); + if (cliSha256 !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256 || serverSha256 !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256) throw new Error("Installed CLI/server bytes differ from their reviewed pins"); + return { schema: "paperclip.e2e.installed-cli-admission/v1", entry, cliRoot, cliSha256, serverRoot, serverEntry, serverSha256, version: cli.version, defaultRuntimeResolution: true, qualificationOverride: false }; +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 0492112083d..e85d98ab737 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,3 +1,4 @@ +import { assertInstalledCliSelection, verifyInstalledCli } from "./installed-cli.js"; import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js"; import { randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; @@ -471,6 +472,8 @@ async function runAttempt(input: { "One isolated runner E2E harness cannot mix profiles or environments", ); } + assertInstalledCliSelection(process.env, executions); + const installedCli = await verifyInstalledCli(process.env, executions); const startedAtMs = Date.now(); const sharedMemoryBaseline = snapshotDarwinSharedMemory(); const temporaryRoot = await mkdtemp( @@ -516,7 +519,8 @@ async function runAttempt(input: { betterAuthSecret, ]); attemptSecrets = credentials; - const runnerBinary = resolvePaperclipRunnerBinaryForHarness( + if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 }); + const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness( executions, repositoryRoot, ); @@ -534,7 +538,7 @@ async function runAttempt(input: { PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(privateDir, "server.log"), PAPERCLIP_RUNNER_BINARY: runnerBinary, PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: - resolvePaperclipRemoteRunnerBinaryForHarness(executions, runnerBinary), + installedCli ? undefined : resolvePaperclipRemoteRunnerBinaryForHarness(executions, runnerBinary), // Vite's optimized dependency cache embeds revision query strings. A // private per-attempt cache prevents an earlier cell or local rebuild // from producing `504 Outdated Optimize Dep` during browser bootstrap. diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 10b1174bfed..2dcbca125cf 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -125,7 +125,7 @@ describe("Pi controls catalog admission", () => { expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); expect(validateRunnerCatalog()).toHaveLength(469); - for (const cell of cells) expect(JSON.parse(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: cell.profile.model }]); + for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); it("pins the Pi 1 profile and versioned coverage while retaining active Stop identity", () => { diff --git a/tests/runner-e2e/pi-native-cases.test.ts b/tests/runner-e2e/pi-native-cases.test.ts index f99a6b78870..227255d9fd0 100644 --- a/tests/runner-e2e/pi-native-cases.test.ts +++ b/tests/runner-e2e/pi-native-cases.test.ts @@ -29,9 +29,9 @@ describe("Pi native Product qualification", () => { it("admits explicit local and remote Pi native candidates and discards ambient admission", () => { const cell = runnerMatrix.find(row => row.suite.id === "pi-native")!; const source = { PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }; - expect(JSON.parse(buildRunnerE2EProcessEnvironment(source, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: cell.profile.model }]); + expect(buildRunnerE2EProcessEnvironment(source, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); const remote = runnerMatrix.find(row => row.suite.id === "pi-native" && row.environment.id === "daytona")!; - expect(JSON.parse(buildRunnerE2EProcessEnvironment(source, [remote]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: remote.profile.model }]); + expect(buildRunnerE2EProcessEnvironment(source, [remote]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); expect(buildRunnerE2EProcessEnvironment(source, []).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); for (const changed of [ { ...cell, suite: { ...cell.suite, manualOnly: false } }, diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 9acb4292a25..846235401be 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -1,3 +1,5 @@ +import { usesInstalledCli, verifyInstalledCli } from "./installed-cli.js"; +import { runnerMatrix } from "./catalog.js"; import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server-stop.js"; import { runnerE2ETypeScriptProcessArgs } from "./web-server-command.js"; import { qualifyLegacyClaudeCli } from "./legacy-claude-cli.js"; @@ -25,6 +27,12 @@ const configPath = required("PAPERCLIP_CONFIG"); const port = required("PAPERCLIP_RUNNER_E2E_PORT"); const repositoryRoot = path.resolve(import.meta.dirname, "../.."); const paperclipCli = path.join(repositoryRoot, "tests/runner-e2e/server-entry.ts"); +const executionIds: string[] = JSON.parse(process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]"); +const selectedExecutions = usesInstalledCli(process.env) ? executionIds.map(id => { + const execution = runnerMatrix.find(row => row.id === id); + if (!execution) throw new Error("Unknown installed CLI proof execution"); + return execution; +}) : []; const { controlDirectory, restartRequestPath, @@ -110,9 +118,11 @@ async function startServer() { if (shutdownRequested()) { throw new Error("Refusing to start Paperclip after wrapper shutdown"); } + // Recheck bytes and dependency resolution on every controller restart. + const installed = await verifyInstalledCli(process.env, selectedExecutions); const candidate = spawn( process.execPath, - runnerE2ETypeScriptProcessArgs(repositoryRoot, paperclipCli, ["onboard", "--yes", "--run"]), + installed ? [installed.entry, "onboard", "--yes", "--run"] : runnerE2ETypeScriptProcessArgs(repositoryRoot, paperclipCli, ["onboard", "--yes", "--run"]), { cwd: repositoryRoot, env: definedServerEnvironment,