diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 57114f5329a..6f12639e351 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1820,3 +1820,39 @@ explicit Pi extended, native, controls and warm suites, which require none of th source-only response barriers. Cursor/Copilot qualification gates stay intact. Pi's historical `qualificationCandidate` fixture selector remains a roster key; it no longer grants an opt-in when the source Pi declaration admits normal use. + +Installed Pi Daytona cells also require the published plugin fixture inputs +`PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN` (canonical package root), +`PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY` (canonical JSON file), +and `PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256`. +The authority schema `paperclip.e2e.installed-daytona-plugin/v1` pins a canonical +`graphRoot` and four public packages (`plugin`, `sdk`, `shared`, `daytona`). +Each has `root`, `packageSha256`, `entry`, and `entrySha256`; `plugin` also has +`manifestSha256` and `workerSha256`. Roots must be the named packages beneath +that graph's `node_modules`. The fixture verifies compiled public exports, +exact dependency versions, resolved package identities, and file pins before +launch and again immediately before the ordinary `/api/plugins/install` request. +It records `installed-daytona-plugin-admission.json` privately. Missing pins in +installed Daytona mode fail; they never select the source plugin as a fallback. +Local cells need no Daytona plugin. Source-development lanes retain their +existing fixture path. These entrypoint checks do not replace the separately +required tar provenance and complete installed dependency inventory audit. +The fixture inputs do not reach the production server environment. + +Before a paid installed local cell, use the same launcher and WebServer chain +without provider work: + +```sh +node --import ./cli/node_modules/tsx/dist/loader.mjs \ + tests/runner-e2e/launch.ts --installed-startup-only \ + --id extended-harnesses.runner-acpx-pi.local.hello-complete \ + --max-automatic-retries 0 +``` + +This mode requires the reviewed installed CLI pins, rejects provider credential +inputs, skips local credential-file loading, and runs only health, browser UI, +and zero-company checks. It retains separate private startup evidence and cannot +produce a passing provider case. Existing process, IPC, and scratch cleanup stay +in force. The installed lane invokes Playwright's public JavaScript bin directly +with the current Node; pnpm's generated bin shim would inject `NODE_PATH` into +the WebServer. Arbitrary ambient `NODE_OPTIONS` and `NODE_PATH` remain rejected. diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 511cb37b34d..10791debf56 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -1,3 +1,4 @@ +import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js"; import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; import { installedCliKeys } from "./installed-cli.js"; import path from "node:path"; @@ -180,7 +181,7 @@ export function buildPaperclipServerEnvironment( ]) { delete result[key]; } - for (const key of [...GENERATED_SERVER_SECRET_KEYS, ...installedCliKeys]) delete result[key]; + for (const key of [...GENERATED_SERVER_SECRET_KEYS, ...installedCliKeys, ...installedDaytonaPluginKeys]) delete result[key]; Object.assign(result, overrides); return result; } diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.ts index a351afebe52..84c5c58c288 100644 --- a/tests/runner-e2e/installed-cli.test.ts +++ b/tests/runner-e2e/installed-cli.test.ts @@ -67,3 +67,28 @@ it.skipIf(process.platform === "win32")("rejects a FIFO entrypoint without waiti execFileSync("/usr/bin/mkfifo", [f.env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI], { timeout: 5000, env: {} }); await expect(verifyInstalledCli(f.env, [cells[0]!])).rejects.toThrow("bounded regular file"); }); + +it("uses the direct public Playwright JS bin without bin-shim Node injection", async () => { + const { runnerE2EPlaywrightInvocation } = await import("./web-server-command.js"); + const f = await fixture(); const cli = join(f.root,"node_modules/@playwright/test/cli.js"); + await mkdir(join(f.root,"node_modules/@playwright/test"),{recursive:true}); + await writeFile(cli,"console.log(JSON.stringify({nodePath:process.env.NODE_PATH??null,nodeOptions:process.env.NODE_OPTIONS??null,args:process.argv.slice(2)}))"); + const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"],true); + expect(invocation.command).toBe(process.execPath); + expect(JSON.parse(execFileSync(invocation.command,invocation.args,{env:{},timeout:5000,encoding:"utf8"}))).toEqual({nodePath:null,nodeOptions:null,args:["--version"]}); + expect(runnerE2EPlaywrightInvocation(f.root,["--version"],false)).toEqual({command:"pnpm",args:["exec","playwright","--version"]}); + for (const key of ["NODE_PATH","NODE_OPTIONS"]) expect(()=>assertInstalledCliSelection({...f.env,[key]:""},[cells[0]!])).toThrow("ambient Node injection"); +}); + +it("startup-only accepts only explicit uncredentialed local hello and zero retries", async () => { + const { assertInstalledStartupOnly } = await import("./installed-cli.js"); + const { parseRunnerSelectors } = await import("./selectors.js"); + const f=await fixture(); const cell=cells.find(e=>e.id==="extended-harnesses.runner-acpx-pi.local.hello-complete")!; + const options=parseRunnerSelectors(["--installed-startup-only","--id",cell.id,"--max-automatic-retries","0"]); + expect(options.installedStartupOnly).toBe(true);expect(()=>assertInstalledStartupOnly(f.env,[cell],options)).not.toThrow(); + for(const delta of [{maxAutomaticRetries:1},{maxParallel:2},{ui:true},{debug:true},{headed:true},{list:true},{matrixJson:true},{ids:[]}])expect(()=>assertInstalledStartupOnly(f.env,[cell],{...options,...delta})).toThrow("startup-only"); + expect(()=>assertInstalledStartupOnly(f.env,[cells.find(e=>e.environment.id==="daytona")!],options)).toThrow("startup-only"); + expect(()=>assertInstalledStartupOnly({...f.env,OPENROUTER_API_KEY:"dummy-not-a-real-key"},[cell],options)).toThrow("credential inputs"); + expect(()=>assertInstalledStartupOnly({...f.env,KIMI_MODEL_API_KEY:"dummy-not-a-real-key"},[cell],options)).toThrow("credential inputs"); + expect(()=>assertInstalledStartupOnly({},[cell],options)).toThrow("startup-only"); +}); diff --git a/tests/runner-e2e/installed-cli.ts b/tests/runner-e2e/installed-cli.ts index b24ddf26166..97dc50a49f2 100644 --- a/tests/runner-e2e/installed-cli.ts +++ b/tests/runner-e2e/installed-cli.ts @@ -5,7 +5,7 @@ import { createHash } from "node:crypto"; import { findPackageJSON } from "node:module"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; -import type { MatrixExecution } from "./types.js"; +import { CREDENTIAL_NAMES, type MatrixExecution } from "./types.js"; export const installedCliKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_CLI", "PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256"] as const; const overrideKeys = ["PAPERCLIP_RUNNER_BINARY", "PAPERCLIP_RUNNER_REMOTE_BINARY_PATH", "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "PAPERCLIP_RUNNER_ACPX_QUALIFICATION", "PAPERCLIP_ACPX_BUILTIN_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST"] as const; @@ -50,3 +50,10 @@ export async function verifyInstalledCli(env: NodeJS.ProcessEnv, executions: rea if (cliSha256 !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256 || serverSha256 !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256) throw new Error("Installed CLI/server bytes differ from their reviewed pins"); return { schema: "paperclip.e2e.installed-cli-admission/v1", entry, cliRoot, cliSha256, serverRoot, serverEntry, serverSha256, version: cli.version, defaultRuntimeResolution: true, qualificationOverride: false }; } + +/** This prep lane cannot create agents or read the normal local credential file. */ +export function assertInstalledStartupOnly(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[], options: { ids: string[]; maxParallel: number; maxAutomaticRetries: number; headed: boolean; ui: boolean; debug: boolean; list: boolean; matrixJson: boolean }) { + assertInstalledCliSelection(env, executions); + if (!usesInstalledCli(env) || executions.length !== 1 || executions[0]?.id !== "extended-harnesses.runner-acpx-pi.local.hello-complete" || options.ids.length !== 1 || options.ids[0] !== executions[0]?.id || options.maxParallel !== 1 || options.maxAutomaticRetries !== 0 || options.headed || options.ui || options.debug || options.list || options.matrixJson) throw new Error("Installed startup-only requires one explicit local Pi hello, no retries or interactive mode"); + if (CREDENTIAL_NAMES.some(key => env[key] !== undefined) || Object.keys(env).some(key => /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/.test(key) && env[key] !== undefined)) throw new Error("Installed startup-only forbids provider credential inputs"); +} diff --git a/tests/runner-e2e/installed-daytona-plugin.test.ts b/tests/runner-e2e/installed-daytona-plugin.test.ts new file mode 100644 index 00000000000..a3d65456116 --- /dev/null +++ b/tests/runner-e2e/installed-daytona-plugin.test.ts @@ -0,0 +1,80 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, expect, it, vi } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { installedDaytonaPluginKeys, verifyInstalledDaytonaPlugin, type InstalledDaytonaPluginAuthority } from "./installed-daytona-plugin.js"; +import { buildPaperclipServerEnvironment } from "./harness-env.js"; +import { setupLiveFixtures } from "./live-fixtures.js"; +import type { RunnerApi } from "./api.js"; +const cell = runnerMatrix.find(e => e.id === "extended-harnesses.runner-acpx-pi.daytona.hello-complete")!; +const roots: string[] = []; +const hash = (v: string) => createHash("sha256").update(v).digest("hex"); +afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { + const root = await realpath(await mkdtemp(join(tmpdir(), "e2e-installed-plugin-"))); roots.push(root); + async function pkg(name: string, version: string, dependencies: Record = {}, extra = {}) { + const dir = join(root, "node_modules", name); const entry = name === "@daytonaio/sdk" ? "./esm/index.js" : "./dist/index.js"; await mkdir(join(dir, name === "@daytonaio/sdk" ? "esm" : "dist"), { recursive: true }); + const content = JSON.stringify({ name, version, type: "module", exports: { ".": { import: name === "@daytonaio/sdk" ? { types: "./esm/index.d.ts", default: entry } : entry } }, dependencies, ...extra }); + await writeFile(join(dir, "package.json"), content); await writeFile(join(dir, entry), "// compiled"); + return { root: dir, packageSha256: hash(content), entry, entrySha256: hash("// compiled") }; + } + const plugin = await pkg("@paperclipai/plugin-daytona", "0.1.1", { "@paperclipai/plugin-sdk": "0.3.1", "@daytonaio/sdk": "0.203.0" }, { paperclipPlugin: { manifest: "./dist/manifest.js", worker: "./dist/worker.js" } }); + await writeFile(join(plugin.root, "dist/manifest.js"), "// manifest"); await writeFile(join(plugin.root, "dist/worker.js"), "// worker"); + const authority: InstalledDaytonaPluginAuthority = { schema: "paperclip.e2e.installed-daytona-plugin/v1", graphRoot: root, plugin: { ...plugin, manifestSha256: hash("// manifest"), workerSha256: hash("// worker") }, sdk: await pkg("@paperclipai/plugin-sdk", "0.3.1", { "@paperclipai/shared": "0.3.1" }), shared: await pkg("@paperclipai/shared", "0.3.1"), daytona: await pkg("@daytonaio/sdk", "0.203.0") }; + const authorityPath = join(root, "authority.json"); + const env: NodeJS.ProcessEnv = { PAPERCLIP_RUNNER_E2E_INSTALLED_CLI: "/reviewed/cli", PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256: "reviewed-separately", PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT: "/reviewed/server", PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256: "reviewed-separately", PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN: plugin.root, PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY: authorityPath }; + async function seal() { const bytes = JSON.stringify(authority); await writeFile(authorityPath, bytes); env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256 = hash(bytes); } + await seal(); return { root, authority, env, seal }; +} +it("admits pinned published plugin and resolved SDK graph without claiming a full graph audit", async () => { + const f = await fixture(); expect(await verifyInstalledDaytonaPlugin(f.env, [cell])).toMatchObject({ packageRoot: f.authority.plugin.root, sourceFallback: false, fullGraphVerified: false, completeGraphAuthorityRequired: true }); +}); +it("fails closed for absent/partial installed Daytona pins but preserves ordinary source and local lanes", async () => { + const f = await fixture(); + for (const key of installedDaytonaPluginKeys) await expect(verifyInstalledDaytonaPlugin({ ...f.env, [key]: undefined }, [cell])).rejects.toThrow("complete pins"); + await expect(verifyInstalledDaytonaPlugin({}, [cell])).resolves.toBeUndefined(); + const local = runnerMatrix.find(e => e.id === "extended-harnesses.runner-acpx-pi.local.hello-complete")!; + await expect(verifyInstalledDaytonaPlugin(f.env, [local])).rejects.toThrow("Daytona selection"); + const noPlugin = { ...f.env }; for (const key of installedDaytonaPluginKeys) delete noPlugin[key]; + await expect(verifyInstalledDaytonaPlugin(noPlugin, [local])).resolves.toBeUndefined(); + await expect(verifyInstalledDaytonaPlugin(noPlugin, [cell])).rejects.toThrow("complete pins"); +}); +it("rejects stale authority, worker, manifest and dependency entry bytes", async () => { + const f = await fixture(); + await expect(verifyInstalledDaytonaPlugin({ ...f.env, PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256: "0".repeat(64) }, [cell])).rejects.toThrow("reviewed pins"); + for (const [path, original] of [[join(f.authority.plugin.root,"dist/worker.js"),"// worker"], [join(f.authority.plugin.root,"dist/manifest.js"),"// manifest"], [join(f.authority.sdk.root,"dist/index.js"),"// compiled"]]) { + await writeFile(path!, "// wrong"); await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("reviewed pins"); await writeFile(path!,original!); + } +}); +it("rejects source exports and workspace dependency versions even with refreshed pins", async () => { + const f = await fixture(); const p = join(f.authority.plugin.root,"package.json"); + const original = { name:"@paperclipai/plugin-daytona",version:"0.1.1",exports:{".":{import:"./dist/index.js"}},paperclipPlugin:{manifest:"./dist/manifest.js",worker:"./dist/worker.js"},dependencies:{"@paperclipai/plugin-sdk":"0.3.1","@daytonaio/sdk":"0.203.0"} }; + for (const manifest of [{ ...original, exports:{".":{import:"./src/index.ts"}} }, { ...original, dependencies:{ ...original.dependencies,"@paperclipai/plugin-sdk":"workspace:*" } }]) { + const bytes=JSON.stringify(manifest);await writeFile(p,bytes);f.authority.plugin.packageSha256=hash(bytes);await f.seal();await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow(/compiled package exports|dependency identity/); + } +}); +it("rejects linked or escaped roots and a shadowed foreign resolved dependency", async () => { + const f = await fixture(); const other = await fixture(); + f.authority.shared=other.authority.shared;await f.seal();await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("escapes"); + f.authority.shared={...other.authority.shared,root:join(f.root,"node_modules/@paperclipai/shared")};await f.seal(); + const nested=join(f.authority.plugin.root,"node_modules/@paperclipai");await mkdir(nested,{recursive:true});await symlink(other.authority.sdk.root,join(nested,"plugin-sdk")); + await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("foreign dependency"); + await rm(nested,{recursive:true});await rm(f.authority.shared.root,{recursive:true});await symlink(other.authority.shared.root,f.authority.shared.root); + await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("canonical and unlinked"); +}); +it.skipIf(process.platform === "win32")("rejects a FIFO authority without waiting for a writer", async () => { + const f=await fixture();const p=f.env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY!;await rm(p);execFileSync("/usr/bin/mkfifo",[p],{timeout:5000,env:{}});await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("bounded unlinked regular file"); +}); +it("uses the verified public package in the existing API and rejects changed bytes before any mutation", async () => { + const f=await fixture(); for(const [k,v]of Object.entries(f.env))vi.stubEnv(k,v!); + const calls:unknown[]=[]; + const api={async post(url:string,data:unknown){calls.push({url,data});throw new Error("stop-after-public-install");}} as unknown as RunnerApi; + const input={api,execution:cell,executionNonce:"nonce",workspacePath:"/tmp/workspace",credentials:{},daytonaImage:"ghcr.io/example/image@sha256:"+"a".repeat(64)}; + await expect(setupLiveFixtures(input)).rejects.toThrow("stop-after-public-install"); + expect(calls).toEqual([{url:"/api/plugins/install",data:{packageName:f.authority.plugin.root,isLocalPath:true}}]); + calls.length=0;await writeFile(join(f.authority.plugin.root,"dist/worker.js"),"// changed");await expect(setupLiveFixtures(input)).rejects.toThrow("reviewed pins");expect(calls).toEqual([]); +}); +it("keeps fixture authority out of the production server environment", async()=>{const f=await fixture();const env=buildPaperclipServerEnvironment(f.env);for(const k of installedDaytonaPluginKeys)expect(env[k]).toBeUndefined();}); diff --git a/tests/runner-e2e/installed-daytona-plugin.ts b/tests/runner-e2e/installed-daytona-plugin.ts new file mode 100644 index 00000000000..b23742f692a --- /dev/null +++ b/tests/runner-e2e/installed-daytona-plugin.ts @@ -0,0 +1,72 @@ +/** Public plugin fixture admission; the complete installed graph is audited separately. */ +import { constants } from "node:fs"; +import { lstat, open, realpath } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { findPackageJSON } from "node:module"; +import { isAbsolute, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { assertInstalledCliSelection, usesInstalledCli } from "./installed-cli.js"; +import type { MatrixExecution } from "./types.js"; + +export const installedDaytonaPluginKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN", "PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY", "PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256"] as const; +type PackagePin = { root: string; packageSha256: string; entry: string; entrySha256: string }; +export interface InstalledDaytonaPluginAuthority { + schema: "paperclip.e2e.installed-daytona-plugin/v1"; + graphRoot: string; + plugin: PackagePin & { manifestSha256: string; workerSha256: string }; + sdk: PackagePin; + shared: PackagePin; + daytona: PackagePin; +} +const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +async function canonical(path: string) { + if (typeof path !== "string" || !isAbsolute(path) || resolve(path) !== path || await realpath(path) !== path) throw new Error("Installed Daytona plugin path must be canonical and unlinked"); +} +async function pinned(path: string, expected: string, maximum = 32 * 1024 * 1024) { + if (!/^[a-f0-9]{64}$/.test(expected)) throw new Error("Installed Daytona plugin requires a SHA-256 pin"); + await canonical(path); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = await handle.stat({ bigint: true }); + if (!before.isFile() || before.size < 1n || before.size > BigInt(maximum) || before.nlink !== 1n) throw new Error("Installed Daytona plugin requires a bounded unlinked regular file"); + const bytes = await handle.readFile(); const after = await handle.stat({ bigint: true }); const named = await lstat(path, { bigint: true }); + if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs || named.dev !== before.dev || named.ino !== before.ino || bytes.length !== Number(before.size) || digest(bytes) !== expected) throw new Error("Installed Daytona plugin bytes differ from reviewed pins"); + return bytes; + } finally { await handle.close(); } +} +function compiledEntry(manifest: any): string | undefined { + const exported = manifest.exports?.["."]; + return typeof exported === "string" ? exported : (typeof exported?.import === "string" ? exported.import : exported?.import?.default) ?? manifest.main; +} +export async function verifyInstalledDaytonaPlugin(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[]) { + const selected = installedDaytonaPluginKeys.some(key => env[key] !== undefined); + const required = usesInstalledCli(env) && executions.some(e => e.environment.id === "daytona"); + if (!selected && !required) return undefined; + assertInstalledCliSelection(env, executions); + if (!usesInstalledCli(env) || !installedDaytonaPluginKeys.every(key => Boolean(env[key])) || executions.some(e => e.environment.id !== "daytona")) throw new Error("Installed Daytona plugin requires complete pins and installed Pi Daytona selection"); + const authorityPath = env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY!; + const authoritySha256 = env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256!; + const a: InstalledDaytonaPluginAuthority = JSON.parse((await pinned(authorityPath, authoritySha256, 65536)).toString()); + if (a.schema !== "paperclip.e2e.installed-daytona-plugin/v1" || a.plugin?.root !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN) throw new Error("Installed Daytona plugin authority differs"); + await canonical(a.graphRoot); + async function packageAt(pin: PackagePin, name: string) { + await canonical(pin.root); + if (pin.root !== join(a.graphRoot, "node_modules", name)) throw new Error("Installed Daytona plugin package escapes its reviewed graph"); + const manifest = JSON.parse((await pinned(join(pin.root, "package.json"), pin.packageSha256, 65536)).toString()); + if (manifest.name !== name || typeof manifest.version !== "string" || !/^\d+\.\d+\.\d+(?:[-+][a-zA-Z0-9.-]+)?$/.test(manifest.version) || pin.entry !== (name === "@daytonaio/sdk" ? "./esm/index.js" : "./dist/index.js") || compiledEntry(manifest) !== pin.entry) throw new Error("Installed Daytona plugin requires published compiled package exports"); + await pinned(join(pin.root, pin.entry), pin.entrySha256); + return manifest; + } + const plugin = await packageAt(a.plugin, "@paperclipai/plugin-daytona"); + const sdk = await packageAt(a.sdk, "@paperclipai/plugin-sdk"); + const shared = await packageAt(a.shared, "@paperclipai/shared"); + const daytona = await packageAt(a.daytona, "@daytonaio/sdk"); + if (plugin.paperclipPlugin?.manifest !== "./dist/manifest.js" || plugin.paperclipPlugin?.worker !== "./dist/worker.js" || plugin.dependencies?.["@paperclipai/plugin-sdk"] !== sdk.version || sdk.dependencies?.["@paperclipai/shared"] !== shared.version || plugin.dependencies?.["@daytonaio/sdk"] !== "0.203.0" || daytona.version !== "0.203.0") throw new Error("Installed Daytona plugin public dependency identity differs"); + await pinned(join(a.plugin.root, "dist/manifest.js"), a.plugin.manifestSha256); + await pinned(join(a.plugin.root, "dist/worker.js"), a.plugin.workerSha256); + for (const [name, parent, pin] of [["@paperclipai/plugin-sdk", join(a.plugin.root, "dist/worker.js"), a.sdk], ["@paperclipai/shared", join(a.sdk.root, a.sdk.entry), a.shared], ["@daytonaio/sdk", join(a.plugin.root, "dist/worker.js"), a.daytona]] as const) { + const found = findPackageJSON(name, pathToFileURL(parent)); + if (!found || await realpath(found) !== join(pin.root, "package.json")) throw new Error("Installed Daytona plugin resolves a foreign dependency"); + } + return { schema: a.schema, authorityPath, authoritySha256, packageRoot: a.plugin.root, graphRoot: a.graphRoot, fullGraphVerified: false, completeGraphAuthorityRequired: true, sourceFallback: false }; +} diff --git a/tests/runner-e2e/installed-startup.spec.ts b/tests/runner-e2e/installed-startup.spec.ts new file mode 100644 index 00000000000..c8c9ee07e17 --- /dev/null +++ b/tests/runner-e2e/installed-startup.spec.ts @@ -0,0 +1,31 @@ +/** Credential-free proof of the real launcher/Playwright/WebServer chain. */ +import { test, expect } from "@playwright/test"; +import { writeFile } from "node:fs/promises"; +import path from "node:path"; + +test("installed controller serves health and UI without creating provider work", async ({ page, request, baseURL }) => { + test.skip(process.env.PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY !== "1"); + expect(baseURL).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + const errors: string[] = []; + page.on("pageerror", error => errors.push(error.message)); + await page.route("**/*", route => { + const url = new URL(route.request().url()); + return url.origin === baseURL ? route.continue() : route.abort("blockedbyclient"); + }); + const health = await request.get(`${baseURL}/api/health`); + expect(health.ok()).toBe(true); + const before = await request.get(`${baseURL}/api/companies`); + expect(before.ok()).toBe(true); expect(await before.json()).toEqual([]); + const response = await page.goto(baseURL!); + expect(response?.ok()).toBe(true); + await expect(page.locator("#root")).not.toBeEmpty(); + await expect(page.getByRole("heading", { name: "What is the name of your organization?", exact: true })).toBeVisible(); + await expect(page.getByRole("textbox", { name: "Name", exact: true })).toBeVisible(); + await expect(page.getByRole("button", { name: "Continue", exact: true })).toBeDisabled(); + expect(errors).toEqual([]); + const after = await request.get(`${baseURL}/api/companies`); + expect(after.ok()).toBe(true); expect(await after.json()).toEqual([]); + const output = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR!; + await page.screenshot({ path: path.join(output, "installed-startup-only.png"), fullPage: true }); + await writeFile(path.join(output, "installed-startup-only.json"), `${JSON.stringify({ status: "health_ui_zero_company_passed", qualified: false, providerCalls: 0, healthStatus: health.status(), uiStatus: response!.status(), companiesBefore: 0, companiesAfter: 0, pageErrors: errors }, null, 2)}\n`, { mode: 0o600 }); +}); diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index e85d98ab737..708c3fecbc3 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,4 +1,6 @@ -import { assertInstalledCliSelection, verifyInstalledCli } from "./installed-cli.js"; +import { runnerE2EPlaywrightInvocation } from "./web-server-command.js"; +import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; +import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js"; import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js"; import { randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; @@ -257,7 +259,7 @@ async function prepareProviderPath( } async function runProcess( - args: string[], + invocation: { command: string; args: string[] }, env: NodeJS.ProcessEnv, timeoutMs: number | null, logPath: string, @@ -265,7 +267,7 @@ async function runProcess( interactive: boolean, ) { const log = createWriteStream(logPath, { flags: "a", mode: 0o600 }); - const child = spawn("pnpm", args, { + const child = spawn(invocation.command, invocation.args, { cwd: repositoryRoot, env, stdio: ["inherit", "pipe", "pipe"], @@ -474,6 +476,7 @@ async function runAttempt(input: { } assertInstalledCliSelection(process.env, executions); const installedCli = await verifyInstalledCli(process.env, executions); + const installedPlugin = await verifyInstalledDaytonaPlugin(process.env, executions); const startedAtMs = Date.now(); const sharedMemoryBaseline = snapshotDarwinSharedMemory(); const temporaryRoot = await mkdtemp( @@ -483,6 +486,7 @@ async function runAttempt(input: { const publishedResultPaths = new Map(); let attemptSecrets: string[] = []; let processCleanupFailed = false; + let startupReceiptPath: string | undefined; try { const paperclipHome = path.join(temporaryRoot, "paperclip-home"); const workspace = path.join(temporaryRoot, "workspace"); @@ -520,6 +524,7 @@ async function runAttempt(input: { ]); attemptSecrets = credentials; if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 }); + if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 }); const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness( executions, repositoryRoot, @@ -531,6 +536,7 @@ async function runAttempt(input: { executions.map((candidate) => candidate.id), ), PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt), + PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY: options.installedStartupOnly ? "1" : undefined, PAPERCLIP_RUNNER_E2E_PORT: String(port), PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot, PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, @@ -565,8 +571,6 @@ async function runAttempt(input: { delete childEnv.DATABASE_MIGRATION_URL; const playwrightArgs = [ - "exec", - "playwright", "test", "--config", "tests/runner-e2e/playwright.config.ts", @@ -586,7 +590,7 @@ async function runAttempt(input: { ) + 5 * 60_000; const processResult = await runProcess( - playwrightArgs, + runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs, Boolean(installedCli)), childEnv, watchdog, path.join(privateDir, "playwright.log"), @@ -596,6 +600,18 @@ async function runAttempt(input: { options.ui || options.debug, ); processCleanupFailed = processResult.processCleanupError !== null; + if (options.installedStartupOnly) { + const proofDir = path.join(resultsRoot, campaignId, "installed-startup-only"); + await mkdir(proofDir, { recursive: true }); + // This is private setup evidence, never a successful provider case/result. + await cp(privateDir, path.join(proofDir, "private"), { recursive: true }); + const probe = JSON.parse(await readFile(path.join(privateDir, "installed-startup-only.json"), "utf8")); + if (processResult.exitCode !== 0 || processResult.timedOut || processResult.spawnError || processCleanupFailed || probe.status !== "health_ui_zero_company_passed" || probe.providerCalls !== 0 || probe.qualified !== false) throw new Error("Installed startup-only actual launch chain failed"); + await assertEmbeddedDatabaseIsolation(configPath, temporaryRoot); + startupReceiptPath = path.join(proofDir, "receipt.json"); + await writeFile(startupReceiptPath, `${JSON.stringify({ status: "installed_launch_chain_pending_scratch_cleanup", qualified: false, providerCalls: 0, credentialsLoaded: false, installedCli, processResult, probe }, null, 2)}\n`, { mode: 0o600 }); + return []; + } const processFailure = processResult.spawnError ? `Playwright failed to start: ${processResult.spawnError}` : processResult.timedOut @@ -824,6 +840,12 @@ async function runAttempt(input: { `Refusing to remove unexpected temporary path ${temporaryRoot}`, ); } + if (startupReceiptPath) { + const receipt = JSON.parse(await readFile(startupReceiptPath, "utf8")); + receipt.status = cleanupError ? "installed_launch_chain_cleanup_failed" : "installed_launch_chain_passed"; + receipt.temporaryRootRemoved = !cleanupError; + await writeFile(startupReceiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); + } if (cleanupError) { const message = `Temporary runner E2E state cleanup failed at ${temporaryRoot}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`; for (const publishedResult of publishedResults) { @@ -848,6 +870,7 @@ async function runAttempt(input: { // Cleanup failure is a failed cell, but must not suppress later cells in // the same campaign. The result above carries the terminal failure. console.error(message); + if (options.installedStartupOnly) throw new Error(message); } } } @@ -939,6 +962,14 @@ async function main() { throw error; } const executions = selectRunnerExecutions(options, runnerMatrix); + if (options.installedStartupOnly) { + assertInstalledStartupOnly(process.env, executions, options); + assertRunnerE2EPrerequisites(executions); + const campaignId = cleanId(process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? `installed-startup-${Date.now()}`); + await runAttempt({ executions, attempt: 1, campaignId, options }); + console.log(`PASS installed launch chain (no provider qualification) -> ${path.join(resultsRoot, campaignId, "installed-startup-only")}`); + return; + } if (options.list) { printList(executions); return; diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 858d63cc156..1cc151a53ba 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,3 +1,4 @@ +import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; import path from "node:path"; import { isManagedHiringCase } from "./chat-cases.js"; import { FixtureRegistry } from "./fixture-registry.js"; @@ -113,8 +114,9 @@ export async function setupLiveFixtures(input: { registry.register({ id: "sandbox-provider", async setup() { + const installed = await verifyInstalledDaytonaPlugin(process.env, [execution]); return api.post("/api/plugins/install", { - packageName: path.resolve( + packageName: installed?.packageRoot ?? path.resolve( import.meta.dirname, "../../packages/plugins/sandbox-providers/daytona", ), diff --git a/tests/runner-e2e/playwright.config.ts b/tests/runner-e2e/playwright.config.ts index 29ab9db6977..09e74cd1683 100644 --- a/tests/runner-e2e/playwright.config.ts +++ b/tests/runner-e2e/playwright.config.ts @@ -45,7 +45,7 @@ const repositoryRoot = path.resolve(import.meta.dirname, "../.."); export default defineConfig({ testDir: ".", - testMatch: "runner.spec.ts", + testMatch: process.env.PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY === "1" ? "installed-startup.spec.ts" : "runner.spec.ts", timeout: Number(process.env.PAPERCLIP_RUNNER_E2E_TEST_TIMEOUT_MS ?? 600_000), expect: { timeout: 30_000 }, fullyParallel: false, diff --git a/tests/runner-e2e/selectors.ts b/tests/runner-e2e/selectors.ts index e890da21e4c..d62c2191be1 100644 --- a/tests/runner-e2e/selectors.ts +++ b/tests/runner-e2e/selectors.ts @@ -5,6 +5,7 @@ export interface RunnerSelectorOptions { all: boolean; list: boolean; matrixJson: boolean; + installedStartupOnly?: boolean; ids: string[]; suites: string[]; groups: string[]; @@ -52,6 +53,7 @@ export function parseRunnerSelectors( if (flag === "--all") options.all = true; else if (flag === "--list") options.list = true; else if (flag === "--matrix-json") options.matrixJson = true; + else if (flag === "--installed-startup-only") options.installedStartupOnly = true; else if (flag === "--headed") options.headed = true; else if (flag === "--ui") options.ui = true; else if (flag === "--debug") options.debug = true; diff --git a/tests/runner-e2e/web-server-command.ts b/tests/runner-e2e/web-server-command.ts index de88292fcbe..7735119d246 100644 --- a/tests/runner-e2e/web-server-command.ts +++ b/tests/runner-e2e/web-server-command.ts @@ -21,3 +21,12 @@ export function runnerE2EWebServerCommand(repositoryRoot: string) { export function runnerE2ETypeScriptProcessArgs(repositoryRoot: string, entry: string, args: string[] = []) { return ["--import", path.join(repositoryRoot, "cli/node_modules/tsx/dist/loader.mjs"), entry, ...args]; } + +// pnpm's generated playwright bin shim adds NODE_PATH, even when the caller +// rejected ambient injection. The installed lane invokes the public JS bin +// with the current Node directly, keeping the WebServer environment closed. +export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[], installed: boolean) { + return installed + ? { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] } + : { command: "pnpm", args: ["exec", "playwright", ...args] }; +}