mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
test(runner): verify installed launch and published Daytona plugin
Invoke the public Playwright JavaScript entry directly for installed Pi tests so pnpm shim NODE_PATH injection cannot cross the closed controller boundary. Add a credential-free health/UI startup proof and a pinned published Daytona plugin fixture path without production runtime overrides. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
4c58da84fd
commit
30edeec300
12 files changed
+306
-10
No files matched your search
@@ -1820,3 +1820,39 @@ explicit Pi extended, native, controls and warm suites, which require none of th
|
||||
source-only response barriers. Cursor/Copilot qualification gates stay intact.
|
||||
Pi's historical `qualificationCandidate` fixture selector remains a roster key;
|
||||
it no longer grants an opt-in when the source Pi declaration admits normal use.
|
||||
|
||||
Installed Pi Daytona cells also require the published plugin fixture inputs
|
||||
`PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN` (canonical package root),
|
||||
`PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY` (canonical JSON file),
|
||||
and `PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256`.
|
||||
The authority schema `paperclip.e2e.installed-daytona-plugin/v1` pins a canonical
|
||||
`graphRoot` and four public packages (`plugin`, `sdk`, `shared`, `daytona`).
|
||||
Each has `root`, `packageSha256`, `entry`, and `entrySha256`; `plugin` also has
|
||||
`manifestSha256` and `workerSha256`. Roots must be the named packages beneath
|
||||
that graph's `node_modules`. The fixture verifies compiled public exports,
|
||||
exact dependency versions, resolved package identities, and file pins before
|
||||
launch and again immediately before the ordinary `/api/plugins/install` request.
|
||||
It records `installed-daytona-plugin-admission.json` privately. Missing pins in
|
||||
installed Daytona mode fail; they never select the source plugin as a fallback.
|
||||
Local cells need no Daytona plugin. Source-development lanes retain their
|
||||
existing fixture path. These entrypoint checks do not replace the separately
|
||||
required tar provenance and complete installed dependency inventory audit.
|
||||
The fixture inputs do not reach the production server environment.
|
||||
|
||||
Before a paid installed local cell, use the same launcher and WebServer chain
|
||||
without provider work:
|
||||
|
||||
```sh
|
||||
node --import ./cli/node_modules/tsx/dist/loader.mjs \
|
||||
tests/runner-e2e/launch.ts --installed-startup-only \
|
||||
--id extended-harnesses.runner-acpx-pi.local.hello-complete \
|
||||
--max-automatic-retries 0
|
||||
```
|
||||
|
||||
This mode requires the reviewed installed CLI pins, rejects provider credential
|
||||
inputs, skips local credential-file loading, and runs only health, browser UI,
|
||||
and zero-company checks. It retains separate private startup evidence and cannot
|
||||
produce a passing provider case. Existing process, IPC, and scratch cleanup stay
|
||||
in force. The installed lane invokes Playwright's public JavaScript bin directly
|
||||
with the current Node; pnpm's generated bin shim would inject `NODE_PATH` into
|
||||
the WebServer. Arbitrary ambient `NODE_OPTIONS` and `NODE_PATH` remain rejected.
|
||||
@@ -1,3 +1,4 @@
|
||||
import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js";
|
||||
import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
|
||||
import { installedCliKeys } from "./installed-cli.js";
|
||||
import path from "node:path";
|
||||
@@ -180,7 +181,7 @@ export function buildPaperclipServerEnvironment(
|
||||
]) {
|
||||
delete result[key];
|
||||
}
|
||||
for (const key of [...GENERATED_SERVER_SECRET_KEYS, ...installedCliKeys]) delete result[key];
|
||||
for (const key of [...GENERATED_SERVER_SECRET_KEYS, ...installedCliKeys, ...installedDaytonaPluginKeys]) delete result[key];
|
||||
Object.assign(result, overrides);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -67,3 +67,28 @@ it.skipIf(process.platform === "win32")("rejects a FIFO entrypoint without waiti
|
||||
execFileSync("/usr/bin/mkfifo", [f.env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI], { timeout: 5000, env: {} });
|
||||
await expect(verifyInstalledCli(f.env, [cells[0]!])).rejects.toThrow("bounded regular file");
|
||||
});
|
||||
|
||||
it("uses the direct public Playwright JS bin without bin-shim Node injection", async () => {
|
||||
const { runnerE2EPlaywrightInvocation } = await import("./web-server-command.js");
|
||||
const f = await fixture(); const cli = join(f.root,"node_modules/@playwright/test/cli.js");
|
||||
await mkdir(join(f.root,"node_modules/@playwright/test"),{recursive:true});
|
||||
await writeFile(cli,"console.log(JSON.stringify({nodePath:process.env.NODE_PATH??null,nodeOptions:process.env.NODE_OPTIONS??null,args:process.argv.slice(2)}))");
|
||||
const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"],true);
|
||||
expect(invocation.command).toBe(process.execPath);
|
||||
expect(JSON.parse(execFileSync(invocation.command,invocation.args,{env:{},timeout:5000,encoding:"utf8"}))).toEqual({nodePath:null,nodeOptions:null,args:["--version"]});
|
||||
expect(runnerE2EPlaywrightInvocation(f.root,["--version"],false)).toEqual({command:"pnpm",args:["exec","playwright","--version"]});
|
||||
for (const key of ["NODE_PATH","NODE_OPTIONS"]) expect(()=>assertInstalledCliSelection({...f.env,[key]:""},[cells[0]!])).toThrow("ambient Node injection");
|
||||
});
|
||||
|
||||
it("startup-only accepts only explicit uncredentialed local hello and zero retries", async () => {
|
||||
const { assertInstalledStartupOnly } = await import("./installed-cli.js");
|
||||
const { parseRunnerSelectors } = await import("./selectors.js");
|
||||
const f=await fixture(); const cell=cells.find(e=>e.id==="extended-harnesses.runner-acpx-pi.local.hello-complete")!;
|
||||
const options=parseRunnerSelectors(["--installed-startup-only","--id",cell.id,"--max-automatic-retries","0"]);
|
||||
expect(options.installedStartupOnly).toBe(true);expect(()=>assertInstalledStartupOnly(f.env,[cell],options)).not.toThrow();
|
||||
for(const delta of [{maxAutomaticRetries:1},{maxParallel:2},{ui:true},{debug:true},{headed:true},{list:true},{matrixJson:true},{ids:[]}])expect(()=>assertInstalledStartupOnly(f.env,[cell],{...options,...delta})).toThrow("startup-only");
|
||||
expect(()=>assertInstalledStartupOnly(f.env,[cells.find(e=>e.environment.id==="daytona")!],options)).toThrow("startup-only");
|
||||
expect(()=>assertInstalledStartupOnly({...f.env,OPENROUTER_API_KEY:"dummy-not-a-real-key"},[cell],options)).toThrow("credential inputs");
|
||||
expect(()=>assertInstalledStartupOnly({...f.env,KIMI_MODEL_API_KEY:"dummy-not-a-real-key"},[cell],options)).toThrow("credential inputs");
|
||||
expect(()=>assertInstalledStartupOnly({},[cell],options)).toThrow("startup-only");
|
||||
});
|
||||
@@ -5,7 +5,7 @@ import { createHash } from "node:crypto";
|
||||
import { findPackageJSON } from "node:module";
|
||||
import { dirname, isAbsolute, join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
import { CREDENTIAL_NAMES, type MatrixExecution } from "./types.js";
|
||||
|
||||
export const installedCliKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_CLI", "PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256"] as const;
|
||||
const overrideKeys = ["PAPERCLIP_RUNNER_BINARY", "PAPERCLIP_RUNNER_REMOTE_BINARY_PATH", "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "PAPERCLIP_RUNNER_ACPX_QUALIFICATION", "PAPERCLIP_ACPX_BUILTIN_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST"] as const;
|
||||
@@ -50,3 +50,10 @@ export async function verifyInstalledCli(env: NodeJS.ProcessEnv, executions: rea
|
||||
if (cliSha256 !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256 || serverSha256 !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256) throw new Error("Installed CLI/server bytes differ from their reviewed pins");
|
||||
return { schema: "paperclip.e2e.installed-cli-admission/v1", entry, cliRoot, cliSha256, serverRoot, serverEntry, serverSha256, version: cli.version, defaultRuntimeResolution: true, qualificationOverride: false };
|
||||
}
|
||||
|
||||
/** This prep lane cannot create agents or read the normal local credential file. */
|
||||
export function assertInstalledStartupOnly(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[], options: { ids: string[]; maxParallel: number; maxAutomaticRetries: number; headed: boolean; ui: boolean; debug: boolean; list: boolean; matrixJson: boolean }) {
|
||||
assertInstalledCliSelection(env, executions);
|
||||
if (!usesInstalledCli(env) || executions.length !== 1 || executions[0]?.id !== "extended-harnesses.runner-acpx-pi.local.hello-complete" || options.ids.length !== 1 || options.ids[0] !== executions[0]?.id || options.maxParallel !== 1 || options.maxAutomaticRetries !== 0 || options.headed || options.ui || options.debug || options.list || options.matrixJson) throw new Error("Installed startup-only requires one explicit local Pi hello, no retries or interactive mode");
|
||||
if (CREDENTIAL_NAMES.some(key => env[key] !== undefined) || Object.keys(env).some(key => /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/.test(key) && env[key] !== undefined)) throw new Error("Installed startup-only forbids provider credential inputs");
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { installedDaytonaPluginKeys, verifyInstalledDaytonaPlugin, type InstalledDaytonaPluginAuthority } from "./installed-daytona-plugin.js";
|
||||
import { buildPaperclipServerEnvironment } from "./harness-env.js";
|
||||
import { setupLiveFixtures } from "./live-fixtures.js";
|
||||
import type { RunnerApi } from "./api.js";
|
||||
const cell = runnerMatrix.find(e => e.id === "extended-harnesses.runner-acpx-pi.daytona.hello-complete")!;
|
||||
const roots: string[] = [];
|
||||
const hash = (v: string) => createHash("sha256").update(v).digest("hex");
|
||||
afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
|
||||
async function fixture() {
|
||||
const root = await realpath(await mkdtemp(join(tmpdir(), "e2e-installed-plugin-"))); roots.push(root);
|
||||
async function pkg(name: string, version: string, dependencies: Record<string,string> = {}, extra = {}) {
|
||||
const dir = join(root, "node_modules", name); const entry = name === "@daytonaio/sdk" ? "./esm/index.js" : "./dist/index.js"; await mkdir(join(dir, name === "@daytonaio/sdk" ? "esm" : "dist"), { recursive: true });
|
||||
const content = JSON.stringify({ name, version, type: "module", exports: { ".": { import: name === "@daytonaio/sdk" ? { types: "./esm/index.d.ts", default: entry } : entry } }, dependencies, ...extra });
|
||||
await writeFile(join(dir, "package.json"), content); await writeFile(join(dir, entry), "// compiled");
|
||||
return { root: dir, packageSha256: hash(content), entry, entrySha256: hash("// compiled") };
|
||||
}
|
||||
const plugin = await pkg("@paperclipai/plugin-daytona", "0.1.1", { "@paperclipai/plugin-sdk": "0.3.1", "@daytonaio/sdk": "0.203.0" }, { paperclipPlugin: { manifest: "./dist/manifest.js", worker: "./dist/worker.js" } });
|
||||
await writeFile(join(plugin.root, "dist/manifest.js"), "// manifest"); await writeFile(join(plugin.root, "dist/worker.js"), "// worker");
|
||||
const authority: InstalledDaytonaPluginAuthority = { schema: "paperclip.e2e.installed-daytona-plugin/v1", graphRoot: root, plugin: { ...plugin, manifestSha256: hash("// manifest"), workerSha256: hash("// worker") }, sdk: await pkg("@paperclipai/plugin-sdk", "0.3.1", { "@paperclipai/shared": "0.3.1" }), shared: await pkg("@paperclipai/shared", "0.3.1"), daytona: await pkg("@daytonaio/sdk", "0.203.0") };
|
||||
const authorityPath = join(root, "authority.json");
|
||||
const env: NodeJS.ProcessEnv = { PAPERCLIP_RUNNER_E2E_INSTALLED_CLI: "/reviewed/cli", PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256: "reviewed-separately", PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT: "/reviewed/server", PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256: "reviewed-separately", PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN: plugin.root, PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY: authorityPath };
|
||||
async function seal() { const bytes = JSON.stringify(authority); await writeFile(authorityPath, bytes); env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256 = hash(bytes); }
|
||||
await seal(); return { root, authority, env, seal };
|
||||
}
|
||||
it("admits pinned published plugin and resolved SDK graph without claiming a full graph audit", async () => {
|
||||
const f = await fixture(); expect(await verifyInstalledDaytonaPlugin(f.env, [cell])).toMatchObject({ packageRoot: f.authority.plugin.root, sourceFallback: false, fullGraphVerified: false, completeGraphAuthorityRequired: true });
|
||||
});
|
||||
it("fails closed for absent/partial installed Daytona pins but preserves ordinary source and local lanes", async () => {
|
||||
const f = await fixture();
|
||||
for (const key of installedDaytonaPluginKeys) await expect(verifyInstalledDaytonaPlugin({ ...f.env, [key]: undefined }, [cell])).rejects.toThrow("complete pins");
|
||||
await expect(verifyInstalledDaytonaPlugin({}, [cell])).resolves.toBeUndefined();
|
||||
const local = runnerMatrix.find(e => e.id === "extended-harnesses.runner-acpx-pi.local.hello-complete")!;
|
||||
await expect(verifyInstalledDaytonaPlugin(f.env, [local])).rejects.toThrow("Daytona selection");
|
||||
const noPlugin = { ...f.env }; for (const key of installedDaytonaPluginKeys) delete noPlugin[key];
|
||||
await expect(verifyInstalledDaytonaPlugin(noPlugin, [local])).resolves.toBeUndefined();
|
||||
await expect(verifyInstalledDaytonaPlugin(noPlugin, [cell])).rejects.toThrow("complete pins");
|
||||
});
|
||||
it("rejects stale authority, worker, manifest and dependency entry bytes", async () => {
|
||||
const f = await fixture();
|
||||
await expect(verifyInstalledDaytonaPlugin({ ...f.env, PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256: "0".repeat(64) }, [cell])).rejects.toThrow("reviewed pins");
|
||||
for (const [path, original] of [[join(f.authority.plugin.root,"dist/worker.js"),"// worker"], [join(f.authority.plugin.root,"dist/manifest.js"),"// manifest"], [join(f.authority.sdk.root,"dist/index.js"),"// compiled"]]) {
|
||||
await writeFile(path!, "// wrong"); await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("reviewed pins"); await writeFile(path!,original!);
|
||||
}
|
||||
});
|
||||
it("rejects source exports and workspace dependency versions even with refreshed pins", async () => {
|
||||
const f = await fixture(); const p = join(f.authority.plugin.root,"package.json");
|
||||
const original = { name:"@paperclipai/plugin-daytona",version:"0.1.1",exports:{".":{import:"./dist/index.js"}},paperclipPlugin:{manifest:"./dist/manifest.js",worker:"./dist/worker.js"},dependencies:{"@paperclipai/plugin-sdk":"0.3.1","@daytonaio/sdk":"0.203.0"} };
|
||||
for (const manifest of [{ ...original, exports:{".":{import:"./src/index.ts"}} }, { ...original, dependencies:{ ...original.dependencies,"@paperclipai/plugin-sdk":"workspace:*" } }]) {
|
||||
const bytes=JSON.stringify(manifest);await writeFile(p,bytes);f.authority.plugin.packageSha256=hash(bytes);await f.seal();await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow(/compiled package exports|dependency identity/);
|
||||
}
|
||||
});
|
||||
it("rejects linked or escaped roots and a shadowed foreign resolved dependency", async () => {
|
||||
const f = await fixture(); const other = await fixture();
|
||||
f.authority.shared=other.authority.shared;await f.seal();await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("escapes");
|
||||
f.authority.shared={...other.authority.shared,root:join(f.root,"node_modules/@paperclipai/shared")};await f.seal();
|
||||
const nested=join(f.authority.plugin.root,"node_modules/@paperclipai");await mkdir(nested,{recursive:true});await symlink(other.authority.sdk.root,join(nested,"plugin-sdk"));
|
||||
await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("foreign dependency");
|
||||
await rm(nested,{recursive:true});await rm(f.authority.shared.root,{recursive:true});await symlink(other.authority.shared.root,f.authority.shared.root);
|
||||
await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("canonical and unlinked");
|
||||
});
|
||||
it.skipIf(process.platform === "win32")("rejects a FIFO authority without waiting for a writer", async () => {
|
||||
const f=await fixture();const p=f.env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY!;await rm(p);execFileSync("/usr/bin/mkfifo",[p],{timeout:5000,env:{}});await expect(verifyInstalledDaytonaPlugin(f.env,[cell])).rejects.toThrow("bounded unlinked regular file");
|
||||
});
|
||||
it("uses the verified public package in the existing API and rejects changed bytes before any mutation", async () => {
|
||||
const f=await fixture(); for(const [k,v]of Object.entries(f.env))vi.stubEnv(k,v!);
|
||||
const calls:unknown[]=[];
|
||||
const api={async post(url:string,data:unknown){calls.push({url,data});throw new Error("stop-after-public-install");}} as unknown as RunnerApi;
|
||||
const input={api,execution:cell,executionNonce:"nonce",workspacePath:"/tmp/workspace",credentials:{},daytonaImage:"ghcr.io/example/image@sha256:"+"a".repeat(64)};
|
||||
await expect(setupLiveFixtures(input)).rejects.toThrow("stop-after-public-install");
|
||||
expect(calls).toEqual([{url:"/api/plugins/install",data:{packageName:f.authority.plugin.root,isLocalPath:true}}]);
|
||||
calls.length=0;await writeFile(join(f.authority.plugin.root,"dist/worker.js"),"// changed");await expect(setupLiveFixtures(input)).rejects.toThrow("reviewed pins");expect(calls).toEqual([]);
|
||||
});
|
||||
it("keeps fixture authority out of the production server environment", async()=>{const f=await fixture();const env=buildPaperclipServerEnvironment(f.env);for(const k of installedDaytonaPluginKeys)expect(env[k]).toBeUndefined();});
|
||||
@@ -0,0 +1,72 @@
|
||||
/** Public plugin fixture admission; the complete installed graph is audited separately. */
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, open, realpath } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
import { findPackageJSON } from "node:module";
|
||||
import { isAbsolute, join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { assertInstalledCliSelection, usesInstalledCli } from "./installed-cli.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
|
||||
export const installedDaytonaPluginKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN", "PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY", "PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256"] as const;
|
||||
type PackagePin = { root: string; packageSha256: string; entry: string; entrySha256: string };
|
||||
export interface InstalledDaytonaPluginAuthority {
|
||||
schema: "paperclip.e2e.installed-daytona-plugin/v1";
|
||||
graphRoot: string;
|
||||
plugin: PackagePin & { manifestSha256: string; workerSha256: string };
|
||||
sdk: PackagePin;
|
||||
shared: PackagePin;
|
||||
daytona: PackagePin;
|
||||
}
|
||||
const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
|
||||
async function canonical(path: string) {
|
||||
if (typeof path !== "string" || !isAbsolute(path) || resolve(path) !== path || await realpath(path) !== path) throw new Error("Installed Daytona plugin path must be canonical and unlinked");
|
||||
}
|
||||
async function pinned(path: string, expected: string, maximum = 32 * 1024 * 1024) {
|
||||
if (!/^[a-f0-9]{64}$/.test(expected)) throw new Error("Installed Daytona plugin requires a SHA-256 pin");
|
||||
await canonical(path);
|
||||
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try {
|
||||
const before = await handle.stat({ bigint: true });
|
||||
if (!before.isFile() || before.size < 1n || before.size > BigInt(maximum) || before.nlink !== 1n) throw new Error("Installed Daytona plugin requires a bounded unlinked regular file");
|
||||
const bytes = await handle.readFile(); const after = await handle.stat({ bigint: true }); const named = await lstat(path, { bigint: true });
|
||||
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs || named.dev !== before.dev || named.ino !== before.ino || bytes.length !== Number(before.size) || digest(bytes) !== expected) throw new Error("Installed Daytona plugin bytes differ from reviewed pins");
|
||||
return bytes;
|
||||
} finally { await handle.close(); }
|
||||
}
|
||||
function compiledEntry(manifest: any): string | undefined {
|
||||
const exported = manifest.exports?.["."];
|
||||
return typeof exported === "string" ? exported : (typeof exported?.import === "string" ? exported.import : exported?.import?.default) ?? manifest.main;
|
||||
}
|
||||
export async function verifyInstalledDaytonaPlugin(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[]) {
|
||||
const selected = installedDaytonaPluginKeys.some(key => env[key] !== undefined);
|
||||
const required = usesInstalledCli(env) && executions.some(e => e.environment.id === "daytona");
|
||||
if (!selected && !required) return undefined;
|
||||
assertInstalledCliSelection(env, executions);
|
||||
if (!usesInstalledCli(env) || !installedDaytonaPluginKeys.every(key => Boolean(env[key])) || executions.some(e => e.environment.id !== "daytona")) throw new Error("Installed Daytona plugin requires complete pins and installed Pi Daytona selection");
|
||||
const authorityPath = env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY!;
|
||||
const authoritySha256 = env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN_AUTHORITY_SHA256!;
|
||||
const a: InstalledDaytonaPluginAuthority = JSON.parse((await pinned(authorityPath, authoritySha256, 65536)).toString());
|
||||
if (a.schema !== "paperclip.e2e.installed-daytona-plugin/v1" || a.plugin?.root !== env.PAPERCLIP_RUNNER_E2E_INSTALLED_DAYTONA_PLUGIN) throw new Error("Installed Daytona plugin authority differs");
|
||||
await canonical(a.graphRoot);
|
||||
async function packageAt(pin: PackagePin, name: string) {
|
||||
await canonical(pin.root);
|
||||
if (pin.root !== join(a.graphRoot, "node_modules", name)) throw new Error("Installed Daytona plugin package escapes its reviewed graph");
|
||||
const manifest = JSON.parse((await pinned(join(pin.root, "package.json"), pin.packageSha256, 65536)).toString());
|
||||
if (manifest.name !== name || typeof manifest.version !== "string" || !/^\d+\.\d+\.\d+(?:[-+][a-zA-Z0-9.-]+)?$/.test(manifest.version) || pin.entry !== (name === "@daytonaio/sdk" ? "./esm/index.js" : "./dist/index.js") || compiledEntry(manifest) !== pin.entry) throw new Error("Installed Daytona plugin requires published compiled package exports");
|
||||
await pinned(join(pin.root, pin.entry), pin.entrySha256);
|
||||
return manifest;
|
||||
}
|
||||
const plugin = await packageAt(a.plugin, "@paperclipai/plugin-daytona");
|
||||
const sdk = await packageAt(a.sdk, "@paperclipai/plugin-sdk");
|
||||
const shared = await packageAt(a.shared, "@paperclipai/shared");
|
||||
const daytona = await packageAt(a.daytona, "@daytonaio/sdk");
|
||||
if (plugin.paperclipPlugin?.manifest !== "./dist/manifest.js" || plugin.paperclipPlugin?.worker !== "./dist/worker.js" || plugin.dependencies?.["@paperclipai/plugin-sdk"] !== sdk.version || sdk.dependencies?.["@paperclipai/shared"] !== shared.version || plugin.dependencies?.["@daytonaio/sdk"] !== "0.203.0" || daytona.version !== "0.203.0") throw new Error("Installed Daytona plugin public dependency identity differs");
|
||||
await pinned(join(a.plugin.root, "dist/manifest.js"), a.plugin.manifestSha256);
|
||||
await pinned(join(a.plugin.root, "dist/worker.js"), a.plugin.workerSha256);
|
||||
for (const [name, parent, pin] of [["@paperclipai/plugin-sdk", join(a.plugin.root, "dist/worker.js"), a.sdk], ["@paperclipai/shared", join(a.sdk.root, a.sdk.entry), a.shared], ["@daytonaio/sdk", join(a.plugin.root, "dist/worker.js"), a.daytona]] as const) {
|
||||
const found = findPackageJSON(name, pathToFileURL(parent));
|
||||
if (!found || await realpath(found) !== join(pin.root, "package.json")) throw new Error("Installed Daytona plugin resolves a foreign dependency");
|
||||
}
|
||||
return { schema: a.schema, authorityPath, authoritySha256, packageRoot: a.plugin.root, graphRoot: a.graphRoot, fullGraphVerified: false, completeGraphAuthorityRequired: true, sourceFallback: false };
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/** Credential-free proof of the real launcher/Playwright/WebServer chain. */
|
||||
import { test, expect } from "@playwright/test";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
test("installed controller serves health and UI without creating provider work", async ({ page, request, baseURL }) => {
|
||||
test.skip(process.env.PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY !== "1");
|
||||
expect(baseURL).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
|
||||
const errors: string[] = [];
|
||||
page.on("pageerror", error => errors.push(error.message));
|
||||
await page.route("**/*", route => {
|
||||
const url = new URL(route.request().url());
|
||||
return url.origin === baseURL ? route.continue() : route.abort("blockedbyclient");
|
||||
});
|
||||
const health = await request.get(`${baseURL}/api/health`);
|
||||
expect(health.ok()).toBe(true);
|
||||
const before = await request.get(`${baseURL}/api/companies`);
|
||||
expect(before.ok()).toBe(true); expect(await before.json()).toEqual([]);
|
||||
const response = await page.goto(baseURL!);
|
||||
expect(response?.ok()).toBe(true);
|
||||
await expect(page.locator("#root")).not.toBeEmpty();
|
||||
await expect(page.getByRole("heading", { name: "What is the name of your organization?", exact: true })).toBeVisible();
|
||||
await expect(page.getByRole("textbox", { name: "Name", exact: true })).toBeVisible();
|
||||
await expect(page.getByRole("button", { name: "Continue", exact: true })).toBeDisabled();
|
||||
expect(errors).toEqual([]);
|
||||
const after = await request.get(`${baseURL}/api/companies`);
|
||||
expect(after.ok()).toBe(true); expect(await after.json()).toEqual([]);
|
||||
const output = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR!;
|
||||
await page.screenshot({ path: path.join(output, "installed-startup-only.png"), fullPage: true });
|
||||
await writeFile(path.join(output, "installed-startup-only.json"), `${JSON.stringify({ status: "health_ui_zero_company_passed", qualified: false, providerCalls: 0, healthStatus: health.status(), uiStatus: response!.status(), companiesBefore: 0, companiesAfter: 0, pageErrors: errors }, null, 2)}\n`, { mode: 0o600 });
|
||||
});
|
||||
@@ -1,4 +1,6 @@
|
||||
import { assertInstalledCliSelection, verifyInstalledCli } from "./installed-cli.js";
|
||||
import { runnerE2EPlaywrightInvocation } from "./web-server-command.js";
|
||||
import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
|
||||
import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js";
|
||||
import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
|
||||
@@ -257,7 +259,7 @@ async function prepareProviderPath(
|
||||
}
|
||||
|
||||
async function runProcess(
|
||||
args: string[],
|
||||
invocation: { command: string; args: string[] },
|
||||
env: NodeJS.ProcessEnv,
|
||||
timeoutMs: number | null,
|
||||
logPath: string,
|
||||
@@ -265,7 +267,7 @@ async function runProcess(
|
||||
interactive: boolean,
|
||||
) {
|
||||
const log = createWriteStream(logPath, { flags: "a", mode: 0o600 });
|
||||
const child = spawn("pnpm", args, {
|
||||
const child = spawn(invocation.command, invocation.args, {
|
||||
cwd: repositoryRoot,
|
||||
env,
|
||||
stdio: ["inherit", "pipe", "pipe"],
|
||||
@@ -474,6 +476,7 @@ async function runAttempt(input: {
|
||||
}
|
||||
assertInstalledCliSelection(process.env, executions);
|
||||
const installedCli = await verifyInstalledCli(process.env, executions);
|
||||
const installedPlugin = await verifyInstalledDaytonaPlugin(process.env, executions);
|
||||
const startedAtMs = Date.now();
|
||||
const sharedMemoryBaseline = snapshotDarwinSharedMemory();
|
||||
const temporaryRoot = await mkdtemp(
|
||||
@@ -483,6 +486,7 @@ async function runAttempt(input: {
|
||||
const publishedResultPaths = new Map<string, string>();
|
||||
let attemptSecrets: string[] = [];
|
||||
let processCleanupFailed = false;
|
||||
let startupReceiptPath: string | undefined;
|
||||
try {
|
||||
const paperclipHome = path.join(temporaryRoot, "paperclip-home");
|
||||
const workspace = path.join(temporaryRoot, "workspace");
|
||||
@@ -520,6 +524,7 @@ async function runAttempt(input: {
|
||||
]);
|
||||
attemptSecrets = credentials;
|
||||
if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 });
|
||||
if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 });
|
||||
const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness(
|
||||
executions,
|
||||
repositoryRoot,
|
||||
@@ -531,6 +536,7 @@ async function runAttempt(input: {
|
||||
executions.map((candidate) => candidate.id),
|
||||
),
|
||||
PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt),
|
||||
PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY: options.installedStartupOnly ? "1" : undefined,
|
||||
PAPERCLIP_RUNNER_E2E_PORT: String(port),
|
||||
PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot,
|
||||
PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir,
|
||||
@@ -565,8 +571,6 @@ async function runAttempt(input: {
|
||||
delete childEnv.DATABASE_MIGRATION_URL;
|
||||
|
||||
const playwrightArgs = [
|
||||
"exec",
|
||||
"playwright",
|
||||
"test",
|
||||
"--config",
|
||||
"tests/runner-e2e/playwright.config.ts",
|
||||
@@ -586,7 +590,7 @@ async function runAttempt(input: {
|
||||
) +
|
||||
5 * 60_000;
|
||||
const processResult = await runProcess(
|
||||
playwrightArgs,
|
||||
runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs, Boolean(installedCli)),
|
||||
childEnv,
|
||||
watchdog,
|
||||
path.join(privateDir, "playwright.log"),
|
||||
@@ -596,6 +600,18 @@ async function runAttempt(input: {
|
||||
options.ui || options.debug,
|
||||
);
|
||||
processCleanupFailed = processResult.processCleanupError !== null;
|
||||
if (options.installedStartupOnly) {
|
||||
const proofDir = path.join(resultsRoot, campaignId, "installed-startup-only");
|
||||
await mkdir(proofDir, { recursive: true });
|
||||
// This is private setup evidence, never a successful provider case/result.
|
||||
await cp(privateDir, path.join(proofDir, "private"), { recursive: true });
|
||||
const probe = JSON.parse(await readFile(path.join(privateDir, "installed-startup-only.json"), "utf8"));
|
||||
if (processResult.exitCode !== 0 || processResult.timedOut || processResult.spawnError || processCleanupFailed || probe.status !== "health_ui_zero_company_passed" || probe.providerCalls !== 0 || probe.qualified !== false) throw new Error("Installed startup-only actual launch chain failed");
|
||||
await assertEmbeddedDatabaseIsolation(configPath, temporaryRoot);
|
||||
startupReceiptPath = path.join(proofDir, "receipt.json");
|
||||
await writeFile(startupReceiptPath, `${JSON.stringify({ status: "installed_launch_chain_pending_scratch_cleanup", qualified: false, providerCalls: 0, credentialsLoaded: false, installedCli, processResult, probe }, null, 2)}\n`, { mode: 0o600 });
|
||||
return [];
|
||||
}
|
||||
const processFailure = processResult.spawnError
|
||||
? `Playwright failed to start: ${processResult.spawnError}`
|
||||
: processResult.timedOut
|
||||
@@ -824,6 +840,12 @@ async function runAttempt(input: {
|
||||
`Refusing to remove unexpected temporary path ${temporaryRoot}`,
|
||||
);
|
||||
}
|
||||
if (startupReceiptPath) {
|
||||
const receipt = JSON.parse(await readFile(startupReceiptPath, "utf8"));
|
||||
receipt.status = cleanupError ? "installed_launch_chain_cleanup_failed" : "installed_launch_chain_passed";
|
||||
receipt.temporaryRootRemoved = !cleanupError;
|
||||
await writeFile(startupReceiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
|
||||
}
|
||||
if (cleanupError) {
|
||||
const message = `Temporary runner E2E state cleanup failed at ${temporaryRoot}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`;
|
||||
for (const publishedResult of publishedResults) {
|
||||
@@ -848,6 +870,7 @@ async function runAttempt(input: {
|
||||
// Cleanup failure is a failed cell, but must not suppress later cells in
|
||||
// the same campaign. The result above carries the terminal failure.
|
||||
console.error(message);
|
||||
if (options.installedStartupOnly) throw new Error(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -939,6 +962,14 @@ async function main() {
|
||||
throw error;
|
||||
}
|
||||
const executions = selectRunnerExecutions(options, runnerMatrix);
|
||||
if (options.installedStartupOnly) {
|
||||
assertInstalledStartupOnly(process.env, executions, options);
|
||||
assertRunnerE2EPrerequisites(executions);
|
||||
const campaignId = cleanId(process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? `installed-startup-${Date.now()}`);
|
||||
await runAttempt({ executions, attempt: 1, campaignId, options });
|
||||
console.log(`PASS installed launch chain (no provider qualification) -> ${path.join(resultsRoot, campaignId, "installed-startup-only")}`);
|
||||
return;
|
||||
}
|
||||
if (options.list) {
|
||||
printList(executions);
|
||||
return;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
|
||||
import path from "node:path";
|
||||
import { isManagedHiringCase } from "./chat-cases.js";
|
||||
import { FixtureRegistry } from "./fixture-registry.js";
|
||||
@@ -113,8 +114,9 @@ export async function setupLiveFixtures(input: {
|
||||
registry.register<PluginRecord>({
|
||||
id: "sandbox-provider",
|
||||
async setup() {
|
||||
const installed = await verifyInstalledDaytonaPlugin(process.env, [execution]);
|
||||
return api.post<PluginRecord>("/api/plugins/install", {
|
||||
packageName: path.resolve(
|
||||
packageName: installed?.packageRoot ?? path.resolve(
|
||||
import.meta.dirname,
|
||||
"../../packages/plugins/sandbox-providers/daytona",
|
||||
),
|
||||
|
||||
@@ -45,7 +45,7 @@ const repositoryRoot = path.resolve(import.meta.dirname, "../..");
|
||||
|
||||
export default defineConfig({
|
||||
testDir: ".",
|
||||
testMatch: "runner.spec.ts",
|
||||
testMatch: process.env.PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY === "1" ? "installed-startup.spec.ts" : "runner.spec.ts",
|
||||
timeout: Number(process.env.PAPERCLIP_RUNNER_E2E_TEST_TIMEOUT_MS ?? 600_000),
|
||||
expect: { timeout: 30_000 },
|
||||
fullyParallel: false,
|
||||
|
||||
@@ -5,6 +5,7 @@ export interface RunnerSelectorOptions {
|
||||
all: boolean;
|
||||
list: boolean;
|
||||
matrixJson: boolean;
|
||||
installedStartupOnly?: boolean;
|
||||
ids: string[];
|
||||
suites: string[];
|
||||
groups: string[];
|
||||
@@ -52,6 +53,7 @@ export function parseRunnerSelectors(
|
||||
if (flag === "--all") options.all = true;
|
||||
else if (flag === "--list") options.list = true;
|
||||
else if (flag === "--matrix-json") options.matrixJson = true;
|
||||
else if (flag === "--installed-startup-only") options.installedStartupOnly = true;
|
||||
else if (flag === "--headed") options.headed = true;
|
||||
else if (flag === "--ui") options.ui = true;
|
||||
else if (flag === "--debug") options.debug = true;
|
||||
|
||||
@@ -21,3 +21,12 @@ export function runnerE2EWebServerCommand(repositoryRoot: string) {
|
||||
export function runnerE2ETypeScriptProcessArgs(repositoryRoot: string, entry: string, args: string[] = []) {
|
||||
return ["--import", path.join(repositoryRoot, "cli/node_modules/tsx/dist/loader.mjs"), entry, ...args];
|
||||
}
|
||||
|
||||
// pnpm's generated playwright bin shim adds NODE_PATH, even when the caller
|
||||
// rejected ambient injection. The installed lane invokes the public JS bin
|
||||
// with the current Node directly, keeping the WebServer environment closed.
|
||||
export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[], installed: boolean) {
|
||||
return installed
|
||||
? { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] }
|
||||
: { command: "pnpm", args: ["exec", "playwright", ...args] };
|
||||
}
|
||||
Reference in new issue
Block a user