From 5e6c1666c14edc954c994e8920966a09de3602ca Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 3 Oct 2026 17:30:11 -0500 Subject: [PATCH] test(cursor): exercise the installed public product without runtime overrides Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 17 ++++++++++++ tests/runner-e2e/installed-release.test.ts | 31 ++++++++++++++++++++++ tests/runner-e2e/installed-release.ts | 31 ++++++++++++++++++++++ tests/runner-e2e/launch.ts | 9 +++++-- tests/runner-e2e/server.ts | 11 +++++--- 5 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/installed-release.test.ts create mode 100644 tests/runner-e2e/installed-release.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index db22cb50173..b53e98e0e1f 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1597,3 +1597,20 @@ The v3 hiring grader uses turn-accounting v2 in both executable guards. It requi The live fixture retries entire bracketed observations, waits for both known task callbacks and attributed replies (including batching), checks untruncated pending-wake diagnostics, and requires two equal settled observations. Silence before outbox enqueue is not delivery. Five-turn generic accounting remains calibrated for no owed notifications; this delegated fixture owes two completions. All actual runs remain counted for usage and cost. Retained original, limited sidecar-v1, initial executable, and stricter v3 assessments remain separately versioned; no models are rerun by the repair. Recovery-state retention uses raw cleanup results before evidence publication. An owner-only resource-admission marker also preserves state if the test worker dies before producing a result. Confirmed bootstrap failures before allocation remove their temporary state and retain their original failure classification. + +## Public installed release smoke + +Set `PAPERCLIP_RUNNER_E2E_INSTALLED_CLI` to the absolute public consumer's +`paperclipai/dist/index.js` for an installed-product acceptance run. Install the +public package graph and run its ordinary `runtime setup cursor` first. The +supervisor launches that compiled CLI from its own package directory; repository +server-entry patches, provider-bin shims, loader injection, provider packs and +native binary overrides are removed from the server environment. Qualification +admission is rejected, so this path requires production admission. Use the +existing browser/API cases and encrypted company-secret fixture path. + +For a diagnosed failure campaign, `PAPERCLIP_RUNNER_E2E_KEEP_FAILED_PRIVATE=1` +retains the attempt's owner-only private directory after a failed case even +when owned cleanup passed. This does not alter the case or cleanup outcome. +Private traces and database files must not be published. Unconfirmed cleanup +always preserves recovery state regardless of this optional diagnostic flag. diff --git a/tests/runner-e2e/installed-release.test.ts b/tests/runner-e2e/installed-release.test.ts new file mode 100644 index 00000000000..7e450403088 --- /dev/null +++ b/tests/runner-e2e/installed-release.test.ts @@ -0,0 +1,31 @@ +import { mkdtempSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { installedReleaseEnvironment, installedReleaseLaunch } from "./installed-release.js"; + +const roots: string[] = []; +afterEach(() => roots.splice(0).forEach(root => rmSync(root, { recursive: true, force: true }))); +describe("ordinary installed release smoke", () => { + it("launches the public compiled CLI from its own consumer root", () => { + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "installed-paperclip-"))); roots.push(root); + mkdirSync(path.join(root, "dist")); + const cli = path.join(root, "dist", "index.js"); writeFileSync(cli, ""); + writeFileSync(path.join(root, "package.json"), JSON.stringify({ name: "paperclipai", version: "0.0.0-verify" })); + expect(installedReleaseLaunch(cli)).toMatchObject({ cwd: root, args: [cli, "onboard", "--yes", "--run"], version: "0.0.0-verify" }); + writeFileSync(path.join(root, "package.json"), JSON.stringify({ name: "other-cli", version: "1.0.0" })); + expect(() => installedReleaseLaunch(cli)).toThrow("public package"); + expect(() => installedReleaseLaunch("relative/index.js")).toThrow("absolute path"); + }); + it("removes repository executables, provider packs, native paths and loader injection", () => { + const source = { PATH: "/repo/packages/node_modules/.bin:/tmp/provider-bin:/usr/bin:/tmp/node/bin", NODE_OPTIONS: "--import /repo/loader", NODE_PATH: "/repo/node_modules", + PAPERCLIP_RUNNER_BINARY: "/repo/runner", PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT: "/private/pack", + PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: "/private/pack", PAPERCLIP_ACPX_BUILTIN_ROOT: "/repo/providers", + PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH: "/repo/context", PAPERCLIP_RUNNER_E2E_PORT: "3100", PAPERCLIP_HOME: "/tmp/home" }; + expect(installedReleaseEnvironment(source, "/repo", "/tmp/provider-bin")).toEqual({ PATH: "/usr/bin:/tmp/node/bin", PAPERCLIP_RUNNER_E2E_PORT: "3100", PAPERCLIP_HOME: "/tmp/home" }); + expect(source.PAPERCLIP_RUNNER_BINARY).toBe("/repo/runner"); + }); + it("rejects qualification admission instead of silently relabeling it as production", () => { + expect(() => installedReleaseEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "cursor" }, "/repo", "/tmp/provider-bin")).toThrow("production admission"); + }); +}); diff --git a/tests/runner-e2e/installed-release.ts b/tests/runner-e2e/installed-release.ts new file mode 100644 index 00000000000..def2391d0f4 --- /dev/null +++ b/tests/runner-e2e/installed-release.ts @@ -0,0 +1,31 @@ +import { readFileSync, realpathSync } from "node:fs"; +import path from "node:path"; + +/** Test supervisor only: launch the actual public CLI, with its own module root. */ +export function installedReleaseLaunch(cliPath: string) { + if (!path.isAbsolute(cliPath)) throw new Error("Installed release CLI must be an absolute path"); + const cli = realpathSync(cliPath); + const root = path.resolve(path.dirname(cli), ".."); + const manifest = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8")); + if (path.basename(cli) !== "index.js" || path.basename(path.dirname(cli)) !== "dist" || + manifest.name !== "paperclipai" || typeof manifest.version !== "string") { + throw new Error("Installed release must use paperclipai/dist/index.js from its public package"); + } + return { cli, cwd: root, args: [cli, "onboard", "--yes", "--run"], version: manifest.version }; +} + +/** Qualification assets must never make a public installed product smoke pass. */ +export function installedReleaseEnvironment(source: NodeJS.ProcessEnv, repositoryRoot: string, providerBin: string): NodeJS.ProcessEnv { + if (source.PAPERCLIP_RUNNER_ACPX_QUALIFICATION) throw new Error("Installed release smoke requires production admission without qualification overrides"); + const result = { ...source }; + for (const key of Object.keys(result)) { + if ((key.startsWith("PAPERCLIP_RUNNER_") && !key.startsWith("PAPERCLIP_RUNNER_E2E_")) || + key.startsWith("PAPERCLIP_ACPX_") || key.startsWith("PAPERCLIP_NATIVE_") || + key === "NODE_PATH" || key === "NODE_OPTIONS") delete result[key]; + } + result.PATH = source.PATH?.split(path.delimiter).filter(entry => + entry && entry !== providerBin && entry !== repositoryRoot && + !entry.startsWith(`${repositoryRoot}${path.sep}`) && !entry.includes(`${path.sep}node_modules${path.sep}`), + ).join(path.delimiter); + return result; +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index ce3f0304aef..14b042e611b 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -485,7 +485,7 @@ async function runAttempt(input: { const publishedResultPaths = new Map(); let attemptSecrets: string[] = []; let processCleanupFailed = false; - const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean }> = []; + const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = []; try { const paperclipHome = path.join(temporaryRoot, "paperclip-home"); const workspace = path.join(temporaryRoot, "workspace"); @@ -630,7 +630,7 @@ async function runAttempt(input: { ); const result = await readResult(resultPath, fallback); // Keep cleanup authority before evidence copying/publication can fail. - rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback }); + rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback, status: result.status }); return enforceResultProcessIntegrity(result, processResult); }), ); @@ -810,6 +810,11 @@ async function runAttempt(input: { // Deleting it after the controller exits would strand uncertain creates. await chmod(temporaryRoot, 0o700); cleanupError = new Error(`Preserving private recovery state after unconfirmed cleanup: ${temporaryRoot}`); + } else if (process.env.PAPERCLIP_RUNNER_E2E_KEEP_FAILED_PRIVATE === "1" && rawCleanupResults.some(result => result.status === "failed")) { + // Explicit diagnosis only. Confirmed resource cleanup stays confirmed; + // keep private traces for investigation without publishing provider data. + await chmod(temporaryRoot, 0o700); + console.warn(`Retained private failed-case diagnostics: ${temporaryRoot}`); } else if ( temporaryRoot.startsWith(`${os.tmpdir()}${path.sep}paperclip-runner-e2e-`) ) { diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 9acb4292a25..bc27bcfac46 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -6,6 +6,7 @@ import { createWriteStream } from "node:fs"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; import path from "node:path"; import { prepareRunnerE2EServerConfig } from "./server-config.js"; +import { installedReleaseEnvironment, installedReleaseLaunch } from "./installed-release.js"; import { assertIsolatedServerEnvironment, buildPaperclipServerEnvironment, @@ -32,7 +33,11 @@ const { } = runnerE2EServerControlPaths(temporaryRoot); const restartTimeoutMs = 180_000; const gracefulStopTimeoutMs = 30_000; -const serverEnvironment = buildPaperclipServerEnvironment(process.env, { +const installedRelease = process.env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI + ? installedReleaseLaunch(process.env.PAPERCLIP_RUNNER_E2E_INSTALLED_CLI) : null; +const serverEnvironment = buildPaperclipServerEnvironment(installedRelease + ? installedReleaseEnvironment(process.env, repositoryRoot, path.join(temporaryRoot, "provider-bin")) + : process.env, { NODE_ENV: "test", PORT: port, // Keep provider caches attempt-private without changing Playwright's browser @@ -112,9 +117,9 @@ async function startServer() { } const candidate = spawn( process.execPath, - runnerE2ETypeScriptProcessArgs(repositoryRoot, paperclipCli, ["onboard", "--yes", "--run"]), + installedRelease?.args ?? runnerE2ETypeScriptProcessArgs(repositoryRoot, paperclipCli, ["onboard", "--yes", "--run"]), { - cwd: repositoryRoot, + cwd: installedRelease?.cwd ?? repositoryRoot, env: definedServerEnvironment, stdio: ["ignore", "pipe", "pipe"], // Playwright signals the wrapper's group. Keep that signal from bypassing