From ab0696ae9d7976ae668c65fe91e8b7427ea69d82 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:11:45 -0500 Subject: [PATCH 1/9] test(runner): isolate cancellation timing and portable snapshot paths Paperclip-Task: rich-acp-production --- .../acpx/installation-integrity.test.ts | 12 +++++-- .../src/drivers/runner-tool-bridge.test.ts | 36 ++++++++++++++----- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts index b97b7c51ac..2318cbf298 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts @@ -1926,9 +1926,15 @@ async function expectOutput( }); const [exitCode] = await once(child, "exit"); expect(exitCode, stderr).toBe(0); - const normalized = process.platform === "darwin" - ? stdout.replace(/\/private\/var\/[^"\s]*\/paperclip-acpx-[^/]+\/0/g, "/proc/self/fd/4") - : stdout; + let normalized = stdout; + if (process.platform === "darwin") { + const snapshotPrefix = join(await realpath(tmpdir()), "paperclip-acpx-") + .replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + normalized = stdout.replace( + new RegExp(`${snapshotPrefix}[^/"\\s]+/0(?=[/"])`, "g"), + "/proc/self/fd/4", + ); + } expect(normalized).toBe(expected); } diff --git a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts index b552adcb28..ff2a412d3b 100644 --- a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts +++ b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts @@ -186,14 +186,12 @@ describe("runner semantic MCP bridge", () => { expect(handler).toHaveBeenCalledTimes(2); }); - it("times out calls and honors MCP cancellation", async () => { - const starts: string[] = []; + it("times out calls", async () => { const bridge = await startRunnerToolBridge({ tools: [tool("documents.read")], timeoutMs: 20, - handler: ({ callId, signal }) => + handler: ({ signal }) => new Promise((_resolve, reject) => { - starts.push(callId); signal.addEventListener( "abort", () => reject(new Error("handler aborted")), @@ -217,15 +215,36 @@ describe("runner semantic MCP bridge", () => { content: [{ text: "Paperclip tool call timed out" }], }, }); + }); + + it("honors MCP cancellation after the handler starts", async () => { + let markStarted!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + const aborted = vi.fn(); + const bridge = await startRunnerToolBridge({ + tools: [tool("documents.read")], + timeoutMs: 60_000, + handler: ({ signal }) => new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => { + aborted(); + reject(new Error("handler aborted")); + }, { once: true }); + markStarted(); + }), + }); + bridges.push(bridge); const pending = rpc(bridge, { id: "cancel-me", method: "tools/call", params: { name: "documents.read", arguments: {} }, }); - await vi.waitFor(() => expect(starts).toContain("cancel-me"), { - interval: 1, - timeout: 15, - }); + // Observe request arrival without racing the separate timeout behavior. + // Retain the rejection for the assertion while avoiding an unhandled fetch + // rejection if a failed test closes the bridge before the request arrives. + void pending.catch(() => undefined); + await started; expect( ( await rpc(bridge, { @@ -240,6 +259,7 @@ describe("runner semantic MCP bridge", () => { content: [{ text: "Paperclip tool call cancelled" }], }, }); + expect(aborted).toHaveBeenCalledOnce(); }); it("preserves successful mutation identity when the result is oversized", async () => { From 6405942a77c2a8f1add333387e29e2624dafae5e Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:35:28 -0500 Subject: [PATCH 2/9] test(runner): drain retained commits before terminal ACK fault Reproduce slow durable event commits in the completed-ACK row while leaving the native two-second deadline intact. Let maintenance enqueue suspend after all retained events drain so the row reliably exercises actual ACK loss and preserves its exact replay assertions. Co-Authored-By: Paperclip --- .../runnerd-codex-transport-settlement.test.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts index b760a59e09..826d5c466f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts @@ -273,7 +273,14 @@ it.each([ builder.queueCommand("turn.stop", { reason: "interrupted original close", }); - if (!missingHome) builder.queueCommand("runner.suspend", {}); + // The outbound-ACK row must reach its injected loss after retained event + // commits finish. An already queued suspend starts the real two-second + // ACK deadline behind that commit queue and can time out without ever + // sending the frame this row is meant to withhold. Maintenance queues its + // own suspend after draining; the other rows keep the old pending command. + if (!missingHome && completedTerminalAck !== "completed") { + builder.queueCommand("runner.suspend", {}); + } // Match the retained production split: runner-owned unacknowledged // output plus another full provider-owned prefix behind the old suspend. const runnerFile = join(original, "runner/runner-state.json"); @@ -349,7 +356,13 @@ it.each([ ), ) .digest("hex"); - const appendEvent = vi.fn(async (_event: PrpEvent) => {}); + const appendEvent = vi.fn(async (_event: PrpEvent) => { + // Reproduce slow durable commits without extending the runtime ACK + // deadline: all 218 retained events still have to commit exactly once. + if (completedTerminalAck === "completed") { + await new Promise((resolveCommit) => setTimeout(resolveCommit, 30)); + } + }); const maintenanceAbort = new AbortController(); let finalAuthorityRevoked = false; const authorize = vi.fn(async () => { From 6331ea1a1d8842b19d1cf43c167b8f25804204fa Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:31:40 -0500 Subject: [PATCH 3/9] fix(runner): load live console helpers from owning modules Co-Authored-By: Paperclip --- .../scripts/live-console-browser-server.mjs | 20 +++++++++++-- .../live-console-browser-server.test.mjs | 30 ++++++++++++++++++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/packages/paperclip-runner/scripts/live-console-browser-server.mjs b/packages/paperclip-runner/scripts/live-console-browser-server.mjs index 777837d3fe..fc67929dea 100644 --- a/packages/paperclip-runner/scripts/live-console-browser-server.mjs +++ b/packages/paperclip-runner/scripts/live-console-browser-server.mjs @@ -11,8 +11,22 @@ import { resolve } from "node:path"; * `PAPERCLIP_LIVE_CONSOLE_DRIVER=codex` swaps in the real Codex app-server driver * behind exactly the same routes. */ -async function loadRunner() { - return import(new URL("../dist/index.js", import.meta.url).href); +export async function loadLiveConsoleRunner(loadModule = (url) => import(url)) { + // Demo helpers are intentionally absent from the public runtime entrypoint. + // Load their owning modules, as the local-runner devtool transport does. + const [server, manifests, scripted, codex] = await Promise.all([ + "../dist/mock-core/live-console-demo-server.js", + "../dist/mock-core/live-console-demo-manifests.js", + "../dist/mock-core/live-console-scripted-driver.js", + "../dist/drivers/codex/codex-app-server-driver.js", + ].map((path) => loadModule(new URL(path, import.meta.url).href))); + return { + assertLiveConsoleLoopbackBindHost: server.assertLiveConsoleLoopbackBindHost, + LiveConsoleDemoServer: server.LiveConsoleDemoServer, + liveConsoleDemoManifestCatalogue: manifests.liveConsoleDemoManifestCatalogue, + LiveConsoleScriptedDriver: scripted.LiveConsoleScriptedDriver, + CodexAppServerDriver: codex.CodexAppServerDriver, + }; } async function createWorkingDirectory() { @@ -22,7 +36,7 @@ async function createWorkingDirectory() { } export function createLiveConsoleBrowserMiddleware(options = {}) { - const load = options.loadRunner ?? loadRunner; + const load = options.loadRunner ?? loadLiveConsoleRunner; const driverMode = options.driverMode ?? process.env.PAPERCLIP_LIVE_CONSOLE_DRIVER ?? "demo"; const chunkDelayMs = Number.parseInt( options.chunkDelayMs ?? process.env.PAPERCLIP_LIVE_CONSOLE_CHUNK_DELAY_MS ?? "45", diff --git a/packages/paperclip-runner/scripts/live-console-browser-server.test.mjs b/packages/paperclip-runner/scripts/live-console-browser-server.test.mjs index cde0b7b97c..3eefda1889 100644 --- a/packages/paperclip-runner/scripts/live-console-browser-server.test.mjs +++ b/packages/paperclip-runner/scripts/live-console-browser-server.test.mjs @@ -1,8 +1,36 @@ import { describe, expect, it } from "vitest"; -import { createLiveConsoleBrowserMiddleware } from "./live-console-browser-server.mjs"; +import { createLiveConsoleBrowserMiddleware, loadLiveConsoleRunner } from "./live-console-browser-server.mjs"; describe("Live console Vite transport bootstrap", () => { + it("loads the demo helpers from their owning modules instead of the public barrel", async () => { + const paths = []; + const runner = await loadLiveConsoleRunner(async (url) => { + const path = new URL(url).pathname; + paths.push(path.slice(path.lastIndexOf("/dist/") + 6)); + // Exercise real exports without requiring generated dist in unit tests. + return import(url.replace("/dist/", "/src/").replace(/\.js$/, ".ts")); + }); + expect(paths).toEqual([ + "mock-core/live-console-demo-server.js", + "mock-core/live-console-demo-manifests.js", + "mock-core/live-console-scripted-driver.js", + "drivers/codex/codex-app-server-driver.js", + ]); + expect(runner.assertLiveConsoleLoopbackBindHost).toBeTypeOf("function"); + expect(runner.LiveConsoleDemoServer).toBeTypeOf("function"); + expect(runner.LiveConsoleScriptedDriver).toBeTypeOf("function"); + expect(runner.CodexAppServerDriver).toBeTypeOf("function"); + expect(runner.liveConsoleDemoManifestCatalogue().length).toBeGreaterThan(0); + expect(() => runner.assertLiveConsoleLoopbackBindHost("0.0.0.0")).toThrow(); + const middleware = createLiveConsoleBrowserMiddleware({ + workingDirectory: "/server-owned-workspace", + loadRunner: async () => runner, + }); + await middleware.prepare("127.0.0.1"); + await middleware.close(); + }); + it("uses the runner's shared bind guard before constructing middleware", async () => { const calls = []; const middleware = createLiveConsoleBrowserMiddleware({ From 623fcc12134a190aa88052490864736409712a18 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:38:36 -0500 Subject: [PATCH 4/9] test(runner): align static replay fixture expectations Co-Authored-By: Paperclip --- .../paperclip-runner/devtools/browser/static-replay.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/paperclip-runner/devtools/browser/static-replay.spec.ts b/packages/paperclip-runner/devtools/browser/static-replay.spec.ts index 98dfaf7288..6fe4489ea0 100644 --- a/packages/paperclip-runner/devtools/browser/static-replay.spec.ts +++ b/packages/paperclip-runner/devtools/browser/static-replay.spec.ts @@ -5,6 +5,7 @@ test("validates and renders the shared Replay fixture", async ({ page }, testInf await expect(page.getByRole("heading", { name: "Live runner diagnostics" })).toBeVisible(); await page.getByRole("button", { name: "Static replay" }).click(); await expect(page.getByRole("heading", { name: "Static protocol replay" })).toBeVisible(); + await page.getByLabel("Fixture", { exact: true }).selectOption("happy-path"); await expect(page.getByTestId("terminal-badge")).toHaveText("Succeeded"); await expect(page.getByTestId("timeline").getByRole("listitem")).toHaveCount(9); await expect(page.getByTestId("timeline")).not.toContainText("workspace_preparing"); @@ -26,7 +27,7 @@ test("shows duplicate and unsupported-version replay states", async ({ page }) = .getByLabel("Fixture", { exact: true }) .selectOption("unsupported-required-version"); await expect(page.getByRole("heading", { name: "Fixture cannot be replayed" })).toBeVisible(); - await expect(page.getByText(/protocolVersion 2 is unsupported/)).toBeVisible(); + await expect(page.getByText(/protocolVersion 3 is unsupported/)).toBeVisible(); }); test("streams a live run and proves replay parity", async ({ page }, testInfo) => { From 01d6fcb8325212f710e7b6a592b2e10a87600f26 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:39:39 -0500 Subject: [PATCH 5/9] docs(runner): link current Rust bridge architecture Paperclip-Task: rich-acp-production Co-Authored-By: Paperclip --- packages/paperclip-runner/docs/capability-live-runnerd-codex.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/paperclip-runner/docs/capability-live-runnerd-codex.md b/packages/paperclip-runner/docs/capability-live-runnerd-codex.md index 7ba6106c9d..42598ff3fd 100644 --- a/packages/paperclip-runner/docs/capability-live-runnerd-codex.md +++ b/packages/paperclip-runner/docs/capability-live-runnerd-codex.md @@ -3,7 +3,7 @@ > **Reference lab, not the production sandbox topology.** This API remains the > runner-lab/session implementation used by UI and recovery tests. Production > sandbox execution and live protocol evals use the Rust-owned bridge described -> in [`../../../doc/plans/2026-08-20-single-daemon-runner-tool-bridge.md`](../../../doc/plans/2026-08-20-single-daemon-runner-tool-bridge.md): external control plane → PRP → one Rust `paperclip-runnerd` → provider. The TypeScript dispatcher below does not run in the sandbox. +> in [the runner architecture](architecture.md): external control plane → PRP → one Rust `paperclip-runnerd` → provider. The TypeScript dispatcher below does not run in the sandbox. Capability binds the provider-neutral semantic catalog to a real package-local `paperclip-runnerd` process and a real Codex app-server session. Paperclip data From d16f56ef71db8b526a5f047e54189e17dc05845c Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:42:27 -0500 Subject: [PATCH 6/9] test(runner): check App icon parity at the server boundary Keep both live and catalog schema assertions through public runner exports while removing the standalone suite dependency on App source. Paperclip-Task: rich-acp-production Co-Authored-By: Paperclip --- .../catalog/semantic-action-catalog.test.ts | 12 --------- .../runner-project-icon-contract.test.ts | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 server/src/__tests__/runner-project-icon-contract.test.ts diff --git a/packages/paperclip-runner/src/catalog/semantic-action-catalog.test.ts b/packages/paperclip-runner/src/catalog/semantic-action-catalog.test.ts index d1835af7bd..544735cfef 100644 --- a/packages/paperclip-runner/src/catalog/semantic-action-catalog.test.ts +++ b/packages/paperclip-runner/src/catalog/semantic-action-catalog.test.ts @@ -20,18 +20,6 @@ const packageRoot = resolve( ); describe("semantic action catalog", () => { - it("advertises only icons accepted by the project API on both tool surfaces", async () => { - const { PROJECT_ICON_NAMES } = await import("../../../shared/src/constants.js"); - const ajv = new Ajv2020({ allErrors: true, allowUnionTypes: true, strict: true }); - for (const schema of [createProjectAction.live.descriptor.inputSchema, paperclipSemanticAction("create_project")!.inputSchema]) { - const validate = ajv.compile(schema); - const input = { name: "Onboarding", idempotencyKey: "onboarding" }; - expect(schema).toMatchObject({ properties: { icon: { enum: [...PROJECT_ICON_NAMES, null] } } }); - for (const icon of [...PROJECT_ICON_NAMES, null]) expect(validate({ ...input, icon }), String(icon)).toBe(true); - expect(validate({ ...input, icon: "users" })).toBe(false); - } - }); - it("limits project repository URLs to HTTPS GitHub repository paths on both tool surfaces", () => { const ajv = new Ajv2020({ allErrors: true, allowUnionTypes: true, strict: true }); for (const schema of [createProjectAction.live.descriptor.inputSchema, paperclipSemanticAction("create_project")!.inputSchema]) { diff --git a/server/src/__tests__/runner-project-icon-contract.test.ts b/server/src/__tests__/runner-project-icon-contract.test.ts new file mode 100644 index 0000000000..04d7c20b7e --- /dev/null +++ b/server/src/__tests__/runner-project-icon-contract.test.ts @@ -0,0 +1,25 @@ +import Ajv2020 from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; +import { + capabilitySemanticToolDescriptor, + paperclipSemanticAction, +} from "@paperclipai/paperclip-runner"; +import { PROJECT_ICON_NAMES } from "@paperclipai/shared"; + +// Cross-package parity belongs at the App boundary: the standalone runner +// must not import the App's shared package, even from its own tests. +describe("runner project icon contract", () => { + it("advertises only icons accepted by the project API on both tool surfaces", () => { + const ajv = new Ajv2020({ allErrors: true, allowUnionTypes: true, strict: true }); + for (const schema of [ + capabilitySemanticToolDescriptor("create_project")!.inputSchema, + paperclipSemanticAction("create_project")!.inputSchema, + ]) { + const validate = ajv.compile(schema); + const input = { name: "Onboarding", idempotencyKey: "onboarding" }; + expect(schema).toMatchObject({ properties: { icon: { enum: [...PROJECT_ICON_NAMES, null] } } }); + for (const icon of [...PROJECT_ICON_NAMES, null]) expect(validate({ ...input, icon }), String(icon)).toBe(true); + expect(validate({ ...input, icon: "users" })).toBe(false); + } + }); +}); From f7ab50455bd6018f4acc065829755865655763ce Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:46:32 -0500 Subject: [PATCH 7/9] fix(runner): parse import gates and pack explicit consumer inputs Use the declared TypeScript parser to distinguish module references from fixture strings. Enforce declared public exports and the ADR0001 private development-only eval harness boundary, including public transitive exposure. Pack absolute dependency inputs from scratch without adopting publisher development toolchains. Co-Authored-By: Paperclip --- .../scripts/check-clean-consumers.mjs | 5 +- .../scripts/check-forbidden-imports.test.mjs | 131 ++++++++++++++++ .../scripts/check-tracked-imports.test.mjs | 26 ++++ .../scripts/lib/forbidden-imports.mjs | 144 +++++++++++++----- .../scripts/lib/tracked-imports.mjs | 11 +- 5 files changed, 276 insertions(+), 41 deletions(-) diff --git a/packages/paperclip-runner/scripts/check-clean-consumers.mjs b/packages/paperclip-runner/scripts/check-clean-consumers.mjs index 65b9310404..bc4ee1ce9e 100644 --- a/packages/paperclip-runner/scripts/check-clean-consumers.mjs +++ b/packages/paperclip-runner/scripts/check-clean-consumers.mjs @@ -78,7 +78,10 @@ try { async function pack(packageRoot, destination) { const before = new Set(await readdir(destination)); - run("npm", ["pack", "--ignore-scripts", "--pack-destination", destination], packageRoot, { quiet: true }); + // These installed dependencies are pack inputs, not our development cwd. + // Keep their publisher devEngines intact without adopting their toolchain; + // normal tarball contents and clean-consumer runtime engine checks still apply. + run("npm", ["pack", resolve(packageRoot), "--ignore-scripts", "--pack-destination", destination], destination, { quiet: true }); const created = (await readdir(destination)) .filter((entry) => entry.endsWith(".tgz") && !before.has(entry)) .sort(); diff --git a/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs b/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs index f4dc12e447..59a620f4f9 100644 --- a/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs +++ b/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs @@ -1,9 +1,13 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, resolve } from "node:path"; import { checkForbiddenImports, defaultPackageRoot, + findSpecifiers, } from "./lib/forbidden-imports.mjs"; test("the package passes its standalone boundary", async () => { @@ -59,3 +63,130 @@ test("a negative fixture proves that an SDK consumer deep import is rejected", a assert.equal(violations.length, 1); assert.match(violations[0].reason, /may not deep-import/); }); + +test("the parser ignores embedded fixtures but preserves executable imports and types", () => { + const source = [ + '// import "comment";', + '/* export * from "comment-export"; */', + 'const fixture = "import value from \\"fixture\\"";', + 'const template = `require("fixture-require"); import("fixture-dynamic")`;', + 'const pattern = /import\\("fixture-regex"\\)/;', + 'import "side-effect";', + 'import type { T } from "types";', + 'export { value } from "barrel";', + 'export * from "star";', + 'import equal = require("equal");', + 'type Imported = import("import-type").T;', + 'const cjs = require("cjs");', + 'const dynamic = import("dynamic", { with: { type: "json" } });', + 'const literal = import(`static-template`);', + 'const escaped = import("escaped\\u002dname");', + 'const interpolation = `text ${import("executed-interpolation")}`;', + ].join("\n"); + const found = findSpecifiers(source); + assert.deepEqual(found.map(({ specifier }) => specifier), [ + "side-effect", "types", "barrel", "star", "equal", "import-type", "cjs", + "dynamic", "static-template", "escaped-name", "executed-interpolation", + ]); + assert.deepEqual(found.map(({ offset }) => source.slice(0, offset).split("\n").length), + [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); +}); + +test("the parser treats JSX fixture text separately from expression imports", () => { + assert.deepEqual(findSpecifiers( + '
require("text") {import("executed")}
', + "fixture.tsx", + ).map(({ specifier }) => specifier), ["executed"]); +}); + +test("Rust include macros and path modules retain their boundary checks", () => { + assert.deepEqual(findSpecifiers([ + 'include!("../included.rs");', + 'include_str!("../text.txt");', + 'include_bytes!("../bytes.bin");', + '#[path = "../module.rs"] mod module;', + ].join("\n"), "source.rs").map(({ specifier }) => specifier), + ["../included.rs", "../text.txt", "../bytes.bin", "../module.rs"]); +}); + +async function fixtureBoundary(t, { manifest = {}, files = {} } = {}) { + const root = await mkdtemp(resolve(tmpdir(), "runner-import-boundary-")); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(resolve(root, "src"), { recursive: true }); + await writeFile(resolve(root, "package.json"), JSON.stringify(manifest)); + for (const [file, source] of Object.entries(files)) { + await mkdir(dirname(resolve(root, file)), { recursive: true }); + await writeFile(resolve(root, file), source); + } + return checkForbiddenImports({ packageRoot: root, scanRoots: ["src"], cargoRoots: [] }); +} + +const kernel = "@paperclipai/paperclip-eval-kernel"; +const harness = "src/eval/workflow-harness.ts"; +const reviewedManifest = { + exports: { "./evals": "./dist/evals.js" }, + devDependencies: { [kernel]: "workspace:*" }, +}; + +test("only exact declared public subpaths are admitted", async (t) => { + const violations = await fixtureBoundary(t, { + manifest: reviewedManifest, + files: { "src/consumer.ts": [ + 'import "@paperclipai/paperclip-runner/evals";', + 'import "@paperclipai/paperclip-runner/evals/private";', + 'import "@paperclipai/paperclip-runner/undeclared";', + ].join("\n") }, + }); + assert.deepEqual(violations.map(({ specifier }) => specifier), [ + "@paperclipai/paperclip-runner/evals/private", "@paperclipai/paperclip-runner/undeclared", + ]); +}); + +test("ADR0001 permits only the private development harness and reviewed package", async (t) => { + assert.deepEqual(await fixtureBoundary(t, { + manifest: reviewedManifest, + files: { [harness]: `import { scenario } from "${kernel}";`, "src/evals.ts": "export const publicApi = true;" }, + }), []); + for (const [file, specifier] of [ + ["src/other.ts", kernel], [harness, `${kernel}/private`], [harness, "@paperclipai/other"], + ]) { + const violations = await fixtureBoundary(t, { + manifest: reviewedManifest, files: { [file]: `import "${specifier}";` }, + }); + assert.equal(violations.length, 1); + assert.equal(violations[0].specifier, specifier); + } + assert.equal((await fixtureBoundary(t, { + files: { [harness]: `import "${kernel}";` }, + })).length, 1, "the exception requires the reviewed devDependency declaration"); +}); + +test("the reviewed kernel is still forbidden in every production dependency group", async (t) => { + for (const group of ["dependencies", "optionalDependencies", "peerDependencies"]) { + const violations = await fixtureBoundary(t, { + manifest: { ...reviewedManifest, [group]: { [kernel]: "workspace:*" } }, + }); + assert.equal(violations.length, 1, group); + assert.equal(violations[0].specifier, kernel); + } +}); + +test("direct or transitive public harness exposure revokes the dev-only exception", async (t) => { + for (const entry of [ + "./dist/eval/workflow-harness.js", + "./dist/evals.js", + ]) { + for (const imported of ["./eval/workflow-harness.js", "./eval/workflow-harness"]) { + const violations = await fixtureBoundary(t, { + manifest: { ...reviewedManifest, exports: { "./evals": { types: "./dist/evals.d.ts", import: entry } } }, + files: { + [harness]: `import "${kernel}";`, + "src/evals.ts": 'export * from "./barrel.js";', + "src/barrel.ts": `export * from "${imported}";`, + }, + }); + assert.equal(violations.length, 1, `${entry} via ${imported}`); + assert.equal(violations[0].specifier, kernel); + } + } +}); diff --git a/packages/paperclip-runner/scripts/check-tracked-imports.test.mjs b/packages/paperclip-runner/scripts/check-tracked-imports.test.mjs index 79778857de..66151ddc83 100644 --- a/packages/paperclip-runner/scripts/check-tracked-imports.test.mjs +++ b/packages/paperclip-runner/scripts/check-tracked-imports.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import { resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { test } from "node:test"; import { @@ -71,3 +74,26 @@ test("the fixture itself is tracked so the package-wide check stays green", asyn assert.ok(tracked.has(resolve(defaultPackageRoot, fixtureRoot, name)), name); } }); + + +test("embedded fixture imports are ignored while real missing imports still fail", async (t) => { + const root = await mkdtemp(resolve(tmpdir(), "runner-tracked-imports-")); + t.after(() => rm(root, { recursive: true, force: true })); + const initialized = spawnSync("git", ["init", "--quiet", root], { encoding: "utf8" }); + assert.equal(initialized.status, 0, initialized.stderr); + await mkdir(resolve(root, "src")); + const consumer = resolve(root, "src/consumer.mts"); + await writeFile(consumer, [ + 'const fixture = `import "./fixture-only.js"; require("./other-fixture.js")`;', + 'const actual = import("./missing.js", { with: { type: "json" } });', + 'export { tracked } from "./tracked.js";', + ].join("\n")); + const tracked = resolve(root, "src/tracked.ts"); + await writeFile(tracked, "export const tracked = true;"); + const violations = await checkTrackedImports({ + packageRoot: root, scanRoots: ["src"], trackedFiles: [consumer, tracked], + }); + assert.deepEqual(violations.map(({ specifier, line, reason }) => ({ specifier, line, reason })), [{ + specifier: "./missing.js", line: 2, reason: "does not resolve to any tracked file", + }]); +}); diff --git a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs index 5132fe4519..2fda4a5377 100644 --- a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs +++ b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs @@ -1,15 +1,17 @@ import { readFile, readdir } from "node:fs/promises"; import { dirname, isAbsolute, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { API } from "typescript/unstable/sync"; +import { createVirtualFileSystem } from "typescript/unstable/fs"; +import { SyntaxKind } from "typescript/unstable/ast"; -const SOURCE_EXTENSIONS = new Set([".cjs", ".js", ".jsx", ".mjs", ".rs", ".ts", ".tsx"]); -const IMPORT_PATTERNS = [ - /\b(?:import|export)\s+(?:type\s+)?(?:[^"'`;]{0,500}?\s+from\s+)?["']([^"']+)["']/g, - /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g, - /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g, +const SOURCE_EXTENSIONS = new Set([".cjs", ".js", ".jsx", ".mjs", ".mts", ".cts", ".rs", ".ts", ".tsx"]); +const RUST_IMPORT_PATTERNS = [ /\b(?:include|include_str|include_bytes)!\s*\(\s*["']([^"']+)["']\s*\)/g, /#\[path\s*=\s*["']([^"']+)["']\]/g, ]; +const REVIEWED_EVAL_KERNEL = "@paperclipai/paperclip-eval-kernel"; +const REVIEWED_EVAL_HARNESS = "src/eval/workflow-harness.ts"; const libraryDirectory = dirname(fileURLToPath(import.meta.url)); export const defaultPackageRoot = resolve(libraryDirectory, "../.."); @@ -44,15 +46,92 @@ async function collectSourceFiles(target) { return files; } -export function findSpecifiers(source) { - const found = []; - for (const pattern of IMPORT_PATTERNS) { - pattern.lastIndex = 0; - for (let match = pattern.exec(source); match !== null; match = pattern.exec(source)) { - found.push({ specifier: match[1], offset: match.index }); +/** Parse a whole scan in one compiler session; never execute scanned source. */ +export function findSpecifiersInFiles(files) { + const results = new Map(files.map(({ file }) => [file, []])); + const javascript = files.filter(({ file }) => extension(file) !== ".rs"); + for (const { file, source } of files.filter(({ file }) => extension(file) === ".rs")) { + for (const pattern of RUST_IMPORT_PATTERNS) { + pattern.lastIndex = 0; + for (const match of source.matchAll(pattern)) { + results.get(file).push({ specifier: match[1], offset: match.index }); + } } } - return found; + if (javascript.length === 0) return results; + const root = "/__paperclip_import_check__"; + const names = javascript.map(({ file }, index) => `input-${index}${extension(file) || ".ts"}`); + const virtualFiles = Object.fromEntries(javascript.map(({ source }, index) => [`${root}/${names[index]}`, source])); + virtualFiles[`${root}/tsconfig.json`] = JSON.stringify({ + compilerOptions: { noLib: true, noResolve: true, allowJs: true }, files: names, + }); + const api = new API({ fs: createVirtualFileSystem(virtualFiles) }); + try { + const snapshot = api.updateSnapshot({ openProjects: [`${root}/tsconfig.json`] }); + const program = snapshot.getProject(`${root}/tsconfig.json`).program; + for (const [index, { file }] of javascript.entries()) { + const sourceFile = program.getSourceFile(`${root}/${names[index]}`); + if (!sourceFile) throw new Error(`Import scanner could not parse ${file}`); + const found = results.get(file); + function add(literal) { + if (literal?.kind === SyntaxKind.StringLiteral || literal?.kind === SyntaxKind.NoSubstitutionTemplateLiteral) { + found.push({ specifier: literal.text, offset: literal.getStart(sourceFile) }); + } + } + function visit(node) { + if (node.kind === SyntaxKind.ImportDeclaration || node.kind === SyntaxKind.ExportDeclaration) add(node.moduleSpecifier); + else if (node.kind === SyntaxKind.ImportEqualsDeclaration && node.moduleReference.kind === SyntaxKind.ExternalModuleReference) add(node.moduleReference.expression); + else if (node.kind === SyntaxKind.ImportType && node.argument.kind === SyntaxKind.LiteralType) add(node.argument.literal); + else if (node.kind === SyntaxKind.CallExpression && ( + node.expression.kind === SyntaxKind.ImportKeyword || + (node.expression.kind === SyntaxKind.Identifier && node.expression.text === "require") + )) add(node.arguments[0]); + node.forEachChild(visit); + } + visit(sourceFile); + } + } finally { + api.close(); + } + return results; +} + +export function findSpecifiers(source, file = "source.ts") { + return findSpecifiersInFiles([{ file, source }]).get(file); +} + +function declaredPublicImports(manifest) { + return new Set(Object.entries(manifest.exports ?? {}) + .filter(([key, target]) => key.startsWith("./") && !key.includes("*") && target !== null) + .map(([key]) => `@paperclipai/paperclip-runner/${key.slice(2)}`)); +} + +function publicSourceGraph(packageRoot, manifest, specifiers) { + const known = new Set(specifiers.keys()); + const sourceCandidates = (path) => { + const source = path.replace(/\/dist\//, "/src/"); + const stem = source.replace(/(?:\.d)?\.(?:[cm]?js|jsx|[cm]?ts|tsx)$/, ""); + // Follow every matching source candidate conservatively, including barrels + // and extensionless imports. A public graph must never hide the dev harness. + return [source, ...[".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", "/index.ts", "/index.tsx", "/index.js"] + .map((suffix) => `${stem}${suffix}`)].filter((candidate) => known.has(candidate)); + }; + const targets = (value) => typeof value === "string" ? [value] : value && typeof value === "object" ? Object.values(value).flatMap(targets) : []; + const exportSources = (value) => targets(value).flatMap((target) => sourceCandidates(resolve(packageRoot, target))); + const queue = exportSources(manifest.exports); + const reachable = new Set(); + while (queue.length > 0) { + const file = queue.pop(); + if (reachable.has(file)) continue; + reachable.add(file); + for (const { specifier } of specifiers.get(file) ?? []) { + if (specifier.startsWith(".")) queue.push(...sourceCandidates(resolve(dirname(file), specifier))); + else if (specifier.startsWith("@paperclipai/paperclip-runner/")) { + queue.push(...exportSources(manifest.exports?.[`./${specifier.slice("@paperclipai/paperclip-runner/".length)}`])); + } + } + } + return reachable; } // The Live console component decision record adapts shadcn/ui and AI Elements @@ -79,16 +158,9 @@ function isForbiddenBrowserPackage(specifier) { ); } -function violationReason({ file, packageRoot, specifier }) { +function violationReason({ file, packageRoot, specifier, publicRunnerImports, reviewedEvalHarness }) { const relativeFile = relative(packageRoot, file).split(/[\\/]/).join("/"); const isExampleConsumer = relativeFile.startsWith("examples/"); - const publicRunnerImports = new Set([ - "@paperclipai/paperclip-runner/browser", - "@paperclipai/paperclip-runner/react", - "@paperclipai/paperclip-runner/standalone", - "@paperclipai/paperclip-runner/testing", - "@paperclipai/paperclip-runner/styles.css", - ]); if ( specifier.startsWith("@paperclipai/paperclip-runner/") && !publicRunnerImports.has(specifier) @@ -101,7 +173,8 @@ function violationReason({ file, packageRoot, specifier }) { if ( specifier.startsWith("@paperclipai/") && specifier !== "@paperclipai/paperclip-runner" && - !publicRunnerImports.has(specifier) + !publicRunnerImports.has(specifier) && + !(specifier === REVIEWED_EVAL_KERNEL && reviewedEvalHarness && relativeFile === REVIEWED_EVAL_HARNESS) ) { return "Paperclip workspace packages are outside the standalone boundary"; } @@ -152,7 +225,7 @@ async function manifestViolations(packageRoot) { const unreviewedDevelopmentDependencies = Object.keys( manifest.devDependencies ?? {}, ).filter( - (name) => name.startsWith("@paperclipai/"), + (name) => name.startsWith("@paperclipai/") && !(name === REVIEWED_EVAL_KERNEL && manifest.devDependencies[name] === "workspace:*"), ); return [...runtimeDependencies, ...unreviewedDevelopmentDependencies] .filter( @@ -221,19 +294,20 @@ export async function checkForbiddenImports({ files.push(...(await collectSourceFiles(resolve(packageRoot, root)))); } - for (const file of files.sort()) { - const source = await readFile(file, "utf8"); - for (const { specifier, offset } of findSpecifiers(source)) { - const reason = violationReason({ file, packageRoot, specifier }); - if (reason === null) { - continue; - } - violations.push({ - file, - line: source.slice(0, offset).split("\n").length, - specifier, - reason, - }); + const manifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); + const sources = await Promise.all(files.sort().map(async (file) => ({ file, source: await readFile(file, "utf8") }))); + const specifiers = findSpecifiersInFiles(sources); + const publicRunnerImports = declaredPublicImports(manifest); + const publicSources = publicSourceGraph(packageRoot, manifest, specifiers); + // ADR0001 permits only this private development harness to use the kernel. + // A public export reaching it revokes the exception, including through barrels. + const reviewedEvalHarness = manifest.devDependencies?.[REVIEWED_EVAL_KERNEL] === "workspace:*" + && !publicSources.has(resolve(packageRoot, REVIEWED_EVAL_HARNESS)); + for (const { file, source } of sources) { + for (const { specifier, offset } of specifiers.get(file)) { + const reason = violationReason({ file, packageRoot, specifier, publicRunnerImports, reviewedEvalHarness }); + if (reason === null) continue; + violations.push({ file, line: source.slice(0, offset).split("\n").length, specifier, reason }); } } diff --git a/packages/paperclip-runner/scripts/lib/tracked-imports.mjs b/packages/paperclip-runner/scripts/lib/tracked-imports.mjs index d11dd14e93..97a98efca8 100644 --- a/packages/paperclip-runner/scripts/lib/tracked-imports.mjs +++ b/packages/paperclip-runner/scripts/lib/tracked-imports.mjs @@ -4,13 +4,13 @@ import { readFile } from "node:fs/promises"; import { dirname, relative, resolve } from "node:path"; import { promisify } from "node:util"; -import { defaultPackageRoot, findSpecifiers } from "./forbidden-imports.mjs"; +import { defaultPackageRoot, findSpecifiersInFiles } from "./forbidden-imports.mjs"; export { defaultPackageRoot }; const execFileAsync = promisify(execFile); -const SCANNED_EXTENSIONS = new Set([".cjs", ".js", ".jsx", ".mjs", ".ts", ".tsx"]); +const SCANNED_EXTENSIONS = new Set([".cjs", ".js", ".jsx", ".mjs", ".mts", ".cts", ".ts", ".tsx"]); // TypeScript resolves an ESM ".js" specifier against its ".ts" source sibling, // so a tracked import may legitimately name a file that never exists on disk. const JS_TO_SOURCE_EXTENSIONS = new Map([ @@ -129,9 +129,10 @@ export async function checkTrackedImports({ .sort(); const suspects = []; - for (const file of files) { - const source = await readFile(file, "utf8"); - for (const { specifier, offset } of findSpecifiers(source)) { + const sources = await Promise.all(files.map(async (file) => ({ file, source: await readFile(file, "utf8") }))); + const specifiers = findSpecifiersInFiles(sources); + for (const { file, source } of sources) { + for (const { specifier, offset } of specifiers.get(file)) { if (!isRelative(specifier) || /[?*]|\$\{/.test(specifier)) { continue; } From c8239a3bab05562cdae8957ed761b8f27606c372 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:48:24 -0500 Subject: [PATCH 8/9] fix(runner): reject dev-only exceptions under wildcard exports Treat unexpanded wildcard export targets as potentially exposing every scanned source, so they cannot bypass the reviewed private eval harness boundary. Co-Authored-By: Paperclip --- .../scripts/check-forbidden-imports.test.mjs | 11 +++++++++++ .../scripts/lib/forbidden-imports.mjs | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs b/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs index 59a620f4f9..faa3b01543 100644 --- a/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs +++ b/packages/paperclip-runner/scripts/check-forbidden-imports.test.mjs @@ -190,3 +190,14 @@ test("direct or transitive public harness exposure revokes the dev-only exceptio } } }); + + +test("unexpanded wildcard exports cannot expose the reviewed development harness", async (t) => { + const violations = await fixtureBoundary(t, { + manifest: { ...reviewedManifest, exports: { "./eval/*": "./dist/eval/*.js" } }, + files: { [harness]: `import "${kernel}";` }, + }); + assert.equal(violations.length, 1); + assert.equal(violations[0].specifier, kernel); + assert.match(violations[0].reason, /outside the standalone boundary/); +}); diff --git a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs index 2fda4a5377..be6b0f6830 100644 --- a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs +++ b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs @@ -117,7 +117,12 @@ function publicSourceGraph(packageRoot, manifest, specifiers) { .map((suffix) => `${stem}${suffix}`)].filter((candidate) => known.has(candidate)); }; const targets = (value) => typeof value === "string" ? [value] : value && typeof value === "object" ? Object.values(value).flatMap(targets) : []; - const exportSources = (value) => targets(value).flatMap((target) => sourceCandidates(resolve(packageRoot, target))); + const exportSources = (value) => targets(value).flatMap((target) => { + // Wildcard exports can expose the development harness. Until this gate + // resolves their full mapping, conservatively treat all sources as public. + if (target.includes("*")) return [...known]; + return sourceCandidates(resolve(packageRoot, target)); + }); const queue = exportSources(manifest.exports); const reachable = new Set(); while (queue.length > 0) { From 11da146c3c39aea5605fdfac31e619fd8024227b Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 13:49:34 -0500 Subject: [PATCH 9/9] test(runner): align issue-thread assertions with current catalog Assert the canonical active-task description and all nine DevTools tabs while retaining document-read, layout, and interaction assertions. Co-Authored-By: Paperclip --- .../devtools/issue-thread/issue-thread.spec.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/paperclip-runner/devtools/issue-thread/issue-thread.spec.ts b/packages/paperclip-runner/devtools/issue-thread/issue-thread.spec.ts index fa9c0f2551..e907fa9513 100644 --- a/packages/paperclip-runner/devtools/issue-thread/issue-thread.spec.ts +++ b/packages/paperclip-runner/devtools/issue-thread/issue-thread.spec.ts @@ -251,7 +251,7 @@ test.describe("Capability issue thread", () => { await always.click(); await panel.getByRole("button", { name: /get_task_context/ }).click(); const dialog = page.getByRole("dialog", { name: "Get active task context" }); - await expect(dialog).toContainText("Read the active mock task, actor, wake, ancestors, budget, and interaction results."); + await expect(dialog).toContainText("Read the active task and actor, including the exact approved Markdown revision when this issue has an accepted plan."); await expect(dialog).toContainText("Input schema"); await dialog.getByRole("button", { name: "Close tool details" }).click(); await expect(dialog).toHaveCount(0); @@ -723,9 +723,12 @@ test.describe("Capability clean-room chat", () => { await page.getByTestId("evidence-toggle").click(); const panel = page.getByTestId("evidence-panel"); const tabs = panel.getByRole("tab"); - await expect(tabs.first()).toHaveText(/Evidence/); - await expect(tabs.nth(1)).toHaveText(/Timeline/); - await expect(panel.locator(".pit-devtools-tabs .pit-icon")).toHaveCount(8); + await expect(tabs).toHaveText([ + "Evidence", "Timeline", "State", "Diff", "Documents", "Protocol", + "Provider trace", "Runtime", "Authority", + ]); + await expect(tabs).toHaveCount(9); + await expect(panel.locator(".pit-devtools-tabs .pit-icon")).toHaveCount(9); const centerOffsets = await tabs.evaluateAll((elements) => elements.map((element) => { const icon = element.querySelector(".pit-tab-glyph")?.getBoundingClientRect(); const label = element.querySelector(".pit-tab-glyph + span")?.getBoundingClientRect();