diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-fixture.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-fixture.ts index 159e5a9e9e2..0c531c75498 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-plugin-fixture.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-fixture.ts @@ -1,10 +1,11 @@ -import { mkdtemp, mkdir, writeFile, readFile, rm } from "node:fs/promises"; +import { mkdtemp, mkdir, writeFile, readFile, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; +import { mirrorPreviewSdk } from "./external-plugin-sdk-mirror.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { PaperclipPluginManifestV1 } from "../../../packages/shared/src/types/plugin.js"; @@ -17,15 +18,17 @@ export function previewFixtureManifest(key: string): PaperclipPluginManifestV1 { tools: [{ name: "preview", displayName: "Preview website", description: "Read the current synthetic website preview", parametersSchema: { type: "object", properties: {}, additionalProperties: false } }], }; } -/** Runtime-only controlled package. CLI scaffolding and package admission happen - * during the coordinator's selected attempt, never during discovery/unit tests. */ +/** CLI scaffolding and package admission run during the selected attempt. + * The explicit offline calibration exercises scaffolding without admission. */ export async function createPreviewFixture(ctx: ChatCatalogContext) { - const temporary = await mkdtemp(path.join(tmpdir(), "paperclip-preview-eval-")); + const temporary = await realpath(await mkdtemp(path.join(tmpdir(), "paperclip-preview-eval-"))); const packageRoot = path.join(temporary, "website-preview"); - const cli = fileURLToPath(new URL("../../../cli/dist/index.js", import.meta.url)); + const cli = fileURLToPath(new URL("../../../cli/src/index.ts", import.meta.url)); + const loader = fileURLToPath(new URL("../../../cli/node_modules/tsx/dist/loader.mjs", import.meta.url)); const sdkUrl = new URL("../../../packages/plugins/sdk/dist/index.js", import.meta.url).href; try { - await promisify(execFile)(process.execPath, [cli, "plugin", "init", "@paperclip-eval/website-preview", "--output", temporary, "--display-name", "Website Preview Eval"], { timeout: 30_000, maxBuffer: 256_000, env: { PATH: process.env.PATH ?? "", NODE_ENV: "test", DO_NOT_TRACK: "1" } }); + const mirror = await mirrorPreviewSdk(fileURLToPath(new URL("../../../packages/plugins/sdk/", import.meta.url)), temporary); + await promisify(execFile)(process.execPath, ["--import", loader, cli, "plugin", "init", "@paperclip-eval/website-preview", "--output", temporary, "--display-name", "Website Preview Eval", "--sdk-path", mirror.sdkPath], { cwd: temporary, timeout: 30_000, maxBuffer: 256_000, env: { HOME: temporary, PATH: process.env.PATH ?? "", NODE_ENV: "test", DO_NOT_TRACK: "1", PAPERCLIP_TELEMETRY_DISABLED: "1" } }); const key = `paperclip.eval-preview-${ctx.input.nonce.replace(/[^a-z0-9]/gi, "").toLowerCase()}`; const manifest = previewFixtureManifest(key); const marker = `Synthetic website preview ${ctx.input.nonce}`; @@ -43,7 +46,7 @@ const plugin=definePlugin({ await writeFile(path.join(packageRoot, "package.json"), JSON.stringify({ name: "@paperclip-eval/website-preview", version: "0.1.0", type: "module", private: true, paperclipPlugin: { manifest: "./dist/manifest.js", worker: "./dist/worker.js" } })); await writeFile(path.join(packageRoot, "dist/manifest.js"), manifestSource); await writeFile(path.join(packageRoot, "dist/worker.js"), workerSource); const digest = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); - await ctx.input.evidence("chat-catalog-vetted-preview-package.json", { synthetic: true, pluginKey: key, manifestSha256: digest(manifestSource), workerSha256: digest(workerSource), cliSha256: digest(await readFile(cli)), sdkEntrySha256: digest(await readFile(fileURLToPath(sdkUrl))) }); + await ctx.input.evidence("chat-catalog-vetted-preview-package.json", { synthetic: true, cliMode: "supported-source-cli-tsx", cliLoaderSha256: digest(await readFile(loader)), pluginKey: key, sdkMirror: { files: mirror.files, archiveWrites: mirror.archiveWrites, packageBuilds: mirror.packageBuilds, packagePacks: mirror.packagePacks }, manifestSha256: digest(manifestSource), workerSha256: digest(workerSource), cliSha256: digest(await readFile(cli)), sdkEntrySha256: digest(await readFile(fileURLToPath(sdkUrl))) }); return { key, packageRoot, marker, async close() { await rm(temporary, { recursive: true, force: true }); } }; } catch { await rm(temporary, { recursive: true, force: true }); throw new Error("Disposable plugin scaffold failed; command output withheld"); } } diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-scaffold-calibration.test.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-scaffold-calibration.test.ts new file mode 100644 index 00000000000..51913939778 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-scaffold-calibration.test.ts @@ -0,0 +1,48 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; +import { expect, it } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { createPreviewFixture } from "./external-plugin-fixture.js"; + +it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_PLUGIN_SCAFFOLD_CALIBRATION !== "1")("qualifies the actual source CLI scaffold with explicit owned SDK custody and no build or pack", async () => { + const root = await mkdtemp(path.join(tmpdir(), "plugin-cli-offline-control-")); + const previousPath = process.env.PATH; + let fixture: Awaited> | undefined; + try { + const bin = path.join(root, "bin"); await mkdir(bin); + const called = path.join(root, "unexpected-pnpm-call"); + await writeFile(path.join(bin, "pnpm"), `#!/bin/sh\nprintf invoked > '${called}'\nexit 91\n`, { mode: 0o700 }); + const cli = fileURLToPath(new URL("../../../cli/dist/index.js", import.meta.url)); + const sourceSdk = fileURLToPath(new URL("../../../packages/plugins/sdk/", import.meta.url)); + const hash = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + const cliBefore = hash(await readFile(cli)); + const original = { failed: false, missingCheckoutDependency: false }; + try { + await promisify(execFile)(process.execPath, [cli, "plugin", "init", "@paperclip-eval/default-sdk-control", "--output", root], { + cwd: root, timeout: 30_000, maxBuffer: 256_000, env: { HOME: root, PATH: bin, NODE_ENV: "test", DO_NOT_TRACK: "1", PAPERCLIP_TELEMETRY_DISABLED: "1" }, + }); + } catch (error) { + original.failed = true; + + original.missingCheckoutDependency = /ERR_MODULE_NOT_FOUND/.test(String((error as { stderr?: string }).stderr)) && /Cannot find package 'zod'/.test(String((error as { stderr?: string }).stderr)); + } + expect(original).toEqual({ failed: true, missingCheckoutDependency: true }); + let evidence: { sdkMirror: { files: Array<{ path: string; sha256: string }> } } | undefined; + process.env.PATH = bin; + const ctx = { input: { nonce: "offline-cli-calibration", evidence: async (_name: string, value: unknown) => { evidence = value as typeof evidence; } } } as unknown as ChatCatalogContext; + fixture = await createPreviewFixture(ctx); + expect(JSON.parse(await readFile(path.join(fixture.packageRoot, "package.json"), "utf8"))).toMatchObject({ name: "@paperclip-eval/website-preview" }); + expect(evidence?.sdkMirror.files.length).toBeGreaterThan(1); + for (const file of evidence!.sdkMirror.files) expect(hash(await readFile(path.join(sourceSdk, file.path)))).toBe(file.sha256); + expect(hash(await readFile(cli))).toBe(cliBefore); + await expect(access(called)).rejects.toThrow(); + } finally { + if (previousPath === undefined) delete process.env.PATH; else process.env.PATH = previousPath; + await fixture?.close(); await rm(root, { recursive: true, force: true }); + } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.test.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.test.ts new file mode 100644 index 00000000000..a2b8d379099 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.test.ts @@ -0,0 +1,29 @@ +import { mkdtemp, mkdir, readFile, rm, realpath, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { expect, it } from "vitest"; +import { mirrorPreviewSdk } from "./external-plugin-sdk-mirror.js"; + +it("retains exact public SDK bytes in a separately owned scaffold workspace", async () => { + const root = await mkdtemp(path.join(tmpdir(), "sdk-custody-control-")); + try { + const source = path.join(root, "source"); await mkdir(path.join(source, "dist"), { recursive: true }); + const manifest = JSON.stringify({ name: "@paperclipai/plugin-sdk", exports: { ".": { import: "./dist/index.js" } } }); + await writeFile(path.join(source, "package.json"), manifest); await writeFile(path.join(source, "dist/index.js"), "export const publicSdk = true;\n"); + const owned = path.join(root, "owned"); await mkdir(owned); + await expect(mirrorPreviewSdk(source, source)).rejects.toThrow("separately owned"); + const result = await mirrorPreviewSdk(source, owned); + expect(result.sdkPath).toBe(path.join(await realpath(owned), "packages/plugins/sdk")); + expect(result.archiveWrites).toBe(false); expect(result.packageBuilds).toBe(0); expect(result.packagePacks).toBe(0); + for (const file of result.files) { + const original = await readFile(path.join(source, file.path)); + expect(await readFile(path.join(result.sdkPath, file.path))).toEqual(original); + expect(file.sha256).toBe(createHash("sha256").update(original).digest("hex")); + } + await expect(mirrorPreviewSdk(source, owned)).rejects.toThrow(); + const other = path.join(root, "other"); await mkdir(other); + await symlink(path.join(source, "package.json"), path.join(source, "dist/escape.js")); + await expect(mirrorPreviewSdk(source, other)).rejects.toThrow("symbolic links"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.ts new file mode 100644 index 00000000000..687851d243c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.ts @@ -0,0 +1,41 @@ +import { createHash } from "node:crypto"; +import { lstat, mkdir, readFile, readdir, realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; + +/** Copy genuine SDK package/dist bytes into the owned scaffold workspace. The + * public CLI can then use its workspace SDK route without packing the archive. */ +export async function mirrorPreviewSdk(source: string, ownedRoot: string) { + const root = await realpath(ownedRoot), sourceRoot = await realpath(source); + if (sourceRoot === root || root.startsWith(sourceRoot + path.sep) || sourceRoot.startsWith(root + path.sep)) + throw new Error("SDK mirror requires a separately owned output root"); + const sdkPath = path.join(root, "packages/plugins/sdk"); + await mkdir(sdkPath, { recursive: true }); + const files: Array<{ path: string; sha256: string; byteSize: number }> = []; + let totalBytes = 0; + async function copy(relative: string): Promise { + const sourcePath = path.join(sourceRoot, relative), targetPath = path.join(sdkPath, relative); + const stat = await lstat(sourcePath); + if (stat.isSymbolicLink()) throw new Error("SDK mirror rejects symbolic links"); + if (stat.isDirectory()) { + await mkdir(targetPath, { recursive: true }); + for (const name of (await readdir(sourcePath)).sort()) await copy(path.join(relative, name)); + return; + } + if (!stat.isFile() || files.length >= 512 || stat.size > 4_194_304 || totalBytes + stat.size > 33_554_432) + throw new Error("SDK mirror exceeds its bounded public package/dist inventory"); + const bytes = await readFile(sourcePath); + await writeFile(targetPath, bytes, { flag: "wx", mode: 0o600 }); + const hash = (body: Buffer) => createHash("sha256").update(body).digest("hex"); + const sha256 = hash(bytes); + if (hash(await readFile(targetPath)) !== sha256) throw new Error("SDK mirror byte custody mismatch"); + totalBytes += bytes.length; + files.push({ path: relative, sha256, byteSize: bytes.length }); + } + await copy("package.json"); + const manifest = JSON.parse(await readFile(path.join(sdkPath, "package.json"), "utf8")); + if (manifest.name !== "@paperclipai/plugin-sdk" || manifest.exports?.["."]?.import !== "./dist/index.js") + throw new Error("SDK mirror is not the actual public plugin SDK package"); + await copy("dist"); + if (!files.some(file => file.path === path.join("dist", "index.js"))) throw new Error("SDK mirror has no real compiled entrypoint"); + return { sdkPath, files, archiveWrites: false as const, packageBuilds: 0 as const, packagePacks: 0 as const }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts index 6a0b2988bfe..dda64b739ca 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts @@ -321,6 +321,9 @@ export const secretDeliveryExternalCases = [ "../../server/src/services/heartbeat/run-preparation.ts", "../../cli/dist/index.js", "../../cli/src/index.ts", + "../../cli/node_modules/tsx/dist/loader.mjs", + "chat-catalog-domains/external-plugin-sdk-mirror.ts", + "../../packages/plugins/sdk/package.json", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts index 273bb93e499..c20ffbc6e1d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts @@ -77,7 +77,7 @@ export const runtimeFailedRunCases = [runtimeDefinition("runs-adapters.inspect-f await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return { worker, task: done.task, ...failure, successfulRun, successArtifactVerified: true, failureLogAvailable }; }, gradeFailedAttempt, { - sourceFiles: ["chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/TaskChatThread.tsx", "../../ui/src/components/task-chat/TaskChatMarker.tsx", "../../ui/src/components/task-chat/TaskChatThreadView.tsx", "../../ui/src/pages/AgentDetail.tsx"], expectedRunCount: 3, attemptTimeoutMs: 720_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 2 } }, async observe(ctx, state) { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-session.ts b/tests/runner-e2e/chat-catalog-domains/runtime-session.ts index c5f6b319db5..6d3cc63deee 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-session.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-session.ts @@ -41,7 +41,7 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi for (const doc of documents) revisions.push(...await ctx.input.api.get(`/api/issues/${state.task.id}/documents/${encodeURIComponent(doc.key!)}/revisions`)); return { ...state, priorComments: comments, priorDocuments: documents, priorRevisions: revisions }; }, gradeSessionReset, { - sourceFiles: ["chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/AgentActionButtons.tsx", "../../ui/src/pages/AgentDetail.tsx"], expectedRunCount: 3, attemptTimeoutMs: 720_000, async act(ctx, state) { await pollUntil({ label: "Canonical session guidance settled", deadlineAt: Date.now() + 150_000,