fix(evals): scaffold preview plugins through the owned source CLI

tests/runner-e2e/chat-catalog-domains/external-plugin-fixture.ts

@@ -1,10 +1,11 @@
-import { mkdtemp, mkdir, writeFile, readFile, rm } from "node:fs/promises";
+import { mkdtemp, mkdir, writeFile, readFile, realpath, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { createHash } from "node:crypto";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
+import { mirrorPreviewSdk } from "./external-plugin-sdk-mirror.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import type { PaperclipPluginManifestV1 } from "../../../packages/shared/src/types/plugin.js";
 
@@ -17,15 +18,17 @@ export function previewFixtureManifest(key: string): PaperclipPluginManifestV1 {
     tools: [{ name: "preview", displayName: "Preview website", description: "Read the current synthetic website preview", parametersSchema: { type: "object", properties: {}, additionalProperties: false } }],
   };
 }
-/** Runtime-only controlled package. CLI scaffolding and package admission happen
- * during the coordinator's selected attempt, never during discovery/unit tests. */
+/** CLI scaffolding and package admission run during the selected attempt.
+ * The explicit offline calibration exercises scaffolding without admission. */
 export async function createPreviewFixture(ctx: ChatCatalogContext) {
-  const temporary = await mkdtemp(path.join(tmpdir(), "paperclip-preview-eval-"));
+  const temporary = await realpath(await mkdtemp(path.join(tmpdir(), "paperclip-preview-eval-")));
   const packageRoot = path.join(temporary, "website-preview");
-  const cli = fileURLToPath(new URL("../../../cli/dist/index.js", import.meta.url));
+  const cli = fileURLToPath(new URL("../../../cli/src/index.ts", import.meta.url));
+  const loader = fileURLToPath(new URL("../../../cli/node_modules/tsx/dist/loader.mjs", import.meta.url));
   const sdkUrl = new URL("../../../packages/plugins/sdk/dist/index.js", import.meta.url).href;
   try {
-    await promisify(execFile)(process.execPath, [cli, "plugin", "init", "@paperclip-eval/website-preview", "--output", temporary, "--display-name", "Website Preview Eval"], { timeout: 30_000, maxBuffer: 256_000, env: { PATH: process.env.PATH ?? "", NODE_ENV: "test", DO_NOT_TRACK: "1" } });
+    const mirror = await mirrorPreviewSdk(fileURLToPath(new URL("../../../packages/plugins/sdk/", import.meta.url)), temporary);
+    await promisify(execFile)(process.execPath, ["--import", loader, cli, "plugin", "init", "@paperclip-eval/website-preview", "--output", temporary, "--display-name", "Website Preview Eval", "--sdk-path", mirror.sdkPath], { cwd: temporary, timeout: 30_000, maxBuffer: 256_000, env: { HOME: temporary, PATH: process.env.PATH ?? "", NODE_ENV: "test", DO_NOT_TRACK: "1", PAPERCLIP_TELEMETRY_DISABLED: "1" } });
     const key = `paperclip.eval-preview-${ctx.input.nonce.replace(/[^a-z0-9]/gi, "").toLowerCase()}`;
     const manifest = previewFixtureManifest(key);
     const marker = `Synthetic website preview ${ctx.input.nonce}`;
@@ -43,7 +46,7 @@ const plugin=definePlugin({
     await writeFile(path.join(packageRoot, "package.json"), JSON.stringify({ name: "@paperclip-eval/website-preview", version: "0.1.0", type: "module", private: true, paperclipPlugin: { manifest: "./dist/manifest.js", worker: "./dist/worker.js" } }));
     await writeFile(path.join(packageRoot, "dist/manifest.js"), manifestSource); await writeFile(path.join(packageRoot, "dist/worker.js"), workerSource);
     const digest = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
-    await ctx.input.evidence("chat-catalog-vetted-preview-package.json", { synthetic: true, pluginKey: key, manifestSha256: digest(manifestSource), workerSha256: digest(workerSource), cliSha256: digest(await readFile(cli)), sdkEntrySha256: digest(await readFile(fileURLToPath(sdkUrl))) });
+    await ctx.input.evidence("chat-catalog-vetted-preview-package.json", { synthetic: true, cliMode: "supported-source-cli-tsx", cliLoaderSha256: digest(await readFile(loader)), pluginKey: key, sdkMirror: { files: mirror.files, archiveWrites: mirror.archiveWrites, packageBuilds: mirror.packageBuilds, packagePacks: mirror.packagePacks }, manifestSha256: digest(manifestSource), workerSha256: digest(workerSource), cliSha256: digest(await readFile(cli)), sdkEntrySha256: digest(await readFile(fileURLToPath(sdkUrl))) });
     return { key, packageRoot, marker, async close() { await rm(temporary, { recursive: true, force: true }); } };
   } catch { await rm(temporary, { recursive: true, force: true }); throw new Error("Disposable plugin scaffold failed; command output withheld"); }
 }

tests/runner-e2e/chat-catalog-domains/external-plugin-scaffold-calibration.test.ts

@@ -0,0 +1,48 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { createHash } from "node:crypto";
+import { expect, it } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { createPreviewFixture } from "./external-plugin-fixture.js";
+
+it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_PLUGIN_SCAFFOLD_CALIBRATION !== "1")("qualifies the actual source CLI scaffold with explicit owned SDK custody and no build or pack", async () => {
+  const root = await mkdtemp(path.join(tmpdir(), "plugin-cli-offline-control-"));
+  const previousPath = process.env.PATH;
+  let fixture: Awaited<ReturnType<typeof createPreviewFixture>> | undefined;
+  try {
+    const bin = path.join(root, "bin"); await mkdir(bin);
+    const called = path.join(root, "unexpected-pnpm-call");
+    await writeFile(path.join(bin, "pnpm"), `#!/bin/sh\nprintf invoked > '${called}'\nexit 91\n`, { mode: 0o700 });
+    const cli = fileURLToPath(new URL("../../../cli/dist/index.js", import.meta.url));
+    const sourceSdk = fileURLToPath(new URL("../../../packages/plugins/sdk/", import.meta.url));
+    const hash = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
+    const cliBefore = hash(await readFile(cli));
+    const original = { failed: false, missingCheckoutDependency: false };
+    try {
+      await promisify(execFile)(process.execPath, [cli, "plugin", "init", "@paperclip-eval/default-sdk-control", "--output", root], {
+        cwd: root, timeout: 30_000, maxBuffer: 256_000, env: { HOME: root, PATH: bin, NODE_ENV: "test", DO_NOT_TRACK: "1", PAPERCLIP_TELEMETRY_DISABLED: "1" },
+      });
+    } catch (error) {
+      original.failed = true;
+
+      original.missingCheckoutDependency = /ERR_MODULE_NOT_FOUND/.test(String((error as { stderr?: string }).stderr)) && /Cannot find package 'zod'/.test(String((error as { stderr?: string }).stderr));
+    }
+    expect(original).toEqual({ failed: true, missingCheckoutDependency: true });
+    let evidence: { sdkMirror: { files: Array<{ path: string; sha256: string }> } } | undefined;
+    process.env.PATH = bin;
+    const ctx = { input: { nonce: "offline-cli-calibration", evidence: async (_name: string, value: unknown) => { evidence = value as typeof evidence; } } } as unknown as ChatCatalogContext;
+    fixture = await createPreviewFixture(ctx);
+    expect(JSON.parse(await readFile(path.join(fixture.packageRoot, "package.json"), "utf8"))).toMatchObject({ name: "@paperclip-eval/website-preview" });
+    expect(evidence?.sdkMirror.files.length).toBeGreaterThan(1);
+    for (const file of evidence!.sdkMirror.files) expect(hash(await readFile(path.join(sourceSdk, file.path)))).toBe(file.sha256);
+    expect(hash(await readFile(cli))).toBe(cliBefore);
+    await expect(access(called)).rejects.toThrow();
+  } finally {
+    if (previousPath === undefined) delete process.env.PATH; else process.env.PATH = previousPath;
+    await fixture?.close(); await rm(root, { recursive: true, force: true });
+  }
+});

tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.test.ts

@@ -0,0 +1,29 @@
+import { mkdtemp, mkdir, readFile, rm, realpath, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { expect, it } from "vitest";
+import { mirrorPreviewSdk } from "./external-plugin-sdk-mirror.js";
+
+it("retains exact public SDK bytes in a separately owned scaffold workspace", async () => {
+  const root = await mkdtemp(path.join(tmpdir(), "sdk-custody-control-"));
+  try {
+    const source = path.join(root, "source"); await mkdir(path.join(source, "dist"), { recursive: true });
+    const manifest = JSON.stringify({ name: "@paperclipai/plugin-sdk", exports: { ".": { import: "./dist/index.js" } } });
+    await writeFile(path.join(source, "package.json"), manifest); await writeFile(path.join(source, "dist/index.js"), "export const publicSdk = true;\n");
+    const owned = path.join(root, "owned"); await mkdir(owned);
+    await expect(mirrorPreviewSdk(source, source)).rejects.toThrow("separately owned");
+    const result = await mirrorPreviewSdk(source, owned);
+    expect(result.sdkPath).toBe(path.join(await realpath(owned), "packages/plugins/sdk"));
+    expect(result.archiveWrites).toBe(false); expect(result.packageBuilds).toBe(0); expect(result.packagePacks).toBe(0);
+    for (const file of result.files) {
+      const original = await readFile(path.join(source, file.path));
+      expect(await readFile(path.join(result.sdkPath, file.path))).toEqual(original);
+      expect(file.sha256).toBe(createHash("sha256").update(original).digest("hex"));
+    }
+    await expect(mirrorPreviewSdk(source, owned)).rejects.toThrow();
+    const other = path.join(root, "other"); await mkdir(other);
+    await symlink(path.join(source, "package.json"), path.join(source, "dist/escape.js"));
+    await expect(mirrorPreviewSdk(source, other)).rejects.toThrow("symbolic links");
+  } finally { await rm(root, { recursive: true, force: true }); }
+});

tests/runner-e2e/chat-catalog-domains/external-plugin-sdk-mirror.ts

@@ -0,0 +1,41 @@
+import { createHash } from "node:crypto";
+import { lstat, mkdir, readFile, readdir, realpath, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+/** Copy genuine SDK package/dist bytes into the owned scaffold workspace. The
+ * public CLI can then use its workspace SDK route without packing the archive. */
+export async function mirrorPreviewSdk(source: string, ownedRoot: string) {
+  const root = await realpath(ownedRoot), sourceRoot = await realpath(source);
+  if (sourceRoot === root || root.startsWith(sourceRoot + path.sep) || sourceRoot.startsWith(root + path.sep))
+    throw new Error("SDK mirror requires a separately owned output root");
+  const sdkPath = path.join(root, "packages/plugins/sdk");
+  await mkdir(sdkPath, { recursive: true });
+  const files: Array<{ path: string; sha256: string; byteSize: number }> = [];
+  let totalBytes = 0;
+  async function copy(relative: string): Promise<void> {
+    const sourcePath = path.join(sourceRoot, relative), targetPath = path.join(sdkPath, relative);
+    const stat = await lstat(sourcePath);
+    if (stat.isSymbolicLink()) throw new Error("SDK mirror rejects symbolic links");
+    if (stat.isDirectory()) {
+      await mkdir(targetPath, { recursive: true });
+      for (const name of (await readdir(sourcePath)).sort()) await copy(path.join(relative, name));
+      return;
+    }
+    if (!stat.isFile() || files.length >= 512 || stat.size > 4_194_304 || totalBytes + stat.size > 33_554_432)
+      throw new Error("SDK mirror exceeds its bounded public package/dist inventory");
+    const bytes = await readFile(sourcePath);
+    await writeFile(targetPath, bytes, { flag: "wx", mode: 0o600 });
+    const hash = (body: Buffer) => createHash("sha256").update(body).digest("hex");
+    const sha256 = hash(bytes);
+    if (hash(await readFile(targetPath)) !== sha256) throw new Error("SDK mirror byte custody mismatch");
+    totalBytes += bytes.length;
+    files.push({ path: relative, sha256, byteSize: bytes.length });
+  }
+  await copy("package.json");
+  const manifest = JSON.parse(await readFile(path.join(sdkPath, "package.json"), "utf8"));
+  if (manifest.name !== "@paperclipai/plugin-sdk" || manifest.exports?.["."]?.import !== "./dist/index.js")
+    throw new Error("SDK mirror is not the actual public plugin SDK package");
+  await copy("dist");
+  if (!files.some(file => file.path === path.join("dist", "index.js"))) throw new Error("SDK mirror has no real compiled entrypoint");
+  return { sdkPath, files, archiveWrites: false as const, packageBuilds: 0 as const, packagePacks: 0 as const };
+}

tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts

@@ -321,6 +321,9 @@ export const secretDeliveryExternalCases = [
         "../../server/src/services/heartbeat/run-preparation.ts",
         "../../cli/dist/index.js",
         "../../cli/src/index.ts",
+        "../../cli/node_modules/tsx/dist/loader.mjs",
+        "chat-catalog-domains/external-plugin-sdk-mirror.ts",
+        "../../packages/plugins/sdk/package.json",
         "../../packages/plugins/create-paperclip-plugin/src/index.ts",
         "../../packages/plugins/sdk/src/index.ts",
         "../../packages/plugins/sdk/dist/index.js",

tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts

@@ -77,7 +77,7 @@ export const runtimeFailedRunCases = [runtimeDefinition("runs-adapters.inspect-f
   await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
   return { worker, task: done.task, ...failure, successfulRun, successArtifactVerified: true, failureLogAvailable };
 }, gradeFailedAttempt, {
-  sourceFiles: ["chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml"],
+  sourceFiles: ["chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/TaskChatThread.tsx", "../../ui/src/components/task-chat/TaskChatMarker.tsx", "../../ui/src/components/task-chat/TaskChatThreadView.tsx", "../../ui/src/pages/AgentDetail.tsx"],
   expectedRunCount: 3, attemptTimeoutMs: 720_000,
   runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 2 } },
   async observe(ctx, state) {

tests/runner-e2e/chat-catalog-domains/runtime-session.ts

@@ -41,7 +41,7 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi
   for (const doc of documents) revisions.push(...await ctx.input.api.get<Entity[]>(`/api/issues/${state.task.id}/documents/${encodeURIComponent(doc.key!)}/revisions`));
   return { ...state, priorComments: comments, priorDocuments: documents, priorRevisions: revisions };
 }, gradeSessionReset, {
-  sourceFiles: ["chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml"],
+  sourceFiles: ["chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/AgentActionButtons.tsx", "../../ui/src/pages/AgentDetail.tsx"],
   expectedRunCount: 3, attemptTimeoutMs: 720_000,
   async act(ctx, state) {
     await pollUntil({ label: "Canonical session guidance settled", deadlineAt: Date.now() + 150_000,