Use editable skill copies in planning comparisons

Calibrate creation and selection through real public APIs before providers. Preserve the cancelled zero-provider setup campaign.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/plan-task-fixture.test.ts

@@ -0,0 +1,79 @@
+import { randomUUID } from "node:crypto";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import express from "express";
+import request from "supertest";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
+import { companySkillRoutes } from "../routes/company-skills.js";
+import { agentRoutes } from "../routes/agents.js";
+import { errorHandler } from "../middleware/error-handler.js";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { preparePlanSkills, selectPlanSkills, verifyPlanSkills, verifyPlanSelection } from "../../../tests/runner-e2e/plan-task-skills.js";
+import { PLAN_SKILLS, PLAN_VARIANTS } from "../../../tests/runner-e2e/plan-task-cases.js";
+import type { RunnerApi } from "../../../tests/runner-e2e/api.js";
+
+const support = await getEmbeddedPostgresTestSupport();
+if (!support.supported) console.warn(`Planning fixture API calibration skipped: ${support.reason}`);
+(support.supported ? describe : describe.skip)("planning fixture public API calibration (no providers)", () => {
+  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let db: ReturnType<typeof createDb>;
+  let root: string;
+  const savedHome = process.env.PAPERCLIP_HOME, savedInstance = process.env.PAPERCLIP_INSTANCE_ID;
+  let api: Pick<RunnerApi, "get" | "post">;
+  let app: express.Express;
+  beforeAll(async () => {
+    tempDb = await startEmbeddedPostgresTestDatabase("paperclip-plan-fixture-");
+    root = await mkdtemp(path.join(os.tmpdir(), "paperclip-plan-fixture-home-"));
+    process.env.PAPERCLIP_HOME = root;
+    process.env.PAPERCLIP_INSTANCE_ID = "default";
+    db = createDb(tempDb.connectionString);
+    app = express();
+    app.use(express.json());
+    app.use((req, _res, next) => { (req as any).actor = { type: "board", source: "local_implicit", isInstanceAdmin: true, userId: "local-board" }; next(); });
+    app.use("/api", companySkillRoutes(db));
+    app.use("/api", agentRoutes(db));
+    app.use(errorHandler);
+    api = {
+      async get<T>(url: string): Promise<T> {
+        const response = await request(app).get(url);
+        if (response.status !== 200) throw new Error(`GET ${url}: ${response.status} ${JSON.stringify(response.body)}`);
+        return response.body;
+      },
+      async post<T>(url: string, body?: unknown): Promise<T> {
+        const response = await request(app).post(url).send(body as object);
+        if (response.status < 200 || response.status >= 300) throw new Error(`POST ${url}: ${response.status} ${JSON.stringify(response.body)}`);
+        return response.body;
+      },
+    };
+  });
+  afterAll(async () => {
+    if (savedHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = savedHome;
+    if (savedInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = savedInstance;
+    await tempDb?.cleanup();
+    if (root) await rm(root, { recursive: true, force: true });
+  });
+  it.each(PLAN_VARIANTS)("admits %s through real creation, readback and selection APIs", async variant => {
+    const companyId = randomUUID(), agentId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "Planning calibration", issuePrefix: `P${companyId.slice(0, 6)}` });
+    await db.insert(agents).values({ id: agentId, companyId, name: "Fixture Lead", role: "general", adapterType: "paperclip_runner", adapterConfig: {} });
+    const before = await api.get<any[]>(`/api/companies/${companyId}/skills`);
+    const bundled = before.find(s => s.key === PLAN_SKILLS[0].key);
+    expect(bundled).toBeDefined();
+    const sources = await preparePlanSkills(api, companyId, variant);
+    expect(sources).toHaveLength(2);
+    expect(sources.every(s => s.key.startsWith(`company/${companyId}/eval-`))).toBe(true);
+    await selectPlanSkills(api, companyId, agentId, sources);
+    await verifyPlanSelection(api, companyId, agentId, sources);
+    await verifyPlanSkills(api, companyId, sources);
+    const after = await api.get<any[]>(`/api/companies/${companyId}/skills`);
+    expect(after.find(s => s.key === bundled.key)?.id).toBe(bundled.id);
+    expect(after.find(s => s.key === bundled.key)?.markdown).toBe(bundled.markdown);
+    expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
+    // Fail closed if a later actor changes either selection or source bytes.
+    const swapped = sources.map(s => ({ ...s, selected: !s.selected }));
+    await expect(verifyPlanSelection(api, companyId, agentId, swapped)).rejects.toThrow("selection differs");
+    await expect(verifyPlanSkills(api, companyId, [{ ...sources[0]!, sha256: "wrong" }])).rejects.toThrow("guidance changed");
+  }, 30_000);
+});