diff --git a/doc/plans/2026-10-05-plan-task-guidance.md b/doc/plans/2026-10-05-plan-task-guidance.md index 2140c80daf6..67ee52afd58 100644 --- a/doc/plans/2026-10-05-plan-task-guidance.md +++ b/doc/plans/2026-10-05-plan-task-guidance.md @@ -16,9 +16,9 @@ its separate checklist remains untouched. Four ordinary execution scenarios: cohesive work; independent specialist outputs; a real prerequisite between specialists; an independent adverse review. -Compare current, short, and disabled planning skills using public skill editing +Compare current, short, and disabled planning skills using public company-copy creation and selection APIs in isolated companies. Archived current skill bytes come from -the base above. The disabled variant is an availability ablation, not a production +the base above. The disabled variant is an unassigned-selection ablation (the library remains discoverable), not a production removal or an accepted-plan continuation test. Other instructions, tools, models, permissions, fixtures, graders, budgets, and deadlines stay matched. @@ -49,3 +49,19 @@ and continuation qualification before a deletion recommendation can be shipped. - Archived source hashes: conversion `08cb036df0e05b1d704dc0cd547c4e37b73078597c072a85ff982a2bb9b3a370`; planning `9c52a44a30ec8d306119da51bf298e9e3e6c382a9a9559ffd3054bf5e0c75f36`. Both match the named base Git blobs exactly. - Generated capability inventory and catalog are synchronized. - No provider call has started. Live outcomes remain unqualified. + +Source review found two fixture defects before provider execution: bundled and +catalog skills cannot be edited, and deleted core skills are automatically +restored. Campaign `37399550253` at `843238f43cb266f6ca0bc9d255881558ffd2eb71` +was cancelled; its paid-cell step was skipped, with zero provider runs and no +behavioral grade. Corrected setup creates editable, byte-identical company copies +and varies their explicit selection. No bundled deletion or in-place edit occurs. + +Corrected fixture admission: all three current/short/unassigned variants pass real +company-skill creation, content readback and native-agent selection APIs against +a disposable PostgreSQL database. No heartbeat run rows were created. The +evaluator's 35 positive/wrong/missing-evidence checks pass, including rejection +of a bare issue-ID URL without the issues route. Product E2E typecheck passes. +Initial local DB checks were skipped until the dependency's missing library +symlinks were restored with its supplied postinstall script; skipped checks were +not counted as passes. diff --git a/server/src/__tests__/plan-task-fixture.test.ts b/server/src/__tests__/plan-task-fixture.test.ts new file mode 100644 index 00000000000..bce94808b47 --- /dev/null +++ b/server/src/__tests__/plan-task-fixture.test.ts @@ -0,0 +1,79 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import express from "express"; +import request from "supertest"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { companySkillRoutes } from "../routes/company-skills.js"; +import { agentRoutes } from "../routes/agents.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { preparePlanSkills, selectPlanSkills, verifyPlanSkills, verifyPlanSelection } from "../../../tests/runner-e2e/plan-task-skills.js"; +import { PLAN_SKILLS, PLAN_VARIANTS } from "../../../tests/runner-e2e/plan-task-cases.js"; +import type { RunnerApi } from "../../../tests/runner-e2e/api.js"; + +const support = await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn(`Planning fixture API calibration skipped: ${support.reason}`); +(support.supported ? describe : describe.skip)("planning fixture public API calibration (no providers)", () => { + let tempDb: Awaited>; + let db: ReturnType; + let root: string; + const savedHome = process.env.PAPERCLIP_HOME, savedInstance = process.env.PAPERCLIP_INSTANCE_ID; + let api: Pick; + let app: express.Express; + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-plan-fixture-"); + root = await mkdtemp(path.join(os.tmpdir(), "paperclip-plan-fixture-home-")); + process.env.PAPERCLIP_HOME = root; + process.env.PAPERCLIP_INSTANCE_ID = "default"; + db = createDb(tempDb.connectionString); + app = express(); + app.use(express.json()); + app.use((req, _res, next) => { (req as any).actor = { type: "board", source: "local_implicit", isInstanceAdmin: true, userId: "local-board" }; next(); }); + app.use("/api", companySkillRoutes(db)); + app.use("/api", agentRoutes(db)); + app.use(errorHandler); + api = { + async get(url: string): Promise { + const response = await request(app).get(url); + if (response.status !== 200) throw new Error(`GET ${url}: ${response.status} ${JSON.stringify(response.body)}`); + return response.body; + }, + async post(url: string, body?: unknown): Promise { + const response = await request(app).post(url).send(body as object); + if (response.status < 200 || response.status >= 300) throw new Error(`POST ${url}: ${response.status} ${JSON.stringify(response.body)}`); + return response.body; + }, + }; + }); + afterAll(async () => { + if (savedHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = savedHome; + if (savedInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = savedInstance; + await tempDb?.cleanup(); + if (root) await rm(root, { recursive: true, force: true }); + }); + it.each(PLAN_VARIANTS)("admits %s through real creation, readback and selection APIs", async variant => { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Planning calibration", issuePrefix: `P${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Fixture Lead", role: "general", adapterType: "paperclip_runner", adapterConfig: {} }); + const before = await api.get(`/api/companies/${companyId}/skills`); + const bundled = before.find(s => s.key === PLAN_SKILLS[0].key); + expect(bundled).toBeDefined(); + const sources = await preparePlanSkills(api, companyId, variant); + expect(sources).toHaveLength(2); + expect(sources.every(s => s.key.startsWith(`company/${companyId}/eval-`))).toBe(true); + await selectPlanSkills(api, companyId, agentId, sources); + await verifyPlanSelection(api, companyId, agentId, sources); + await verifyPlanSkills(api, companyId, sources); + const after = await api.get(`/api/companies/${companyId}/skills`); + expect(after.find(s => s.key === bundled.key)?.id).toBe(bundled.id); + expect(after.find(s => s.key === bundled.key)?.markdown).toBe(bundled.markdown); + expect(await db.select().from(heartbeatRuns)).toHaveLength(0); + // Fail closed if a later actor changes either selection or source bytes. + const swapped = sources.map(s => ({ ...s, selected: !s.selected })); + await expect(verifyPlanSelection(api, companyId, agentId, swapped)).rejects.toThrow("selection differs"); + await expect(verifyPlanSkills(api, companyId, [{ ...sources[0]!, sha256: "wrong" }])).rejects.toThrow("guidance changed"); + }, 30_000); +}); diff --git a/tests/runner-e2e/PLAN-TASK-GUIDANCE.md b/tests/runner-e2e/PLAN-TASK-GUIDANCE.md index e8aefcd7ee4..0f785eece54 100644 --- a/tests/runner-e2e/PLAN-TASK-GUIDANCE.md +++ b/tests/runner-e2e/PLAN-TASK-GUIDANCE.md @@ -7,10 +7,14 @@ saved upstream result; and an independent adverse audit. Each scenario compares current (archived from master `72ff3a9f27e581a27acb49771e8658bbb0bbaa47`), short (checkout production files), -and disabled planning skills. The fixture uses the normal company skill editing, -selection and removal APIs before any task starts. Both runtime plan conversion +and disabled planning skills. The fixture creates editable company-owned copies through the public API, +preserving each source file byte-for-byte, then explicitly selects those copies +before any task starts. Bundled/catalog skills remain read-only. Both runtime plan conversion and catalog task planning change together. All agents get the same treatment. -Disabled removes both from this disposable company, not from the product catalog. +Disabled leaves those copies unassigned, with an empty explicit skill selection. +The company library and automatically restored core inventory remain available; +this is a selection ablation, not proof that a determined agent cannot discover +unassigned guidance. The eval does not exercise automatic accepted-plan selection or installed-skill migration, and cannot by itself qualify deleting that wiring. @@ -44,3 +48,9 @@ skill invocation or cognitive consumption. Word/UTF-8 byte counts are exact file measurements; provider context, token usage and billing coverage come from actual retained run records. Cleanup, credential scanning and public projection use the existing Product E2E pipeline. Do not publish raw sessions or hidden reasoning. + +Setup attempt [37399550253](https://github.com/paperclipai/paperclip/actions/runs/37399550253) +on source `843238f43cb266f6ca0bc9d255881558ffd2eb71` was cancelled after source +review identified unsupported bundled edits and automatic core reinstallation. +The paid-cell step was skipped: zero provider runs and no behavioral grade. +The corrected fixture uses company copies and selection only. diff --git a/tests/runner-e2e/plan-task-cases.ts b/tests/runner-e2e/plan-task-cases.ts index 3fdb787a089..f2312feef79 100644 --- a/tests/runner-e2e/plan-task-cases.ts +++ b/tests/runner-e2e/plan-task-cases.ts @@ -15,7 +15,7 @@ export const PLAN_SKILLS = [ ] as const; export const planHash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); export const planDefinitionDigest = planHash([ - "plan-task-cases.ts", "plan-task-flow.ts", "plan-task-scoring.ts", + "plan-task-cases.ts", "plan-task-flow.ts", "plan-task-scoring.ts", "plan-task-skills.ts", "fixtures/plan-task-guidance/current-conversion.md", "fixtures/plan-task-guidance/current-planning.md", ].map(file => readFileSync(new URL(file, import.meta.url), "utf8")).join("\0")); diff --git a/tests/runner-e2e/plan-task-flow.ts b/tests/runner-e2e/plan-task-flow.ts index e22eb2afa4a..adb122763d1 100644 --- a/tests/runner-e2e/plan-task-flow.ts +++ b/tests/runner-e2e/plan-task-flow.ts @@ -1,5 +1,5 @@ import { expect, type Page } from "@playwright/test"; -import { mkdir, readFile } from "node:fs/promises"; +import { mkdir } from "node:fs/promises"; import path from "node:path"; import { pollUntil, type RunnerApi } from "./api.js"; import { collectChatRunEvidence } from "./chat-flow.js"; @@ -7,8 +7,9 @@ import { FixtureRegistry } from "./fixture-registry.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; import { createTaskThroughUi } from "./user-actions.js"; -import { PLAN_BASE_SHA, PLAN_BUDGET_CENTS, PLAN_MAX_RUNS, PLAN_SKILLS, parsePlanCase, planDefinitionDigest, planHash, planScenario } from "./plan-task-cases.js"; +import { PLAN_BASE_SHA, PLAN_BUDGET_CENTS, PLAN_MAX_RUNS, parsePlanCase, planDefinitionDigest, planScenario } from "./plan-task-cases.js"; import { gradePlanTask, type PlanCheck, type PlanDocument, type PlanObservation, type PlanRow } from "./plan-task-scoring.js"; +import { preparePlanSkills, selectPlanSkills, verifyPlanSelection, verifyPlanSkills } from "./plan-task-skills.js"; type Input = { page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; @@ -55,24 +56,7 @@ export async function runPlanTaskFlow(input: Input) { // Public fixture APIs and normal skill versioning only. No provider begins // until the served bytes and all three agents' selections are verified. await api.patch(`${company}/budgets`, { budgetMonthlyCents: PLAN_BUDGET_CENTS }); - const catalog = await api.get("/api/skills/catalog?kind=bundled&q=task-planning"); - const planning = catalog.find(s => s.key === PLAN_SKILLS[1].key); - if (!planning) throw new Error("Bundled task-planning catalog entry is missing"); - await api.post(`${company}/skills/install-catalog`, { catalogSkillId: planning.id }); - const library = await api.get(`${company}/skills`); - for (const skill of PLAN_SKILLS) { - const installed = library.find(s => s.key === skill.key); - if (!installed) throw new Error(`Missing production planning skill ${skill.key}`); - const relative = variant === "current" ? skill.current : skill.short; - const content = await readFile(new URL(`../../${relative}`, import.meta.url), "utf8"); - if (variant !== "disabled") { - await api.patch(`${company}/skills/${installed.id}/files`, { path: "SKILL.md", content }); - const served = await api.get<{ content: string }>(`${company}/skills/${installed.id}/files?path=SKILL.md`); - if (planHash(served.content) !== planHash(content)) throw new Error(`Served skill differs from selected source ${relative}`); - } - source.skills.push({ key: skill.key, id: installed.id, sourcePath: relative, sha256: planHash(content), - bytes: Buffer.byteLength(content), words: content.split(/\s+/).filter(Boolean).length, selected: variant !== "disabled" }); - } + source.skills = await preparePlanSkills(api, f.company.id, variant); const registry = new FixtureRegistry(); for (const [id, name, role, capabilities] of [ ["alex", "Alex Metrics", "engineer", "Owns arithmetic verification, operational metrics, and signed order summaries."], @@ -90,20 +74,12 @@ export async function runPlanTaskFlow(input: Input) { const team = await registry.setupAll(); alexId = (team.values.get("alex") as PlanRow).id; rileyId = (team.values.get("riley") as PlanRow).id; - const selected = variant === "disabled" ? [] : PLAN_SKILLS.map(s => s.key); for (const id of [f.agent.id, alexId, rileyId]) { await api.patch(`/api/agents/${id}/permissions`, { canCreateAgents: false, canAssignTasks: id === f.agent.id }); await api.patch(`/api/agents/${id}/budgets`, { budgetMonthlyCents: PLAN_BUDGET_CENTS }); - await api.post(`/api/agents/${id}/skills/sync?companyId=${f.company.id}`, { desiredSkills: selected, mode: "replace" }); - const skills = await api.get(`/api/agents/${id}/skills?companyId=${f.company.id}`); - if (JSON.stringify([...(skills.desiredSkills ?? [])].sort()) !== JSON.stringify([...selected].sort())) throw new Error("Agent planning skill selection differs from selected variant"); + const skills = await selectPlanSkills(api, f.company.id, id, source.skills); source.agents.push({ id, skills }); } - if (variant === "disabled") { - for (const skill of source.skills) await api.delete(`${company}/skills/${skill.id}`); - const after = await api.get(`${company}/skills`); - if (after.some(s => PLAN_SKILLS.some(wanted => wanted.key === s.key))) throw new Error("Disabled planning skills remain in company library"); - } await input.evidence("plan-task-source.json", source); await api.patch("/api/instance/settings/experimental", { enableClassicTaskInterface: false }); await createTaskThroughUi({ page, issuePrefix: f.company.issuePrefix!, agentName: f.agent.name, @@ -121,21 +97,8 @@ export async function runPlanTaskFlow(input: Input) { return same >= 2; }, reject: o => o.runs.length > PLAN_MAX_RUNS ? "bounded run count exceeded" : o.runs.some(r => ["failed", "timed_out", "cancelled"].includes(r.status)) ? "actual run failed; retain original failure" : undefined }); - const finalLibrary = await api.get(`${company}/skills`); - for (const skill of source.skills) { - const found = finalLibrary.find(s => s.key === skill.key); - if (variant === "disabled") { - if (found) throw new Error("Disabled planning guidance was reinstalled; comparison is uncomparable"); - } else { - if (!found) throw new Error("Selected planning guidance disappeared; comparison is uncomparable"); - const served = await api.get<{ content: string }>(`${company}/skills/${found.id}/files?path=SKILL.md`); - if (planHash(served.content) !== skill.sha256) throw new Error("Planning guidance changed during execution; comparison is uncomparable"); - } - } - for (const id of [f.agent.id, alexId, rileyId]) { - const skills = await api.get(`/api/agents/${id}/skills?companyId=${f.company.id}`); - if (JSON.stringify([...(skills.desiredSkills ?? [])].sort()) !== JSON.stringify([...selected].sort())) throw new Error("Planning skill selection changed during execution; comparison is uncomparable"); - } + await verifyPlanSkills(api, f.company.id, source.skills); + for (const id of [f.agent.id, alexId, rileyId]) await verifyPlanSelection(api, f.company.id, id, source.skills); checks = grade().checks; input.observe(parent, observed.runs, checks); await page.goto(`/${f.company.issuePrefix}/issues/${parent.identifier ?? parent.id}`, { waitUntil: "domcontentloaded" }); diff --git a/tests/runner-e2e/plan-task-scoring.test.ts b/tests/runner-e2e/plan-task-scoring.test.ts index a1ee9ee1ce9..e209fb7131f 100644 --- a/tests/runner-e2e/plan-task-scoring.test.ts +++ b/tests/runner-e2e/plan-task-scoring.test.ts @@ -99,7 +99,7 @@ describe("planning guidance outcome calibration", () => { const recovery = valid("cohesive"); recovery.observation.issues[0]!.scheduledRetry = { reason: "retry" }; expect(passed(recovery, "tasks-completed")).toBe(false); }); - it.each(["/T/issues/other#document-result", "https://unrelated.example/T/issues/T-parent#document-result", "/T/issues/T-parent#document-other"])("rejects unrelated result link %s", url => { + it.each(["/T-parent#document-result", "/T/issues/other#document-result", "https://unrelated.example/T/issues/T-parent#document-result", "/T/issues/T-parent#document-other"])("rejects unrelated result link %s", url => { const input = valid("cohesive"); input.observation.comments[0]!.body = `[Result](${url})`; expect(passed(input, "visible-result-link")).toBe(false); }); diff --git a/tests/runner-e2e/plan-task-scoring.ts b/tests/runner-e2e/plan-task-scoring.ts index 7d038dd11f5..dba36fb252c 100644 --- a/tests/runner-e2e/plan-task-scoring.ts +++ b/tests/runner-e2e/plan-task-scoring.ts @@ -89,7 +89,9 @@ export function gradePlanTask(input: { try { const url = new URL(match[1]!, input.origin); const parts = url.pathname.split("/").filter(Boolean); - const id = parts[parts.indexOf("issues") + 1]; + const issueIndex = parts.indexOf("issues"); + if (issueIndex < 0) return false; + const id = parts[issueIndex + 1]; return url.origin === new URL(input.origin).origin && url.hash === "#document-result" && [parent?.id, parent?.identifier].includes(id); } catch { return false; } })), diff --git a/tests/runner-e2e/plan-task-skills.ts b/tests/runner-e2e/plan-task-skills.ts new file mode 100644 index 00000000000..2f33bf6f349 --- /dev/null +++ b/tests/runner-e2e/plan-task-skills.ts @@ -0,0 +1,54 @@ +import { readFile } from "node:fs/promises"; +import type { RunnerApi } from "./api.js"; +import { PLAN_SKILLS, planHash, type PlanVariant } from "./plan-task-cases.js"; + +type SkillApi = Pick; +export type PlanSkillSource = { + originalKey: string; key: string; id: string; sourcePath: string; + sha256: string; bytes: number; words: number; selected: boolean; +}; + +/** Bundled/catalog files are read-only and core inventory is automatically + * restored. Editable company copies preserve the source bytes and names; + * selection, rather than deletion, implements the unassigned control. */ +export async function preparePlanSkills(api: SkillApi, companyId: string, variant: PlanVariant) { + const company = `/api/companies/${companyId}`; + const skills: PlanSkillSource[] = []; + for (const skill of PLAN_SKILLS) { + const relative = variant === "current" ? skill.current : skill.short; + const content = await readFile(new URL(`../../${relative}`, import.meta.url), "utf8"); + const name = skill.key.split("/").at(-1)!; + const created = await api.post<{ id: string; key: string; sourceType: string }>(`${company}/skills`, { + name, slug: `eval-${name}`, markdown: content, sharingScope: "company", + }); + if (created.sourceType !== "local_path") throw new Error("Planning fixture did not create an editable company skill"); + const served = await api.get<{ content: string; editable: boolean }>(`${company}/skills/${created.id}/files?path=SKILL.md`); + if (!served.editable || planHash(served.content) !== planHash(content)) throw new Error(`Served skill differs from editable source ${relative}`); + skills.push({ originalKey: skill.key, key: created.key, id: created.id, sourcePath: relative, + sha256: planHash(content), bytes: Buffer.byteLength(content), words: content.split(/\s+/).filter(Boolean).length, + selected: variant !== "disabled" }); + } + return skills; +} + +export async function selectPlanSkills(api: SkillApi, companyId: string, agentId: string, sources: PlanSkillSource[]) { + const selected = sources.filter(s => s.selected).map(s => s.key); + await api.post(`/api/agents/${agentId}/skills/sync?companyId=${companyId}`, { desiredSkills: selected, mode: "replace" }); + return verifyPlanSelection(api, companyId, agentId, sources); +} + +export async function verifyPlanSelection(api: Pick, companyId: string, agentId: string, sources: PlanSkillSource[]) { + const selected = sources.filter(s => s.selected).map(s => s.key); + const snapshot = await api.get<{ desiredSkills: string[] }>(`/api/agents/${agentId}/skills?companyId=${companyId}`); + if (JSON.stringify([...(snapshot.desiredSkills ?? [])].sort()) !== JSON.stringify([...selected].sort())) { + throw new Error("Planning skill selection differs from selected variant; comparison is uncomparable"); + } + return snapshot; +} + +export async function verifyPlanSkills(api: Pick, companyId: string, sources: PlanSkillSource[]) { + for (const skill of sources) { + const served = await api.get<{ content: string }>(`/api/companies/${companyId}/skills/${skill.id}/files?path=SKILL.md`); + if (planHash(served.content) !== skill.sha256) throw new Error("Planning guidance changed during execution; comparison is uncomparable"); + } +}