Require native planning skill invocation evidence in comparisons

Bind explicit assigned skill inputs to immutable content digests and provider acceptance. Preserve separate outcome and exposure gates, and calibrate the supported native path without providers.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/plan-task-fixture.test.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import express from "express";
@@ -11,7 +11,11 @@ import { agentRoutes } from "../routes/agents.js";
 import { errorHandler } from "../middleware/error-handler.js";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 import { preparePlanSkills, selectPlanSkills, verifyPlanSkills, verifyPlanSelection } from "../../../tests/runner-e2e/plan-task-skills.js";
-import { PLAN_SKILLS, PLAN_VARIANTS } from "../../../tests/runner-e2e/plan-task-cases.js";
+import { planInvocations, planInvocationPrompt } from "../../../tests/runner-e2e/plan-task-exposure.js";
+import { nativeTaskSkillInputs } from "../../../packages/paperclip-runner/src/backends/runtime-context.js";
+import { resolveRunnerdCodexSkillInputs } from "../../../packages/paperclip-runner/src/live/runnerd-codex-transport.js";
+import type { NativeRuntimeContextSnapshot } from "../../../packages/paperclip-runner/src/contracts/runtime-context.js";
+import { PLAN_SKILLS, PLAN_VARIANTS, planHash } from "../../../tests/runner-e2e/plan-task-cases.js";
 import type { RunnerApi } from "../../../tests/runner-e2e/api.js";
 
 const support = await getEmbeddedPostgresTestSupport();
@@ -64,7 +68,28 @@ if (!support.supported) console.warn(`Planning fixture API calibration skipped:
     const sources = await preparePlanSkills(api, companyId, variant);
     expect(sources).toHaveLength(2);
     expect(sources.every(s => s.key.startsWith(`company/${companyId}/eval-`))).toBe(true);
-    await selectPlanSkills(api, companyId, agentId, sources);
+    const selected = await selectPlanSkills(api, companyId, agentId, sources);
+    const invocations = planInvocations(sources, selected);
+    const contextSkills = [];
+    for (const source of sources.filter(s => s.selected)) {
+      const entry = (selected as any).entries.find((e: any) => e.key === source.key && e.desired);
+      const bytes = await readFile(path.join(entry.sourcePath, "SKILL.md"));
+      expect(planHash(bytes)).toBe(source.sha256);
+      const mode = (await stat(path.join(entry.sourcePath, "SKILL.md"))).mode & 0o555;
+      expect(mode).toBe(0o444);
+      const digest = planHash(JSON.stringify([{ path: "SKILL.md", sha256: planHash(bytes), mode, size: bytes.length }]));
+      expect(invocations.find(i => i.name === entry.runtimeName)?.assetDigest).toBe(digest);
+      contextSkills.push({ key: source.key, runtimeName: entry.runtimeName, versionId: entry.currentVersionId,
+        bundle: { rootPath: path.join(root, "runtime-context-assets/bundles", digest) } });
+    }
+    const context = { skills: contextSkills } as NativeRuntimeContextSnapshot;
+    const inputs = nativeTaskSkillInputs(planInvocationPrompt("Business task", invocations), context);
+    expect(inputs).toHaveLength(variant === "disabled" ? 0 : 2);
+    expect(inputs.map(i => i.name)).toEqual(invocations.map(i => i.name));
+    const providerHome = path.join(root, "isolated-codex-home");
+    expect(resolveRunnerdCodexSkillInputs(inputs, context, providerHome)).toEqual(inputs.map(i => ({
+      ...i, path: path.join(providerHome, "skills", i.name, "SKILL.md"),
+    })));
     await verifyPlanSelection(api, companyId, agentId, sources);
     await verifyPlanSkills(api, companyId, sources);
     const after = await api.get<any[]>(`/api/companies/${companyId}/skills`);

tests/runner-e2e/PLAN-TASK-GUIDANCE.md

@@ -18,6 +18,18 @@ unassigned guidance.
 The eval does not exercise automatic accepted-plan selection or installed-skill
 migration, and cannot by itself qualify deleting that wiring.
 
+The business prompts do not prescribe task counts or particular tools. Enabled
+variants append explicit references to the two assigned runtime skill names; the
+unassigned control appends none. This measures explicitly invoked guidance, not
+automatic discovery. A separate exposure gate requires the initial lead run to
+submit exactly those native skill inputs, bound to the immutable SKILL.md file
+digests, and records provider turn acceptance. The controller emits the submission
+receipt before a provider turn ID exists: this is a run-bound submission contract,
+not a claimed provider call-ID join or proof of cognitive consumption. Native
+protocol calibration separately verifies mapping into the isolated provider skill
+directory. Missing or wrong exposure fails qualification even when the output
+checks pass.
+
 The scenario prompts do not prescribe task counts or particular tools. They do
 specify which specialist is accountable and what business output is required.
 The grader checks independent arithmetic, actual latest document revisions and

tests/runner-e2e/plan-task-cases.ts

@@ -15,7 +15,7 @@ export const PLAN_SKILLS = [
 ] as const;
 export const planHash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex");
 export const planDefinitionDigest = planHash([
-  "plan-task-cases.ts", "plan-task-flow.ts", "plan-task-scoring.ts", "plan-task-skills.ts", "plan-task-ui.ts",
+  "plan-task-cases.ts", "plan-task-flow.ts", "plan-task-scoring.ts", "plan-task-skills.ts", "plan-task-ui.ts", "plan-task-exposure.ts",
   "fixtures/plan-task-guidance/current-conversion.md", "fixtures/plan-task-guidance/current-planning.md",
 ].map(file => readFileSync(new URL(file, import.meta.url), "utf8")).join("\0"));
 

tests/runner-e2e/plan-task-exposure.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import { planExposure, planInvocationPrompt } from "./plan-task-exposure.js";
+
+const digest = "a".repeat(64);
+function fixture(enabled = true) {
+  const skillInputs = enabled ? [{ type: "skill", name: "planning", path: `/private/runtime-context-assets/bundles/${digest}/SKILL.md` }] : [];
+  return { runs: [{ id: "lead-run", agentId: "lead", nativeIssueId: "parent", startedAt: "2026-10-05T00:00:00Z" }],
+    evidence: [{ runId: "lead-run", events: [
+      { runId: "lead-run", eventType: "turn.submitted", payload: { prpEvent: { payload: { skillInputs } } } },
+      { runId: "lead-run", eventType: "turn.accepted", payload: { prpEvent: { turnId: "turn", payload: { turnId: "turn" } } } },
+      { runId: "lead-run", eventType: "turn.started", payload: { prpEvent: { turnId: "turn", payload: {} } } },
+    ] }], leadId: "lead", parentId: "parent", invocations: enabled ? [{ name: "planning", assetDigest: digest }] : [] };
+}
+
+describe("planning native invocation evidence", () => {
+  it.each([true, false])("accepts exact enabled=%s delivery and accepted-turn receipts", enabled => expect(planExposure(fixture(enabled)).passed).toBe(true));
+  it.each(["missing", "wrong-run", "wrong-turn", "wrong-path", "wrong-digest", "wrong-name", "duplicate", "prose-only", "unaccepted"])("rejects %s delivery evidence", problem => {
+    const value = fixture();
+    const event = value.evidence[0]!.events[0]!;
+    const skills = event.payload.prpEvent.payload.skillInputs!;
+    if (problem === "missing") value.evidence = [];
+    if (problem === "wrong-run") event.runId = "other-run";
+    if (problem === "wrong-turn") value.evidence[0]!.events[1]!.payload.prpEvent.turnId = "other-turn";
+    if (problem === "wrong-path") skills[0]!.path = `/unverified/${digest}/SKILL.md`;
+    if (problem === "wrong-digest") skills[0]!.path = skills[0]!.path.replace(digest, "b".repeat(64));
+    if (problem === "wrong-name") skills[0]!.name = "other";
+    if (problem === "duplicate") skills.push({ ...skills[0]! });
+    if (problem === "prose-only") event.eventType = "agentMessage";
+    if (problem === "unaccepted") value.evidence[0]!.events.splice(1, 1);
+    expect(planExposure(value).passed).toBe(false);
+  });
+  it("rejects a disabled control with an actual skill input", () => {
+    const value = fixture(); value.invocations = [];
+    expect(planExposure(value).passed).toBe(false);
+  });
+  it("keeps the business prompt intact and invokes only assigned guidance", () => {
+    expect(planInvocationPrompt("Business task", [])).toBe("Business task");
+    expect(planInvocationPrompt("Business task", [{ name: "planning", assetDigest: digest }]))
+      .toBe("Business task\n\nUse the assigned planning guidance: /planning.");
+  });
+});

tests/runner-e2e/plan-task-exposure.ts

@@ -0,0 +1,46 @@
+import path from "node:path";
+import { createHash } from "node:crypto";
+import type { PlanRow } from "./plan-task-scoring.js";
+import type { PlanSkillSource } from "./plan-task-skills.js";
+
+export type PlanInvocation = { name: string; assetDigest: string };
+
+/** Native context uses one immutable SKILL.md file for each company copy. */
+export function planInvocations(sources: PlanSkillSource[], snapshot: PlanRow): PlanInvocation[] {
+  return sources.filter(s => s.selected).map(source => {
+    const entries = snapshot.entries?.filter((e: PlanRow) => e.key === source.key && e.desired);
+    const entry = entries?.length === 1 ? entries[0] : undefined;
+    if (!entry || !/^[a-zA-Z0-9_-]+$/.test(entry.runtimeName)) throw new Error("Missing unique assigned planning runtime name");
+    const manifest = [{ path: "SKILL.md", sha256: source.sha256, mode: 0o444, size: source.bytes }];
+    return { name: entry.runtimeName, assetDigest: createHash("sha256").update(JSON.stringify(manifest)).digest("hex") };
+  });
+}
+
+export function planInvocationPrompt(prompt: string, invocations: PlanInvocation[]) {
+  return invocations.length ? `${prompt}\n\nUse the assigned planning guidance: ${invocations.map(s => `/${s.name}`).join(" and ")}.` : prompt;
+}
+
+/** Exact initial lead run identity; provider turn acceptance joins its actual turn ID.
+ * turn.submitted is emitted before the provider assigns that ID, so its skill
+ * receipt establishes submission within this run, not a provider call-ID join. */
+export function planExposure(input: { runs: PlanRow[]; evidence: PlanRow[]; leadId: string; parentId: string; invocations: PlanInvocation[] }) {
+  const initial = input.runs.filter(r => r.agentId === input.leadId && r.nativeIssueId === input.parentId)
+    .sort((a, b) => Date.parse(a.startedAt) - Date.parse(b.startedAt))[0];
+  const records = input.evidence.filter(r => r.runId === initial?.id);
+  const events: PlanRow[] = records.length === 1 && Array.isArray(records[0].events) ? records[0].events : [];
+  const submitted = events.filter(e => e.runId === initial?.id && e.eventType === "turn.submitted");
+  const prp = submitted.length === 1 ? submitted[0].payload?.prpEvent : undefined;
+  const actual = prp?.payload?.skillInputs ?? [];
+  const accepts = events.filter(e => e.runId === initial?.id && e.eventType === "turn.accepted");
+  const acceptedTurn = accepts.length === 1 ? accepts[0].payload?.prpEvent : undefined;
+  const accepted = typeof acceptedTurn?.turnId === "string" && acceptedTurn.turnId.length > 0 &&
+    acceptedTurn.payload?.turnId === acceptedTurn.turnId && events.some(e => e.runId === initial?.id && e.eventType === "turn.started" &&
+      e.payload?.prpEvent?.turnId === acceptedTurn.turnId);
+  const expected = input.invocations;
+  const matched = Array.isArray(actual) && actual.length === expected.length && expected.every(wanted => actual.filter((s: PlanRow) =>
+    s.type === "skill" && s.name === wanted.name && typeof s.path === "string" &&
+    path.isAbsolute(s.path) && s.path.endsWith(`/runtime-context-assets/bundles/${wanted.assetDigest}/SKILL.md`)).length === 1);
+  return { id: "assigned-guidance-delivered", passed: Boolean(initial && prp && accepted && matched),
+    detail: `Initial lead run must submit exactly ${expected.length} native skill inputs tied to the verified immutable file digests, and have a provider-accepted turn. This verifies the submission contract, not model consumption.`,
+    selectedCount: expected.length, observedInputCount: Array.isArray(actual) ? actual.length : null };
+}

tests/runner-e2e/plan-task-flow.ts

@@ -9,6 +9,7 @@ import type { MatrixExecution } from "./types.js";
 import { createPlanTaskThroughUi } from "./plan-task-ui.js";
 import { PLAN_BASE_SHA, PLAN_BUDGET_CENTS, PLAN_MAX_RUNS, parsePlanCase, planDefinitionDigest, planScenario } from "./plan-task-cases.js";
 import { gradePlanTask, type PlanCheck, type PlanDocument, type PlanObservation, type PlanRow } from "./plan-task-scoring.js";
+import { planExposure, planInvocationPrompt, planInvocations, type PlanInvocation } from "./plan-task-exposure.js";
 import { preparePlanSkills, selectPlanSkills, verifyPlanSelection, verifyPlanSkills } from "./plan-task-skills.js";
 
 type Input = {
@@ -28,6 +29,13 @@ export async function runPlanTaskFlow(input: Input) {
   let parent: PlanRow | undefined, alexId = "", rileyId = "", failure: string | undefined;
   let observed: PlanObservation = { issues: [], runs: [], documents: [], comments: [], activity: [], interactions: [], wakes: [] };
   let checks: PlanCheck[] = [];
+  let invocations: PlanInvocation[] = [];
+  let runEvidence: PlanRow[] = [];
+  const exposure = () => planExposure({ runs: observed.runs, evidence: runEvidence, leadId: f.agent.id, parentId: parent?.id ?? "", invocations });
+  async function captureRuns() {
+    runEvidence = await Promise.all(observed.runs.map(run => collectChatRunEvidence(api, run as Parameters<typeof collectChatRunEvidence>[1])
+      .catch(error => ({ runId: run.id, evidenceError: String(error) }))));
+  }
   const grade = () => gradePlanTask({ caseId, marker: scenario.marker, parentId: parent?.id ?? "", leadId: f.agent.id,
     alexId, rileyId, observation: observed, maxRuns: PLAN_MAX_RUNS, origin: api.baseURL });
 
@@ -79,11 +87,14 @@ export async function runPlanTaskFlow(input: Input) {
       await api.patch(`/api/agents/${id}/budgets`, { budgetMonthlyCents: PLAN_BUDGET_CENTS });
       const skills = await selectPlanSkills(api, f.company.id, id, source.skills);
       source.agents.push({ id, skills });
+      if (id === f.agent.id) invocations = planInvocations(source.skills, skills);
     }
+    source.invocations = invocations;
+    source.requestPrompt = planInvocationPrompt(scenario.prompt, invocations);
     await input.evidence("plan-task-source.json", source);
     await api.patch("/api/instance/settings/experimental", { enableClassicTaskInterface: false });
     const created = await createPlanTaskThroughUi({ page, companyId: f.company.id, issuePrefix: f.company.issuePrefix!,
-      agentName: f.agent.name, prompt: scenario.prompt });
+      agentName: f.agent.name, prompt: source.requestPrompt });
     parent = await pollUntil({ label: "browser-created planning-guidance task", deadlineAt: input.deadlineAt,
       load: () => api.get<PlanRow>(`/api/issues/${created.id}`), accept: i => i.id === created.id });
     if (!parent) throw new Error("Browser task not found");
@@ -99,7 +110,8 @@ export async function runPlanTaskFlow(input: Input) {
         o.runs.some(r => ["failed", "timed_out", "cancelled"].includes(r.status)) ? "actual run failed; retain original failure" : undefined });
     await verifyPlanSkills(api, f.company.id, source.skills);
     for (const id of [f.agent.id, alexId, rileyId]) await verifyPlanSelection(api, f.company.id, id, source.skills);
-    checks = grade().checks;
+    await captureRuns();
+    checks = [...grade().checks, exposure()];
     input.observe(parent, observed.runs, checks);
     await page.goto(`/${f.company.issuePrefix}/issues/${parent.identifier ?? parent.id}`, { waitUntil: "domcontentloaded" });
     await expect(page.getByRole("heading", { name: parent.title, exact: true })).toBeVisible();
@@ -109,7 +121,7 @@ export async function runPlanTaskFlow(input: Input) {
     return { issue: parent, runs: observed.runs, checks };
   } catch (error) {
     failure = error instanceof Error ? error.message : String(error);
-    checks = [...grade().checks, { id: "workflow-completed", passed: false, detail: failure }];
+    checks = [...grade().checks, exposure(), { id: "workflow-completed", passed: false, detail: failure }];
     if (parent) input.observe(parent, observed.runs, checks);
     throw error;
   } finally {
@@ -117,11 +129,11 @@ export async function runPlanTaskFlow(input: Input) {
     try { await observe(); } catch (error) { captureError = String(error); }
     const finalSkills = await Promise.all([f.agent.id, alexId, rileyId].filter(Boolean).map(async id => ({ id,
       skills: await api.get(`/api/agents/${id}/skills?companyId=${f.company.id}`).catch(error => ({ error: String(error) })) })));
-    await input.evidence("plan-task-runs.json", await Promise.all(observed.runs.map(run => collectChatRunEvidence(api, run as Parameters<typeof collectChatRunEvidence>[1])
-      .catch(error => ({ runId: run.id, evidenceError: String(error) })))));
+    await captureRuns();
+    await input.evidence("plan-task-runs.json", runEvidence);
     await input.evidence("plan-task-guidance.json", { variant, caseId, scenario, source, finalSkills, parentId: parent?.id,
       leadId: f.agent.id, alexId, rileyId, budgetCents: PLAN_BUDGET_CENTS, maxRuns: PLAN_MAX_RUNS,
-      observation: observed, result: grade(), failure, captureError });
+      observation: observed, result: grade(), exposure: exposure(), failure, captureError });
     await input.evidence("api-state.json", { capturePhase: "plan-task-final", issue: parent, runs: observed.runs, checks, failure, captureError });
   }
 }