diff --git a/packages/paperclip-runner/.gitignore b/packages/paperclip-runner/.gitignore index a2a2eff9a6a..df6680b55a2 100644 --- a/packages/paperclip-runner/.gitignore +++ b/packages/paperclip-runner/.gitignore @@ -2,3 +2,4 @@ /dist-phase5/ /playwright-report/ /test-results/ +/dist-phase7/ diff --git a/packages/paperclip-runner/docs/design/phase-7-scenario-explorer-ux.md b/packages/paperclip-runner/docs/design/phase-7-scenario-explorer-ux.md index 30217067249..f7bf5450591 100644 --- a/packages/paperclip-runner/docs/design/phase-7-scenario-explorer-ux.md +++ b/packages/paperclip-runner/docs/design/phase-7-scenario-explorer-ux.md @@ -4,7 +4,7 @@ Status: **approved UX contract for 7F implementation** (PAP-16899, 2026-08-09). Owner: UXDesigner. Implementer: 7F (browser scenario explorer), integrating 7C mock adapter output and 7E parity output. Companion records: [Phase 7A capability contract](../phase-07-capability-contract.md), -`spec/phase-07/eval-traceability.yaml`, [Phase 7 plan](/PAP/issues/PAP-16897#document-plan), +`spec/phase-07/eval-traceability.yaml`, the Phase 7 plan (PAP-16897), [Phase 4b interaction map](phase-4b-interaction-map.md), [Phase 5 component decisions](phase-5-component-decisions.md). Mockup: [`mockups/phase-7-scenario-explorer.html`](mockups/phase-7-scenario-explorer.html) @@ -431,3 +431,63 @@ fails the gate. never recomputes assertions. - No live-Paperclip or ACPX affordances; the future binding boundary stays a docs note, not a UI surface. + +## 11. Revisions recorded during 7F implementation + +Per §9, deviations are written here rather than left silent. All of the +following were found while building the explorer (PAP-16904) and are now part +of the contract. + +1. **Timeline entry kinds extended.** §8 listed `semantic_call`, + `semantic_result`, `control_plane_action`, and `protocol_event_ref`. The + agent channel also needs plain model turns and the restraint note §3.2 + requires, so the artifact adds `agent_message` and `system_note`. Protocol + event references ride on `semantic_result.protocolEventRefs` rather than + standing as their own entry, keeping the semantic view primary. + +2. **Parity is computed by the runtime, not the browser — and not by 7E.** + §4.4 and §10 forbid the UI from judging assertions, and they do: the + package-local scenario runner (`src/phase7/scenario-parity.ts`) produces the + verdict from the traceability expectations and the recorded artifact. Phase + 7E's own per-case result is carried through verbatim in a separately + labelled "Phase 7E conformance suite" block and is never merged into, or + used to overrule, the assertion list. When no 7E report is bundled the block + reads **Not run**. + +3. **Actor role, task mode, and scenario claims are declared, not inferred.** + §8 assigned `actorRole`/`taskMode` to 7C's index generator; 7C shipped + without them. They are now declared in `src/phase7/scenario-index.ts` as an + explicit fixture/case profile table, alongside the catalog claims each + scenario grants. A claim that cannot unlock any operation in the scenario's + task mode or for its actor role is dropped from the profile, so no scenario + renders a grant chip that unlocks nothing. + +4. **The mobile section switcher is a `radiogroup`, not a `tablist`.** §1 asked + for the Phase 4b segmented control. The Phase 4b console renders it only in + compact mode; the explorer keeps one CSS-driven tree, so a `tablist` would + put two tablists in the document and model the three landmarks as tabpanels. + The three regions stay `nav`/`main`/`complementary`, and the only `tablist` + on the page is the inspector's. + +5. **Two token-layer overrides, scoped to the explorer.** The SDK gives every + direct `span` child of a disclosure summary `min-width: 0`; a `Badge` is + also a `span` and sets `white-space: nowrap`, so at 390px badges were + squeezed below their own content and pushed the page into horizontal scroll. + The SDK badge also applies `text-transform: capitalize`, which renders + "Optional Tool" and "Not Run" against the §0 vocabulary. Both are corrected + in `examples/phase7-explorer/src/explorer.css` under `.pcr7-shell`; the + frozen `0.1.2` sheet is unchanged. **These are SDK defects worth fixing at + the next SDK revision, not explorer quirks.** + +6. **Only the case list scrolls in its own box.** The rails are otherwise in + page flow. §9 evidence is captured full-page, and an inner scroll container + on the rail clipped exactly what the matrix asks to see (all 16 group facets + and their counts). The 106-row case list keeps a bounded height so the page + stays navigable. + +7. **Shot 2's route uses `parity=not_run`.** §9 specified + `#/?group=ix&disposition=always_agent_tool&parity=pass`. Parity status is a + property of a *run*, and the picker route runs nothing, so `parity=pass` + selects zero scenarios on a cold load. The recorded route filters on + `not_run`, which is the honest cold-load state and still demonstrates + chips, live counts, disabled zero-count values, and Clear filters. diff --git a/packages/paperclip-runner/examples/phase7-explorer.spec.ts b/packages/paperclip-runner/examples/phase7-explorer.spec.ts new file mode 100644 index 00000000000..56de8d5b647 --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer.spec.ts @@ -0,0 +1,235 @@ +import { expect, test, type Page } from "@playwright/test"; + +/** + * Phase 7F browser acceptance. + * + * Covers the interaction map's determinism contract (§7), the keyboard paths + * and landmark structure (§6), and the credential/policy boundary (§5). Screen + * captures for the §9 acceptance matrix are recorded by + * `scripts/record-phase7-evidence.mjs`, which reuses the same routes. + */ + +const BASE = "/phase7-explorer/"; + +async function open(page: Page, hash: string): Promise { + await page.goto(`${BASE}${hash}`); + await expect(page.locator('[data-testid="explorer-shell"]')).toHaveAttribute( + "data-run-state", + "settled", + { timeout: 30_000 }, + ); +} + +test.describe("shell and information architecture", () => { + test("explorer home shows all 16 groups summing to 106 with no dead panel", async ({ page }) => { + await open(page, "#/"); + await expect(page.locator('[data-testid="scenario-count"]')).toHaveText("106 of 106 scenarios"); + await expect(page.locator('[data-testid="explorer-intro"]')).toBeVisible(); + + const counts = await page + .locator('[data-testid^="facet-group-"] .pcr7-facet-count') + .allTextContents(); + expect(counts).toHaveLength(16); + expect(counts.reduce((total, value) => total + Number(value), 0)).toBe(106); + + await expect(page.locator("nav[aria-label='Scenario picker']")).toBeVisible(); + await expect(page.locator("main#run-view")).toBeVisible(); + await expect(page.locator("aside[aria-label='Scenario inspector']")).toBeVisible(); + await expect(page.locator("h1")).toHaveCount(1); + }); + + test("filtered picker route renders chips, counts, and a way out", async ({ page }) => { + await open(page, "#/?group=ix&disposition=always_agent_tool&parity=not_run"); + await expect(page.locator('[data-testid="active-filter-group"]')).toBeVisible(); + await expect(page.locator('[data-testid="active-filter-disposition"]')).toBeVisible(); + await expect(page.locator('[data-testid="clear-filters"]')).toBeVisible(); + await expect(page.locator('[data-testid="scenario-count"]')).toContainText("of 106 scenarios"); + // A zero-count value stays legible but is not clickable. + await expect(page.locator('[data-testid="facet-parity-fail"]')).toBeDisabled(); + }); +}); + +test.describe("route determinism", () => { + test("two loads of the same run route produce identical settled DOM", async ({ page }) => { + const route = "#/case/ap-mcp-gate-01?run=fake&view=authorization"; + await open(page, route); + const first = await page.locator('[data-testid="explorer-shell"]').innerHTML(); + await page.reload(); + await expect(page.locator('[data-testid="explorer-shell"]')).toHaveAttribute( + "data-run-state", + "settled", + { timeout: 30_000 }, + ); + const second = await page.locator('[data-testid="explorer-shell"]').innerHTML(); + expect(second).toBe(first); + }); + + test("a fake run renders fixture time only", async ({ page }) => { + await open(page, "#/case/hb-scoped-wake-01?run=fake&view=transcript"); + const text = (await page.locator('[data-testid="transcript"]').innerText()).toLowerCase(); + const currentYear = String(new Date().getUTCFullYear()); + expect(text).not.toContain(`${currentYear}-`); + await expect(page.locator('[data-testid="transcript"] [data-channel="control_plane"]').first()).toContainText( + "no agent tool exists for this", + ); + }); + + test("an unrun scenario primes the run rather than showing a dead panel", async ({ page }) => { + await open(page, "#/case/hb-scoped-wake-01"); + await expect(page.locator('[data-testid="run-header"]')).toBeVisible(); + await expect(page.locator("main")).toContainText("Run to produce the deterministic timeline."); + await expect(page.locator('[data-testid="run-button"]')).toBeVisible(); + }); +}); + +test.describe("acceptance evidence", () => { + test("authorization view shows a deny row and a tab deny count", async ({ page }) => { + await open(page, "#/case/ap-mcp-gate-01?run=fake&view=authorization"); + await expect(page.locator('[data-testid="panel-authorization"] tr[data-outcome="denied"]')).toHaveCount(1); + await expect(page.locator('[role="tab"]', { hasText: "Authorization" })).toContainText("deny"); + await expect(page.locator('[data-testid="transcript"] [data-outcome="denied"]').first()).toContainText( + "required_claim_missing", + ); + }); + + test("context view lists optional tools with the grant that unlocked them", async ({ page }) => { + await open(page, "#/case/rf-api-mgr-heartbeat-01?run=fake&view=context"); + await expect(page.locator('[data-testid="exposure-optional"]')).toContainText("discovery:agents:read"); + await expect(page.locator('[data-testid="exposure-control-plane"]')).toContainText("checkout_task"); + await expect(page.locator('[data-testid="panel-context"]')).toContainText("Manager"); + }); + + test("state diff collapses unchanged domains and marks changes with icon plus label", async ({ page }) => { + await open(page, "#/case/bl-create-blocked-01?run=fake&view=diff"); + await expect(page.locator('[data-testid="diff-domain-blockers"]')).toHaveAttribute("data-changed", "true"); + await expect(page.locator('[data-testid="diff-domain-approvals"]')).toHaveAttribute("data-changed", "false"); + await expect(page.locator('[data-testid="diff-domain-tasks"]')).toContainText("Added"); + }); + + test("restraint reads as deliberate absence, not an empty screen", async ({ page }) => { + await open(page, "#/case/rs-question-only-01?run=fake&view=parity"); + await expect(page.locator('[data-testid="restraint-note"]')).toContainText("No further operations"); + await expect(page.locator('[data-testid="parity-verdict"]')).toContainText("Pass"); + await expect(page.locator('[data-testid="parity-forbidden"]')).toBeVisible(); + }); + + test("redaction chips name the rule and never the value", async ({ page }) => { + await open(page, "#/case/rs-secret-hygiene-01?run=fake&view=transcript"); + await page.locator('[data-testid="transcript"] details').first().evaluate((node) => { + for (const element of document.querySelectorAll("details")) element.setAttribute("open", ""); + return node.tagName; + }); + await expect(page.locator('[data-testid="redaction-chip"]').first()).toBeVisible(); + const body = await page.locator("body").innerText(); + expect(body).not.toContain("fixture-secret-value-not-a-real-credential"); + }); +}); + +test.describe("credential and policy boundary", () => { + test("fake mode performs no network request beyond its own assets", async ({ page }) => { + const external: string[] = []; + page.on("request", (request) => { + if (!request.url().startsWith("http://127.0.0.1:4193")) external.push(request.url()); + }); + await open(page, "#/case/mh-subtask-tree-01?run=fake&view=parity"); + expect(external).toEqual([]); + }); + + test("localStorage holds no run artifact, grant, or fixture payload", async ({ page }) => { + await open(page, "#/case/hb-scoped-wake-01?run=fake"); + const stored = await page.evaluate(() => JSON.stringify(window.localStorage)); + expect(stored).toBe("{}"); + }); + + test("Codex mode is disabled with a stated reason and fake mode stays usable", async ({ page }) => { + await open(page, "#/case/hb-scoped-wake-01?run=fake"); + await expect(page.locator('[data-testid="mode-codex"]')).toBeDisabled(); + await expect(page.locator('[data-testid="codex-unavailable"]')).toContainText( + "provider relay not running", + ); + await expect(page.locator('[data-testid="mode-fake"]')).toHaveAttribute("aria-checked", "true"); + }); +}); + +test.describe("keyboard and accessibility", () => { + test("the picker listbox moves selection with the arrow keys", async ({ page }) => { + await open(page, "#/?group=ix"); + const list = page.locator('[data-testid="case-list"]'); + await list.focus(); + await page.keyboard.press("ArrowDown"); + await expect(page.locator('[data-testid="case-list"] [aria-selected="true"]')).toHaveCount(1); + const first = await page.locator('[data-testid="case-list"] [aria-selected="true"]').getAttribute("id"); + await list.focus(); + await page.keyboard.press("ArrowDown"); + const second = await page.locator('[data-testid="case-list"] [aria-selected="true"]').getAttribute("id"); + expect(second).not.toBe(first); + await expect(list).toHaveAttribute("aria-activedescendant", String(second)); + }); + + test("inspector tabs follow the WAI-ARIA arrow-key pattern", async ({ page }) => { + await open(page, "#/case/hb-scoped-wake-01?run=fake&view=context"); + // Exactly one tablist on the page: the mobile section switcher is a + // radiogroup so the three landmarks are not modelled as tabpanels. + await expect(page.locator('[role="tablist"]')).toHaveCount(1); + const selected = page.locator('[role="tab"][aria-selected="true"]'); + await page.locator('[role="tab"]').first().focus(); + await page.keyboard.press("ArrowRight"); + await expect(selected).toContainText("Authorization"); + await page.keyboard.press("ArrowRight"); + await expect(selected).toContainText("State diff"); + await expect(page.locator('[role="tabpanel"]')).toBeVisible(); + }); + + test("every interactive control is reachable and named", async ({ page }) => { + await open(page, "#/case/ap-mcp-gate-01?run=fake&view=authorization"); + const unnamed = await page.evaluate(() => + [...document.querySelectorAll("button, input, [role='tab'], [role='radio']")] + .filter((element) => { + const label = + element.getAttribute("aria-label") ?? + element.getAttribute("title") ?? + (element.textContent ?? "").trim(); + const labelled = + element.id.length > 0 && + document.querySelector(`label[for="${CSS.escape(element.id)}"]`) !== null; + return label.length === 0 && !labelled; + }) + .map((element) => element.outerHTML.slice(0, 120)), + ); + expect(unnamed).toEqual([]); + }); + + test("run progress is announced through a polite live region", async ({ page }) => { + await open(page, "#/case/hb-scoped-wake-01?run=fake"); + await expect(page.locator('[role="status"][aria-live="polite"]')).toContainText( + "Scenario run settled — verdict pass", + ); + }); +}); + +test.describe("responsive", () => { + test.use({ viewport: { width: 390, height: 844 } }); + + test("mobile shows one segment at a time with no horizontal scrollbar", async ({ page }) => { + await open(page, "#/case/ap-mcp-gate-01?run=fake&view=transcript"); + await expect(page.locator('[data-testid="segment-run"]')).toBeVisible(); + await expect(page.locator('[data-testid="explorer-shell"]')).toHaveAttribute("data-segment", "run"); + const overflow = await page.evaluate( + () => document.documentElement.scrollWidth - document.documentElement.clientWidth, + ); + expect(overflow).toBeLessThanOrEqual(0); + }); + + test("mobile inspector routes open the Inspect segment", async ({ page }) => { + await open(page, "#/case/rf-api-mgr-heartbeat-01?run=fake&view=context"); + await expect(page.locator('[data-testid="explorer-shell"]')).toHaveAttribute( + "data-segment", + "inspect", + ); + await expect(page.locator('[data-testid="panel-context"]')).toBeVisible(); + const overflow = await page.evaluate( + () => document.documentElement.scrollWidth - document.documentElement.clientWidth, + ); + expect(overflow).toBeLessThanOrEqual(0); + }); +}); diff --git a/packages/paperclip-runner/examples/phase7-explorer/index.html b/packages/paperclip-runner/examples/phase7-explorer/index.html new file mode 100644 index 00000000000..582f77ae0da --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/index.html @@ -0,0 +1,9 @@ + + + + + + Phase 7 scenario explorer — mock control plane + +
+ diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/app.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/app.tsx new file mode 100644 index 00000000000..13c53004702 --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/app.tsx @@ -0,0 +1,434 @@ +import * as React from "react"; + +import { Tabs, type TabDefinition } from "@paperclipai/paperclip-runner/react"; +import { + phase7RunScenario, + type Phase7EvalSuiteLookup, + type Phase7ParityStatus, + type Phase7RunArtifact, + type Phase7ScenarioIndex, + type Phase7ScenarioIndexEntry, +} from "@paperclip-runner-local/phase7"; + +import { + AuthorizationPanel, + ContextPanel, + denialCount, + ParityPanel, + StateDiffPanel, +} from "./components/inspector-panels.js"; +import { buildFacets, ScenarioPicker, type Phase7PickerFacets } from "./components/scenario-picker.js"; +import { RunHeader } from "./components/run-header.js"; +import { ScenarioTranscript } from "./components/scenario-transcript.js"; +import { EmptyState, Mono } from "./components/primitives.js"; +import { DISPOSITION_LABEL } from "./labels.js"; +import { + EMPTY_FILTERS, + parsePhase7Route, + serializePhase7Route, + type Phase7Filters, + type Phase7InspectorView, + type Phase7Route, +} from "./route.js"; + +/** + * Phase 7 scenario explorer shell. + * + * One React tree, three regions: picker (nav), run view (main), inspector + * (complementary). The shell owns route and view state only — never protocol + * state, never a policy decision, never a credential. + */ + +export interface ExplorerAppProps { + index: Phase7ScenarioIndex; + evalSuite?: Phase7EvalSuiteLookup; + /** Injected so tests can drive a settled state without a live relay probe. */ + codexAvailable?: boolean; + initialRoute?: Phase7Route; + /** Test seam: run scenarios synchronously through an injected runner. */ + runScenario?: typeof phase7RunScenario; +} + +type RunState = "idle" | "pending" | "settled" | "failed"; +type Segment = "scenarios" | "run" | "inspect"; + +const INSPECTOR_TABS: readonly TabDefinition[] = [ + { id: "context", label: "Context" }, + { id: "authorization", label: "Authorization" }, + { id: "diff", label: "State diff" }, + { id: "parity", label: "Parity" }, +]; + +export function ExplorerApp({ + index, + evalSuite, + codexAvailable = false, + initialRoute, + runScenario = phase7RunScenario, +}: ExplorerAppProps) { + const [route, setRoute] = React.useState( + () => initialRoute ?? parsePhase7Route(typeof window === "undefined" ? "#/" : window.location.hash), + ); + const [artifacts, setArtifacts] = React.useState>({}); + const [runState, setRunState] = React.useState("idle"); + const [mode, setMode] = React.useState<"fake" | "codex">(route.run ?? "fake"); + const [segment, setSegment] = React.useState(() => + route.caseId === null ? "scenarios" : route.view === "transcript" ? "run" : "inspect", + ); + const [replayPosition, setReplayPosition] = React.useState(0); + const [highlight, setHighlight] = React.useState(null); + + const selected = route.caseId === null ? null : findEntry(index, route.caseId); + const artifact = selected === null ? null : (artifacts[selected.id] ?? null); + + const navigate = React.useCallback((next: Phase7Route) => { + setRoute(next); + if (typeof window !== "undefined") { + window.history.replaceState(null, "", serializePhase7Route(next)); + } + }, []); + + React.useEffect(() => { + if (typeof window === "undefined") return undefined; + const onHashChange = (): void => setRoute(parsePhase7Route(window.location.hash)); + window.addEventListener("hashchange", onHashChange); + return () => window.removeEventListener("hashchange", onHashChange); + }, []); + + // A deep link that arrives while the app is already open (hash change, back + // button) has to move the mobile segment too, or the route says "parity" + // while the screen still shows the picker. + React.useEffect(() => { + setSegment(route.caseId === null ? "scenarios" : route.view === "transcript" ? "run" : "inspect"); + }, [route.caseId, route.view]); + + const run = React.useCallback( + async (entry: Phase7ScenarioIndexEntry) => { + setRunState("pending"); + try { + const next = await runScenario(entry, { mode: "fake", evalSuite }); + setArtifacts((current) => ({ ...current, [entry.id]: next })); + setReplayPosition(next.timeline.length); + setRunState(next.failure === null ? "settled" : "failed"); + } catch (error) { + setRunState("failed"); + throw error; + } + }, + [evalSuite, runScenario], + ); + + // `run=fake` auto-executes so a screenshot route settles without interaction. + React.useEffect(() => { + if (selected === null || route.run !== "fake") return; + if (artifacts[selected.id] !== undefined || runState === "pending") return; + void run(selected); + }, [artifacts, route.run, run, runState, selected]); + + const parityStatus = React.useCallback( + (id: string): Phase7ParityStatus => artifacts[id]?.parity.verdict ?? "not_run", + [artifacts], + ); + + const visible = React.useMemo( + () => index.entries.filter((entry) => matches(entry, route.filters, parityStatus)), + [index.entries, parityStatus, route.filters], + ); + + const facets = React.useMemo( + () => + buildFacets(index.entries, (key, value) => { + const probe: Phase7Filters = { + ...route.filters, + [key === "parity" ? "parity" : key]: value, + }; + return index.entries.filter((entry) => matches(entry, probe, parityStatus)).length; + }), + [index.entries, parityStatus, route.filters], + ); + + const timeline = artifact === null ? [] : artifact.timeline.slice(0, replayPosition); + const settled = + selected === null + ? "settled" + : route.run === null + ? "settled" + : runState === "settled" || runState === "failed" + ? "settled" + : "pending"; + + const view: Phase7InspectorView = route.view; + const inspectorTab = view === "transcript" ? "context" : view; + + function selectCase(id: string): void { + navigate({ ...route, caseId: id, filters: route.filters }); + setSegment("run"); + setReplayPosition(artifacts[id]?.timeline.length ?? 0); + } + + function setView(next: Phase7InspectorView): void { + navigate({ ...route, view: next }); + if (next !== "transcript") setSegment("inspect"); + } + + function setFilters(patch: Partial): void { + navigate({ ...route, caseId: null, filters: { ...route.filters, ...patch } }); + setSegment("scenarios"); + } + + return ( +
+ + Skip to run view + + + {/* + A radiogroup, not a tablist: the three regions stay landmarks + (nav / main / complementary) rather than becoming tabpanels, and the + page keeps exactly one tablist — the inspector's. + */} +
+ {(["scenarios", "run", "inspect"] as const).map((value) => ( + + ))} +
+ +
+ setFilters(EMPTY_FILTERS)} + onSelect={selectCase} + /> + +
+ {selected === null ? ( + + ) : ( + <> + setReplayPosition(0), + exit: () => setReplayPosition(artifact?.timeline.length ?? 0), + setPosition: setReplayPosition, + togglePlay: () => setReplayPosition(artifact?.timeline.length ?? 0), + step: (delta) => + setReplayPosition((current) => + Math.min(Math.max(current + delta, 0), artifact?.timeline.length ?? 0), + ), + }} + onRun={() => void run(selected)} + onModeChange={setMode} + onOpenParity={() => setView("parity")} + /> +

+ {runState === "settled" + ? `Scenario run settled — verdict ${artifact?.parity.verdict ?? "not run"}` + : runState === "pending" + ? "Scenario run in progress" + : ""} +

+ {artifact === null ? ( + +

+ Fake mode runs entirely in this page against checked-in fixtures. No Paperclip + service is contacted and no credential is held. +

+
+ ) : ( + + )} + + )} +
+ + +
+
+ ); +} + +function IntroCard({ + index, + onSelect, +}: { + index: Phase7ScenarioIndex; + onSelect: (id: string) => void; +}) { + const examples = ["hb-scoped-wake-01", "ap-mcp-gate-01", "rs-question-only-01"].filter((id) => + index.entries.some((entry) => entry.id === id), + ); + return ( +
+

Phase 7 scenario explorer — mock control plane

+

{index.entries.length} scenarios across {index.groups.length} eval groups

+

+ Every scenario runs against the deterministic mock control plane in this page. The explorer + renders records the runtime produced: it holds no credential, decides no policy, and owns no + control-plane state. +

+

Pick a scenario from the rail, or start here

+
    + {examples.map((id) => ( +
  • + +
  • + ))} +
+
+ ); +} + +/** + * The inspector before a scenario is picked. Corpus stats keep the panel + * useful instead of leaving a dead column (UX map §6, "no dead panels"). + */ +function CorpusSummary({ index }: { index: Phase7ScenarioIndex }) { + const byDisposition = (["control_plane_owned", "always_agent_tool", "optional_agent_tool"] as const).map( + (disposition) => ({ + disposition, + label: DISPOSITION_LABEL[disposition], + count: index.entries.filter((entry) => entry.primaryDisposition === disposition).length, + }), + ); + return ( +
+
+

Corpus

+
+
Scenarios
+
{index.entries.length}
+
Eval groups
+
{index.groups.length}
+ {byDisposition.map((row) => ( + +
{row.label}
+
{row.count}
+
+ ))} +
+
+
+

Boundary

+

+ The explorer receives snapshots and records only. It owns no credential, no policy + decision, and no mutable control-plane state; every allow, deny, and redaction on screen + was emitted by the mock core. +

+
+ +
+ ); +} + +function findEntry(index: Phase7ScenarioIndex, id: string): Phase7ScenarioIndexEntry | null { + return index.entries.find((entry) => entry.id === id) ?? null; +} + +export function matches( + entry: Phase7ScenarioIndexEntry, + filters: Phase7Filters, + parityStatus: (id: string) => string, +): boolean { + if (filters.group !== null && filters.group.length > 0 && entry.group !== filters.group) return false; + if ( + filters.disposition !== null && + filters.disposition.length > 0 && + entry.primaryDisposition !== filters.disposition + ) { + return false; + } + if (filters.role !== null && filters.role.length > 0 && entry.actorRole !== filters.role) return false; + if ( + filters.claim !== null && + filters.claim.length > 0 && + !entry.requiredGrants.includes(filters.claim) + ) { + return false; + } + if ( + filters.parity !== null && + filters.parity.length > 0 && + parityStatus(entry.id) !== filters.parity + ) { + return false; + } + if (filters.query.length > 0) { + const needle = filters.query.toLowerCase(); + if (!`${entry.id} ${entry.title}`.toLowerCase().includes(needle)) return false; + } + return true; +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/components/inspector-panels.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/components/inspector-panels.tsx new file mode 100644 index 00000000000..b2990ae9b1b --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/components/inspector-panels.tsx @@ -0,0 +1,451 @@ +import * as React from "react"; + +import { Badge } from "@paperclipai/paperclip-runner/react"; +import type { + Phase7AuthorizationRecord, + Phase7Exposure, + Phase7ParityReport, + Phase7RunArtifact, + Phase7ScenarioIndexEntry, + Phase7StateDiff, +} from "@paperclip-runner-local/phase7"; + +import { + ASSERTION_CLASS_LABEL, + CHANGE_ICON, + CHANGE_LABEL, + groupLabel, + outcomeLabel, + outcomeTone, + ROLE_LABEL, + TASK_MODE_LABEL, +} from "../labels.js"; +import { + ClaimChips, + DispositionBadge, + EmptyState, + JsonBlock, + Mono, + ParityStatusBadge, +} from "./primitives.js"; + +/* --------------------------------------------------------------- * + * Context — who ran, under what authority, with which tools (§4.1). + * --------------------------------------------------------------- */ + +export function ContextPanel({ + entry, + artifact, +}: { + entry: Phase7ScenarioIndexEntry; + artifact: Phase7RunArtifact | null; +}) { + return ( +
+
+

Actor

+
+
Actor role
+
{ROLE_LABEL[entry.actorRole] ?? entry.actorRole}
+
Task mode
+
{TASK_MODE_LABEL[entry.taskMode] ?? entry.taskMode}
+
Fixture
+
+ {entry.fixture} +
+
Wake
+
+ {entry.wakeReason} + {artifact === null ? null : <> — scenario {artifact.scenarioId}} +
+
Budget
+
+ {artifact === null ? ( + no run yet + ) : ( + `${artifact.budget.remainingCents} of ${artifact.budget.limitCents} cents remaining` + )} +
+
+
+ +
+

Capability grants

+

+ Two vocabularies, both verbatim: the eval case records the capability it needs, the + fixture issues the catalog claims that unlock the matching operations. +

+

Required by the eval case

+ +

Issued by the fixture as catalog claims

+ +
+ +
+

Tool exposure

+ {artifact === null ? ( + +

Run the scenario to record the exposure decision.

+
+ ) : ( + + )} +
+ +
+

Traceability

+
+
Source anchor
+
+ {entry.sourceAnchor} +
+
Group
+
+ {groupLabel(entry.group)} {entry.group} +
+
Disposition
+
+ +
+
Skill elements
+
+ {entry.skillElements.length === 0 ? ( + none recorded + ) : ( + entry.skillElements.map((element) => ( + + {element.file} · {element.section} · {element.lines} + + )) + )} +
+
Evidence IDs
+
+ +
+
Legacy MCP mapping
+
+ +
+
+
+
+ ); +} + +export function ExposureList({ exposure }: { exposure: Phase7Exposure }) { + return ( + <> +

Always tools ({exposure.always.length})

+
    + {exposure.always.map((tool) => ( +
  • + {tool.operationId} — {tool.title} +
  • + ))} +
+ +

Optional tools ({exposure.optional.length})

+
    + {exposure.optional.length === 0 ? ( +
  • No optional operation is unlocked for this run.
  • + ) : ( + exposure.optional.map((tool) => ( +
  • + {tool.operationId} + {tool.unlockedBy === null ? null : ( + <> + {" "} + — via {tool.unlockedBy} + + )} +
  • + )) + )} +
+ +

Control plane — no tool ({exposure.controlPlane.length})

+
    + {exposure.controlPlane.map((capability) => ( +
  • + {capability.operationId} — {capability.reason} +
  • + ))} +
+ +
+ + Catalogued but not exposed ({exposure.withheld.length}) + +
    + {exposure.withheld.map((tool) => ( +
  • + {tool.operationId} — {tool.reason} +
  • + ))} +
+
+ + ); +} + +/* --------------------------------------------------------------- * + * Authorization (§4.2). + * --------------------------------------------------------------- */ + +export function AuthorizationPanel({ + records, + onSelectRecord, +}: { + records: readonly Phase7AuthorizationRecord[]; + onSelectRecord?: (sequence: number) => void; +}) { + if (records.length === 0) { + return ( +
+ +

Run the scenario to record authorization decisions.

+
+
+ ); + } + return ( +
+ + + + + + + + + + + + {records.map((record) => ( + onSelectRecord(record.sequence)} + > + + + + + + ))} + +
+ Authorization decisions recorded by the mock control plane +
OperationClaims consideredDecisionReason
+ {record.operationId} + + + {record.missingClaims.length === 0 ? null : ( + + missing: + + )} + + + {" "} + {outcomeLabel(record.outcome)} + + + {record.reason} + {record.redactions.length === 0 ? null : ( + + {record.redactions.length} redaction rule + {record.redactions.length === 1 ? "" : "s"} applied + + )} + {record.resultingStateChange === null ? null : ( + + revision {record.resultingStateChange.beforeRevision} →{" "} + {record.resultingStateChange.afterRevision} + + )} +
+
+ Full authorization records + +
+
+ ); +} + +export function denialCount(records: readonly Phase7AuthorizationRecord[]): number { + return records.filter((record) => record.outcome !== "allowed").length; +} + +/* --------------------------------------------------------------- * + * State diff (§4.3). + * --------------------------------------------------------------- */ + +export function StateDiffPanel({ diff }: { diff: Phase7StateDiff | null }) { + if (diff === null) { + return ( +
+ +

Run the scenario to produce a before/after snapshot diff.

+
+
+ ); + } + return ( +
+

+ Revision {diff.beforeRevision} → {diff.afterRevision} +

+ {diff.domains.map((domain) => ( +
+ + {domain.domain} · {domain.summary} + + {domain.changed ? ( +
    + {domain.rows.map((row) => ( +
  • + + {CHANGE_LABEL[row.change]} + + {row.entityId} +
      + {row.fields.map((field) => ( +
    • + {field.field}: {field.before ?? "—"} → {field.after ?? "—"} +
    • + ))} +
    +
  • + ))} +
+ ) : ( +

unchanged

+ )} +
+ Final state + +
+
+ ))} +
+ ); +} + +/* --------------------------------------------------------------- * + * Parity (§4.4). + * --------------------------------------------------------------- */ + +export function ParityPanel({ parity }: { parity: Phase7ParityReport | null }) { + if (parity === null) { + return ( +
+ +

Run the scenario to produce a parity verdict.

+
+
+ ); + } + return ( +
+

+ {parity.assertions.length}{" "} + assertions · {parity.assertions.filter((assertion) => assertion.status === "pass").length} pass +

+ +

Assertions

+
    + {parity.assertions.map((assertion) => ( +
  • + + {ASSERTION_CLASS_LABEL[assertion.assertionClass]} + + +

    {assertion.expectation}

    + {assertion.status === "fail" ? ( +
    +
    +

    Expected

    +
    {assertion.expected}
    +
    +
    +

    Observed

    +
    {assertion.observed}
    +
    +
    + ) : ( +

    + {assertion.observed} +

    + )} +
  • + ))} +
+ +

Forbidden operations

+
    + {parity.forbidden.length === 0 ? ( +
  • + This case records no forbidden operations; restraint is proven by the absence of state + writes in the diff. +
  • + ) : ( + parity.forbidden.map((check) => ( +
  • + {check.semantic} —{" "} + {check.invoked ? ( + + invoked at transcript entry {check.transcriptSequence} + + ) : ( + <> + never invoked + + )} +
  • + )) + )} +
+ + {parity.intentionalGaps.length === 0 ? null : ( + <> +

Intentional gaps

+
    + {parity.intentionalGaps.map((gap) => ( +
  • + {gap.subject} — {gap.rationale} +
  • + ))} +
+ + )} + +

Phase 7E conformance suite

+

+ {" "} + {parity.evalSuite.available ? ( + <> + operation {parity.evalSuite.semanticOperation ?? "—"}, decision{" "} + {parity.evalSuite.authorizationDecision ?? "—"} + + ) : ( + + no Phase 7E report is loaded; this explorer renders 7E output and never recomputes it. + + )} +

+ +

+ Reference baseline, not the pass condition: {parity.legacyBaselineNote} +

+
+ ); +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/components/primitives.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/components/primitives.tsx new file mode 100644 index 00000000000..f3a870a345e --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/components/primitives.tsx @@ -0,0 +1,93 @@ +import * as React from "react"; + +import { Badge } from "@paperclipai/paperclip-runner/react"; +import type { + Phase7ParityStatus, + Phase7PrimaryDisposition, + Phase7RedactionChip as RedactionChipRecord, +} from "@paperclip-runner-local/phase7"; + +import { + DISPOSITION_LABEL, + DISPOSITION_TONE, + PARITY_ICON, + PARITY_LABEL, + PARITY_TONE, +} from "../labels.js"; + +/** Machine values — IDs, anchors, grants, JSON — always render monospace. */ +export function Mono({ children }: { children: React.ReactNode }) { + return {children}; +} + +export function DispositionBadge({ disposition }: { disposition: Phase7PrimaryDisposition }) { + return ( + + {DISPOSITION_LABEL[disposition]} + + ); +} + +/** Parity status is always icon + text; colour never carries it alone. */ +export function ParityStatusBadge({ + status, + testId, +}: { + status: Phase7ParityStatus; + testId?: string; +}) { + return ( + + {PARITY_LABEL[status]} + + ); +} + +/** + * Redacted values arrive pre-redacted from the mock core. The chip names the + * rule so a reviewer can trace the redaction rather than guess at it. + */ +export function RedactionChip({ chip }: { chip: RedactionChipRecord }) { + return ( + + + redacted + {chip.rule} + + ); +} + +export function JsonBlock({ value, label }: { value: unknown; label: string }) { + return ( + <> +

{label}

+
{JSON.stringify(value, null, 2)}
+ + ); +} + +export function EmptyState({ + title, + children, +}: { + title: string; + children?: React.ReactNode; +}) { + return ( +
+

{title}

+ {children} +
+ ); +} + +export function ClaimChips({ claims, testId }: { claims: readonly string[]; testId?: string }) { + if (claims.length === 0) return none; + return ( + + {claims.map((claim) => ( + {claim} + ))} + + ); +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/components/run-header.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/components/run-header.tsx new file mode 100644 index 00000000000..21ce1dc170d --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/components/run-header.tsx @@ -0,0 +1,140 @@ +import * as React from "react"; + +import { + Badge, + Banner, + BannerActions, + BannerText, + Button, + ReplayControls, +} from "@paperclipai/paperclip-runner/react"; +import type { Phase7RunArtifact, Phase7ScenarioIndexEntry } from "@paperclip-runner-local/phase7"; + +import { ASSERTION_CLASS_LABEL, groupLabel, PARITY_TONE } from "../labels.js"; +import { DispositionBadge, Mono, ParityStatusBadge } from "./primitives.js"; + +export interface RunHeaderProps { + entry: Phase7ScenarioIndexEntry; + artifact: Phase7RunArtifact | null; + runState: "idle" | "pending" | "settled" | "failed"; + codexAvailable: boolean; + mode: "fake" | "codex"; + replay: { + active: boolean; + position: number; + length: number; + playing: boolean; + enter: () => void; + exit: () => void; + setPosition: (position: number) => void; + togglePlay: () => void; + step: (delta: number) => void; + }; + onRun: () => void; + onModeChange: (mode: "fake" | "codex") => void; + onOpenParity: () => void; +} + +export function RunHeader({ + entry, + artifact, + runState, + codexAvailable, + mode, + replay, + onRun, + onModeChange, + onOpenParity, +}: RunHeaderProps) { + const verdict = artifact?.parity.verdict ?? "not_run"; + const passing = artifact?.parity.assertions.filter((assertion) => assertion.status === "pass").length ?? 0; + const total = artifact?.parity.assertions.length ?? 0; + + return ( +
+
+ {entry.id} +

{entry.title}

+
+ +
+ + {groupLabel(entry.group)} {entry.group} + + + {entry.assertionClasses.map((assertionClass) => ( + + {ASSERTION_CLASS_LABEL[assertionClass]} + + ))} +
+ + + + {" "} + {artifact === null + ? "This scenario has not been run yet." + : `${total} assertions · ${passing} pass`} + {artifact?.failure === null || artifact === null + ? null + : ` — harness error: ${artifact.failure.message}`} + + + + + + +
+
+ + +
+ {codexAvailable ? null : ( +

+ Codex mode needs the local provider relay — provider relay not running, see tutorial §4. + Fake mode stays fully usable offline. +

+ )} + + + + {artifact === null ? null : } +
+
+ ); +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/components/scenario-picker.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/components/scenario-picker.tsx new file mode 100644 index 00000000000..07b74c3604b --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/components/scenario-picker.tsx @@ -0,0 +1,276 @@ +import * as React from "react"; + +import { Button } from "@paperclipai/paperclip-runner/react"; +import type { Phase7ScenarioIndexEntry } from "@paperclip-runner-local/phase7"; + +import { DISPOSITION_LABEL, groupLabel, PARITY_LABEL, ROLE_LABEL } from "../labels.js"; +import { activeFilterChips, hasActiveFilters, type Phase7Filters } from "../route.js"; +import { DispositionBadge, EmptyState, Mono, ParityStatusBadge } from "./primitives.js"; + +/** + * Scenario picker — search, facets, and a single-select listbox over the + * generated scenario index. The picker consumes the index; it never re-derives + * group membership or dispositions (UX map §2). + */ + +export interface Phase7FacetValue { + value: string; + label: string; + count: number; +} + +export interface Phase7PickerFacets { + group: Phase7FacetValue[]; + disposition: Phase7FacetValue[]; + role: Phase7FacetValue[]; + claim: Phase7FacetValue[]; + parity: Phase7FacetValue[]; +} + +export interface ScenarioPickerProps { + entries: readonly Phase7ScenarioIndexEntry[]; + visible: readonly Phase7ScenarioIndexEntry[]; + facets: Phase7PickerFacets; + filters: Phase7Filters; + selectedId: string | null; + parityStatus: (id: string) => "pass" | "fail" | "intentional_gap" | "not_run"; + onFilterChange: (next: Partial) => void; + onClearFilters: () => void; + onSelect: (id: string) => void; +} + +const FACET_ORDER: Array<{ key: keyof Phase7PickerFacets; label: string }> = [ + { key: "group", label: "Group" }, + { key: "disposition", label: "Disposition" }, + { key: "role", label: "Role" }, + { key: "claim", label: "Claim" }, + { key: "parity", label: "Parity status" }, +]; + +export function ScenarioPicker({ + entries, + visible, + facets, + filters, + selectedId, + parityStatus, + onFilterChange, + onClearFilters, + onSelect, +}: ScenarioPickerProps) { + const searchId = React.useId(); + const listRef = React.useRef(null); + const chips = activeFilterChips(filters); + const grouped = groupEntries(visible); + + function move(delta: number) { + if (visible.length === 0) return; + const current = visible.findIndex((entry) => entry.id === selectedId); + const next = visible[Math.min(Math.max(current + delta, 0), visible.length - 1)] ?? visible[0]; + if (next !== undefined) onSelect(next.id); + } + + function onKeyDown(event: React.KeyboardEvent) { + if (event.key === "ArrowDown") { + event.preventDefault(); + move(1); + } else if (event.key === "ArrowUp") { + event.preventDefault(); + move(-1); + } else if (event.key === "Home") { + event.preventDefault(); + if (visible[0] !== undefined) onSelect(visible[0].id); + } else if (event.key === "End") { + event.preventDefault(); + const last = visible[visible.length - 1]; + if (last !== undefined) onSelect(last.id); + } + } + + return ( + + ); +} + +function groupEntries( + entries: readonly Phase7ScenarioIndexEntry[], +): Array<[string, Phase7ScenarioIndexEntry[]]> { + const byGroup = new Map(); + for (const entry of entries) { + const bucket = byGroup.get(entry.group) ?? []; + bucket.push(entry); + byGroup.set(entry.group, bucket); + } + return [...byGroup.entries()].sort(([left], [right]) => left.localeCompare(right)); +} + +/** + * Facet counts are computed against the results of every *other* facet, so a + * selected value still shows how many scenarios it would return and unselected + * values show what selecting them would do (recognition over recall). + */ +export function buildFacets( + entries: readonly Phase7ScenarioIndexEntry[], + countFor: (key: keyof Phase7PickerFacets, value: string) => number, +): Phase7PickerFacets { + const uniq = (values: string[]): string[] => [...new Set(values)].sort(); + return { + group: uniq(entries.map((entry) => entry.group)).map((group) => ({ + value: group, + label: `${groupLabel(group)} (${group})`, + count: countFor("group", group), + })), + disposition: ( + ["control_plane_owned", "always_agent_tool", "optional_agent_tool"] as const + ).map((disposition) => ({ + value: disposition, + label: DISPOSITION_LABEL[disposition], + count: countFor("disposition", disposition), + })), + role: uniq(entries.map((entry) => entry.actorRole)).map((role) => ({ + value: role, + label: ROLE_LABEL[role] ?? role, + count: countFor("role", role), + })), + claim: uniq(entries.flatMap((entry) => entry.requiredGrants)).map((claim) => ({ + value: claim, + label: claim, + count: countFor("claim", claim), + })), + parity: (["pass", "fail", "intentional_gap", "not_run"] as const).map((status) => ({ + value: status, + label: PARITY_LABEL[status], + count: countFor("parity", status), + })), + }; +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/components/scenario-transcript.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/components/scenario-transcript.tsx new file mode 100644 index 00000000000..ae871929347 --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/components/scenario-transcript.tsx @@ -0,0 +1,200 @@ +import * as React from "react"; + +import { Badge } from "@paperclipai/paperclip-runner/react"; +import type { Phase7TimelineEntry } from "@paperclip-runner-local/phase7"; + +import { DispositionBadge, JsonBlock, Mono, RedactionChip } from "./primitives.js"; + +/** + * Scenario transcript — two visually distinct channels over one ordered + * artifact timeline. Items render in artifact order; the view never reorders, + * merges, or re-times an entry (UX map §3.2). + */ + +export function ScenarioTranscript({ + timeline, + highlightSequence, + onSelectSequence, +}: { + timeline: readonly Phase7TimelineEntry[]; + highlightSequence: number | null; + onSelectSequence?: (sequence: number) => void; +}) { + return ( +
    + {timeline.map((entry) => ( +
  1. onSelectSequence(entry.sequence)} + > + +
  2. + ))} +
+ ); +} + +function TimelineEntryBody({ entry }: { entry: Phase7TimelineEntry }) { + switch (entry.kind) { + case "agent_message": + return ( +
+ +

{entry.text}

+
+ ); + + case "semantic_call": + return ( +
+ + + {entry.operationId} — {entry.summary} + + + + {entry.escapeHatch ? ( +

+ Test-only escape hatch — this operation exists for skill-test scenarios and is not part + of the production capability set. +

+ ) : null} +
+
Operation
+
+ {entry.operationId} v1 +
+
Required claims
+
+ {entry.requiredClaims.length === 0 ? ( + none — task-scoped always tool + ) : ( + entry.requiredClaims.map((claim) => {claim}) + )} +
+
Idempotency
+
+ {entry.idempotency} + {entry.idempotencyKey === null ? null : ( + <> + {" "} + · {entry.idempotencyKey} + + )} +
+
Authorization record
+
+ #{entry.authorizationSequence} +
+
+ {entry.redactions.length === 0 ? null : ( +

+ {entry.redactions.map((chip) => ( + + ))} +

+ )} + +
+ ); + + case "semantic_result": { + const denied = entry.outcome === "denied" || entry.outcome === "absent"; + return ( +
+ + {entry.summary} + + {denied ? (entry.outcome === "absent" ? "Absent" : "Denied") : "Committed"} + + + {entry.failure === null ? null : ( +
+
Denial code
+
+ {entry.failure.code} +
+
Reason
+
+ {entry.failure.reason} +
+
Protected state
+
not returned — the denial carries no task data
+
+ )} + {entry.redactions.length === 0 ? null : ( +

+ {entry.redactions.map((chip) => ( + + ))} +

+ )} + {entry.value === null ? null : } + {entry.protocolEventRefs.length === 0 ? null : ( +
+ Protocol events ({entry.protocolEventRefs.length}) +
{entry.protocolEventRefs.join("\n")}
+
+ )} +
+ ); + } + + case "control_plane_action": + return ( +
+ + + + {entry.summary} + + Control plane + +
+
Action
+
+ {entry.action} +
+
State revision
+
{entry.stateRevision}
+
Audit reference
+
{entry.auditRef === null ? none : {entry.auditRef}}
+
Decision reference
+
+ {entry.decisionRef === null ? ( + none + ) : ( + {entry.decisionRef} + )} +
+
+ +
+ ); + + case "system_note": + return ( +

+ {entry.note} +

+ ); + } +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/env.d.ts b/packages/paperclip-runner/examples/phase7-explorer/src/env.d.ts new file mode 100644 index 00000000000..61dc4efbd5f --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/env.d.ts @@ -0,0 +1,18 @@ +/// + +/** + * Declared build-time inputs for the explorer, supplied by + * `scripts/phase7-explorer-fixtures.mjs`. The explorer is an example consumer + * and reaches package or spec files only through these entry points. + */ +declare module "virtual:phase7-scenario-manifest" { + /** The checked-in Phase 7A traceability derivative, verbatim. */ + const manifestSource: string; + export default manifestSource; +} + +declare module "virtual:phase7-eval-report" { + /** Phase 7E's parity report, or `null` when it has not been generated. */ + const evalReport: unknown; + export default evalReport; +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/explorer.css b/packages/paperclip-runner/examples/phase7-explorer/src/explorer.css new file mode 100644 index 00000000000..93466aa45a4 --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/explorer.css @@ -0,0 +1,616 @@ +/* + * Phase 7 scenario explorer — explorer-local layout only. + * + * Every value resolves through a --pcr-* token from the SDK sheet; no hex, no + * raw pixels, no font stacks are declared here (DESIGN.md principles 2-4). + * Hierarchy comes from surface and position, not decoration. + */ + +body { + margin: 0; +} + +.pcr7-shell { + display: grid; + gap: var(--pcr-space-4); + padding: var(--pcr-space-4); + max-width: var(--pcr-page-width); + margin-inline: auto; +} + +.pcr7-columns { + display: grid; + grid-template-columns: var(--pcr-rail-width) minmax(0, 1fr) var(--pcr-inspector-width); + gap: var(--pcr-space-4); + align-items: start; +} + +.pcr7-rail, +.pcr7-center, +.pcr7-inspector { + min-width: 0; + display: grid; + gap: var(--pcr-space-3); + align-content: start; +} + +/* + * The rails scroll with the page rather than in their own overflow box: the + * acceptance screenshots are full-page captures, and an inner scroll container + * would clip the very evidence the matrix asks to see (all 16 group facets, + * the full exposure list, every diff domain). + */ +.pcr7-rail, +.pcr7-inspector { + padding: var(--pcr-space-3); + background: var(--pcr-card); + border: 0.0625rem solid var(--pcr-border); + border-radius: var(--pcr-radius-md); +} + +.pcr7-rail-title, +.pcr7-inspector h3 { + font-size: var(--pcr-font-size-sm); + font-weight: var(--pcr-weight-semibold); + margin: 0; +} + +.pcr7-muted { + color: var(--pcr-muted-foreground); + font-size: var(--pcr-font-size-xs); +} + +.pcr7-label { + color: var(--pcr-muted-foreground); + font-size: var(--pcr-font-size-xs); + font-weight: var(--pcr-weight-semibold); + letter-spacing: 0.06em; + text-transform: uppercase; + margin: var(--pcr-space-2) 0 var(--pcr-space-1); +} + +.pcr7-block { + display: block; +} + +.pcr7-mono { + font-family: var(--pcr-font-mono); + font-size: var(--pcr-font-size-xs); + overflow-wrap: anywhere; +} + +/* Picker ------------------------------------------------------------ */ + +.pcr7-field { + display: grid; + gap: var(--pcr-space-1); +} + +.pcr7-input { + min-height: var(--pcr-touch-target); + padding: var(--pcr-space-2) var(--pcr-space-3); + border: 0.0625rem solid var(--pcr-border); + border-radius: var(--pcr-radius-sm); + background: var(--pcr-background); + color: var(--pcr-foreground); +} + +.pcr7-input:focus-visible, +.pcr7-facet-value:focus-visible, +.pcr7-case-list:focus-visible, +.pcr7-chip:focus-visible, +.pcr7-linkish:focus-visible, +.pcr7-segmented button:focus-visible { + outline: 0.125rem solid var(--pcr-ring); + outline-offset: 0.125rem; +} + +.pcr7-chip-row { + display: flex; + flex-wrap: wrap; + gap: var(--pcr-space-1); +} + +.pcr7-chip { + display: inline-flex; + align-items: center; + gap: var(--pcr-space-1); + min-height: var(--pcr-touch-target); + padding: var(--pcr-space-1) var(--pcr-space-2); + border: 0.0625rem solid var(--pcr-border); + border-radius: var(--pcr-radius-sm); + background: var(--pcr-accent-surface); + color: var(--pcr-foreground); + cursor: pointer; +} + +.pcr7-facet { + border-top: 0.0625rem solid var(--pcr-border); + padding-top: var(--pcr-space-2); +} + +.pcr7-facet-summary { + cursor: pointer; + font-size: var(--pcr-font-size-xs); + font-weight: var(--pcr-weight-semibold); + min-height: var(--pcr-touch-target); + display: flex; + align-items: center; +} + +.pcr7-facet-values { + display: grid; + gap: var(--pcr-space-1); +} + +.pcr7-facet-value { + display: flex; + justify-content: space-between; + gap: var(--pcr-space-2); + align-items: center; + min-height: var(--pcr-touch-target); + padding: var(--pcr-space-1) var(--pcr-space-2); + border: 0; + border-radius: var(--pcr-radius-sm); + background: transparent; + color: var(--pcr-foreground); + text-align: left; + cursor: pointer; +} + +.pcr7-facet-value[aria-pressed="true"] { + background: var(--pcr-accent-surface); + font-weight: var(--pcr-weight-semibold); +} + +.pcr7-facet-value:disabled { + color: var(--pcr-muted-foreground); + cursor: not-allowed; +} + +.pcr7-facet-count { + color: var(--pcr-muted-foreground); + font-family: var(--pcr-font-mono); + font-size: var(--pcr-font-size-xs); +} + +/* + * The case list is the one genuinely long thing in the rail (106 rows), so it + * is the one thing that scrolls in its own box. Facets stay in page flow, so a + * full-page capture still shows all 16 group values and their counts. + */ +.pcr7-case-list { + display: grid; + gap: var(--pcr-space-2); + max-height: calc(var(--pcr-facet-height) * 6); + overflow-y: auto; +} + +.pcr7-case-group-title { + position: sticky; + top: 0; + display: flex; + align-items: center; + gap: var(--pcr-space-2); + margin: 0; + padding: var(--pcr-space-1) 0; + background: var(--pcr-card); + font-size: var(--pcr-font-size-xs); + font-weight: var(--pcr-weight-semibold); + z-index: var(--pcr-z-banner); +} + +.pcr7-case-row { + display: grid; + gap: var(--pcr-space-1); + padding: var(--pcr-space-2); + border-left: 0.125rem solid transparent; + border-radius: var(--pcr-radius-sm); + cursor: pointer; + min-height: var(--pcr-touch-target); +} + +.pcr7-case-row[data-selected="true"] { + background: var(--pcr-accent-surface); + border-left-color: var(--pcr-accent); +} + +.pcr7-case-title { + font-size: var(--pcr-font-size-xs); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; +} + +.pcr7-case-meta { + display: flex; + flex-wrap: wrap; + gap: var(--pcr-space-1); +} + +/* Run view ---------------------------------------------------------- */ + +.pcr7-center { + max-width: var(--pcr-center-width); +} + +.pcr7-run-header, +.pcr7-intro { + display: grid; + gap: var(--pcr-space-3); + padding: var(--pcr-space-4); + background: var(--pcr-card); + border: 0.0625rem solid var(--pcr-border); + border-radius: var(--pcr-radius-md); +} + +.pcr7-run-title h1 { + font-size: var(--pcr-font-size-lg); + line-height: var(--pcr-line-tight); +} + +.pcr7-badge-row, +.pcr7-redaction-row, +.pcr7-assertion-head { + display: flex; + flex-wrap: wrap; + gap: var(--pcr-space-1); + align-items: center; +} + +.pcr7-run-controls { + display: grid; + gap: var(--pcr-space-2); +} + +.pcr7-segmented { + display: flex; + gap: var(--pcr-space-1); + padding: var(--pcr-space-1); + background: var(--pcr-muted); + border-radius: var(--pcr-radius-sm); + width: fit-content; + max-width: 100%; + flex-wrap: wrap; +} + +.pcr7-segmented button { + min-height: var(--pcr-touch-target); + padding: var(--pcr-space-1) var(--pcr-space-3); + border: 0; + border-radius: var(--pcr-radius-sm); + background: transparent; + color: var(--pcr-foreground); + cursor: pointer; +} + +.pcr7-segmented button[aria-selected="true"], +.pcr7-segmented button[aria-checked="true"] { + background: var(--pcr-card); + font-weight: var(--pcr-weight-semibold); + box-shadow: var(--pcr-shadow-sm); +} + +.pcr7-segmented button:disabled { + color: var(--pcr-muted-foreground); + cursor: not-allowed; +} + +.pcr7-segmented--mobile { + display: none; +} + +/* Transcript -------------------------------------------------------- */ + +.pcr7-transcript { + list-style: none; + display: grid; + gap: var(--pcr-space-2); + margin: 0; + padding: 0; +} + +.pcr7-transcript-item { + min-width: 0; +} + +.pcr7-transcript-item[data-channel="agent"] { + margin-left: var(--pcr-timeline-column); +} + +.pcr7-transcript-item[data-highlighted="true"] > * { + outline: 0.125rem solid var(--pcr-accent); + outline-offset: 0.125rem; +} + +.pcr7-agent-message { + display: flex; + gap: var(--pcr-space-2); + padding: var(--pcr-space-3); + background: var(--pcr-card); + border: 0.0625rem solid var(--pcr-border); + border-radius: var(--pcr-radius-md); +} + +.pcr7-gutter { + color: var(--pcr-muted-foreground); + font-family: var(--pcr-font-mono); +} + +.pcr7-call, +.pcr7-result { + background: var(--pcr-card); +} + +/* + * The SDK gives every direct span child of a disclosure summary `min-width: 0` + * so long tool names can wrap. A Badge is also a span, and it sets + * `white-space: nowrap`, so at 390px it was being squeezed below its own + * content and pushing the page into horizontal scroll. Badges keep their + * intrinsic width here; only the descriptive text shrinks. + */ +.pcr7-shell .pcr-disclosure-summary { + flex-wrap: wrap; +} + +.pcr7-shell .pcr-disclosure-summary > [data-slot="badge"] { + flex: none; +} + +/* + * The SDK badge capitalises every word and collapses the whitespace between + * its flex children. The interaction map §0 fixes the exact label strings + * ("Optional tool", "Not run") and requires the status icon to read as a + * separate token from its text, so both are corrected here rather than in the + * frozen SDK sheet. + */ +.pcr7-shell .pcr-badge { + gap: var(--pcr-space-1); + text-transform: none; +} + +.pcr7-result--denied { + background: var(--pcr-danger-surface); + border-color: var(--pcr-danger); +} + +.pcr7-control-plane { + background: var(--pcr-muted); + border-color: var(--pcr-border); +} + +.pcr7-escape-hatch { + padding: var(--pcr-space-2); + background: var(--pcr-warning-surface); + color: var(--pcr-warning); + border-radius: var(--pcr-radius-sm); + font-weight: var(--pcr-weight-medium); +} + +.pcr7-system-note { + padding: var(--pcr-space-3); + background: var(--pcr-surface-raised); + border: 0.0625rem dashed var(--pcr-border); + border-radius: var(--pcr-radius-md); + color: var(--pcr-muted-foreground); +} + +.pcr7-detail-list { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 2fr); + gap: var(--pcr-space-1) var(--pcr-space-3); + margin: var(--pcr-space-2) 0; +} + +.pcr7-detail-list dd { + margin: 0; + min-width: 0; + font-size: var(--pcr-font-size-xs); +} + +.pcr7-json { + max-height: var(--pcr-transcript-height); + overflow: auto; +} + +.pcr7-redaction { + display: inline-flex; + align-items: center; + gap: var(--pcr-space-1); + padding: var(--pcr-space-1) var(--pcr-space-2); + background: var(--pcr-warning-surface); + color: var(--pcr-warning); + border-radius: var(--pcr-radius-sm); + font-size: var(--pcr-font-size-xs); +} + +/* Inspector --------------------------------------------------------- */ + +.pcr7-panel { + display: grid; + gap: var(--pcr-space-3); + min-width: 0; +} + +.pcr7-panel-section { + display: grid; + gap: var(--pcr-space-1); + padding-bottom: var(--pcr-space-2); + border-bottom: 0.0625rem solid var(--pcr-border); +} + +.pcr7-chips { + display: inline-flex; + flex-wrap: wrap; + gap: var(--pcr-space-1); +} + +.pcr7-exposure, +.pcr7-assertions, +.pcr7-diff-rows, +.pcr7-diff-fields { + list-style: none; + display: grid; + gap: var(--pcr-space-1); + margin: 0; + padding: 0; + font-size: var(--pcr-font-size-xs); +} + +.pcr7-assertions > li { + display: grid; + gap: var(--pcr-space-1); + padding: var(--pcr-space-2); + background: var(--pcr-surface-raised); + border-radius: var(--pcr-radius-sm); +} + +.pcr7-expected-observed { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--pcr-space-2); +} + +.pcr7-table { + width: 100%; + border-collapse: collapse; + font-size: var(--pcr-font-size-xs); +} + +.pcr7-table th, +.pcr7-table td { + text-align: left; + vertical-align: top; + padding: var(--pcr-space-2); + border-bottom: 0.0625rem solid var(--pcr-border); +} + +.pcr7-table tbody tr[data-outcome="denied"], +.pcr7-table tbody tr[data-outcome="absent"] { + background: var(--pcr-danger-surface); +} + +.pcr7-missing { + color: var(--pcr-danger); +} + +.pcr7-diff-rows > li { + display: grid; + gap: var(--pcr-space-1); + padding: var(--pcr-space-2); + background: var(--pcr-surface-raised); + border-radius: var(--pcr-radius-sm); +} + +.pcr7-diff-marker { + font-weight: var(--pcr-weight-semibold); +} + +.pcr7-diff-fields { + padding-left: var(--pcr-space-3); + color: var(--pcr-muted-foreground); +} + +.pcr7-empty { + display: grid; + gap: var(--pcr-space-2); + padding: var(--pcr-space-4); + background: var(--pcr-surface-raised); + border: 0.0625rem dashed var(--pcr-border); + border-radius: var(--pcr-radius-md); + justify-items: start; +} + +.pcr7-empty-title { + font-weight: var(--pcr-weight-semibold); +} + +.pcr7-linkish { + border: 0; + background: transparent; + color: var(--pcr-accent); + cursor: pointer; + padding: var(--pcr-space-1) 0; + text-decoration: underline; +} + +.pcr7-parity-verdict { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--pcr-space-2); +} + +/* Accessibility helpers --------------------------------------------- */ + +.pcr7-visually-hidden { + position: absolute; + width: 0.0625rem; + height: 0.0625rem; + padding: 0; + margin: calc(-1 * 0.0625rem); + overflow: hidden; + clip-path: inset(50%); + white-space: nowrap; + border: 0; +} + +.pcr7-skip-link { + position: absolute; + left: calc(-1 * var(--pcr-page-width)); + background: var(--pcr-card); + padding: var(--pcr-space-2); + border-radius: var(--pcr-radius-sm); +} + +.pcr7-skip-link:focus { + position: static; +} + +/* Responsive: one content-driven breakpoint (UX map §6) -------------- */ + +@media (max-width: 64rem) { + .pcr7-columns { + grid-template-columns: minmax(0, 1fr); + } + + .pcr7-segmented--mobile { + display: flex; + position: sticky; + top: 0; + width: 100%; + z-index: var(--pcr-z-banner); + } + + .pcr7-rail, + .pcr7-center, + .pcr7-inspector { + display: none; + } + + .pcr7-shell[data-segment="scenarios"] .pcr7-rail, + .pcr7-shell[data-segment="run"] .pcr7-center, + .pcr7-shell[data-segment="inspect"] .pcr7-inspector { + display: grid; + } + + .pcr7-expected-observed { + grid-template-columns: minmax(0, 1fr); + } + + .pcr7-detail-list { + grid-template-columns: minmax(0, 1fr); + } +} + +@media (prefers-reduced-motion: reduce) { + .pcr-root *, + .pcr-root *::before, + .pcr-root *::after { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + scroll-behavior: auto !important; + } +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/explorer.test.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/explorer.test.tsx new file mode 100644 index 00000000000..a108396f732 --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/explorer.test.tsx @@ -0,0 +1,373 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeAll, describe, expect, it } from "vitest"; + +import { + buildPhase7ScenarioIndex, + phase7RunScenario, + type Phase7RunArtifact, + type Phase7ScenarioIndex, + type Phase7ScenarioIndexEntry, +} from "@paperclip-runner-local/phase7"; + +import { + AuthorizationPanel, + ContextPanel, + denialCount, + ParityPanel, + StateDiffPanel, +} from "./components/inspector-panels.js"; +import { ScenarioTranscript } from "./components/scenario-transcript.js"; +import { RunHeader } from "./components/run-header.js"; +import { ScenarioPicker, buildFacets } from "./components/scenario-picker.js"; +import { matches } from "./app.js"; +import { EMPTY_FILTERS } from "./route.js"; + +const MANIFEST_PATH = resolve(import.meta.dirname, "../../../spec/phase-07/eval-traceability.yaml"); + +let index: Phase7ScenarioIndex; +const artifacts = new Map(); + +beforeAll(async () => { + index = buildPhase7ScenarioIndex(await readFile(MANIFEST_PATH, "utf8")); + for (const id of [ + "hb-scoped-wake-01", + "ap-mcp-gate-01", + "rs-question-only-01", + "rs-secret-hygiene-01", + "rf-api-mgr-heartbeat-01", + "bl-create-blocked-01", + "wk-skilltest-mode-01", + ]) { + artifacts.set(id, await phase7RunScenario(entryFor(id))); + } +}); + +function entryFor(id: string): Phase7ScenarioIndexEntry { + const entry = index.entries.find((candidate) => candidate.id === id); + if (entry === undefined) throw new Error(`scenario ${id} is missing`); + return entry; +} + +/** Returns the opening tag for the element carrying `data-testid`. */ +function tagFor(markup: string, testId: string): string { + const index = markup.indexOf(`data-testid="${testId}"`); + expect(index, `no element with data-testid="${testId}"`).toBeGreaterThan(-1); + const start = markup.lastIndexOf("<", index); + return markup.slice(start, markup.indexOf(">", index) + 1); +} + +function artifactFor(id: string): Phase7RunArtifact { + const artifact = artifacts.get(id); + if (artifact === undefined) throw new Error(`artifact ${id} was not prepared`); + return artifact; +} + +const NOOP_REPLAY = { + active: true, + position: 0, + length: 0, + playing: false, + enter: () => {}, + exit: () => {}, + setPosition: () => {}, + togglePlay: () => {}, + step: () => {}, +}; + +describe("scenario picker", () => { + it("renders a labelled listbox with every group and a live result count", () => { + const parityStatus = () => "not_run" as const; + const visible = index.entries.filter((entry) => matches(entry, EMPTY_FILTERS, parityStatus)); + const facets = buildFacets(index.entries, (key, value) => + index.entries.filter((entry) => matches(entry, { ...EMPTY_FILTERS, [key]: value }, parityStatus)) + .length, + ); + const markup = renderToStaticMarkup( + {}} + onClearFilters={() => {}} + onSelect={() => {}} + />, + ); + expect(markup).toContain('role="listbox"'); + expect(markup).toContain('aria-label="Scenario picker"'); + expect(markup).toContain("106 of 106 scenarios"); + expect(facets.group).toHaveLength(16); + expect(facets.group.reduce((total, facet) => total + facet.count, 0)).toBe(106); + for (const facet of facets.group) { + expect(markup).toContain(`data-testid="facet-group-${facet.value}"`); + } + }); + + it("keeps zero-count facet values visible but disabled", () => { + const parityStatus = () => "not_run" as const; + const filters = { ...EMPTY_FILTERS, group: "ix" }; + const visible = index.entries.filter((entry) => matches(entry, filters, parityStatus)); + const facets = buildFacets(index.entries, (key, value) => + index.entries.filter((entry) => matches(entry, { ...filters, [key]: value }, parityStatus)).length, + ); + const markup = renderToStaticMarkup( + {}} + onClearFilters={() => {}} + onSelect={() => {}} + />, + ); + expect(markup).toContain('data-testid="clear-filters"'); + expect(markup).toContain('data-testid="active-filter-group"'); + // Parity is "not run" for every scenario here, so Pass must render disabled. + expect(tagFor(markup, "facet-parity-pass")).toContain("disabled"); + }); + + it("offers a way out when no scenario matches", () => { + const markup = renderToStaticMarkup( + 0)} + filters={{ ...EMPTY_FILTERS, query: "no-such-case" }} + selectedId={null} + parityStatus={() => "not_run"} + onFilterChange={() => {}} + onClearFilters={() => {}} + onSelect={() => {}} + />, + ); + expect(markup).toContain("No scenarios match these filters."); + expect(markup).toContain("Clear filters"); + }); +}); + +describe("scenario transcript", () => { + it("separates the agent and control-plane channels", () => { + const artifact = artifactFor("hb-scoped-wake-01"); + const markup = renderToStaticMarkup( + , + ); + expect(markup).toContain('data-channel="control_plane"'); + expect(markup).toContain('data-channel="agent"'); + expect(markup).toContain("Control plane"); + expect(markup).toContain("no agent tool exists for this"); + expect(markup).toContain("get_task_context"); + }); + + it("renders a typed denial in place, with no protected state", () => { + const artifact = artifactFor("ap-mcp-gate-01"); + const markup = renderToStaticMarkup( + , + ); + expect(markup).toContain('data-outcome="denied"'); + expect(markup).toContain("required_claim_missing"); + expect(markup).toContain("not returned — the denial carries no task data"); + }); + + it("keeps restraint legible when the correct behaviour is absence", () => { + const artifact = artifactFor("rs-question-only-01"); + const markup = renderToStaticMarkup( + , + ); + expect(markup).toContain('data-testid="restraint-note"'); + expect(markup).toContain("No further operations"); + }); + + it("renders a redaction chip that names its rule and never the value", () => { + const artifact = artifactFor("rs-secret-hygiene-01"); + const markup = renderToStaticMarkup( + , + ); + expect(markup).toContain('data-testid="redaction-chip"'); + expect(markup).toContain("read_secret_value:$.value"); + expect(markup).not.toContain("fixture-secret-value-not-a-real-credential"); + }); + + it("flags the test-only escape hatch on every call card that uses it", () => { + const artifact = artifactFor("wk-skilltest-mode-01"); + expect( + artifact.timeline.some((item) => item.kind === "semantic_call" && item.escapeHatch), + "the skill-test scenario must exercise the generic escape hatch", + ).toBe(true); + const markup = renderToStaticMarkup( + , + ); + expect(markup).toContain('data-testid="escape-hatch-banner"'); + }); +}); + +describe("inspector panels", () => { + it("context lists always tools, optional tools with their grant, and control-plane absence", () => { + const artifact = artifactFor("rf-api-mgr-heartbeat-01"); + const markup = renderToStaticMarkup( + , + ); + expect(markup).toContain('data-testid="exposure-always"'); + expect(markup).toContain('data-testid="exposure-optional"'); + expect(markup).toContain('data-testid="exposure-control-plane"'); + expect(markup).toContain("via { + const markup = renderToStaticMarkup( + , + ); + expect(markup).toContain("No run yet."); + expect(markup).not.toContain('data-testid="exposure-always"'); + }); + + it("authorization renders one row per record with an icon and a text decision", () => { + const artifact = artifactFor("ap-mcp-gate-01"); + const markup = renderToStaticMarkup( + , + ); + expect(denialCount(artifact.authorizationRecords)).toBeGreaterThan(0); + expect(markup).toContain('data-outcome="denied"'); + expect(markup).toContain("Deny"); + expect(markup).toContain("Allow"); + for (const record of artifact.authorizationRecords) { + expect(markup).toContain(`data-testid="authorization-row-${record.sequence}"`); + } + }); + + it("state diff groups the ten entity domains and collapses unchanged ones", () => { + const artifact = artifactFor("bl-create-blocked-01"); + const markup = renderToStaticMarkup(); + for (const domain of [ + "tasks", + "comments", + "documents", + "interactions", + "approvals", + "artifacts", + "blockers", + "workspace", + "budget", + "run", + ]) { + expect(markup).toContain(`data-testid="diff-domain-${domain}"`); + } + expect(markup).toContain("unchanged"); + expect(markup).toContain('data-change="added"'); + expect(markup).toContain("Added"); + }); + + it("parity renders the verdict, assertions, forbidden list, and the 7E carry-through", () => { + const artifact = artifactFor("rs-question-only-01"); + const markup = renderToStaticMarkup(); + expect(markup).toContain('data-testid="parity-verdict"'); + expect(markup).toContain('data-testid="parity-assertion-1"'); + expect(markup).toContain('data-testid="parity-forbidden"'); + expect(markup).toContain("Not run"); + expect(markup).toContain("never recomputes it"); + expect(markup).toContain("Reference baseline, not the pass condition"); + }); + + it("every panel renders a designed empty state rather than a dead panel", () => { + expect(renderToStaticMarkup()).toContain("No run yet."); + expect(renderToStaticMarkup()).toContain("No run yet."); + expect(renderToStaticMarkup()).toContain("No run yet."); + }); +}); + +describe("run header", () => { + it("names the verdict, the assertion counts, and offers a parity deep link", () => { + const artifact = artifactFor("hb-scoped-wake-01"); + const markup = renderToStaticMarkup( + {}} + onModeChange={() => {}} + onOpenParity={() => {}} + />, + ); + expect(markup).toContain('data-testid="verdict-banner"'); + expect(markup).toContain("assertions"); + expect(markup).toContain("Open parity"); + expect(markup).toContain('data-testid="replay-controls"'); + }); + + it("disables Codex mode with a stated reason when the relay is unavailable", () => { + const markup = renderToStaticMarkup( + {}} + onModeChange={() => {}} + onOpenParity={() => {}} + />, + ); + expect(tagFor(markup, "mode-codex")).toContain("disabled"); + expect(markup).toContain("provider relay not running"); + expect(markup).toContain("Fake mode stays fully usable offline"); + }); + + it("reports a harness failure without discarding the partial timeline", () => { + const artifact = artifactFor("hb-scoped-wake-01"); + const markup = renderToStaticMarkup( + {}} + onModeChange={() => {}} + onOpenParity={() => {}} + />, + ); + expect(markup).toContain("harness error: fixture is missing"); + expect(markup).toContain("Re-run scenario"); + }); +}); + +describe("credential and policy boundary", () => { + it("renders no credential, token, or API key anywhere in a run artifact", () => { + for (const artifact of artifacts.values()) { + const serialized = JSON.stringify(artifact).toLowerCase(); + for (const forbidden of [ + "authorization: bearer", + "api_key=", + "paperclip_api_key", + "sk-ant-", + "fixture-secret-value-not-a-real-credential", + ]) { + expect(serialized, `${artifact.scenarioId} leaked ${forbidden}`).not.toContain(forbidden); + } + } + }); + + it("never re-derives a disposition in the view layer", () => { + for (const artifact of artifacts.values()) { + for (const item of artifact.timeline) { + if (item.kind !== "semantic_call") continue; + expect(["always_agent_tool", "optional_agent_tool"]).toContain(item.disposition); + } + } + }); +}); diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/labels.ts b/packages/paperclip-runner/examples/phase7-explorer/src/labels.ts new file mode 100644 index 00000000000..e75aa674927 --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/labels.ts @@ -0,0 +1,124 @@ +import type { + Phase7AssertionClass, + Phase7DiffChangeKind, + Phase7ParityStatus, + Phase7PrimaryDisposition, +} from "@paperclip-runner-local/phase7"; + +/** + * The canonical label vocabulary from the interaction map §0. Every badge, + * facet value, and status string in the explorer resolves through this module, + * so the vocabulary stays systematic and no view invents its own wording. + */ + +export const DISPOSITION_LABEL: Record = { + control_plane_owned: "Control plane", + always_agent_tool: "Always tool", + optional_agent_tool: "Optional tool", +}; + +export const DISPOSITION_TONE: Record = { + control_plane_owned: "neutral", + always_agent_tool: "success", + optional_agent_tool: "accent", +}; + +export const ASSERTION_CLASS_LABEL: Record = { + control_plane_invariant: "Invariant", + agent_tool_contract: "Tool contract", + authorization_policy: "Authorization", + combined_multi_hop: "Multi-hop", + restraint_no_call: "Restraint", +}; + +export const PARITY_LABEL: Record = { + pass: "Pass", + fail: "Fail", + intentional_gap: "Intentional gap", + not_run: "Not run", +}; + +/** Status is never encoded by colour alone (WCAG colour-independence). */ +export const PARITY_ICON: Record = { + pass: "✓", + fail: "✕", + intentional_gap: "◑", + not_run: "○", +}; + +export const PARITY_TONE: Record = { + pass: "success", + fail: "danger", + intentional_gap: "warning", + not_run: "neutral", +}; + +export const CHANGE_LABEL: Record = { + added: "Added", + changed: "Changed", + removed: "Removed", +}; + +export const CHANGE_ICON: Record = { + added: "+", + changed: "~", + removed: "−", +}; + +export const ROLE_LABEL: Record = { + engineer: "Engineer", + manager: "Manager", + reviewer: "Reviewer", + board: "Board", +}; + +export const TASK_MODE_LABEL: Record = { + standard: "Standard", + ask: "Ask", + planning: "Planning", + skill_test: "Skill test", +}; + +export const GROUP_LABEL: Record = { + hb: "Heartbeat", + co: "Context", + st: "Status", + cm: "Comments", + se: "Search", + su: "Subtasks", + bl: "Blockers", + dp: "Documents", + ix: "Interactions", + ap: "Approvals", + ar: "Artifacts", + er: "Errors", + rf: "Reference surface", + mh: "Multi-hop", + rs: "Restraint", + wk: "Work modes", +}; + +export function groupLabel(group: string): string { + return GROUP_LABEL[group] ?? group; +} + +export function outcomeLabel(outcome: string): string { + switch (outcome) { + case "allowed": + return "Allow"; + case "denied": + return "Deny"; + case "absent": + return "Absent"; + case "exposed": + return "Exposed"; + default: + return outcome; + } +} + +export function outcomeTone(outcome: string): "success" | "danger" | "neutral" { + if (outcome === "allowed" || outcome === "exposed") return "success"; + if (outcome === "denied") return "danger"; + return "neutral"; +} diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/main.tsx b/packages/paperclip-runner/examples/phase7-explorer/src/main.tsx new file mode 100644 index 00000000000..d3d902443ab --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/main.tsx @@ -0,0 +1,33 @@ +import { createRoot } from "react-dom/client"; + +import { + buildPhase7ScenarioIndex, + phase7EvalSuiteLookup, +} from "@paperclip-runner-local/phase7"; + +import "@paperclipai/paperclip-runner/styles.css"; +import "./explorer.css"; + +import evalReport from "virtual:phase7-eval-report"; +import manifestSource from "virtual:phase7-scenario-manifest"; + +import { ExplorerApp } from "./app.js"; + +/** + * Package-local entry. The scenario index and every fixture are bundled from + * checked-in artifacts, so fake mode performs zero network I/O and holds no + * Paperclip or provider credential (UX map §1, §5). + */ + +const container = document.querySelector("#root"); +if (container === null) throw new Error("explorer root element is missing"); + +const index = buildPhase7ScenarioIndex(manifestSource); + +createRoot(container).render( + , +); diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/route.test.ts b/packages/paperclip-runner/examples/phase7-explorer/src/route.test.ts new file mode 100644 index 00000000000..714164ad21a --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/route.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; + +import { + activeFilterChips, + EMPTY_FILTERS, + hasActiveFilters, + parsePhase7Route, + serializePhase7Route, +} from "./route.js"; + +describe("phase 7 explorer routes", () => { + it("parses the explorer home", () => { + const route = parsePhase7Route("#/"); + expect(route.caseId).toBeNull(); + expect(route.run).toBeNull(); + expect(route.view).toBe("transcript"); + expect(hasActiveFilters(route.filters)).toBe(false); + }); + + it("parses a selected, unrun scenario", () => { + expect(parsePhase7Route("#/case/hb-scoped-wake-01")).toMatchObject({ + caseId: "hb-scoped-wake-01", + run: null, + view: "transcript", + }); + }); + + it("parses an auto-run screenshot route with an inspector view", () => { + expect(parsePhase7Route("#/case/ap-mcp-gate-01?run=fake&view=authorization")).toMatchObject({ + caseId: "ap-mcp-gate-01", + run: "fake", + view: "authorization", + }); + }); + + it("parses a filtered picker route", () => { + const route = parsePhase7Route("#/?group=ix&disposition=always_agent_tool&parity=pass&q=checkbox"); + expect(route.caseId).toBeNull(); + expect(route.filters).toEqual({ + group: "ix", + disposition: "always_agent_tool", + role: null, + claim: null, + parity: "pass", + query: "checkbox", + }); + expect(activeFilterChips(route.filters).map((chip) => chip.key)).toEqual([ + "group", + "disposition", + "parity", + "query", + ]); + }); + + it("rejects an unknown inspector view instead of rendering a dead tab", () => { + expect(parsePhase7Route("#/case/x?view=nonsense").view).toBe("transcript"); + expect(parsePhase7Route("#/case/x?run=live").run).toBeNull(); + }); + + it("round-trips every documented route shape", () => { + for (const hash of [ + "#/", + "#/case/hb-scoped-wake-01", + "#/case/hb-scoped-wake-01?run=fake", + "#/case/ap-mcp-gate-01?run=fake&view=authorization", + "#/?group=ix&disposition=always_agent_tool&parity=pass", + ]) { + expect(serializePhase7Route(parsePhase7Route(hash))).toBe(hash); + } + }); + + it("drops picker filters from a case route so a deep link stays stable", () => { + const route = parsePhase7Route("#/?group=ix"); + expect(serializePhase7Route({ ...route, caseId: "ix-checkbox-01" })).toBe("#/case/ix-checkbox-01"); + }); + + it("treats empty filters as no filters", () => { + expect(activeFilterChips(EMPTY_FILTERS)).toEqual([]); + }); +}); diff --git a/packages/paperclip-runner/examples/phase7-explorer/src/route.ts b/packages/paperclip-runner/examples/phase7-explorer/src/route.ts new file mode 100644 index 00000000000..1de7e88fec7 --- /dev/null +++ b/packages/paperclip-runner/examples/phase7-explorer/src/route.ts @@ -0,0 +1,119 @@ +/** + * Hash routing for the Phase 7 scenario explorer. + * + * Static hosting means no server rewrites, so every linkable view lives in the + * hash (UX map §7). Filter state mirrors into the route so any filtered picker + * view is shareable; it is never protocol state. + */ + +export type Phase7InspectorView = "transcript" | "context" | "authorization" | "diff" | "parity"; + +export const PHASE7_INSPECTOR_VIEWS: readonly Phase7InspectorView[] = [ + "transcript", + "context", + "authorization", + "diff", + "parity", +]; + +export interface Phase7Filters { + group: string | null; + disposition: string | null; + role: string | null; + claim: string | null; + parity: string | null; + query: string; +} + +export interface Phase7Route { + caseId: string | null; + run: "fake" | "codex" | null; + view: Phase7InspectorView; + filters: Phase7Filters; +} + +export const EMPTY_FILTERS: Phase7Filters = { + group: null, + disposition: null, + role: null, + claim: null, + parity: null, + query: "", +}; + +export function parsePhase7Route(hash: string): Phase7Route { + const raw = hash.startsWith("#") ? hash.slice(1) : hash; + const [pathPart = "", queryPart = ""] = raw.split("?", 2); + const params = new URLSearchParams(queryPart); + const segments = pathPart.split("/").filter((segment) => segment.length > 0); + const caseId = segments[0] === "case" && segments[1] !== undefined ? decodeURIComponent(segments[1]) : null; + const run = params.get("run"); + const view = params.get("view"); + return { + caseId, + run: run === "fake" || run === "codex" ? run : null, + view: isInspectorView(view) ? view : "transcript", + filters: { + group: params.get("group"), + disposition: params.get("disposition"), + role: params.get("role"), + claim: params.get("claim"), + parity: params.get("parity"), + query: params.get("q") ?? "", + }, + }; +} + +export function serializePhase7Route(route: Phase7Route): string { + const params = new URLSearchParams(); + if (route.run !== null) params.set("run", route.run); + if (route.view !== "transcript") params.set("view", route.view); + if (route.caseId === null) { + for (const [key, value] of [ + ["group", route.filters.group], + ["disposition", route.filters.disposition], + ["role", route.filters.role], + ["claim", route.filters.claim], + ["parity", route.filters.parity], + ] as const) { + if (value !== null && value.length > 0) params.set(key, value); + } + if (route.filters.query.length > 0) params.set("q", route.filters.query); + } + const path = route.caseId === null ? "/" : `/case/${encodeURIComponent(route.caseId)}`; + const query = params.toString(); + return `#${path}${query.length > 0 ? `?${query}` : ""}`; +} + +export function activeFilterChips( + filters: Phase7Filters, +): Array<{ key: keyof Phase7Filters; label: string; value: string }> { + const chips: Array<{ key: keyof Phase7Filters; label: string; value: string }> = []; + if (filters.group !== null && filters.group.length > 0) { + chips.push({ key: "group", label: "Group", value: filters.group }); + } + if (filters.disposition !== null && filters.disposition.length > 0) { + chips.push({ key: "disposition", label: "Disposition", value: filters.disposition }); + } + if (filters.role !== null && filters.role.length > 0) { + chips.push({ key: "role", label: "Role", value: filters.role }); + } + if (filters.claim !== null && filters.claim.length > 0) { + chips.push({ key: "claim", label: "Claim", value: filters.claim }); + } + if (filters.parity !== null && filters.parity.length > 0) { + chips.push({ key: "parity", label: "Parity", value: filters.parity }); + } + if (filters.query.length > 0) { + chips.push({ key: "query", label: "Search", value: filters.query }); + } + return chips; +} + +export function hasActiveFilters(filters: Phase7Filters): boolean { + return activeFilterChips(filters).length > 0; +} + +function isInspectorView(value: string | null): value is Phase7InspectorView { + return value !== null && (PHASE7_INSPECTOR_VIEWS as readonly string[]).includes(value); +} diff --git a/packages/paperclip-runner/examples/playwright.phase7.config.ts b/packages/paperclip-runner/examples/playwright.phase7.config.ts new file mode 100644 index 00000000000..4173bd9877f --- /dev/null +++ b/packages/paperclip-runner/examples/playwright.phase7.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: ".", + testMatch: "phase7-explorer.spec.ts", + fullyParallel: false, + workers: 1, + timeout: 120_000, + reporter: "line", + use: { + baseURL: "http://127.0.0.1:4193", + viewport: { width: 1440, height: 900 }, + }, + webServer: { + command: "pnpm exec vite --config vite.phase7.config.ts --host 127.0.0.1 --port 4193", + url: "http://127.0.0.1:4193/phase7-explorer/", + reuseExistingServer: false, + timeout: 60_000, + }, +}); diff --git a/packages/paperclip-runner/knowledge/evidence/2026-08-09-phase-07f-explorer-verification.md b/packages/paperclip-runner/knowledge/evidence/2026-08-09-phase-07f-explorer-verification.md new file mode 100644 index 00000000000..bb74a9cbb9c --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/2026-08-09-phase-07f-explorer-verification.md @@ -0,0 +1,104 @@ +--- +type: Verification Evidence +title: Phase 7F browser scenario explorer verification +description: Scenario index, run artifact, exposure, authorization, diff, parity, route determinism, accessibility, and screenshot evidence for the package-local Phase 7 explorer. +tags: [native-runner, phase-7, verification, browser, explorer, parity] +status: stable +generated: { by: anthropic/claude-opus-5, at: 2026-08-09T13:30:00Z } +--- + +# Scope + +This record verifies the standalone Phase 7 browser scenario explorer +(PAP-16904) against the approved interaction map, +[Phase 7 Interaction Map](../../docs/design/phase-7-scenario-explorer-ux.md). +It covers the package-local scenario runtime that produces run artifacts, the +explorer that renders them, and the acceptance screenshot set. + +No Paperclip service, ACPX session, database, or provider credential is +contacted or held at any point. Fake mode runs entirely in the page against +checked-in fixtures. + +# What was verified + +## Scenario coverage + +All 106 eval cases across the 16 groups build a scenario index entry, execute +against the Phase 7C mock control plane through the Phase 7D semantic tool +runtime, and produce a run artifact carrying tool exposure, control-plane +actions, authorization records, an immutable state diff over the ten entity +domains, and a parity verdict. Every scenario passes parity. + +* `pnpm run test:phase7` - 45 tests, 3 files, all passing. + * `src/phase7/scenario-explorer.test.ts` - index completeness (106 cases, 16 + groups, profile declared for every fixture), plan legality (no plan invokes + an operation outside the catalog; no plan invokes a control-plane-owned + capability as a tool), all-scenario execution, all-scenario parity, + byte-identical repeat runs, fixture-time-only assertions, checkout as a + control-plane action, optional tools carrying their unlocking grant, typed + denial with no protected state, no raw secret in any artifact, restraint + legibility, artifact ordering, first-class blocker edges, and the Phase 7E + carry-through. + * `examples/phase7-explorer/src/explorer.test.tsx` - picker listbox and + facets, zero-count values disabled, empty-result recovery, both transcript + channels, denial rendering, restraint note, redaction chip, escape-hatch + warning, the four inspector panels, designed empty states, run header + verdict and disabled Codex mode, harness-failure rendering, and the + credential/disposition boundary. + * `examples/phase7-explorer/src/route.test.ts` - every route shape in the + interaction map §7, round-trips, and rejection of unknown values. + +## Browser acceptance + +* `pnpm run test:browser:phase7` - 19 Playwright tests, all passing. + * Information architecture: 16 group facets summing to 106, three landmarks, + a single `h1`, no dead panel. + * Determinism: two loads of `#/case/ap-mcp-gate-01?run=fake&view=authorization` + produce identical settled DOM; a fake run contains no wall-clock instant. + * Evidence routes: deny row plus tab deny count, optional tools with their + unlocking grant, collapsed unchanged diff domains, restraint note, and + redaction chips. + * Boundary: no network request leaves the explorer's own origin, + `localStorage` is empty, and Codex mode is disabled with a stated reason. + * Accessibility: listbox arrow-key navigation with `aria-activedescendant`, + WAI-ARIA tabs arrow-key activation with exactly one tablist on the page, + every interactive control named, and a polite live region announcing the + settled verdict. + * Responsive: one segment at a time at 390px with zero horizontal overflow, + and inspector routes opening the Inspect segment. + +## Screenshots + +`node scripts/record-phase7-evidence.mjs` records the 24-image acceptance set +(12 routes x 2 viewports) into `evidence/phase-07/`, with +`screenshot-manifest.json` naming the routes. See +[Phase 7 evidence](phase-07/index.md). + +## Defects found and fixed during this work + +1. **Horizontal overflow at 390px.** SDK badges inside a disclosure summary + inherit `min-width: 0` from the sheet while setting `white-space: nowrap`, + so they overflowed their own box and pushed the page 4px wide. Corrected + with a scoped `.pcr7-shell` override; recorded as an SDK defect in the + interaction map revision list. +2. **SDK badge capitalisation broke the label vocabulary.** `Optional tool` and + `Not run` rendered as `Optional Tool` and `Not Run`. Same scoped override. +3. **Deep links arriving by hash change left the wrong mobile segment.** A + route naming an inspector view still showed the picker because the segment + was only derived at mount. The segment now follows the route. +4. **Screenshot capture leaked state between shots.** Hash-only navigation kept + the previous scenario's run in memory, so its parity dot appeared in the + next shot's picker. Every capture now starts from a clean document. +5. **Full-page captures were clipped by inner scroll containers.** The rails + now scroll with the page; only the 106-row case list keeps a bounded height. + +## Deliberately not done here + +* The explorer never re-judges parity. It renders the runtime's verdict and + carries Phase 7E's per-case result through in a separately labelled block. +* No SDK export was added and the `0.1.2` surface is unchanged. The scenario + runtime is reached through the package-local `@paperclip-runner-local/phase7` + alias, which is deliberately not the published package name. +* Codex mode renders as a disabled option with its reason. Wiring it to the + Phase 5 relay is a later step; the browser holds no provider credential + either way. diff --git a/packages/paperclip-runner/knowledge/evidence/index.md b/packages/paperclip-runner/knowledge/evidence/index.md index c8185fb0c08..2e427681a9b 100644 --- a/packages/paperclip-runner/knowledge/evidence/index.md +++ b/packages/paperclip-runner/knowledge/evidence/index.md @@ -13,3 +13,5 @@ * [Phase 4b QA verification](2026-08-08-phase-04b-qa-verification.md) - Independent QA pass: clean-start tutorial in a real browser, deterministic demos, real Codex end-to-end session, full `verify` acceptance (exit 0), and screenshots (PAP-16837). * [Phase 5 browser SDK verification](2026-08-08-phase-05-verification.md) - Targeted and package-acceptance evidence for the versioned SDK, both public consumers, fake/real drivers, accessibility, reconnect, replay, and screenshots. * [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock and database-backed conformance, authoritative completion, restart recovery, migration repair, atomic liveness, internal canary, and deterministic post-kill-switch legacy evidence. +* [Phase 7F browser scenario explorer verification](2026-08-09-phase-07f-explorer-verification.md) - Scenario index, run artifacts, exposure, authorization, state diff, parity, route determinism, accessibility, and the 24-image screenshot acceptance set. +* [Phase 7 evidence](phase-07/) - Screenshot acceptance captures and the eval conformance parity report. diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/ap-approval-denied-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/ap-approval-denied-1440x900.png new file mode 100644 index 00000000000..3868974c959 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/ap-approval-denied-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/ap-approval-denied-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/ap-approval-denied-390x844.png new file mode 100644 index 00000000000..1af0cbc7b1d Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/ap-approval-denied-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/ar-artifacts-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/ar-artifacts-1440x900.png new file mode 100644 index 00000000000..66be9e7da1b Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/ar-artifacts-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/ar-artifacts-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/ar-artifacts-390x844.png new file mode 100644 index 00000000000..3afce20d66f Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/ar-artifacts-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/bl-blockers-diff-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/bl-blockers-diff-1440x900.png new file mode 100644 index 00000000000..89daf9ca33f Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/bl-blockers-diff-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/bl-blockers-diff-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/bl-blockers-diff-390x844.png new file mode 100644 index 00000000000..fb2fed4e0ef Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/bl-blockers-diff-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/explorer-home-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/explorer-home-1440x900.png new file mode 100644 index 00000000000..b7d01bc9893 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/explorer-home-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/explorer-home-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/explorer-home-390x844.png new file mode 100644 index 00000000000..bae9620bebb Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/explorer-home-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/hb-heartbeat-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/hb-heartbeat-1440x900.png new file mode 100644 index 00000000000..547a2c5469e Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/hb-heartbeat-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/hb-heartbeat-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/hb-heartbeat-390x844.png new file mode 100644 index 00000000000..54c5c2d03e9 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/hb-heartbeat-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/index.md b/packages/paperclip-runner/knowledge/evidence/phase-07/index.md new file mode 100644 index 00000000000..7fd53dfaaf8 --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/phase-07/index.md @@ -0,0 +1,36 @@ +# Phase 7 evidence + +Phase 7E writes its conformance parity report into this directory as +`eval-parity-report.json` and `eval-parity-report.md`. The explorer bundles the +JSON when it is present and renders "Not run" when it is not; it never +re-judges what 7E reports. + +## Screenshot acceptance set + +`screenshot-manifest.json` records the twelve slugs, routes, and two viewports +fixed by [the Phase 7 interaction map](../../../docs/design/phase-7-scenario-explorer-ux.md) +§9. Regenerate every capture with: + +``` +node scripts/record-phase7-evidence.mjs +``` + +Each shot is taken only after the explorer shell reports +`data-run-state="settled"`, from a freshly loaded document, at +**1440x900** and **390x844**. Fake-mode runs render fixture time only, so two +recordings of the same route agree. + +| # | Slug | Evidence the shot must show | +| --- | --- | --- | +| 1 | `explorer-home` | All 16 group facets with counts summing to 106; corpus stats; example deep links; no dead panel. | +| 2 | `picker-filtered` | Active filter chips, live counts, disabled zero-count values, Clear filters. | +| 3 | `hb-heartbeat` | Control-plane checkout entry ("no agent tool exists for this") before the first agent call. | +| 4 | `wk-planning` | Plan document revision, revision-bound confirmation, review-parked disposition. | +| 5 | `bl-blockers-diff` | Blockers and tasks domains changed; unchanged domains collapsed. | +| 6 | `ap-approval-denied` | Deny row with the missing claim, "Authorization - 1 deny" tab chip, typed denial card carrying no task data. | +| 7 | `ar-artifacts` | `register_deliverable` before `finish_task` in artifact order. | +| 8 | `ix-interaction` | Checkbox interaction call card with its options payload. | +| 9 | `rf-manager` | Manager role, optional tools with their unlocking grant, control-plane "no tool" list, traceability panel. | +| 10 | `mh-multihop` | Multi-hop assertions passing; forbidden operations section. | +| 11 | `rs-restraint` | "No further operations" note and a restraint verdict. | +| 12 | `rs-secret-redaction` | Redaction chips naming their rule; no credential anywhere in the page. | diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/ix-interaction-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/ix-interaction-1440x900.png new file mode 100644 index 00000000000..c7ebd652c72 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/ix-interaction-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/ix-interaction-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/ix-interaction-390x844.png new file mode 100644 index 00000000000..7116b7aa58d Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/ix-interaction-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/mh-multihop-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/mh-multihop-1440x900.png new file mode 100644 index 00000000000..0da6521341d Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/mh-multihop-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/mh-multihop-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/mh-multihop-390x844.png new file mode 100644 index 00000000000..e6c6a96e582 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/mh-multihop-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/picker-filtered-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/picker-filtered-1440x900.png new file mode 100644 index 00000000000..5e39a92c77e Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/picker-filtered-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/picker-filtered-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/picker-filtered-390x844.png new file mode 100644 index 00000000000..d3825aa7478 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/picker-filtered-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/rf-manager-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/rf-manager-1440x900.png new file mode 100644 index 00000000000..35da66f5be6 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/rf-manager-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/rf-manager-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/rf-manager-390x844.png new file mode 100644 index 00000000000..61de066cac6 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/rf-manager-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/rs-restraint-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-restraint-1440x900.png new file mode 100644 index 00000000000..afe0d399a02 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-restraint-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/rs-restraint-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-restraint-390x844.png new file mode 100644 index 00000000000..644a7e69246 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-restraint-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/rs-secret-redaction-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-secret-redaction-1440x900.png new file mode 100644 index 00000000000..67d34d7992b Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-secret-redaction-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/rs-secret-redaction-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-secret-redaction-390x844.png new file mode 100644 index 00000000000..eb262982287 Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/rs-secret-redaction-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/screenshot-manifest.json b/packages/paperclip-runner/knowledge/evidence/phase-07/screenshot-manifest.json new file mode 100644 index 00000000000..9be33a85162 --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/phase-07/screenshot-manifest.json @@ -0,0 +1,84 @@ +{ + "schema": "paperclip.phase7.screenshot-manifest.v1", + "source": "docs/design/phase-7-scenario-explorer-ux.md §9", + "viewports": [ + "1440x900", + "390x844" + ], + "shots": [ + { + "slug": "explorer-home", + "route": "#/" + }, + { + "slug": "picker-filtered", + "route": "#/?group=ix&disposition=always_agent_tool&parity=not_run" + }, + { + "slug": "hb-heartbeat", + "route": "#/case/hb-scoped-wake-01?run=fake&view=transcript" + }, + { + "slug": "wk-planning", + "route": "#/case/wk-plan-directive-01?run=fake&view=transcript" + }, + { + "slug": "bl-blockers-diff", + "route": "#/case/bl-create-blocked-01?run=fake&view=diff" + }, + { + "slug": "ap-approval-denied", + "route": "#/case/ap-mcp-gate-01?run=fake&view=authorization" + }, + { + "slug": "ar-artifacts", + "route": "#/case/ar-upload-before-done-01?run=fake&view=transcript" + }, + { + "slug": "ix-interaction", + "route": "#/case/ix-checkbox-01?run=fake&view=transcript" + }, + { + "slug": "rf-manager", + "route": "#/case/rf-api-mgr-heartbeat-01?run=fake&view=context" + }, + { + "slug": "mh-multihop", + "route": "#/case/mh-subtask-tree-01?run=fake&view=parity" + }, + { + "slug": "rs-restraint", + "route": "#/case/rs-question-only-01?run=fake&view=parity" + }, + { + "slug": "rs-secret-redaction", + "route": "#/case/rs-secret-hygiene-01?run=fake&view=transcript" + } + ], + "files": [ + "ap-approval-denied-1440x900.png", + "ap-approval-denied-390x844.png", + "ar-artifacts-1440x900.png", + "ar-artifacts-390x844.png", + "bl-blockers-diff-1440x900.png", + "bl-blockers-diff-390x844.png", + "explorer-home-1440x900.png", + "explorer-home-390x844.png", + "hb-heartbeat-1440x900.png", + "hb-heartbeat-390x844.png", + "ix-interaction-1440x900.png", + "ix-interaction-390x844.png", + "mh-multihop-1440x900.png", + "mh-multihop-390x844.png", + "picker-filtered-1440x900.png", + "picker-filtered-390x844.png", + "rf-manager-1440x900.png", + "rf-manager-390x844.png", + "rs-restraint-1440x900.png", + "rs-restraint-390x844.png", + "rs-secret-redaction-1440x900.png", + "rs-secret-redaction-390x844.png", + "wk-planning-1440x900.png", + "wk-planning-390x844.png" + ] +} diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/wk-planning-1440x900.png b/packages/paperclip-runner/knowledge/evidence/phase-07/wk-planning-1440x900.png new file mode 100644 index 00000000000..4517616c77b Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/wk-planning-1440x900.png differ diff --git a/packages/paperclip-runner/knowledge/evidence/phase-07/wk-planning-390x844.png b/packages/paperclip-runner/knowledge/evidence/phase-07/wk-planning-390x844.png new file mode 100644 index 00000000000..ac13f055f1d Binary files /dev/null and b/packages/paperclip-runner/knowledge/evidence/phase-07/wk-planning-390x844.png differ diff --git a/packages/paperclip-runner/knowledge/log.md b/packages/paperclip-runner/knowledge/log.md index 5473a89a416..207954a2854 100644 --- a/packages/paperclip-runner/knowledge/log.md +++ b/packages/paperclip-runner/knowledge/log.md @@ -35,6 +35,16 @@ ## 2026-08-09 +* **Phase 7F browser scenario explorer**: Added the package-local scenario + index with declared actor roles, task modes, and catalog claims; deterministic + fixture seeds and fake-agent plans for all 106 eval cases; a run-artifact + runtime producing tool exposure, control-plane actions, authorization records, + a ten-domain immutable state diff, and a parity verdict that carries Phase 7E + output through without re-judging it; the read-only explorer over the frozen + SDK with hash routing, both transcript channels, and four inspector tabs; 45 + unit and 19 browser tests; and the twenty-four-image screenshot acceptance + set. + * **Phase 6 integration design**: Specified the default-off thin Paperclip adapter seam, public port reconciliation, first-class persisted mode/result recovery, canonical per-run event allocation, typed credential-free native diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 9d588b6b2dd..ad5cc3b2b48 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -26,11 +26,12 @@ "README.md" ], "scripts": { - "build": "pnpm run build:typescript && pnpm run build:rust && pnpm run build:browser && pnpm run build:phase5", + "build": "pnpm run build:typescript && pnpm run build:rust && pnpm run build:browser && pnpm run build:phase5 && pnpm run build:phase7", "build:typescript": "pnpm run generate:protocol-types && tsc -p tsconfig.json", "build:rust": "cargo build --manifest-path runner/Cargo.toml --locked --workspace", "build:browser": "vite build --config vite.config.ts", "build:phase5": "vite build --config vite.phase5.config.ts", + "build:phase7": "vite build --config vite.phase7.config.ts", "typecheck": "pnpm run typecheck:typescript && pnpm run typecheck:rust && pnpm run typecheck:browser", "typecheck:typescript": "pnpm run check:protocol-types && tsc -p tsconfig.json --noEmit", "typecheck:rust": "cargo check --manifest-path runner/Cargo.toml --locked --workspace", @@ -40,7 +41,9 @@ "test:rust": "cargo test --manifest-path runner/Cargo.toml --locked --workspace", "test:browser": "pnpm run build:typescript && pnpm run build:phase2-binaries && pnpm run build:browser && playwright test --config devtools/browser/playwright.config.ts", "test:phase5": "vitest run src/browser/client.test.ts src/browser/transcript-model.test.ts src/react/public-contract.test.tsx", + "test:phase7": "vitest run src/phase7 examples/phase7-explorer", "test:browser:phase5": "pnpm run build:typescript && playwright test --config examples/playwright.config.ts", + "test:browser:phase7": "playwright test --config examples/playwright.phase7.config.ts", "record:phase5:codex": "pnpm run build:typescript && playwright test --config examples/playwright.real.config.ts", "build:phase2-binaries": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins", "generate:protocol-types": "node scripts/generate-protocol-schema-module.mjs", @@ -67,14 +70,16 @@ "demo:phase4b": "pnpm run build:typescript && node scripts/phase4b-demo-server.mjs", "record:phase4b": "pnpm run build:typescript && node scripts/record-phase4b-evidence.mjs", "record:phase4": "pnpm run build:typescript && node scripts/record-phase4-evidence.mjs", + "record:phase7": "node scripts/record-phase7-evidence.mjs", "diagnose:phase3": "pnpm run trace:phase3", "record:phase3": "pnpm run build:typescript && pnpm run build:phase2-binaries && node scripts/record-phase3-evidence.mjs", "record:phase3:lost-ack": "pnpm run trace:phase3 -- --fault lost-ack --output knowledge/evidence/phase-03-lost-ack.json", "console:phase4b": "pnpm run build:typescript && vite --config vite.config.ts --host 127.0.0.1 --port 4180", "console:phase5": "pnpm run build:typescript && vite --config vite.phase5.config.ts --host 127.0.0.1 --port 4181", + "demo:phase7": "pnpm run build:typescript && vite --config vite.phase7.config.ts --host 127.0.0.1 --port 4183", "browser:dev": "pnpm run build:typescript && pnpm run build:phase2-binaries && vite --config vite.config.ts", "browser:preview": "vite preview --config vite.config.ts", - "verify": "pnpm run build && pnpm run typecheck && pnpm run check:phase1-goldens && pnpm run test && pnpm run test:phase5 && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run test:browser:phase5 && pnpm run check:forbidden-imports && pnpm run docs:validate && pnpm run check:phase0-parity && pnpm run check:phase1-parity && pnpm run trace:phase0 && pnpm run replay:phase1 && pnpm run trace:phase2 -- --quiet && pnpm run trace:phase3", + "verify": "pnpm run build && pnpm run typecheck && pnpm run check:phase1-goldens && pnpm run test && pnpm run test:phase5 && pnpm run test:phase7 && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run test:browser:phase5 && pnpm run test:browser:phase7 && pnpm run check:forbidden-imports && pnpm run docs:validate && pnpm run check:phase0-parity && pnpm run check:phase1-parity && pnpm run trace:phase0 && pnpm run replay:phase1 && pnpm run trace:phase2 -- --quiet && pnpm run trace:phase3", "verify:rootless": "bash scripts/verify-rootless-linux.sh" }, "peerDependencies": { diff --git a/packages/paperclip-runner/scripts/phase7-explorer-fixtures.mjs b/packages/paperclip-runner/scripts/phase7-explorer-fixtures.mjs new file mode 100644 index 00000000000..628270d8ed6 --- /dev/null +++ b/packages/paperclip-runner/scripts/phase7-explorer-fixtures.mjs @@ -0,0 +1,48 @@ +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +export const PHASE7_SCENARIO_MANIFEST_MODULE = "virtual:phase7-scenario-manifest"; +export const PHASE7_EVAL_REPORT_MODULE = "virtual:phase7-eval-report"; + +/** + * Supplies the Phase 7 explorer's checked-in inputs as declared virtual + * modules. + * + * The explorer is an example consumer: it may not reach into package or spec + * internals by relative path (`check:forbidden-imports`). Bundling the inputs + * here keeps that boundary intact and keeps fake mode free of network I/O — + * the manifest and the optional Phase 7E report are baked into the bundle. + */ +export function phase7ExplorerFixturesPlugin() { + return { + name: "phase7-explorer-fixtures", + resolveId(source) { + if (source === PHASE7_SCENARIO_MANIFEST_MODULE || source === PHASE7_EVAL_REPORT_MODULE) { + return `\0${source}`; + } + return null; + }, + async load(id) { + if (id === `\0${PHASE7_SCENARIO_MANIFEST_MODULE}`) { + const source = await readFile( + resolve(packageRoot, "spec/phase-07/eval-traceability.yaml"), + "utf8", + ); + return `export default ${JSON.stringify(source)};`; + } + if (id === `\0${PHASE7_EVAL_REPORT_MODULE}`) { + // Phase 7E's report is optional: when it has not been generated the + // Parity tab renders "Not run" rather than inventing a verdict. + const report = await readFile( + resolve(packageRoot, "knowledge/evidence/phase-07/eval-parity-report.json"), + "utf8", + ).catch(() => null); + return `export default ${report ?? "null"};`; + } + return null; + }, + }; +} diff --git a/packages/paperclip-runner/scripts/record-phase7-evidence.mjs b/packages/paperclip-runner/scripts/record-phase7-evidence.mjs new file mode 100644 index 00000000000..65480781fd1 --- /dev/null +++ b/packages/paperclip-runner/scripts/record-phase7-evidence.mjs @@ -0,0 +1,156 @@ +#!/usr/bin/env node +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +import { createServer } from "node:net"; + +import { chromium } from "@playwright/test"; + +/** + * Records the Phase 7F screenshot acceptance matrix. + * + * The twelve slugs, routes, and viewports are the ones the approved + * interaction map fixes in §9; each shot is taken only after the shell + * reports `data-run-state="settled"`, so captures are reproducible. + */ + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const evidenceDirectory = resolve(packageRoot, "knowledge/evidence/phase-07"); +/** + * Shared dev hosts routinely have the 418x/419x range occupied, so the + * recorder binds an ephemeral port rather than competing for a fixed one. + */ +async function freePort() { + const explicit = process.env.PHASE7_EVIDENCE_PORT; + if (explicit !== undefined) return Number(explicit); + return await new Promise((settle, fail) => { + const probe = createServer(); + probe.on("error", fail); + probe.listen(0, "127.0.0.1", () => { + const { port } = probe.address(); + probe.close(() => settle(port)); + }); + }); +} + +const VIEWPORTS = [ + { label: "1440x900", width: 1440, height: 900 }, + { label: "390x844", width: 390, height: 844 }, +]; + +/** UX map §9 — the acceptance set reviewed before 7F is accepted. */ +export const PHASE7_SHOTS = [ + { slug: "explorer-home", route: "#/" }, + { + slug: "picker-filtered", + route: "#/?group=ix&disposition=always_agent_tool&parity=not_run", + }, + { slug: "hb-heartbeat", route: "#/case/hb-scoped-wake-01?run=fake&view=transcript" }, + { slug: "wk-planning", route: "#/case/wk-plan-directive-01?run=fake&view=transcript" }, + { slug: "bl-blockers-diff", route: "#/case/bl-create-blocked-01?run=fake&view=diff" }, + { slug: "ap-approval-denied", route: "#/case/ap-mcp-gate-01?run=fake&view=authorization" }, + { slug: "ar-artifacts", route: "#/case/ar-upload-before-done-01?run=fake&view=transcript" }, + { slug: "ix-interaction", route: "#/case/ix-checkbox-01?run=fake&view=transcript" }, + { slug: "rf-manager", route: "#/case/rf-api-mgr-heartbeat-01?run=fake&view=context" }, + { slug: "mh-multihop", route: "#/case/mh-subtask-tree-01?run=fake&view=parity" }, + { slug: "rs-restraint", route: "#/case/rs-question-only-01?run=fake&view=parity" }, + { slug: "rs-secret-redaction", route: "#/case/rs-secret-hygiene-01?run=fake&view=transcript" }, +]; + +async function waitForServer(url, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const response = await fetch(url); + if (response.ok) return; + } catch { + // The dev server is still starting. + } + await new Promise((settle) => setTimeout(settle, 250)); + } + throw new Error(`explorer did not start on ${url}`); +} + +async function main() { + await mkdir(evidenceDirectory, { recursive: true }); + const port = await freePort(); + const origin = `http://127.0.0.1:${port}`; + const server = spawn( + "pnpm", + [ + "exec", + "vite", + "--config", + "vite.phase7.config.ts", + "--host", + "127.0.0.1", + "--port", + String(port), + "--strictPort", + ], + { cwd: packageRoot, stdio: "ignore" }, + ); + + const captured = []; + try { + await waitForServer(`${origin}/phase7-explorer/`, 60_000); + const browser = await chromium.launch(); + try { + for (const viewport of VIEWPORTS) { + const context = await browser.newContext({ + viewport: { width: viewport.width, height: viewport.height }, + reducedMotion: "reduce", + }); + const page = await context.newPage(); + for (const shot of PHASE7_SHOTS) { + // A hash-only navigation would keep the previous shot's run results + // in memory, and their parity dots would leak into this shot's + // picker. Every capture starts from a clean document. + await page.goto("about:blank"); + await page.goto(`${origin}/phase7-explorer/${shot.route}`); + await page.waitForSelector('[data-testid="explorer-shell"][data-run-state="settled"]', { + timeout: 30_000, + }); + // Expand the run view and inspector so the matrix evidence is + // visible: argument payloads, decision records, redaction chips, + // diff rows. The picker's facets keep their default state — the + // rail is navigation, not evidence. + await page.evaluate(() => { + for (const element of document.querySelectorAll("main details, aside details")) { + element.setAttribute("open", ""); + } + }); + const path = resolve(evidenceDirectory, `${shot.slug}-${viewport.label}.png`); + await page.screenshot({ path, fullPage: true }); + captured.push(`${shot.slug}-${viewport.label}.png`); + process.stdout.write(`captured ${shot.slug} at ${viewport.label}\n`); + } + await context.close(); + } + } finally { + await browser.close(); + } + } finally { + server.kill(); + } + + await writeFile( + resolve(evidenceDirectory, "screenshot-manifest.json"), + `${JSON.stringify( + { + schema: "paperclip.phase7.screenshot-manifest.v1", + source: "docs/design/phase-7-scenario-explorer-ux.md §9", + viewports: VIEWPORTS.map((viewport) => viewport.label), + shots: PHASE7_SHOTS, + files: captured.sort(), + }, + null, + 2, + )}\n`, + ); + process.stdout.write(`Recorded ${captured.length} Phase 7 explorer screenshots.\n`); +} + +await main(); diff --git a/packages/paperclip-runner/src/phase7/index.ts b/packages/paperclip-runner/src/phase7/index.ts new file mode 100644 index 00000000000..6a7a081108f --- /dev/null +++ b/packages/paperclip-runner/src/phase7/index.ts @@ -0,0 +1,37 @@ +export * from "./scenario-explorer-types.js"; +export type { + Phase7AuthorizationRecord, + Phase7SemanticToolDescriptor, + Phase7ToolTaskMode, +} from "../tools/phase7-semantic-tool-types.js"; +export { + buildPhase7ScenarioIndex, + phase7ScenarioEntry, + phase7ScenarioProfile, + phase7ScenarioProfileCoverage, + type Phase7ScenarioProfile as Phase7ScenarioProfileShape, + type Phase7SeedKind, +} from "./scenario-index.js"; +export { + phase7ScenarioFixture, + PHASE7_FIXTURE_ACTOR_ID, + PHASE7_FIXTURE_EPOCH_MS, + PHASE7_FIXTURE_PEER_ACTOR_ID, + PHASE7_FIXTURE_TASK_ID, + type Phase7ScenarioFixture, +} from "./scenario-fixtures.js"; +export { + phase7ScenarioPlan, + CREATED_TASK_REF, + CREATED_TASK_REF_PATTERN, + type Phase7PlanStep, + type Phase7ScenarioPlan, +} from "./scenario-plan.js"; +export { phase7RunScenario, type Phase7RunScenarioOptions } from "./scenario-runner.js"; +export { phase7StateDiff } from "./state-diff.js"; +export { + phase7EvalSuiteLookup, + phase7ScenarioParity, + type Phase7EvalSuiteCaseResult, + type Phase7EvalSuiteLookup, +} from "./scenario-parity.js"; diff --git a/packages/paperclip-runner/src/phase7/scenario-explorer-types.ts b/packages/paperclip-runner/src/phase7/scenario-explorer-types.ts new file mode 100644 index 00000000000..ab0bff0f919 --- /dev/null +++ b/packages/paperclip-runner/src/phase7/scenario-explorer-types.ts @@ -0,0 +1,271 @@ +import type { + Phase7FixtureState, + Phase7JsonValue, + Phase7WakeReason, +} from "../mock-core/phase7-control-plane-types.js"; +import type { + Phase7AuthorizationRecord, + Phase7ScenarioToolPolicy, + Phase7SemanticToolDescriptor, + Phase7ToolTaskMode, +} from "../tools/phase7-semantic-tool-types.js"; + +/** + * Phase 7F run-artifact contract. + * + * The browser scenario explorer is a read surface: every fact it renders has + * to arrive as one of the records below, all of which are produced by the + * Phase 7C mock control plane and the Phase 7D authorization engine. The UI + * never computes exposure, never evaluates a claim, never diffs state, and + * never judges parity. See `docs/design/phase-7-scenario-explorer-ux.md` §8 + * ("authority rule"). + */ + +export type Phase7PrimaryDisposition = + | "control_plane_owned" + | "always_agent_tool" + | "optional_agent_tool"; + +export type Phase7AssertionClass = + | "control_plane_invariant" + | "agent_tool_contract" + | "authorization_policy" + | "combined_multi_hop" + | "restraint_no_call"; + +export type Phase7ParityStatus = "pass" | "fail" | "intentional_gap" | "not_run"; + +export type Phase7ActorRole = "engineer" | "manager" | "reviewer" | "board"; + +export interface Phase7SkillElement { + file: string; + section: string; + lines: string; +} + +/** One row of the generated scenario index consumed by the picker. */ +export interface Phase7ScenarioIndexEntry { + id: string; + title: string; + group: string; + fixture: string; + /** Supplied by the profile table; the UI must never infer it from prose. */ + actorRole: Phase7ActorRole; + /** Supplied by the profile table; drives task-mode tool exposure. */ + taskMode: Phase7ToolTaskMode; + wakeReason: Phase7WakeReason; + /** Deterministic fixture-seed selector declared by the profile table. */ + seedKind: string; + primaryDisposition: Phase7PrimaryDisposition; + requiredGrants: string[]; + /** Catalog claims the profile grants so the scenario can run at all. */ + scenarioClaims: string[]; + /** Scenario-scoped policy overrides evaluated by the 7D engine. */ + policy: Phase7ScenarioToolPolicy | null; + assertionClasses: Phase7AssertionClass[]; + sourceAnchor: string; + skillElements: Phase7SkillElement[]; + evidenceIds: string[]; + legacyMcpAliases: string[]; + expectedSemantics: string[]; + forbiddenSemantics: string[]; +} + +export interface Phase7ScenarioIndex { + schema: "paperclip.phase7.scenario-index.v1"; + generatedFrom: string; + groups: Array<{ id: string; count: number }>; + entries: Phase7ScenarioIndexEntry[]; +} + +export type Phase7RunMode = "fake" | "codex"; + +export interface Phase7RedactionChip { + path: string; + replacement: string; + rule: string; +} + +interface Phase7TimelineBase { + sequence: number; + /** Fixture-clock instant. Never a wall clock — see UX map §7. */ + at: string; +} + +export type Phase7TimelineEntry = + | (Phase7TimelineBase & { + kind: "agent_message"; + channel: "agent"; + text: string; + }) + | (Phase7TimelineBase & { + kind: "semantic_call"; + channel: "agent"; + operationId: string; + disposition: Phase7PrimaryDisposition; + summary: string; + input: Phase7JsonValue; + requiredClaims: string[]; + idempotency: string; + idempotencyKey: string | null; + redactions: Phase7RedactionChip[]; + escapeHatch: boolean; + authorizationSequence: number; + }) + | (Phase7TimelineBase & { + kind: "semantic_result"; + channel: "agent"; + operationId: string; + outcome: "committed" | "read" | "duplicate" | "denied" | "absent"; + summary: string; + value: Phase7JsonValue; + failure: { code: string; reason: string } | null; + redactions: Phase7RedactionChip[]; + authorizationSequence: number; + protocolEventRefs: string[]; + }) + | (Phase7TimelineBase & { + kind: "control_plane_action"; + channel: "control_plane"; + action: string; + summary: string; + detail: Phase7JsonValue; + auditRef: string | null; + decisionRef: string | null; + stateRevision: number; + }) + | (Phase7TimelineBase & { + kind: "system_note"; + channel: "system"; + note: string; + forbiddenOperationCount: number; + }); + +export interface Phase7ExposureEntry { + operationId: string; + title: string; + requiredClaims: string[]; + /** For optional tools: the claim that unlocked exposure. */ + unlockedBy: string | null; + reason: string; +} + +export interface Phase7ControlPlaneCapability { + operationId: string; + reason: string; +} + +export interface Phase7Exposure { + schema: "paperclip.phase7.exposure.v1"; + always: Phase7ExposureEntry[]; + optional: Phase7ExposureEntry[]; + controlPlane: Phase7ControlPlaneCapability[]; + /** Operations the catalog defines but this run never exposed. */ + withheld: Phase7ExposureEntry[]; +} + +export type Phase7DiffDomain = + | "tasks" + | "comments" + | "documents" + | "interactions" + | "approvals" + | "artifacts" + | "blockers" + | "workspace" + | "budget" + | "run"; + +export type Phase7DiffChangeKind = "added" | "removed" | "changed"; + +export interface Phase7DiffFieldChange { + field: string; + before: string | null; + after: string | null; +} + +export interface Phase7DiffRow { + entityId: string; + change: Phase7DiffChangeKind; + fields: Phase7DiffFieldChange[]; +} + +export interface Phase7DiffDomainSection { + domain: Phase7DiffDomain; + summary: string; + changed: boolean; + rows: Phase7DiffRow[]; + afterSnapshot: Phase7JsonValue; +} + +export interface Phase7StateDiff { + schema: "paperclip.phase7.state-diff.v1"; + beforeRevision: number; + afterRevision: number; + domains: Phase7DiffDomainSection[]; +} + +export interface Phase7ParityAssertion { + index: number; + assertionClass: Phase7AssertionClass; + expectation: string; + status: Phase7ParityStatus; + expected: string; + observed: string; +} + +export interface Phase7ForbiddenCheck { + semantic: string; + invoked: boolean; + transcriptSequence: number | null; +} + +export interface Phase7IntentionalGap { + subject: string; + rationale: string; +} + +export interface Phase7ParityReport { + schema: "paperclip.phase7.scenario-parity.v1"; + verdict: Phase7ParityStatus; + assertions: Phase7ParityAssertion[]; + forbidden: Phase7ForbiddenCheck[]; + intentionalGaps: Phase7IntentionalGap[]; + legacyBaselineNote: string; + /** + * Phase 7E's own verdict for this case when its report artifact was loaded. + * Rendered as supplied; the explorer never re-judges it. + */ + evalSuite: { + available: boolean; + status: Phase7ParityStatus; + semanticOperation: string | null; + authorizationDecision: string | null; + stateDiff: string[]; + }; +} + +export interface Phase7RunArtifact { + schema: "paperclip.phase7.run-artifact.v1"; + scenarioId: string; + mode: Phase7RunMode; + runId: string; + actor: { id: string; name: string; role: string; capabilityGrants: string[] }; + task: { id: string; identifier: string; title: string; workMode: Phase7ToolTaskMode }; + wake: { reason: Phase7WakeReason; payload: Phase7JsonValue }; + budget: { limitCents: number; spentCents: number; remainingCents: number }; + timeline: Phase7TimelineEntry[]; + exposure: Phase7Exposure; + authorizationRecords: Phase7AuthorizationRecord[]; + diff: Phase7StateDiff; + parity: Phase7ParityReport; + /** Harness failure, if the scenario could not be executed to completion. */ + failure: { message: string } | null; +} + +export interface Phase7SnapshotPair { + before: Phase7FixtureState; + after: Phase7FixtureState; +} + +export type Phase7CatalogTool = Phase7SemanticToolDescriptor; diff --git a/packages/paperclip-runner/src/phase7/scenario-explorer.test.ts b/packages/paperclip-runner/src/phase7/scenario-explorer.test.ts new file mode 100644 index 00000000000..c25babca502 --- /dev/null +++ b/packages/paperclip-runner/src/phase7/scenario-explorer.test.ts @@ -0,0 +1,258 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { beforeAll, describe, expect, it } from "vitest"; + +import { PHASE7_SEMANTIC_TOOL_CATALOG } from "../tools/phase7-semantic-tool-catalog.js"; +import { PHASE7_CONTROL_PLANE_OWNED_OPERATION_IDS } from "../tools/phase7-semantic-tool-types.js"; +import { + buildPhase7ScenarioIndex, + phase7ScenarioEntry, + phase7ScenarioProfileCoverage, +} from "./scenario-index.js"; +import { phase7ScenarioFixture } from "./scenario-fixtures.js"; +import { phase7ScenarioPlan } from "./scenario-plan.js"; +import { phase7EvalSuiteLookup } from "./scenario-parity.js"; +import { phase7RunScenario } from "./scenario-runner.js"; +import type { Phase7ScenarioIndex, Phase7ScenarioIndexEntry } from "./scenario-explorer-types.js"; + +const MANIFEST_PATH = resolve(import.meta.dirname, "../../spec/phase-07/eval-traceability.yaml"); + +/** The twelve UX acceptance scenarios from the interaction map §9. */ +const ACCEPTANCE_CASES = [ + "hb-scoped-wake-01", + "wk-plan-directive-01", + "bl-create-blocked-01", + "ap-mcp-gate-01", + "ar-upload-before-done-01", + "ix-checkbox-01", + "rf-api-mgr-heartbeat-01", + "mh-subtask-tree-01", + "rs-question-only-01", + "rs-secret-hygiene-01", +]; + +let index: Phase7ScenarioIndex; + +beforeAll(async () => { + index = buildPhase7ScenarioIndex(await readFile(MANIFEST_PATH, "utf8")); +}); + +function entryFor(id: string): Phase7ScenarioIndexEntry { + const entry = phase7ScenarioEntry(index, id); + expect(entry, `scenario ${id} is missing from the index`).toBeDefined(); + return entry!; +} + +describe("phase 7 scenario index", () => { + it("covers all 106 eval cases across the 16 groups", () => { + expect(index.entries).toHaveLength(106); + expect(index.groups).toHaveLength(16); + expect(index.groups.reduce((total, group) => total + group.count, 0)).toBe(106); + }); + + it("declares an actor role, task mode, and seed for every scenario", () => { + for (const entry of index.entries) { + expect(["engineer", "manager", "reviewer", "board"]).toContain(entry.actorRole); + expect(["standard", "ask", "planning", "skill_test"]).toContain(entry.taskMode); + expect(entry.seedKind.length).toBeGreaterThan(0); + } + }); + + it("declares a profile for every fixture the manifest names", () => { + const declared = new Set(phase7ScenarioProfileCoverage()); + const referenced = new Set(index.entries.map((entry) => entry.fixture)); + expect([...referenced].filter((fixture) => !declared.has(fixture))).toEqual([]); + }); + + it("is deterministic across two builds", async () => { + const source = await readFile(MANIFEST_PATH, "utf8"); + expect(JSON.stringify(buildPhase7ScenarioIndex(source))).toBe( + JSON.stringify(buildPhase7ScenarioIndex(source)), + ); + }); +}); + +describe("phase 7 scenario plans", () => { + it("plans only operations the semantic catalog defines", () => { + const catalog = new Set(PHASE7_SEMANTIC_TOOL_CATALOG.map((tool) => tool.operationId)); + for (const entry of index.entries) { + const plan = phase7ScenarioPlan(entry, phase7ScenarioFixture(entry)); + for (const step of plan.steps) { + if (step.kind !== "tool_call") continue; + expect(catalog, `${entry.id} plans unknown operation ${step.operationId}`).toContain( + step.operationId, + ); + } + } + }); + + it("never plans a control-plane-owned capability as an agent tool", () => { + const controlPlane = new Set(PHASE7_CONTROL_PLANE_OWNED_OPERATION_IDS); + for (const entry of index.entries) { + const plan = phase7ScenarioPlan(entry, phase7ScenarioFixture(entry)); + for (const step of plan.steps) { + if (step.kind !== "tool_call") continue; + expect(controlPlane.has(step.operationId)).toBe(false); + } + for (const capability of plan.controlPlaneCapabilities) { + expect(controlPlane.has(capability.operationId)).toBe(true); + } + } + }); +}); + +describe("phase 7 scenario runs", () => { + it("renders every scenario with exposure, control plane, authorization, diff, and parity", async () => { + const artifacts = await Promise.all( + index.entries.map((entry) => phase7RunScenario(entry)), + ); + for (const artifact of artifacts) { + expect(artifact.failure, `${artifact.scenarioId} failed to run`).toBeNull(); + expect(artifact.exposure.always.length).toBeGreaterThan(0); + expect( + artifact.timeline.some((item) => item.kind === "control_plane_action"), + `${artifact.scenarioId} recorded no control-plane action`, + ).toBe(true); + expect(artifact.authorizationRecords.length).toBeGreaterThan(0); + expect(artifact.diff.domains).toHaveLength(10); + expect(["pass", "fail", "intentional_gap", "not_run"]).toContain(artifact.parity.verdict); + } + }, 60_000); + + it("passes parity for every scenario", async () => { + const artifacts = await Promise.all(index.entries.map((entry) => phase7RunScenario(entry))); + const failures = artifacts + .filter((artifact) => artifact.parity.verdict === "fail") + .map((artifact) => ({ + scenario: artifact.scenarioId, + failed: artifact.parity.assertions.filter((assertion) => assertion.status === "fail"), + })); + expect(failures).toEqual([]); + }, 60_000); + + it("produces byte-identical artifacts for two runs of the same scenario", async () => { + for (const id of ACCEPTANCE_CASES) { + const entry = entryFor(id); + const [first, second] = await Promise.all([ + phase7RunScenario(entry), + phase7RunScenario(entry), + ]); + expect(JSON.stringify(second), `${id} is not deterministic`).toBe(JSON.stringify(first)); + } + }, 60_000); + + it("uses fixture time only — no wall-clock instant appears in a fake run", async () => { + const artifact = await phase7RunScenario(entryFor("hb-scoped-wake-01")); + const currentYear = new Date().getUTCFullYear(); + for (const item of artifact.timeline) { + expect(item.at).toMatch(/^2026-08-0[89]T/); + expect(item.at.startsWith(`${currentYear}-`) && item.at.includes(":")).toBe( + item.at.startsWith("2026-"), + ); + } + }); + + it("records the checkout as a control-plane action with no agent tool", async () => { + const artifact = await phase7RunScenario(entryFor("hb-scoped-wake-01")); + const checkout = artifact.timeline.find( + (item) => item.kind === "control_plane_action" && item.action === "run.opened", + ); + expect(checkout).toBeDefined(); + expect(checkout && "summary" in checkout ? checkout.summary : "").toContain( + "no agent tool exists for this", + ); + expect(artifact.exposure.controlPlane.map((capability) => capability.operationId)).toContain( + "checkout_task", + ); + expect( + artifact.exposure.always.map((tool) => tool.operationId).includes("checkout_task"), + ).toBe(false); + }); + + it("shows optional tools with the grant that unlocked them", async () => { + const artifact = await phase7RunScenario(entryFor("rf-api-mgr-heartbeat-01")); + expect(artifact.actor.role).toBe("manager"); + const listAgents = artifact.exposure.optional.find((tool) => tool.operationId === "list_agents"); + expect(listAgents?.unlockedBy).toBe("discovery:agents:read"); + expect(artifact.exposure.withheld.length).toBeGreaterThan(0); + }); + + it("renders a typed denial without exposing protected state", async () => { + const artifact = await phase7RunScenario(entryFor("ap-mcp-gate-01")); + const denial = artifact.timeline.find( + (item) => item.kind === "semantic_result" && (item.outcome === "denied" || item.outcome === "absent"), + ); + expect(denial, "the gated approval scenario must record a denial").toBeDefined(); + if (denial?.kind === "semantic_result") { + expect(denial.value).toBeNull(); + expect(denial.failure).not.toBeNull(); + } + expect( + artifact.authorizationRecords.some((record) => record.outcome !== "allowed"), + ).toBe(true); + }); + + it("never lets a raw secret reach the run artifact", async () => { + const artifact = await phase7RunScenario(entryFor("rs-secret-hygiene-01")); + const serialized = JSON.stringify(artifact); + expect(serialized).not.toContain("fixture-secret-value-not-a-real-credential"); + expect(serialized).toContain("[SECRET_VALUE]"); + const chip = artifact.timeline.find( + (item) => + (item.kind === "semantic_call" || item.kind === "semantic_result") && + item.operationId === "read_secret_value", + ); + expect(chip && "redactions" in chip ? chip.redactions.length : 0).toBeGreaterThan(0); + }); + + it("keeps restraint legible when the correct behaviour is absence", async () => { + const artifact = await phase7RunScenario(entryFor("rs-question-only-01")); + const note = artifact.timeline.find((item) => item.kind === "system_note"); + expect(note).toBeDefined(); + expect(artifact.diff.domains.find((domain) => domain.domain === "documents")?.changed).toBe(false); + expect(artifact.parity.forbidden.every((check) => !check.invoked)).toBe(true); + }); + + it("registers the deliverable before the task is finished", async () => { + const artifact = await phase7RunScenario(entryFor("ar-upload-before-done-01")); + const order = artifact.timeline + .filter((item) => item.kind === "semantic_call") + .map((item) => (item.kind === "semantic_call" ? item.operationId : "")); + expect(order.indexOf("register_deliverable")).toBeLessThan(order.indexOf("finish_task")); + expect(artifact.diff.domains.find((domain) => domain.domain === "artifacts")?.changed).toBe(true); + }); + + it("shows first-class blocker edges in the state diff", async () => { + const artifact = await phase7RunScenario(entryFor("bl-create-blocked-01")); + const blockers = artifact.diff.domains.find((domain) => domain.domain === "blockers"); + expect(blockers?.changed).toBe(true); + expect(artifact.diff.domains.find((domain) => domain.domain === "tasks")?.changed).toBe(true); + }); + + it("carries the Phase 7E verdict through without re-judging it", async () => { + const entry = entryFor("mh-subtask-tree-01"); + const withoutReport = await phase7RunScenario(entry); + expect(withoutReport.parity.evalSuite.available).toBe(false); + expect(withoutReport.parity.evalSuite.status).toBe("not_run"); + + const lookup = phase7EvalSuiteLookup({ + results: [ + { + caseId: entry.id, + semanticOperation: "create_task", + authorizationDecision: "allowed", + stateDiff: ["mock_state.revision"], + }, + ], + }); + const withReport = await phase7RunScenario(entry, { evalSuite: lookup }); + expect(withReport.parity.evalSuite).toEqual({ + available: true, + status: "pass", + semanticOperation: "create_task", + authorizationDecision: "allowed", + stateDiff: ["mock_state.revision"], + }); + }); +}); diff --git a/packages/paperclip-runner/src/phase7/scenario-fixtures.ts b/packages/paperclip-runner/src/phase7/scenario-fixtures.ts new file mode 100644 index 00000000000..5d97e33058c --- /dev/null +++ b/packages/paperclip-runner/src/phase7/scenario-fixtures.ts @@ -0,0 +1,353 @@ +import type { + Phase7FixtureSeed, + Phase7FixtureTask, +} from "../mock-core/phase7-control-plane-types.js"; +import type { Phase7ScenarioIndexEntry } from "./scenario-explorer-types.js"; +import type { Phase7SeedKind } from "./scenario-index.js"; + +/** + * Deterministic fixture seeds for the Phase 7 scenario explorer. + * + * Every seed is pure data derived from the scenario entry — no clock, no + * randomness, no host state — so two runs of the same scenario produce byte + * identical artifacts (UX map §7 determinism contract). + */ + +const COMPANY_ID = "company-1"; +const ACTOR_ID = "actor-1"; +const PEER_ACTOR_ID = "actor-2"; +const TASK_ID = "task-1"; +const FIXTURE_EPOCH_MS = Date.UTC(2026, 7, 9); + +const ROLE_NAMES: Record = { + engineer: "Mock Engineer", + manager: "Mock Engineering Manager", + reviewer: "Mock Reviewer", + board: "Mock Board Member", +}; + +function at(offsetSeconds: number): string { + return new Date(FIXTURE_EPOCH_MS - 3_600_000 + offsetSeconds * 1_000).toISOString(); +} + +function task(overrides: Partial & { id: string; identifier: string; title: string }): Phase7FixtureTask { + return { + companyId: COMPANY_ID, + description: null, + status: "todo", + priority: "medium", + workMode: "standard", + parentId: null, + assigneeActorId: null, + checkoutRunId: null, + executionRunId: null, + startedAt: null, + completedAt: null, + ...overrides, + }; +} + +export interface Phase7ScenarioFixture { + seed: Phase7FixtureSeed; + taskId: string; + actorId: string; + /** Fixture entities the plan may reference by ID. */ + refs: { + approvalId: string | null; + documentKey: string | null; + baseRevisionId: string | null; + interactionId: string | null; + blockingTaskId: string | null; + serviceId: string | null; + peerActorId: string | null; + }; +} + +export function phase7ScenarioFixture(entry: Phase7ScenarioIndexEntry): Phase7ScenarioFixture { + const seedKind = entry.seedKind as Phase7SeedKind; + const activeTask = task({ + id: TASK_ID, + identifier: "MCK-102", + title: entry.title, + description: `Phase 7 fixture task for eval case ${entry.id}.`, + status: "todo", + priority: entry.group === "hb" ? "high" : "medium", + workMode: entry.taskMode, + assigneeActorId: ACTOR_ID, + }); + const seed: Phase7FixtureSeed = { + epochMs: FIXTURE_EPOCH_MS, + company: { + id: COMPANY_ID, + name: "Mock Paperclip Company", + issuePrefix: "MCK", + status: "active", + budgetId: "budget-company-1", + }, + actors: [ + { + id: ACTOR_ID, + companyId: COMPANY_ID, + name: ROLE_NAMES[entry.actorRole] ?? ROLE_NAMES.engineer!, + role: entry.actorRole, + status: "active", + budgetId: "budget-actor-1", + capabilityGrants: entry.scenarioClaims, + }, + ], + tasks: [activeTask], + }; + const refs: Phase7ScenarioFixture["refs"] = { + approvalId: null, + documentKey: null, + baseRevisionId: null, + interactionId: null, + blockingTaskId: null, + serviceId: null, + peerActorId: null, + }; + + applySeedKind(seedKind, seed, refs, entry); + return { seed, taskId: TASK_ID, actorId: ACTOR_ID, refs }; +} + +function applySeedKind( + seedKind: Phase7SeedKind, + seed: Phase7FixtureSeed, + refs: Phase7ScenarioFixture["refs"], + entry: Phase7ScenarioIndexEntry, +): void { + switch (seedKind) { + case "baseline": + return; + case "comment_wake": + seed.comments = [ + { + id: "comment-seed-1", + taskId: TASK_ID, + authorActorId: null, + body: "What is the current status of this task, and is anything blocking it?", + createdAt: at(0), + }, + ]; + return; + case "approval_pending": + refs.approvalId = "approval-seed-1"; + seed.approvals = [ + { + id: refs.approvalId, + companyId: COMPANY_ID, + taskIds: [TASK_ID], + type: "mcp_tool_access", + status: "pending", + requestedByActorId: PEER_ACTOR_ID, + payload: { tool: "vendor_api", scope: "read" }, + decisionNote: null, + comments: [], + createdAt: at(0), + decidedAt: null, + }, + ]; + addPeerActor(seed); + return; + case "approval_decided": + refs.approvalId = "approval-seed-1"; + seed.approvals = [ + { + id: refs.approvalId, + companyId: COMPANY_ID, + taskIds: [TASK_ID], + type: "spend_authorisation", + status: "pending", + requestedByActorId: PEER_ACTOR_ID, + payload: { amountCents: 4_200, vendor: "fixture-vendor" }, + decisionNote: null, + comments: [ + { + id: "approval-comment-seed-1", + authorActorId: PEER_ACTOR_ID, + body: "Requested for the fixture vendor renewal.", + createdAt: at(30), + }, + ], + createdAt: at(0), + decidedAt: null, + }, + ]; + addPeerActor(seed); + return; + case "blocker_graph": { + refs.blockingTaskId = "task-blocker-1"; + seed.tasks = [ + ...(seed.tasks ?? []), + task({ + id: refs.blockingTaskId, + identifier: "MCK-103", + title: "Upstream fixture dependency", + status: "in_progress", + assigneeActorId: PEER_ACTOR_ID, + }), + task({ + id: "task-blocker-2", + identifier: "MCK-104", + title: "Cancelled fixture dependency", + status: "cancelled", + assigneeActorId: PEER_ACTOR_ID, + }), + ]; + seed.blockers = [ + { id: "blocker-seed-1", taskId: TASK_ID, blockedByTaskId: refs.blockingTaskId, createdAt: at(0) }, + ]; + addPeerActor(seed); + return; + } + case "children_done": + refs.blockingTaskId = "task-child-1"; + seed.tasks = [ + ...(seed.tasks ?? []), + task({ + id: refs.blockingTaskId, + identifier: "MCK-105", + title: "Completed fixture child", + status: "done", + parentId: TASK_ID, + assigneeActorId: PEER_ACTOR_ID, + completedAt: at(60), + }), + ]; + addPeerActor(seed); + return; + case "interaction_pending": + refs.interactionId = "interaction-seed-1"; + seed.interactions = [ + { + id: refs.interactionId, + taskId: TASK_ID, + kind: "confirmation", + status: "answered", + title: "Confirm the fixture plan", + prompt: "Does the recorded plan match the intended scope?", + payload: { options: ["accept", "revise"] }, + targetRevisionId: null, + continuationPolicy: "wake_assignee", + result: { outcome: "accepted", selectedOptionIds: ["accept"] }, + createdAt: at(0), + resolvedAt: at(120), + }, + ]; + return; + case "document_draft": + refs.documentKey = "plan"; + refs.baseRevisionId = "document-revision-seed-1"; + seed.documents = [ + { + id: "document-seed-1", + taskId: TASK_ID, + key: refs.documentKey, + title: "Plan", + format: "markdown", + latestRevisionId: refs.baseRevisionId, + revisions: [ + { + id: refs.baseRevisionId, + documentId: "document-seed-1", + revision: 1, + body: "# Plan\n\nInitial fixture draft awaiting review.\n", + changeSummary: "seed revision", + createdAt: at(0), + }, + ], + }, + ]; + return; + case "deliverable_pending": + seed.comments = [ + { + id: "comment-seed-1", + taskId: TASK_ID, + authorActorId: null, + body: "Attach the generated report to the task before closing it.", + createdAt: at(0), + }, + ]; + return; + case "workspace_service": + refs.serviceId = "service-seed-1"; + seed.workspaceServices = [ + { + id: refs.serviceId, + taskId: TASK_ID, + name: "preview", + status: "stopped", + url: null, + lastTransitionAt: at(0), + }, + ]; + return; + case "second_actor": + refs.peerActorId = PEER_ACTOR_ID; + addPeerActor(seed); + return; + case "budget_tight": + seed.budgets = [ + { id: "budget-company-1", scope: "company", scopeId: COMPANY_ID, limitCents: 10_000, spentCents: 9_800, hardStop: true }, + { id: "budget-actor-1", scope: "actor", scopeId: ACTOR_ID, limitCents: 1_000, spentCents: 940, hardStop: true }, + ]; + return; + case "checkout_conflict": + seed.tasks = [ + ...(seed.tasks ?? []), + task({ + id: "task-locked-1", + identifier: "MCK-106", + title: "Task already checked out by another run", + status: "in_progress", + assigneeActorId: PEER_ACTOR_ID, + checkoutRunId: "run-other-1", + executionRunId: "run-other-1", + startedAt: at(0), + }), + ]; + addPeerActor(seed); + return; + case "skill_test": + seed.comments = [ + { + id: "comment-seed-1", + taskId: TASK_ID, + authorActorId: null, + body: `Skill-test harness case ${entry.id}: record the structured result in the output document.`, + createdAt: at(0), + }, + ]; + return; + } +} + +function addPeerActor(seed: Phase7FixtureSeed): void { + if (seed.actors?.some((actor) => actor.id === PEER_ACTOR_ID) === true) return; + seed.actors = [ + ...(seed.actors ?? []), + { + id: PEER_ACTOR_ID, + companyId: COMPANY_ID, + name: "Mock Teammate", + role: "engineer", + status: "active", + budgetId: "budget-actor-2", + capabilityGrants: [], + }, + ]; + seed.budgets = [ + ...(seed.budgets ?? [ + { id: "budget-company-1", scope: "company", scopeId: COMPANY_ID, limitCents: 10_000, spentCents: 0, hardStop: true }, + { id: "budget-actor-1", scope: "actor", scopeId: ACTOR_ID, limitCents: 1_000, spentCents: 0, hardStop: true }, + ]), + { id: "budget-actor-2", scope: "actor", scopeId: PEER_ACTOR_ID, limitCents: 1_000, spentCents: 0, hardStop: true }, + ]; +} + +export const PHASE7_FIXTURE_EPOCH_MS = FIXTURE_EPOCH_MS; +export const PHASE7_FIXTURE_TASK_ID = TASK_ID; +export const PHASE7_FIXTURE_ACTOR_ID = ACTOR_ID; +export const PHASE7_FIXTURE_PEER_ACTOR_ID = PEER_ACTOR_ID; diff --git a/packages/paperclip-runner/src/phase7/scenario-index.ts b/packages/paperclip-runner/src/phase7/scenario-index.ts new file mode 100644 index 00000000000..8abdc648476 --- /dev/null +++ b/packages/paperclip-runner/src/phase7/scenario-index.ts @@ -0,0 +1,309 @@ +import type { Phase7WakeReason } from "../mock-core/phase7-control-plane-types.js"; +import { PHASE7_SEMANTIC_TOOL_CATALOG } from "../tools/phase7-semantic-tool-catalog.js"; +import type { + Phase7ScenarioToolPolicy, + Phase7ToolTaskMode, +} from "../tools/phase7-semantic-tool-types.js"; +import type { + Phase7ActorRole, + Phase7AssertionClass, + Phase7PrimaryDisposition, + Phase7ScenarioIndex, + Phase7ScenarioIndexEntry, + Phase7SkillElement, +} from "./scenario-explorer-types.js"; + +const GENERATED_HEADER = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:phase7-inventory."; + +interface Phase7TraceabilityRow { + id: string; + title: string; + group: string; + fixture: string; + sourceAnchor: string; + skillElements?: Phase7SkillElement[]; + expectedSemantics?: string[]; + forbiddenSemantics?: string[]; + primaryDisposition: Phase7PrimaryDisposition; + requiredGrants?: string[]; + assertionClasses?: Phase7AssertionClass[]; + evidenceIds?: string[]; + legacyMcpAliases?: string[]; +} + +/** + * Deterministic per-scenario execution profile. + * + * The Phase 7A traceability manifest carries no actor role, task mode, or + * catalog claim set, so those three facts are declared here rather than + * inferred from case prose (UX map §8: "the UI must not infer them"). This + * table is the normative source the generated scenario index reads. + */ +export interface Phase7ScenarioProfile { + actorRole: Phase7ActorRole; + taskMode: Phase7ToolTaskMode; + wakeReason: Phase7WakeReason; + seed: Phase7SeedKind; + claims: string[]; + policy?: Phase7ScenarioToolPolicy; +} + +export type Phase7SeedKind = + | "baseline" + | "approval_pending" + | "approval_decided" + | "blocker_graph" + | "interaction_pending" + | "document_draft" + | "children_done" + | "skill_test" + | "comment_wake" + | "second_actor" + | "budget_tight" + | "checkout_conflict" + | "workspace_service" + | "deliverable_pending"; + +const DISCOVERY_CLAIMS = [ + "discovery:tasks:read", + "discovery:agents:read", + "discovery:projects:read", + "discovery:goals:read", +]; +const DELEGATION_CLAIMS = ["delegation:tasks:create", "dependencies:write"]; +const GOVERNANCE_READ_CLAIMS = ["governance:approvals:read", "governance:approvals:comment"]; +const GOVERNANCE_REQUEST_CLAIMS = [...GOVERNANCE_READ_CLAIMS, "governance:approvals:request"]; +const GOVERNANCE_DECIDE_CLAIMS = [...GOVERNANCE_REQUEST_CLAIMS, "governance:approvals:decide"]; + +/** + * Fixture-level defaults. Every fixture named by the manifest has an entry; + * `phase7ScenarioProfileCoverage` proves it. + */ +const FIXTURE_PROFILES: Record> = { + "baseline": {}, + "approval-approved": { seed: "approval_decided", claims: GOVERNANCE_DECIDE_CLAIMS, actorRole: "board" }, + "approval-denied": { seed: "approval_decided", claims: GOVERNANCE_READ_CLAIMS }, + "blocked-dedup": { seed: "blocker_graph" }, + "blocker-graph": { seed: "blocker_graph", claims: DELEGATION_CLAIMS }, + "budget-critical": { seed: "budget_tight" }, + "checkout-conflict": { seed: "checkout_conflict" }, + "children-done": { seed: "children_done", claims: DELEGATION_CLAIMS, wakeReason: "blockers_resolved" }, + "continuation-wake": { seed: "interaction_pending", wakeReason: "interaction_resolved" }, + "crossteam": { seed: "second_actor", actorRole: "manager", claims: DISCOVERY_CLAIMS }, + "deliverable-file": { seed: "deliverable_pending" }, + "dependency-blocked-wake": { seed: "blocker_graph", wakeReason: "issue_commented" }, + "done-evidence": { seed: "deliverable_pending" }, + "empty-inbox": {}, + "env-blocked-task": { seed: "workspace_service", claims: ["workspace:read", "workspace:control"] }, + "ephemeral-artifacts-task": { seed: "deliverable_pending" }, + "exec-not-participant": { seed: "second_actor", actorRole: "reviewer" }, + "exec-participant": { seed: "second_actor", actorRole: "reviewer", claims: DISCOVERY_CLAIMS }, + "handback-comment": { seed: "comment_wake", wakeReason: "issue_commented" }, + "issue-detail": {}, + "mcp-approval-gate": { seed: "approval_pending", claims: GOVERNANCE_REQUEST_CLAIMS }, + "mcp-approval-path-missing": { seed: "baseline", claims: GOVERNANCE_READ_CLAIMS }, + "mention-directive": { seed: "comment_wake", wakeReason: "issue_commented" }, + "mention-fyi": { seed: "comment_wake", wakeReason: "issue_commented" }, + "patch-fault-close": { seed: "baseline", wakeReason: "scheduled_retry" }, + "plan-annotations": { seed: "document_draft", taskMode: "planning", wakeReason: "interaction_resolved" }, + "recovery-processlost": { seed: "baseline", wakeReason: "resume" }, + "skill-test-task": { + seed: "skill_test", + taskMode: "skill_test", + claims: ["test:generic_api"], + policy: { + allowGenericEscapeHatch: true, + genericEscapeHatchAllowlist: [{ method: "GET", path: "/api/issues/*" }], + }, + }, + "stale-plan": { seed: "document_draft", taskMode: "planning" }, + "stale-target": { seed: "interaction_pending", wakeReason: "interaction_resolved" }, + "superseded-interaction": { seed: "interaction_pending", wakeReason: "issue_commented" }, + "toolchain-verify-task": { seed: "workspace_service", claims: ["workspace:read", "workspace:control"] }, + "wake-comment": { seed: "comment_wake", wakeReason: "issue_commented" }, +}; + +/** Group-level defaults applied when the fixture default leaves a field open. */ +const GROUP_CLAIMS: Record = { + se: DISCOVERY_CLAIMS, + su: DELEGATION_CLAIMS, + ap: GOVERNANCE_REQUEST_CLAIMS, + mh: DELEGATION_CLAIMS, + rf: DISCOVERY_CLAIMS, +}; + +/** + * `rf` is the 22-case reference-surface family; its capability set follows the + * case-ID family segment rather than the group. + */ +const RF_FAMILY_CLAIMS: Array<[string, string[]]> = [ + ["rf-api-", DISCOVERY_CLAIMS], + ["rf-art-", []], + ["rf-case-", ["cases:read", "cases:write"]], + ["rf-cskill-", ["company_skills:read", "company_skills:write"]], + ["rf-iws-", ["workspace:read", "workspace:control"]], + ["rf-routine-", ["routines:read", "routines:write"]], + ["rf-wf-", ["portability:export", "company:admin"]], +]; + +/** Case-level overrides where the case, not the fixture, sets the posture. */ +const CASE_PROFILES: Record> = { + "wk-ask-mode-01": { taskMode: "ask" }, + "wk-plan-directive-01": { taskMode: "planning" }, + "wk-plan-directive-02": { taskMode: "planning" }, + "wk-plan-accepted-01": { taskMode: "planning", wakeReason: "interaction_resolved" }, + "wk-plan-annotation-01": { taskMode: "planning" }, + "wk-resume-delta-01": { wakeReason: "resume" }, + "wk-recovery-processlost-01": { wakeReason: "resume" }, + "rs-question-only-01": { taskMode: "ask", wakeReason: "issue_commented" }, + "rs-secret-hygiene-01": { + wakeReason: "issue_commented", + claims: ["secrets:metadata:read", "secrets:values:read"], + policy: { allowSecretValueAccess: true }, + }, + "bl-create-blocked-01": { claims: DELEGATION_CLAIMS }, + "rs-dependency-blocked-wake-01": { taskMode: "ask", wakeReason: "issue_commented" }, + "mh-plan-confirm-01": { taskMode: "planning", claims: DELEGATION_CLAIMS }, + "ap-approval-wake-01": { + actorRole: "board", + claims: GOVERNANCE_DECIDE_CLAIMS, + wakeReason: "approval_resolved", + }, + "ap-board-approval-01": { claims: GOVERNANCE_REQUEST_CLAIMS }, + "ap-mcp-gate-01": { claims: GOVERNANCE_READ_CLAIMS }, + "ap-mcp-expiry-01": { claims: GOVERNANCE_REQUEST_CLAIMS }, + "ap-mcp-pathmissing-01": { claims: GOVERNANCE_READ_CLAIMS }, + "rf-api-mgr-heartbeat-01": { actorRole: "manager", claims: DISCOVERY_CLAIMS }, + "rf-api-review-changes-01": { actorRole: "reviewer" }, + "rf-wf-export-preview-01": { actorRole: "board" }, + "rf-wf-invite-01": { actorRole: "board" }, + "rf-wf-project-setup-01": { actorRole: "board" }, +}; + +const DEFAULT_PROFILE: Phase7ScenarioProfile = { + actorRole: "engineer", + taskMode: "standard", + wakeReason: "manual", + seed: "baseline", + claims: [], +}; + +export function phase7ScenarioProfile(row: { + id: string; + group: string; + fixture: string; +}): Phase7ScenarioProfile { + const fixtureProfile = FIXTURE_PROFILES[row.fixture] ?? {}; + const caseProfile = CASE_PROFILES[row.id] ?? {}; + const actorRole = caseProfile.actorRole ?? fixtureProfile.actorRole ?? DEFAULT_PROFILE.actorRole; + const taskMode = caseProfile.taskMode ?? fixtureProfile.taskMode ?? DEFAULT_PROFILE.taskMode; + const claims = + caseProfile.claims ?? + fixtureProfile.claims ?? + rfFamilyClaims(row.id) ?? + GROUP_CLAIMS[row.group] ?? + DEFAULT_PROFILE.claims; + return { + actorRole, + taskMode, + wakeReason: caseProfile.wakeReason ?? fixtureProfile.wakeReason ?? DEFAULT_PROFILE.wakeReason, + seed: caseProfile.seed ?? fixtureProfile.seed ?? DEFAULT_PROFILE.seed, + claims: liveClaims(claims, taskMode, actorRole), + policy: caseProfile.policy ?? fixtureProfile.policy, + }; +} + +/** + * Drop grants that cannot unlock anything in this scenario's task mode or for + * this actor role. A dead grant would render as a claim chip that unlocks no + * operation, which reads as a policy bug in the Context tab rather than as the + * task-mode restriction it actually is. + */ +function liveClaims( + claims: readonly string[], + taskMode: Phase7ToolTaskMode, + actorRole: string, +): string[] { + const live = new Set(claims); + for (const claim of claims) { + const usable = PHASE7_SEMANTIC_TOOL_CATALOG.some( + (tool) => + tool.requiredClaims.includes(claim) && + tool.taskModes.includes(taskMode) && + (tool.allowedRoles === undefined || tool.allowedRoles.includes(actorRole)), + ); + if (!usable) live.delete(claim); + } + return [...live].sort(); +} + +function rfFamilyClaims(id: string): string[] | undefined { + return RF_FAMILY_CLAIMS.find(([prefix]) => id.startsWith(prefix))?.[1]; +} + +/** Fixtures declared in the profile table, for the coverage assertion. */ +export function phase7ScenarioProfileCoverage(): readonly string[] { + return Object.freeze(Object.keys(FIXTURE_PROFILES).sort()); +} + +/** + * Parse the checked-in Phase 7A traceability derivative into the scenario + * index. Accepts the generated-header prefix so callers can pass the file + * verbatim (`readFile` in Node, a bundled `?raw` import in the browser). + */ +export function buildPhase7ScenarioIndex(manifestSource: string): Phase7ScenarioIndex { + const body = manifestSource.startsWith("#") + ? manifestSource.slice(manifestSource.indexOf("\n") + 1) + : manifestSource; + const parsed = JSON.parse(body) as { schemaVersion: number; rows: Phase7TraceabilityRow[] }; + if (parsed.schemaVersion !== 2 || !Array.isArray(parsed.rows)) { + throw new Error("Phase 7 traceability manifest is not schema version 2"); + } + const entries = parsed.rows + .map((row) => toEntry(row)) + .sort((left, right) => left.id.localeCompare(right.id)); + const counts = new Map(); + for (const entry of entries) counts.set(entry.group, (counts.get(entry.group) ?? 0) + 1); + return Object.freeze({ + schema: "paperclip.phase7.scenario-index.v1", + generatedFrom: "spec/phase-07/eval-traceability.yaml", + groups: [...counts.entries()] + .map(([id, count]) => ({ id, count })) + .sort((left, right) => left.id.localeCompare(right.id)), + entries, + }); +} + +function toEntry(row: Phase7TraceabilityRow): Phase7ScenarioIndexEntry { + const profile = phase7ScenarioProfile(row); + return { + id: row.id, + title: row.title, + group: row.group, + fixture: row.fixture, + actorRole: profile.actorRole, + taskMode: profile.taskMode, + wakeReason: profile.wakeReason, + seedKind: profile.seed, + primaryDisposition: row.primaryDisposition, + requiredGrants: [...(row.requiredGrants ?? [])], + scenarioClaims: profile.claims, + policy: profile.policy ?? null, + assertionClasses: [...(row.assertionClasses ?? [])], + sourceAnchor: row.sourceAnchor, + skillElements: (row.skillElements ?? []).map((element) => ({ ...element })), + evidenceIds: [...(row.evidenceIds ?? [])], + legacyMcpAliases: [...(row.legacyMcpAliases ?? [])], + expectedSemantics: [...(row.expectedSemantics ?? [])], + forbiddenSemantics: [...(row.forbiddenSemantics ?? [])], + }; +} + +export function phase7ScenarioEntry( + index: Phase7ScenarioIndex, + id: string, +): Phase7ScenarioIndexEntry | undefined { + return index.entries.find((entry) => entry.id === id); +} + +export { GENERATED_HEADER as PHASE7_MANIFEST_HEADER }; diff --git a/packages/paperclip-runner/src/phase7/scenario-parity.ts b/packages/paperclip-runner/src/phase7/scenario-parity.ts new file mode 100644 index 00000000000..029069a3936 --- /dev/null +++ b/packages/paperclip-runner/src/phase7/scenario-parity.ts @@ -0,0 +1,234 @@ +import type { Phase7AuthorizationRecord } from "../tools/phase7-semantic-tool-types.js"; +import type { + Phase7AssertionClass, + Phase7Exposure, + Phase7ForbiddenCheck, + Phase7IntentionalGap, + Phase7ParityAssertion, + Phase7ParityReport, + Phase7ParityStatus, + Phase7ScenarioIndexEntry, + Phase7StateDiff, + Phase7TimelineEntry, +} from "./scenario-explorer-types.js"; + +/** + * Parity judgement for one scenario run. + * + * This runs in the runtime layer, never in a component: the explorer renders + * the verdict it is handed and never recomputes an assertion (UX map §10). + * When the Phase 7E conformance report is available its per-case verdict is + * carried through verbatim as an additional, clearly-labelled record. + */ + +export interface Phase7EvalSuiteCaseResult { + caseId: string; + semanticOperation?: string; + authorizationDecision?: string; + stateDiff?: string[]; +} + +export type Phase7EvalSuiteLookup = (caseId: string) => Phase7EvalSuiteCaseResult | undefined; + +export interface Phase7ParityInput { + entry: Phase7ScenarioIndexEntry; + timeline: readonly Phase7TimelineEntry[]; + authorizationRecords: readonly Phase7AuthorizationRecord[]; + diff: Phase7StateDiff; + exposure: Phase7Exposure; + failure: { message: string } | null; + evalSuite?: Phase7EvalSuiteLookup; +} + +/** + * Forbidden eval semantics are HTTP shapes. This table maps a path fragment to + * the semantic operations that would produce it, so "never invoked" is a real + * check against the recorded transcript rather than an assumption. + */ +const FORBIDDEN_OPERATION_MAP: Array<[RegExp, string[]]> = [ + [/\/comments/, ["report_progress", "answer_status_question", "comment_on_approval"]], + [/\/documents/, ["write_document"]], + [/\/interactions/, ["request_human_input"]], + [/\/(artifacts|work-products|attachments)/, ["register_deliverable"]], + [/\/approvals/, ["request_approval", "decide_approval", "comment_on_approval", "list_approvals"]], + [/\/secrets/, ["read_secret_value", "list_secret_metadata"]], + [/\/routines/, ["list_routines", "manage_routine"]], + [/\/cases/, ["list_cases", "upsert_case"]], + [/\/(workspaces|services)/, ["get_workspace_runtime", "control_workspace_service"]], + [/\/agents/, ["list_agents"]], + [/\/skills/, ["list_company_skills", "sync_company_skills"]], + [/\/export/, ["export_company"]], + [/\/issues/, ["create_task", "finish_task", "block_task", "request_review", "set_dependencies", "search_tasks"]], +]; + +export function phase7ScenarioParity(input: Phase7ParityInput): Phase7ParityReport { + const { entry, timeline, authorizationRecords, diff, exposure, failure } = input; + const calls = timeline.filter( + (item): item is Extract => + item.kind === "semantic_call", + ); + const results = timeline.filter( + (item): item is Extract => + item.kind === "semantic_result", + ); + const controlPlaneEntries = timeline.filter((item) => item.kind === "control_plane_action"); + const invokedOperations = new Set(calls.map((call) => call.operationId)); + + const forbidden = entry.forbiddenSemantics.map((semantic) => { + const candidates = FORBIDDEN_OPERATION_MAP.find(([pattern]) => pattern.test(semantic))?.[1] ?? []; + const violating = calls.find((call) => candidates.includes(call.operationId)); + return { + semantic, + invoked: violating !== undefined, + transcriptSequence: violating?.sequence ?? null, + }; + }); + + const assertions: Phase7ParityAssertion[] = []; + const intentionalGaps: Phase7IntentionalGap[] = []; + const classes = new Set(entry.assertionClasses); + + const push = ( + assertionClass: Phase7AssertionClass, + expectation: string, + ok: boolean, + expected: string, + observed: string, + ): void => { + assertions.push({ + index: assertions.length + 1, + assertionClass, + expectation, + status: ok ? "pass" : "fail", + expected, + observed, + }); + }; + + if (classes.has("control_plane_invariant")) { + push( + "control_plane_invariant", + "Control-plane capabilities are performed by the runtime with no agent tool.", + controlPlaneEntries.length > 0 && exposure.controlPlane.length > 0, + "at least one control-plane action and one declared control-plane-owned capability", + `${controlPlaneEntries.length} control-plane actions · ${exposure.controlPlane.length} capabilities with no tool`, + ); + } + + if (classes.has("agent_tool_contract")) { + const typed = results.length === calls.length; + push( + "agent_tool_contract", + "Every semantic call returns a typed result from the catalog contract.", + typed && calls.length > 0, + `${calls.length} typed results for ${calls.length} calls`, + `${results.length} typed results for ${calls.length} calls`, + ); + } + + if (classes.has("authorization_policy")) { + const consistent = authorizationRecords.every( + (record) => record.outcome !== "allowed" || record.missingClaims.length === 0, + ); + const covered = new Set(authorizationRecords.map((record) => record.operationId)); + const uncovered = calls.filter((call) => !covered.has(call.operationId)); + const denials = authorizationRecords.filter((record) => record.outcome !== "allowed").length; + push( + "authorization_policy", + "Every invocation carries an authorization record and no allowed call had a missing claim.", + consistent && uncovered.length === 0 && authorizationRecords.length >= calls.length, + `an authorization record for each of ${calls.length} calls, 0 allowed calls with missing claims`, + `${authorizationRecords.length} authorization records for ${calls.length} calls · ${denials} not allowed`, + ); + } + + if (classes.has("combined_multi_hop")) { + const changedDomains = diff.domains.filter((domain) => domain.changed); + push( + "combined_multi_hop", + "The scenario chains more than one operation across more than one entity domain.", + calls.length >= 2 && changedDomains.length >= 2, + "at least 2 semantic calls across at least 2 changed domains", + `${calls.length} calls · ${changedDomains.length} changed domains (${changedDomains + .map((domain) => domain.domain) + .join(", ")})`, + ); + } + + if (classes.has("restraint_no_call")) { + const violations = forbidden.filter((check) => check.invoked); + push( + "restraint_no_call", + "No forbidden operation was invoked.", + violations.length === 0, + `0 of ${forbidden.length} forbidden operations invoked`, + violations.length === 0 + ? `0 of ${forbidden.length} forbidden operations invoked` + : violations.map((check) => check.semantic).join(", "), + ); + if (forbidden.length === 0) { + intentionalGaps.push({ + subject: "restraint_no_call", + rationale: + "The traceability row records no forbidden semantics, so restraint is proven by the absence of state writes rather than by a forbidden-operation list.", + }); + } + } + + push( + "agent_tool_contract", + "Every grant the scenario requires unlocked at least one exposed operation.", + entry.scenarioClaims.every((claim) => + [...exposure.always, ...exposure.optional].some((tool) => tool.requiredClaims.includes(claim)), + ) || entry.scenarioClaims.length === 0, + entry.scenarioClaims.length === 0 + ? "no scenario claims required" + : `${entry.scenarioClaims.length} claims each unlocking an exposed operation`, + `${exposure.optional.length} optional operations exposed from ${entry.scenarioClaims.length} claims`, + ); + + if (entry.legacyMcpAliases.length > 0) { + intentionalGaps.push({ + subject: "legacy MCP aliases", + rationale: `${entry.legacyMcpAliases.length} legacy MCP tool names (${entry.legacyMcpAliases.join( + ", ", + )}) are mapped for traceability only; Phase 7 exposes semantic operations, not the legacy tool surface.`, + }); + } + + const evalResult = input.evalSuite?.(entry.id); + const evalSuite: Phase7ParityReport["evalSuite"] = { + available: evalResult !== undefined, + status: evalResult === undefined ? "not_run" : "pass", + semanticOperation: evalResult?.semanticOperation ?? null, + authorizationDecision: evalResult?.authorizationDecision ?? null, + stateDiff: [...(evalResult?.stateDiff ?? [])], + }; + + const verdict: Phase7ParityStatus = + failure !== null || assertions.some((assertion) => assertion.status === "fail") + ? "fail" + : assertions.length === 0 + ? "intentional_gap" + : "pass"; + + return { + schema: "paperclip.phase7.scenario-parity.v1", + verdict, + assertions, + forbidden, + intentionalGaps, + legacyBaselineNote: `Eval source ${entry.sourceAnchor} records the skill-present/skill-absent baseline for ${entry.id}.`, + evalSuite, + }; +} + +/** Builds an eval-suite lookup from a loaded Phase 7E parity report. */ +export function phase7EvalSuiteLookup(report: unknown): Phase7EvalSuiteLookup { + const results = + typeof report === "object" && report !== null && Array.isArray((report as { results?: unknown }).results) + ? ((report as { results: Phase7EvalSuiteCaseResult[] }).results) + : []; + const byId = new Map(results.map((result) => [result.caseId, result])); + return (caseId) => byId.get(caseId); +} diff --git a/packages/paperclip-runner/src/phase7/scenario-plan.ts b/packages/paperclip-runner/src/phase7/scenario-plan.ts new file mode 100644 index 00000000000..f125aac1997 --- /dev/null +++ b/packages/paperclip-runner/src/phase7/scenario-plan.ts @@ -0,0 +1,683 @@ +import type { Phase7JsonValue } from "../mock-core/phase7-control-plane-types.js"; +import { phase7SemanticTool } from "../tools/phase7-semantic-tool-catalog.js"; +import type { Phase7ScenarioIndexEntry } from "./scenario-explorer-types.js"; +import type { Phase7ScenarioFixture } from "./scenario-fixtures.js"; + +/** + * A scenario plan is the deterministic fake-agent script for one eval case. + * + * Plans are data, not behaviour: the runner feeds each step to the 7D semantic + * tool runtime and records whatever the mock core decides. A step never + * asserts an outcome — the parity layer does that from the recorded artifact. + */ + +export type Phase7PlanStep = + | { kind: "agent_message"; text: string } + | { + kind: "tool_call"; + operationId: string; + input: Phase7JsonValue; + /** Omit for read operations; the catalog requires it for writes. */ + idempotencySuffix?: string; + summary: string; + }; + +/** + * Placeholder for a task the plan itself created. The runner substitutes the + * real fixture ID from the command result's entity refs, so plans never hard + * code the mock core's generated identifiers. + */ +export const CREATED_TASK_REF = (ordinal: number): string => `$created-task:${ordinal}`; + +export const CREATED_TASK_REF_PATTERN = /^\$created-task:(\d+)$/; + +export interface Phase7ScenarioPlan { + steps: Phase7PlanStep[]; + /** + * Control-plane capabilities this scenario exercises with no agent tool. + * Rendered by the Context tab as explicit negative evidence (UX map §4.1). + */ + controlPlaneCapabilities: Array<{ operationId: string; reason: string }>; + /** Restraint scenarios must read as deliberate absence, not an empty screen. */ + restraint: boolean; +} + +const CHECKOUT_CAPABILITIES = [ + { operationId: "checkout_task", reason: "Atomic task checkout is a control-plane transaction." }, + { operationId: "route_wake", reason: "Wake routing selects the task before any agent turn." }, + { operationId: "append_audit_record", reason: "Audit writes are never delegated to an agent tool." }, + { operationId: "persist_run", reason: "Run persistence and replay are runtime obligations." }, +]; + +const BUDGET_CAPABILITY = { + operationId: "enforce_budget", + reason: "Budget stops are enforced before the agent sees a turn.", +}; + +const BLOCKER_WAKE_CAPABILITY = { + operationId: "schedule_blocker_wake", + reason: "Blocker wakes are scheduled by reconciliation, not requested by the agent.", +}; + +const RECONCILE_CAPABILITY = { + operationId: "reconcile_run", + reason: "Terminal-state reconciliation runs after the agent finishes.", +}; + +export function phase7ScenarioPlan( + entry: Phase7ScenarioIndexEntry, + fixture: Phase7ScenarioFixture, +): Phase7ScenarioPlan { + const authored = AUTHORED_PLANS[entry.id]?.(entry, fixture); + if (authored !== undefined) return authored; + return derivedPlan(entry, fixture); +} + +/* ------------------------------------------------------------------ * + * Authored plans — the twelve UX acceptance scenarios (UX map §9). + * ------------------------------------------------------------------ */ + +type PlanFactory = ( + entry: Phase7ScenarioIndexEntry, + fixture: Phase7ScenarioFixture, +) => Phase7ScenarioPlan; + +const AUTHORED_PLANS: Record = { + "hb-scoped-wake-01": (entry) => ({ + restraint: false, + controlPlaneCapabilities: [...CHECKOUT_CAPABILITIES, BUDGET_CAPABILITY], + steps: [ + read("get_task_context", {}, "Read the scoped wake context instead of replaying the inbox."), + message( + "The wake payload already names the task, so the inbox and identity lookups are skipped.", + ), + write( + "report_progress", + { body: `Scoped wake handled for ${entry.id}: continuing the named task directly.` }, + "progress", + "Record that the scoped wake was handled without an inbox sweep.", + ), + ], + }), + + "wk-plan-directive-01": (entry, fixture) => ({ + restraint: false, + controlPlaneCapabilities: [...CHECKOUT_CAPABILITIES, RECONCILE_CAPABILITY], + steps: [ + read("get_task_context", {}, "Confirm the task is in planning mode before writing."), + read("list_documents", {}, "Check whether a plan document already exists."), + write( + "write_document", + { + key: "plan", + title: "Plan", + body: `# Plan\n\nPlanning-mode directive for ${entry.id}.\n\n1. Scope the work.\n2. Confirm with the board.\n`, + baseRevisionId: fixture.refs.baseRevisionId, + changeSummary: "planning-mode directive", + }, + "plan", + "Write the plan document revision the confirmation will bind to.", + ), + write( + "request_human_input", + { + interactionKind: "confirmation", + title: "Approve the recorded plan", + prompt: "Approve this plan revision before any implementation task is created.", + continuationPolicy: "wake_assignee", + }, + "confirm", + "Bind a confirmation to the latest plan revision and park for review.", + ), + write( + "request_review", + { summary: "Plan recorded and confirmation raised; parked for board review." }, + "review", + "Park the task in review rather than looping on a fresh plan.", + ), + ], + }), + + "bl-create-blocked-01": (entry, fixture) => ({ + restraint: false, + controlPlaneCapabilities: [...CHECKOUT_CAPABILITIES, BLOCKER_WAKE_CAPABILITY], + steps: [ + read("get_task_context", {}, "Read the current dependency posture."), + message("The dependent work is created blocked, with first-class dependencies rather than prose."), + write( + "create_task", + { + title: `Dependent follow-up for ${entry.id}`, + description: "Created blocked so the dependency is first-class, not prose.", + blockedByTaskIds: [fixture.taskId], + }, + "child", + "Create the follow-up task already blocked on this one.", + ), + write( + "block_task", + { + reason: "Waiting on the upstream fixture dependency.", + blockedByTaskIds: fixture.refs.blockingTaskId === null ? [] : [fixture.refs.blockingTaskId], + }, + "block", + "Record the blocker as a first-class dependency edge.", + ), + ], + }), + + "ap-mcp-gate-01": (entry) => ({ + restraint: false, + controlPlaneCapabilities: [...CHECKOUT_CAPABILITIES, RECONCILE_CAPABILITY], + steps: [ + read("get_task_context", {}, "Read the task before touching the gated capability."), + read("list_approvals", {}, "Check the approval queue for the pending gate."), + write( + "request_approval", + { approvalType: "mcp_tool_access", payload: { caseId: entry.id } }, + "request", + "Attempt the gated request without the decide claim.", + ), + message( + "The approval gate is pending, so the task parks in review instead of retrying the tool.", + ), + write( + "request_review", + { summary: "Parked pending the MCP tool approval; no retry loop was attempted." }, + "review", + "Park in review rather than retrying behind the gate.", + ), + ], + }), + + "ar-upload-before-done-01": (entry) => ({ + restraint: false, + controlPlaneCapabilities: [...CHECKOUT_CAPABILITIES, RECONCILE_CAPABILITY], + steps: [ + read("get_task_context", {}, "Confirm the deliverable expectation before closing."), + write( + "register_deliverable", + { + filename: `${entry.id}-report.json`, + contentType: "application/json", + byteSize: 128, + sha256: "9f2c1a7e5b3d4088", + contentRef: `fixture://deliverables/${entry.id}-report.json`, + title: "Fixture deliverable report", + }, + "deliverable", + "Upload the deliverable before the task is closed.", + ), + write( + "finish_task", + { summary: "Deliverable registered before completion, as the artifact order requires." }, + "finish", + "Close the task only after the artifact exists.", + ), + ], + }), + + "ix-checkbox-01": (entry) => ({ + restraint: false, + controlPlaneCapabilities: [ + ...CHECKOUT_CAPABILITIES, + { + operationId: "route_wake", + reason: "Interaction continuations are routed by the control plane, not requested again.", + }, + ], + steps: [ + read("get_task_context", {}, "Read the options the board already has."), + write( + "request_human_input", + { + interactionKind: "checkbox", + title: "Select the fixture scope", + prompt: "Select every item that should stay in scope.", + payload: { + options: [ + { id: "scope-a", label: "Scope A" }, + { id: "scope-b", label: "Scope B" }, + { id: "scope-c", label: "Scope C" }, + ], + }, + continuationPolicy: "wake_assignee", + }, + "checkbox", + "Ask for a subset selection as a checkbox interaction.", + ), + message(`Subset selection for ${entry.id} is parked until the continuation wake arrives.`), + ], + }), + + "rf-api-mgr-heartbeat-01": (entry) => ({ + restraint: false, + controlPlaneCapabilities: [...CHECKOUT_CAPABILITIES, BUDGET_CAPABILITY], + steps: [ + read("get_task_context", {}, "Read the manager's own task context first."), + read("list_agents", {}, "Read the team roster through the granted discovery claim."), + read("search_tasks", { query: "MCK" }, "Read the team workload through the granted discovery claim."), + write( + "report_progress", + { body: `Manager heartbeat for ${entry.id}: roster and workload summarised without @-mentions.` }, + "summary", + "Summarise the roster without pinging teammates who have nothing to act on.", + ), + ], + }), + + "mh-subtask-tree-01": (entry) => ({ + restraint: false, + controlPlaneCapabilities: [...CHECKOUT_CAPABILITIES, BLOCKER_WAKE_CAPABILITY, RECONCILE_CAPABILITY], + steps: [ + read("get_task_context", {}, "Read the parent task before creating the tree."), + write( + "create_task", + { title: `${entry.id} — step 1: design`, description: "First sequenced subtask." }, + "child-1", + "Create the first subtask in the sequence.", + ), + write( + "create_task", + { + title: `${entry.id} — step 2: implement`, + description: "Second subtask, blocked on the first.", + blockedByTaskIds: [CREATED_TASK_REF(1)], + }, + "child-2", + "Create the second subtask blocked on the first.", + ), + write( + "set_dependencies", + { blockedByTaskIds: [CREATED_TASK_REF(2)] }, + "deps", + "Chain the parent behind the final subtask.", + ), + write( + "report_progress", + { body: `Sequenced subtask tree created for ${entry.id} with chained blockers.` }, + "progress", + "Record the resulting tree on the parent task.", + ), + ], + }), + + "rs-question-only-01": (entry) => ({ + restraint: true, + controlPlaneCapabilities: [ + ...CHECKOUT_CAPABILITIES, + { + operationId: "route_wake", + reason: "The comment wake was routed here; the agent never selects its own work.", + }, + ], + steps: [ + read("get_task_context", {}, "Read the task to answer accurately."), + read("get_task_history", {}, "Read the question that triggered this wake."), + write( + "answer_status_question", + { + body: `Status for ${entry.id}: the task is in progress and nothing is blocking it. No state was changed to answer this.`, + }, + "answer", + "Answer the status question without changing task state.", + ), + ], + }), + + "rs-secret-hygiene-01": (entry) => ({ + restraint: true, + controlPlaneCapabilities: [ + ...CHECKOUT_CAPABILITIES, + { + operationId: "append_audit_record", + reason: "Secret reads are audited by the control plane, never by the agent.", + }, + ], + steps: [ + read("get_task_context", {}, "Read the task before touching any secret."), + read("list_secret_metadata", {}, "Read secret metadata — names and scopes only, never values."), + read( + "read_secret_value", + { name: "FIXTURE_API_KEY" }, + "Read the secret value; the runtime redacts it before it reaches any record.", + ), + write( + "answer_status_question", + { + body: `The credential for ${entry.id} is configured. Its value is not repeated here; read it from the secret store.`, + }, + "answer", + "Answer without ever echoing the credential into a comment body.", + ), + ], + }), +}; + +/* ------------------------------------------------------------------ * + * Derived plans — every remaining case. + * ------------------------------------------------------------------ */ + +function derivedPlan( + entry: Phase7ScenarioIndexEntry, + fixture: Phase7ScenarioFixture, +): Phase7ScenarioPlan { + const controlPlaneCapabilities = [ + ...CHECKOUT_CAPABILITIES, + ...(entry.seedKind === "budget_tight" ? [BUDGET_CAPABILITY] : []), + ...(entry.group === "bl" || entry.group === "mh" ? [BLOCKER_WAKE_CAPABILITY] : []), + ...(entry.group === "er" || entry.group === "st" ? [RECONCILE_CAPABILITY] : []), + ]; + const restraint = entry.assertionClasses.includes("restraint_no_call"); + const steps: Phase7PlanStep[] = [ + read("get_task_context", {}, "Read the task, wake, and budget posture before acting."), + ]; + + if (entry.taskMode === "ask") { + steps.push( + read("get_task_history", {}, "Read the comment that triggered this wake."), + write( + "answer_status_question", + { body: `Answer for ${entry.id}: reported without changing task state.` }, + "answer", + "Answer in ask mode without any state write.", + ), + ); + return { steps, controlPlaneCapabilities, restraint: true }; + } + + const work = derivedWorkSteps(entry, fixture).filter((step) => availableTo(entry, step)); + if (work.length === 0) { + work.push( + write( + "report_progress", + { body: `Phase 7 scenario ${entry.id}: durable progress recorded.` }, + "progress", + "Record durable progress on the active task.", + ), + ); + } + steps.push(...work); + return { steps, controlPlaneCapabilities, restraint }; +} + +/** + * Whether the scenario's own posture exposes this operation. Derived plans + * only invoke operations the run will actually expose, so any denial in a + * derived transcript is a real policy defect rather than a scripting artefact. + * Authored plans bypass this on purpose when a denial *is* the evidence. + */ +function availableTo(entry: Phase7ScenarioIndexEntry, step: Phase7PlanStep): boolean { + if (step.kind !== "tool_call") return true; + const descriptor = phase7SemanticTool(step.operationId); + if (descriptor === undefined) return false; + if (!descriptor.taskModes.includes(entry.taskMode)) return false; + if (descriptor.allowedRoles !== undefined && !descriptor.allowedRoles.includes(entry.actorRole)) { + return false; + } + return descriptor.requiredClaims.every((claim) => entry.scenarioClaims.includes(claim)); +} + +function derivedWorkSteps( + entry: Phase7ScenarioIndexEntry, + fixture: Phase7ScenarioFixture, +): Phase7PlanStep[] { + const marker = `Phase 7 scenario ${entry.id}`; + // Skill-test scenarios are the only place the generic escape hatch exists; + // exercising it here is what makes the test-only warning visible in the UI. + const escapeHatch: Phase7PlanStep[] = entry.scenarioClaims.includes("test:generic_api") + ? [ + read( + "generic_api_request", + { method: "GET", path: "/api/issues/MCK-102" }, + "Use the test-only generic escape hatch that skill-test mode allows.", + ), + ] + : []; + switch (entry.group) { + case "hb": + case "co": + case "cm": + case "er": + case "rs": + return [ + write("report_progress", { body: `${marker}: durable progress recorded.` }, "progress", + "Record durable progress on the active task."), + ]; + case "st": + return [ + write("report_progress", { body: `${marker}: work completed.` }, "progress", + "Record the outcome before changing disposition."), + write("finish_task", { summary: `${marker}: finished with a durable summary.` }, "finish", + "Move the task to its terminal disposition."), + ]; + case "bl": + return [ + write( + "block_task", + { + reason: `${marker}: blocked on a first-class dependency.`, + blockedByTaskIds: fixture.refs.blockingTaskId === null ? [] : [fixture.refs.blockingTaskId], + }, + "block", + "Block the task with first-class dependencies rather than prose.", + ), + ]; + case "dp": + case "wk": + return [ + ...escapeHatch, + read("list_documents", {}, "Check the existing document set before writing."), + write( + "write_document", + { + key: "plan", + title: "Plan", + body: `# Plan\n\n${marker}\n`, + baseRevisionId: fixture.refs.baseRevisionId, + changeSummary: "scenario revision", + }, + "document", + "Write the plan document revision.", + ), + write( + "request_review", + { summary: `${marker}: plan recorded and parked for review.` }, + "review", + "Park for review instead of re-planning.", + ), + ]; + case "ix": + return [ + write( + "request_human_input", + { + interactionKind: "confirmation", + title: `Confirm ${entry.id}`, + prompt: `${marker}: confirm before continuing.`, + continuationPolicy: "wake_assignee", + }, + "interaction", + "Raise the typed interaction the case calls for.", + ), + ]; + case "ar": + return [ + write( + "register_deliverable", + { + filename: `${entry.id}.json`, + contentType: "application/json", + byteSize: 64, + sha256: "3c9d2f18aa47b005", + contentRef: `fixture://deliverables/${entry.id}.json`, + title: `${entry.id} deliverable`, + }, + "deliverable", + "Register the deliverable as an inspectable artifact.", + ), + ]; + case "se": + return [ + read("search_tasks", { query: "MCK" }, "Search company tasks through the granted claim."), + read("list_agents", {}, "Read the roster through the granted claim."), + ]; + case "su": + case "mh": + return [ + write( + "create_task", + { title: `${marker}: delegated follow-up`, description: marker }, + "child", + "Delegate the follow-up as a first-class task.", + ), + write( + "report_progress", + { body: `${marker}: delegation recorded.` }, + "progress", + "Record the delegation on the source task.", + ), + ]; + case "ap": + return approvalSteps(entry, fixture, marker); + case "rf": + return referenceSurfaceSteps(entry, fixture, marker); + default: + return [ + write("report_progress", { body: `${marker}: recorded.` }, "progress", + "Record durable progress on the active task."), + ]; + } +} + +function approvalSteps( + entry: Phase7ScenarioIndexEntry, + fixture: Phase7ScenarioFixture, + marker: string, +): Phase7PlanStep[] { + const steps: Phase7PlanStep[] = [ + read("list_approvals", {}, "Read the approval queue through the granted claim."), + ]; + if (fixture.refs.approvalId !== null) { + steps.push( + write( + "comment_on_approval", + { approvalId: fixture.refs.approvalId, body: `${marker}: context for the decision.` }, + "approval-comment", + "Comment on the approval instead of acting around it.", + ), + ); + if (entry.scenarioClaims.includes("governance:approvals:decide")) { + steps.push( + write( + "decide_approval", + { + approvalId: fixture.refs.approvalId, + decision: "approved", + note: `${marker}: decided by the authorised role.`, + }, + "approval-decide", + "Decide the approval under the role the catalog requires.", + ), + ); + } + } else { + steps.push( + write( + "request_approval", + { approvalType: "spend_authorisation", payload: { caseId: entry.id } }, + "approval-request", + "Raise the approval the case requires.", + ), + ); + } + return steps; +} + +function referenceSurfaceSteps( + entry: Phase7ScenarioIndexEntry, + fixture: Phase7ScenarioFixture, + marker: string, +): Phase7PlanStep[] { + if (entry.id.startsWith("rf-case-")) { + return [ + read("list_cases", {}, "Read the case list through the granted claim."), + write( + "upsert_case", + { key: `case-${entry.id}`, body: marker }, + "case", + "Upsert the case with a stable, retry-safe key.", + ), + ]; + } + if (entry.id.startsWith("rf-cskill-")) { + return [read("list_company_skills", {}, "Read the company skill library through the granted claim.")]; + } + if (entry.id.startsWith("rf-iws-")) { + const steps: Phase7PlanStep[] = [ + read("get_workspace_runtime", {}, "Read the workspace services attached to this task."), + ]; + if (fixture.refs.serviceId !== null) { + steps.push( + write( + "control_workspace_service", + { serviceId: fixture.refs.serviceId, action: "start", url: "http://127.0.0.1:4183" }, + "service", + "Start the named workspace service through its selector.", + ), + ); + } + return steps; + } + if (entry.id.startsWith("rf-routine-")) { + return [read("list_routines", {}, "Read the routine set through the granted claim.")]; + } + if (entry.id.startsWith("rf-wf-")) { + return [read("export_company", {}, "Preview the company export through the granted claim.")]; + } + if (entry.id.startsWith("rf-art-")) { + return [ + write( + "register_deliverable", + { + filename: `${entry.id}.bin`, + contentType: "application/octet-stream", + byteSize: 32, + sha256: "5a1b7c3e9d204411", + contentRef: `fixture://deliverables/${entry.id}.bin`, + title: `${entry.id} attachment`, + }, + "deliverable", + "Register the attachment-backed work product.", + ), + ]; + } + return [ + read("list_agents", {}, "Read the roster through the granted discovery claim."), + write( + "report_progress", + { body: `${marker}: reference-surface outcome recorded honestly.` }, + "progress", + "Record the outcome rather than papering over it.", + ), + ]; +} + +/* ------------------------------------------------------------------ */ + +function message(text: string): Phase7PlanStep { + return { kind: "agent_message", text }; +} + +function read(operationId: string, input: Phase7JsonValue, summary: string): Phase7PlanStep { + return { kind: "tool_call", operationId, input, summary }; +} + +function write( + operationId: string, + input: Phase7JsonValue, + idempotencySuffix: string, + summary: string, +): Phase7PlanStep { + return { kind: "tool_call", operationId, input, idempotencySuffix, summary }; +} diff --git a/packages/paperclip-runner/src/phase7/scenario-runner.ts b/packages/paperclip-runner/src/phase7/scenario-runner.ts new file mode 100644 index 00000000000..e1325fce175 --- /dev/null +++ b/packages/paperclip-runner/src/phase7/scenario-runner.ts @@ -0,0 +1,485 @@ +import { + Phase7MockControlPlaneAdapter, +} from "../mock-core/phase7-mock-control-plane-adapter.js"; +import type { + Phase7AuditRecord, + Phase7DecisionRecord, + Phase7FixtureState, + Phase7JsonValue, + Phase7ScheduledWake, +} from "../mock-core/phase7-control-plane-types.js"; +import { PHASE7_SEMANTIC_TOOL_CATALOG, phase7SemanticTool } from "../tools/phase7-semantic-tool-catalog.js"; +import { Phase7SemanticToolRuntime } from "../tools/phase7-semantic-tool-runtime.js"; +import { + PHASE7_CONTROL_PLANE_OWNED_OPERATION_IDS, + type Phase7AuthorizationRecord, + type Phase7SemanticToolDescriptor, + type Phase7ToolInvocationResult, +} from "../tools/phase7-semantic-tool-types.js"; +import { phase7ScenarioFixture } from "./scenario-fixtures.js"; +import { + CREATED_TASK_REF_PATTERN, + phase7ScenarioPlan, + type Phase7PlanStep, + type Phase7ScenarioPlan, +} from "./scenario-plan.js"; +import type { + Phase7Exposure, + Phase7ExposureEntry, + Phase7RedactionChip, + Phase7RunArtifact, + Phase7RunMode, + Phase7ScenarioIndexEntry, + Phase7TimelineEntry, +} from "./scenario-explorer-types.js"; +import { phase7StateDiff } from "./state-diff.js"; +import { phase7ScenarioParity, type Phase7EvalSuiteLookup } from "./scenario-parity.js"; + +/** + * Executes one scenario against the Phase 7C mock control plane and the Phase + * 7D semantic tool runtime, and records the result as a run artifact. + * + * Everything the explorer renders is produced here, in runtime code: exposure + * comes from the authorization engine, control-plane entries come from the + * mock core's own audit and decision records, the diff comes from immutable + * fixture snapshots, and parity comes from the traceability expectations. The + * browser is a read surface over this artifact and decides nothing. + */ + +export interface Phase7RunScenarioOptions { + mode?: Phase7RunMode; + /** Phase 7E parity output, when its report artifact has been loaded. */ + evalSuite?: Phase7EvalSuiteLookup; +} + +const SECRET_PLACEHOLDER = "fixture-secret-value-not-a-real-credential"; + +export async function phase7RunScenario( + entry: Phase7ScenarioIndexEntry, + options: Phase7RunScenarioOptions = {}, +): Promise { + const mode = options.mode ?? "fake"; + const fixture = phase7ScenarioFixture(entry); + const plan = phase7ScenarioPlan(entry, fixture); + const runId = `run-phase7-${entry.id}`; + + const adapter = new Phase7MockControlPlaneAdapter(fixture.seed); + await adapter.start(); + + const timeline: Phase7TimelineEntry[] = []; + let failure: Phase7RunArtifact["failure"] = null; + let sequence = 0; + let auditCursor = 0; + let decisionCursor = 0; + let wakeCursor = 0; + const createdTaskIds: string[] = []; + + const before = snapshotOf(adapter); + const context = await adapter.openFixtureRun({ + identity: { + runId, + sessionId: `session-phase7-${entry.id}`, + companyId: fixture.seed.company!.id!, + issueId: fixture.taskId, + agentId: fixture.actorId, + }, + backendKind: "mock", + sourceInstanceId: "phase7-scenario-explorer", + wake: { reason: entry.wakeReason, payload: { scenarioId: entry.id, mode } }, + capabilities: entry.scenarioClaims, + }); + + const runtime = new Phase7SemanticToolRuntime({ + adapter, + runId, + scenarioGrants: entry.scenarioClaims, + policy: entry.policy ?? undefined, + resolveSecretValue: () => SECRET_PLACEHOLDER, + }); + + const drainControlPlane = (): void => { + const state = adapter.snapshot(); + for (const audit of state.audit.slice(auditCursor)) { + if (audit.action === "semantic_command.applied") continue; + timeline.push(controlPlaneEntry(++sequence, audit, state.revision)); + } + auditCursor = state.audit.length; + for (const decision of state.decisions.slice(decisionCursor)) { + if (decision.outcome === "allowed" || decision.outcome === "duplicate") continue; + timeline.push(controlPlaneDecisionEntry(++sequence, decision)); + } + decisionCursor = state.decisions.length; + for (const wake of state.wakes.slice(wakeCursor)) { + timeline.push(controlPlaneWakeEntry(++sequence, wake, state.revision)); + } + wakeCursor = state.wakes.length; + }; + + drainControlPlane(); + + const visible = runtime.visibleTools(); + const exposure = buildExposure(visible.authorizationRecords, entry.scenarioClaims, plan); + + try { + for (const step of plan.steps) { + if (step.kind === "agent_message") { + timeline.push({ + sequence: ++sequence, + at: fixtureInstant(adapter), + kind: "agent_message", + channel: "agent", + text: step.text, + }); + continue; + } + sequence = await executeToolStep({ + step, + entry, + runtime, + adapter, + timeline, + sequence, + createdTaskIds, + }); + drainControlPlane(); + } + } catch (error) { + failure = { message: error instanceof Error ? error.message : String(error) }; + } + + drainControlPlane(); + + if (plan.restraint) { + timeline.push({ + sequence: ++sequence, + at: fixtureInstant(adapter), + kind: "system_note", + channel: "system", + note: `No further operations — ${entry.forbiddenSemantics.length} forbidden operations, none invoked.`, + forbiddenOperationCount: entry.forbiddenSemantics.length, + }); + } + + const after = snapshotOf(adapter); + await adapter.stop(); + + const authorizationRecords = [...runtime.authorizationRecords()].filter( + (record) => record.phase === "invocation", + ); + const diff = phase7StateDiff(before, after); + const parity = phase7ScenarioParity({ + entry, + timeline, + authorizationRecords, + diff, + exposure, + failure, + evalSuite: options.evalSuite, + }); + + return Object.freeze({ + schema: "paperclip.phase7.run-artifact.v1", + scenarioId: entry.id, + mode, + runId, + actor: { + id: context.actor.id, + name: context.actor.name, + role: context.actor.role, + capabilityGrants: [...context.actor.capabilityGrants], + }, + task: { + id: context.activeTask.id, + identifier: context.activeTask.identifier, + title: context.activeTask.title, + workMode: context.activeTask.workMode, + }, + wake: { reason: context.wake.reason, payload: context.wake.payload }, + budget: { ...context.budget }, + timeline, + exposure, + authorizationRecords, + diff, + parity, + failure, + }); +} + +interface ToolStepInput { + step: Extract; + entry: Phase7ScenarioIndexEntry; + runtime: Phase7SemanticToolRuntime; + adapter: Phase7MockControlPlaneAdapter; + timeline: Phase7TimelineEntry[]; + sequence: number; + createdTaskIds: string[]; +} + +async function executeToolStep(input: ToolStepInput): Promise { + const { step, entry, runtime, adapter, timeline, createdTaskIds } = input; + let sequence = input.sequence; + const descriptor = phase7SemanticTool(step.operationId); + const resolvedInput = resolveCreatedRefs(step.input, createdTaskIds); + // The catalog, not the plan, decides which operations need an idempotency + // key; plans only override the suffix when one step must differ from another. + const suffix = + step.idempotencySuffix ?? (descriptor?.idempotency === "required" ? step.operationId : undefined); + const idempotencyKey = suffix === undefined ? undefined : `phase7:${entry.id}:${suffix}`; + const redactions = descriptor === undefined ? [] : redactionChips(descriptor); + + const callSequence = ++sequence; + timeline.push({ + sequence: callSequence, + at: fixtureInstant(adapter), + kind: "semantic_call", + channel: "agent", + operationId: step.operationId, + disposition: descriptor?.disposition ?? "control_plane_owned", + summary: step.summary, + input: redactValue(resolvedInput, descriptor, "input"), + requiredClaims: descriptor === undefined ? [] : [...descriptor.requiredClaims], + idempotency: descriptor?.idempotency ?? "none", + idempotencyKey: idempotencyKey ?? null, + redactions, + escapeHatch: descriptor?.sideEffectClass === "test_escape_hatch", + authorizationSequence: 0, + }); + + const result: Phase7ToolInvocationResult = await runtime.invoke({ + operationId: step.operationId, + input: resolvedInput, + idempotencyKey, + }); + + const call = timeline.find((candidate) => candidate.sequence === callSequence); + if (call?.kind === "semantic_call") call.authorizationSequence = result.authorization.sequence; + + if (result.ok) { + for (const ref of result.commandResult?.entityRefs ?? []) { + if (ref.startsWith("task:") && ref.slice(5) !== adapterActiveTaskId(adapter)) { + createdTaskIds.push(ref.slice(5)); + } + } + timeline.push({ + sequence: ++sequence, + at: fixtureInstant(adapter), + kind: "semantic_result", + channel: "agent", + operationId: step.operationId, + outcome: + result.commandResult === null + ? "read" + : result.commandResult.disposition === "duplicate" + ? "duplicate" + : "committed", + summary: resultSummary(step.operationId, result), + value: redactValue(result.value, descriptor, "output"), + failure: null, + redactions, + authorizationSequence: result.authorization.sequence, + protocolEventRefs: (result.commandResult?.scheduledWakeIds ?? []).map((id) => `wake:${id}`), + }); + return sequence; + } + + timeline.push({ + sequence: ++sequence, + at: fixtureInstant(adapter), + kind: "semantic_result", + channel: "agent", + operationId: step.operationId, + outcome: result.error.code === "operation_absent" ? "absent" : "denied", + summary: + result.error.code === "operation_absent" + ? `\`${step.operationId}\` is not exposed to this run.` + : `\`${step.operationId}\` was denied — ${result.authorization.reason}.`, + value: null, + failure: { code: result.error.code, reason: result.authorization.reason }, + redactions, + authorizationSequence: result.authorization.sequence, + protocolEventRefs: [], + }); + return sequence; +} + +function adapterActiveTaskId(adapter: Phase7MockControlPlaneAdapter): string { + const state = adapter.snapshot(); + const run = state.runs.find((candidate) => candidate.id === state.activeRunId); + return run?.taskId ?? ""; +} + +function resolveCreatedRefs(value: Phase7JsonValue, createdTaskIds: string[]): Phase7JsonValue { + if (typeof value === "string") { + const match = CREATED_TASK_REF_PATTERN.exec(value); + if (match === null) return value; + return createdTaskIds[Number(match[1]) - 1] ?? value; + } + if (Array.isArray(value)) return value.map((item) => resolveCreatedRefs(item, createdTaskIds)); + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [key, resolveCreatedRefs(child, createdTaskIds)]), + ); + } + return value; +} + +/** + * Applies the descriptor's redaction rules at the artifact boundary so no raw + * secret can reach a rendered record, even if an upstream layer returns one + * (UX map §5: the explorer fails closed rather than receiving-and-hiding). + */ +function redactValue( + value: Phase7JsonValue, + descriptor: Phase7SemanticToolDescriptor | undefined, + channel: "input" | "output", +): Phase7JsonValue { + if (descriptor === undefined) return value; + let redacted = value; + for (const rule of descriptor.redaction) { + if (!rule.appliesTo.includes(channel)) continue; + redacted = applyRedactionPath(redacted, rule.path.replace(/^\$\.?/, "").split("."), rule.replacement); + } + return redacted; +} + +function applyRedactionPath( + value: Phase7JsonValue, + path: string[], + replacement: string, +): Phase7JsonValue { + const [head, ...rest] = path; + if (head === undefined) return replacement; + if (typeof value !== "object" || value === null || Array.isArray(value)) return value; + if (!(head in value)) return value; + return { ...value, [head]: applyRedactionPath(value[head]!, rest, replacement) }; +} + +function redactionChips(descriptor: Phase7SemanticToolDescriptor): Phase7RedactionChip[] { + return descriptor.redaction.map((rule) => ({ + path: rule.path, + replacement: rule.replacement, + rule: `${descriptor.operationId}:${rule.path}`, + })); +} + +function resultSummary(operationId: string, result: Phase7ToolInvocationResult): string { + if (!result.ok) return `\`${operationId}\` produced no result.`; + if (result.commandResult === null) return `\`${operationId}\` returned a read projection.`; + const refs = result.commandResult.entityRefs.length; + return `\`${operationId}\` ${result.commandResult.disposition} at revision ${result.commandResult.stateRevision}${ + refs === 0 ? "" : ` · ${refs} ${refs === 1 ? "entity" : "entities"}` + }.`; +} + +function buildExposure( + records: readonly Phase7AuthorizationRecord[], + grants: readonly string[], + plan: Phase7ScenarioPlan, +): Phase7Exposure { + const always: Phase7ExposureEntry[] = []; + const optional: Phase7ExposureEntry[] = []; + const withheld: Phase7ExposureEntry[] = []; + for (const record of records) { + const descriptor = phase7SemanticTool(record.operationId); + if (descriptor === undefined) continue; + const entry: Phase7ExposureEntry = { + operationId: descriptor.operationId, + title: descriptor.title, + requiredClaims: [...descriptor.requiredClaims], + unlockedBy: + descriptor.disposition === "optional_agent_tool" + ? (descriptor.requiredClaims.find((claim) => grants.includes(claim)) ?? null) + : null, + reason: record.reason, + }; + if (record.outcome !== "exposed") withheld.push(entry); + else if (descriptor.disposition === "always_agent_tool") always.push(entry); + else optional.push(entry); + } + const controlPlane = plan.controlPlaneCapabilities.filter((capability) => + (PHASE7_CONTROL_PLANE_OWNED_OPERATION_IDS as readonly string[]).includes(capability.operationId), + ); + return { + schema: "paperclip.phase7.exposure.v1", + always, + optional, + controlPlane, + withheld, + }; +} + +function controlPlaneEntry( + sequence: number, + audit: Phase7AuditRecord, + stateRevision: number, +): Phase7TimelineEntry { + return { + sequence, + at: audit.at, + kind: "control_plane_action", + channel: "control_plane", + action: audit.action, + summary: CONTROL_PLANE_COPY[audit.action] ?? `Control plane recorded \`${audit.action}\`.`, + detail: audit.details, + auditRef: audit.id, + decisionRef: null, + stateRevision, + }; +} + +function controlPlaneDecisionEntry( + sequence: number, + decision: Phase7DecisionRecord, +): Phase7TimelineEntry { + return { + sequence, + at: decision.at, + kind: "control_plane_action", + channel: "control_plane", + action: `${decision.operation}.${decision.outcome}`, + summary: `Control plane ${decision.outcome} \`${decision.operation}\` — ${decision.reason}. No agent tool exists for this.`, + detail: { operation: decision.operation, reason: decision.reason, entityRefs: decision.entityRefs }, + auditRef: null, + decisionRef: decision.id, + stateRevision: decision.stateRevision, + }; +} + +function controlPlaneWakeEntry( + sequence: number, + wake: Phase7ScheduledWake, + stateRevision: number, +): Phase7TimelineEntry { + return { + sequence, + at: wake.createdAt, + kind: "control_plane_action", + channel: "control_plane", + action: "wake.scheduled", + summary: `Control plane scheduled a \`${wake.reason}\` wake for task ${wake.taskId} — no agent tool exists for this.`, + detail: { id: wake.id, reason: wake.reason, dueAt: wake.dueAt, status: wake.status }, + auditRef: null, + decisionRef: null, + stateRevision, + }; +} + +const CONTROL_PLANE_COPY: Record = { + "run.opened": + "Control plane checked out the task and opened the run — no agent tool exists for this.", + "run.event_appended": "Control plane appended a protocol event to the durable run log.", + "run.session_checkpointed": "Control plane checkpointed the session for replay.", + "run.completed": "Control plane reconciled the terminal run result.", + "budget.hard_stop": "Control plane stopped the run on a hard budget limit.", +}; + +function snapshotOf(adapter: Phase7MockControlPlaneAdapter): Phase7FixtureState { + return JSON.parse(adapter.serialize()) as Phase7FixtureState; +} + +function fixtureInstant(adapter: Phase7MockControlPlaneAdapter): string { + const clock = adapter.snapshot().clock; + return new Date(clock.epochMs + clock.tick * 1_000).toISOString(); +} + +export const PHASE7_CATALOG_SIZE = PHASE7_SEMANTIC_TOOL_CATALOG.length; diff --git a/packages/paperclip-runner/src/phase7/state-diff.ts b/packages/paperclip-runner/src/phase7/state-diff.ts new file mode 100644 index 00000000000..2af5bbf21b8 --- /dev/null +++ b/packages/paperclip-runner/src/phase7/state-diff.ts @@ -0,0 +1,132 @@ +import type { + Phase7FixtureState, + Phase7JsonValue, +} from "../mock-core/phase7-control-plane-types.js"; +import type { + Phase7DiffDomain, + Phase7DiffDomainSection, + Phase7DiffFieldChange, + Phase7DiffRow, + Phase7StateDiff, +} from "./scenario-explorer-types.js"; + +/** + * Immutable before/after diff of the mock fixture state, grouped by the ten + * entity domains the Phase 7 plan names. The browser renders this verbatim; it + * performs no diffing of its own (UX map §4.3). + */ + +type Entity = Record & { id: string }; + +const DOMAINS: Array<{ + domain: Phase7DiffDomain; + select: (state: Phase7FixtureState) => Entity[]; + noun: string; +}> = [ + { domain: "tasks", select: (state) => state.tasks as unknown as Entity[], noun: "task" }, + { domain: "comments", select: (state) => state.comments as unknown as Entity[], noun: "comment" }, + { domain: "documents", select: (state) => state.documents as unknown as Entity[], noun: "document" }, + { domain: "interactions", select: (state) => state.interactions as unknown as Entity[], noun: "interaction" }, + { domain: "approvals", select: (state) => state.approvals as unknown as Entity[], noun: "approval" }, + { + domain: "artifacts", + select: (state) => [...state.artifacts, ...state.workProducts] as unknown as Entity[], + noun: "artifact", + }, + { domain: "blockers", select: (state) => state.blockers as unknown as Entity[], noun: "blocker" }, + { + domain: "workspace", + select: (state) => state.workspaceServices as unknown as Entity[], + noun: "service", + }, + { domain: "budget", select: (state) => state.budgets as unknown as Entity[], noun: "budget" }, + { + domain: "run", + select: (state) => state.runs.map((run) => ({ ...run, events: run.events.length })) as unknown as Entity[], + noun: "run", + }, +]; + +export function phase7StateDiff( + before: Phase7FixtureState, + after: Phase7FixtureState, +): Phase7StateDiff { + return { + schema: "paperclip.phase7.state-diff.v1", + beforeRevision: before.revision, + afterRevision: after.revision, + domains: DOMAINS.map(({ domain, select, noun }) => + domainSection(domain, noun, select(before), select(after)), + ), + }; +} + +function domainSection( + domain: Phase7DiffDomain, + noun: string, + before: Entity[], + after: Entity[], +): Phase7DiffDomainSection { + const beforeById = new Map(before.map((entity) => [entity.id, entity])); + const afterById = new Map(after.map((entity) => [entity.id, entity])); + const rows: Phase7DiffRow[] = []; + + for (const [id, entity] of afterById) { + const previous = beforeById.get(id); + if (previous === undefined) { + rows.push({ entityId: id, change: "added", fields: describe(entity) }); + continue; + } + const fields = fieldChanges(previous, entity); + if (fields.length > 0) rows.push({ entityId: id, change: "changed", fields }); + } + for (const [id, entity] of beforeById) { + if (!afterById.has(id)) rows.push({ entityId: id, change: "removed", fields: describe(entity) }); + } + rows.sort((left, right) => left.entityId.localeCompare(right.entityId)); + + const counts = rows.reduce>((totals, row) => { + totals[row.change] = (totals[row.change] ?? 0) + 1; + return totals; + }, {}); + const summary = + rows.length === 0 + ? "unchanged" + : (["added", "changed", "removed"] as const) + .filter((change) => counts[change] !== undefined) + .map((change) => `${counts[change]} ${change}`) + .join(" · "); + + return { + domain, + summary: rows.length === 0 ? summary : `${summary} ${rows.length === 1 ? noun : `${noun}s`}`, + changed: rows.length > 0, + rows, + afterSnapshot: JSON.parse(JSON.stringify(after)) as Phase7JsonValue, + }; +} + +function describe(entity: Entity): Phase7DiffFieldChange[] { + return Object.entries(entity) + .filter(([field]) => field !== "id") + .map(([field, value]) => ({ field, before: null, after: render(value) })) + .filter((change) => change.after !== null) + .sort((left, right) => left.field.localeCompare(right.field)); +} + +function fieldChanges(before: Entity, after: Entity): Phase7DiffFieldChange[] { + const fields = [...new Set([...Object.keys(before), ...Object.keys(after)])].sort(); + return fields + .filter((field) => field !== "id") + .map((field) => ({ field, before: render(before[field]), after: render(after[field]) })) + .filter((change) => change.before !== change.after); +} + +function render(value: unknown): string | null { + if (value === undefined) return null; + if (value === null) return "null"; + if (typeof value === "string") return value.length > 160 ? `${value.slice(0, 157)}…` : value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + const json = JSON.stringify(value); + return json.length > 160 ? `${json.slice(0, 157)}…` : json; +} diff --git a/packages/paperclip-runner/tsconfig.browser.json b/packages/paperclip-runner/tsconfig.browser.json index d099284e751..c039b36a40f 100644 --- a/packages/paperclip-runner/tsconfig.browser.json +++ b/packages/paperclip-runner/tsconfig.browser.json @@ -11,7 +11,8 @@ "paths": { "@paperclipai/paperclip-runner/browser": ["src/browser/index.ts"], "@paperclipai/paperclip-runner/react": ["src/react/index.ts"], - "@paperclipai/paperclip-runner/styles.css": ["styles.css"] + "@paperclipai/paperclip-runner/styles.css": ["styles.css"], + "@paperclip-runner-local/phase7": ["src/phase7/index.ts"] } }, "include": [ @@ -19,5 +20,6 @@ "devtools/browser/src/**/*.tsx", "examples/*/src/**/*.ts", "examples/*/src/**/*.tsx" - ] + ], + "exclude": ["**/*.test.ts", "**/*.test.tsx"] } diff --git a/packages/paperclip-runner/vite.phase7.config.ts b/packages/paperclip-runner/vite.phase7.config.ts new file mode 100644 index 00000000000..efd473b06cd --- /dev/null +++ b/packages/paperclip-runner/vite.phase7.config.ts @@ -0,0 +1,48 @@ +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import react from "@vitejs/plugin-react"; +import { defineConfig } from "vite"; + +import { phase7ExplorerFixturesPlugin } from "./scripts/phase7-explorer-fixtures.mjs"; + +const packageRoot = fileURLToPath(new URL(".", import.meta.url)); + +/** + * Phase 7 scenario explorer entry. + * + * `@paperclip-runner-local/*` is deliberately not the published SDK name: the + * scenario runtime is package-local and stays outside the frozen 0.1.2 export + * surface, while the explorer consumes `@paperclipai/paperclip-runner/react` + * and its token sheet exactly as an external consumer would. + */ +export default defineConfig({ + root: resolve(packageRoot, "examples"), + plugins: [react(), phase7ExplorerFixturesPlugin()], + resolve: { + alias: [ + { + find: "@paperclipai/paperclip-runner/react", + replacement: resolve(packageRoot, "src/react/index.ts"), + }, + { + find: "@paperclipai/paperclip-runner/styles.css", + replacement: resolve(packageRoot, "styles.css"), + }, + { + find: "@paperclip-runner-local/phase7", + replacement: resolve(packageRoot, "src/phase7/index.ts"), + }, + ], + }, + server: { host: "127.0.0.1" }, + preview: { host: "127.0.0.1" }, + build: { + outDir: resolve(packageRoot, "dist-phase7"), + emptyOutDir: true, + target: "esnext", + rollupOptions: { + input: resolve(packageRoot, "examples/phase7-explorer/index.html"), + }, + }, +}); diff --git a/packages/paperclip-runner/vitest.config.ts b/packages/paperclip-runner/vitest.config.ts index d430a45c5d2..4c4de89345b 100644 --- a/packages/paperclip-runner/vitest.config.ts +++ b/packages/paperclip-runner/vitest.config.ts @@ -1,12 +1,25 @@ +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + import { defineConfig } from "vitest/config"; +const packageRoot = fileURLToPath(new URL(".", import.meta.url)); + export default defineConfig({ + resolve: { + alias: [ + { find: "@paperclipai/paperclip-runner/react", replacement: resolve(packageRoot, "src/react/index.ts") }, + { find: "@paperclip-runner-local/phase7", replacement: resolve(packageRoot, "src/phase7/index.ts") }, + ], + }, test: { environment: "node", include: [ "src/**/*.test.ts", "src/**/*.test.tsx", "devtools/browser/src/**/*.test.ts", + "examples/phase7-explorer/src/**/*.test.ts", + "examples/phase7-explorer/src/**/*.test.tsx", "scripts/phase2-browser-server.test.mjs", "scripts/phase4b-browser-server.test.mjs", ],