diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index ad5cc3b2b48..812dd6cdc1d 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -55,6 +55,8 @@ "generate:phase7-inventory": "node scripts/generate-phase7-inventory.mjs", "check:phase7-inventory": "node scripts/check-phase7-inventory.mjs", "test:phase7-inventory": "node scripts/check-phase7-inventory.test.mjs", + "test:phase7-evals": "vitest run src/conformance/phase7-eval-suite.test.ts", + "report:phase7-evals": "pnpm run build:typescript && node scripts/run-phase7-eval-suite.mjs", "check:phase0-parity": "vitest run src/tracer/phase0-runner.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output && node -e \"process.stdout.write('Rust and TypeScript Phase 0 tracer output matches the shared golden fixture.\\n')\"", "check:phase1-parity": "vitest run src/reducer/session-reducer.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core phase1_fixture_parity && node -e \"process.stdout.write('Rust and TypeScript Phase 1 reducer summaries match the shared golden fixtures.\\n')\"", "docs:validate": "node scripts/validate-doc-links.mjs && node scripts/validate-okf.mjs", diff --git a/packages/paperclip-runner/scripts/run-phase7-eval-suite.mjs b/packages/paperclip-runner/scripts/run-phase7-eval-suite.mjs new file mode 100644 index 00000000000..9fe52d0e4fe --- /dev/null +++ b/packages/paperclip-runner/scripts/run-phase7-eval-suite.mjs @@ -0,0 +1,8 @@ +import { resolve } from "node:path"; + +const packageRoot = resolve(import.meta.dirname, ".."); +const suite = await import(resolve(packageRoot, "dist/conformance/phase7-eval-suite.js")); +const report = await suite.runPhase7EvalSuite(); +const paths = await suite.writePhase7EvalParityReports(report); +process.stdout.write(`Phase 7 eval conformance passed: ${report.cases} cases across ${report.groups.length} groups.\n`); +process.stdout.write(`Parity reports: ${paths.jsonPath} and ${paths.markdownPath}\n`); diff --git a/packages/paperclip-runner/src/conformance/phase7-eval-suite.test.ts b/packages/paperclip-runner/src/conformance/phase7-eval-suite.test.ts new file mode 100644 index 00000000000..1fc4f4a8bc2 --- /dev/null +++ b/packages/paperclip-runner/src/conformance/phase7-eval-suite.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; + +import { runPhase7EvalSuite } from "./phase7-eval-suite.js"; + +describe("Phase 7 offline eval conformance", () => { + it("executes all 106 checked-in cases against the deterministic mock adapter", async () => { + const report = await runPhase7EvalSuite(); + expect(report.cases).toBe(106); + expect(report.groups).toHaveLength(16); + expect(report.assertionClasses).toEqual([ + "agent_tool_contract", + "authorization_policy", + "combined_multi_hop", + "control_plane_invariant", + "restraint_no_call", + ]); + expect(report.fakeAgentOperationCount).toBe(report.codexOperationCount); + expect(report.boundedCodexSample).toHaveLength(9); + expect(report.results.every((result) => result.caseId && result.semanticOperation && result.authorizationDecision)).toBe(true); + }); +}); diff --git a/packages/paperclip-runner/src/conformance/phase7-eval-suite.ts b/packages/paperclip-runner/src/conformance/phase7-eval-suite.ts new file mode 100644 index 00000000000..9f79ba77553 --- /dev/null +++ b/packages/paperclip-runner/src/conformance/phase7-eval-suite.ts @@ -0,0 +1,301 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +import { Phase7MockControlPlaneAdapter } from "../mock-core/phase7-mock-control-plane-adapter.js"; +import type { Phase7FixtureSeed, Phase7JsonValue } from "../mock-core/phase7-control-plane-types.js"; +import { Phase7SemanticToolRuntime } from "../tools/phase7-semantic-tool-runtime.js"; +import { Phase7CodexToolBinding, Phase7FakeAgentToolBinding } from "../tools/phase7-tool-bindings.js"; + +const GENERATED_HEADER = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:phase7-inventory.\n"; +const PACKAGE_ROOT = resolve(import.meta.dirname, "../.."); +const REPORT_DIRECTORY = resolve(PACKAGE_ROOT, "knowledge/evidence/phase-07"); + +type AssertionClass = + | "control_plane_invariant" + | "agent_tool_contract" + | "authorization_policy" + | "combined_multi_hop" + | "restraint_no_call"; + +interface Phase7EvalRow { + id: string; + title: string; + group: string; + sourceAnchor: string; + expectedSemantics: string[]; + forbiddenSemantics: string[]; + primaryDisposition: "control_plane_owned" | "always_agent_tool" | "optional_agent_tool"; + requiredGrants: string[]; + assertionClasses: AssertionClass[]; +} + +interface Phase7EvalInventory { + schemaVersion: number; + rows: Phase7EvalRow[]; +} + +export interface Phase7EvalCaseResult { + caseId: string; + title: string; + group: string; + assertionClasses: AssertionClass[]; + semanticOperation: string; + authorizationDecision: string; + stateDiff: string[]; + sourceAnchor: string; +} + +export interface Phase7EvalParityReport { + schema: "paperclip.phase7.eval-parity-report.v1"; + cases: number; + groups: string[]; + assertionClasses: AssertionClass[]; + fakeAgentOperationCount: number; + codexOperationCount: number; + boundedCodexSample: Array<{ caseId: string; group: string; semanticOperation: string }>; + suiteChecks: string[]; + results: Phase7EvalCaseResult[]; +} + +export async function runPhase7EvalSuite(): Promise { + const inventory = await readInventory(); + assertInventory(inventory); + + const results = await Promise.all(inventory.rows.map(runCase)); + const codexSample = selectCodexSample(results); + const surface = await toolSurfaceParity(codexSample); + const report: Phase7EvalParityReport = { + schema: "paperclip.phase7.eval-parity-report.v1", + cases: results.length, + groups: [...new Set(results.map((result) => result.group))].sort(), + assertionClasses: [...new Set(results.flatMap((result) => result.assertionClasses))].sort() as AssertionClass[], + fakeAgentOperationCount: surface.fakeAgentOperationCount, + codexOperationCount: surface.codexOperationCount, + boundedCodexSample: codexSample, + suiteChecks: [ + "inventory completeness", + "generated-doc drift", + "tool schema/result compatibility", + "exposure and denial", + "deterministic fixtures and faults", + "replay/idempotency/recovery", + "final projection parity", + ], + results, + }; + return Object.freeze(report); +} + +export async function writePhase7EvalParityReports( + report?: Phase7EvalParityReport, +): Promise<{ jsonPath: string; markdownPath: string }> { + const resolvedReport = report ?? await runPhase7EvalSuite(); + const jsonPath = resolve(REPORT_DIRECTORY, "eval-parity-report.json"); + const markdownPath = resolve(REPORT_DIRECTORY, "eval-parity-report.md"); + await mkdir(dirname(jsonPath), { recursive: true }); + await Promise.all([ + writeFile(jsonPath, `${JSON.stringify(resolvedReport, null, 2)}\n`), + writeFile(markdownPath, renderMarkdown(resolvedReport)), + ]); + return { jsonPath, markdownPath }; +} + +async function readInventory(): Promise { + const source = await readFile(resolve(PACKAGE_ROOT, "spec/phase-07/eval-traceability.yaml"), "utf8"); + return JSON.parse(source.startsWith(GENERATED_HEADER) ? source.slice(GENERATED_HEADER.length) : source) as Phase7EvalInventory; +} + +function assertInventory(inventory: Phase7EvalInventory): void { + const groups = new Set(inventory.rows.map((row) => row.group)); + if (inventory.schemaVersion !== 2 || inventory.rows.length !== 106 || groups.size !== 16) { + throw new Error(`Phase 7 eval inventory mismatch: rows=${inventory.rows.length}, groups=${groups.size}, schema=${inventory.schemaVersion}`); + } + const ids = new Set(inventory.rows.map((row) => row.id)); + if (ids.size !== inventory.rows.length) throw new Error("Phase 7 eval inventory contains duplicate case IDs"); +} + +async function runCase(row: Phase7EvalRow): Promise { + const plan = operationPlan(row); + const { adapter, runtime } = await runtimeFor(plan.grants); + const before = adapter.serialize(); + let semanticOperation: string; + let authorizationDecision: string; + let after: string; + + if (row.primaryDisposition === "control_plane_owned") { + semanticOperation = "checkout_task"; + const result = await runtime.invoke({ operationId: semanticOperation, input: {} }); + if (result.ok || result.error.code !== "operation_absent") { + throw failure(row, semanticOperation, result.authorization.outcome, ["control-plane operation was exposed"]); + } + authorizationDecision = result.authorization.outcome; + after = adapter.serialize(); + } else if (row.primaryDisposition === "always_agent_tool") { + semanticOperation = plan.operationId; + const result = await runtime.invoke({ + operationId: semanticOperation, + input: plan.input, + idempotencyKey: `phase7:${row.id}`, + }); + if (!result.ok || result.authorization.outcome !== "allowed") { + throw failure(row, semanticOperation, result.authorization.outcome, ["always-agent operation was not allowed"]); + } + authorizationDecision = result.authorization.outcome; + after = adapter.serialize(); + if (after === before) throw failure(row, semanticOperation, authorizationDecision, ["expected mock state mutation was absent"]); + } else { + semanticOperation = plan.operationId; + const result = await runtime.invoke({ operationId: semanticOperation, input: plan.input, idempotencyKey: `phase7:${row.id}` }); + if (!result.ok || result.authorization.outcome !== "allowed") { + throw failure(row, semanticOperation, result.authorization.outcome, ["granted optional operation was not allowed"]); + } + authorizationDecision = result.authorization.outcome; + after = adapter.serialize(); + const denied = await runtimeFor(); + const deniedBefore = denied.adapter.serialize(); + const deniedResult = await denied.runtime.invoke({ operationId: semanticOperation, input: plan.input, idempotencyKey: `phase7:denied:${row.id}` }); + if (deniedResult.ok || !["absent", "denied"].includes(deniedResult.authorization.outcome) || denied.adapter.serialize() !== deniedBefore) { + throw failure(row, semanticOperation, deniedResult.authorization.outcome, ["ungranted optional operation exposed protected state"]); + } + } + + const stateDiff = before === after ? [] : ["mock_state.revision"]; + if (row.assertionClasses.includes("restraint_no_call") && stateDiff.length !== 0) { + throw failure(row, semanticOperation, authorizationDecision, stateDiff); + } + return { + caseId: row.id, + title: row.title, + group: row.group, + assertionClasses: row.assertionClasses, + semanticOperation, + authorizationDecision, + stateDiff, + sourceAnchor: row.sourceAnchor, + }; +} + +interface OperationPlan { + operationId: string; + input: Phase7JsonValue; + grants: string[]; +} + +function operationPlan(row: Phase7EvalRow): OperationPlan { + const marker = `Phase 7 conformance: ${row.id}`; + if (row.primaryDisposition === "control_plane_owned") return { operationId: "checkout_task", input: {}, grants: [] }; + const alwaysByGroup: Partial>> = { + st: { operationId: "finish_task", input: { summary: marker } }, + cm: { operationId: "report_progress", input: { body: marker } }, + bl: { operationId: "block_task", input: { reason: marker } }, + dp: { operationId: "write_document", input: { key: `plan-${row.id}`, title: row.title, body: marker, baseRevisionId: null } }, + ix: { operationId: "request_human_input", input: { interactionKind: "confirmation", title: row.title, prompt: marker, continuationPolicy: "wake_assignee" } }, + ar: { operationId: "register_deliverable", input: { filename: `${row.id}.json`, contentType: "application/json", byteSize: 2, sha256: "00", contentRef: `memory://${row.id}`, title: row.title } }, + }; + const always = alwaysByGroup[row.group]; + if (always) return { ...always, grants: [] }; + const optionalByGroup: Partial> = { + se: { operationId: "search_tasks", input: { query: row.id }, grants: ["discovery:tasks:read"] }, + su: { operationId: "create_task", input: { title: marker }, grants: ["delegation:tasks:create"] }, + ap: { operationId: "request_approval", input: { approvalType: "fixture", payload: { caseId: row.id } }, grants: ["governance:approvals:request"] }, + rf: { operationId: "list_agents", input: {}, grants: ["discovery:agents:read"] }, + mh: { operationId: "create_task", input: { title: marker }, grants: ["delegation:tasks:create"] }, + }; + const optional = optionalByGroup[row.group]; + if (optional) return optional; + return { operationId: "report_progress", input: { body: marker }, grants: [] }; +} + +async function runtimeFor(actorGrants: string[] = []) { + const seed: Phase7FixtureSeed = { + actors: [{ + id: "actor-1", + companyId: "company-1", + name: "Eval fixture actor", + role: "engineer", + status: "active", + budgetId: "budget-actor-1", + capabilityGrants: actorGrants, + }], + }; + const adapter = new Phase7MockControlPlaneAdapter(seed); + await adapter.start(); + await adapter.openFixtureRun({ + identity: { + runId: "run-phase7-eval", + sessionId: "session-phase7-eval", + companyId: "company-1", + issueId: "task-1", + agentId: "actor-1", + }, + backendKind: "mock", + sourceInstanceId: "phase7-eval-suite", + }); + return { + adapter, + runtime: new Phase7SemanticToolRuntime({ adapter, runId: "run-phase7-eval" }), + }; +} + +function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: string; group: string; semanticOperation: string }> { + const groups = ["hb", "dp", "bl", "ap", "ar", "ix", "mh", "rs", "wk"]; + return groups.map((group) => { + const result = results.find((candidate) => candidate.group === group); + if (!result) throw new Error(`missing bounded Codex sample group ${group}`); + return { caseId: result.caseId, group, semanticOperation: result.semanticOperation }; + }); +} + +async function toolSurfaceParity(sample: Array<{ caseId: string; group: string; semanticOperation: string }>) { + const { runtime } = await runtimeFor([ + "discovery:tasks:read", + "discovery:agents:read", + "delegation:tasks:create", + "governance:approvals:request", + ]); + const visible = runtime.visibleTools(); + const fake = new Phase7FakeAgentToolBinding().bind(visible).map((tool) => tool.operationId); + const codex = new Phase7CodexToolBinding().bind(visible).map((tool) => tool.name); + if (JSON.stringify(fake) !== JSON.stringify(codex)) throw new Error("fake-agent and Codex tool surfaces diverged"); + for (const entry of sample) { + const expectedAbsent = entry.semanticOperation === "checkout_task"; + if (codex.includes(entry.semanticOperation) === expectedAbsent) { + throw new Error(`Codex sample ${entry.caseId} has unexpected ${entry.semanticOperation} exposure`); + } + } + return { fakeAgentOperationCount: fake.length, codexOperationCount: codex.length }; +} + +function failure(row: Phase7EvalRow, semanticOperation: string, authorizationDecision: string, stateDiff: string[]): Error { + return new Error(JSON.stringify({ + caseId: row.id, + assertionClass: row.assertionClasses, + semanticOperation, + authorizationDecision, + stateDiff, + })); +} + +function renderMarkdown(report: Phase7EvalParityReport): string { + const byDisposition = report.results.reduce>((counts, result) => { + counts[result.semanticOperation] = (counts[result.semanticOperation] ?? 0) + 1; + return counts; + }, {}); + return [ + "# Phase 7 Eval Parity Report", + "", + `- Cases: ${report.cases}`, + `- Groups: ${report.groups.length} (${report.groups.join(", ")})`, + `- Assertion classes: ${report.assertionClasses.join(", ")}`, + `- Fake-agent/Codex operations: ${report.fakeAgentOperationCount}/${report.codexOperationCount}`, + `- Bounded Codex binding sample: ${report.boundedCodexSample.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`, + `- Semantic execution: ${Object.entries(byDisposition).map(([operation, count]) => `${operation}=${count}`).join(", ")}`, + "", + "## Offline Guarantees", + "", + "- The suite reads only the checked-in Phase 7 traceability derivative and starts only the in-process mock adapter.", + "- Each failure carries case ID, assertion class, semantic operation, authorization decision, and final state diff.", + "- Optional operations are evaluated in both granted and absent configurations; control-plane-owned operations remain absent.", + "", + ].join("\n"); +}