From 0a0eb19713ce8e3b57e76cae6b37e6ada502c470 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 11 Aug 2026 15:21:28 +0000 Subject: [PATCH] feat(runner): define package and eval boundaries Co-Authored-By: Paperclip --- packages/paperclip-eval-kernel/README.md | 14 + packages/paperclip-eval-kernel/package.json | 28 ++ packages/paperclip-eval-kernel/src/index.ts | 98 +++++++ .../test/kernel.test.mjs | 54 ++++ packages/paperclip-eval-kernel/tsconfig.json | 8 + packages/paperclip-runner/README.md | 15 ++ ...-runner-testing-eval-package-boundaries.md | 129 +++++++++ .../paperclip-runner/docs/architecture.md | 17 +- .../docs/capability-live-runnerd-codex.md | 2 +- packages/paperclip-runner/docs/index.md | 1 + .../docs/protocol-compatibility.md | 12 + packages/paperclip-runner/package.json | 20 +- .../scripts/check-clean-consumers.mjs | 248 ++++++++++++++++++ .../scripts/check-package-boundaries.mjs | 53 ++++ .../scripts/lib/forbidden-imports.mjs | 8 + .../src/compatibility.test.ts | 72 +++++ .../paperclip-runner/src/compatibility.ts | 149 +++++++++++ .../conformance/semantic-conformance.test.ts | 49 ++++ .../src/conformance/semantic-conformance.ts | 104 ++++++++ packages/paperclip-runner/src/index.ts | 31 +-- packages/paperclip-runner/src/testing.ts | 15 ++ .../native-finalization-recovery.test.ts | 2 +- .../native-session-resumption.test.ts | 6 +- .../paperclip-control-plane-port.test.ts | 10 +- 24 files changed, 1105 insertions(+), 40 deletions(-) create mode 100644 packages/paperclip-eval-kernel/README.md create mode 100644 packages/paperclip-eval-kernel/package.json create mode 100644 packages/paperclip-eval-kernel/src/index.ts create mode 100644 packages/paperclip-eval-kernel/test/kernel.test.mjs create mode 100644 packages/paperclip-eval-kernel/tsconfig.json create mode 100644 packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md create mode 100644 packages/paperclip-runner/scripts/check-clean-consumers.mjs create mode 100644 packages/paperclip-runner/scripts/check-package-boundaries.mjs create mode 100644 packages/paperclip-runner/src/compatibility.test.ts create mode 100644 packages/paperclip-runner/src/compatibility.ts create mode 100644 packages/paperclip-runner/src/conformance/semantic-conformance.test.ts create mode 100644 packages/paperclip-runner/src/conformance/semantic-conformance.ts create mode 100644 packages/paperclip-runner/src/testing.ts diff --git a/packages/paperclip-eval-kernel/README.md b/packages/paperclip-eval-kernel/README.md new file mode 100644 index 00000000000..a7b97a0667e --- /dev/null +++ b/packages/paperclip-eval-kernel/README.md @@ -0,0 +1,14 @@ +# Paperclip Eval Kernel + +`@paperclipai/paperclip-eval-kernel` is the generic, provider-neutral matrix +orchestrator owned by Paperclip Evals. It contains no Paperclip scenario corpus, +provider configuration, product fixture, scorer, or report template. + +Consumers pass scenario and candidate values plus execution and scoring +callbacks. Candidate `preflight` hooks should call the runner package's +`assertPaperclipRunnerCompatibility` before any provider work starts. This keeps +catalog, protocol, runner-client, control-plane-adapter, testkit, corpus, and +provider-operation incompatibilities explicit. + +Paperclip App may consume this package only as a development dependency for CI +or parity tests. `@paperclipai/paperclip-runner` has no runtime dependency on it. diff --git a/packages/paperclip-eval-kernel/package.json b/packages/paperclip-eval-kernel/package.json new file mode 100644 index 00000000000..6869f2f41ff --- /dev/null +++ b/packages/paperclip-eval-kernel/package.json @@ -0,0 +1,28 @@ +{ + "name": "@paperclipai/paperclip-eval-kernel", + "version": "0.1.0", + "description": "Provider-neutral evaluation matrix kernel without Paperclip scenario content", + "type": "module", + "files": [ + "dist", + "README.md" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "pnpm run build && node --test test/*.test.mjs" + }, + "engines": { + "node": ">=20" + }, + "license": "MIT", + "devDependencies": { + "typescript": "^5.7.3" + } +} diff --git a/packages/paperclip-eval-kernel/src/index.ts b/packages/paperclip-eval-kernel/src/index.ts new file mode 100644 index 00000000000..792406249e7 --- /dev/null +++ b/packages/paperclip-eval-kernel/src/index.ts @@ -0,0 +1,98 @@ +export const PAPERCLIP_EVAL_KERNEL_COMPATIBILITY = Object.freeze({ + schema: "paperclip.eval-kernel.compatibility.v1" as const, + packageName: "@paperclipai/paperclip-eval-kernel" as const, + packageVersion: "0.1.0" as const, + apiVersion: 1 as const, +}); + +export interface PaperclipEvalScenario { + readonly id: string; + readonly input: TInput; +} + +export interface PaperclipEvalCandidate { + readonly id: string; + readonly config: TCandidate; + /** Fail-closed runner/catalog/provider compatibility check. */ + readonly preflight?: () => void | Promise; +} + +export interface PaperclipEvalResult { + readonly scenarioId: string; + readonly candidateId: string; + readonly output: TOutput; + readonly score: TScore; +} + +export class PaperclipEvalKernelConfigurationError extends Error { + readonly code = "paperclip_eval_kernel_configuration_invalid" as const; + + constructor(message: string) { + super(message); + this.name = "PaperclipEvalKernelConfigurationError"; + } +} + +/** + * Generic deterministic matrix orchestration. Scenario definitions, provider + * configuration, scorers, reports, and persistence remain caller-owned. + */ +export async function runPaperclipEvalMatrix< + TInput, + TCandidate, + TOutput, + TScore, +>(input: { + readonly scenarios: readonly PaperclipEvalScenario[]; + readonly candidates: readonly PaperclipEvalCandidate[]; + readonly execute: (context: { + readonly scenario: PaperclipEvalScenario; + readonly candidate: PaperclipEvalCandidate; + }) => Promise; + readonly score: (context: { + readonly scenario: PaperclipEvalScenario; + readonly candidate: PaperclipEvalCandidate; + readonly output: TOutput; + }) => Promise | TScore; +}): Promise[]> { + assertUniqueNonEmptyIds("scenario", input.scenarios); + assertUniqueNonEmptyIds("candidate", input.candidates); + + for (const candidate of input.candidates) { + await candidate.preflight?.(); + } + + const results: PaperclipEvalResult[] = []; + for (const scenario of input.scenarios) { + for (const candidate of input.candidates) { + const output = await input.execute({ scenario, candidate }); + const score = await input.score({ scenario, candidate, output }); + results.push(Object.freeze({ + scenarioId: scenario.id, + candidateId: candidate.id, + output, + score, + })); + } + } + return Object.freeze(results); +} + +function assertUniqueNonEmptyIds( + kind: "scenario" | "candidate", + values: readonly { readonly id: string }[], +): void { + if (values.length === 0) { + throw new PaperclipEvalKernelConfigurationError(`${kind} list must not be empty`); + } + const ids = new Set(); + for (const value of values) { + if (value.id.trim().length === 0) { + throw new PaperclipEvalKernelConfigurationError(`${kind} id must not be empty`); + } + if (ids.has(value.id)) { + throw new PaperclipEvalKernelConfigurationError(`duplicate ${kind} id: ${value.id}`); + } + ids.add(value.id); + } +} diff --git a/packages/paperclip-eval-kernel/test/kernel.test.mjs b/packages/paperclip-eval-kernel/test/kernel.test.mjs new file mode 100644 index 00000000000..38d0cf7e0ca --- /dev/null +++ b/packages/paperclip-eval-kernel/test/kernel.test.mjs @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + PAPERCLIP_EVAL_KERNEL_COMPATIBILITY, + PaperclipEvalKernelConfigurationError, + runPaperclipEvalMatrix, +} from "../dist/index.js"; + +test("runs a caller-owned scenario/candidate matrix", async () => { + const results = await runPaperclipEvalMatrix({ + scenarios: [{ id: "scenario-a", input: { value: 2 } }], + candidates: [{ id: "candidate-a", config: { multiplier: 3 } }], + execute: async ({ scenario, candidate }) => scenario.input.value * candidate.config.multiplier, + score: ({ output }) => ({ passed: output === 6 }), + }); + assert.equal(PAPERCLIP_EVAL_KERNEL_COMPATIBILITY.apiVersion, 1); + assert.deepEqual(results, [{ + scenarioId: "scenario-a", + candidateId: "candidate-a", + output: 6, + score: { passed: true }, + }]); +}); + +test("fails before execution when compatibility preflight fails", async () => { + let executed = false; + await assert.rejects( + runPaperclipEvalMatrix({ + scenarios: [{ id: "scenario-a", input: null }], + candidates: [{ + id: "candidate-a", + config: null, + preflight: () => { throw new Error("paperclip_runner_incompatible"); }, + }], + execute: async () => { executed = true; }, + score: () => null, + }), + /paperclip_runner_incompatible/, + ); + assert.equal(executed, false); +}); + +test("rejects duplicate scenario ids", async () => { + await assert.rejects( + runPaperclipEvalMatrix({ + scenarios: [{ id: "duplicate", input: 1 }, { id: "duplicate", input: 2 }], + candidates: [{ id: "candidate-a", config: null }], + execute: async () => null, + score: () => null, + }), + PaperclipEvalKernelConfigurationError, + ); +}); diff --git a/packages/paperclip-eval-kernel/tsconfig.json b/packages/paperclip-eval-kernel/tsconfig.json new file mode 100644 index 00000000000..c99ec7b71f9 --- /dev/null +++ b/packages/paperclip-eval-kernel/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index 359d8486f9b..dabae8de12f 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -13,6 +13,18 @@ session and issue-thread surfaces, a public browser/React SDK, a standalone adapter demo, and a deterministic mock control plane. None of these surfaces imports or starts Paperclip's server, UI, CLI, or production database. +## Public package surfaces + +- `@paperclipai/paperclip-runner` — production contracts, clients/backends, + PRP validation/replay, canonical catalog/dispatcher, and compatibility check. +- `@paperclipai/paperclip-runner/testing` — deterministic mocks plus PRP and + semantic conformance kits. Tests and Paperclip Evals import this explicitly. +- `@paperclipai/paperclip-eval-kernel` — separately packed generic eval matrix + orchestration, permitted in Paperclip App only as a development dependency. + +The package root has no mock or scenario exports, and no Paperclip Evals runtime +dependency. See [ADR 0001](docs/adr/0001-runner-testing-eval-package-boundaries.md). + ## Quick start From the repository root: @@ -84,6 +96,8 @@ and JSON content; see the protocol-server tutorial for direct `curl` examples. | `check:forbidden-imports` | Reject TypeScript imports and Cargo path dependencies that cross into Paperclip core. | | `check:tracked-imports` | Reject tracked imports and `package.json` entry points that only resolve against untracked files, so a clean checkout of any commit builds. | | `check:numbered-milestones` | Reject numbered construction-milestone names in tracked package paths and source. | +| `check:package-boundaries` | Enforce the acyclic runtime/testing/eval dependency and manifest boundary. | +| `check:clean-consumers` | Pack runner and eval-kernel tarballs and install them in two clean consumers. | | `check:conformance-parity` | Require byte-for-byte equivalent Rust and TypeScript tracer output. | | `check:replay-goldens` | Require all reducer snapshots and cross-language summaries to match checked goldens. | | `check:replay-parity` | Run TypeScript and Rust against the same Replay fixture summaries. | @@ -121,6 +135,7 @@ and JSON content; see the protocol-server tutorial for direct `curl` examples. ## Navigate - [Architecture and dependency boundary](docs/architecture.md) +- [ADR 0001: runner, testing, and eval package boundaries](docs/adr/0001-runner-testing-eval-package-boundaries.md) - [Tutorial index](docs/index.md) - [Conformance hand-run tutorial](docs/tutorials/conformance-standalone-tracer.md) - [Replay hand-run tutorial](docs/tutorials/replay.md) diff --git a/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md b/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md new file mode 100644 index 00000000000..3677f7c0fb1 --- /dev/null +++ b/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md @@ -0,0 +1,129 @@ +# ADR 0001: Runner, Testing, and Eval Package Boundaries + +- Status: Accepted +- Date: 2026-08-11 +- Decision owners: Paperclip App and Paperclip Evals + +## Context + +The runner package previously exported production contracts, deterministic +mocks, conformance fixtures, scenario/eval helpers, and demo servers from one +root. A workspace checkout hid two packaging defects: consumers could use +undeclared deep surfaces, and the public semantic dispatcher loaded `ajv` even +though `ajv` was only a development dependency. + +Paperclip Evals must consume App contracts and test helpers without making the +App runtime depend on the Evals repository. Paperclip App may use generic eval +orchestration in CI, but scenario content and provider experiments are not +product runtime dependencies. + +## Decision + +Ownership is: + +| Owner | Stable responsibility | +|---|---| +| Paperclip App | PRP schemas and fixtures, canonical semantic catalog and dispatcher, runnerd/client interfaces, `ControlPlanePort`, the production binding, deterministic mock, and mock/real parity fixtures | +| Paperclip Evals | Scenario corpus, candidate matrices, provider configuration, scorers, experiment reports, and eval orchestration | + +The production binding remains App code at +`server/src/services/native-runtime/paperclip-control-plane-port.ts`. It +implements the App-owned `ControlPlanePort`; the runner package never imports +the server, database, UI, or CLI. + +Public runner exports are: + +| Export | Stability and purpose | +|---|---| +| `@paperclipai/paperclip-runner` | Runtime contracts, runner clients/backends, PRP validation/replay, canonical catalog/dispatcher, and compatibility preflight | +| `@paperclipai/paperclip-runner/testing` | Deterministic mocks, PRP port conformance, and provider-neutral semantic conformance kit | +| `./browser`, `./react`, `./standalone`, `./styles.css` | Existing explicitly named UI/standalone consumers | + +Mock adapters and conformance constants are no longer package-root exports. +Tests and Evals must use `./testing`. Scenario content, matrices, reports, and +provider configuration are not public App package exports. + +The testkit remains an App `./testing` subpath rather than a separate package. +It shares the runner protocol/catalog release cadence, and no independent +consumer or version cadence currently justifies another package. Split it only +after an independent release requirement exists; a directory preference is not +sufficient. + +The generic matrix engine is a separate +`@paperclipai/paperclip-eval-kernel` package. It contains only structural matrix +orchestration and callbacks: no scenario, provider, product fixture, scorer, or +report content. It has no dependency on the runner package. Paperclip App may +install it only as a development dependency; the runner's dependency, +optional-dependency, and peer-dependency sets must remain free of Evals +packages. + +The dependency graph is acyclic: + +```text +Paperclip App production binding + | + v +@paperclipai/paperclip-runner (runtime contracts) + ^ + | +Paperclip Evals ----> @paperclipai/paperclip-runner/testing + | + +-------------> @paperclipai/paperclip-eval-kernel + +Paperclip App tests/CI --dev-only--> @paperclipai/paperclip-eval-kernel +``` + +No arrow points from App runtime to Paperclip Evals. + +## Compatibility and versioning + +Package semver describes distribution compatibility. Independently versioned +contracts are published in `PAPERCLIP_RUNNER_COMPATIBILITY`: + +| Component | Current contract | Compatibility rule | +|---|---:|---| +| Canonical catalog | 1 | Operation removals, renames, placement changes, or incompatible schemas require a new contract version | +| PRP | 1 | Highest overlapping required protocol version; no overlap fails closed | +| Runner client | 1 | Breaking runnerd/client interface changes require a new version | +| Control-plane adapter | 1 | Breaking `ControlPlanePort` or production-binding expectations require a new version | +| Testkit | 1 | Breaking mock seed, vector, observation, or conformance behavior requires a new version | +| Eval corpus | 1 | Runner declares a supported inclusive corpus-version range; out-of-range bundles fail before execution | +| Eval kernel API | 1 | Versioned by `PAPERCLIP_EVAL_KERNEL_COMPATIBILITY`, independently of scenario content | + +`assertPaperclipRunnerCompatibility` performs a fail-closed preflight. It emits +`paperclip_runner_incompatible` with stable issue codes for component mismatch, +unsupported corpus versions, unknown catalog operations, missing provider +capability declarations, and provider-operation gaps. Provider-specific runtime +errors must not stand in for this preflight. + +## Clean-consumer proof + +Run: + +```sh +pnpm --filter @paperclipai/paperclip-runner check:package-boundaries +pnpm --filter @paperclipai/paperclip-runner check:clean-consumers +``` + +The second command builds and packs both packages, then installs tarballs into +two newly created consumers: + +1. An Evals-side consumer imports only declared runner root and `./testing` + exports, executes the deterministic PRP conformance kit, and proves an + unsupported provider operation fails explicitly. +2. An App-side consumer installs the eval kernel as a development dependency, + runs a generic matrix after runner compatibility preflight, and proves the + runner manifest has no Evals runtime dependency. + +Neither consumer uses a workspace protocol, source-relative import, or deep +package path. This is the packaging gate; workspace tests alone are not proof. + +## Consequences + +- Existing tests importing mock/conformance values from the package root must + migrate to `@paperclipai/paperclip-runner/testing`. +- `ajv` is a runtime dependency because the public dispatcher imports it. +- The semantic conformance kit defines normalized comparison; real App service + adapters and risk-weighted vectors may evolve behind its testkit version. +- Scenario corpus changes do not force an App runtime release unless their + declared compatibility requirement changes. diff --git a/packages/paperclip-runner/docs/architecture.md b/packages/paperclip-runner/docs/architecture.md index 794f5925a66..3dba631cf14 100644 --- a/packages/paperclip-runner/docs/architecture.md +++ b/packages/paperclip-runner/docs/architecture.md @@ -13,11 +13,26 @@ v byte-identical Conformance result -Paperclip core (later) --> implements ControlPlanePort / NativeSessionBackend +Paperclip App production binding --> implements ControlPlanePort +Paperclip Evals --> consumes packed runtime + ./testing exports ``` The dependency arrow always points from an implementation toward a contract. The standalone package does not reach backward into a Paperclip implementation. +The production implementation lives in +`server/src/services/native-runtime/paperclip-control-plane-port.ts`; it depends +on the public port, never the reverse. The accepted package/export ownership is +recorded in [ADR 0001](adr/0001-runner-testing-eval-package-boundaries.md). + +## Public package surfaces + +- The package root is runtime-only: PRP, runner/client contracts, normalized + backends, catalog/dispatcher, and compatibility preflight. +- `./testing` contains deterministic mocks and conformance kits. +- `@paperclipai/paperclip-eval-kernel` contains generic orchestration only and + is permitted in App CI/tests as a development dependency, never at runtime. +- Paperclip Evals owns scenario content, matrices, provider configuration, + scoring, and reports. ## Core contracts diff --git a/packages/paperclip-runner/docs/capability-live-runnerd-codex.md b/packages/paperclip-runner/docs/capability-live-runnerd-codex.md index 180b7339eef..8a3df440873 100644 --- a/packages/paperclip-runner/docs/capability-live-runnerd-codex.md +++ b/packages/paperclip-runner/docs/capability-live-runnerd-codex.md @@ -29,7 +29,7 @@ the separate skillless, network-disabled workspace permission profile. Import the package root and construct `CapabilityLiveSessionService`: ```ts -import { CapabilityLiveSessionService } from "@paperclipai/paperclip-runner"; +import { CapabilityLiveSessionService } from "@paperclipai/paperclip-runner/testing"; const service = new CapabilityLiveSessionService({ store }); const session = await service.create({ workingDirectory }); diff --git a/packages/paperclip-runner/docs/index.md b/packages/paperclip-runner/docs/index.md index 9a680a50d44..52eaef0e48e 100644 --- a/packages/paperclip-runner/docs/index.md +++ b/packages/paperclip-runner/docs/index.md @@ -24,6 +24,7 @@ Each tutorial explains the capability it exercises and the contract it proves. - [Architecture and dependency boundary](architecture.md) - [PRP compatibility and versioning policy](protocol-compatibility.md) +- [ADR 0001: runner, testing, and eval package boundaries](adr/0001-runner-testing-eval-package-boundaries.md) - [Local protocol and supervision](local-runner.md) - [Durable transport and recovery](durable-recovery.md) - [Codex skillless Codex driver](codex-driver.md) diff --git a/packages/paperclip-runner/docs/protocol-compatibility.md b/packages/paperclip-runner/docs/protocol-compatibility.md index 429697aeba0..6d521fd8835 100644 --- a/packages/paperclip-runner/docs/protocol-compatibility.md +++ b/packages/paperclip-runner/docs/protocol-compatibility.md @@ -111,3 +111,15 @@ and deployment policy remain separately reviewed work. Breaking changes require a new required version. Additive optional fields may remain in v1 only when old consumers can safely ignore them. + +## Package-level compatibility + +PRP is one independently versioned component of the runner bundle. Catalog, +runner-client, control-plane-adapter, testkit, and eval-corpus compatibility is +declared by `PAPERCLIP_RUNNER_COMPATIBILITY` and checked before execution by +`assertPaperclipRunnerCompatibility`. A mismatch fails with +`paperclip_runner_incompatible` and stable per-issue codes; a provider-specific +tool error is not a compatibility negotiation mechanism. + +See [ADR 0001](adr/0001-runner-testing-eval-package-boundaries.md) for the +component rules and clean-consumer packaging gate. diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index bc0923ff90d..da1ee95751e 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -9,6 +9,10 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js" }, + "./testing": { + "types": "./dist/testing.d.ts", + "import": "./dist/testing.js" + }, "./browser": { "types": "./dist/browser/index.d.ts", "import": "./dist/browser/index.js" @@ -65,6 +69,8 @@ "check:forbidden-imports": "node scripts/check-forbidden-imports.mjs", "check:tracked-imports": "node scripts/check-tracked-imports.mjs", "check:numbered-milestones": "node scripts/check-numbered-milestones.mjs", + "check:package-boundaries": "node scripts/check-package-boundaries.mjs", + "check:clean-consumers": "node scripts/check-clean-consumers.mjs", "generate:capability-inventory": "node scripts/generate-capability-inventory.mjs", "check:capability-inventory": "node scripts/check-capability-inventory.mjs", "test:capability-inventory": "node scripts/check-capability-inventory.test.mjs", @@ -105,20 +111,30 @@ "console:sdk": "pnpm run build:typescript && vite --config vite.sdk.config.ts --host 127.0.0.1 --port 4181", "browser:dev": "pnpm run build:typescript && pnpm run build:runner-binaries && vite --config vite.config.ts", "browser:preview": "vite preview --config vite.config.ts", - "verify": "pnpm run build && pnpm run typecheck && pnpm run check:replay-goldens && pnpm run test && pnpm run test:sdk && pnpm run test:scenarios && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run test:browser:sdk && pnpm run test:browser:scenarios && pnpm run check:forbidden-imports && pnpm run check:tracked-imports && pnpm run check:numbered-milestones && pnpm run docs:validate && pnpm run check:conformance-parity && pnpm run check:replay-parity && pnpm run trace:conformance && pnpm run replay:fixture && pnpm run trace:local-runner -- --quiet && pnpm run trace:durable-recovery", + "verify": "pnpm run build && pnpm run typecheck && pnpm run check:replay-goldens && pnpm run test && pnpm run test:sdk && pnpm run test:scenarios && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run test:browser:sdk && pnpm run test:browser:scenarios && pnpm run check:forbidden-imports && pnpm run check:tracked-imports && pnpm run check:numbered-milestones && pnpm run check:package-boundaries && pnpm run check:clean-consumers && pnpm run docs:validate && pnpm run check:conformance-parity && pnpm run check:replay-parity && pnpm run trace:conformance && pnpm run replay:fixture && pnpm run trace:local-runner -- --quiet && pnpm run trace:durable-recovery", "verify:rootless": "bash scripts/verify-rootless-linux.sh" }, + "dependencies": { + "ajv": "^8.20.0" + }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + }, "devDependencies": { "@playwright/test": "^1.61.1", "@types/node": "^22.20.1", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^4.7.0", - "ajv": "^8.20.0", "axe-core": "^4.12.1", "json-schema-to-ts": "^3.1.1", "react": "^19.2.7", diff --git a/packages/paperclip-runner/scripts/check-clean-consumers.mjs b/packages/paperclip-runner/scripts/check-clean-consumers.mjs new file mode 100644 index 00000000000..6246af5cbd9 --- /dev/null +++ b/packages/paperclip-runner/scripts/check-clean-consumers.mjs @@ -0,0 +1,248 @@ +import { spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { tmpdir } from "node:os"; + +const runnerRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const evalKernelRoot = resolve(runnerRoot, "../paperclip-eval-kernel"); +const scratchParent = process.env.PAPERCLIP_RUN_SCRATCH_DIR + ?? process.env.PAPERCLIP_SCRATCH_DIR + ?? tmpdir(); +await mkdir(scratchParent, { recursive: true }); +const scratchRoot = await mkdtemp(join(scratchParent, "paperclip-package-consumers-")); +const artifactsRoot = resolve(scratchRoot, "artifacts"); +await mkdir(artifactsRoot, { recursive: true }); + +try { + run("pnpm", ["run", "build:typescript"], runnerRoot); + run("pnpm", ["run", "build"], evalKernelRoot); + const runnerTarball = await pack(runnerRoot, artifactsRoot); + const evalKernelTarball = await pack(evalKernelRoot, artifactsRoot); + const runtimeDependencyTarballs = await packRunnerRuntimeDependencies(artifactsRoot); + + await verifyEvalsConsumer( + resolve(scratchRoot, "evals-consumer"), + runnerTarball, + runtimeDependencyTarballs, + ); + await verifyAppDevConsumer( + resolve(scratchRoot, "app-dev-consumer"), + runnerTarball, + evalKernelTarball, + runtimeDependencyTarballs, + ); + + const runnerManifest = JSON.parse(await readFile(resolve(runnerRoot, "package.json"), "utf8")); + const runtimeDependencies = { + ...(runnerManifest.dependencies ?? {}), + ...(runnerManifest.optionalDependencies ?? {}), + ...(runnerManifest.peerDependencies ?? {}), + }; + if ("@paperclipai/paperclip-eval-kernel" in runtimeDependencies) { + throw new Error("runner tarball has a runtime dependency on Paperclip Evals"); + } + process.stdout.write("Clean-consumer pack/install checks passed for Evals -> App and App dev -> eval kernel.\n"); +} finally { + if (process.env.PAPERCLIP_KEEP_PACKAGE_CONSUMERS !== "1") { + await rm(scratchRoot, { recursive: true, force: true }); + } else { + process.stdout.write(`Kept clean-consumer scratch at ${scratchRoot}\n`); + } +} + +async function pack(packageRoot, destination) { + const before = new Set(await readdir(destination)); + run("pnpm", ["pack", "--pack-destination", destination], packageRoot, { quiet: true }); + const created = (await readdir(destination)) + .filter((entry) => entry.endsWith(".tgz") && !before.has(entry)) + .sort(); + if (created.length !== 1) { + throw new Error(`Expected one tarball from ${packageRoot}, found ${created.join(", ") || "none"}`); + } + return resolve(destination, created[0]); +} + +async function packRunnerRuntimeDependencies(destination) { + const ajvRoot = resolve(runnerRoot, "node_modules/ajv"); + const ajvModules = dirname(await realpath(ajvRoot)); + const packageNames = [ + "ajv", + "fast-deep-equal", + "fast-uri", + "json-schema-traverse", + "require-from-string", + ]; + const tarballs = {}; + for (const packageName of packageNames) { + const packageRoot = packageName === "ajv" ? ajvRoot : resolve(ajvModules, packageName); + tarballs[packageName] = await pack(packageRoot, destination); + } + return tarballs; +} + +function localOverrides(tarballs) { + return Object.fromEntries( + Object.entries(tarballs).map(([packageName, tarball]) => [packageName, `file:${tarball}`]), + ); +} + +async function verifyEvalsConsumer(consumerRoot, runnerTarball, runtimeDependencyTarballs) { + await mkdir(consumerRoot, { recursive: true }); + await writeFile(resolve(consumerRoot, "package.json"), `${JSON.stringify({ + name: "paperclip-evals-clean-consumer", + private: true, + type: "module", + packageManager: "pnpm@9.15.4", + dependencies: { + "@paperclipai/paperclip-runner": `file:${runnerTarball}`, + }, + pnpm: { overrides: localOverrides(runtimeDependencyTarballs) }, + }, null, 2)}\n`); + await writeFile(resolve(consumerRoot, "verify.mjs"), ` +import * as runtime from "@paperclipai/paperclip-runner"; +import * as testing from "@paperclipai/paperclip-runner/testing"; + +if ("MockControlPlaneAdapter" in runtime || "runControlPlanePortConformance" in runtime) { + throw new Error("test helpers leaked through the runtime root"); +} +if (typeof testing.MockControlPlaneAdapter !== "function") { + throw new Error("deterministic mock is absent from ./testing"); +} +const adapter = new testing.MockControlPlaneAdapter(); +const report = await testing.runControlPlanePortConformance({ + port: adapter, + start: () => adapter.start(), + stop: () => adapter.stop(), +}); +if (report.eventCount !== 3) throw new Error("packed conformance kit returned the wrong event count"); + +runtime.assertPaperclipRunnerCompatibility({ + consumer: "paperclip-evals-clean-consumer", + components: { catalog: 1, protocol: 1, runnerClient: 1, controlPlaneAdapter: 1, testkit: 1 }, + evalCorpusVersion: 1, + requiredOperationIds: ["finish_task"], + provider: { id: "clean-provider", supportedOperationIds: ["finish_task"] }, +}); +let failedExplicitly = false; +try { + runtime.assertPaperclipRunnerCompatibility({ + consumer: "incompatible-provider", + requiredOperationIds: ["finish_task"], + provider: { id: "missing-finish", supportedOperationIds: [] }, + }); +} catch (error) { + failedExplicitly = error?.code === "paperclip_runner_incompatible" + && error.issues?.[0]?.code === "provider_operation_unsupported"; +} +if (!failedExplicitly) throw new Error("provider incompatibility did not fail explicitly"); + +const normalized = { + authorization: { outcome: "allowed" }, + state: { status: "done" }, + effects: [], + audit: [], +}; +await testing.runSemanticConformanceKit({ + vectors: [{ id: "finish", operationId: "finish_task", input: {} }], + adapters: [ + { id: "mock", execute: async () => normalized }, + { id: "real", execute: async () => ({ audit: [], effects: [], state: { status: "done" }, authorization: { outcome: "allowed" } }) }, + ], +}); +`); + installAndRun(consumerRoot); +} + +async function verifyAppDevConsumer( + consumerRoot, + runnerTarball, + evalKernelTarball, + runtimeDependencyTarballs, +) { + await mkdir(consumerRoot, { recursive: true }); + await writeFile(resolve(consumerRoot, "package.json"), `${JSON.stringify({ + name: "paperclip-app-dev-clean-consumer", + private: true, + type: "module", + packageManager: "pnpm@9.15.4", + dependencies: { + "@paperclipai/paperclip-runner": `file:${runnerTarball}`, + }, + devDependencies: { + "@paperclipai/paperclip-eval-kernel": `file:${evalKernelTarball}`, + }, + pnpm: { overrides: localOverrides(runtimeDependencyTarballs) }, + }, null, 2)}\n`); + await writeFile(resolve(consumerRoot, "verify.mjs"), ` +import { + assertPaperclipRunnerCompatibility, +} from "@paperclipai/paperclip-runner"; +import { + PAPERCLIP_EVAL_KERNEL_COMPATIBILITY, + runPaperclipEvalMatrix, +} from "@paperclipai/paperclip-eval-kernel"; + +if (PAPERCLIP_EVAL_KERNEL_COMPATIBILITY.apiVersion !== 1) { + throw new Error("unexpected eval-kernel API version"); +} +const results = await runPaperclipEvalMatrix({ + scenarios: [{ id: "finish", input: { expected: "done" } }], + candidates: [{ + id: "runner-v1", + config: { result: "done" }, + preflight: () => assertPaperclipRunnerCompatibility({ + consumer: "paperclip-app-dev-clean-consumer", + components: { catalog: 1, protocol: 1, runnerClient: 1, controlPlaneAdapter: 1, testkit: 1 }, + evalCorpusVersion: 1, + requiredOperationIds: ["finish_task"], + provider: { id: "clean-provider", supportedOperationIds: ["finish_task"] }, + }), + }], + execute: async ({ candidate }) => candidate.config.result, + score: ({ scenario, output }) => ({ passed: output === scenario.input.expected }), +}); +if (results.length !== 1 || results[0].score.passed !== true) { + throw new Error("packed eval kernel did not execute the dev-only matrix"); +} +`); + installAndRun(consumerRoot); +} + +function installAndRun(consumerRoot) { + run("pnpm", [ + "install", + "--offline", + "--ignore-scripts", + "--lockfile=false", + "--store-dir", + resolve(consumerRoot, ".pnpm-store"), + "--config.auto-install-peers=false", + "--reporter=append-only", + ], consumerRoot, { env: { NODE_ENV: "development" } }); + run(process.execPath, ["verify.mjs"], consumerRoot); +} + +function run(command, args, cwd, { quiet = false, env = {} } = {}) { + const usesPnpm = command === "pnpm"; + const executable = usesPnpm + ? (process.platform === "win32" ? "corepack.cmd" : "corepack") + : command; + const effectiveArgs = usesPnpm ? ["pnpm@9.15.4", ...args] : args; + const result = spawnSync(executable, effectiveArgs, { + cwd, + encoding: "utf8", + env: { ...process.env, CI: "true", ...env }, + stdio: quiet ? ["ignore", "pipe", "pipe"] : ["ignore", "inherit", "inherit"], + ...(quiet ? { maxBuffer: 32 * 1024 * 1024 } : {}), + }); + if (result.status !== 0) { + if (quiet) { + process.stderr.write(result.stdout ?? ""); + process.stderr.write(result.stderr ?? ""); + } + if (result.error !== undefined) process.stderr.write(`${String(result.error)}\n`); + if (result.signal !== null) process.stderr.write(`Terminated by signal ${result.signal}\n`); + throw new Error(`${command} ${args.join(" ")} failed in ${cwd}`); + } +} diff --git a/packages/paperclip-runner/scripts/check-package-boundaries.mjs b/packages/paperclip-runner/scripts/check-package-boundaries.mjs new file mode 100644 index 00000000000..6344b8b17ac --- /dev/null +++ b/packages/paperclip-runner/scripts/check-package-boundaries.mjs @@ -0,0 +1,53 @@ +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const evalKernelRoot = resolve(packageRoot, "../paperclip-eval-kernel"); +const runnerManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); +const evalKernelManifest = JSON.parse(await readFile(resolve(evalKernelRoot, "package.json"), "utf8")); +const runtimeIndex = await readFile(resolve(packageRoot, "src/index.ts"), "utf8"); + +const violations = []; +if (runnerManifest.exports?.["./testing"] === undefined) { + violations.push("runner package must declare the ./testing export"); +} +for (const privatePath of ["mock-core", "conformance", "scenarios", "src/tools", "./tools/"]) { + if (runtimeIndex.includes(privatePath)) { + violations.push(`runtime root must not export test/eval path: ${privatePath}`); + } +} + +const runtimeDependencyGroups = [ + runnerManifest.dependencies ?? {}, + runnerManifest.optionalDependencies ?? {}, + runnerManifest.peerDependencies ?? {}, +]; +for (const dependency of runtimeDependencyGroups.flatMap((group) => Object.keys(group))) { + if (dependency.includes("eval")) { + violations.push(`runner runtime must not depend on eval package: ${dependency}`); + } +} +if (runnerManifest.dependencies?.ajv === undefined || runnerManifest.devDependencies?.ajv !== undefined) { + violations.push("ajv must be declared as a runtime dependency because the public dispatcher imports it"); +} + +for (const dependency of Object.keys({ + ...(evalKernelManifest.dependencies ?? {}), + ...(evalKernelManifest.optionalDependencies ?? {}), + ...(evalKernelManifest.peerDependencies ?? {}), +})) { + if (dependency === "@paperclipai/paperclip-runner") { + violations.push("generic eval kernel must use structural callbacks, not a runner runtime dependency"); + } +} +if (evalKernelManifest.files?.some((entry) => /scenario|fixture|corpus/.test(entry))) { + violations.push("generic eval kernel package inventory must not include scenario content"); +} + +if (violations.length > 0) { + process.stderr.write(`Package boundary check failed:\n${violations.map((item) => `- ${item}`).join("\n")}\n`); + process.exitCode = 1; +} else { + process.stdout.write("Package boundary check passed: runtime, testing, and eval-kernel ownership is acyclic.\n"); +} diff --git a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs index 69fe2f11de5..b42a209a402 100644 --- a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs +++ b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs @@ -82,10 +82,12 @@ function isForbiddenBrowserPackage(specifier) { function violationReason({ file, packageRoot, specifier }) { const relativeFile = relative(packageRoot, file).split(/[\\/]/).join("/"); const isExampleConsumer = relativeFile.startsWith("examples/"); + const isCleanConsumerHarness = relativeFile === "scripts/check-clean-consumers.mjs"; const publicRunnerImports = new Set([ "@paperclipai/paperclip-runner/browser", "@paperclipai/paperclip-runner/react", "@paperclipai/paperclip-runner/standalone", + "@paperclipai/paperclip-runner/testing", "@paperclipai/paperclip-runner/styles.css", ]); if ( @@ -102,6 +104,12 @@ function violationReason({ file, packageRoot, specifier }) { specifier !== "@paperclipai/paperclip-runner" && !publicRunnerImports.has(specifier) ) { + if ( + isCleanConsumerHarness + && specifier === "@paperclipai/paperclip-eval-kernel" + ) { + return null; + } return "Paperclip workspace packages are outside the standalone boundary"; } diff --git a/packages/paperclip-runner/src/compatibility.test.ts b/packages/paperclip-runner/src/compatibility.test.ts new file mode 100644 index 00000000000..93a6c1053eb --- /dev/null +++ b/packages/paperclip-runner/src/compatibility.test.ts @@ -0,0 +1,72 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +import { + PAPERCLIP_RUNNER_COMPATIBILITY, + PaperclipRunnerCompatibilityError, + assertPaperclipRunnerCompatibility, +} from "./compatibility.js"; + +describe("runner compatibility preflight", () => { + it("keeps the compatibility manifest synchronized with the package version", () => { + const manifestPath = fileURLToPath(new URL("../package.json", import.meta.url)); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + expect(PAPERCLIP_RUNNER_COMPATIBILITY.packageVersion).toBe(manifest.version); + }); + + it("accepts the supported product/test/eval bundle", () => { + expect(assertPaperclipRunnerCompatibility({ + consumer: "paperclip-evals", + components: { + catalog: 1, + protocol: 1, + runnerClient: 1, + controlPlaneAdapter: 1, + testkit: 1, + }, + evalCorpusVersion: 1, + requiredOperationIds: ["get_task_context", "finish_task"], + provider: { + id: "codex-app-server", + supportedOperationIds: ["get_task_context", "finish_task"], + }, + })).toBe(PAPERCLIP_RUNNER_COMPATIBILITY); + }); + + it("fails with stable component, corpus, catalog, and provider issues", () => { + expect(() => assertPaperclipRunnerCompatibility({ + consumer: "incompatible-eval-bundle", + components: { protocol: 2 }, + evalCorpusVersion: 2, + requiredOperationIds: ["missing_operation", "finish_task"], + provider: { id: "limited-provider", supportedOperationIds: [] }, + })).toThrow(PaperclipRunnerCompatibilityError); + + try { + assertPaperclipRunnerCompatibility({ + consumer: "incompatible-eval-bundle", + components: { protocol: 2 }, + evalCorpusVersion: 2, + requiredOperationIds: ["missing_operation", "finish_task"], + provider: { id: "limited-provider", supportedOperationIds: [] }, + }); + } catch (error) { + expect(error).toBeInstanceOf(PaperclipRunnerCompatibilityError); + expect((error as PaperclipRunnerCompatibilityError).code).toBe("paperclip_runner_incompatible"); + expect((error as PaperclipRunnerCompatibilityError).issues.map((issue) => issue.code)).toEqual([ + "component_version_mismatch", + "eval_corpus_version_unsupported", + "catalog_operation_unknown", + "provider_operation_unsupported", + ]); + } + }); + + it("requires explicit provider capabilities when operations are required", () => { + expect(() => assertPaperclipRunnerCompatibility({ + consumer: "undeclared-provider", + requiredOperationIds: ["finish_task"], + })).toThrow(/provider_capabilities_missing/); + }); +}); diff --git a/packages/paperclip-runner/src/compatibility.ts b/packages/paperclip-runner/src/compatibility.ts new file mode 100644 index 00000000000..905745d7223 --- /dev/null +++ b/packages/paperclip-runner/src/compatibility.ts @@ -0,0 +1,149 @@ +import { capabilityCanonicalOperation } from "./catalog/index.js"; + +export const PAPERCLIP_RUNNER_COMPATIBILITY = Object.freeze({ + schema: "paperclip.runner.compatibility.v1" as const, + packageName: "@paperclipai/paperclip-runner" as const, + packageVersion: "0.1.2" as const, + components: Object.freeze({ + catalog: 1, + protocol: 1, + runnerClient: 1, + controlPlaneAdapter: 1, + testkit: 1, + }), + evalCorpus: Object.freeze({ minimum: 1, maximum: 1 }), +}); + +export type PaperclipRunnerCompatibilityComponent = + keyof typeof PAPERCLIP_RUNNER_COMPATIBILITY.components; + +export type PaperclipRunnerCompatibilityIssueCode = + | "component_version_mismatch" + | "eval_corpus_version_unsupported" + | "catalog_operation_unknown" + | "provider_capabilities_missing" + | "provider_operation_unsupported"; + +export interface PaperclipRunnerCompatibilityIssue { + readonly code: PaperclipRunnerCompatibilityIssueCode; + readonly component?: PaperclipRunnerCompatibilityComponent | "evalCorpus" | "provider"; + readonly expected?: string; + readonly received?: string; + readonly operationId?: string; + readonly providerId?: string; + readonly message: string; +} + +export interface PaperclipProviderCompatibility { + readonly id: string; + readonly supportedOperationIds: readonly string[]; +} + +export interface PaperclipRunnerCompatibilityRequirement { + readonly consumer: string; + readonly components?: Partial>>; + readonly evalCorpusVersion?: number; + readonly requiredOperationIds?: readonly string[]; + readonly provider?: PaperclipProviderCompatibility; +} + +export class PaperclipRunnerCompatibilityError extends Error { + readonly code = "paperclip_runner_incompatible" as const; + readonly issues: readonly PaperclipRunnerCompatibilityIssue[]; + + constructor( + readonly consumer: string, + issues: readonly PaperclipRunnerCompatibilityIssue[], + ) { + super( + `Paperclip runner compatibility check failed for ${consumer}: ${issues + .map((issue) => `${issue.code}: ${issue.message}`) + .join("; ")}`, + ); + this.name = "PaperclipRunnerCompatibilityError"; + this.issues = Object.freeze(issues.map((issue) => Object.freeze({ ...issue }))); + } +} + +/** + * Fail-closed preflight shared by production bindings, testkit consumers, and + * eval bundles. Provider limitations are reported as stable compatibility + * issues instead of being inferred from provider-specific failures later. + */ +export function assertPaperclipRunnerCompatibility( + requirement: PaperclipRunnerCompatibilityRequirement, +): typeof PAPERCLIP_RUNNER_COMPATIBILITY { + const issues: PaperclipRunnerCompatibilityIssue[] = []; + + for (const [component, received] of Object.entries(requirement.components ?? {}) as Array< + [PaperclipRunnerCompatibilityComponent, number] + >) { + const expected = PAPERCLIP_RUNNER_COMPATIBILITY.components[component]; + if (received !== expected) { + issues.push({ + code: "component_version_mismatch", + component, + expected: String(expected), + received: String(received), + message: `${component} contract v${received} is incompatible with v${expected}`, + }); + } + } + + if (requirement.evalCorpusVersion !== undefined) { + const { minimum, maximum } = PAPERCLIP_RUNNER_COMPATIBILITY.evalCorpus; + if ( + requirement.evalCorpusVersion < minimum + || requirement.evalCorpusVersion > maximum + ) { + issues.push({ + code: "eval_corpus_version_unsupported", + component: "evalCorpus", + expected: `${minimum}..${maximum}`, + received: String(requirement.evalCorpusVersion), + message: `eval corpus v${requirement.evalCorpusVersion} is outside supported range ${minimum}..${maximum}`, + }); + } + } + + const requiredOperations = [...new Set(requirement.requiredOperationIds ?? [])].sort(); + const knownOperations: string[] = []; + for (const operationId of requiredOperations) { + if (capabilityCanonicalOperation(operationId) === undefined) { + issues.push({ + code: "catalog_operation_unknown", + component: "catalog", + operationId, + message: `required operation ${operationId} is absent from the canonical catalog`, + }); + } else { + knownOperations.push(operationId); + } + } + + if (knownOperations.length > 0 && requirement.provider === undefined) { + issues.push({ + code: "provider_capabilities_missing", + component: "provider", + message: "required operations need an explicit provider capability declaration", + }); + } else if (requirement.provider !== undefined) { + const supported = new Set(requirement.provider.supportedOperationIds); + for (const operationId of knownOperations) { + if (!supported.has(operationId)) { + issues.push({ + code: "provider_operation_unsupported", + component: "provider", + operationId, + providerId: requirement.provider.id, + message: `provider ${requirement.provider.id} does not support required operation ${operationId}`, + }); + } + } + } + + if (issues.length > 0) { + throw new PaperclipRunnerCompatibilityError(requirement.consumer, issues); + } + return PAPERCLIP_RUNNER_COMPATIBILITY; +} diff --git a/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts b/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts new file mode 100644 index 00000000000..d8b372560cf --- /dev/null +++ b/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; + +import { + SemanticConformanceMismatchError, + runSemanticConformanceKit, + type SemanticConformanceAdapter, + type SemanticConformanceObservation, +} from "./semantic-conformance.js"; + +const allowed: SemanticConformanceObservation = { + authorization: { outcome: "allowed" }, + state: { task: { status: "done" } }, + effects: [{ kind: "issue_status", status: "done" }], + audit: [{ action: "finish_task" }], +}; + +function adapter(id: string, observation: SemanticConformanceObservation): SemanticConformanceAdapter { + return { id, execute: async () => structuredClone(observation) }; +} + +describe("semantic conformance kit", () => { + it("accepts normalized mock/real observations independent of object key order", async () => { + const report = await runSemanticConformanceKit({ + vectors: [{ id: "finish", operationId: "finish_task", input: { summary: "done" } }], + adapters: [ + adapter("mock", allowed), + adapter("real", { + audit: [{ action: "finish_task" }], + effects: [{ status: "done", kind: "issue_status" }], + state: { task: { status: "done" } }, + authorization: { outcome: "allowed" }, + }), + ], + }); + expect(report.schema).toBe("paperclip.semantic-conformance-report.v1"); + expect(report.rows).toHaveLength(1); + expect(report.rows[0]?.adapterIds).toEqual(["mock", "real"]); + }); + + it("fails explicitly when a provider/control-plane adapter diverges", async () => { + await expect(runSemanticConformanceKit({ + vectors: [{ id: "finish", operationId: "finish_task", input: {} }], + adapters: [ + adapter("mock", allowed), + adapter("real", { ...allowed, authorization: { outcome: "denied", code: "forbidden" } }), + ], + })).rejects.toBeInstanceOf(SemanticConformanceMismatchError); + }); +}); diff --git a/packages/paperclip-runner/src/conformance/semantic-conformance.ts b/packages/paperclip-runner/src/conformance/semantic-conformance.ts new file mode 100644 index 00000000000..d59dbc5a407 --- /dev/null +++ b/packages/paperclip-runner/src/conformance/semantic-conformance.ts @@ -0,0 +1,104 @@ +export type SemanticConformanceAuthorization = + | { readonly outcome: "allowed" } + | { readonly outcome: "denied"; readonly code: string }; + +export interface SemanticConformanceVector { + readonly id: string; + readonly operationId: string; + readonly input: unknown; +} + +export interface SemanticConformanceObservation { + readonly authorization: SemanticConformanceAuthorization; + readonly state: unknown; + readonly effects: readonly unknown[]; + readonly audit: readonly unknown[]; +} + +export interface SemanticConformanceAdapter { + readonly id: string; + execute(vector: SemanticConformanceVector): Promise; +} + +export interface SemanticConformanceReportRow { + readonly vectorId: string; + readonly operationId: string; + readonly adapterIds: readonly string[]; + readonly observation: SemanticConformanceObservation; +} + +export interface SemanticConformanceReport { + readonly schema: "paperclip.semantic-conformance-report.v1"; + readonly rows: readonly SemanticConformanceReportRow[]; +} + +export class SemanticConformanceMismatchError extends Error { + readonly code = "semantic_conformance_mismatch" as const; + + constructor( + readonly vectorId: string, + readonly baselineAdapterId: string, + readonly mismatchedAdapterId: string, + ) { + super( + `Semantic conformance mismatch for ${vectorId}: ${mismatchedAdapterId} differs from ${baselineAdapterId}`, + ); + this.name = "SemanticConformanceMismatchError"; + } +} + +/** + * Executes provider-neutral command vectors against two or more adapters and + * compares their normalized authorization, state, effects, and audit output. + * Adapters own setup/teardown and normalization; this package owns comparison. + */ +export async function runSemanticConformanceKit(input: { + readonly vectors: readonly SemanticConformanceVector[]; + readonly adapters: readonly SemanticConformanceAdapter[]; +}): Promise { + if (input.adapters.length < 2) { + throw new Error("semantic_conformance_requires_two_adapters"); + } + const rows: SemanticConformanceReportRow[] = []; + for (const vector of input.vectors) { + const observations = await Promise.all( + input.adapters.map(async (adapter) => ({ + adapter, + observation: await adapter.execute(vector), + })), + ); + const baseline = observations[0]!; + const baselineJson = canonicalJson(baseline.observation); + for (const candidate of observations.slice(1)) { + if (canonicalJson(candidate.observation) !== baselineJson) { + throw new SemanticConformanceMismatchError( + vector.id, + baseline.adapter.id, + candidate.adapter.id, + ); + } + } + rows.push(Object.freeze({ + vectorId: vector.id, + operationId: vector.operationId, + adapterIds: Object.freeze(input.adapters.map((adapter) => adapter.id)), + observation: structuredClone(baseline.observation), + })); + } + return Object.freeze({ + schema: "paperclip.semantic-conformance-report.v1", + rows: Object.freeze(rows), + }); +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 802c9a64c8a..e7edd97ea82 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -8,41 +8,14 @@ export * from "./contracts/codex.js"; export * from "./contracts/types.js"; export * from "./backends/harness-driver-backend.js"; export * from "./backends/codex-native-backend.js"; -export * from "./conformance/control-plane-port.js"; export * from "./native-session-runtime.js"; export * from "./drivers/codex/app-server-transport.js"; export * from "./drivers/codex/codex-app-server-driver.js"; -export * from "./mock-core/mock-control-plane-adapter.js"; -export * from "./mock-core/capability-control-plane-types.js"; -export * from "./mock-core/capability-mock-control-plane-adapter.js"; -export * from "./standalone/standalone-demo.js"; -export * from "./mock-core/codex-runner.js"; -export * from "./mock-core/live-console-demo-server.js"; -export * from "./mock-core/live-console-demo-manifests.js"; -export * from "./mock-core/live-console-scripted-driver.js"; -export * from "./mock-core/local-runner.js"; -export * from "./mock-core/durable-recovery.js"; -export * from "./protocol/conformance-fixture.js"; export * from "./protocol/replay-contract.js"; -export * from "./protocol/live-console-fixture.js"; export * from "./protocol/replay-loader.js"; export * from "./reducer/session-reducer.js"; -export * from "./tracer/conformance-runner.js"; -export * from "./tracer/replay.js"; -export * from "./tracer/local-runner-live.js"; export * from "./generated/capability-contract.js"; -export * from "./tools/index.js"; +export * from "./semantic-tools/index.js"; export * as acceptedCapabilitySemanticTools from "./semantic-tools/index.js"; export * from "./catalog/index.js"; -export * from "./live/clean-room.js"; -export * from "./live/live-session.js"; -export * from "./live/runnerd-codex-transport.js"; -export * from "./live/turn-stream.js"; -export * from "./live/evidence-redaction.js"; -export { projectCapabilityIssueThread } from "./issue-thread/live-projection.js"; -export { - toCapabilityPublicThreadView, - type CapabilityPublicIssueThreadView, - type CapabilityPublicThreadViewOptions, -} from "./issue-thread/public-view.js"; -export * as capabilityIssueThread from "./issue-thread/index.js"; +export * from "./compatibility.js"; diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts new file mode 100644 index 00000000000..293ac726159 --- /dev/null +++ b/packages/paperclip-runner/src/testing.ts @@ -0,0 +1,15 @@ +/** + * Public test-only surface for deterministic adapters and conformance kits. + * + * Production Paperclip code imports the package root. Tests, Paperclip Evals, + * and other conformance consumers import this explicit subpath so test helpers + * never become an accidental runtime dependency. + */ +export * from "./index.js"; +export * from "./conformance/control-plane-port.js"; +export * from "./conformance/semantic-conformance.js"; +export * from "./mock-core/mock-control-plane-adapter.js"; +export * from "./mock-core/capability-control-plane-types.js"; +export * from "./mock-core/capability-mock-control-plane-adapter.js"; +export * from "./protocol/conformance-fixture.js"; +export * from "./live/live-session.js"; diff --git a/server/src/__tests__/native-finalization-recovery.test.ts b/server/src/__tests__/native-finalization-recovery.test.ts index 8393bfab2be..04f79dea14f 100644 --- a/server/src/__tests__/native-finalization-recovery.test.ts +++ b/server/src/__tests__/native-finalization-recovery.test.ts @@ -17,7 +17,7 @@ import { CONTROL_PLANE_CONFORMANCE_OPEN, CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL, -} from "@paperclipai/paperclip-runner"; +} from "@paperclipai/paperclip-runner/testing"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; diff --git a/server/src/__tests__/native-session-resumption.test.ts b/server/src/__tests__/native-session-resumption.test.ts index 335d69532d6..e85c5c9782f 100644 --- a/server/src/__tests__/native-session-resumption.test.ts +++ b/server/src/__tests__/native-session-resumption.test.ts @@ -23,14 +23,16 @@ import { workAssessments, } from "@paperclipai/db"; import { - CONTROL_PLANE_CONFORMANCE_RESULT, - CONTROL_PLANE_CONFORMANCE_TERMINAL, type NativeExecutionInputV1, type NativeSession, type NativeSessionBackend, type PersistedNativeSession, type PrpEvent, } from "@paperclipai/paperclip-runner"; +import { + CONTROL_PLANE_CONFORMANCE_RESULT, + CONTROL_PLANE_CONFORMANCE_TERMINAL, +} from "@paperclipai/paperclip-runner/testing"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; import { diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index 4967fa5b7b9..8eece0daaf9 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -20,16 +20,18 @@ import { workAssessments, } from "@paperclipai/db"; import { - CONTROL_PLANE_CONFORMANCE_RESULT, - CONTROL_PLANE_CONFORMANCE_TERMINAL, - CONTROL_PLANE_CONFORMANCE_OPEN, executeNativeSession, - runControlPlanePortConformance, type NativeSessionBackend, type PrpEvent, type PrpStructuredRunResult, type PrpTerminalState, } from "@paperclipai/paperclip-runner"; +import { + CONTROL_PLANE_CONFORMANCE_RESULT, + CONTROL_PLANE_CONFORMANCE_TERMINAL, + CONTROL_PLANE_CONFORMANCE_OPEN, + runControlPlanePortConformance, +} from "@paperclipai/paperclip-runner/testing"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { finalizeNativeRun } from "./native-run-finalizer.js";