feat(runner): define package and eval boundaries

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-eval-kernel/README.md

@@ -0,0 +1,14 @@
+# Paperclip Eval Kernel
+
+`@paperclipai/paperclip-eval-kernel` is the generic, provider-neutral matrix
+orchestrator owned by Paperclip Evals. It contains no Paperclip scenario corpus,
+provider configuration, product fixture, scorer, or report template.
+
+Consumers pass scenario and candidate values plus execution and scoring
+callbacks. Candidate `preflight` hooks should call the runner package's
+`assertPaperclipRunnerCompatibility` before any provider work starts. This keeps
+catalog, protocol, runner-client, control-plane-adapter, testkit, corpus, and
+provider-operation incompatibilities explicit.
+
+Paperclip App may consume this package only as a development dependency for CI
+or parity tests. `@paperclipai/paperclip-runner` has no runtime dependency on it.

packages/paperclip-eval-kernel/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@paperclipai/paperclip-eval-kernel",
+  "version": "0.1.0",
+  "description": "Provider-neutral evaluation matrix kernel without Paperclip scenario content",
+  "type": "module",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "pnpm run build && node --test test/*.test.mjs"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "license": "MIT",
+  "devDependencies": {
+    "typescript": "^5.7.3"
+  }
+}

packages/paperclip-eval-kernel/src/index.ts

@@ -0,0 +1,98 @@
+export const PAPERCLIP_EVAL_KERNEL_COMPATIBILITY = Object.freeze({
+  schema: "paperclip.eval-kernel.compatibility.v1" as const,
+  packageName: "@paperclipai/paperclip-eval-kernel" as const,
+  packageVersion: "0.1.0" as const,
+  apiVersion: 1 as const,
+});
+
+export interface PaperclipEvalScenario<TInput = unknown> {
+  readonly id: string;
+  readonly input: TInput;
+}
+
+export interface PaperclipEvalCandidate<TCandidate = unknown> {
+  readonly id: string;
+  readonly config: TCandidate;
+  /** Fail-closed runner/catalog/provider compatibility check. */
+  readonly preflight?: () => void | Promise<void>;
+}
+
+export interface PaperclipEvalResult<TOutput = unknown, TScore = unknown> {
+  readonly scenarioId: string;
+  readonly candidateId: string;
+  readonly output: TOutput;
+  readonly score: TScore;
+}
+
+export class PaperclipEvalKernelConfigurationError extends Error {
+  readonly code = "paperclip_eval_kernel_configuration_invalid" as const;
+
+  constructor(message: string) {
+    super(message);
+    this.name = "PaperclipEvalKernelConfigurationError";
+  }
+}
+
+/**
+ * Generic deterministic matrix orchestration. Scenario definitions, provider
+ * configuration, scorers, reports, and persistence remain caller-owned.
+ */
+export async function runPaperclipEvalMatrix<
+  TInput,
+  TCandidate,
+  TOutput,
+  TScore,
+>(input: {
+  readonly scenarios: readonly PaperclipEvalScenario<TInput>[];
+  readonly candidates: readonly PaperclipEvalCandidate<TCandidate>[];
+  readonly execute: (context: {
+    readonly scenario: PaperclipEvalScenario<TInput>;
+    readonly candidate: PaperclipEvalCandidate<TCandidate>;
+  }) => Promise<TOutput>;
+  readonly score: (context: {
+    readonly scenario: PaperclipEvalScenario<TInput>;
+    readonly candidate: PaperclipEvalCandidate<TCandidate>;
+    readonly output: TOutput;
+  }) => Promise<TScore> | TScore;
+}): Promise<readonly PaperclipEvalResult<TOutput, TScore>[]> {
+  assertUniqueNonEmptyIds("scenario", input.scenarios);
+  assertUniqueNonEmptyIds("candidate", input.candidates);
+
+  for (const candidate of input.candidates) {
+    await candidate.preflight?.();
+  }
+
+  const results: PaperclipEvalResult<TOutput, TScore>[] = [];
+  for (const scenario of input.scenarios) {
+    for (const candidate of input.candidates) {
+      const output = await input.execute({ scenario, candidate });
+      const score = await input.score({ scenario, candidate, output });
+      results.push(Object.freeze({
+        scenarioId: scenario.id,
+        candidateId: candidate.id,
+        output,
+        score,
+      }));
+    }
+  }
+  return Object.freeze(results);
+}
+
+function assertUniqueNonEmptyIds(
+  kind: "scenario" | "candidate",
+  values: readonly { readonly id: string }[],
+): void {
+  if (values.length === 0) {
+    throw new PaperclipEvalKernelConfigurationError(`${kind} list must not be empty`);
+  }
+  const ids = new Set<string>();
+  for (const value of values) {
+    if (value.id.trim().length === 0) {
+      throw new PaperclipEvalKernelConfigurationError(`${kind} id must not be empty`);
+    }
+    if (ids.has(value.id)) {
+      throw new PaperclipEvalKernelConfigurationError(`duplicate ${kind} id: ${value.id}`);
+    }
+    ids.add(value.id);
+  }
+}

packages/paperclip-eval-kernel/test/kernel.test.mjs

@@ -0,0 +1,54 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  PAPERCLIP_EVAL_KERNEL_COMPATIBILITY,
+  PaperclipEvalKernelConfigurationError,
+  runPaperclipEvalMatrix,
+} from "../dist/index.js";
+
+test("runs a caller-owned scenario/candidate matrix", async () => {
+  const results = await runPaperclipEvalMatrix({
+    scenarios: [{ id: "scenario-a", input: { value: 2 } }],
+    candidates: [{ id: "candidate-a", config: { multiplier: 3 } }],
+    execute: async ({ scenario, candidate }) => scenario.input.value * candidate.config.multiplier,
+    score: ({ output }) => ({ passed: output === 6 }),
+  });
+  assert.equal(PAPERCLIP_EVAL_KERNEL_COMPATIBILITY.apiVersion, 1);
+  assert.deepEqual(results, [{
+    scenarioId: "scenario-a",
+    candidateId: "candidate-a",
+    output: 6,
+    score: { passed: true },
+  }]);
+});
+
+test("fails before execution when compatibility preflight fails", async () => {
+  let executed = false;
+  await assert.rejects(
+    runPaperclipEvalMatrix({
+      scenarios: [{ id: "scenario-a", input: null }],
+      candidates: [{
+        id: "candidate-a",
+        config: null,
+        preflight: () => { throw new Error("paperclip_runner_incompatible"); },
+      }],
+      execute: async () => { executed = true; },
+      score: () => null,
+    }),
+    /paperclip_runner_incompatible/,
+  );
+  assert.equal(executed, false);
+});
+
+test("rejects duplicate scenario ids", async () => {
+  await assert.rejects(
+    runPaperclipEvalMatrix({
+      scenarios: [{ id: "duplicate", input: 1 }, { id: "duplicate", input: 2 }],
+      candidates: [{ id: "candidate-a", config: null }],
+      execute: async () => null,
+      score: () => null,
+    }),
+    PaperclipEvalKernelConfigurationError,
+  );
+});

packages/paperclip-eval-kernel/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src/**/*.ts"]
+}

packages/paperclip-runner/README.md

@@ -13,6 +13,18 @@ session and issue-thread surfaces, a public browser/React SDK, a standalone
 adapter demo, and a deterministic mock control plane. None of these surfaces
 imports or starts Paperclip's server, UI, CLI, or production database.
 
+## Public package surfaces
+
+- `@paperclipai/paperclip-runner` — production contracts, clients/backends,
+  PRP validation/replay, canonical catalog/dispatcher, and compatibility check.
+- `@paperclipai/paperclip-runner/testing` — deterministic mocks plus PRP and
+  semantic conformance kits. Tests and Paperclip Evals import this explicitly.
+- `@paperclipai/paperclip-eval-kernel` — separately packed generic eval matrix
+  orchestration, permitted in Paperclip App only as a development dependency.
+
+The package root has no mock or scenario exports, and no Paperclip Evals runtime
+dependency. See [ADR 0001](docs/adr/0001-runner-testing-eval-package-boundaries.md).
+
 ## Quick start
 
 From the repository root:
@@ -84,6 +96,8 @@ and JSON content; see the protocol-server tutorial for direct `curl` examples.
 | `check:forbidden-imports` | Reject TypeScript imports and Cargo path dependencies that cross into Paperclip core. |
 | `check:tracked-imports` | Reject tracked imports and `package.json` entry points that only resolve against untracked files, so a clean checkout of any commit builds. |
 | `check:numbered-milestones` | Reject numbered construction-milestone names in tracked package paths and source. |
+| `check:package-boundaries` | Enforce the acyclic runtime/testing/eval dependency and manifest boundary. |
+| `check:clean-consumers` | Pack runner and eval-kernel tarballs and install them in two clean consumers. |
 | `check:conformance-parity` | Require byte-for-byte equivalent Rust and TypeScript tracer output. |
 | `check:replay-goldens` | Require all reducer snapshots and cross-language summaries to match checked goldens. |
 | `check:replay-parity` | Run TypeScript and Rust against the same Replay fixture summaries. |
@@ -121,6 +135,7 @@ and JSON content; see the protocol-server tutorial for direct `curl` examples.
 ## Navigate
 
 - [Architecture and dependency boundary](docs/architecture.md)
+- [ADR 0001: runner, testing, and eval package boundaries](docs/adr/0001-runner-testing-eval-package-boundaries.md)
 - [Tutorial index](docs/index.md)
 - [Conformance hand-run tutorial](docs/tutorials/conformance-standalone-tracer.md)
 - [Replay hand-run tutorial](docs/tutorials/replay.md)

packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md

@@ -0,0 +1,129 @@
+# ADR 0001: Runner, Testing, and Eval Package Boundaries
+
+- Status: Accepted
+- Date: 2026-08-11
+- Decision owners: Paperclip App and Paperclip Evals
+
+## Context
+
+The runner package previously exported production contracts, deterministic
+mocks, conformance fixtures, scenario/eval helpers, and demo servers from one
+root. A workspace checkout hid two packaging defects: consumers could use
+undeclared deep surfaces, and the public semantic dispatcher loaded `ajv` even
+though `ajv` was only a development dependency.
+
+Paperclip Evals must consume App contracts and test helpers without making the
+App runtime depend on the Evals repository. Paperclip App may use generic eval
+orchestration in CI, but scenario content and provider experiments are not
+product runtime dependencies.
+
+## Decision
+
+Ownership is:
+
+| Owner | Stable responsibility |
+|---|---|
+| Paperclip App | PRP schemas and fixtures, canonical semantic catalog and dispatcher, runnerd/client interfaces, `ControlPlanePort`, the production binding, deterministic mock, and mock/real parity fixtures |
+| Paperclip Evals | Scenario corpus, candidate matrices, provider configuration, scorers, experiment reports, and eval orchestration |
+
+The production binding remains App code at
+`server/src/services/native-runtime/paperclip-control-plane-port.ts`. It
+implements the App-owned `ControlPlanePort`; the runner package never imports
+the server, database, UI, or CLI.
+
+Public runner exports are:
+
+| Export | Stability and purpose |
+|---|---|
+| `@paperclipai/paperclip-runner` | Runtime contracts, runner clients/backends, PRP validation/replay, canonical catalog/dispatcher, and compatibility preflight |
+| `@paperclipai/paperclip-runner/testing` | Deterministic mocks, PRP port conformance, and provider-neutral semantic conformance kit |
+| `./browser`, `./react`, `./standalone`, `./styles.css` | Existing explicitly named UI/standalone consumers |
+
+Mock adapters and conformance constants are no longer package-root exports.
+Tests and Evals must use `./testing`. Scenario content, matrices, reports, and
+provider configuration are not public App package exports.
+
+The testkit remains an App `./testing` subpath rather than a separate package.
+It shares the runner protocol/catalog release cadence, and no independent
+consumer or version cadence currently justifies another package. Split it only
+after an independent release requirement exists; a directory preference is not
+sufficient.
+
+The generic matrix engine is a separate
+`@paperclipai/paperclip-eval-kernel` package. It contains only structural matrix
+orchestration and callbacks: no scenario, provider, product fixture, scorer, or
+report content. It has no dependency on the runner package. Paperclip App may
+install it only as a development dependency; the runner's dependency,
+optional-dependency, and peer-dependency sets must remain free of Evals
+packages.
+
+The dependency graph is acyclic:
+
+```text
+Paperclip App production binding
+            |
+            v
+@paperclipai/paperclip-runner (runtime contracts)
+            ^
+            |
+Paperclip Evals ----> @paperclipai/paperclip-runner/testing
+      |
+      +-------------> @paperclipai/paperclip-eval-kernel
+
+Paperclip App tests/CI --dev-only--> @paperclipai/paperclip-eval-kernel
+```
+
+No arrow points from App runtime to Paperclip Evals.
+
+## Compatibility and versioning
+
+Package semver describes distribution compatibility. Independently versioned
+contracts are published in `PAPERCLIP_RUNNER_COMPATIBILITY`:
+
+| Component | Current contract | Compatibility rule |
+|---|---:|---|
+| Canonical catalog | 1 | Operation removals, renames, placement changes, or incompatible schemas require a new contract version |
+| PRP | 1 | Highest overlapping required protocol version; no overlap fails closed |
+| Runner client | 1 | Breaking runnerd/client interface changes require a new version |
+| Control-plane adapter | 1 | Breaking `ControlPlanePort` or production-binding expectations require a new version |
+| Testkit | 1 | Breaking mock seed, vector, observation, or conformance behavior requires a new version |
+| Eval corpus | 1 | Runner declares a supported inclusive corpus-version range; out-of-range bundles fail before execution |
+| Eval kernel API | 1 | Versioned by `PAPERCLIP_EVAL_KERNEL_COMPATIBILITY`, independently of scenario content |
+
+`assertPaperclipRunnerCompatibility` performs a fail-closed preflight. It emits
+`paperclip_runner_incompatible` with stable issue codes for component mismatch,
+unsupported corpus versions, unknown catalog operations, missing provider
+capability declarations, and provider-operation gaps. Provider-specific runtime
+errors must not stand in for this preflight.
+
+## Clean-consumer proof
+
+Run:
+
+```sh
+pnpm --filter @paperclipai/paperclip-runner check:package-boundaries
+pnpm --filter @paperclipai/paperclip-runner check:clean-consumers
+```
+
+The second command builds and packs both packages, then installs tarballs into
+two newly created consumers:
+
+1. An Evals-side consumer imports only declared runner root and `./testing`
+   exports, executes the deterministic PRP conformance kit, and proves an
+   unsupported provider operation fails explicitly.
+2. An App-side consumer installs the eval kernel as a development dependency,
+   runs a generic matrix after runner compatibility preflight, and proves the
+   runner manifest has no Evals runtime dependency.
+
+Neither consumer uses a workspace protocol, source-relative import, or deep
+package path. This is the packaging gate; workspace tests alone are not proof.
+
+## Consequences
+
+- Existing tests importing mock/conformance values from the package root must
+  migrate to `@paperclipai/paperclip-runner/testing`.
+- `ajv` is a runtime dependency because the public dispatcher imports it.
+- The semantic conformance kit defines normalized comparison; real App service
+  adapters and risk-weighted vectors may evolve behind its testkit version.
+- Scenario corpus changes do not force an App runtime release unless their
+  declared compatibility requirement changes.

packages/paperclip-runner/docs/architecture.md

@@ -13,11 +13,26 @@
                                       v
                          byte-identical Conformance result
 
-Paperclip core (later) --> implements ControlPlanePort / NativeSessionBackend
+Paperclip App production binding --> implements ControlPlanePort
+Paperclip Evals --> consumes packed runtime + ./testing exports
 ```
 
 The dependency arrow always points from an implementation toward a contract.
 The standalone package does not reach backward into a Paperclip implementation.
+The production implementation lives in
+`server/src/services/native-runtime/paperclip-control-plane-port.ts`; it depends
+on the public port, never the reverse. The accepted package/export ownership is
+recorded in [ADR 0001](adr/0001-runner-testing-eval-package-boundaries.md).
+
+## Public package surfaces
+
+- The package root is runtime-only: PRP, runner/client contracts, normalized
+  backends, catalog/dispatcher, and compatibility preflight.
+- `./testing` contains deterministic mocks and conformance kits.
+- `@paperclipai/paperclip-eval-kernel` contains generic orchestration only and
+  is permitted in App CI/tests as a development dependency, never at runtime.
+- Paperclip Evals owns scenario content, matrices, provider configuration,
+  scoring, and reports.
 
 ## Core contracts
 

packages/paperclip-runner/docs/capability-live-runnerd-codex.md

@@ -29,7 +29,7 @@ the separate skillless, network-disabled workspace permission profile.
 Import the package root and construct `CapabilityLiveSessionService`:
 
 ```ts
-import { CapabilityLiveSessionService } from "@paperclipai/paperclip-runner";
+import { CapabilityLiveSessionService } from "@paperclipai/paperclip-runner/testing";
 
 const service = new CapabilityLiveSessionService({ store });
 const session = await service.create({ workingDirectory });

packages/paperclip-runner/docs/index.md

@@ -24,6 +24,7 @@ Each tutorial explains the capability it exercises and the contract it proves.
 
 - [Architecture and dependency boundary](architecture.md)
 - [PRP compatibility and versioning policy](protocol-compatibility.md)
+- [ADR 0001: runner, testing, and eval package boundaries](adr/0001-runner-testing-eval-package-boundaries.md)
 - [Local protocol and supervision](local-runner.md)
 - [Durable transport and recovery](durable-recovery.md)
 - [Codex skillless Codex driver](codex-driver.md)

packages/paperclip-runner/docs/protocol-compatibility.md

@@ -111,3 +111,15 @@ and deployment policy remain separately reviewed work.
 
 Breaking changes require a new required version. Additive optional fields may
 remain in v1 only when old consumers can safely ignore them.
+
+## Package-level compatibility
+
+PRP is one independently versioned component of the runner bundle. Catalog,
+runner-client, control-plane-adapter, testkit, and eval-corpus compatibility is
+declared by `PAPERCLIP_RUNNER_COMPATIBILITY` and checked before execution by
+`assertPaperclipRunnerCompatibility`. A mismatch fails with
+`paperclip_runner_incompatible` and stable per-issue codes; a provider-specific
+tool error is not a compatibility negotiation mechanism.
+
+See [ADR 0001](adr/0001-runner-testing-eval-package-boundaries.md) for the
+component rules and clean-consumer packaging gate.

packages/paperclip-runner/package.json

@@ -9,6 +9,10 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     },
+    "./testing": {
+      "types": "./dist/testing.d.ts",
+      "import": "./dist/testing.js"
+    },
     "./browser": {
       "types": "./dist/browser/index.d.ts",
       "import": "./dist/browser/index.js"
@@ -65,6 +69,8 @@
     "check:forbidden-imports": "node scripts/check-forbidden-imports.mjs",
     "check:tracked-imports": "node scripts/check-tracked-imports.mjs",
     "check:numbered-milestones": "node scripts/check-numbered-milestones.mjs",
+    "check:package-boundaries": "node scripts/check-package-boundaries.mjs",
+    "check:clean-consumers": "node scripts/check-clean-consumers.mjs",
     "generate:capability-inventory": "node scripts/generate-capability-inventory.mjs",
     "check:capability-inventory": "node scripts/check-capability-inventory.mjs",
     "test:capability-inventory": "node scripts/check-capability-inventory.test.mjs",
@@ -105,20 +111,30 @@
     "console:sdk": "pnpm run build:typescript && vite --config vite.sdk.config.ts --host 127.0.0.1 --port 4181",
     "browser:dev": "pnpm run build:typescript && pnpm run build:runner-binaries && vite --config vite.config.ts",
     "browser:preview": "vite preview --config vite.config.ts",
-    "verify": "pnpm run build && pnpm run typecheck && pnpm run check:replay-goldens && pnpm run test && pnpm run test:sdk && pnpm run test:scenarios && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run test:browser:sdk && pnpm run test:browser:scenarios && pnpm run check:forbidden-imports && pnpm run check:tracked-imports && pnpm run check:numbered-milestones && pnpm run docs:validate && pnpm run check:conformance-parity && pnpm run check:replay-parity && pnpm run trace:conformance && pnpm run replay:fixture && pnpm run trace:local-runner -- --quiet && pnpm run trace:durable-recovery",
+    "verify": "pnpm run build && pnpm run typecheck && pnpm run check:replay-goldens && pnpm run test && pnpm run test:sdk && pnpm run test:scenarios && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run test:browser:sdk && pnpm run test:browser:scenarios && pnpm run check:forbidden-imports && pnpm run check:tracked-imports && pnpm run check:numbered-milestones && pnpm run check:package-boundaries && pnpm run check:clean-consumers && pnpm run docs:validate && pnpm run check:conformance-parity && pnpm run check:replay-parity && pnpm run trace:conformance && pnpm run replay:fixture && pnpm run trace:local-runner -- --quiet && pnpm run trace:durable-recovery",
     "verify:rootless": "bash scripts/verify-rootless-linux.sh"
   },
+  "dependencies": {
+    "ajv": "^8.20.0"
+  },
   "peerDependencies": {
     "react": ">=18",
     "react-dom": ">=18"
   },
+  "peerDependenciesMeta": {
+    "react": {
+      "optional": true
+    },
+    "react-dom": {
+      "optional": true
+    }
+  },
   "devDependencies": {
     "@playwright/test": "^1.61.1",
     "@types/node": "^22.20.1",
     "@types/react": "^19.2.17",
     "@types/react-dom": "^19.2.3",
     "@vitejs/plugin-react": "^4.7.0",
-    "ajv": "^8.20.0",
     "axe-core": "^4.12.1",
     "json-schema-to-ts": "^3.1.1",
     "react": "^19.2.7",

packages/paperclip-runner/scripts/check-clean-consumers.mjs

@@ -0,0 +1,248 @@
+import { spawnSync } from "node:child_process";
+import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { tmpdir } from "node:os";
+
+const runnerRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const evalKernelRoot = resolve(runnerRoot, "../paperclip-eval-kernel");
+const scratchParent = process.env.PAPERCLIP_RUN_SCRATCH_DIR
+  ?? process.env.PAPERCLIP_SCRATCH_DIR
+  ?? tmpdir();
+await mkdir(scratchParent, { recursive: true });
+const scratchRoot = await mkdtemp(join(scratchParent, "paperclip-package-consumers-"));
+const artifactsRoot = resolve(scratchRoot, "artifacts");
+await mkdir(artifactsRoot, { recursive: true });
+
+try {
+  run("pnpm", ["run", "build:typescript"], runnerRoot);
+  run("pnpm", ["run", "build"], evalKernelRoot);
+  const runnerTarball = await pack(runnerRoot, artifactsRoot);
+  const evalKernelTarball = await pack(evalKernelRoot, artifactsRoot);
+  const runtimeDependencyTarballs = await packRunnerRuntimeDependencies(artifactsRoot);
+
+  await verifyEvalsConsumer(
+    resolve(scratchRoot, "evals-consumer"),
+    runnerTarball,
+    runtimeDependencyTarballs,
+  );
+  await verifyAppDevConsumer(
+    resolve(scratchRoot, "app-dev-consumer"),
+    runnerTarball,
+    evalKernelTarball,
+    runtimeDependencyTarballs,
+  );
+
+  const runnerManifest = JSON.parse(await readFile(resolve(runnerRoot, "package.json"), "utf8"));
+  const runtimeDependencies = {
+    ...(runnerManifest.dependencies ?? {}),
+    ...(runnerManifest.optionalDependencies ?? {}),
+    ...(runnerManifest.peerDependencies ?? {}),
+  };
+  if ("@paperclipai/paperclip-eval-kernel" in runtimeDependencies) {
+    throw new Error("runner tarball has a runtime dependency on Paperclip Evals");
+  }
+  process.stdout.write("Clean-consumer pack/install checks passed for Evals -> App and App dev -> eval kernel.\n");
+} finally {
+  if (process.env.PAPERCLIP_KEEP_PACKAGE_CONSUMERS !== "1") {
+    await rm(scratchRoot, { recursive: true, force: true });
+  } else {
+    process.stdout.write(`Kept clean-consumer scratch at ${scratchRoot}\n`);
+  }
+}
+
+async function pack(packageRoot, destination) {
+  const before = new Set(await readdir(destination));
+  run("pnpm", ["pack", "--pack-destination", destination], packageRoot, { quiet: true });
+  const created = (await readdir(destination))
+    .filter((entry) => entry.endsWith(".tgz") && !before.has(entry))
+    .sort();
+  if (created.length !== 1) {
+    throw new Error(`Expected one tarball from ${packageRoot}, found ${created.join(", ") || "none"}`);
+  }
+  return resolve(destination, created[0]);
+}
+
+async function packRunnerRuntimeDependencies(destination) {
+  const ajvRoot = resolve(runnerRoot, "node_modules/ajv");
+  const ajvModules = dirname(await realpath(ajvRoot));
+  const packageNames = [
+    "ajv",
+    "fast-deep-equal",
+    "fast-uri",
+    "json-schema-traverse",
+    "require-from-string",
+  ];
+  const tarballs = {};
+  for (const packageName of packageNames) {
+    const packageRoot = packageName === "ajv" ? ajvRoot : resolve(ajvModules, packageName);
+    tarballs[packageName] = await pack(packageRoot, destination);
+  }
+  return tarballs;
+}
+
+function localOverrides(tarballs) {
+  return Object.fromEntries(
+    Object.entries(tarballs).map(([packageName, tarball]) => [packageName, `file:${tarball}`]),
+  );
+}
+
+async function verifyEvalsConsumer(consumerRoot, runnerTarball, runtimeDependencyTarballs) {
+  await mkdir(consumerRoot, { recursive: true });
+  await writeFile(resolve(consumerRoot, "package.json"), `${JSON.stringify({
+    name: "paperclip-evals-clean-consumer",
+    private: true,
+    type: "module",
+    packageManager: "pnpm@9.15.4",
+    dependencies: {
+      "@paperclipai/paperclip-runner": `file:${runnerTarball}`,
+    },
+    pnpm: { overrides: localOverrides(runtimeDependencyTarballs) },
+  }, null, 2)}\n`);
+  await writeFile(resolve(consumerRoot, "verify.mjs"), `
+import * as runtime from "@paperclipai/paperclip-runner";
+import * as testing from "@paperclipai/paperclip-runner/testing";
+
+if ("MockControlPlaneAdapter" in runtime || "runControlPlanePortConformance" in runtime) {
+  throw new Error("test helpers leaked through the runtime root");
+}
+if (typeof testing.MockControlPlaneAdapter !== "function") {
+  throw new Error("deterministic mock is absent from ./testing");
+}
+const adapter = new testing.MockControlPlaneAdapter();
+const report = await testing.runControlPlanePortConformance({
+  port: adapter,
+  start: () => adapter.start(),
+  stop: () => adapter.stop(),
+});
+if (report.eventCount !== 3) throw new Error("packed conformance kit returned the wrong event count");
+
+runtime.assertPaperclipRunnerCompatibility({
+  consumer: "paperclip-evals-clean-consumer",
+  components: { catalog: 1, protocol: 1, runnerClient: 1, controlPlaneAdapter: 1, testkit: 1 },
+  evalCorpusVersion: 1,
+  requiredOperationIds: ["finish_task"],
+  provider: { id: "clean-provider", supportedOperationIds: ["finish_task"] },
+});
+let failedExplicitly = false;
+try {
+  runtime.assertPaperclipRunnerCompatibility({
+    consumer: "incompatible-provider",
+    requiredOperationIds: ["finish_task"],
+    provider: { id: "missing-finish", supportedOperationIds: [] },
+  });
+} catch (error) {
+  failedExplicitly = error?.code === "paperclip_runner_incompatible"
+    && error.issues?.[0]?.code === "provider_operation_unsupported";
+}
+if (!failedExplicitly) throw new Error("provider incompatibility did not fail explicitly");
+
+const normalized = {
+  authorization: { outcome: "allowed" },
+  state: { status: "done" },
+  effects: [],
+  audit: [],
+};
+await testing.runSemanticConformanceKit({
+  vectors: [{ id: "finish", operationId: "finish_task", input: {} }],
+  adapters: [
+    { id: "mock", execute: async () => normalized },
+    { id: "real", execute: async () => ({ audit: [], effects: [], state: { status: "done" }, authorization: { outcome: "allowed" } }) },
+  ],
+});
+`);
+  installAndRun(consumerRoot);
+}
+
+async function verifyAppDevConsumer(
+  consumerRoot,
+  runnerTarball,
+  evalKernelTarball,
+  runtimeDependencyTarballs,
+) {
+  await mkdir(consumerRoot, { recursive: true });
+  await writeFile(resolve(consumerRoot, "package.json"), `${JSON.stringify({
+    name: "paperclip-app-dev-clean-consumer",
+    private: true,
+    type: "module",
+    packageManager: "pnpm@9.15.4",
+    dependencies: {
+      "@paperclipai/paperclip-runner": `file:${runnerTarball}`,
+    },
+    devDependencies: {
+      "@paperclipai/paperclip-eval-kernel": `file:${evalKernelTarball}`,
+    },
+    pnpm: { overrides: localOverrides(runtimeDependencyTarballs) },
+  }, null, 2)}\n`);
+  await writeFile(resolve(consumerRoot, "verify.mjs"), `
+import {
+  assertPaperclipRunnerCompatibility,
+} from "@paperclipai/paperclip-runner";
+import {
+  PAPERCLIP_EVAL_KERNEL_COMPATIBILITY,
+  runPaperclipEvalMatrix,
+} from "@paperclipai/paperclip-eval-kernel";
+
+if (PAPERCLIP_EVAL_KERNEL_COMPATIBILITY.apiVersion !== 1) {
+  throw new Error("unexpected eval-kernel API version");
+}
+const results = await runPaperclipEvalMatrix({
+  scenarios: [{ id: "finish", input: { expected: "done" } }],
+  candidates: [{
+    id: "runner-v1",
+    config: { result: "done" },
+    preflight: () => assertPaperclipRunnerCompatibility({
+      consumer: "paperclip-app-dev-clean-consumer",
+      components: { catalog: 1, protocol: 1, runnerClient: 1, controlPlaneAdapter: 1, testkit: 1 },
+      evalCorpusVersion: 1,
+      requiredOperationIds: ["finish_task"],
+      provider: { id: "clean-provider", supportedOperationIds: ["finish_task"] },
+    }),
+  }],
+  execute: async ({ candidate }) => candidate.config.result,
+  score: ({ scenario, output }) => ({ passed: output === scenario.input.expected }),
+});
+if (results.length !== 1 || results[0].score.passed !== true) {
+  throw new Error("packed eval kernel did not execute the dev-only matrix");
+}
+`);
+  installAndRun(consumerRoot);
+}
+
+function installAndRun(consumerRoot) {
+  run("pnpm", [
+    "install",
+    "--offline",
+    "--ignore-scripts",
+    "--lockfile=false",
+    "--store-dir",
+    resolve(consumerRoot, ".pnpm-store"),
+    "--config.auto-install-peers=false",
+    "--reporter=append-only",
+  ], consumerRoot, { env: { NODE_ENV: "development" } });
+  run(process.execPath, ["verify.mjs"], consumerRoot);
+}
+
+function run(command, args, cwd, { quiet = false, env = {} } = {}) {
+  const usesPnpm = command === "pnpm";
+  const executable = usesPnpm
+    ? (process.platform === "win32" ? "corepack.cmd" : "corepack")
+    : command;
+  const effectiveArgs = usesPnpm ? ["pnpm@9.15.4", ...args] : args;
+  const result = spawnSync(executable, effectiveArgs, {
+    cwd,
+    encoding: "utf8",
+    env: { ...process.env, CI: "true", ...env },
+    stdio: quiet ? ["ignore", "pipe", "pipe"] : ["ignore", "inherit", "inherit"],
+    ...(quiet ? { maxBuffer: 32 * 1024 * 1024 } : {}),
+  });
+  if (result.status !== 0) {
+    if (quiet) {
+      process.stderr.write(result.stdout ?? "");
+      process.stderr.write(result.stderr ?? "");
+    }
+    if (result.error !== undefined) process.stderr.write(`${String(result.error)}\n`);
+    if (result.signal !== null) process.stderr.write(`Terminated by signal ${result.signal}\n`);
+    throw new Error(`${command} ${args.join(" ")} failed in ${cwd}`);
+  }
+}

packages/paperclip-runner/scripts/check-package-boundaries.mjs

@@ -0,0 +1,53 @@
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const evalKernelRoot = resolve(packageRoot, "../paperclip-eval-kernel");
+const runnerManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));
+const evalKernelManifest = JSON.parse(await readFile(resolve(evalKernelRoot, "package.json"), "utf8"));
+const runtimeIndex = await readFile(resolve(packageRoot, "src/index.ts"), "utf8");
+
+const violations = [];
+if (runnerManifest.exports?.["./testing"] === undefined) {
+  violations.push("runner package must declare the ./testing export");
+}
+for (const privatePath of ["mock-core", "conformance", "scenarios", "src/tools", "./tools/"]) {
+  if (runtimeIndex.includes(privatePath)) {
+    violations.push(`runtime root must not export test/eval path: ${privatePath}`);
+  }
+}
+
+const runtimeDependencyGroups = [
+  runnerManifest.dependencies ?? {},
+  runnerManifest.optionalDependencies ?? {},
+  runnerManifest.peerDependencies ?? {},
+];
+for (const dependency of runtimeDependencyGroups.flatMap((group) => Object.keys(group))) {
+  if (dependency.includes("eval")) {
+    violations.push(`runner runtime must not depend on eval package: ${dependency}`);
+  }
+}
+if (runnerManifest.dependencies?.ajv === undefined || runnerManifest.devDependencies?.ajv !== undefined) {
+  violations.push("ajv must be declared as a runtime dependency because the public dispatcher imports it");
+}
+
+for (const dependency of Object.keys({
+  ...(evalKernelManifest.dependencies ?? {}),
+  ...(evalKernelManifest.optionalDependencies ?? {}),
+  ...(evalKernelManifest.peerDependencies ?? {}),
+})) {
+  if (dependency === "@paperclipai/paperclip-runner") {
+    violations.push("generic eval kernel must use structural callbacks, not a runner runtime dependency");
+  }
+}
+if (evalKernelManifest.files?.some((entry) => /scenario|fixture|corpus/.test(entry))) {
+  violations.push("generic eval kernel package inventory must not include scenario content");
+}
+
+if (violations.length > 0) {
+  process.stderr.write(`Package boundary check failed:\n${violations.map((item) => `- ${item}`).join("\n")}\n`);
+  process.exitCode = 1;
+} else {
+  process.stdout.write("Package boundary check passed: runtime, testing, and eval-kernel ownership is acyclic.\n");
+}

packages/paperclip-runner/scripts/lib/forbidden-imports.mjs

@@ -82,10 +82,12 @@ function isForbiddenBrowserPackage(specifier) {
 function violationReason({ file, packageRoot, specifier }) {
   const relativeFile = relative(packageRoot, file).split(/[\\/]/).join("/");
   const isExampleConsumer = relativeFile.startsWith("examples/");
+  const isCleanConsumerHarness = relativeFile === "scripts/check-clean-consumers.mjs";
   const publicRunnerImports = new Set([
     "@paperclipai/paperclip-runner/browser",
     "@paperclipai/paperclip-runner/react",
     "@paperclipai/paperclip-runner/standalone",
+    "@paperclipai/paperclip-runner/testing",
     "@paperclipai/paperclip-runner/styles.css",
   ]);
   if (
@@ -102,6 +104,12 @@ function violationReason({ file, packageRoot, specifier }) {
     specifier !== "@paperclipai/paperclip-runner" &&
     !publicRunnerImports.has(specifier)
   ) {
+    if (
+      isCleanConsumerHarness
+      && specifier === "@paperclipai/paperclip-eval-kernel"
+    ) {
+      return null;
+    }
     return "Paperclip workspace packages are outside the standalone boundary";
   }
 

packages/paperclip-runner/src/compatibility.test.ts

@@ -0,0 +1,72 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+
+import {
+  PAPERCLIP_RUNNER_COMPATIBILITY,
+  PaperclipRunnerCompatibilityError,
+  assertPaperclipRunnerCompatibility,
+} from "./compatibility.js";
+
+describe("runner compatibility preflight", () => {
+  it("keeps the compatibility manifest synchronized with the package version", () => {
+    const manifestPath = fileURLToPath(new URL("../package.json", import.meta.url));
+    const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+    expect(PAPERCLIP_RUNNER_COMPATIBILITY.packageVersion).toBe(manifest.version);
+  });
+
+  it("accepts the supported product/test/eval bundle", () => {
+    expect(assertPaperclipRunnerCompatibility({
+      consumer: "paperclip-evals",
+      components: {
+        catalog: 1,
+        protocol: 1,
+        runnerClient: 1,
+        controlPlaneAdapter: 1,
+        testkit: 1,
+      },
+      evalCorpusVersion: 1,
+      requiredOperationIds: ["get_task_context", "finish_task"],
+      provider: {
+        id: "codex-app-server",
+        supportedOperationIds: ["get_task_context", "finish_task"],
+      },
+    })).toBe(PAPERCLIP_RUNNER_COMPATIBILITY);
+  });
+
+  it("fails with stable component, corpus, catalog, and provider issues", () => {
+    expect(() => assertPaperclipRunnerCompatibility({
+      consumer: "incompatible-eval-bundle",
+      components: { protocol: 2 },
+      evalCorpusVersion: 2,
+      requiredOperationIds: ["missing_operation", "finish_task"],
+      provider: { id: "limited-provider", supportedOperationIds: [] },
+    })).toThrow(PaperclipRunnerCompatibilityError);
+
+    try {
+      assertPaperclipRunnerCompatibility({
+        consumer: "incompatible-eval-bundle",
+        components: { protocol: 2 },
+        evalCorpusVersion: 2,
+        requiredOperationIds: ["missing_operation", "finish_task"],
+        provider: { id: "limited-provider", supportedOperationIds: [] },
+      });
+    } catch (error) {
+      expect(error).toBeInstanceOf(PaperclipRunnerCompatibilityError);
+      expect((error as PaperclipRunnerCompatibilityError).code).toBe("paperclip_runner_incompatible");
+      expect((error as PaperclipRunnerCompatibilityError).issues.map((issue) => issue.code)).toEqual([
+        "component_version_mismatch",
+        "eval_corpus_version_unsupported",
+        "catalog_operation_unknown",
+        "provider_operation_unsupported",
+      ]);
+    }
+  });
+
+  it("requires explicit provider capabilities when operations are required", () => {
+    expect(() => assertPaperclipRunnerCompatibility({
+      consumer: "undeclared-provider",
+      requiredOperationIds: ["finish_task"],
+    })).toThrow(/provider_capabilities_missing/);
+  });
+});

packages/paperclip-runner/src/compatibility.ts

@@ -0,0 +1,149 @@
+import { capabilityCanonicalOperation } from "./catalog/index.js";
+
+export const PAPERCLIP_RUNNER_COMPATIBILITY = Object.freeze({
+  schema: "paperclip.runner.compatibility.v1" as const,
+  packageName: "@paperclipai/paperclip-runner" as const,
+  packageVersion: "0.1.2" as const,
+  components: Object.freeze({
+    catalog: 1,
+    protocol: 1,
+    runnerClient: 1,
+    controlPlaneAdapter: 1,
+    testkit: 1,
+  }),
+  evalCorpus: Object.freeze({ minimum: 1, maximum: 1 }),
+});
+
+export type PaperclipRunnerCompatibilityComponent =
+  keyof typeof PAPERCLIP_RUNNER_COMPATIBILITY.components;
+
+export type PaperclipRunnerCompatibilityIssueCode =
+  | "component_version_mismatch"
+  | "eval_corpus_version_unsupported"
+  | "catalog_operation_unknown"
+  | "provider_capabilities_missing"
+  | "provider_operation_unsupported";
+
+export interface PaperclipRunnerCompatibilityIssue {
+  readonly code: PaperclipRunnerCompatibilityIssueCode;
+  readonly component?: PaperclipRunnerCompatibilityComponent | "evalCorpus" | "provider";
+  readonly expected?: string;
+  readonly received?: string;
+  readonly operationId?: string;
+  readonly providerId?: string;
+  readonly message: string;
+}
+
+export interface PaperclipProviderCompatibility {
+  readonly id: string;
+  readonly supportedOperationIds: readonly string[];
+}
+
+export interface PaperclipRunnerCompatibilityRequirement {
+  readonly consumer: string;
+  readonly components?: Partial<Readonly<Record<PaperclipRunnerCompatibilityComponent, number>>>;
+  readonly evalCorpusVersion?: number;
+  readonly requiredOperationIds?: readonly string[];
+  readonly provider?: PaperclipProviderCompatibility;
+}
+
+export class PaperclipRunnerCompatibilityError extends Error {
+  readonly code = "paperclip_runner_incompatible" as const;
+  readonly issues: readonly PaperclipRunnerCompatibilityIssue[];
+
+  constructor(
+    readonly consumer: string,
+    issues: readonly PaperclipRunnerCompatibilityIssue[],
+  ) {
+    super(
+      `Paperclip runner compatibility check failed for ${consumer}: ${issues
+        .map((issue) => `${issue.code}: ${issue.message}`)
+        .join("; ")}`,
+    );
+    this.name = "PaperclipRunnerCompatibilityError";
+    this.issues = Object.freeze(issues.map((issue) => Object.freeze({ ...issue })));
+  }
+}
+
+/**
+ * Fail-closed preflight shared by production bindings, testkit consumers, and
+ * eval bundles. Provider limitations are reported as stable compatibility
+ * issues instead of being inferred from provider-specific failures later.
+ */
+export function assertPaperclipRunnerCompatibility(
+  requirement: PaperclipRunnerCompatibilityRequirement,
+): typeof PAPERCLIP_RUNNER_COMPATIBILITY {
+  const issues: PaperclipRunnerCompatibilityIssue[] = [];
+
+  for (const [component, received] of Object.entries(requirement.components ?? {}) as Array<
+    [PaperclipRunnerCompatibilityComponent, number]
+  >) {
+    const expected = PAPERCLIP_RUNNER_COMPATIBILITY.components[component];
+    if (received !== expected) {
+      issues.push({
+        code: "component_version_mismatch",
+        component,
+        expected: String(expected),
+        received: String(received),
+        message: `${component} contract v${received} is incompatible with v${expected}`,
+      });
+    }
+  }
+
+  if (requirement.evalCorpusVersion !== undefined) {
+    const { minimum, maximum } = PAPERCLIP_RUNNER_COMPATIBILITY.evalCorpus;
+    if (
+      requirement.evalCorpusVersion < minimum
+      || requirement.evalCorpusVersion > maximum
+    ) {
+      issues.push({
+        code: "eval_corpus_version_unsupported",
+        component: "evalCorpus",
+        expected: `${minimum}..${maximum}`,
+        received: String(requirement.evalCorpusVersion),
+        message: `eval corpus v${requirement.evalCorpusVersion} is outside supported range ${minimum}..${maximum}`,
+      });
+    }
+  }
+
+  const requiredOperations = [...new Set(requirement.requiredOperationIds ?? [])].sort();
+  const knownOperations: string[] = [];
+  for (const operationId of requiredOperations) {
+    if (capabilityCanonicalOperation(operationId) === undefined) {
+      issues.push({
+        code: "catalog_operation_unknown",
+        component: "catalog",
+        operationId,
+        message: `required operation ${operationId} is absent from the canonical catalog`,
+      });
+    } else {
+      knownOperations.push(operationId);
+    }
+  }
+
+  if (knownOperations.length > 0 && requirement.provider === undefined) {
+    issues.push({
+      code: "provider_capabilities_missing",
+      component: "provider",
+      message: "required operations need an explicit provider capability declaration",
+    });
+  } else if (requirement.provider !== undefined) {
+    const supported = new Set(requirement.provider.supportedOperationIds);
+    for (const operationId of knownOperations) {
+      if (!supported.has(operationId)) {
+        issues.push({
+          code: "provider_operation_unsupported",
+          component: "provider",
+          operationId,
+          providerId: requirement.provider.id,
+          message: `provider ${requirement.provider.id} does not support required operation ${operationId}`,
+        });
+      }
+    }
+  }
+
+  if (issues.length > 0) {
+    throw new PaperclipRunnerCompatibilityError(requirement.consumer, issues);
+  }
+  return PAPERCLIP_RUNNER_COMPATIBILITY;
+}

packages/paperclip-runner/src/conformance/semantic-conformance.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  SemanticConformanceMismatchError,
+  runSemanticConformanceKit,
+  type SemanticConformanceAdapter,
+  type SemanticConformanceObservation,
+} from "./semantic-conformance.js";
+
+const allowed: SemanticConformanceObservation = {
+  authorization: { outcome: "allowed" },
+  state: { task: { status: "done" } },
+  effects: [{ kind: "issue_status", status: "done" }],
+  audit: [{ action: "finish_task" }],
+};
+
+function adapter(id: string, observation: SemanticConformanceObservation): SemanticConformanceAdapter {
+  return { id, execute: async () => structuredClone(observation) };
+}
+
+describe("semantic conformance kit", () => {
+  it("accepts normalized mock/real observations independent of object key order", async () => {
+    const report = await runSemanticConformanceKit({
+      vectors: [{ id: "finish", operationId: "finish_task", input: { summary: "done" } }],
+      adapters: [
+        adapter("mock", allowed),
+        adapter("real", {
+          audit: [{ action: "finish_task" }],
+          effects: [{ status: "done", kind: "issue_status" }],
+          state: { task: { status: "done" } },
+          authorization: { outcome: "allowed" },
+        }),
+      ],
+    });
+    expect(report.schema).toBe("paperclip.semantic-conformance-report.v1");
+    expect(report.rows).toHaveLength(1);
+    expect(report.rows[0]?.adapterIds).toEqual(["mock", "real"]);
+  });
+
+  it("fails explicitly when a provider/control-plane adapter diverges", async () => {
+    await expect(runSemanticConformanceKit({
+      vectors: [{ id: "finish", operationId: "finish_task", input: {} }],
+      adapters: [
+        adapter("mock", allowed),
+        adapter("real", { ...allowed, authorization: { outcome: "denied", code: "forbidden" } }),
+      ],
+    })).rejects.toBeInstanceOf(SemanticConformanceMismatchError);
+  });
+});

packages/paperclip-runner/src/conformance/semantic-conformance.ts

@@ -0,0 +1,104 @@
+export type SemanticConformanceAuthorization =
+  | { readonly outcome: "allowed" }
+  | { readonly outcome: "denied"; readonly code: string };
+
+export interface SemanticConformanceVector {
+  readonly id: string;
+  readonly operationId: string;
+  readonly input: unknown;
+}
+
+export interface SemanticConformanceObservation {
+  readonly authorization: SemanticConformanceAuthorization;
+  readonly state: unknown;
+  readonly effects: readonly unknown[];
+  readonly audit: readonly unknown[];
+}
+
+export interface SemanticConformanceAdapter {
+  readonly id: string;
+  execute(vector: SemanticConformanceVector): Promise<SemanticConformanceObservation>;
+}
+
+export interface SemanticConformanceReportRow {
+  readonly vectorId: string;
+  readonly operationId: string;
+  readonly adapterIds: readonly string[];
+  readonly observation: SemanticConformanceObservation;
+}
+
+export interface SemanticConformanceReport {
+  readonly schema: "paperclip.semantic-conformance-report.v1";
+  readonly rows: readonly SemanticConformanceReportRow[];
+}
+
+export class SemanticConformanceMismatchError extends Error {
+  readonly code = "semantic_conformance_mismatch" as const;
+
+  constructor(
+    readonly vectorId: string,
+    readonly baselineAdapterId: string,
+    readonly mismatchedAdapterId: string,
+  ) {
+    super(
+      `Semantic conformance mismatch for ${vectorId}: ${mismatchedAdapterId} differs from ${baselineAdapterId}`,
+    );
+    this.name = "SemanticConformanceMismatchError";
+  }
+}
+
+/**
+ * Executes provider-neutral command vectors against two or more adapters and
+ * compares their normalized authorization, state, effects, and audit output.
+ * Adapters own setup/teardown and normalization; this package owns comparison.
+ */
+export async function runSemanticConformanceKit(input: {
+  readonly vectors: readonly SemanticConformanceVector[];
+  readonly adapters: readonly SemanticConformanceAdapter[];
+}): Promise<SemanticConformanceReport> {
+  if (input.adapters.length < 2) {
+    throw new Error("semantic_conformance_requires_two_adapters");
+  }
+  const rows: SemanticConformanceReportRow[] = [];
+  for (const vector of input.vectors) {
+    const observations = await Promise.all(
+      input.adapters.map(async (adapter) => ({
+        adapter,
+        observation: await adapter.execute(vector),
+      })),
+    );
+    const baseline = observations[0]!;
+    const baselineJson = canonicalJson(baseline.observation);
+    for (const candidate of observations.slice(1)) {
+      if (canonicalJson(candidate.observation) !== baselineJson) {
+        throw new SemanticConformanceMismatchError(
+          vector.id,
+          baseline.adapter.id,
+          candidate.adapter.id,
+        );
+      }
+    }
+    rows.push(Object.freeze({
+      vectorId: vector.id,
+      operationId: vector.operationId,
+      adapterIds: Object.freeze(input.adapters.map((adapter) => adapter.id)),
+      observation: structuredClone(baseline.observation),
+    }));
+  }
+  return Object.freeze({
+    schema: "paperclip.semantic-conformance-report.v1",
+    rows: Object.freeze(rows),
+  });
+}
+
+function canonicalJson(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+  if (typeof value === "object" && value !== null) {
+    const record = value as Record<string, unknown>;
+    return `{${Object.keys(record)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
+      .join(",")}}`;
+  }
+  return JSON.stringify(value) ?? "undefined";
+}

packages/paperclip-runner/src/index.ts

@@ -8,41 +8,14 @@ export * from "./contracts/codex.js";
 export * from "./contracts/types.js";
 export * from "./backends/harness-driver-backend.js";
 export * from "./backends/codex-native-backend.js";
-export * from "./conformance/control-plane-port.js";
 export * from "./native-session-runtime.js";
 export * from "./drivers/codex/app-server-transport.js";
 export * from "./drivers/codex/codex-app-server-driver.js";
-export * from "./mock-core/mock-control-plane-adapter.js";
-export * from "./mock-core/capability-control-plane-types.js";
-export * from "./mock-core/capability-mock-control-plane-adapter.js";
-export * from "./standalone/standalone-demo.js";
-export * from "./mock-core/codex-runner.js";
-export * from "./mock-core/live-console-demo-server.js";
-export * from "./mock-core/live-console-demo-manifests.js";
-export * from "./mock-core/live-console-scripted-driver.js";
-export * from "./mock-core/local-runner.js";
-export * from "./mock-core/durable-recovery.js";
-export * from "./protocol/conformance-fixture.js";
 export * from "./protocol/replay-contract.js";
-export * from "./protocol/live-console-fixture.js";
 export * from "./protocol/replay-loader.js";
 export * from "./reducer/session-reducer.js";
-export * from "./tracer/conformance-runner.js";
-export * from "./tracer/replay.js";
-export * from "./tracer/local-runner-live.js";
 export * from "./generated/capability-contract.js";
-export * from "./tools/index.js";
+export * from "./semantic-tools/index.js";
 export * as acceptedCapabilitySemanticTools from "./semantic-tools/index.js";
 export * from "./catalog/index.js";
-export * from "./live/clean-room.js";
-export * from "./live/live-session.js";
-export * from "./live/runnerd-codex-transport.js";
-export * from "./live/turn-stream.js";
-export * from "./live/evidence-redaction.js";
-export { projectCapabilityIssueThread } from "./issue-thread/live-projection.js";
-export {
-  toCapabilityPublicThreadView,
-  type CapabilityPublicIssueThreadView,
-  type CapabilityPublicThreadViewOptions,
-} from "./issue-thread/public-view.js";
-export * as capabilityIssueThread from "./issue-thread/index.js";
+export * from "./compatibility.js";

packages/paperclip-runner/src/testing.ts

@@ -0,0 +1,15 @@
+/**
+ * Public test-only surface for deterministic adapters and conformance kits.
+ *
+ * Production Paperclip code imports the package root. Tests, Paperclip Evals,
+ * and other conformance consumers import this explicit subpath so test helpers
+ * never become an accidental runtime dependency.
+ */
+export * from "./index.js";
+export * from "./conformance/control-plane-port.js";
+export * from "./conformance/semantic-conformance.js";
+export * from "./mock-core/mock-control-plane-adapter.js";
+export * from "./mock-core/capability-control-plane-types.js";
+export * from "./mock-core/capability-mock-control-plane-adapter.js";
+export * from "./protocol/conformance-fixture.js";
+export * from "./live/live-session.js";

server/src/__tests__/native-finalization-recovery.test.ts

@@ -17,7 +17,7 @@ import {
   CONTROL_PLANE_CONFORMANCE_OPEN,
   CONTROL_PLANE_CONFORMANCE_RESULT,
   CONTROL_PLANE_CONFORMANCE_TERMINAL,
-} from "@paperclipai/paperclip-runner";
+} from "@paperclipai/paperclip-runner/testing";
 import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js";
 import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js";

server/src/__tests__/native-session-resumption.test.ts

@@ -23,14 +23,16 @@ import {
   workAssessments,
 } from "@paperclipai/db";
 import {
-  CONTROL_PLANE_CONFORMANCE_RESULT,
-  CONTROL_PLANE_CONFORMANCE_TERMINAL,
   type NativeExecutionInputV1,
   type NativeSession,
   type NativeSessionBackend,
   type PersistedNativeSession,
   type PrpEvent,
 } from "@paperclipai/paperclip-runner";
+import {
+  CONTROL_PLANE_CONFORMANCE_RESULT,
+  CONTROL_PLANE_CONFORMANCE_TERMINAL,
+} from "@paperclipai/paperclip-runner/testing";
 import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";
 import {

server/src/services/native-runtime/paperclip-control-plane-port.test.ts

@@ -20,16 +20,18 @@ import {
   workAssessments,
 } from "@paperclipai/db";
 import {
-  CONTROL_PLANE_CONFORMANCE_RESULT,
-  CONTROL_PLANE_CONFORMANCE_TERMINAL,
-  CONTROL_PLANE_CONFORMANCE_OPEN,
   executeNativeSession,
-  runControlPlanePortConformance,
   type NativeSessionBackend,
   type PrpEvent,
   type PrpStructuredRunResult,
   type PrpTerminalState,
 } from "@paperclipai/paperclip-runner";
+import {
+  CONTROL_PLANE_CONFORMANCE_RESULT,
+  CONTROL_PLANE_CONFORMANCE_TERMINAL,
+  CONTROL_PLANE_CONFORMANCE_OPEN,
+  runControlPlanePortConformance,
+} from "@paperclipai/paperclip-runner/testing";
 import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
 import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
 import { finalizeNativeRun } from "./native-run-finalizer.js";