diff --git a/packages/paperclip-runner/spec/prp-v1-expressiveness-audit.md b/packages/paperclip-runner/spec/prp-v1-expressiveness-audit.md new file mode 100644 index 00000000000..47253cd4b35 --- /dev/null +++ b/packages/paperclip-runner/spec/prp-v1-expressiveness-audit.md @@ -0,0 +1,102 @@ +# PRP v1 expressiveness audit + +## Verdict + +PRP v1 is sufficient for runner lifecycle, replay integrity, negotiated runner +features, runtime/issue-thread request routing, structured work disposition, +and terminal causality. It is **not** yet sufficient to claim a provider-neutral +semantic-control-plane protocol: tool invocation, authorization denial, +redaction, optimistic concurrency, mutation receipts, and budget stop reasons +are presently represented only by event or request `payload` extension space. + +`prp-v1-expressiveness-crosswalk.json` is the machine-checked source for this +verdict. Its Vitest gate rejects an unclassified required fact, a fact justified +only by a permissive field, an unbounded control-plane-local fact, or an +accepted additive change without required fixtures. + +## Evidence boundary + +This audit compares the canonical capability authority and its traceability +inputs, the real-surface ledger, the deterministic mock, live Codex/runnerd +transport, and PRP schemas/replay fixtures. The wire contract is not the tool +catalog: capability placement controls exposure, while PRP carries execution +and causality. `generic_api_request` remains test-only and cannot satisfy any +coverage row. + +## What v1 proves directly + +- `capabilities.schema.json` negotiates driver identity, session reuse, + steering, interruption, resume, runtime requests, structured results, typed + events, and explicit unsupported features. +- Events have stable source instance/event ids, source ordering, run/session/ + turn/item correlation, priority, and an explicit v1 schema version. Replay + rejects unsupported required event versions, requires byte-equivalent + duplicate source events, records source gaps, and is side-effect free. +- Commands have controller ordering and preconditions; runtime and issue-thread + requests have typed kinds/statuses; results require completion claims, + verification, artifacts, and blocker/yield continuations where applicable. +- `run.terminal` is a typed terminal envelope with terminal state, work + assessment, and issue-status decision references. Existing golden replay, + duplicate-event, unknown-optional-fields, and unsupported-required-version + fixtures demonstrate the compatibility boundary. + +## Required additive v1 envelope + +The smallest correction is one optional, versioned `semantic_tool` envelope +for `mcp_app.tool_input` / `mcp_app.tool_result` (or equivalent new event +types), plus an optional `terminal.stopReason` envelope. It must define: + +- operation id, call id, correlation ids, idempotency key, outcome, stable + code, retryability, and audit/operation receipt id; +- admitted input/output references or digests, redaction disposition, and only + safe identifiers; never raw credentials, hidden-company identifiers, or + secret payloads; +- authorization boundary (`company`, `actor`, `active_task`, `grant`, + `governed_action`, `lock`, or `revision`), current revision where safe, and + a deterministic conflict/duplicate result; +- artifact/work-product references linked to the semantic receipt and terminal + result; immutable interaction/approval document or decision targets; and + budget stop reason/aggregate/decision id. + +This is additive because existing v1 consumers can ignore the optional typed +envelope. It is not permission to add speculative semantic events: the +crosswalk names the exact facts and the conformance vectors that must land +with any schema change. + +## Classification and fixture plan + +The crosswalk assigns every required fact to exactly one of: direct v1, +compositional v1 with a listed invariant, control-plane-local, additive v1, +missing fixture/docs, or breaking v2 work. Direct and compositional support is +never inferred merely from `additionalProperties: true`. + +Before an additive change graduates to direct support, add these fixtures and +golden projections: + +1. semantic tool happy path and artifact registration; +2. denied operation with redaction assertions and no fallback call; +3. stale optimistic write conflict and exact duplicate/retry receipt; +4. immutable interaction/approval target plus wake/monitor causal chain; +5. budget/cost terminal stop reason; +6. unknown optional envelope fields accepted without changing v1 projection; +7. unknown required envelope version rejected fail-closed. + +Each mutation fixture must replay to a golden summary/snapshot and be executed +through mock and real adapters with normalized receipt/state-diff comparison. + +## Explicit exclusions + +Checkout/release, task selection, wake routing, budget enforcement, audit and +run persistence/replay, assignment, and monitor management are control-plane +actions, not runner-wire semantic operations. Cross-company discovery, broad +audit access, destructive document lifecycle, and company administration are +breaking v2/product-governance work. The runner must report typed unavailable +or denied outcomes rather than tunnel those operations through generic payloads. + +## Review checklist + +- Golden replay, duplicate/retry, and unknown-version fixtures accompany every + accepted protocol schema/event addition. +- A semantic receipt is typed and safe before any surface is marked supported. +- Control-plane-local decisions stay out of the semantic tool wire. +- Fixture/document debt is recorded separately from a protocol semantic gap. diff --git a/packages/paperclip-runner/spec/prp-v1-expressiveness-crosswalk.json b/packages/paperclip-runner/spec/prp-v1-expressiveness-crosswalk.json new file mode 100644 index 00000000000..041025cdb51 --- /dev/null +++ b/packages/paperclip-runner/spec/prp-v1-expressiveness-crosswalk.json @@ -0,0 +1,20 @@ +{ + "$schema": "./prp-v1-expressiveness-crosswalk.schema.json", + "version": 1, + "facts": [ + {"id":"semantic-tool-exposure","classification":"additive_v1_change","wire":"capabilities.schema.json","evidence":["capabilities.schema.json#/properties/unsupported","spec/capability/capabilities.yaml"],"invariant":"Advertised operations are a typed operation-id set with grant, task-mode, and redaction disposition; generic_api_request never counts as coverage.","correction":{"kind":"additive_schema","target":"capabilities.semanticTools","fixtures":["semantic-tool-denied.json","semantic-tool-unsupported.json"]}}, + {"id":"semantic-tool-call-result","classification":"additive_v1_change","wire":"event.schema.json","evidence":["event.schema.json#/properties/eventType","docs/capability-semantic-tools.md"],"invariant":"Each semantic invocation has one call id, operation id, input digest/redaction disposition, and exactly one typed result receipt.","correction":{"kind":"additive_event","target":"semantic_tool.call|result","fixtures":["semantic-tool-happy-path.json","semantic-tool-duplicate-retry.json"]}}, + {"id":"semantic-tool-denial-and-redaction","classification":"additive_v1_change","wire":"event.schema.json","evidence":["event.schema.json#/properties/eventType","docs/capability-authorization-and-exposure.md"],"invariant":"A denied operation yields a typed safe receipt naming the failed boundary; secret values and hidden identifiers are absent.","correction":{"kind":"additive_schema","target":"semantic_tool.receipt","fixtures":["semantic-tool-denied.json","semantic-tool-redaction.json"]}}, + {"id":"optimistic-write-conflict","classification":"additive_v1_change","wire":"event.schema.json","evidence":["event.schema.json#/properties/eventType","docs/capability-mock-control-plane-port.md"],"invariant":"A mutation receipt carries operation id, idempotency key, outcome, stable code, retryability, and safe current revision when applicable.","correction":{"kind":"additive_schema","target":"semantic_tool.receipt","fixtures":["document-conflict.json","document-duplicate-retry.json"]}}, + {"id":"artifacts-and-deliverables","classification":"compositional_v1_with_invariant","wire":"result.schema.json","evidence":["result.schema.json#/properties/artifacts","protocol/fixtures/replay/happy-path.json"],"invariant":"Every artifact ref is produced by a typed semantic receipt or registered control-plane work product, and is linked to the terminal result by run id."}, + {"id":"governance-links-and-interactions","classification":"compositional_v1_with_invariant","wire":"request.schema.json","evidence":["request.schema.json#/oneOf/1","event.schema.json#/properties/eventType"],"invariant":"Interaction and approval targets are immutable revision/decision references carried in a typed target envelope, not an unconstrained request payload."}, + {"id":"blockers-monitors-and-wakes","classification":"compositional_v1_with_invariant","wire":"result.schema.json","evidence":["result.schema.json#/properties/blocker","result.schema.json#/properties/continuation","event.schema.json#/properties/eventType"],"invariant":"A blocked or yielded result names a bounded owner/action or continuation kind; wake routing remains a control-plane decision and emits only safe causal identifiers."}, + {"id":"budget-cost-stop-reason","classification":"additive_v1_change","wire":"terminal.schema.json","evidence":["terminal.schema.json#/properties/runTerminalState","docs/capability-disposition.md"],"invariant":"A budget/cost stop includes a typed stop reason, observed limit class, safe usage aggregate, and control-plane decision id.","correction":{"kind":"additive_schema","target":"terminal.stopReason","fixtures":["budget-stop.json"]}}, + {"id":"audit-and-idempotency-receipts","classification":"additive_v1_change","wire":"event.schema.json","evidence":["event.schema.json#/required","protocol/fixtures/replay/duplicate-event.json"],"invariant":"Source-event idempotency is already replay-safe; semantic mutation idempotency additionally requires a stable operation receipt/audit ref.","correction":{"kind":"additive_schema","target":"semantic_tool.receipt","fixtures":["semantic-tool-duplicate-retry.json"]}}, + {"id":"capability-negotiation-and-unsupported","classification":"direct_v1","wire":"capabilities.schema.json","evidence":["capabilities.schema.json#/required","capabilities.schema.json#/properties/unsupported","protocol/fixtures/replay/unsupported-required-version.json"],"invariant":"The controller binds behavior to the negotiated typed booleans/version and treats unsupported required versions as fail-closed."}, + {"id":"end-to-end-causality","classification":"compositional_v1_with_invariant","wire":"event.schema.json","evidence":["event.schema.json#/required","terminal.schema.json#/properties/workAssessmentId","protocol/fixtures/replay/golden/happy-path.snapshot.json"],"invariant":"Run, session, turn, item, source-event, work-assessment, status-decision, request, and receipt identifiers form an acyclic trace from command to terminal decision."}, + {"id":"control-plane-local-lifecycle","classification":"control_plane_local","wire":"none","evidence":["docs/capability-disposition.md","spec/capability/capabilities.yaml"],"invariant":"Checkout/release, wake routing, budget enforcement, audit persistence, run persistence/replay, assignment, and monitor management are not runner tool calls and must not be serialized as semantic operations."}, + {"id":"future-cross-company-and-destructive-admin","classification":"breaking_v2_work","wire":"none","evidence":["docs/capability-disposition.md","docs/capability-future-binding-boundary.md"],"invariant":"Cross-company discovery, broad audit access, destructive document lifecycle, and company administration require separately governed product contracts; v1 must reject rather than tunnel them through generic payloads."}, + {"id":"existing-fixture-and-doc-coverage","classification":"missing_fixture_docs","wire":"protocol/fixtures/replay","evidence":["protocol/fixtures/replay/duplicate-event.json","protocol/fixtures/replay/unknown-optional-fields.json","src/protocol/replay-contract.test.ts"],"invariant":"Every accepted additive correction ships a golden replay, duplicate/retry, redaction/denial, and unknown-version fixture before it is called direct support."} + ] +} diff --git a/packages/paperclip-runner/spec/prp-v1-expressiveness-crosswalk.schema.json b/packages/paperclip-runner/spec/prp-v1-expressiveness-crosswalk.schema.json new file mode 100644 index 00000000000..65022afe12e --- /dev/null +++ b/packages/paperclip-runner/spec/prp-v1-expressiveness-crosswalk.schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://paperclip.dev/schemas/runner/prp-v1-expressiveness-crosswalk.schema.json", + "title": "PRP v1 expressiveness crosswalk", + "description": "Machine-checked source for the PRP v1 expressiveness audit. Every required semantic-parity fact is classified against exactly one support strategy, justified by concrete wire evidence and an invariant that never rests on a permissive payload field.", + "type": "object", + "additionalProperties": false, + "required": ["version", "facts"], + "properties": { + "$schema": { "type": "string" }, + "version": { "const": 1 }, + "facts": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/fact" } + } + }, + "$defs": { + "fact": { + "type": "object", + "additionalProperties": false, + "required": ["id", "classification", "wire", "evidence", "invariant"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" }, + "classification": { + "enum": [ + "direct_v1", + "compositional_v1_with_invariant", + "control_plane_local", + "additive_v1_change", + "missing_fixture_docs", + "breaking_v2_work" + ] + }, + "wire": { "type": "string", "minLength": 1 }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "invariant": { + "type": "string", + "minLength": 1, + "description": "Support justification. Must not rest on a permissive payload field.", + "not": { "pattern": "^[Pp]ayload" } + }, + "correction": { "$ref": "#/$defs/correction" } + }, + "allOf": [ + { + "$comment": "Control-plane-local and breaking v2 facts are not runner-wire operations, so they carry no wire target.", + "if": { + "properties": { + "classification": { + "enum": ["control_plane_local", "breaking_v2_work"] + } + } + }, + "then": { "properties": { "wire": { "const": "none" } } } + }, + { + "$comment": "An accepted additive v1 change must name the correction target and the fixtures that graduate it to direct support.", + "if": { + "properties": { + "classification": { "const": "additive_v1_change" } + } + }, + "then": { "required": ["correction"] } + } + ] + }, + "correction": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "target", "fixtures"], + "properties": { + "kind": { + "enum": ["additive_schema", "additive_event"] + }, + "target": { "type": "string", "minLength": 1 }, + "fixtures": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "pattern": "\\.json$" } + } + } + } + } +} diff --git a/packages/paperclip-runner/src/protocol/prp-v1-expressiveness-crosswalk.test.ts b/packages/paperclip-runner/src/protocol/prp-v1-expressiveness-crosswalk.test.ts new file mode 100644 index 00000000000..acb6dd4aaf0 --- /dev/null +++ b/packages/paperclip-runner/src/protocol/prp-v1-expressiveness-crosswalk.test.ts @@ -0,0 +1,67 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; +import Ajv2020 from "ajv/dist/2020"; +import { describe, expect, it } from "vitest"; + +type Classification = + | "direct_v1" + | "compositional_v1_with_invariant" + | "control_plane_local" + | "additive_v1_change" + | "missing_fixture_docs" + | "breaking_v2_work"; + +interface Fact { + id: string; + classification: Classification; + wire: string; + evidence: string[]; + invariant: string; + correction?: { kind: string; target: string; fixtures: string[] }; +} + +const specDir = resolve( + dirname(fileURLToPath(import.meta.url)), + "..", + "..", + "spec", +); +const crosswalkPath = resolve(specDir, "prp-v1-expressiveness-crosswalk.json"); +const schemaPath = resolve(specDir, "prp-v1-expressiveness-crosswalk.schema.json"); +const crosswalk = JSON.parse(readFileSync(crosswalkPath, "utf8")) as { facts: Fact[] }; +const crosswalkSchema = JSON.parse(readFileSync(schemaPath, "utf8")); + +describe("PRP v1 expressiveness crosswalk", () => { + it("conforms to its self-describing JSON schema", () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }); + const validate = ajv.compile(crosswalkSchema); + const valid = validate(crosswalk); + expect(validate.errors ?? [], JSON.stringify(validate.errors, null, 2)).toEqual([]); + expect(valid).toBe(true); + }); + + it("classifies every required audit fact with concrete evidence", () => { + const expected = [ + "semantic-tool-exposure", "semantic-tool-call-result", "semantic-tool-denial-and-redaction", + "optimistic-write-conflict", "artifacts-and-deliverables", "governance-links-and-interactions", + "blockers-monitors-and-wakes", "budget-cost-stop-reason", "audit-and-idempotency-receipts", + "capability-negotiation-and-unsupported", "end-to-end-causality", + "control-plane-local-lifecycle", "future-cross-company-and-destructive-admin", + "existing-fixture-and-doc-coverage", + ]; + expect(crosswalk.facts.map((fact) => fact.id).sort()).toEqual(expected.sort()); + for (const fact of crosswalk.facts) { + expect(fact.evidence.length).toBeGreaterThan(0); + expect(fact.invariant.length).toBeGreaterThan(0); + expect(fact.invariant).not.toMatch(/^payload/i); + if (fact.classification === "control_plane_local" || fact.classification === "breaking_v2_work") { + expect(fact.wire).toBe("none"); + } + if (fact.classification === "additive_v1_change") { + expect(fact.correction?.fixtures.length).toBeGreaterThan(0); + expect(fact.correction?.target.length).toBeGreaterThan(0); + } + } + }); +});