From db349f767096a561f3e342056fa550314bdc500f Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 11 Aug 2026 20:49:39 +0000 Subject: [PATCH] feat(runner): publish eval integration contract Co-Authored-By: Paperclip --- packages/paperclip-runner/README.md | 4 + ...-runner-testing-eval-package-boundaries.md | 12 +- .../paperclip-runner/docs/architecture.md | 10 +- .../docs/evals-integration.md | 91 +++ packages/paperclip-runner/docs/index.md | 1 + .../docs/protocol-compatibility.md | 19 + packages/paperclip-runner/package.json | 4 + .../evals/harness-driver-conformance.json | 29 + .../evals/native-execution-seeded.json | 258 +++++++++ .../schemas/native-execution.schema.json | 188 ++++++ packages/paperclip-runner/runner/Cargo.lock | 2 +- packages/paperclip-runner/runner/Cargo.toml | 2 +- .../runner-core/src/bin/paperclip-runnerd.rs | 41 ++ .../scripts/check-clean-consumers.mjs | 90 ++- .../scripts/check-package-boundaries.mjs | 3 + .../scripts/lib/forbidden-imports.mjs | 1 + .../paperclip-runner/src/compatibility.ts | 5 + .../src/conformance/harness-driver.test.ts | 38 ++ .../src/conformance/harness-driver.ts | 239 ++++++++ .../src/contracts/harness-driver.ts | 22 + .../src/evals/build-metadata.ts | 58 ++ .../src/evals/compatibility.test.ts | 120 ++++ .../src/evals/compatibility.ts | 244 ++++++++ packages/paperclip-runner/src/evals/index.ts | 32 ++ .../src/evals/native-execution.test.ts | 132 +++++ .../src/evals/native-execution.ts | 538 ++++++++++++++++++ .../src/evals/runnerd-artifact.test.ts | 57 ++ .../src/evals/runnerd-artifact.ts | 207 +++++++ .../mock-core/deterministic-harness-driver.ts | 353 ++++++++++++ packages/paperclip-runner/src/testing.ts | 2 + 30 files changed, 2790 insertions(+), 12 deletions(-) create mode 100644 packages/paperclip-runner/docs/evals-integration.md create mode 100644 packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json create mode 100644 packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json create mode 100644 packages/paperclip-runner/protocol/schemas/native-execution.schema.json create mode 100644 packages/paperclip-runner/src/conformance/harness-driver.test.ts create mode 100644 packages/paperclip-runner/src/conformance/harness-driver.ts create mode 100644 packages/paperclip-runner/src/evals/build-metadata.ts create mode 100644 packages/paperclip-runner/src/evals/compatibility.test.ts create mode 100644 packages/paperclip-runner/src/evals/compatibility.ts create mode 100644 packages/paperclip-runner/src/evals/index.ts create mode 100644 packages/paperclip-runner/src/evals/native-execution.test.ts create mode 100644 packages/paperclip-runner/src/evals/native-execution.ts create mode 100644 packages/paperclip-runner/src/evals/runnerd-artifact.test.ts create mode 100644 packages/paperclip-runner/src/evals/runnerd-artifact.ts create mode 100644 packages/paperclip-runner/src/mock-core/deterministic-harness-driver.ts diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index 54e87d49276..d62a1f21cb8 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -17,6 +17,9 @@ imports or starts Paperclip's server, UI, CLI, or production database. - `@paperclipai/paperclip-runner` — production contracts, clients/backends, PRP validation/replay, canonical catalog/dispatcher, and compatibility check. +- `@paperclipai/paperclip-runner/evals` — versioned native-attempt/build + metadata, explicit digest-verified runnerd artifact resolution, and complete + App/Evals compatibility negotiation. - `@paperclipai/paperclip-runner/testing` — deterministic mocks plus PRP and semantic conformance kits. Tests and Paperclip Evals import this explicitly. - `@paperclipai/paperclip-eval-kernel` — separately packed generic eval matrix @@ -164,6 +167,7 @@ and JSON content; see the protocol-server tutorial for direct `curl` examples. - [Capability live runnerd/Codex loop](docs/capability-live-runnerd-codex.md) - [Capability issue-thread UI](docs/capability-issue-thread-ui.md) - [PRP compatibility/versioning policy](docs/protocol-compatibility.md) +- [Paperclip Evals integration contract](docs/evals-integration.md) - [PRP v1 expressiveness audit](spec/prp-v1-expressiveness-audit.md) - [Cumulative end-to-end tutorial](docs/tutorials/end-to-end.md) - [OKF knowledge bundle](knowledge/) diff --git a/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md b/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md index 3677f7c0fb1..b992711714a 100644 --- a/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md +++ b/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md @@ -36,6 +36,7 @@ Public runner exports are: | Export | Stability and purpose | |---|---| | `@paperclipai/paperclip-runner` | Runtime contracts, runner clients/backends, PRP validation/replay, canonical catalog/dispatcher, and compatibility preflight | +| `@paperclipai/paperclip-runner/evals` | Versioned native-attempt/build metadata, explicit runnerd artifact resolver, and complete App/Evals compatibility negotiation | | `@paperclipai/paperclip-runner/testing` | Deterministic mocks, PRP port conformance, and provider-neutral semantic conformance kit | | `./browser`, `./react`, `./standalone`, `./styles.css` | Existing explicitly named UI/standalone consumers | @@ -85,6 +86,10 @@ contracts are published in `PAPERCLIP_RUNNER_COMPATIBILITY`: | Canonical catalog | 1 | Operation removals, renames, placement changes, or incompatible schemas require a new contract version | | PRP | 1 | Highest overlapping required protocol version; no overlap fails closed | | Runner client | 1 | Breaking runnerd/client interface changes require a new version | +| runnerd artifact | 1 | Binary metadata/package disagreement or digest mismatch fails before launch | +| Harness driver | 1 | Breaking descriptor/config/session/conformance behavior requires a new version | +| Native execution | 1 | Breaking App attempt-bundle semantics require a new schema version and converter | +| Evals integration | 1 | Breaking package/binary/catalog/driver join behavior requires a new version | | Control-plane adapter | 1 | Breaking `ControlPlanePort` or production-binding expectations require a new version | | Testkit | 1 | Breaking mock seed, vector, observation, or conformance behavior requires a new version | | Eval corpus | 1 | Runner declares a supported inclusive corpus-version range; out-of-range bundles fail before execution | @@ -108,9 +113,10 @@ pnpm --filter @paperclipai/paperclip-runner check:clean-consumers The second command builds and packs both packages, then installs tarballs into two newly created consumers: -1. An Evals-side consumer imports only declared runner root and `./testing` - exports, executes the deterministic PRP conformance kit, and proves an - unsupported provider operation fails explicitly. +1. An Evals-side consumer imports only declared runner root, `./evals`, and + `./testing` exports; validates the native bundle; executes deterministic PRP + and harness-driver conformance; resolves a separately copied runnerd by + explicit path/digest; and proves provider/driver mismatches fail explicitly. 2. An App-side consumer installs the eval kernel as a development dependency, runs a generic matrix after runner compatibility preflight, and proves the runner manifest has no Evals runtime dependency. diff --git a/packages/paperclip-runner/docs/architecture.md b/packages/paperclip-runner/docs/architecture.md index 3dba631cf14..e4306fa9491 100644 --- a/packages/paperclip-runner/docs/architecture.md +++ b/packages/paperclip-runner/docs/architecture.md @@ -14,7 +14,7 @@ byte-identical Conformance result Paperclip App production binding --> implements ControlPlanePort -Paperclip Evals --> consumes packed runtime + ./testing exports +Paperclip Evals --> consumes packed runtime + ./evals + ./testing exports ``` The dependency arrow always points from an implementation toward a contract. @@ -28,6 +28,9 @@ recorded in [ADR 0001](adr/0001-runner-testing-eval-package-boundaries.md). - The package root is runtime-only: PRP, runner/client contracts, normalized backends, catalog/dispatcher, and compatibility preflight. +- `./evals` is the stable integration join: native attempt schema/fixture, + build metadata, complete package/binary/PRP/catalog/driver negotiation, and + explicit digest-verified runnerd artifact resolution. - `./testing` contains deterministic mocks and conformance kits. - `@paperclipai/paperclip-eval-kernel` contains generic orchestration only and is permitted in App CI/tests as a development dependency, never at runtime. @@ -67,6 +70,11 @@ boundary and checks the same fixture summaries. Local runner adds the package-lo `paperclip-runnerd` and `fake-harness` binaries without changing that dependency direction. +Evals receives runnerd as a separate content-addressed artifact. The package +does not discover or bundle a host-specific executable: the consumer supplies +an explicit path and digest, then reads versioned metadata from that exact +binary. See [the Evals integration contract](evals-integration.md). + ## Language ownership - Rust is the production direction for deterministic runner behavior, diff --git a/packages/paperclip-runner/docs/evals-integration.md b/packages/paperclip-runner/docs/evals-integration.md new file mode 100644 index 00000000000..4398c801b99 --- /dev/null +++ b/packages/paperclip-runner/docs/evals-integration.md @@ -0,0 +1,91 @@ +# Paperclip Evals Integration Contract + +## Stable consumer inputs + +Paperclip Evals consumes two explicit App artifacts: + +1. a packed/released `@paperclipai/paperclip-runner` package; and +2. an explicit `paperclip-runnerd` executable path plus its + `sha256:` digest. + +The consumer must not import App source paths, search a workspace for a binary, +or infer compatibility from a process failure. The package exposes three +relevant entry points: + +- package root: PRP, runner, semantic-tool, and control-plane runtime contracts; +- `./evals`: build metadata, native-attempt schema/fixture, compatibility + negotiation, semantic receipts/catalog, and explicit runnerd resolution; +- `./testing`: deterministic driver/control-plane fakes and conformance kits. + +`resolvePaperclipRunnerdArtifact` resolves only the supplied path, verifies its +bytes against the supplied digest, and invokes that exact executable with +`--build-metadata`. It never searches `PATH` or the App repository. + +## Versioned join + +`assertPaperclipRunnerEvalCompatibility` checks the complete join before any +provider process starts: + +| Dimension | V1 rule | +| --- | --- | +| Package | Exact package version matches loaded build metadata. | +| Binary | runnerd package version and binary-artifact contract match the package. | +| PRP | Package, runnerd, and consumer version ranges have a common version. | +| Semantic catalog | Contract version and canonical content digest both match. | +| Harness driver | Contract and negotiated PRP versions match and every required capability is explicitly true. | +| Native output | Consumer and runnerd both select `paperclip-runner/native-execution/v1`. | + +Failures use `paperclip_runner_eval_incompatible` and include all detected +component issues with expected and received values. A catalog, driver, or PRP +mismatch cannot degrade into an attempted run. + +## Native attempt bundle + +`paperclip-runner/native-execution/v1` is the App-owned raw attempt. Its checked +schema is `protocol/schemas/native-execution.schema.json`; the no-spend seeded +fixture is `protocol/fixtures/evals/native-execution-seeded.json`. + +The bundle pins run/case/config/attempt identity, case and config digests, +package/binary/catalog/driver versions and digests, ordered PRP events, semantic +tool definitions/calls/results/denials, terminal state, check-ready +observations, usage/cost/time/request totals, transcript completeness, and +content-addressed artifacts. Unknown additive fields survive parsing. + +The parser fails closed on an unknown native schema, malformed digest, event +run mismatch, missing or conflicting terminal, semantic indexes that omit or +reinterpret their PRP tool envelopes, denied results without matching denial +receipts, inconsistent token totals, or ambiguous incomplete transcripts. +Unknown additive fields are retained at nested contract objects as well as the +bundle root. The seeded fixture deliberately preserves a rejected governed +tool effect as `denied`; it never turns that effect into a successful mutation. + +Evals owns conversion of this bundle into Evalbook ledger, scores, and reports. +The App package does not implement an Evalbook environment loader, importer, +grid, comparison, or report renderer. + +## Deterministic conformance + +`runHarnessDriverConformance` with `DeterministicHarnessDriver` covers, without +network or provider credentials: + +- capability and unsupported-feature description; +- valid/invalid config handling; +- session open, turn, snapshot, recovery, close, and cancel lifecycle; +- paired provider-neutral semantic tool events; +- PRP event validation; +- interrupt and cancelled terminal behavior; +- usage reporting; and +- complete transcript accounting. + +The input fixture is +`protocol/fixtures/evals/harness-driver-conformance.json`. The packed +clean-consumer gate builds and copies runnerd into an isolated artifact +directory, packs the package, installs it offline, resolves only declared +exports, validates the native fixture, runs driver conformance, verifies binary +metadata/digest, and exercises both compatible and incompatible negotiation. + +Run it with: + +```sh +pnpm --filter @paperclipai/paperclip-runner check:clean-consumers +``` diff --git a/packages/paperclip-runner/docs/index.md b/packages/paperclip-runner/docs/index.md index 9d7c629b249..06a82e69102 100644 --- a/packages/paperclip-runner/docs/index.md +++ b/packages/paperclip-runner/docs/index.md @@ -23,6 +23,7 @@ Each tutorial explains the capability it exercises and the contract it proves. ## Reference - [Architecture and dependency boundary](architecture.md) +- [Paperclip Evals integration contract](evals-integration.md) - [PRP compatibility and versioning policy](protocol-compatibility.md) - [PRP v1 expressiveness audit](../spec/prp-v1-expressiveness-audit.md) - [ADR 0001: runner, testing, and eval package boundaries](adr/0001-runner-testing-eval-package-boundaries.md) diff --git a/packages/paperclip-runner/docs/protocol-compatibility.md b/packages/paperclip-runner/docs/protocol-compatibility.md index b8a1f2d8fea..c7e57a84b03 100644 --- a/packages/paperclip-runner/docs/protocol-compatibility.md +++ b/packages/paperclip-runner/docs/protocol-compatibility.md @@ -155,3 +155,22 @@ tool error is not a compatibility negotiation mechanism. See [ADR 0001](adr/0001-runner-testing-eval-package-boundaries.md) for the component rules and clean-consumer packaging gate. + +## Evals integration negotiation + +The packed `./evals` entry point adds a stricter execution preflight for the +App/Evals join. `assertPaperclipRunnerEvalCompatibility` requires simultaneous +agreement on package semver, runnerd build metadata, a common PRP version, +semantic catalog version and SHA-256 digest, harness-driver contract and +required capabilities, and the native-execution version. It reports +`paperclip_runner_eval_incompatible` with expected/received values for every +mismatch and must run before launching a provider. + +runnerd itself reports `paperclip-runner/runnerd-build-metadata/v1` from +`--build-metadata`. The consumer passes its path and expected content digest to +`resolvePaperclipRunnerdArtifact`; implicit PATH or source-tree discovery is +not part of the contract. Native attempt output is +`paperclip-runner/native-execution/v1`, whose parser accepts unknown additive +fields but rejects unknown required versions and inconsistent terminal, +semantic-denial, usage, or transcript facts. Full fields and the deterministic +gate are documented in [the Evals integration contract](evals-integration.md). diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 08e912900f4..d48495d6d71 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -13,6 +13,10 @@ "types": "./dist/testing.d.ts", "import": "./dist/testing.js" }, + "./evals": { + "types": "./dist/evals/index.d.ts", + "import": "./dist/evals/index.js" + }, "./browser": { "types": "./dist/browser/index.d.ts", "import": "./dist/browser/index.js" diff --git a/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json b/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json new file mode 100644 index 00000000000..670fc5b980d --- /dev/null +++ b/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json @@ -0,0 +1,29 @@ +{ + "schema": "paperclip-runner/harness-driver-conformance-fixture/v1", + "validConfig": { + "schema": "paperclip-runner/deterministic-harness-driver-config/v1", + "scenario": "semantic-tool-success" + }, + "invalidConfig": { + "schema": "paperclip-runner/deterministic-harness-driver-config/v1", + "scenario": "unknown" + }, + "completionMessage": "Run deterministic semantic-tool conformance.", + "interruptMessage": "[conformance:interrupt] Keep the turn active until interrupted.", + "requiredCapabilities": [ + "resume", + "typedEvents", + "interruption", + "structuredResult", + "read", + "reconciliation", + "usage", + "dynamicTools" + ], + "unsupportedFeatures": [ + "steering", + "runtimeRequestResolution", + "goals", + "threadLineage" + ] +} diff --git a/packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json b/packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json new file mode 100644 index 00000000000..5c0a62f64b1 --- /dev/null +++ b/packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json @@ -0,0 +1,258 @@ +{ + "schema": "paperclip-runner/native-execution/v1", + "provenance": "seeded", + "identity": { + "runId": "run_evals_seeded", + "caseId": "case_semantic_denial", + "configId": "config_deterministic_v1", + "attemptId": "attempt_semantic_denial_0001" + }, + "input": { + "caseSha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "configSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "deterministicSeed": "paperclip-evals-v1-seed" + }, + "runner": { + "package": { + "name": "@paperclipai/paperclip-runner", + "version": "0.1.2" + }, + "binary": { + "name": "paperclip-runnerd", + "sha256": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "prpVersion": 1, + "nativeExecutionVersion": 1, + "catalogVersion": 1, + "catalogSha256": "sha256:b6c2723c0b2a041f7b23c00a6043bd81d69ef3d299f67b6dbd689afb7c36dc01", + "driverContractVersion": 1, + "driverKind": "paperclip-deterministic", + "driverVersion": "1.0.0" + }, + "events": [ + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_01", + "sourceSeq": 1, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded", + "eventType": "mcp_app.tool_input", + "schemaVersion": 1, + "priority": 1, + "emittedAt": "2026-08-11T12:00:00.000Z", + "payload": { + "semantic_tool": { + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "input", + "operationId": "decide_approval", + "callId": "call_evals_seeded", + "correlation": { + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded" + }, + "idempotencyKey": "decide_evals_seeded", + "content": { + "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "redactionDisposition": "redacted", + "references": [] + } + } + } + }, + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_02", + "sourceSeq": 2, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded", + "eventType": "mcp_app.tool_result", + "schemaVersion": 1, + "priority": 1, + "emittedAt": "2026-08-11T12:00:00.010Z", + "payload": { + "semantic_tool": { + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "result", + "operationId": "decide_approval", + "callId": "call_evals_seeded", + "correlation": { + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded" + }, + "idempotencyKey": "decide_evals_seeded", + "content": { + "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "redactionDisposition": "redacted", + "references": [] + }, + "outcome": "denied", + "code": "required_claim_missing", + "retryable": false, + "authorizationBoundary": "grant", + "operationReceiptId": "receipt_evals_seeded", + "auditReceiptId": "audit_evals_seeded" + } + } + }, + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_03", + "sourceSeq": 3, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "eventType": "run.result.proposed", + "schemaVersion": 1, + "priority": 0, + "emittedAt": "2026-08-11T12:00:00.020Z", + "payload": { + "schema": "paperclip.run_result.v1", + "reportedWorkDisposition": "needs_review", + "summary": "The governed tool effect was rejected and preserved as a denial.", + "completionClaim": { + "contractRevision": "evals-native-execution-v1", + "objectiveSatisfied": false, + "criteria": [], + "remainingWork": [ + { + "description": "An authorized approver must decide.", + "blocksCompletion": true + } + ] + }, + "evidence": [{ "receiptId": "receipt_evals_seeded" }], + "verification": [ + { + "commandOrCheck": "rejected tool effect stays denied", + "status": "passed" + } + ], + "attentionRequests": [], + "artifacts": [] + } + }, + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_04", + "sourceSeq": 4, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "eventType": "run.terminal", + "schemaVersion": 1, + "priority": 0, + "emittedAt": "2026-08-11T12:00:00.030Z", + "payload": { + "schema": "paperclip.prp.terminal.v1", + "turnTerminalState": "completed", + "runTerminalState": "succeeded", + "reportedWorkDisposition": "needs_review", + "workAssessmentId": "assessment_evals_seeded", + "statusDecisionId": "decision_evals_seeded" + } + } + ], + "semanticTools": { + "definitions": [ + { + "operationId": "decide_approval", + "schemaVersion": 1, + "availability": "denied" + } + ], + "calls": [ + { + "callId": "call_evals_seeded", + "operationId": "decide_approval", + "eventId": "evals_seeded_01" + } + ], + "results": [ + { + "callId": "call_evals_seeded", + "operationId": "decide_approval", + "eventId": "evals_seeded_02", + "outcome": "denied" + } + ], + "denials": [ + { + "callId": "call_evals_seeded", + "code": "required_claim_missing", + "authorizationBoundary": "grant", + "retryable": false + } + ] + }, + "terminal": { + "schema": "paperclip.prp.terminal.v1", + "turnTerminalState": "completed", + "runTerminalState": "succeeded", + "reportedWorkDisposition": "needs_review", + "workAssessmentId": "assessment_evals_seeded", + "statusDecisionId": "decision_evals_seeded" + }, + "observations": { + "checks": [ + { + "id": "tool-effect-denied", + "passed": true, + "receiptId": "receipt_evals_seeded" + } + ] + }, + "usage": { + "inputTokens": 12, + "outputTokens": 8, + "totalTokens": 20, + "requestCount": 1, + "durationMs": 30, + "cost": { + "currency": "USD", + "amountMicros": 0 + } + }, + "transcript": { + "uri": "artifact://native/attempt_semantic_denial_0001/transcript.jsonl", + "sha256": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "byteSize": 2048, + "mediaType": "application/x-ndjson", + "complete": true, + "eventCount": 4, + "omissionReason": null + }, + "artifactRoot": { + "uri": "artifact://native/attempt_semantic_denial_0001/", + "sha256": "sha256:1212121212121212121212121212121212121212121212121212121212121212", + "byteSize": 4096, + "mediaType": "application/vnd.paperclip.runner-artifact-root+json" + }, + "artifacts": [ + { + "uri": "artifact://native/attempt_semantic_denial_0001/denial-receipt.json", + "sha256": "sha256:1313131313131313131313131313131313131313131313131313131313131313", + "byteSize": 512, + "mediaType": "application/json" + } + ], + "failure": null, + "x_fixturePurpose": "Proves seeded provenance, denied effects, and additive field preservation." +} diff --git a/packages/paperclip-runner/protocol/schemas/native-execution.schema.json b/packages/paperclip-runner/protocol/schemas/native-execution.schema.json new file mode 100644 index 00000000000..86dfc266327 --- /dev/null +++ b/packages/paperclip-runner/protocol/schemas/native-execution.schema.json @@ -0,0 +1,188 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://paperclip.dev/schemas/runner/native-execution/v1.json", + "title": "Paperclip runner native execution bundle", + "type": "object", + "required": [ + "schema", + "provenance", + "identity", + "input", + "runner", + "events", + "semanticTools", + "terminal", + "observations", + "usage", + "transcript", + "artifactRoot", + "artifacts", + "failure" + ], + "properties": { + "schema": { "const": "paperclip-runner/native-execution/v1" }, + "provenance": { "enum": ["seeded", "replay", "live"] }, + "identity": { + "type": "object", + "required": ["runId", "caseId", "configId", "attemptId"], + "properties": { + "runId": { "$ref": "#/$defs/nonEmpty" }, + "caseId": { "$ref": "#/$defs/nonEmpty" }, + "configId": { "$ref": "#/$defs/nonEmpty" }, + "attemptId": { "$ref": "#/$defs/nonEmpty" } + }, + "additionalProperties": true + }, + "input": { + "type": "object", + "required": ["caseSha256", "configSha256", "deterministicSeed"], + "properties": { + "caseSha256": { "$ref": "#/$defs/sha256" }, + "configSha256": { "$ref": "#/$defs/sha256" }, + "deterministicSeed": { "$ref": "#/$defs/nonEmpty" } + }, + "additionalProperties": true + }, + "runner": { + "type": "object", + "required": [ + "package", + "binary", + "prpVersion", + "nativeExecutionVersion", + "catalogVersion", + "catalogSha256", + "driverContractVersion", + "driverKind", + "driverVersion" + ], + "properties": { + "package": { + "type": "object", + "required": ["name", "version"], + "properties": { + "name": { "$ref": "#/$defs/nonEmpty" }, + "version": { "$ref": "#/$defs/nonEmpty" } + }, + "additionalProperties": true + }, + "binary": { + "type": "object", + "required": ["name", "sha256"], + "properties": { + "name": { "$ref": "#/$defs/nonEmpty" }, + "sha256": { "$ref": "#/$defs/sha256" } + }, + "additionalProperties": true + }, + "prpVersion": { "$ref": "#/$defs/version" }, + "nativeExecutionVersion": { "$ref": "#/$defs/version" }, + "catalogVersion": { "$ref": "#/$defs/version" }, + "catalogSha256": { "$ref": "#/$defs/sha256" }, + "driverContractVersion": { "$ref": "#/$defs/version" }, + "driverKind": { "$ref": "#/$defs/nonEmpty" }, + "driverVersion": { "$ref": "#/$defs/nonEmpty" } + }, + "additionalProperties": true + }, + "events": { + "type": "array", + "minItems": 1, + "items": { "$ref": "https://paperclip.dev/schemas/prp/v1/event.schema.json" } + }, + "semanticTools": { + "type": "object", + "required": ["definitions", "calls", "results", "denials"], + "properties": { + "definitions": { "type": "array", "items": { "type": "object" } }, + "calls": { "type": "array", "items": { "type": "object" } }, + "results": { "type": "array", "items": { "type": "object" } }, + "denials": { "type": "array", "items": { "type": "object" } } + }, + "additionalProperties": true + }, + "terminal": { "$ref": "https://paperclip.dev/schemas/prp/v1/terminal.schema.json" }, + "observations": { "type": "object", "additionalProperties": true }, + "usage": { + "type": "object", + "required": [ + "inputTokens", + "outputTokens", + "totalTokens", + "requestCount", + "durationMs", + "cost" + ], + "properties": { + "inputTokens": { "type": "integer", "minimum": 0 }, + "outputTokens": { "type": "integer", "minimum": 0 }, + "totalTokens": { "type": "integer", "minimum": 0 }, + "requestCount": { "type": "integer", "minimum": 0 }, + "durationMs": { "type": "integer", "minimum": 0 }, + "cost": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["currency", "amountMicros"], + "properties": { + "currency": { "$ref": "#/$defs/nonEmpty" }, + "amountMicros": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": true + } + ] + } + }, + "additionalProperties": true + }, + "transcript": { + "allOf": [ + { "$ref": "#/$defs/contentRef" }, + { + "type": "object", + "required": ["complete", "eventCount", "omissionReason"], + "properties": { + "complete": { "type": "boolean" }, + "eventCount": { "type": "integer", "minimum": 0 }, + "omissionReason": { "type": ["string", "null"] } + } + } + ] + }, + "artifactRoot": { "$ref": "#/$defs/contentRef" }, + "artifacts": { "type": "array", "items": { "$ref": "#/$defs/contentRef" } }, + "failure": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["classification", "code", "message"], + "properties": { + "classification": { "$ref": "#/$defs/nonEmpty" }, + "code": { "$ref": "#/$defs/nonEmpty" }, + "message": { "$ref": "#/$defs/nonEmpty" } + }, + "additionalProperties": true + } + ] + } + }, + "additionalProperties": true, + "$defs": { + "nonEmpty": { "type": "string", "minLength": 1 }, + "version": { "type": "integer", "minimum": 1 }, + "sha256": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "contentRef": { + "type": "object", + "required": ["uri", "sha256", "byteSize", "mediaType"], + "properties": { + "uri": { "$ref": "#/$defs/nonEmpty" }, + "sha256": { "$ref": "#/$defs/sha256" }, + "byteSize": { "type": "integer", "minimum": 0 }, + "mediaType": { "$ref": "#/$defs/nonEmpty" } + }, + "additionalProperties": true + } + } +} diff --git a/packages/paperclip-runner/runner/Cargo.lock b/packages/paperclip-runner/runner/Cargo.lock index e609c2e5bd9..92aac34b4c2 100644 --- a/packages/paperclip-runner/runner/Cargo.lock +++ b/packages/paperclip-runner/runner/Cargo.lock @@ -177,7 +177,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "paperclip-runner-core" -version = "0.0.0" +version = "0.1.2" dependencies = [ "aes-gcm", "hmac", diff --git a/packages/paperclip-runner/runner/Cargo.toml b/packages/paperclip-runner/runner/Cargo.toml index 58aeff2a863..f390d360d4f 100644 --- a/packages/paperclip-runner/runner/Cargo.toml +++ b/packages/paperclip-runner/runner/Cargo.toml @@ -3,7 +3,7 @@ members = ["crates/runner-core"] resolver = "2" [workspace.package] -version = "0.0.0" +version = "0.1.2" edition = "2021" license = "MIT" publish = false diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs index f6cb56d21c5..f9fab1a101a 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs @@ -6,6 +6,26 @@ use paperclip_runner_core::durable::{ capture_bootstrap_ticket, run_durable_runner, BootstrapTicket, DurableRunnerConfig, }; use paperclip_runner_core::local_runner::{run_local_runner, LocalRunnerError, RunnerConfig}; +use serde_json::json; + +const RUNNERD_BUILD_METADATA_SCHEMA: &str = "paperclip-runner/runnerd-build-metadata/v1"; + +fn build_metadata() -> serde_json::Value { + json!({ + "schema": RUNNERD_BUILD_METADATA_SCHEMA, + "binaryName": "paperclip-runnerd", + "packageName": "@paperclipai/paperclip-runner", + "packageVersion": env!("CARGO_PKG_VERSION"), + "binaryContractVersion": 1, + "nativeExecutionVersion": 1, + "harnessDriverVersion": 1, + "prp": { + "name": "paperclip.runner", + "minimumVersion": 1, + "maximumVersion": 1 + } + }) +} fn value(args: &[String], name: &str) -> Result { let index = args @@ -162,6 +182,10 @@ fn run_durable_mode( fn run(bootstrap_ticket: Option) -> Result<(), LocalRunnerError> { let args = std::env::args().skip(1).collect::>(); + if args.as_slice() == ["--build-metadata"] { + println!("{}", build_metadata()); + return Ok(()); + } if args .iter() .any(|argument| argument == "--codex-app-server-proxy") @@ -187,6 +211,23 @@ fn run(bootstrap_ticket: Option) -> Result<(), LocalRunnerError }) } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_metadata_names_the_versioned_eval_contracts() { + let metadata = build_metadata(); + assert_eq!(metadata["schema"], RUNNERD_BUILD_METADATA_SCHEMA); + assert_eq!(metadata["packageVersion"], "0.1.2"); + assert_eq!(metadata["binaryContractVersion"], 1); + assert_eq!(metadata["nativeExecutionVersion"], 1); + assert_eq!(metadata["harnessDriverVersion"], 1); + assert_eq!(metadata["prp"]["minimumVersion"], 1); + assert_eq!(metadata["prp"]["maximumVersion"], 1); + } +} + fn main() -> ExitCode { // Capture and remove the bootstrap capability before argument parsing or child work. let bootstrap_ticket = match capture_bootstrap_ticket() { diff --git a/packages/paperclip-runner/scripts/check-clean-consumers.mjs b/packages/paperclip-runner/scripts/check-clean-consumers.mjs index 6246af5cbd9..7d1cd304a2d 100644 --- a/packages/paperclip-runner/scripts/check-clean-consumers.mjs +++ b/packages/paperclip-runner/scripts/check-clean-consumers.mjs @@ -1,5 +1,6 @@ import { spawnSync } from "node:child_process"; -import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { chmod, copyFile, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; @@ -16,15 +17,18 @@ await mkdir(artifactsRoot, { recursive: true }); try { run("pnpm", ["run", "build:typescript"], runnerRoot); + run("pnpm", ["run", "build:runner-binaries"], runnerRoot); run("pnpm", ["run", "build"], evalKernelRoot); const runnerTarball = await pack(runnerRoot, artifactsRoot); const evalKernelTarball = await pack(evalKernelRoot, artifactsRoot); const runtimeDependencyTarballs = await packRunnerRuntimeDependencies(artifactsRoot); + const runnerdArtifact = await stageRunnerdArtifact(artifactsRoot); await verifyEvalsConsumer( resolve(scratchRoot, "evals-consumer"), runnerTarball, runtimeDependencyTarballs, + runnerdArtifact, ); await verifyAppDevConsumer( resolve(scratchRoot, "app-dev-consumer"), @@ -42,7 +46,7 @@ try { if ("@paperclipai/paperclip-eval-kernel" in runtimeDependencies) { throw new Error("runner tarball has a runtime dependency on Paperclip Evals"); } - process.stdout.write("Clean-consumer pack/install checks passed for Evals -> App and App dev -> eval kernel.\n"); + process.stdout.write("Clean-consumer pack/install checks passed for Evals -> App package/runnerd and App dev -> eval kernel.\n"); } finally { if (process.env.PAPERCLIP_KEEP_PACKAGE_CONSUMERS !== "1") { await rm(scratchRoot, { recursive: true, force: true }); @@ -81,13 +85,28 @@ async function packRunnerRuntimeDependencies(destination) { return tarballs; } +async function stageRunnerdArtifact(destination) { + const suffix = process.platform === "win32" ? ".exe" : ""; + const source = resolve(runnerRoot, `runner/target/debug/paperclip-runnerd${suffix}`); + const executablePath = resolve(destination, `paperclip-runnerd${suffix}`); + await copyFile(source, executablePath); + if (process.platform !== "win32") await chmod(executablePath, 0o755); + const sha256 = `sha256:${createHash("sha256").update(await readFile(executablePath)).digest("hex")}`; + return { executablePath, sha256 }; +} + function localOverrides(tarballs) { return Object.fromEntries( Object.entries(tarballs).map(([packageName, tarball]) => [packageName, `file:${tarball}`]), ); } -async function verifyEvalsConsumer(consumerRoot, runnerTarball, runtimeDependencyTarballs) { +async function verifyEvalsConsumer( + consumerRoot, + runnerTarball, + runtimeDependencyTarballs, + runnerdArtifact, +) { await mkdir(consumerRoot, { recursive: true }); await writeFile(resolve(consumerRoot, "package.json"), `${JSON.stringify({ name: "paperclip-evals-clean-consumer", @@ -101,6 +120,7 @@ async function verifyEvalsConsumer(consumerRoot, runnerTarball, runtimeDependenc }, null, 2)}\n`); await writeFile(resolve(consumerRoot, "verify.mjs"), ` import * as runtime from "@paperclipai/paperclip-runner"; +import * as evals from "@paperclipai/paperclip-runner/evals"; import * as testing from "@paperclipai/paperclip-runner/testing"; if ("MockControlPlaneAdapter" in runtime || "runControlPlanePortConformance" in runtime) { @@ -117,6 +137,40 @@ const report = await testing.runControlPlanePortConformance({ }); if (report.eventCount !== 3) throw new Error("packed conformance kit returned the wrong event count"); +const nativeBundle = await evals.loadPaperclipNativeExecutionFixture(); +if (nativeBundle.schema !== "paperclip-runner/native-execution/v1") { + throw new Error("packed native execution fixture is unavailable"); +} +if (nativeBundle.semanticTools.results[0]?.outcome !== "denied") { + throw new Error("native execution fixture lost its rejected tool effect"); +} +const driverConformance = await testing.runHarnessDriverConformance({ + driver: new testing.DeterministicHarnessDriver(), +}); +if (!driverConformance.checks.transcriptCompleteness || driverConformance.semanticToolCallCount !== 1) { + throw new Error("packed harness-driver conformance did not cover transcript/tools"); +} +const runnerd = await evals.resolvePaperclipRunnerdArtifact({ + executablePath: process.env.PAPERCLIP_RUNNERD_ARTIFACT, + expectedSha256: process.env.PAPERCLIP_RUNNERD_SHA256, +}); +const integration = evals.assertPaperclipRunnerEvalCompatibility({ + consumer: "paperclip-evals-clean-consumer", + packageVersion: evals.PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + runnerd: runnerd.buildMetadata, + nativeExecutionVersion: 1, + prp: { minimumVersion: 1, maximumVersion: 1 }, + catalog: evals.PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog, + driver: { + contractVersion: driverConformance.contractVersion, + descriptor: driverConformance.descriptor, + requiredCapabilities: ["typedEvents", "interruption", "usage", "dynamicTools"], + }, +}); +if (integration.negotiatedPrpVersion !== 1) { + throw new Error("package/binary PRP negotiation returned the wrong version"); +} + runtime.assertPaperclipRunnerCompatibility({ consumer: "paperclip-evals-clean-consumer", components: { catalog: 1, protocol: 1, runnerClient: 1, controlPlaneAdapter: 1, testkit: 1 }, @@ -137,6 +191,27 @@ try { } if (!failedExplicitly) throw new Error("provider incompatibility did not fail explicitly"); +let driverMismatchFailed = false; +try { + evals.assertPaperclipRunnerEvalCompatibility({ + consumer: "incompatible-driver", + packageVersion: evals.PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + runnerd: runnerd.buildMetadata, + nativeExecutionVersion: 1, + prp: { minimumVersion: 1, maximumVersion: 1 }, + catalog: evals.PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog, + driver: { + contractVersion: 2, + descriptor: driverConformance.descriptor, + requiredCapabilities: [], + }, + }); +} catch (error) { + driverMismatchFailed = error?.code === "paperclip_runner_eval_incompatible" + && error.issues?.some((issue) => issue.code === "driver_contract_version_mismatch"); +} +if (!driverMismatchFailed) throw new Error("driver mismatch did not fail explicitly"); + const normalized = { authorization: { outcome: "allowed" }, state: { status: "done" }, @@ -151,7 +226,10 @@ await testing.runSemanticConformanceKit({ ], }); `); - installAndRun(consumerRoot); + installAndRun(consumerRoot, { + PAPERCLIP_RUNNERD_ARTIFACT: runnerdArtifact.executablePath, + PAPERCLIP_RUNNERD_SHA256: runnerdArtifact.sha256, + }); } async function verifyAppDevConsumer( @@ -209,7 +287,7 @@ if (results.length !== 1 || results[0].score.passed !== true) { installAndRun(consumerRoot); } -function installAndRun(consumerRoot) { +function installAndRun(consumerRoot, extraEnv = {}) { run("pnpm", [ "install", "--offline", @@ -220,7 +298,7 @@ function installAndRun(consumerRoot) { "--config.auto-install-peers=false", "--reporter=append-only", ], consumerRoot, { env: { NODE_ENV: "development" } }); - run(process.execPath, ["verify.mjs"], consumerRoot); + run(process.execPath, ["verify.mjs"], consumerRoot, { env: extraEnv }); } function run(command, args, cwd, { quiet = false, env = {} } = {}) { diff --git a/packages/paperclip-runner/scripts/check-package-boundaries.mjs b/packages/paperclip-runner/scripts/check-package-boundaries.mjs index 6344b8b17ac..2d48dbcf004 100644 --- a/packages/paperclip-runner/scripts/check-package-boundaries.mjs +++ b/packages/paperclip-runner/scripts/check-package-boundaries.mjs @@ -12,6 +12,9 @@ const violations = []; if (runnerManifest.exports?.["./testing"] === undefined) { violations.push("runner package must declare the ./testing export"); } +if (runnerManifest.exports?.["./evals"] === undefined) { + violations.push("runner package must declare the ./evals integration export"); +} for (const privatePath of ["mock-core", "conformance", "scenarios", "src/tools", "./tools/"]) { if (runtimeIndex.includes(privatePath)) { violations.push(`runtime root must not export test/eval path: ${privatePath}`); diff --git a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs index b42a209a402..103d9c11bb1 100644 --- a/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs +++ b/packages/paperclip-runner/scripts/lib/forbidden-imports.mjs @@ -88,6 +88,7 @@ function violationReason({ file, packageRoot, specifier }) { "@paperclipai/paperclip-runner/react", "@paperclipai/paperclip-runner/standalone", "@paperclipai/paperclip-runner/testing", + "@paperclipai/paperclip-runner/evals", "@paperclipai/paperclip-runner/styles.css", ]); if ( diff --git a/packages/paperclip-runner/src/compatibility.ts b/packages/paperclip-runner/src/compatibility.ts index 905745d7223..2a4001a49d6 100644 --- a/packages/paperclip-runner/src/compatibility.ts +++ b/packages/paperclip-runner/src/compatibility.ts @@ -7,7 +7,12 @@ export const PAPERCLIP_RUNNER_COMPATIBILITY = Object.freeze({ components: Object.freeze({ catalog: 1, protocol: 1, + prp: 1, runnerClient: 1, + runnerdBinary: 1, + harnessDriver: 1, + nativeExecution: 1, + evalIntegration: 1, controlPlaneAdapter: 1, testkit: 1, }), diff --git a/packages/paperclip-runner/src/conformance/harness-driver.test.ts b/packages/paperclip-runner/src/conformance/harness-driver.test.ts new file mode 100644 index 00000000000..9287ab38eca --- /dev/null +++ b/packages/paperclip-runner/src/conformance/harness-driver.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; + +import { DeterministicHarnessDriver } from "../mock-core/deterministic-harness-driver.js"; +import { + loadHarnessDriverConformanceFixture, + runHarnessDriverConformance, +} from "./harness-driver.js"; + +describe("harness-driver conformance V1", () => { + it("ships a deterministic fixture covering the complete Evals hook set", async () => { + const fixture = await loadHarnessDriverConformanceFixture(); + expect(fixture.requiredCapabilities).toContain("dynamicTools"); + expect(fixture.unsupportedFeatures).toContain("steering"); + + const report = await runHarnessDriverConformance({ + driver: new DeterministicHarnessDriver(), + fixture, + }); + expect(report).toMatchObject({ + schema: "paperclip-runner/harness-driver-conformance-report/v1", + contractVersion: 1, + eventCount: 7, + semanticToolCallCount: 1, + checks: { + capabilityDescription: true, + configValidation: true, + sessionLifecycle: true, + sessionRecovery: true, + semanticTools: true, + eventValidation: true, + interruptAndCancel: true, + usage: true, + transcriptCompleteness: true, + unsupportedFeatures: true, + }, + }); + }); +}); diff --git a/packages/paperclip-runner/src/conformance/harness-driver.ts b/packages/paperclip-runner/src/conformance/harness-driver.ts new file mode 100644 index 00000000000..37f620230ea --- /dev/null +++ b/packages/paperclip-runner/src/conformance/harness-driver.ts @@ -0,0 +1,239 @@ +import { readFile } from "node:fs/promises"; + +import { + HARNESS_DRIVER_CONTRACT_VERSION, + type HarnessDriver, + type HarnessDriverDescriptor, +} from "../contracts/harness-driver.js"; +import { validatePrpEvent, type PrpEvent } from "../protocol/replay-contract.js"; + +export const HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA = + "paperclip-runner/harness-driver-conformance-fixture/v1" as const; +export const HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA = + "paperclip-runner/harness-driver-conformance-report/v1" as const; + +export const harnessDriverConformanceFixtureUrl = new URL( + "../../protocol/fixtures/evals/harness-driver-conformance.json", + import.meta.url, +); + +export interface HarnessDriverConformanceFixture { + schema: typeof HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA; + validConfig: Record; + invalidConfig: Record; + completionMessage: string; + interruptMessage: string; + requiredCapabilities: string[]; + unsupportedFeatures: string[]; +} + +export interface HarnessDriverConformanceReport { + schema: typeof HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA; + contractVersion: typeof HARNESS_DRIVER_CONTRACT_VERSION; + descriptor: HarnessDriverDescriptor; + checks: { + capabilityDescription: true; + configValidation: true; + sessionLifecycle: true; + sessionRecovery: true; + semanticTools: true; + eventValidation: true; + interruptAndCancel: true; + usage: true; + transcriptCompleteness: true; + unsupportedFeatures: true; + }; + eventCount: number; + semanticToolCallCount: number; + usage: Record; +} + +function record(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + return value as Record; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${path} must be a non-empty string`); + } + return value; +} + +function strings(value: unknown, path: string): string[] { + if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) { + throw new Error(`${path} must be an array of strings`); + } + return [...value] as string[]; +} + +export function parseHarnessDriverConformanceFixture( + value: unknown, +): HarnessDriverConformanceFixture { + const fixture = record(value, "fixture"); + if (fixture.schema !== HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA) { + throw new Error( + `fixture.schema must be ${HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA}`, + ); + } + return { + schema: HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA, + validConfig: structuredClone(record(fixture.validConfig, "fixture.validConfig")), + invalidConfig: structuredClone(record(fixture.invalidConfig, "fixture.invalidConfig")), + completionMessage: text(fixture.completionMessage, "fixture.completionMessage"), + interruptMessage: text(fixture.interruptMessage, "fixture.interruptMessage"), + requiredCapabilities: strings(fixture.requiredCapabilities, "fixture.requiredCapabilities"), + unsupportedFeatures: strings(fixture.unsupportedFeatures, "fixture.unsupportedFeatures"), + }; +} + +export async function loadHarnessDriverConformanceFixture( + url: URL = harnessDriverConformanceFixtureUrl, +): Promise { + return parseHarnessDriverConformanceFixture( + JSON.parse(await readFile(url, "utf8")) as unknown, + ); +} + +async function collectEvents(events: AsyncIterable): Promise { + const collected: PrpEvent[] = []; + for await (const event of events) { + const validation = validatePrpEvent(event); + if (!validation.ok) { + throw new Error( + `driver conformance received invalid event: ${validation.issues[0]?.message ?? "unknown issue"}`, + ); + } + collected.push(structuredClone(validation.event)); + } + return collected; +} + +/** Run the deterministic, provider-free harness-driver V1 contract. */ +export async function runHarnessDriverConformance(input: { + driver: HarnessDriver; + fixture?: HarnessDriverConformanceFixture; +}): Promise { + const fixture = input.fixture ?? await loadHarnessDriverConformanceFixture(); + const descriptor = await input.driver.descriptor(); + if (!descriptor.kind || !descriptor.displayName || !descriptor.version) { + throw new Error("driver descriptor is missing stable identity fields"); + } + for (const capability of fixture.requiredCapabilities) { + if (descriptor.capabilities[capability as keyof typeof descriptor.capabilities] !== true) { + throw new Error(`driver capability ${capability} is required by conformance V1`); + } + } + for (const unsupported of fixture.unsupportedFeatures) { + if (!descriptor.capabilities.unsupported?.includes(unsupported)) { + throw new Error(`driver must explicitly describe unsupported feature ${unsupported}`); + } + } + + if (input.driver.validateConfig === undefined) { + throw new Error("driver does not expose deterministic config validation"); + } + const valid = await input.driver.validateConfig(fixture.validConfig); + const invalid = await input.driver.validateConfig(fixture.invalidConfig); + if (!valid.ok || invalid.ok || invalid.issues.length === 0) { + throw new Error("driver config validation did not accept valid and reject invalid input"); + } + + const completeSession = await input.driver.openSession({ + runId: "run_driver_conformance_complete", + normalizedSessionId: "session_driver_conformance_complete", + workingDirectory: "/deterministic/conformance", + }); + await completeSession.startTurn({ message: { role: "user", text: fixture.completionMessage } }); + const completeEvents = await collectEvents(completeSession.events()); + const semanticInputs = completeEvents.filter((event) => event.eventType === "mcp_app.tool_input"); + const semanticResults = completeEvents.filter((event) => event.eventType === "mcp_app.tool_result"); + if (semanticInputs.length === 0 || semanticInputs.length !== semanticResults.length) { + throw new Error("driver conformance requires paired semantic tool input/result events"); + } + const snapshot = await completeSession.snapshot(); + if (snapshot.semanticResult === undefined || snapshot.semanticResult === null) { + throw new Error("completed driver session did not persist a semantic result"); + } + const usage = await completeSession.usage?.(); + if (usage === undefined || usage === null) { + throw new Error("driver conformance requires a deterministic usage snapshot"); + } + const transcript = await completeSession.transcript?.(); + if ( + transcript === undefined + || !transcript.complete + || transcript.eventCount !== completeEvents.length + || transcript.events.length !== completeEvents.length + || transcript.omissionReason !== null + ) { + throw new Error("driver conformance transcript is incomplete or inconsistent"); + } + if (input.driver.recoverSession === undefined) { + throw new Error("driver declares resume but does not expose session recovery"); + } + const recovery = await input.driver.recoverSession(snapshot); + if ( + !recovery.recovered + || recovery.session === undefined + || recovery.session.ids().driverSessionId !== snapshot.driverSessionId + ) { + throw new Error("driver did not recover the persisted conformance session identity"); + } + const recoveredSnapshot = await recovery.session.snapshot(); + if ( + recoveredSnapshot.runId !== snapshot.runId + || recoveredSnapshot.normalizedSessionId !== snapshot.normalizedSessionId + || recoveredSnapshot.semanticResult?.fingerprint !== snapshot.semanticResult?.fingerprint + ) { + throw new Error("driver recovery did not preserve the persisted conformance state"); + } + await recovery.session.close({ reason: "conformance_recovery_complete" }); + await completeSession.close({ reason: "conformance_complete" }); + + const interruptedSession = await input.driver.openSession({ + runId: "run_driver_conformance_interrupt", + normalizedSessionId: "session_driver_conformance_interrupt", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await interruptedSession.startTurn({ + message: { role: "user", text: fixture.interruptMessage }, + }); + if (interruptedSession.interrupt === undefined) { + throw new Error("driver conformance requires interrupt support"); + } + await interruptedSession.interrupt({ turnId, reason: "conformance_interrupt" }); + const interruptedEvents = await collectEvents(interruptedSession.events()); + if ( + !interruptedEvents.some((event) => event.eventType === "turn.interrupted") + || !interruptedEvents.some((event) => + event.eventType === "run.terminal" + && record(event.payload, "terminal payload").runTerminalState === "cancelled") + ) { + throw new Error("driver interrupt did not produce an explicit cancelled terminal"); + } + await interruptedSession.close({ reason: "conformance_cancel", force: true }); + + return { + schema: HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA, + contractVersion: HARNESS_DRIVER_CONTRACT_VERSION, + descriptor: structuredClone(descriptor), + checks: { + capabilityDescription: true, + configValidation: true, + sessionLifecycle: true, + sessionRecovery: true, + semanticTools: true, + eventValidation: true, + interruptAndCancel: true, + usage: true, + transcriptCompleteness: true, + unsupportedFeatures: true, + }, + eventCount: completeEvents.length, + semanticToolCallCount: semanticInputs.length, + usage: structuredClone(usage), + }; +} diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index 14fc9c323f9..b7b8ee8d3fe 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -4,6 +4,26 @@ import type { } from "../protocol/replay-contract.js"; import type { NativeSessionCapabilities, NativeUserMessage } from "./types.js"; +export const HARNESS_DRIVER_CONTRACT_VERSION = 1 as const; + +export interface HarnessDriverConfigIssue { + path: string; + code: string; + message: string; +} + +export type HarnessDriverConfigValidation = + | { ok: true; config: Record; issues: [] } + | { ok: false; config: null; issues: HarnessDriverConfigIssue[] }; + +export interface HarnessTranscriptSnapshot { + schema: "paperclip-runner/harness-transcript/v1"; + complete: boolean; + eventCount: number; + events: PrpEvent[]; + omissionReason: string | null; +} + export interface HarnessDriverDescriptor { kind: string; displayName: string; @@ -322,6 +342,7 @@ export interface HarnessSession { read?(): Promise>; reconcile?(): Promise>; usage?(): Promise | null>; + transcript?(): Promise; snapshot(): Promise; close(input: { reason: string; force?: boolean }): Promise; } @@ -329,6 +350,7 @@ export interface HarnessSession { /** A local harness implementation hidden behind the runner daemon boundary. */ export interface HarnessDriver { descriptor(): Promise; + validateConfig?(config: unknown): Promise; openSession(input: OpenHarnessSessionInput): Promise; recoverSession?( snapshot: PersistedHarnessSession, diff --git a/packages/paperclip-runner/src/evals/build-metadata.ts b/packages/paperclip-runner/src/evals/build-metadata.ts new file mode 100644 index 00000000000..f19f334781b --- /dev/null +++ b/packages/paperclip-runner/src/evals/build-metadata.ts @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto"; + +import { PAPERCLIP_RUNNER_COMPATIBILITY } from "../compatibility.js"; +import { PRP_PROTOCOL_NAME, PRP_PROTOCOL_VERSION } from "../protocol/replay-contract.js"; +import { canonicalCapabilitySemanticCatalog } from "../semantic-tools/catalog.js"; + +export const PAPERCLIP_RUNNER_BUILD_METADATA_SCHEMA = + "paperclip-runner/build-metadata/v1" as const; +export const PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA = + "paperclip-runner/native-execution/v1" as const; +export const PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA = + "paperclip-runner/evals-integration/v1" as const; +export const PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA = + "paperclip-runner/runnerd-build-metadata/v1" as const; + +export const PAPERCLIP_RUNNER_SEMANTIC_CATALOG_SHA256 = + `sha256:${createHash("sha256") + .update(canonicalCapabilitySemanticCatalog()) + .digest("hex")}` as const; + +/** + * App-owned release metadata that Evals pins beside every native attempt. + * Contract versions are independent from package semver so consumers can give + * a precise mismatch instead of guessing from a package version. + */ +export const PAPERCLIP_RUNNER_BUILD_METADATA = Object.freeze({ + schema: PAPERCLIP_RUNNER_BUILD_METADATA_SCHEMA, + package: Object.freeze({ + name: PAPERCLIP_RUNNER_COMPATIBILITY.packageName, + version: PAPERCLIP_RUNNER_COMPATIBILITY.packageVersion, + }), + contracts: Object.freeze({ + evalIntegration: PAPERCLIP_RUNNER_COMPATIBILITY.components.evalIntegration, + nativeExecution: PAPERCLIP_RUNNER_COMPATIBILITY.components.nativeExecution, + runnerdArtifact: PAPERCLIP_RUNNER_COMPATIBILITY.components.runnerdBinary, + prp: PRP_PROTOCOL_VERSION, + semanticCatalog: PAPERCLIP_RUNNER_COMPATIBILITY.components.catalog, + harnessDriver: PAPERCLIP_RUNNER_COMPATIBILITY.components.harnessDriver, + controlPlaneAdapter: PAPERCLIP_RUNNER_COMPATIBILITY.components.controlPlaneAdapter, + testkit: PAPERCLIP_RUNNER_COMPATIBILITY.components.testkit, + }), + prp: Object.freeze({ + name: PRP_PROTOCOL_NAME, + minimumVersion: PRP_PROTOCOL_VERSION, + maximumVersion: PRP_PROTOCOL_VERSION, + }), + semanticCatalog: Object.freeze({ + version: PAPERCLIP_RUNNER_COMPATIBILITY.components.catalog, + sha256: PAPERCLIP_RUNNER_SEMANTIC_CATALOG_SHA256, + }), + runnerd: Object.freeze({ + binaryName: "paperclip-runnerd" as const, + metadataSchema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + digestAlgorithm: "sha256" as const, + }), +}); + +export type PaperclipRunnerBuildMetadata = typeof PAPERCLIP_RUNNER_BUILD_METADATA; diff --git a/packages/paperclip-runner/src/evals/compatibility.test.ts b/packages/paperclip-runner/src/evals/compatibility.test.ts new file mode 100644 index 00000000000..d4ce823ffec --- /dev/null +++ b/packages/paperclip-runner/src/evals/compatibility.test.ts @@ -0,0 +1,120 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +import { + PAPERCLIP_RUNNER_BUILD_METADATA, + PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, +} from "./build-metadata.js"; +import { + PaperclipRunnerEvalCompatibilityError, + assertPaperclipRunnerEvalCompatibility, + type PaperclipRunnerEvalCompatibilityRequirement, +} from "./compatibility.js"; + +function compatible(): PaperclipRunnerEvalCompatibilityRequirement { + return { + consumer: "paperclip-evals", + packageVersion: PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + runnerd: { + schema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", + packageVersion: PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + binaryContractVersion: 1, + nativeExecutionVersion: 1, + harnessDriverVersion: 1, + prp: { name: "paperclip.runner", minimumVersion: 1, maximumVersion: 1 }, + }, + nativeExecutionVersion: 1, + prp: { minimumVersion: 1, maximumVersion: 1 }, + catalog: { + version: PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.version, + sha256: PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.sha256, + }, + driver: { + contractVersion: 1, + requiredCapabilities: ["typedEvents", "interruption", "usage", "dynamicTools"], + descriptor: { + kind: "paperclip-deterministic", + displayName: "Deterministic", + version: "1.0.0", + protocolVersion: "prp.v1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + usage: true, + dynamicTools: true, + }, + }, + }, + }; +} + +describe("Paperclip Evals integration compatibility", () => { + it("keeps build metadata synchronized with package semver", () => { + const packageJson = JSON.parse(readFileSync( + fileURLToPath(new URL("../../package.json", import.meta.url)), + "utf8", + )); + expect(PAPERCLIP_RUNNER_BUILD_METADATA.package.version).toBe(packageJson.version); + }); + + it("negotiates package, binary, PRP, catalog, and driver V1", () => { + expect(assertPaperclipRunnerEvalCompatibility(compatible())).toMatchObject({ + schema: "paperclip-runner/evals-integration/v1", + consumer: "paperclip-evals", + negotiatedPrpVersion: 1, + driverKind: "paperclip-deterministic", + }); + }); + + it("fails closed with actionable codes for every independently versioned input", () => { + const requirement = compatible(); + requirement.packageVersion = "9.0.0"; + requirement.runnerd.packageVersion = "8.0.0"; + requirement.runnerd.binaryContractVersion = 2; + requirement.runnerd.nativeExecutionVersion = 2; + requirement.runnerd.harnessDriverVersion = 2; + requirement.nativeExecutionVersion = 2; + requirement.prp = { minimumVersion: 2, maximumVersion: 2 }; + requirement.runnerd.prp = { name: "paperclip.runner", minimumVersion: 2, maximumVersion: 2 }; + requirement.catalog = { version: 2, sha256: `sha256:${"0".repeat(64)}` }; + requirement.driver.contractVersion = 2; + requirement.driver.descriptor.capabilities.dynamicTools = false; + + try { + assertPaperclipRunnerEvalCompatibility(requirement); + throw new Error("expected compatibility failure"); + } catch (error) { + expect(error).toBeInstanceOf(PaperclipRunnerEvalCompatibilityError); + expect((error as PaperclipRunnerEvalCompatibilityError).issues.map((item) => item.code)) + .toEqual([ + "package_version_mismatch", + "binary_package_version_mismatch", + "binary_contract_version_mismatch", + "native_execution_version_mismatch", + "prp_version_no_overlap", + "catalog_version_mismatch", + "catalog_digest_mismatch", + "driver_contract_version_mismatch", + "driver_capability_unsupported", + ]); + expect((error as Error).message).toContain("runnerd was built for package 8.0.0"); + } + }); + + it("rejects a driver that does not speak the negotiated PRP version", () => { + const requirement = compatible(); + requirement.driver.descriptor.protocolVersion = "prp.v2"; + + expect(() => assertPaperclipRunnerEvalCompatibility(requirement)).toThrow( + expect.objectContaining({ + issues: [expect.objectContaining({ code: "driver_protocol_version_mismatch" })], + }), + ); + }); +}); diff --git a/packages/paperclip-runner/src/evals/compatibility.ts b/packages/paperclip-runner/src/evals/compatibility.ts new file mode 100644 index 00000000000..2e62540e7e4 --- /dev/null +++ b/packages/paperclip-runner/src/evals/compatibility.ts @@ -0,0 +1,244 @@ +import type { HarnessDriverDescriptor } from "../contracts/harness-driver.js"; +import type { NativeSessionCapabilities } from "../contracts/types.js"; +import { negotiateProtocolVersion } from "../protocol/replay-contract.js"; +import { + PAPERCLIP_RUNNER_BUILD_METADATA, + PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA, +} from "./build-metadata.js"; +import type { PaperclipRunnerdBuildMetadata } from "./runnerd-artifact.js"; + +export type RequiredHarnessDriverCapability = Exclude< + keyof NativeSessionCapabilities, + "unsupported" +>; + +export type PaperclipRunnerEvalCompatibilityIssueCode = + | "package_version_mismatch" + | "binary_package_version_mismatch" + | "binary_contract_version_mismatch" + | "native_execution_version_mismatch" + | "prp_name_mismatch" + | "prp_version_no_overlap" + | "catalog_version_mismatch" + | "catalog_digest_mismatch" + | "driver_contract_version_mismatch" + | "driver_protocol_version_mismatch" + | "driver_capability_unsupported"; + +export interface PaperclipRunnerEvalCompatibilityIssue { + code: PaperclipRunnerEvalCompatibilityIssueCode; + component: "package" | "binary" | "nativeExecution" | "prp" | "catalog" | "driver"; + expected: string; + received: string; + message: string; +} + +export interface PaperclipRunnerEvalCompatibilityRequirement { + consumer: string; + packageVersion: string; + runnerd: PaperclipRunnerdBuildMetadata; + nativeExecutionVersion: number; + prp: { minimumVersion: number; maximumVersion: number }; + catalog: { version: number; sha256: string }; + driver: { + contractVersion: number; + descriptor: HarnessDriverDescriptor; + requiredCapabilities: readonly RequiredHarnessDriverCapability[]; + }; +} + +export interface PaperclipRunnerEvalCompatibilityReceipt { + schema: typeof PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA; + consumer: string; + packageVersion: string; + runnerdPackageVersion: string; + negotiatedPrpVersion: number; + catalogSha256: string; + driverKind: string; + driverVersion: string; +} + +export class PaperclipRunnerEvalCompatibilityError extends Error { + readonly code = "paperclip_runner_eval_incompatible" as const; + + constructor( + readonly consumer: string, + readonly issues: readonly PaperclipRunnerEvalCompatibilityIssue[], + ) { + super( + `Paperclip runner eval compatibility check failed for ${consumer}: ${issues + .map((issue) => `${issue.code}: ${issue.message}`) + .join("; ")}`, + ); + this.name = "PaperclipRunnerEvalCompatibilityError"; + this.issues = Object.freeze(issues.map((issue) => Object.freeze({ ...issue }))); + } +} + +/** + * Negotiate the complete App/Evals join before starting a provider process. + * Every independently versioned input is checked and all mismatches are + * returned together so remediation is actionable. + */ +export function assertPaperclipRunnerEvalCompatibility( + requirement: PaperclipRunnerEvalCompatibilityRequirement, +): PaperclipRunnerEvalCompatibilityReceipt { + const expected = PAPERCLIP_RUNNER_BUILD_METADATA; + const issues: PaperclipRunnerEvalCompatibilityIssue[] = []; + const issue = ( + code: PaperclipRunnerEvalCompatibilityIssueCode, + component: PaperclipRunnerEvalCompatibilityIssue["component"], + expectedValue: string | number, + receivedValue: string | number, + message: string, + ): void => { + issues.push({ + code, + component, + expected: String(expectedValue), + received: String(receivedValue), + message, + }); + }; + + if (requirement.packageVersion !== expected.package.version) { + issue( + "package_version_mismatch", + "package", + expected.package.version, + requirement.packageVersion, + `package ${requirement.packageVersion} does not match the loaded contract package ${expected.package.version}`, + ); + } + if (requirement.runnerd.packageVersion !== expected.package.version) { + issue( + "binary_package_version_mismatch", + "binary", + expected.package.version, + requirement.runnerd.packageVersion, + `runnerd was built for package ${requirement.runnerd.packageVersion}, not ${expected.package.version}`, + ); + } + if (requirement.runnerd.binaryContractVersion !== expected.contracts.runnerdArtifact) { + issue( + "binary_contract_version_mismatch", + "binary", + expected.contracts.runnerdArtifact, + requirement.runnerd.binaryContractVersion, + "runnerd artifact contract is incompatible with this package", + ); + } + if ( + requirement.nativeExecutionVersion !== expected.contracts.nativeExecution + || requirement.runnerd.nativeExecutionVersion !== expected.contracts.nativeExecution + ) { + issue( + "native_execution_version_mismatch", + "nativeExecution", + expected.contracts.nativeExecution, + `${requirement.nativeExecutionVersion}/${requirement.runnerd.nativeExecutionVersion}`, + "consumer and runnerd must both emit paperclip-runner/native-execution/v1", + ); + } + + if (requirement.runnerd.prp.name !== expected.prp.name) { + issue( + "prp_name_mismatch", + "prp", + expected.prp.name, + requirement.runnerd.prp.name, + "runnerd speaks a different protocol family", + ); + } + const packageAndConsumerPrp = negotiateProtocolVersion( + { min: expected.prp.minimumVersion, max: expected.prp.maximumVersion }, + { min: requirement.prp.minimumVersion, max: requirement.prp.maximumVersion }, + ); + const negotiatedPrpVersion = packageAndConsumerPrp === null + ? null + : negotiateProtocolVersion( + { min: packageAndConsumerPrp, max: packageAndConsumerPrp }, + { + min: requirement.runnerd.prp.minimumVersion, + max: requirement.runnerd.prp.maximumVersion, + }, + ); + if (negotiatedPrpVersion === null) { + issue( + "prp_version_no_overlap", + "prp", + `${expected.prp.minimumVersion}..${expected.prp.maximumVersion}`, + `consumer ${requirement.prp.minimumVersion}..${requirement.prp.maximumVersion}; runnerd ${requirement.runnerd.prp.minimumVersion}..${requirement.runnerd.prp.maximumVersion}`, + "package, runnerd, and consumer have no common PRP version", + ); + } + + if (requirement.catalog.version !== expected.semanticCatalog.version) { + issue( + "catalog_version_mismatch", + "catalog", + expected.semanticCatalog.version, + requirement.catalog.version, + "semantic catalog contract version is incompatible", + ); + } + if (requirement.catalog.sha256 !== expected.semanticCatalog.sha256) { + issue( + "catalog_digest_mismatch", + "catalog", + expected.semanticCatalog.sha256, + requirement.catalog.sha256, + "semantic catalog content digest does not match the loaded package", + ); + } + if ( + requirement.driver.contractVersion !== expected.contracts.harnessDriver + || requirement.runnerd.harnessDriverVersion !== expected.contracts.harnessDriver + ) { + issue( + "driver_contract_version_mismatch", + "driver", + expected.contracts.harnessDriver, + `${requirement.driver.contractVersion}/${requirement.runnerd.harnessDriverVersion}`, + "consumer and runnerd harness-driver contract versions must match the package", + ); + } + if ( + negotiatedPrpVersion !== null + && requirement.driver.descriptor.protocolVersion !== `prp.v${negotiatedPrpVersion}` + ) { + issue( + "driver_protocol_version_mismatch", + "driver", + `prp.v${negotiatedPrpVersion}`, + String(requirement.driver.descriptor.protocolVersion), + `driver ${requirement.driver.descriptor.kind}@${requirement.driver.descriptor.version} does not speak the negotiated PRP version`, + ); + } + for (const capability of [...new Set(requirement.driver.requiredCapabilities)].sort()) { + if (requirement.driver.descriptor.capabilities[capability] !== true) { + issue( + "driver_capability_unsupported", + "driver", + `${capability}=true`, + `${capability}=${String(requirement.driver.descriptor.capabilities[capability])}`, + `driver ${requirement.driver.descriptor.kind}@${requirement.driver.descriptor.version} does not support required capability ${capability}`, + ); + } + } + + if (issues.length > 0) { + throw new PaperclipRunnerEvalCompatibilityError(requirement.consumer, issues); + } + + return { + schema: PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA, + consumer: requirement.consumer, + packageVersion: expected.package.version, + runnerdPackageVersion: requirement.runnerd.packageVersion, + negotiatedPrpVersion: negotiatedPrpVersion!, + catalogSha256: expected.semanticCatalog.sha256, + driverKind: requirement.driver.descriptor.kind, + driverVersion: requirement.driver.descriptor.version, + }; +} diff --git a/packages/paperclip-runner/src/evals/index.ts b/packages/paperclip-runner/src/evals/index.ts new file mode 100644 index 00000000000..5c2b5457b3c --- /dev/null +++ b/packages/paperclip-runner/src/evals/index.ts @@ -0,0 +1,32 @@ +/** Stable App-owned integration surface for Paperclip Evals. */ +export * from "./build-metadata.js"; +export * from "./compatibility.js"; +export * from "./native-execution.js"; +export * from "./runnerd-artifact.js"; + +export { + PRP_PROTOCOL_NAME, + PRP_PROTOCOL_VERSION, + negotiateProtocolVersion, + validatePrpEvent, + validatePrpFixture, + validatePrpStructuredRunResult, + type PrpCapabilities, + type PrpEvent, + type PrpSemanticToolEnvelope, + type PrpStructuredRunResult, + type PrpTerminalState, +} from "../protocol/replay-contract.js"; +export * from "../protocol/semantic-tool-receipts.js"; +export { + CAPABILITY_SEMANTIC_TOOL_CATALOG, + canonicalCapabilitySemanticCatalog, + capabilitySemanticToolDescriptor, +} from "../semantic-tools/catalog.js"; +export type { + HarnessDriver, + HarnessDriverDescriptor, + HarnessSession, + OpenHarnessSessionInput, + PersistedHarnessSession, +} from "../contracts/harness-driver.js"; diff --git a/packages/paperclip-runner/src/evals/native-execution.test.ts b/packages/paperclip-runner/src/evals/native-execution.test.ts new file mode 100644 index 00000000000..257ee247def --- /dev/null +++ b/packages/paperclip-runner/src/evals/native-execution.test.ts @@ -0,0 +1,132 @@ +import { readFile } from "node:fs/promises"; +import Ajv2020 from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; + +import { prpSchemaBundle } from "../protocol/generated/schema-bundle.js"; +import { + loadPaperclipNativeExecutionFixture, + paperclipNativeExecutionFixtureUrl, + paperclipNativeExecutionSchemaUrl, + parsePaperclipNativeExecution, +} from "./native-execution.js"; +import { PAPERCLIP_RUNNER_BUILD_METADATA } from "./build-metadata.js"; + +describe("paperclip-runner/native-execution/v1", () => { + it("keeps the shipped seeded fixture valid against the published JSON Schema", async () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }); + for (const schema of Object.values(prpSchemaBundle)) ajv.addSchema(schema); + const schema = JSON.parse( + await readFile(paperclipNativeExecutionSchemaUrl, "utf8"), + ) as Record; + const validate = ajv.compile(schema); + const fixture = JSON.parse( + await readFile(paperclipNativeExecutionFixtureUrl, "utf8"), + ) as unknown; + + expect(validate(fixture), JSON.stringify(validate.errors)).toBe(true); + }); + + it("loads the shipped seeded attempt and preserves nested additive fields", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(bundle.provenance).toBe("seeded"); + expect(bundle.semanticTools.results).toEqual([ + expect.objectContaining({ outcome: "denied", callId: "call_evals_seeded" }), + ]); + expect(bundle.semanticTools.denials).toHaveLength(1); + expect(bundle.transcript).toMatchObject({ complete: true, eventCount: 4 }); + expect(bundle.usage.cost).toEqual({ currency: "USD", amountMicros: 0 }); + expect(bundle.runner.package).toEqual(PAPERCLIP_RUNNER_BUILD_METADATA.package); + expect(bundle.runner.catalogSha256).toBe( + PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.sha256, + ); + expect(bundle.x_fixturePurpose).toContain("additive field preservation"); + + const extended = structuredClone(bundle); + extended.identity.x_identity = "identity-extension"; + extended.input.x_input = "input-extension"; + extended.runner.x_runner = "runner-extension"; + extended.runner.package.x_package = "package-extension"; + extended.runner.binary.x_binary = "binary-extension"; + extended.semanticTools.x_semanticTools = "semantic-extension"; + extended.usage.x_usage = "usage-extension"; + extended.usage.cost!.x_cost = "cost-extension"; + + expect(parsePaperclipNativeExecution(extended)).toMatchObject({ + identity: { x_identity: "identity-extension" }, + input: { x_input: "input-extension" }, + runner: { + x_runner: "runner-extension", + package: { x_package: "package-extension" }, + binary: { x_binary: "binary-extension" }, + }, + semanticTools: { x_semanticTools: "semantic-extension" }, + usage: { x_usage: "usage-extension", cost: { x_cost: "cost-extension" } }, + }); + }); + + it("rejects unknown native schema versions", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(() => parsePaperclipNativeExecution({ ...bundle, schema: "paperclip-runner/native-execution/v2" })) + .toThrow(/unsupported schema/); + }); + + it("rejects a terminal that conflicts with the event stream", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(() => parsePaperclipNativeExecution({ + ...bundle, + terminal: { ...bundle.terminal, runTerminalState: "failed" }, + })).toThrow(/must equal the run\.terminal event payload/); + }); + + it("rejects missing denial receipts and incomplete transcript ambiguity", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(() => parsePaperclipNativeExecution({ + ...bundle, + semanticTools: { ...bundle.semanticTools, denials: [] }, + })).toThrow(/needs a denial receipt/); + expect(() => parsePaperclipNativeExecution({ + ...bundle, + transcript: { ...bundle.transcript, complete: false, omissionReason: null }, + })).toThrow(/omissionReason.*required/); + }); + + it("rejects semantic indexes that reinterpret or omit PRP tool events", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + const mismatched = structuredClone(bundle); + const resultPayload = mismatched.events[1]!.payload as { + semantic_tool: { outcome: string }; + }; + resultPayload.semantic_tool.outcome = "succeeded"; + expect(() => parsePaperclipNativeExecution(mismatched)) + .toThrow(/does not match semantic envelope/); + + const mismatchedCorrelation = structuredClone(bundle); + const inputPayload = mismatchedCorrelation.events[0]!.payload as { + semantic_tool: { correlation: { runId: string } }; + }; + inputPayload.semantic_tool.correlation.runId = "other_run"; + expect(() => parsePaperclipNativeExecution(mismatchedCorrelation)) + .toThrow(/correlation\.runId.*enclosing event/); + + expect(() => parsePaperclipNativeExecution({ + ...bundle, + semanticTools: { ...bundle.semanticTools, calls: [] }, + })).toThrow(/no matching call|tool-input event .* is not indexed/); + }); + + it("rejects unordered native events", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + const events = structuredClone(bundle.events); + events[1] = { ...events[1]!, sourceSeq: 1 }; + expect(() => parsePaperclipNativeExecution({ ...bundle, events })) + .toThrow(/must be ordered after 1/); + + const duplicateIds = structuredClone(bundle.events); + duplicateIds[1] = { + ...duplicateIds[1]!, + sourceEventId: duplicateIds[0]!.sourceEventId, + }; + expect(() => parsePaperclipNativeExecution({ ...bundle, events: duplicateIds })) + .toThrow(/sourceEventId.*unique/); + }); +}); diff --git a/packages/paperclip-runner/src/evals/native-execution.ts b/packages/paperclip-runner/src/evals/native-execution.ts new file mode 100644 index 00000000000..9252a99ac57 --- /dev/null +++ b/packages/paperclip-runner/src/evals/native-execution.ts @@ -0,0 +1,538 @@ +import { readFile } from "node:fs/promises"; + +import { + validatePrpEvent, + type PrpEvent, + type PrpTerminalState, +} from "../protocol/replay-contract.js"; +import { PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA } from "./build-metadata.js"; + +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; +const TOOL_OUTCOMES = new Set([ + "succeeded", + "denied", + "conflict", + "duplicate", + "unavailable", + "failed", +]); + +export const paperclipNativeExecutionSchemaUrl = new URL( + "../../protocol/schemas/native-execution.schema.json", + import.meta.url, +); +export const paperclipNativeExecutionFixtureUrl = new URL( + "../../protocol/fixtures/evals/native-execution-seeded.json", + import.meta.url, +); + +export interface NativeExecutionContentRef { + uri: string; + sha256: string; + byteSize: number; + mediaType: string; + [key: string]: unknown; +} + +export interface PaperclipNativeExecutionV1 { + schema: typeof PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA; + provenance: "seeded" | "replay" | "live"; + identity: { + runId: string; + caseId: string; + configId: string; + attemptId: string; + [key: string]: unknown; + }; + input: { + caseSha256: string; + configSha256: string; + deterministicSeed: string; + [key: string]: unknown; + }; + runner: { + package: { name: string; version: string; [key: string]: unknown }; + binary: { name: string; sha256: string; [key: string]: unknown }; + prpVersion: number; + nativeExecutionVersion: number; + catalogVersion: number; + catalogSha256: string; + driverContractVersion: number; + driverKind: string; + driverVersion: string; + [key: string]: unknown; + }; + events: PrpEvent[]; + semanticTools: { + definitions: Array>; + calls: Array<{ callId: string; operationId: string; eventId: string; [key: string]: unknown }>; + results: Array<{ + callId: string; + operationId: string; + eventId: string; + outcome: string; + [key: string]: unknown; + }>; + denials: Array<{ + callId: string; + code: string; + authorizationBoundary: string; + retryable: boolean; + [key: string]: unknown; + }>; + [key: string]: unknown; + }; + terminal: PrpTerminalState; + observations: Record; + usage: { + inputTokens: number; + outputTokens: number; + totalTokens: number; + requestCount: number; + durationMs: number; + cost: { currency: string; amountMicros: number; [key: string]: unknown } | null; + [key: string]: unknown; + }; + transcript: NativeExecutionContentRef & { + complete: boolean; + eventCount: number; + omissionReason: string | null; + }; + artifactRoot: NativeExecutionContentRef; + artifacts: NativeExecutionContentRef[]; + failure: { + classification: string; + code: string; + message: string; + [key: string]: unknown; + } | null; + [key: string]: unknown; +} + +export class PaperclipNativeExecutionError extends Error { + readonly code = "paperclip_native_execution_invalid" as const; + + constructor( + message: string, + readonly path: string, + ) { + super(`${path}: ${message}`); + this.name = "PaperclipNativeExecutionError"; + } +} + +function fail(path: string, message: string): never { + throw new PaperclipNativeExecutionError(message, path); +} + +function record(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail(path, "must be an object"); + } + return value as Record; +} + +function array(value: unknown, path: string): unknown[] { + if (!Array.isArray(value)) fail(path, "must be an array"); + return value; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + fail(path, "must be a non-empty string"); + } + return value; +} + +function integer(value: unknown, path: string, minimum = 0): number { + if (!Number.isSafeInteger(value) || (value as number) < minimum) { + fail(path, `must be an integer >= ${minimum}`); + } + return value as number; +} + +function boolean(value: unknown, path: string): boolean { + if (typeof value !== "boolean") fail(path, "must be a boolean"); + return value; +} + +function digest(value: unknown, path: string): string { + const parsed = text(value, path); + if (!SHA256_PATTERN.test(parsed)) fail(path, "must use sha256:<64 lowercase hex>"); + return parsed; +} + +function contentRef(value: unknown, path: string): NativeExecutionContentRef { + const ref = record(value, path); + return { + ...structuredClone(ref), + uri: text(ref.uri, `${path}.uri`), + sha256: digest(ref.sha256, `${path}.sha256`), + byteSize: integer(ref.byteSize, `${path}.byteSize`), + mediaType: text(ref.mediaType, `${path}.mediaType`), + } as NativeExecutionContentRef; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const object = value as Record; + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function semanticEntry( + value: unknown, + path: string, + fields: readonly string[], +): Record { + const entry = record(value, path); + for (const field of fields) text(entry[field], `${path}.${field}`); + return structuredClone(entry); +} + +/** + * Validate the App-native attempt bundle while preserving unknown additive + * fields. Required identities, digests, event bindings, terminal consistency, + * semantic-tool outcomes, usage, and transcript completeness fail closed. + */ +export function parsePaperclipNativeExecution( + value: unknown, +): PaperclipNativeExecutionV1 { + const bundle = record(value, "bundle"); + if (bundle.schema !== PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA) { + fail( + "bundle.schema", + `unsupported schema ${String(bundle.schema)}; expected ${PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA}`, + ); + } + if (bundle.provenance !== "seeded" && bundle.provenance !== "replay" && bundle.provenance !== "live") { + fail("bundle.provenance", "must be seeded, replay, or live"); + } + + const identity = record(bundle.identity, "bundle.identity"); + const parsedIdentity = { + ...structuredClone(identity), + runId: text(identity.runId, "bundle.identity.runId"), + caseId: text(identity.caseId, "bundle.identity.caseId"), + configId: text(identity.configId, "bundle.identity.configId"), + attemptId: text(identity.attemptId, "bundle.identity.attemptId"), + }; + const input = record(bundle.input, "bundle.input"); + const parsedInput = { + ...structuredClone(input), + caseSha256: digest(input.caseSha256, "bundle.input.caseSha256"), + configSha256: digest(input.configSha256, "bundle.input.configSha256"), + deterministicSeed: text(input.deterministicSeed, "bundle.input.deterministicSeed"), + }; + + const runner = record(bundle.runner, "bundle.runner"); + const runnerPackage = record(runner.package, "bundle.runner.package"); + const runnerBinary = record(runner.binary, "bundle.runner.binary"); + const parsedRunner = { + ...structuredClone(runner), + package: { + ...structuredClone(runnerPackage), + name: text(runnerPackage.name, "bundle.runner.package.name"), + version: text(runnerPackage.version, "bundle.runner.package.version"), + }, + binary: { + ...structuredClone(runnerBinary), + name: text(runnerBinary.name, "bundle.runner.binary.name"), + sha256: digest(runnerBinary.sha256, "bundle.runner.binary.sha256"), + }, + prpVersion: integer(runner.prpVersion, "bundle.runner.prpVersion", 1), + nativeExecutionVersion: integer( + runner.nativeExecutionVersion, + "bundle.runner.nativeExecutionVersion", + 1, + ), + catalogVersion: integer(runner.catalogVersion, "bundle.runner.catalogVersion", 1), + catalogSha256: digest(runner.catalogSha256, "bundle.runner.catalogSha256"), + driverContractVersion: integer( + runner.driverContractVersion, + "bundle.runner.driverContractVersion", + 1, + ), + driverKind: text(runner.driverKind, "bundle.runner.driverKind"), + driverVersion: text(runner.driverVersion, "bundle.runner.driverVersion"), + }; + + const sourceCursors = new Map(); + const sourceEventIds = new Set(); + const events = array(bundle.events, "bundle.events").map((event, index) => { + const validation = validatePrpEvent(event); + if (!validation.ok) { + fail( + `bundle.events[${index}]${validation.issues[0]?.path ?? ""}`, + validation.issues[0]?.message ?? "invalid PRP event", + ); + } + if (validation.event.runId !== parsedIdentity.runId) { + fail(`bundle.events[${index}].runId`, "must match bundle.identity.runId"); + } + if (sourceEventIds.has(validation.event.sourceEventId)) { + fail(`bundle.events[${index}].sourceEventId`, "must be unique within the bundle"); + } + sourceEventIds.add(validation.event.sourceEventId); + const previousSourceSeq = sourceCursors.get(validation.event.sourceInstanceId) ?? 0; + if (validation.event.sourceSeq <= previousSourceSeq) { + fail( + `bundle.events[${index}].sourceSeq`, + `must be ordered after ${previousSourceSeq} for source ${validation.event.sourceInstanceId}`, + ); + } + sourceCursors.set(validation.event.sourceInstanceId, validation.event.sourceSeq); + return structuredClone(validation.event); + }); + if (events.length === 0) fail("bundle.events", "must contain native events"); + + const terminal = record(bundle.terminal, "bundle.terminal") as unknown as PrpTerminalState; + if (terminal.schema !== "paperclip.prp.terminal.v1") { + fail("bundle.terminal.schema", "must be paperclip.prp.terminal.v1"); + } + const terminalEvents = events.filter((event) => event.eventType === "run.terminal"); + if (terminalEvents.length !== 1) { + fail("bundle.events", "must contain exactly one run.terminal event"); + } + if (canonicalJson(terminalEvents[0]?.payload) !== canonicalJson(terminal)) { + fail("bundle.terminal", "must equal the run.terminal event payload"); + } + + const semanticTools = record(bundle.semanticTools, "bundle.semanticTools"); + const definitions = array(semanticTools.definitions, "bundle.semanticTools.definitions") + .map((entry, index) => semanticEntry(entry, `bundle.semanticTools.definitions[${index}]`, ["operationId"])); + const calls = array(semanticTools.calls, "bundle.semanticTools.calls").map((entry, index) => { + const parsed = semanticEntry(entry, `bundle.semanticTools.calls[${index}]`, [ + "callId", + "operationId", + "eventId", + ]); + return parsed as PaperclipNativeExecutionV1["semanticTools"]["calls"][number]; + }); + const results = array(semanticTools.results, "bundle.semanticTools.results").map((entry, index) => { + const path = `bundle.semanticTools.results[${index}]`; + const parsed = semanticEntry(entry, path, ["callId", "operationId", "eventId", "outcome"]); + if (!TOOL_OUTCOMES.has(parsed.outcome as string)) fail(`${path}.outcome`, "is unsupported"); + return parsed as PaperclipNativeExecutionV1["semanticTools"]["results"][number]; + }); + const denials = array(semanticTools.denials, "bundle.semanticTools.denials").map((entry, index) => { + const path = `bundle.semanticTools.denials[${index}]`; + const parsed = semanticEntry(entry, path, ["callId", "code", "authorizationBoundary"]); + parsed.retryable = boolean(parsed.retryable, `${path}.retryable`); + return parsed as PaperclipNativeExecutionV1["semanticTools"]["denials"][number]; + }); + const callById = new Map(calls.map((call) => [call.callId, call])); + if (callById.size !== calls.length) { + fail("bundle.semanticTools.calls", "callId values must be unique"); + } + const resultByCallId = new Map(results.map((result) => [result.callId, result])); + if (resultByCallId.size !== results.length) { + fail("bundle.semanticTools.results", "callId values must be unique"); + } + const eventById = new Map(events.map((event) => [event.sourceEventId, event])); + const referencedCallEventIds = new Set(); + for (const call of calls) { + const event = eventById.get(call.eventId); + if (event?.eventType !== "mcp_app.tool_input") { + fail("bundle.semanticTools.calls", `call ${call.callId} does not reference a tool-input event`); + } + const envelope = semanticToolEnvelope(event, "input", `event ${call.eventId}`); + if (envelope.callId !== call.callId || envelope.operationId !== call.operationId) { + fail( + "bundle.semanticTools.calls", + `call ${call.callId} does not match semantic envelope ${String(envelope.callId)}/${String(envelope.operationId)}`, + ); + } + if (referencedCallEventIds.has(call.eventId)) { + fail("bundle.semanticTools.calls", `event ${call.eventId} is referenced more than once`); + } + referencedCallEventIds.add(call.eventId); + } + const referencedResultEventIds = new Set(); + for (const result of results) { + const call = callById.get(result.callId); + if (call === undefined || call.operationId !== result.operationId) { + fail( + "bundle.semanticTools.results", + `result ${result.callId} has no matching call with operation ${result.operationId}`, + ); + } + const event = eventById.get(result.eventId); + if (event?.eventType !== "mcp_app.tool_result") { + fail("bundle.semanticTools.results", `result ${result.callId} does not reference a tool-result event`); + } + const envelope = semanticToolEnvelope(event, "result", `event ${result.eventId}`); + if ( + envelope.callId !== result.callId + || envelope.operationId !== result.operationId + || envelope.outcome !== result.outcome + ) { + fail( + "bundle.semanticTools.results", + `result ${result.callId} does not match semantic envelope ${String(envelope.callId)}/${String(envelope.operationId)}/${String(envelope.outcome)}`, + ); + } + if (referencedResultEventIds.has(result.eventId)) { + fail("bundle.semanticTools.results", `event ${result.eventId} is referenced more than once`); + } + referencedResultEventIds.add(result.eventId); + if (result.outcome === "denied" && !denials.some((denial) => denial.callId === result.callId)) { + fail("bundle.semanticTools.denials", `denied result ${result.callId} needs a denial receipt`); + } + } + for (const event of events) { + if (event.eventType === "mcp_app.tool_input" && !referencedCallEventIds.has(event.sourceEventId)) { + fail("bundle.semanticTools.calls", `tool-input event ${event.sourceEventId} is not indexed`); + } + if (event.eventType === "mcp_app.tool_result" && !referencedResultEventIds.has(event.sourceEventId)) { + fail("bundle.semanticTools.results", `tool-result event ${event.sourceEventId} is not indexed`); + } + } + const denialCallIds = new Set(); + for (const denial of denials) { + if (denialCallIds.has(denial.callId)) { + fail("bundle.semanticTools.denials", `call ${denial.callId} has duplicate denial receipts`); + } + denialCallIds.add(denial.callId); + const result = resultByCallId.get(denial.callId); + if (result?.outcome !== "denied") { + fail("bundle.semanticTools.denials", `denial ${denial.callId} has no denied result`); + } + const resultEvent = eventById.get(result.eventId)!; + const envelope = semanticToolEnvelope( + resultEvent, + "result", + `event ${result.eventId}`, + ); + if ( + envelope.code !== denial.code + || envelope.authorizationBoundary !== denial.authorizationBoundary + || envelope.retryable !== denial.retryable + ) { + fail( + "bundle.semanticTools.denials", + `denial ${denial.callId} does not match its semantic result receipt`, + ); + } + } + + const usage = record(bundle.usage, "bundle.usage"); + const inputTokens = integer(usage.inputTokens, "bundle.usage.inputTokens"); + const outputTokens = integer(usage.outputTokens, "bundle.usage.outputTokens"); + const totalTokens = integer(usage.totalTokens, "bundle.usage.totalTokens"); + if (totalTokens !== inputTokens + outputTokens) { + fail("bundle.usage.totalTokens", "must equal inputTokens + outputTokens"); + } + const costRecord = usage.cost === null ? null : record(usage.cost, "bundle.usage.cost"); + const parsedUsage = { + ...structuredClone(usage), + inputTokens, + outputTokens, + totalTokens, + requestCount: integer(usage.requestCount, "bundle.usage.requestCount"), + durationMs: integer(usage.durationMs, "bundle.usage.durationMs"), + cost: costRecord === null + ? null + : { + ...structuredClone(costRecord), + currency: text(costRecord.currency, "bundle.usage.cost.currency"), + amountMicros: integer(costRecord.amountMicros, "bundle.usage.cost.amountMicros"), + }, + }; + + const transcriptRecord = record(bundle.transcript, "bundle.transcript"); + const transcript = { + ...contentRef(transcriptRecord, "bundle.transcript"), + complete: boolean(transcriptRecord.complete, "bundle.transcript.complete"), + eventCount: integer(transcriptRecord.eventCount, "bundle.transcript.eventCount"), + omissionReason: transcriptRecord.omissionReason === null + ? null + : text(transcriptRecord.omissionReason, "bundle.transcript.omissionReason"), + }; + if (transcript.complete && transcript.eventCount !== events.length) { + fail("bundle.transcript.eventCount", "must equal events.length for a complete transcript"); + } + if (!transcript.complete && transcript.omissionReason === null) { + fail("bundle.transcript.omissionReason", "is required when the transcript is incomplete"); + } + + const observations = structuredClone(record(bundle.observations, "bundle.observations")); + const artifactRoot = contentRef(bundle.artifactRoot, "bundle.artifactRoot"); + const artifacts = array(bundle.artifacts, "bundle.artifacts") + .map((entry, index) => contentRef(entry, `bundle.artifacts[${index}]`)); + const failure = bundle.failure === null + ? null + : (() => { + const parsed = record(bundle.failure, "bundle.failure"); + return { + ...structuredClone(parsed), + classification: text(parsed.classification, "bundle.failure.classification"), + code: text(parsed.code, "bundle.failure.code"), + message: text(parsed.message, "bundle.failure.message"), + }; + })(); + + return { + ...structuredClone(bundle), + schema: PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA, + provenance: bundle.provenance, + identity: parsedIdentity, + input: parsedInput, + runner: parsedRunner, + events, + semanticTools: { ...structuredClone(semanticTools), definitions, calls, results, denials }, + terminal: structuredClone(terminal), + observations, + usage: parsedUsage, + transcript, + artifactRoot, + artifacts, + failure, + } as PaperclipNativeExecutionV1; +} + +function semanticToolEnvelope( + event: PrpEvent, + phase: "input" | "result", + path: string, +): Record { + const payload = record(event.payload, `${path}.payload`); + const envelope = record(payload.semantic_tool, `${path}.payload.semantic_tool`); + if ( + envelope.schema !== "paperclip.prp.semantic_tool.v1" + || envelope.schemaVersion !== 1 + || envelope.phase !== phase + ) { + fail( + `${path}.payload.semantic_tool`, + `must be a paperclip.prp.semantic_tool.v1 ${phase} envelope`, + ); + } + const correlation = record(envelope.correlation, `${path}.payload.semantic_tool.correlation`); + for (const [field, eventValue] of [ + ["runId", event.runId], + ["normalizedSessionId", event.normalizedSessionId], + ["turnId", event.turnId], + ["itemId", event.itemId], + ] as const) { + if (correlation[field] !== eventValue) { + fail( + `${path}.payload.semantic_tool.correlation.${field}`, + `must match the enclosing event ${field}`, + ); + } + } + return envelope; +} + +export async function loadPaperclipNativeExecutionFixture( + url: URL = paperclipNativeExecutionFixtureUrl, +): Promise { + const value = JSON.parse(await readFile(url, "utf8")) as unknown; + return parsePaperclipNativeExecution(value); +} diff --git a/packages/paperclip-runner/src/evals/runnerd-artifact.test.ts b/packages/paperclip-runner/src/evals/runnerd-artifact.test.ts new file mode 100644 index 00000000000..6d013505538 --- /dev/null +++ b/packages/paperclip-runner/src/evals/runnerd-artifact.test.ts @@ -0,0 +1,57 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA } from "./build-metadata.js"; +import { + PaperclipRunnerdArtifactError, + parsePaperclipRunnerdBuildMetadata, + resolvePaperclipRunnerdArtifact, +} from "./runnerd-artifact.js"; + +const valid = { + schema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", + packageVersion: "0.1.2", + binaryContractVersion: 1, + nativeExecutionVersion: 1, + harnessDriverVersion: 1, + prp: { name: "paperclip.runner", minimumVersion: 1, maximumVersion: 1 }, +}; + +describe("runnerd artifact metadata", () => { + it("parses the exact runnerd identity", () => { + expect(parsePaperclipRunnerdBuildMetadata(valid)).toEqual(valid); + }); + + it("rejects an unknown binary metadata schema", () => { + expect(() => parsePaperclipRunnerdBuildMetadata({ ...valid, schema: "runnerd/v2" })) + .toThrow(PaperclipRunnerdArtifactError); + expect(() => parsePaperclipRunnerdBuildMetadata({ ...valid, schema: "runnerd/v2" })) + .toThrow(/unsupported/); + }); + + it("rejects an invalid or mismatched explicit artifact digest before execution", async () => { + await expect(resolvePaperclipRunnerdArtifact({ + executablePath: "/does/not/matter", + expectedSha256: "not-a-digest", + })).rejects.toMatchObject({ issue: "digest_invalid" }); + + const root = await mkdtemp(join(tmpdir(), "paperclip-runnerd-artifact-")); + const executablePath = join(root, "paperclip-runnerd"); + try { + await writeFile(executablePath, "not the expected binary"); + await expect(resolvePaperclipRunnerdArtifact({ + executablePath, + expectedSha256: `sha256:${"0".repeat(64)}`, + })).rejects.toMatchObject({ + issue: "digest_mismatch", + message: expect.stringContaining("observed sha256:"), + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/paperclip-runner/src/evals/runnerd-artifact.ts b/packages/paperclip-runner/src/evals/runnerd-artifact.ts new file mode 100644 index 00000000000..773eeb14989 --- /dev/null +++ b/packages/paperclip-runner/src/evals/runnerd-artifact.ts @@ -0,0 +1,207 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile, realpath, stat } from "node:fs/promises"; +import { promisify } from "node:util"; + +import { + PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + PAPERCLIP_RUNNER_BUILD_METADATA, +} from "./build-metadata.js"; + +const execFileAsync = promisify(execFile); +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; + +export interface PaperclipRunnerdBuildMetadata { + schema: typeof PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA; + binaryName: "paperclip-runnerd"; + packageName: "@paperclipai/paperclip-runner"; + packageVersion: string; + binaryContractVersion: number; + nativeExecutionVersion: number; + harnessDriverVersion: number; + prp: { + name: string; + minimumVersion: number; + maximumVersion: number; + }; +} + +export interface PaperclipRunnerdArtifact { + executablePath: string; + sha256: string; + byteSize: number; + buildMetadata: PaperclipRunnerdBuildMetadata; +} + +export class PaperclipRunnerdArtifactError extends Error { + readonly code = "paperclip_runnerd_artifact_invalid" as const; + + constructor( + message: string, + readonly issue: + | "path_invalid" + | "digest_invalid" + | "digest_mismatch" + | "metadata_unavailable" + | "metadata_invalid", + ) { + super(message); + this.name = "PaperclipRunnerdArtifactError"; + } +} + +function record(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new PaperclipRunnerdArtifactError(`${path} must be an object`, "metadata_invalid"); + } + return value as Record; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new PaperclipRunnerdArtifactError(`${path} must be a non-empty string`, "metadata_invalid"); + } + return value; +} + +function version(value: unknown, path: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new PaperclipRunnerdArtifactError(`${path} must be a positive integer`, "metadata_invalid"); + } + return value as number; +} + +/** Parse the runnerd response without accepting a look-alike binary. */ +export function parsePaperclipRunnerdBuildMetadata( + value: unknown, +): PaperclipRunnerdBuildMetadata { + const metadata = record(value, "runnerd build metadata"); + if (metadata.schema !== PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA) { + throw new PaperclipRunnerdArtifactError( + `runnerd metadata schema ${String(metadata.schema)} is unsupported; expected ${PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA}`, + "metadata_invalid", + ); + } + if (metadata.binaryName !== "paperclip-runnerd") { + throw new PaperclipRunnerdArtifactError( + `runnerd metadata names unexpected binary ${String(metadata.binaryName)}`, + "metadata_invalid", + ); + } + if (metadata.packageName !== PAPERCLIP_RUNNER_BUILD_METADATA.package.name) { + throw new PaperclipRunnerdArtifactError( + `runnerd metadata names unexpected package ${String(metadata.packageName)}`, + "metadata_invalid", + ); + } + const prp = record(metadata.prp, "runnerd build metadata.prp"); + const minimumVersion = version( + prp.minimumVersion, + "runnerd build metadata.prp.minimumVersion", + ); + const maximumVersion = version( + prp.maximumVersion, + "runnerd build metadata.prp.maximumVersion", + ); + if (minimumVersion > maximumVersion) { + throw new PaperclipRunnerdArtifactError( + "runnerd build metadata.prp minimumVersion must not exceed maximumVersion", + "metadata_invalid", + ); + } + return { + schema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + binaryName: "paperclip-runnerd", + packageName: PAPERCLIP_RUNNER_BUILD_METADATA.package.name, + packageVersion: text(metadata.packageVersion, "runnerd build metadata.packageVersion"), + binaryContractVersion: version( + metadata.binaryContractVersion, + "runnerd build metadata.binaryContractVersion", + ), + nativeExecutionVersion: version( + metadata.nativeExecutionVersion, + "runnerd build metadata.nativeExecutionVersion", + ), + harnessDriverVersion: version( + metadata.harnessDriverVersion, + "runnerd build metadata.harnessDriverVersion", + ), + prp: { + name: text(prp.name, "runnerd build metadata.prp.name"), + minimumVersion, + maximumVersion, + }, + }; +} + +/** + * Resolve one explicitly supplied runnerd artifact, verify its content digest, + * and read version metadata from that exact executable. This never searches + * PATH or the App source tree. + */ +export async function resolvePaperclipRunnerdArtifact(input: { + executablePath: string; + expectedSha256: string; + metadataTimeoutMs?: number; +}): Promise { + if (!SHA256_PATTERN.test(input.expectedSha256)) { + throw new PaperclipRunnerdArtifactError( + "expectedSha256 must use the sha256:<64 lowercase hex> form", + "digest_invalid", + ); + } + + let executablePath: string; + let fileStat: Awaited>; + let bytes: Buffer; + try { + executablePath = await realpath(input.executablePath); + fileStat = await stat(executablePath); + if (!fileStat.isFile()) throw new Error("path is not a file"); + bytes = await readFile(executablePath); + } catch (error) { + throw new PaperclipRunnerdArtifactError( + `runnerd artifact ${input.executablePath} is unavailable: ${error instanceof Error ? error.message : String(error)}`, + "path_invalid", + ); + } + + const observedSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + if (observedSha256 !== input.expectedSha256) { + throw new PaperclipRunnerdArtifactError( + `runnerd artifact digest mismatch: expected ${input.expectedSha256}, observed ${observedSha256}`, + "digest_mismatch", + ); + } + + let stdout: string; + try { + ({ stdout } = await execFileAsync(executablePath, ["--build-metadata"], { + encoding: "utf8", + timeout: input.metadataTimeoutMs ?? 5_000, + maxBuffer: 64 * 1024, + })); + } catch (error) { + throw new PaperclipRunnerdArtifactError( + `runnerd artifact did not return build metadata: ${error instanceof Error ? error.message : String(error)}`, + "metadata_unavailable", + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(stdout) as unknown; + } catch (error) { + throw new PaperclipRunnerdArtifactError( + `runnerd build metadata is not JSON: ${error instanceof Error ? error.message : String(error)}`, + "metadata_invalid", + ); + } + + return { + executablePath, + sha256: observedSha256, + byteSize: fileStat.size, + buildMetadata: parsePaperclipRunnerdBuildMetadata(parsed), + }; +} diff --git a/packages/paperclip-runner/src/mock-core/deterministic-harness-driver.ts b/packages/paperclip-runner/src/mock-core/deterministic-harness-driver.ts new file mode 100644 index 00000000000..4d3c5646633 --- /dev/null +++ b/packages/paperclip-runner/src/mock-core/deterministic-harness-driver.ts @@ -0,0 +1,353 @@ +import { + type HarnessDriver, + type HarnessDriverConfigValidation, + type HarnessDriverDescriptor, + type HarnessSession, + type HarnessSessionRecoveryResult, + type HarnessTranscriptSnapshot, + type OpenHarnessSessionInput, + type PersistedHarnessSession, +} from "../contracts/harness-driver.js"; +import { + createPrpSemanticToolInputEnvelope, + createPrpSemanticToolResultEnvelope, +} from "../protocol/semantic-tool-receipts.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; + +export const DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA = + "paperclip-runner/deterministic-harness-driver-config/v1" as const; + +export interface DeterministicHarnessDriverConfig { + schema: typeof DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA; + scenario: "semantic-tool-success"; +} + +const DEFAULT_CONFIG: DeterministicHarnessDriverConfig = { + schema: DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA, + scenario: "semantic-tool-success", +}; + +function parseConfig(value: unknown): HarnessDriverConfigValidation { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return { + ok: false, + config: null, + issues: [{ path: "/", code: "type", message: "config must be an object" }], + }; + } + const config = value as Record; + if (config.schema !== DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA) { + return { + ok: false, + config: null, + issues: [{ + path: "/schema", + code: "unsupported_schema", + message: `schema must be ${DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA}`, + }], + }; + } + if (config.scenario !== "semantic-tool-success") { + return { + ok: false, + config: null, + issues: [{ + path: "/scenario", + code: "unsupported_scenario", + message: "scenario must be semantic-tool-success", + }], + }; + } + return { ok: true, config: structuredClone(config), issues: [] }; +} + +/** Provider-free fake used by packed consumers and App/Evals conformance. */ +export class DeterministicHarnessDriver implements HarnessDriver { + readonly #config: DeterministicHarnessDriverConfig; + + constructor(config: DeterministicHarnessDriverConfig = DEFAULT_CONFIG) { + const validation = parseConfig(config); + if (!validation.ok) throw new Error(validation.issues[0]?.message ?? "invalid config"); + this.#config = structuredClone(config); + } + + async descriptor(): Promise { + return { + kind: "paperclip-deterministic", + displayName: "Paperclip deterministic harness driver", + version: "1.0.0", + protocolVersion: "prp.v1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + read: true, + reconciliation: true, + usage: true, + dynamicTools: true, + runtimeRequestResolution: false, + goals: false, + threadLineage: false, + unsupported: ["steering", "runtimeRequestResolution", "goals", "threadLineage"], + }, + }; + } + + async validateConfig(config: unknown): Promise { + return parseConfig(config); + } + + async openSession(input: OpenHarnessSessionInput): Promise { + return new DeterministicHarnessSession(input, this.#config); + } + + async recoverSession( + snapshot: PersistedHarnessSession, + ): Promise { + if ( + snapshot.driverKind !== "paperclip-deterministic" + || snapshot.runId === undefined + || snapshot.normalizedSessionId === undefined + ) { + return { recovered: false, reason: "snapshot does not belong to the deterministic driver" }; + } + return { + recovered: true, + session: new DeterministicHarnessSession({ + runId: snapshot.runId, + normalizedSessionId: snapshot.normalizedSessionId, + workingDirectory: "/deterministic/recovered", + }, this.#config, snapshot), + }; + } +} + +class DeterministicHarnessSession implements HarnessSession { + readonly #input: OpenHarnessSessionInput; + readonly #config: DeterministicHarnessDriverConfig; + #events: PrpEvent[] = []; + #semanticResult: PrpStructuredRunResult | null = null; + #turnId: string | null = null; + #closed = false; + #active = false; + #nextSourceSeq = 1; + + constructor( + input: OpenHarnessSessionInput, + config: DeterministicHarnessDriverConfig, + snapshot?: PersistedHarnessSession, + ) { + this.#input = structuredClone(input); + this.#config = structuredClone(config); + this.#turnId = snapshot?.activeTurnId ?? snapshot?.semanticResult?.turnId ?? null; + this.#semanticResult = snapshot?.semanticResult?.result ?? null; + } + + ids() { + return { + driverSessionId: `driver_${this.#input.normalizedSessionId}`, + providerSessionId: null, + displayId: this.#input.normalizedSessionId, + }; + } + + async *events(): AsyncIterable { + for (const event of this.#events) yield structuredClone(event); + } + + async startTurn(input: { message: { role: "user"; text: string } }): Promise<{ turnId: string }> { + this.#assertOpen(); + if (this.#turnId !== null) throw new Error("deterministic session accepts one turn"); + this.#turnId = `turn_${this.#input.runId}`; + this.#active = true; + this.#events.push( + this.#event("session.started", {}, { turn: false }), + this.#event("turn.started", { message: input.message.text }), + ); + if (input.message.text.includes("[conformance:interrupt]")) { + return { turnId: this.#turnId }; + } + + const itemId = `item_${this.#input.runId}`; + const callId = `call_${this.#input.runId}`; + const correlation = { + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + turnId: this.#turnId, + itemId, + }; + const inputEnvelope = createPrpSemanticToolInputEnvelope({ + operationId: "finish_task", + callId, + correlation, + idempotencyKey: `finish_${this.#input.runId}`, + content: { summary: "Deterministic conformance complete." }, + }); + const resultEnvelope = createPrpSemanticToolResultEnvelope({ + operationId: "finish_task", + callId, + correlation, + idempotencyKey: `finish_${this.#input.runId}`, + content: { disposition: "applied", status: "done" }, + outcome: "succeeded", + code: "applied", + retryable: false, + authorizationBoundary: "active_task", + auditReceiptId: `audit_${this.#input.runId}`, + }); + this.#events.push( + this.#event("mcp_app.tool_input", { semantic_tool: inputEnvelope }, { itemId }), + this.#event("mcp_app.tool_result", { semantic_tool: resultEnvelope }, { itemId }), + ); + this.#semanticResult = completedResult(); + const terminal = completedTerminal(); + this.#events.push( + this.#event("run.result.proposed", this.#semanticResult), + this.#event("turn.completed", {}), + this.#event("run.terminal", terminal, { turn: false }), + ); + this.#active = false; + return { turnId: this.#turnId }; + } + + async interrupt(input: { turnId?: string; reason?: string }): Promise { + this.#assertOpen(); + if (!this.#active || this.#turnId === null) throw new Error("no active turn to interrupt"); + if (input.turnId !== undefined && input.turnId !== this.#turnId) { + throw new Error(`turn ${input.turnId} is not active`); + } + this.#events.push( + this.#event("turn.interrupted", { reason: input.reason ?? "interrupted" }), + this.#event("run.terminal", cancelledTerminal(), { turn: false }), + ); + this.#active = false; + } + + async read(): Promise> { + return { scenario: this.#config.scenario, eventCount: this.#events.length }; + } + + async reconcile(): Promise> { + return { active: this.#active, closed: this.#closed, eventCount: this.#events.length }; + } + + async usage(): Promise> { + return { + inputTokens: this.#turnId === null ? 0 : 12, + outputTokens: this.#semanticResult === null ? 0 : 8, + totalTokens: this.#turnId === null ? 0 : this.#semanticResult === null ? 12 : 20, + requestCount: this.#turnId === null ? 0 : 1, + cost: { currency: "USD", amountMicros: 0 }, + }; + } + + async transcript(): Promise { + return { + schema: "paperclip-runner/harness-transcript/v1", + complete: true, + eventCount: this.#events.length, + events: structuredClone(this.#events), + omissionReason: null, + }; + } + + async snapshot(): Promise { + return { + driverKind: "paperclip-deterministic", + driverSessionId: `driver_${this.#input.normalizedSessionId}`, + providerSessionId: null, + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + activeTurnId: this.#active ? this.#turnId : null, + lastSourceSequence: this.#events.length, + semanticResult: this.#semanticResult === null || this.#turnId === null + ? null + : { + result: structuredClone(this.#semanticResult), + fingerprint: JSON.stringify(this.#semanticResult), + turnId: this.#turnId, + }, + terminalTurns: this.#events.some((event) => event.eventType === "run.terminal") && this.#turnId !== null + ? [{ turnId: this.#turnId, fingerprint: `terminal_${this.#input.runId}` }] + : [], + }; + } + + async close(input: { reason: string; force?: boolean }): Promise { + if (this.#closed) return; + if (this.#active) { + await this.interrupt({ reason: input.reason }); + } + this.#closed = true; + } + + #event( + eventType: PrpEvent["eventType"], + payload: Record | PrpStructuredRunResult | PrpTerminalState, + options: { itemId?: string; turn?: boolean } = {}, + ): PrpEvent { + const sourceSeq = this.#nextSourceSeq++; + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `event_${this.#input.runId}_${sourceSeq}`, + sourceSeq, + sourceInstanceId: `runner_${this.#input.runId}`, + sourceKind: "runner", + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + ...(options.turn === false || this.#turnId === null ? {} : { turnId: this.#turnId }), + ...(options.itemId === undefined ? {} : { itemId: options.itemId }), + eventType, + schemaVersion: 1, + priority: eventType === "run.terminal" || eventType === "run.result.proposed" ? 0 : 1, + emittedAt: new Date(Date.UTC(2026, 7, 11, 12, 0, sourceSeq)).toISOString(), + payload: structuredClone(payload), + } as PrpEvent; + } + + #assertOpen(): void { + if (this.#closed) throw new Error("deterministic session is closed"); + } +} + +function completedResult(): PrpStructuredRunResult { + return { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Deterministic harness-driver conformance completed.", + completionClaim: { + contractRevision: "harness-driver-conformance-v1", + objectiveSatisfied: true, + criteria: [], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "deterministic harness flow", status: "passed" }], + attentionRequests: [], + artifacts: [], + }; +} + +function completedTerminal(): PrpTerminalState { + return { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }; +} + +function cancelledTerminal(): PrpTerminalState { + return { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "interrupted", + runTerminalState: "cancelled", + reportedWorkDisposition: "yielded", + }; +} diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts index 862c5784b94..71198fbe9a2 100644 --- a/packages/paperclip-runner/src/testing.ts +++ b/packages/paperclip-runner/src/testing.ts @@ -9,7 +9,9 @@ export * from "./index.js"; export * from "./conformance/control-plane-port.js"; export * from "./conformance/semantic-conformance.js"; export * from "./conformance/capability-semantic-conformance.js"; +export * from "./conformance/harness-driver.js"; export * from "./mock-core/mock-control-plane-adapter.js"; +export * from "./mock-core/deterministic-harness-driver.js"; export * from "./mock-core/capability-control-plane-types.js"; export * from "./mock-core/capability-mock-control-plane-adapter.js"; export * from "./protocol/conformance-fixture.js";