From 7eadc714d239368eda4fdbfdbdcf9d6a958f761f Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Mon, 5 Oct 2026 20:21:01 -0700 Subject: [PATCH] Verify native semantic input against its raw wire digest (#15301) ## Thinking Path > - Paperclip manages AI agents and their work. > - Native runners send authenticated semantic tool inputs to the control plane. > - The runner hashes the complete input, but the receiver used a redacted receipt hash. > - Protected fields and credential-like document text can therefore fail integrity validation even when the input is unchanged. > - This pull request verifies the complete input with the existing canonical hash function. > - Receipt redaction and permanent rejection of altered input remain in place. ## Linked Issues or Issue Description **What happened?** The Rust runner preserves tool arguments and hashes their canonical JSON. The TypeScript receiver instead redacts the input before hashing. A valid input such as a synthetic document containing `Bearer fixture_token_123456` fails with `native_event_replay_conflict`. A digest of redacted input can also pass without proving the original protected values. **Expected behavior** Verify the complete transmitted input after authentication and exact scope checks. Reject any incorrect digest before durable commit, dispatch, or ACK. **Steps to reproduce** Run the new authenticated controller regressions against the prior receiver. The protected-field and credential-like document cases fail, and the redacted-digest rejection case receives an ACK. The same tests pass with this change. **Paperclip version or commit** Reproduced from source at `858094ba8123c7edb56623597cd96f0391f7e2d4` with synthetic fixtures. Applies to native runner deployments. Related: #14937 preserves semantic input bytes for execution. GitHub issue and PR searches found no duplicate fix; #14591 touches a separate question-draft contract. ## What Changed - Use the existing bounded raw canonical digest for incoming semantic and MCP tool inputs. - Keep receipt and diagnostic redaction unchanged. - Share four digest fixtures between Rust and the authenticated TypeScript controller, including protected fields, document text, Unicode keys, and number boundaries. - Test raw acceptance, altered protected values, forged and redacted digests, canonicalization limits, permanent reconnect fences, and authentication/scope rejection. - Document the separate wire and receipt contracts and the unchanged recovery fence. ## Verification - Before the production fix, the new selected regressions produced three expected failures and eight passes. - Focused controller, receipt, and semantic-tool suites: 138 tests passed. - Rust shared digest fixture test: 1 test passed. - Full workspace typecheck and build passed; the final receiver delta also passed its TypeScript typecheck. Canonical Linux PR CI passed the full aggregate test gate, runner checks, build, and canary dry run at `efcd38e2d8a9fa6340db4f4e863b6febbe8ca32a`. - Independent review passed, including 18 independently run authenticated regressions and the Rust golden test, plus both matching-digest size-limit cases after the test-only follow-up. Greptile is 5/5 on the current head with no unresolved threads. Diff whitespace and local secret/PII scan passed; fixtures are synthetic. ## Risks The verifier remains strict: there is no redacted-digest fallback. Existing authentication, scope, sequence, replay, settlement, and authorization checks remain in force. Receipt storage and redaction behavior are unchanged. This patch does not clear failed-run fences or replay prior work. Synthetic tests prove the protocol mismatch; they do not identify the contents of any historical rejected input. ## Model Used OpenAI Codex (GPT-6), with reasoning, code execution, and independent agent review. The exact serving model identifier and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../paperclip-runner/docs/durable-recovery.md | 9 + .../tests/semantic_input_digest.rs | 18 ++ .../durable-prp-control-plane.test.ts | 161 +++++++++++++++++- .../durable-prp-control-plane.ts | 19 ++- .../test/fixtures/semantic-input-digests.json | 42 +++++ 5 files changed, 240 insertions(+), 9 deletions(-) create mode 100644 packages/paperclip-runner/runner/crates/runner-core/tests/semantic_input_digest.rs create mode 100644 packages/paperclip-runner/test/fixtures/semantic-input-digests.json diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 8ee0b22c975..0a9d5d9a7a1 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -95,6 +95,15 @@ welcome and every control envelope against the authenticated connection, runner, environment lease, run, normalized session, turn, item, protocol, lease ID, expiry, and revocation metadata before applying an ACK or command. +Semantic input integrity uses SHA-256 of the complete transmitted input's +canonical JSON, with UTF-16 object-key order and JavaScript number formatting. +The controller checks this proof after authentication and exact run/session/ +turn/item correlation, before committing, dispatching, or acknowledging the +input. Receipt redaction has a separate digest; a redacted digest cannot stand +in for wire integrity, including when only a protected field changes. A failed +integrity proof keeps the existing operator-required recovery fence. Updating +the verifier does not clear a previously failed run or replay its work. + The daemon captures and removes the bootstrap environment variable before it parses arguments or starts child work. Secret buffers are overwritten when they are dropped. It resolves the destination once before sending a bearer value and diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/semantic_input_digest.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/semantic_input_digest.rs new file mode 100644 index 00000000000..1ef2c8f34b1 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/semantic_input_digest.rs @@ -0,0 +1,18 @@ +use paperclip_runner_core::provider_bridge::semantic_value_digest; +use serde_json::Value; + +#[test] +fn semantic_input_digests_match_the_authenticated_controller_fixtures() { + let fixtures: Value = serde_json::from_str(include_str!( + "../../../../test/fixtures/semantic-input-digests.json" + )) + .expect("parse shared semantic input digest fixtures"); + for fixture in fixtures.as_array().expect("digest fixture array") { + assert_eq!( + semantic_value_digest(&fixture["input"]), + fixture["digest"].as_str().expect("fixture digest"), + "fixture {}", + fixture["name"] + ); + } +} diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 9a1e145f6e8..8980c1c4e37 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -26,6 +26,7 @@ import { validatePrpEvent } from "../protocol/replay-contract.js"; import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; import { DurablePrpControlPlane, + durableRecoveryInternals, SemanticToolNotDispatchedError, inspectWarmRunTransition, spawnRunner, @@ -1164,7 +1165,17 @@ async function receiveSecure( return value; } -function semanticInputEvent(sourceSeq = 1): Record { +const semanticInputDigestFixtures = JSON.parse(readFileSync( + new URL("../../test/fixtures/semantic-input-digests.json", import.meta.url), + "utf8", +)) as Array<{ name: string; input: Record; digest: string }>; + +function rawSemanticInputDigest(input: unknown): string { + return `sha256:${createHash("sha256") + .update(durableRecoveryInternals.canonicalJson(input)).digest("hex")}`; +} + +function semanticInputEvent(sourceSeq = 1, input: Record = {}): Record { return { protocol: "paperclip.runner", version: 1, @@ -1204,11 +1215,11 @@ function semanticInputEvent(sourceSeq = 1): Record { }, idempotencyKey: null, content: { - digest: digestPaperclipSemanticContent({}), + digest: rawSemanticInputDigest(input), redactionDisposition: "digest_only", references: [], }, - input: {}, + input, }, }, }, @@ -1687,6 +1698,144 @@ describe.sequential("DurablePrpControlPlane", () => { }, ); + it.each(semanticInputDigestFixtures.flatMap((fixture) => + ["semantic_tool.input", "mcp_app.tool_input"].map((eventType) => ({ ...fixture, eventType })), + ))( + "accepts the Rust wire digest and preserves exact input: $eventType / $name", + async ({ input, digest, eventType }) => { + // The JSON transport normalizes negative zero before the receiver sees it. + const wireInput = JSON.parse(JSON.stringify(input)) as Record; + const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-raw-input-")); + const onProtocolIntegrityError = vi.fn(); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, + onProtocolIntegrityError, onSemanticToolInput, + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const event = semanticInputEvent(1, wireInput); + (event.payload as any).eventType = eventType; + const semantic = (event.payload as any).payload.semantic_tool; + // This shared golden also runs against the Rust producer's hash. + semantic.content.digest = digest; + expect(rawSemanticInputDigest(input)).toBe(digest); + sendSecure(client, event); + const frames = [await receiveSecure(client), await receiveSecure(client)]; + expect(frames).toContainEqual(expect.objectContaining({ + kind: "ack", payload: expect.objectContaining({ ackedSourceSeq: 1 }), + })); + const command = frames.find((frame) => frame?.kind === "command")!.payload as any; + expect(command.payload.inputDigest).toBe(digest.slice("sha256:".length)); + expect(onSemanticToolInput).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ input: wireInput })); + expect(core.store.state.committedEvents).toHaveLength(1); + expect(core.store.state.committedEvents[0]?.envelope).toEqual(event); + expect(onProtocolIntegrityError).not.toHaveBeenCalled(); + client.socket.destroy(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } + }, + ); + + it.each([ + "redacted_digest", "protected_field_tampering", "forged_digest", + "over_node_limit", "over_depth_limit", + ] as const)( + "rejects invalid raw semantic integrity without commit, dispatch or ACK: %s", + async (fault) => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-raw-integrity-")); + const onProtocolIntegrityError = vi.fn(); + const onCommittedEvent = vi.fn(async () => undefined); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, + onProtocolIntegrityError, onCommittedEvent, onSemanticToolInput, + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const input = { password: "fixture-before", safe: true }; + const event = semanticInputEvent(1, input); + const semantic = (event.payload as any).payload.semantic_tool; + const redactedDigest = digestPaperclipSemanticContent(input); + expect(redactedDigest).not.toBe(semantic.content.digest); + if (fault === "redacted_digest") semantic.content.digest = redactedDigest; + if (fault === "forged_digest") semantic.content.digest = `sha256:${"0".repeat(64)}`; + if (fault === "protected_field_tampering") { + semantic.input = { ...input, password: "fixture-after" }; + // Redacted receipt hashes intentionally cannot prove byte integrity. + expect(digestPaperclipSemanticContent(semantic.input)).toBe(redactedDigest); + } + if (fault === "over_node_limit") semantic.input = { values: Array(10_001).fill(0) }; + if (fault === "over_depth_limit") { + let nested: unknown = {}; + for (let depth = 0; depth < 65; depth += 1) nested = { child: nested }; + semantic.input = nested; + } + if (fault === "over_node_limit" || fault === "over_depth_limit") { + // Each fixture object has one key, so JSON.stringify is canonical. + // Hash independently of the bounded helper: the limits themselves + // must reject even a matching raw digest and keep the permanent fence. + semantic.content.digest = `sha256:${createHash("sha256") + .update(JSON.stringify(semantic.input)).digest("hex")}`; + } + sendSecure(client, event); + await expect(receiveSecure(client)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ + code: "native_event_replay_conflict", reason: "semantic_input_digest_mismatch", + recovery: "operator_required", + })); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(core.store.state.committedEvents).toEqual([]); + expect(onCommittedEvent).not.toHaveBeenCalled(); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + expect(JSON.stringify(onProtocolIntegrityError.mock.calls)).not.toContain("fixture-before"); + expect(JSON.stringify(onProtocolIntegrityError.mock.calls)).not.toContain("fixture-after"); + const replay = (await authenticate(core, client.leaseToken!))!; + sendSecure(replay, semanticInputEvent()); + await expect(receiveSecure(replay)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } + }, + ); + + it.each(["authentication", "envelope", "event", "correlation"] as const)( + "rejects raw semantic input outside its exact authenticated scope: %s", + async (boundary) => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-raw-scope-")); + const onProtocolIntegrityError = vi.fn(); + const onCommittedEvent = vi.fn(async () => undefined); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, + onProtocolIntegrityError, onCommittedEvent, onSemanticToolInput, + }); + try { + await core.start(); + const wrongRunId = "00000000-0000-4000-8000-000000000999"; + const client = await authenticate(core, core.issueBootstrapTicket(), + boundary === "authentication" ? { ...identity, runId: wrongRunId } : identity); + if (boundary === "authentication") { + expect(client).toBeNull(); + } else { + const event = semanticInputEvent(1, { password: "fixture-only" }); + const payload = event.payload as any; + if (boundary === "envelope") event.runId = wrongRunId; + if (boundary === "event") payload.runId = wrongRunId; + if (boundary === "correlation") payload.payload.semantic_tool.correlation.runId = wrongRunId; + sendSecure(client!, event); + await expect(receiveSecure(client!)).resolves.toBeNull(); + } + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(core.store.state.committedEvents).toEqual([]); + expect(onCommittedEvent).not.toHaveBeenCalled(); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + expect(onProtocolIntegrityError).not.toHaveBeenCalled(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } + }, + ); + it("latches an authenticated semantic integrity fault without ACK or dispatch and still permits suspension", async () => { const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-integrity-")); const onProtocolIntegrityError = vi.fn(); @@ -1848,7 +1997,7 @@ describe.sequential("DurablePrpControlPlane", () => { ).semantic_tool as Record; semantic.input = { changed: true }; (semantic.content as Record).digest = - digestPaperclipSemanticContent(semantic.input); + rawSemanticInputDigest(semantic.input); sendSecure(client, changed); await expect(receiveSecure(client)).resolves.toBeNull(); expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1); @@ -2232,13 +2381,13 @@ describe.sequential("DurablePrpControlPlane", () => { const event = semanticInputEvent(); const semantic = ((event.payload as any).payload as any).semantic_tool; semantic.input = args; - semantic.content.digest = digestPaperclipSemanticContent(args); + semantic.content.digest = rawSemanticInputDigest(args); sendSecure(client, event); const frames = [await receiveSecure(client), await receiveSecure(client)]; const wire = frames.find((frame) => frame?.kind === "command")!.payload as any; expect(wire.payload.result.content).toBe(content); expect(wire.payload.input).toBeUndefined(); - expect(wire.payload.inputDigest).toBe(digestPaperclipSemanticContent(args).slice("sha256:".length)); + expect(wire.payload.inputDigest).toBe(rawSemanticInputDigest(args).slice("sha256:".length)); expect(Buffer.byteLength(JSON.stringify(wire))).toBeLessThan(1024 * 1024); sendSecure(client, { protocol: "paperclip.runner", version: 1, kind: "command_result", payload: { commandId: wire.commandId, controllerSeq: wire.controllerSeq, commandType: wire.type, diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index dfcaf5bb8dd..c7f8cc9e876 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -35,7 +35,6 @@ import { validatePrpEvent, type PrpEvent, } from "../protocol/replay-contract.js"; -import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; import { type DurableRecoveryCommittedEvent, type DurableRecoveryCoreCommand, @@ -416,6 +415,18 @@ function canonicalDigest(value: unknown): string { return createHash("sha256").update(canonicalJson(value)).digest("hex"); } +function matchesSemanticInputDigest(input: unknown, digest: unknown): boolean { + try { + // Wire integrity covers the complete input, including protected fields. + // Receipt redaction can erase those differences and has a separate hash. + return digest === `sha256:${canonicalDigest(input)}`; + } catch { + // Canonicalization bounds must keep the same permanent integrity fence. + // Never attach an input-derived error or payload to the diagnostic. + return false; + } +} + function exactIdentity(value: unknown): value is DurableRecoveryIdentity { return ( isRecord(value) && @@ -3141,8 +3152,10 @@ export class DurablePrpControlPlane { if ( isSemanticInput && semantic !== undefined && - (semantic.content as Record).digest !== - digestPaperclipSemanticContent(semantic.input) + !matchesSemanticInputDigest( + semantic.input, + (semantic.content as Record).digest, + ) ) { // Only the authenticated, schema-valid, exactly correlated input may // permanently fail its owner. Never commit, dispatch, or ACK these bytes. diff --git a/packages/paperclip-runner/test/fixtures/semantic-input-digests.json b/packages/paperclip-runner/test/fixtures/semantic-input-digests.json new file mode 100644 index 00000000000..db605462674 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/semantic-input-digests.json @@ -0,0 +1,42 @@ +[ + { + "name": "plain document", + "input": { + "body": "Plain fixture document" + }, + "digest": "sha256:2ba38770004c4e7672a5097c4d8e3241c62750b21e034903796fb035fcf9a2a4" + }, + { + "name": "protected fields", + "input": { + "password": "fixture-only-not-a-real-credential", + "safe": true + }, + "digest": "sha256:98ecc80f0e6d2d4b99ce3620d18a21392356f99def55ed599c1b1c002ccd2d47" + }, + { + "name": "credential-like document text", + "input": { + "body": "Example Bearer fixture_token_123456\nhttps://example.test/callback?token=fixture-only" + }, + "digest": "sha256:aa7f429cc5dd63e85fd11aa93e0659fa1c7d5041f5f35843fef62092ff86fdc0" + }, + { + "name": "canonical Unicode and numbers", + "input": { + "": "private-use key", + "😀": "astral key", + "values": [ + -0, + 0.000001, + 9.99999e-7, + 100000000000000000000, + 1e+21, + true, + null, + "日本語\n\"text\"" + ] + }, + "digest": "sha256:3104a5e5db1fe2c6dcdaa0bfcf362d7bd36441e7e08905a48ae03be06eea9dc4" + } +]