From 9f6f020da6aa6ff85f679001a8ce0eb2dd8f4884 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 05:45:40 -0500 Subject: [PATCH] test(runner): grade Pi file validation from native streamed evidence Accept unresolved edit targets only before the exact terminal target, and bind validation to ordered command progress plus a successful native structured output receipt. Calibrate against a sanitized actual stream and keep independent artifact and download checks. Co-Authored-By: Paperclip --- .../pi-file-evidence-actual-stream.json | 798 ++++++++++++++++++ tests/runner-e2e/pi-file-evidence.test.ts | 108 ++- tests/runner-e2e/pi-file-evidence.ts | 46 +- 3 files changed, 921 insertions(+), 31 deletions(-) create mode 100644 tests/runner-e2e/fixtures/pi-file-evidence-actual-stream.json diff --git a/tests/runner-e2e/fixtures/pi-file-evidence-actual-stream.json b/tests/runner-e2e/fixtures/pi-file-evidence-actual-stream.json new file mode 100644 index 00000000000..752408372cf --- /dev/null +++ b/tests/runner-e2e/fixtures/pi-file-evidence-actual-stream.json @@ -0,0 +1,798 @@ +{ + "schema": "paperclip.e2e.sanitized-pi-file-stream.v1", + "description": "44 retained native Rust-projected edit/bash rows; only fixture nonce and execution identities replaced. Timestamps, private identities and unrelated events omitted. This calibrates the oracle, not the original failed attempt result.", + "events": [ + { + "seq": 71, + "sourceSeq": 46, + "eventType": "tool.execution.started", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "edit (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 72, + "sourceSeq": 47, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 73, + "sourceSeq": 48, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 74, + "sourceSeq": 49, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 75, + "sourceSeq": 50, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 76, + "sourceSeq": 51, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 77, + "sourceSeq": 52, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 78, + "sourceSeq": 53, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 79, + "sourceSeq": 54, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 80, + "sourceSeq": 55, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 81, + "sourceSeq": 56, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 82, + "sourceSeq": 57, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 83, + "sourceSeq": 58, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 84, + "sourceSeq": 59, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "tool call (pending)", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 85, + "sourceSeq": 60, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": "extended-fixture.txt", + "exitCode": null, + "progress": "tool call (pending): extended-fixture.txt", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 86, + "sourceSeq": 61, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": "extended-fixture.txt", + "exitCode": null, + "progress": "tool call (pending): extended-fixture.txt", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 88, + "sourceSeq": 63, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "running", + "target": "extended-fixture.txt", + "exitCode": null, + "progress": "tool call (in_progress): extended-fixture.txt", + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 89, + "sourceSeq": 64, + "eventType": "tool.execution.completed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "edit", + "operation": "edit", + "transport": "builtin", + "status": "completed", + "target": "extended-fixture.txt", + "exitCode": null, + "progress": null, + "output": "null", + "outputTruncated": false, + "executionId": "edit" + } + }, + { + "seq": 92, + "sourceSeq": 66, + "eventType": "tool.execution.started", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 93, + "sourceSeq": 67, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 94, + "sourceSeq": 68, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 95, + "sourceSeq": 69, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 96, + "sourceSeq": 70, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 97, + "sourceSeq": 71, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 98, + "sourceSeq": 72, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 99, + "sourceSeq": 73, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 100, + "sourceSeq": 74, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 101, + "sourceSeq": 75, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 102, + "sourceSeq": 76, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 103, + "sourceSeq": 77, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 104, + "sourceSeq": 78, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 105, + "sourceSeq": 79, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 106, + "sourceSeq": 80, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 107, + "sourceSeq": 81, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 108, + "sourceSeq": 82, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 109, + "sourceSeq": 83, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 110, + "sourceSeq": 84, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 111, + "sourceSeq": 85, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "bash (pending): [terminal] validate", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 112, + "sourceSeq": 86, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (pending): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 113, + "sourceSeq": 87, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (pending): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 115, + "sourceSeq": 89, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (in_progress): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'", + "output": "null", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 116, + "sourceSeq": 90, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (in_progress): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'", + "output": "{\"content\":[]}", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 117, + "sourceSeq": 91, + "eventType": "tool.execution.progressed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "running", + "target": null, + "exitCode": null, + "progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (in_progress): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'", + "output": "{\"content\":[{\"type\":\"text\",\"text\":\"PI-VALIDATED-fixture\\n\"}],\"details\":{}}", + "outputTruncated": false, + "executionId": "validate" + } + }, + { + "seq": 118, + "sourceSeq": 92, + "eventType": "tool.execution.completed", + "payload": { + "schema": "paperclip.tool.execution.v1", + "name": "bash", + "operation": "execute", + "transport": "builtin", + "status": "completed", + "target": null, + "exitCode": null, + "progress": null, + "output": "{\"content\":[{\"type\":\"text\",\"text\":\"PI-VALIDATED-fixture\\n\"}],\"structuredContent\":{\"output\":\"PI-VALIDATED-fixture\\n\",\"truncated\":false,\"exit_code\":0,\"wall_time_seconds\":0}}", + "outputTruncated": false, + "executionId": "validate" + } + } + ] +} diff --git a/tests/runner-e2e/pi-file-evidence.test.ts b/tests/runner-e2e/pi-file-evidence.test.ts index 212ae8ac9f1..64981c7ab67 100644 --- a/tests/runner-e2e/pi-file-evidence.test.ts +++ b/tests/runner-e2e/pi-file-evidence.test.ts @@ -6,21 +6,27 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { collectPiFileEvidence, gradePiCopyback, gradePiFileEvidence, piFileContract, seedPiFile } from "./pi-file-evidence.js"; -import { canonicalProviderEventsFromAcpxRuntimeEvent } from "../../packages/paperclip-runner/src/provider-events.js"; +import { canonicalProviderEventsFromAcpxRuntimeEvent, createAcpxToolEventNormalizer } from "../../packages/paperclip-runner/src/provider-events.js"; import { safeAcpxLocations } from "../../packages/paperclip-runner/src/drivers/acpx/safe-locations.js"; import { classifyFailure } from "./failure-classifier.js"; import { runnerMatrix } from "./catalog.js"; function fixture() { const c = piFileContract("fixture"), attachmentId = "12345678-1234-1234-1234-123456789abc"; - const tool = (seq: number, executionId: string, name: string, operation: string, completed: boolean, target: string | null, output = "") => { - // Pi emits absolute edit locations and uses the bash command as title. - // Exercise sidecar path normalization followed by the common projection; - // direct raw ACP projection would incorrectly lose the workspace target. - const canonical = canonicalProviderEventsFromAcpxRuntimeEvent({ type: "tool_call", toolCallId: executionId, - tag: completed ? "tool_call_update" : "tool_call", title: name, kind: operation, - status: completed ? "completed" : "pending", locations: safeAcpxLocations(target ? [{ path: `/workspace/${target}` }] : [], "/workspace", operation, name), rawOutput: output }, executionId, "turn")[0]!; + const normalize = createAcpxToolEventNormalizer(); + const tool = (seq: number, executionId: string, name: string, operation: string, + phase: "start" | "progress" | "end", target: string | null, text = "", rawOutput: unknown = undefined) => { + // Sanitized shape of the retained Pi stream: missing opening location, + // later resolved edit arguments, and stable bash name with command progress. + // This exercises TS normalization/projection, not the Rust implementation. + const event = normalize({ type: "tool_call", toolCallId: executionId, + tag: phase === "start" ? "tool_call" : "tool_call_update", title: name, kind: operation, + status: phase === "end" ? "completed" : phase === "start" ? "pending" : "in_progress", + locations: safeAcpxLocations(target ? [{ path: `/workspace/${target}` }] : [], "/workspace", operation, name), + text, rawOutput }); + const canonical = canonicalProviderEventsFromAcpxRuntimeEvent(event, executionId, "turn")[0]!; return { companyId: "company", runId: "run", seq, protocolSchemaVersion: 1, eventType: canonical.eventType, + sourceInstanceId: "runner", sourceSeq: seq, sourceEventId: `runner:run:${seq}`, payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "runner", sourceSeq: seq, sourceEventId: `runner:run:${seq}`, runId: "run", turnId: "turn", normalizedSessionId: "session", ...canonical } } }; }; @@ -33,42 +39,70 @@ function fixture() { publish: { operationId: "register_deliverable", input: { filename: c.filename, contentRef: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 }, result: { disposition: "applied", commandId: `deliverable-prepared:${attachmentId}`, entityRefs: [attachmentId, "product", "comment"] } }, } } }, - events: [tool(1, "edit", "edit", "edit", false, c.filename), tool(2, "edit", "edit", "edit", true, c.filename), - tool(3, "validate", c.validationCommand, "execute", false, null), tool(4, "validate", c.validationCommand, "execute", true, null, c.validationMarker)], + events: [tool(1, "edit", "edit", "edit", "start", null), + tool(2, "edit", "edit", "edit", "progress", c.filename), + tool(3, "edit", "edit", "edit", "end", c.filename), + tool(4, "validate", "bash", "execute", "start", null, "bash (pending): [terminal] validate"), + tool(5, "validate", c.validationCommand, "execute", "progress", null, `${c.validationCommand} (in_progress): ${c.validationCommand}`), + tool(6, "validate", c.validationCommand, "execute", "end", null, "", { + content: [{ type: "text", text: `${c.validationMarker}\n` }], + structuredContent: { output: `${c.validationMarker}\n`, truncated: false, exit_code: 0, wall_time_seconds: 0 }, + })], attachments: [{ id: attachmentId, companyId: "company", issueId: "issue", originatingRunId: "run", createdByAgentId: "agent", originalFilename: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 }], activity: [{ action: "issue.attachment_added", companyId: "company", runId: "run", actorId: "agent", entityId: "issue", details: { attachmentId, source: "paperclip_runner_protocol" } }], }; } +function changeOutput(f: ReturnType, change: (output: any) => void) { + const p = f.events[5]!.payload.prpEvent.payload, output = JSON.parse(p.output as string); + change(output); p.output = JSON.stringify(output); +} describe("Pi edit, validation and public artifact oracle", () => { + it("calibrates against the sanitized 44-row actual native stream without regrading the paid attempt", async () => { + const retained = JSON.parse(await readFile(new URL("./fixtures/pi-file-evidence-actual-stream.json", import.meta.url), "utf8")); + const f = fixture(); + expect(retained.events).toHaveLength(44); + f.events = retained.events.map((x: any) => ({ companyId: "company", runId: "run", seq: x.seq, + protocolSchemaVersion: 1, eventType: x.eventType, sourceInstanceId: "runner", sourceSeq: x.sourceSeq, + sourceEventId: `runner:run:${x.sourceSeq}`, payload: { prpEvent: { + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "runner", + sourceSeq: x.sourceSeq, sourceEventId: `runner:run:${x.sourceSeq}`, runId: "run", turnId: "turn", + normalizedSessionId: "session", eventType: x.eventType, payload: x.payload, + } } })); + expect(gradePiFileEvidence(f).verification).toMatchObject({ nativeStructuredExitCode: 0, typedExitCode: null }); + }); it("accepts actual correlated lifecycles, independent bytes and a run-bound registered download", () => { const result = gradePiFileEvidence(fixture()); expect(result.passed).toBe(true); expect(result.diff.text).toContain("-ready-fixture\n+verified-fixture\n"); expect(result.limits.join(" ")).toContain("not attested"); expect(result.verification.providerExecutionStatus).toBe("completed"); - expect(result.verification.commandTitle).toBe(piFileContract("fixture").validationCommand); + expect(result.verification.command).toBe(piFileContract("fixture").validationCommand); + expect(result.verification).toMatchObject({ commandEvidence: "correlated_native_progress", + exitEvidence: "native_structured_output_receipt", nativeStructuredExitCode: 0, typedExitCode: null }); + expect(fixture().events.map(e => e.payload.prpEvent.payload.target)).toEqual([null, "extended-fixture.txt", "extended-fixture.txt", null, null, null]); + expect(fixture().events.slice(3).every(e => e.payload.prpEvent.payload.name === "bash")).toBe(true); expect(result.verification.nativeFileAttribution).toBe("workspace_relative_display_target"); }); const mutations: Array<[string, (f: ReturnType) => void]> = [ ["absent seed", f => { f.seed = {} as any; }], ["wrong final bytes", f => { f.workspaceBytes = Buffer.from("wrong\n"); }], - ["final-only write", f => { f.events[0]!.payload.prpEvent.payload.name = "write"; f.events[1]!.payload.prpEvent.payload.name = "write"; }], - ["missing edit", f => { f.events = f.events.slice(2); }], - ["missing validation", f => { f.events = f.events.slice(0, 2); }], - ["echo-only validation", f => { f.events[2]!.payload.prpEvent.payload.name = `echo ${piFileContract("fixture").validationMarker}`; f.events[3]!.payload.prpEvent.payload.name = f.events[2]!.payload.prpEvent.payload.name; }], - ["failed validation", f => { f.events[3]!.payload.prpEvent.payload.status = "failed"; }], + ["final-only write", f => { f.events[0]!.payload.prpEvent.payload.name = "write"; f.events[2]!.payload.prpEvent.payload.name = "write"; }], + ["missing edit", f => { f.events = f.events.slice(3); }], + ["missing validation", f => { f.events = f.events.slice(0, 3); }], + ["echo-only validation", f => { f.events[3]!.payload.prpEvent.payload.name = `echo ${piFileContract("fixture").validationMarker}`; f.events[5]!.payload.prpEvent.payload.name = f.events[3]!.payload.prpEvent.payload.name; }], + ["failed validation", f => { f.events[5]!.payload.prpEvent.payload.status = "failed"; }], ["unfinished validation", f => { f.events.pop(); }], - ["unrelated validation turn", f => { f.events[2]!.payload.prpEvent.turnId = "other"; f.events[3]!.payload.prpEvent.turnId = "other"; }], + ["unrelated validation turn", f => { f.events[3]!.payload.prpEvent.turnId = "other"; f.events[5]!.payload.prpEvent.turnId = "other"; }], ["foreign native session", f => { f.run.nativeSessionId = "other"; }], ["foreign native producer", f => { f.run.runnerInstanceId = "other"; }], - ["foreign event run", f => { f.events[1]!.payload.prpEvent.runId = "other"; }], - ["duplicate event", f => { f.events.push(structuredClone(f.events[3]!)); }], - ["missing tool start", f => { f.events.splice(2, 1); }], - ["truncated validation output", f => { f.events[3]!.payload.prpEvent.payload.outputTruncated = true; }], - ["missing native target", f => { f.events[0]!.payload.prpEvent.payload.target = null; f.events[1]!.payload.prpEvent.payload.target = null; }], - ["wrong target", f => { f.events[1]!.payload.prpEvent.payload.target = "other.txt"; }], + ["foreign event run", f => { f.events[2]!.payload.prpEvent.runId = "other"; }], + ["duplicate event", f => { f.events.push(structuredClone(f.events[5]!)); }], + ["missing tool start", f => { f.events.splice(3, 1); }], + ["truncated validation output", f => { f.events[5]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing native target", f => { f.events[0]!.payload.prpEvent.payload.target = null; f.events[2]!.payload.prpEvent.payload.target = null; }], + ["wrong target", f => { f.events[2]!.payload.prpEvent.payload.target = "other.txt"; }], ["missing registration", f => { f.run.resultJson.semanticToolReceipts = {} as any; }], ["rejected registration", f => { f.run.resultJson.semanticToolReceipts.publish.result.disposition = "denied"; }], ["wrong registered hash", f => { f.run.resultJson.semanticToolReceipts.publish.input.sha256 = "wrong"; }], @@ -79,6 +113,34 @@ describe("Pi edit, validation and public artifact oracle", () => { ["wrong download bytes", f => { f.downloadedBytes = Buffer.from("wrong\n"); }], ["missing publication activity", f => { f.activity = []; }], ["foreign publication run", f => { f.activity[0]!.runId = "other"; }], + ["conflicting resolved target", f => { f.events[1]!.payload.prpEvent.payload.target = "other.txt"; }], + ["target regresses after resolution", f => { f.events[0]!.payload.prpEvent.payload.target = piFileContract("fixture").filename; f.events[1]!.payload.prpEvent.payload.target = null; }], + ["missing terminal target", f => { f.events[2]!.payload.prpEvent.payload.target = null; }], + ["absent command proof", f => { f.events[4]!.payload.prpEvent.payload.progress = null; }], + ["echo-only command proof", f => { f.events[4]!.payload.prpEvent.payload.progress = "echo PI-VALIDATED-fixture (in_progress): echo PI-VALIDATED-fixture"; }], + ["conflicting command before exact proof", f => { f.events[3]!.payload.prpEvent.payload.progress = "node other.js (pending): node other.js"; }], + ["late command proof", f => { f.events[5]!.payload.prpEvent.payload.progress = f.events[4]!.payload.prpEvent.payload.progress; f.events[4]!.payload.prpEvent.payload.progress = null; }], + ["plain echoed marker output", f => { f.events[5]!.payload.prpEvent.payload.output = piFileContract("fixture").validationMarker; }], + ["nonzero native exit", f => { changeOutput(f, o => { o.structuredContent.exit_code = 1; }); }], + ["missing native exit", f => { changeOutput(f, o => { delete o.structuredContent.exit_code; }); }], + ["truncated native output", f => { changeOutput(f, o => { o.structuredContent.truncated = true; }); }], + ["false marker substring", f => { changeOutput(f, o => { o.structuredContent.output = "failed before PI-VALIDATED-fixture\n"; }); }], + ["contradictory content", f => { changeOutput(f, o => { o.content[0].text = "failed"; }); }], + ["contradictory typed exit", f => { f.events[5]!.payload.prpEvent.payload.exitCode = 1; }], + ["foreign command proof execution", f => { f.events[4]!.payload.prpEvent.payload.executionId = "foreign"; }], + ["foreign command proof turn", f => { f.events[4]!.payload.prpEvent.turnId = "foreign"; }], + ["foreign command proof session", f => { f.events[4]!.payload.prpEvent.normalizedSessionId = "foreign"; }], + ["foreign command proof producer", f => { + const row = f.events[4]!, e = row.payload.prpEvent; + e.sourceInstanceId = row.sourceInstanceId = "foreign"; e.sourceEventId = row.sourceEventId = `foreign:run:${e.sourceSeq}`; + }], + ["durable source differs from envelope", f => { f.events[4]!.sourceInstanceId = "foreign"; }], + ["durable event identity differs from envelope", f => { f.events[4]!.sourceEventId = "runner:run:999"; }], + ["durable source sequence differs from envelope", f => { f.events[4]!.sourceSeq = 999; }], + ["command proof after terminal", f => { + const e = f.events[4]!; f.events.splice(4, 1); e.seq = 7; e.payload.prpEvent.sourceSeq = 7; + e.sourceSeq = 7; e.sourceEventId = e.payload.prpEvent.sourceEventId = "runner:run:7"; f.events.push(e); + }], ]; it.each(mutations)("rejects %s even when the model claims completion", (_name, change) => { const f = fixture(); change(f); expect(() => gradePiFileEvidence(f)).toThrow("Pi file evidence:"); diff --git a/tests/runner-e2e/pi-file-evidence.ts b/tests/runner-e2e/pi-file-evidence.ts index 1a3e96326e2..7837666633f 100644 --- a/tests/runner-e2e/pi-file-evidence.ts +++ b/tests/runner-e2e/pi-file-evidence.ts @@ -58,7 +58,8 @@ export function gradePiCopyback(run: Row, lease: Row, companyId: string, environ } /** Only the common public projection is authority here. Native raw arguments, - * full diff blocks and typed command exit codes are deliberately not invented. */ + * full diff blocks and typed command exit codes are deliberately not invented. + * The native structured output receipt is distinct from that nullable typed field. */ export function gradePiFileEvidence(input: { nonce: string; companyId: string; issueId: string; agentId: string; run: Row; environment: "local" | "daytona"; environmentId: string; lease?: Row; @@ -84,6 +85,7 @@ export function gradePiFileEvidence(input: { && Number.isSafeInteger(row.seq) && row.seq > 0 && !seqs.has(row.seq) && typeof e.sourceInstanceId === "string" && e.sourceInstanceId.length > 0 && Number.isSafeInteger(e.sourceSeq) && e.sourceSeq > (sourceSeqs.get(e.sourceInstanceId) ?? 0) + && row.sourceInstanceId === e.sourceInstanceId && row.sourceSeq === e.sourceSeq && row.sourceEventId === e.sourceEventId && e.sourceEventId === `${e.sourceInstanceId}:${e.runId}:${e.sourceSeq}` && !seen.has(e.sourceEventId), "foreign, duplicated, or reordered durable evidence"); seqs.add(row.seq); seen.add(e.sourceEventId); sourceSeqs.set(e.sourceInstanceId, e.sourceSeq); return { row, e, p: rec(e.payload) }; @@ -104,20 +106,46 @@ export function gradePiFileEvidence(input: { && starts.length === 1 && starts[0]!.row.seq < end.row.seq && matches.filter(x => x.e.eventType === "tool.execution.completed").length === 1 && matches.every(x => x.p.schema === end.p.schema && x.p.name === end.p.name && x.p.operation === end.p.operation - && x.p.target === end.p.target && x.e.sourceKind === "runner" && x.e.turnId === end.e.turnId + && x.row.seq >= starts[0]!.row.seq && x.row.seq <= end.row.seq + && x.e.sourceKind === "runner" && x.e.turnId === end.e.turnId && x.e.normalizedSessionId === end.e.normalizedSessionId && x.e.sourceInstanceId === end.e.sourceInstanceId - && (x === end || x.p.status === "running")), "incomplete or unsuccessful native tool lifecycle"); - return starts[0]!; + && (x === end || (x.p.status === "running" + && ["tool.execution.started", "tool.execution.progressed"].includes(x.e.eventType)))), "incomplete or unsuccessful native tool lifecycle"); + // Native arguments arrive incrementally. An unresolved target is allowed + // only before the first exact target; conflicting or regressed targets fail. + let resolved = false; + for (const x of matches) { + requireEvidence(x.p.target === end.p.target || (!resolved && x.p.target === null), "conflicting native tool target"); + if (x.p.target !== null) resolved = true; + } + return { start: starts[0]!, matches }; } lifecycle(edit); const validations = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "execute" - && x.p.name === c.validationCommand && typeof x.p.output === "string" && x.p.output.includes(c.validationMarker)); + && x.p.name === "bash"); requireEvidence(validations.length === 1, "one observed provider validation execution"); - const validation = validations[0]!, start = lifecycle(validation); + const validation = validations[0]!, { start, matches } = lifecycle(validation); requireEvidence(start.row.seq > edit.row.seq && validation.e.turnId === edit.e.turnId && validation.e.normalizedSessionId === edit.e.normalizedSessionId && validation.e.sourceInstanceId === edit.e.sourceInstanceId && validation.p.outputTruncated === false, "validation must follow the edit in the same native turn"); + // The common lifecycle preserves the opening name (bash). ACPX's resolved + // command is retained as progress, rather than replacing that stable name. + const commandProgress = `${c.validationCommand} (in_progress): ${c.validationCommand}`; + const allowedProgress = new Set([`bash (pending): [terminal] ${validation.p.executionId}`, + `${c.validationCommand} (pending): ${c.validationCommand}`, commandProgress]); + requireEvidence(matches.some(x => x !== validation && x.p.progress === commandProgress) + && matches.every(x => x.p.progress === null || allowedProgress.has(x.p.progress)), + "exact correlated command progress before completion required"); + requireEvidence(typeof validation.p.output === "string" && Buffer.byteLength(validation.p.output) <= 16_384, + "bounded native validation receipt required"); + let output: Row = {}; + try { output = rec(JSON.parse(validation.p.output)); } catch { /* Rejected below; plain marker text is insufficient. */ } + const structured = rec(output.structuredContent), expectedOutput = `${c.validationMarker}\n`; + requireEvidence(structured.exit_code === 0 && structured.truncated === false && structured.output === expectedOutput + && Array.isArray(output.content) && output.content.length === 1 + && rec(output.content[0]).type === "text" && rec(output.content[0]).text === expectedOutput + && (validation.p.exitCode === null || validation.p.exitCode === 0), "successful native structured validation receipt required"); const attachments = input.attachments.map(rec).filter(a => a.id === input.attachmentId); requireEvidence(attachments.length === 1, "one public attachment"); const a = attachments[0]!; @@ -144,11 +172,13 @@ export function gradePiFileEvidence(input: { validationExecutionId: validation.p.executionId, attachmentId: a.id, verification: { kind: "independent_exact_bytes", beforeSha256: sha(c.before), afterSha256: c.sha256, downloadedSha256: sha(input.downloadedBytes), byteSize: c.byteSize, providerExecutionStatus: validation.p.status, - commandTitle: validation.p.name, nativeFileTarget: edit.p.target, + command: c.validationCommand, providerToolName: validation.p.name, commandEvidence: "correlated_native_progress", + exitEvidence: "native_structured_output_receipt", nativeStructuredExitCode: structured.exit_code, + typedExitCode: validation.p.exitCode, nativeFileTarget: edit.p.target, nativeFileAttribution: "workspace_relative_display_target" }, diff: { source: "independent_seed_and_workspace_bytes", path: c.filename, text: `--- ${c.filename}\n+++ ${c.filename}\n@@ -1 +1 @@\n-${c.before.trimEnd()}\n+${c.after.trimEnd()}\n` }, - limits: ["Validation command title is checked exactly; raw arguments and typed exitCode are not projected, so raw invocation/exit code are not attested.", + limits: ["Exact command progress and the native structured output receipt are checked; raw arguments and nullable typed exitCode are not attested by those distinct fields.", "Diff is computed from independently checked workspace bytes; it does not qualify native diff presentation or private invocation metadata."], }; }