Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-08 01:57:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(runner): grade Pi file validation from native streamed evidence

Browse Source 
Accept unresolved edit targets only before the exact terminal target, and bind validation to ordered command progress plus a successful native structured output receipt. Calibrate against a sanitized actual stream and keep independent artifact and download checks.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-02 05:46:33 -05:00
1 parent 7980fa3f60
commit 9f6f020da6
3 files changed
+921 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/fixtures/pi-file-evidence-actual-stream.json
+798
View File
@@ -0,0 +1,798 @@
{
"schema": "paperclip.e2e.sanitized-pi-file-stream.v1",
"description": "44 retained native Rust-projected edit/bash rows; only fixture nonce and execution identities replaced. Timestamps, private identities and unrelated events omitted. This calibrates the oracle, not the original failed attempt result.",
"events": [
{
"seq": 71,
"sourceSeq": 46,
"eventType": "tool.execution.started",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "edit (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 72,
"sourceSeq": 47,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 73,
"sourceSeq": 48,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 74,
"sourceSeq": 49,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 75,
"sourceSeq": 50,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 76,
"sourceSeq": 51,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 77,
"sourceSeq": 52,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 78,
"sourceSeq": 53,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 79,
"sourceSeq": 54,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 80,
"sourceSeq": 55,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 81,
"sourceSeq": 56,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 82,
"sourceSeq": 57,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 83,
"sourceSeq": 58,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 84,
"sourceSeq": 59,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "tool call (pending)",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 85,
"sourceSeq": 60,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": "extended-fixture.txt",
"exitCode": null,
"progress": "tool call (pending): extended-fixture.txt",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 86,
"sourceSeq": 61,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": "extended-fixture.txt",
"exitCode": null,
"progress": "tool call (pending): extended-fixture.txt",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 88,
"sourceSeq": 63,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "running",
"target": "extended-fixture.txt",
"exitCode": null,
"progress": "tool call (in_progress): extended-fixture.txt",
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 89,
"sourceSeq": 64,
"eventType": "tool.execution.completed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "edit",
"operation": "edit",
"transport": "builtin",
"status": "completed",
"target": "extended-fixture.txt",
"exitCode": null,
"progress": null,
"output": "null",
"outputTruncated": false,
"executionId": "edit"
}
},
{
"seq": 92,
"sourceSeq": 66,
"eventType": "tool.execution.started",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 93,
"sourceSeq": 67,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 94,
"sourceSeq": 68,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 95,
"sourceSeq": 69,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 96,
"sourceSeq": 70,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 97,
"sourceSeq": 71,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 98,
"sourceSeq": 72,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 99,
"sourceSeq": 73,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 100,
"sourceSeq": 74,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 101,
"sourceSeq": 75,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 102,
"sourceSeq": 76,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 103,
"sourceSeq": 77,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 104,
"sourceSeq": 78,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 105,
"sourceSeq": 79,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 106,
"sourceSeq": 80,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 107,
"sourceSeq": 81,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 108,
"sourceSeq": 82,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 109,
"sourceSeq": 83,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 110,
"sourceSeq": 84,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 111,
"sourceSeq": 85,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "bash (pending): [terminal] validate",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 112,
"sourceSeq": 86,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (pending): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 113,
"sourceSeq": 87,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (pending): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 115,
"sourceSeq": 89,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (in_progress): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'",
"output": "null",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 116,
"sourceSeq": 90,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (in_progress): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'",
"output": "{\"content\":[]}",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 117,
"sourceSeq": 91,
"eventType": "tool.execution.progressed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "running",
"target": null,
"exitCode": null,
"progress": "node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")' (in_progress): node -e 'if(require(\"node:fs\").readFileSync(\"extended-fixture.txt\",\"utf8\")!==\"verified-fixture\\n\")throw Error(\"mismatch\");console.log(\"PI-VALIDATED-fixture\")'",
"output": "{\"content\":[{\"type\":\"text\",\"text\":\"PI-VALIDATED-fixture\\n\"}],\"details\":{}}",
"outputTruncated": false,
"executionId": "validate"
}
},
{
"seq": 118,
"sourceSeq": 92,
"eventType": "tool.execution.completed",
"payload": {
"schema": "paperclip.tool.execution.v1",
"name": "bash",
"operation": "execute",
"transport": "builtin",
"status": "completed",
"target": null,
"exitCode": null,
"progress": null,
"output": "{\"content\":[{\"type\":\"text\",\"text\":\"PI-VALIDATED-fixture\\n\"}],\"structuredContent\":{\"output\":\"PI-VALIDATED-fixture\\n\",\"truncated\":false,\"exit_code\":0,\"wall_time_seconds\":0}}",
"outputTruncated": false,
"executionId": "validate"
}
}
]
}
tests/runner-e2e/pi-file-evidence.test.ts
+85 -23
View File
@@ -6,21 +6,27 @@ import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { describe, expect, it, vi } from "vitest";
import { collectPiFileEvidence, gradePiCopyback, gradePiFileEvidence, piFileContract, seedPiFile } from "./pi-file-evidence.js";
import { canonicalProviderEventsFromAcpxRuntimeEvent } from "../../packages/paperclip-runner/src/provider-events.js";
import { canonicalProviderEventsFromAcpxRuntimeEvent, createAcpxToolEventNormalizer } from "../../packages/paperclip-runner/src/provider-events.js";
import { safeAcpxLocations } from "../../packages/paperclip-runner/src/drivers/acpx/safe-locations.js";
import { classifyFailure } from "./failure-classifier.js";
import { runnerMatrix } from "./catalog.js";
function fixture() {
const c = piFileContract("fixture"), attachmentId = "12345678-1234-1234-1234-123456789abc";
const tool = (seq: number, executionId: string, name: string, operation: string, completed: boolean, target: string | null, output = "") => {
// Pi emits absolute edit locations and uses the bash command as title.
// Exercise sidecar path normalization followed by the common projection;
// direct raw ACP projection would incorrectly lose the workspace target.
const canonical = canonicalProviderEventsFromAcpxRuntimeEvent({ type: "tool_call", toolCallId: executionId,
tag: completed ? "tool_call_update" : "tool_call", title: name, kind: operation,
status: completed ? "completed" : "pending", locations: safeAcpxLocations(target ? [{ path: `/workspace/${target}` }] : [], "/workspace", operation, name), rawOutput: output }, executionId, "turn")[0]!;
const normalize = createAcpxToolEventNormalizer();
const tool = (seq: number, executionId: string, name: string, operation: string,
phase: "start" | "progress" | "end", target: string | null, text = "", rawOutput: unknown = undefined) => {
// Sanitized shape of the retained Pi stream: missing opening location,
// later resolved edit arguments, and stable bash name with command progress.
// This exercises TS normalization/projection, not the Rust implementation.
const event = normalize({ type: "tool_call", toolCallId: executionId,
tag: phase === "start" ? "tool_call" : "tool_call_update", title: name, kind: operation,
status: phase === "end" ? "completed" : phase === "start" ? "pending" : "in_progress",
locations: safeAcpxLocations(target ? [{ path: `/workspace/${target}` }] : [], "/workspace", operation, name),
text, rawOutput });
const canonical = canonicalProviderEventsFromAcpxRuntimeEvent(event, executionId, "turn")[0]!;
return { companyId: "company", runId: "run", seq, protocolSchemaVersion: 1, eventType: canonical.eventType,
sourceInstanceId: "runner", sourceSeq: seq, sourceEventId: `runner:run:${seq}`,
payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "runner",
sourceSeq: seq, sourceEventId: `runner:run:${seq}`, runId: "run", turnId: "turn", normalizedSessionId: "session", ...canonical } } };
};
@@ -33,42 +39,70 @@ function fixture() {
publish: { operationId: "register_deliverable", input: { filename: c.filename, contentRef: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 },
result: { disposition: "applied", commandId: `deliverable-prepared:${attachmentId}`, entityRefs: [attachmentId, "product", "comment"] } },
} } },
events: [tool(1, "edit", "edit", "edit", false, c.filename), tool(2, "edit", "edit", "edit", true, c.filename),
tool(3, "validate", c.validationCommand, "execute", false, null), tool(4, "validate", c.validationCommand, "execute", true, null, c.validationMarker)],
events: [tool(1, "edit", "edit", "edit", "start", null),
tool(2, "edit", "edit", "edit", "progress", c.filename),
tool(3, "edit", "edit", "edit", "end", c.filename),
tool(4, "validate", "bash", "execute", "start", null, "bash (pending): [terminal] validate"),
tool(5, "validate", c.validationCommand, "execute", "progress", null, `${c.validationCommand} (in_progress): ${c.validationCommand}`),
tool(6, "validate", c.validationCommand, "execute", "end", null, "", {
content: [{ type: "text", text: `${c.validationMarker}\n` }],
structuredContent: { output: `${c.validationMarker}\n`, truncated: false, exit_code: 0, wall_time_seconds: 0 },
})],
attachments: [{ id: attachmentId, companyId: "company", issueId: "issue", originatingRunId: "run", createdByAgentId: "agent",
originalFilename: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 }],
activity: [{ action: "issue.attachment_added", companyId: "company", runId: "run", actorId: "agent", entityId: "issue",
details: { attachmentId, source: "paperclip_runner_protocol" } }],
};
}
function changeOutput(f: ReturnType<typeof fixture>, change: (output: any) => void) {
const p = f.events[5]!.payload.prpEvent.payload, output = JSON.parse(p.output as string);
change(output); p.output = JSON.stringify(output);
}
describe("Pi edit, validation and public artifact oracle", () => {
it("calibrates against the sanitized 44-row actual native stream without regrading the paid attempt", async () => {
const retained = JSON.parse(await readFile(new URL("./fixtures/pi-file-evidence-actual-stream.json", import.meta.url), "utf8"));
const f = fixture();
expect(retained.events).toHaveLength(44);
f.events = retained.events.map((x: any) => ({ companyId: "company", runId: "run", seq: x.seq,
protocolSchemaVersion: 1, eventType: x.eventType, sourceInstanceId: "runner", sourceSeq: x.sourceSeq,
sourceEventId: `runner:run:${x.sourceSeq}`, payload: { prpEvent: {
schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "runner",
sourceSeq: x.sourceSeq, sourceEventId: `runner:run:${x.sourceSeq}`, runId: "run", turnId: "turn",
normalizedSessionId: "session", eventType: x.eventType, payload: x.payload,
} } }));
expect(gradePiFileEvidence(f).verification).toMatchObject({ nativeStructuredExitCode: 0, typedExitCode: null });
});
it("accepts actual correlated lifecycles, independent bytes and a run-bound registered download", () => {
const result = gradePiFileEvidence(fixture());
expect(result.passed).toBe(true);
expect(result.diff.text).toContain("-ready-fixture\n+verified-fixture\n");
expect(result.limits.join(" ")).toContain("not attested");
expect(result.verification.providerExecutionStatus).toBe("completed");
expect(result.verification.commandTitle).toBe(piFileContract("fixture").validationCommand);
expect(result.verification.command).toBe(piFileContract("fixture").validationCommand);
expect(result.verification).toMatchObject({ commandEvidence: "correlated_native_progress",
exitEvidence: "native_structured_output_receipt", nativeStructuredExitCode: 0, typedExitCode: null });
expect(fixture().events.map(e => e.payload.prpEvent.payload.target)).toEqual([null, "extended-fixture.txt", "extended-fixture.txt", null, null, null]);
expect(fixture().events.slice(3).every(e => e.payload.prpEvent.payload.name === "bash")).toBe(true);
expect(result.verification.nativeFileAttribution).toBe("workspace_relative_display_target");
});
const mutations: Array<[string, (f: ReturnType<typeof fixture>) => void]> = [
["absent seed", f => { f.seed = {} as any; }],
["wrong final bytes", f => { f.workspaceBytes = Buffer.from("wrong\n"); }],
["final-only write", f => { f.events[0]!.payload.prpEvent.payload.name = "write"; f.events[1]!.payload.prpEvent.payload.name = "write"; }],
["missing edit", f => { f.events = f.events.slice(2); }],
["missing validation", f => { f.events = f.events.slice(0, 2); }],
["echo-only validation", f => { f.events[2]!.payload.prpEvent.payload.name = `echo ${piFileContract("fixture").validationMarker}`; f.events[3]!.payload.prpEvent.payload.name = f.events[2]!.payload.prpEvent.payload.name; }],
["failed validation", f => { f.events[3]!.payload.prpEvent.payload.status = "failed"; }],
["final-only write", f => { f.events[0]!.payload.prpEvent.payload.name = "write"; f.events[2]!.payload.prpEvent.payload.name = "write"; }],
["missing edit", f => { f.events = f.events.slice(3); }],
["missing validation", f => { f.events = f.events.slice(0, 3); }],
["echo-only validation", f => { f.events[3]!.payload.prpEvent.payload.name = `echo ${piFileContract("fixture").validationMarker}`; f.events[5]!.payload.prpEvent.payload.name = f.events[3]!.payload.prpEvent.payload.name; }],
["failed validation", f => { f.events[5]!.payload.prpEvent.payload.status = "failed"; }],
["unfinished validation", f => { f.events.pop(); }],
["unrelated validation turn", f => { f.events[2]!.payload.prpEvent.turnId = "other"; f.events[3]!.payload.prpEvent.turnId = "other"; }],
["unrelated validation turn", f => { f.events[3]!.payload.prpEvent.turnId = "other"; f.events[5]!.payload.prpEvent.turnId = "other"; }],
["foreign native session", f => { f.run.nativeSessionId = "other"; }],
["foreign native producer", f => { f.run.runnerInstanceId = "other"; }],
["foreign event run", f => { f.events[1]!.payload.prpEvent.runId = "other"; }],
["duplicate event", f => { f.events.push(structuredClone(f.events[3]!)); }],
["missing tool start", f => { f.events.splice(2, 1); }],
["truncated validation output", f => { f.events[3]!.payload.prpEvent.payload.outputTruncated = true; }],
["missing native target", f => { f.events[0]!.payload.prpEvent.payload.target = null; f.events[1]!.payload.prpEvent.payload.target = null; }],
["wrong target", f => { f.events[1]!.payload.prpEvent.payload.target = "other.txt"; }],
["foreign event run", f => { f.events[2]!.payload.prpEvent.runId = "other"; }],
["duplicate event", f => { f.events.push(structuredClone(f.events[5]!)); }],
["missing tool start", f => { f.events.splice(3, 1); }],
["truncated validation output", f => { f.events[5]!.payload.prpEvent.payload.outputTruncated = true; }],
["missing native target", f => { f.events[0]!.payload.prpEvent.payload.target = null; f.events[2]!.payload.prpEvent.payload.target = null; }],
["wrong target", f => { f.events[2]!.payload.prpEvent.payload.target = "other.txt"; }],
["missing registration", f => { f.run.resultJson.semanticToolReceipts = {} as any; }],
["rejected registration", f => { f.run.resultJson.semanticToolReceipts.publish.result.disposition = "denied"; }],
["wrong registered hash", f => { f.run.resultJson.semanticToolReceipts.publish.input.sha256 = "wrong"; }],
@@ -79,6 +113,34 @@ describe("Pi edit, validation and public artifact oracle", () => {
["wrong download bytes", f => { f.downloadedBytes = Buffer.from("wrong\n"); }],
["missing publication activity", f => { f.activity = []; }],
["foreign publication run", f => { f.activity[0]!.runId = "other"; }],
["conflicting resolved target", f => { f.events[1]!.payload.prpEvent.payload.target = "other.txt"; }],
["target regresses after resolution", f => { f.events[0]!.payload.prpEvent.payload.target = piFileContract("fixture").filename; f.events[1]!.payload.prpEvent.payload.target = null; }],
["missing terminal target", f => { f.events[2]!.payload.prpEvent.payload.target = null; }],
["absent command proof", f => { f.events[4]!.payload.prpEvent.payload.progress = null; }],
["echo-only command proof", f => { f.events[4]!.payload.prpEvent.payload.progress = "echo PI-VALIDATED-fixture (in_progress): echo PI-VALIDATED-fixture"; }],
["conflicting command before exact proof", f => { f.events[3]!.payload.prpEvent.payload.progress = "node other.js (pending): node other.js"; }],
["late command proof", f => { f.events[5]!.payload.prpEvent.payload.progress = f.events[4]!.payload.prpEvent.payload.progress; f.events[4]!.payload.prpEvent.payload.progress = null; }],
["plain echoed marker output", f => { f.events[5]!.payload.prpEvent.payload.output = piFileContract("fixture").validationMarker; }],
["nonzero native exit", f => { changeOutput(f, o => { o.structuredContent.exit_code = 1; }); }],
["missing native exit", f => { changeOutput(f, o => { delete o.structuredContent.exit_code; }); }],
["truncated native output", f => { changeOutput(f, o => { o.structuredContent.truncated = true; }); }],
["false marker substring", f => { changeOutput(f, o => { o.structuredContent.output = "failed before PI-VALIDATED-fixture\n"; }); }],
["contradictory content", f => { changeOutput(f, o => { o.content[0].text = "failed"; }); }],
["contradictory typed exit", f => { f.events[5]!.payload.prpEvent.payload.exitCode = 1; }],
["foreign command proof execution", f => { f.events[4]!.payload.prpEvent.payload.executionId = "foreign"; }],
["foreign command proof turn", f => { f.events[4]!.payload.prpEvent.turnId = "foreign"; }],
["foreign command proof session", f => { f.events[4]!.payload.prpEvent.normalizedSessionId = "foreign"; }],
["foreign command proof producer", f => {
const row = f.events[4]!, e = row.payload.prpEvent;
e.sourceInstanceId = row.sourceInstanceId = "foreign"; e.sourceEventId = row.sourceEventId = `foreign:run:${e.sourceSeq}`;
}],
["durable source differs from envelope", f => { f.events[4]!.sourceInstanceId = "foreign"; }],
["durable event identity differs from envelope", f => { f.events[4]!.sourceEventId = "runner:run:999"; }],
["durable source sequence differs from envelope", f => { f.events[4]!.sourceSeq = 999; }],
["command proof after terminal", f => {
const e = f.events[4]!; f.events.splice(4, 1); e.seq = 7; e.payload.prpEvent.sourceSeq = 7;
e.sourceSeq = 7; e.sourceEventId = e.payload.prpEvent.sourceEventId = "runner:run:7"; f.events.push(e);
}],
];
it.each(mutations)("rejects %s even when the model claims completion", (_name, change) => {
const f = fixture(); change(f); expect(() => gradePiFileEvidence(f)).toThrow("Pi file evidence:");
tests/runner-e2e/pi-file-evidence.ts
+38 -8
View File
@@ -58,7 +58,8 @@ export function gradePiCopyback(run: Row, lease: Row, companyId: string, environ
}
/** Only the common public projection is authority here. Native raw arguments,
* full diff blocks and typed command exit codes are deliberately not invented. */
* full diff blocks and typed command exit codes are deliberately not invented.
* The native structured output receipt is distinct from that nullable typed field. */
export function gradePiFileEvidence(input: {
nonce: string; companyId: string; issueId: string; agentId: string; run: Row;
environment: "local" | "daytona"; environmentId: string; lease?: Row;
@@ -84,6 +85,7 @@ export function gradePiFileEvidence(input: {
&& Number.isSafeInteger(row.seq) && row.seq > 0 && !seqs.has(row.seq)
&& typeof e.sourceInstanceId === "string" && e.sourceInstanceId.length > 0
&& Number.isSafeInteger(e.sourceSeq) && e.sourceSeq > (sourceSeqs.get(e.sourceInstanceId) ?? 0)
&& row.sourceInstanceId === e.sourceInstanceId && row.sourceSeq === e.sourceSeq && row.sourceEventId === e.sourceEventId
&& e.sourceEventId === `${e.sourceInstanceId}:${e.runId}:${e.sourceSeq}` && !seen.has(e.sourceEventId), "foreign, duplicated, or reordered durable evidence");
seqs.add(row.seq); seen.add(e.sourceEventId); sourceSeqs.set(e.sourceInstanceId, e.sourceSeq);
return { row, e, p: rec(e.payload) };
@@ -104,20 +106,46 @@ export function gradePiFileEvidence(input: {
&& starts.length === 1 && starts[0]!.row.seq < end.row.seq
&& matches.filter(x => x.e.eventType === "tool.execution.completed").length === 1
&& matches.every(x => x.p.schema === end.p.schema && x.p.name === end.p.name && x.p.operation === end.p.operation
&& x.p.target === end.p.target && x.e.sourceKind === "runner" && x.e.turnId === end.e.turnId
&& x.row.seq >= starts[0]!.row.seq && x.row.seq <= end.row.seq
&& x.e.sourceKind === "runner" && x.e.turnId === end.e.turnId
&& x.e.normalizedSessionId === end.e.normalizedSessionId && x.e.sourceInstanceId === end.e.sourceInstanceId
&& (x === end || x.p.status === "running")), "incomplete or unsuccessful native tool lifecycle");
return starts[0]!;
&& (x === end || (x.p.status === "running"
&& ["tool.execution.started", "tool.execution.progressed"].includes(x.e.eventType)))), "incomplete or unsuccessful native tool lifecycle");
// Native arguments arrive incrementally. An unresolved target is allowed
// only before the first exact target; conflicting or regressed targets fail.
let resolved = false;
for (const x of matches) {
requireEvidence(x.p.target === end.p.target || (!resolved && x.p.target === null), "conflicting native tool target");
if (x.p.target !== null) resolved = true;
}
return { start: starts[0]!, matches };
}
lifecycle(edit);
const validations = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "execute"
&& x.p.name === c.validationCommand && typeof x.p.output === "string" && x.p.output.includes(c.validationMarker));
&& x.p.name === "bash");
requireEvidence(validations.length === 1, "one observed provider validation execution");
const validation = validations[0]!, start = lifecycle(validation);
const validation = validations[0]!, { start, matches } = lifecycle(validation);
requireEvidence(start.row.seq > edit.row.seq && validation.e.turnId === edit.e.turnId
&& validation.e.normalizedSessionId === edit.e.normalizedSessionId
&& validation.e.sourceInstanceId === edit.e.sourceInstanceId && validation.p.outputTruncated === false,
"validation must follow the edit in the same native turn");
// The common lifecycle preserves the opening name (bash). ACPX's resolved
// command is retained as progress, rather than replacing that stable name.
const commandProgress = `${c.validationCommand} (in_progress): ${c.validationCommand}`;
const allowedProgress = new Set([`bash (pending): [terminal] ${validation.p.executionId}`,
`${c.validationCommand} (pending): ${c.validationCommand}`, commandProgress]);
requireEvidence(matches.some(x => x !== validation && x.p.progress === commandProgress)
&& matches.every(x => x.p.progress === null || allowedProgress.has(x.p.progress)),
"exact correlated command progress before completion required");
requireEvidence(typeof validation.p.output === "string" && Buffer.byteLength(validation.p.output) <= 16_384,
"bounded native validation receipt required");
let output: Row = {};
try { output = rec(JSON.parse(validation.p.output)); } catch { /* Rejected below; plain marker text is insufficient. */ }
const structured = rec(output.structuredContent), expectedOutput = `${c.validationMarker}\n`;
requireEvidence(structured.exit_code === 0 && structured.truncated === false && structured.output === expectedOutput
&& Array.isArray(output.content) && output.content.length === 1
&& rec(output.content[0]).type === "text" && rec(output.content[0]).text === expectedOutput
&& (validation.p.exitCode === null || validation.p.exitCode === 0), "successful native structured validation receipt required");
const attachments = input.attachments.map(rec).filter(a => a.id === input.attachmentId);
requireEvidence(attachments.length === 1, "one public attachment");
const a = attachments[0]!;
@@ -144,11 +172,13 @@ export function gradePiFileEvidence(input: {
validationExecutionId: validation.p.executionId, attachmentId: a.id,
verification: { kind: "independent_exact_bytes", beforeSha256: sha(c.before), afterSha256: c.sha256,
downloadedSha256: sha(input.downloadedBytes), byteSize: c.byteSize, providerExecutionStatus: validation.p.status,
commandTitle: validation.p.name, nativeFileTarget: edit.p.target,
command: c.validationCommand, providerToolName: validation.p.name, commandEvidence: "correlated_native_progress",
exitEvidence: "native_structured_output_receipt", nativeStructuredExitCode: structured.exit_code,
typedExitCode: validation.p.exitCode, nativeFileTarget: edit.p.target,
nativeFileAttribution: "workspace_relative_display_target" },
diff: { source: "independent_seed_and_workspace_bytes", path: c.filename,
text: `--- ${c.filename}\n+++ ${c.filename}\n@@ -1 +1 @@\n-${c.before.trimEnd()}\n+${c.after.trimEnd()}\n` },
limits: ["Validation command title is checked exactly; raw arguments and typed exitCode are not projected, so raw invocation/exit code are not attested.",
limits: ["Exact command progress and the native structured output receipt are checked; raw arguments and nullable typed exitCode are not attested by those distinct fields.",
"Diff is computed from independently checked workspace bytes; it does not qualify native diff presentation or private invocation metadata."],
};
}