From 7710736ca3924c655c5b0efd172cfd3c0173766a Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 28 Sep 2026 16:13:23 -0500 Subject: [PATCH] Retain Pi denial proof and failed file qualification accounting Preserve the original version 2 source, missing file oracle and terminal receipt, and aggregate billing attribution across the delayed charge. Co-Authored-By: Paperclip --- doc/architecture/runner-pi-capabilities.md | 20 ++- .../native-denial-proof.darwin-arm64.json | 121 ++++++++++++++++++ ...d-qualification-progress.darwin-arm64.json | 85 +++++++++++- .../product-file-failure.darwin-arm64.json | 90 +++++++++++++ 4 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 packages/paperclip-runner/test-fixtures/pi-acp/native-denial-proof.darwin-arm64.json create mode 100644 packages/paperclip-runner/test-fixtures/pi-acp/product-file-failure.darwin-arm64.json diff --git a/doc/architecture/runner-pi-capabilities.md b/doc/architecture/runner-pi-capabilities.md index aeb25ab096e..aef70da9df6 100644 --- a/doc/architecture/runner-pi-capabilities.md +++ b/doc/architecture/runner-pi-capabilities.md @@ -324,16 +324,18 @@ every field in Pi's native event stream. These are explicit follow-ups: ## Authenticated local qualification progress The [sanitized attempt ledger](../../packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json) -retains all ten attempts, including seven failures before a model prompt. One +retains all twelve attempts, including seven failures before a model prompt. One Runner protocol turn reached the exact model and successfully called `get_task_context`, `get_task_history`, `list_documents`, and `read_document`. The canonical case permits those extra orientation reads, but also requires a completed turn; it timed out after 120s without terminal usage. The later Product hello journey passes; the canonical Runner protocol case has not yet passed. -Exclusive OpenRouter key billing increased by **$0.006755083**, including +Exclusive OpenRouter key billing increased by **$0.017750126**, including $0.000351509 from the sleep-interrupted Product attempt and $0.000654767 -from the successful hello journey. The seven pre-prompt failures had two post-attempt +from the successful hello journey, plus $0.010995043 across the restrictive-denial +and file-edit batch. A delayed charge crossed the latter attempt baselines, so +that batch is accounted as an aggregate without per-generation attribution. The seven pre-prompt failures had two post-attempt observations with zero delta. This is measured provider spend, separate from Pi's stale catalog pricing estimate. A missing terminal receipt remains unknown, not free execution. Each attempt had no automatic retries, a $2 reservation and a $0.50 billing watchdog; @@ -469,3 +471,15 @@ are rejected. Earlier local hello, restrictive-denial and Linux initialization proofs remain tied to their version 2 execution source. Version 3 must complete fresh immutable-pack checks and paid qualification before those behaviors can be claimed for its bytes. + + +The [restrictive-denial proof](../../packages/paperclip-runner/test-fixtures/pi-acp/native-denial-proof.darwin-arm64.json) +retains the version 2 real native write rejection: the original offered decision +was durably recorded before responding, the matching tool failed, the prompt +settled, and an independent filesystem check found no forbidden marker. This is +direct ACP delivery proof; it does not exercise PRP restart replay. + +The [file-edit failure](../../packages/paperclip-runner/test-fixtures/pi-acp/product-file-failure.darwin-arm64.json) +retains the candidate timeout, five generic finish errors, missing file oracle, +unknown terminal usage and successful process cleanup. These failures are kept +alongside the version 3 corrections rather than overwritten by a future retry. diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/native-denial-proof.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/native-denial-proof.darwin-arm64.json new file mode 100644 index 00000000000..d3502462644 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/native-denial-proof.darwin-arm64.json @@ -0,0 +1,121 @@ +{ + "schema": "paperclip.pi-native-denial-proof.v1", + "sourceRevision": "dd78df1ef8b279c30c710c9b7a7f9fda22e321d6", + "providerPackDigest": "sha256:f75d3de7814276508f3706edb6e4510ce1dcecbc3e8fa674991fd554e5a75273", + "profileDigest": "sha256:0f687e38cb3c607a01fab80f03e19bbbf5cb53a8afb1fc40d71f9ab54551cff1", + "actualSelectedModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "modelPromptRequests": 1, + "permissionRequests": 1, + "decisions": [ + { + "requestId": 0, + "toolTitle": "Pi write", + "options": [ + { + "optionId": "allow_once", + "name": "Allow once", + "kind": "allow_once" + }, + { + "optionId": "allow_always", + "name": "Allow for this session", + "kind": "allow_always" + }, + { + "optionId": "reject_once", + "name": "Deny", + "kind": "reject_once" + } + ], + "decisionOptionId": "reject_once", + "persistedBeforeResponse": true, + "pipeWriteAcknowledged": true, + "toolCallIdSha256": "caa61c37ddbe3dddc3a077f55a4450da3c70e4f98e8913e8a229bb3fc507307b", + "durableOriginalDecisionRecordSha256": "68ab7a1eb4cf5ea29417223f66448b3cba53d2dc4c37c5de3617c52804379436" + } + ], + "providerFailedToolUpdates": [ + { + "status": "failed", + "toolCallIdSha256": "caa61c37ddbe3dddc3a077f55a4450da3c70e4f98e8913e8a229bb3fc507307b" + } + ], + "settlement": { + "stopReason": "end_turn", + "usage": { + "inputTokens": 1735, + "outputTokens": 125, + "cachedReadTokens": 1792, + "cachedWriteTokens": 0, + "totalTokens": 3652, + "_meta": { + "paperclipPi": { + "provenance": "assistant_message_receipts", + "costUsd": 0.00032807600000000006, + "costSource": "pi_pricing_estimate" + } + } + }, + "metadata": null + }, + "oracle": { + "markerExistedBefore": false, + "markerExistsAfter": false, + "method": "independent filesystem stat before cleanup" + }, + "limits": [ + "Direct ACP permission delivery and owned extension denial; durable PRP restart replay is not exercised.", + "Pipe write acknowledgement is followed by the matching failed tool update and settled prompt." + ], + "executionTrees": { + "packages/paperclip-runner/src": "a8123cb4396c77ce0127d5ccf49d3584b63a9a58", + "packages/paperclip-runner/scripts": "9ede97769bb3f2bbfa8e02be220b5ce06e438a99", + "packages/paperclip-runner/runner": "2cc84e150aecbd5faf2c8bcecc4b6352c24dbca8", + "packages/paperclip-runner/protocol": "1d50458fb572d0c75f9a398f419209c52c23ba6f", + "patches": "1e2f846b8b2dda202b79ee83baeb0860a3b9ab64", + "server/src": "b267adc1e0ad36955b68314829cfd4256ab88d37", + "tests/runner-e2e": "5ff4b8a85a14e3e005e3e240c145899c0bc742b5" + }, + "environment": { + "platform": "darwin", + "architecture": "arm64", + "execution": "local immutable ACP wrapper and owned extension" + }, + "identityHandling": "Original JSON-RPC id, toolCallId and offered optionId were persisted privately before responding. Public toolCallId hashes preserve request/update equality without exposing invocation identifiers.", + "cost": { + "catalogEstimateUsd": 0.000328076, + "estimateSource": "pi_pricing_estimate", + "observedKeyDeltaBeforeNextCaseUsd": 0, + "providerBillVerified": false, + "status": "accounted_in_aggregate", + "explanation": "Three immediate post-probe billing observations were unchanged, followed by a delayed increase before the file case. All spend is accounted in the combined exclusive-key delta; no free-inference or per-generation attribution claim.", + "aggregateBillingGroup": "pi-denial-file-01", + "aggregateDenialAndFileDeltaUsd": 0.010995043 + }, + "supervision": { + "reservationUsd": 2, + "combinedDenialAndFileReservationUsd": 4, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 180, + "deadlineClocks": [ + "wall", + "monotonic" + ], + "sleepAssertion": "caffeinate -i", + "automaticRetries": 0 + }, + "cleanup": { + "wrapperClosed": true, + "immutableSnapshotClosed": true, + "workspaceRemoved": true, + "residualAttemptProcesses": 0 + }, + "qualificationStatus": "pending", + "publicationBoundary": { + "credentialsIncluded": false, + "hiddenReasoningIncluded": false, + "rawTranscriptsIncluded": false, + "providerSessionIdsIncluded": false + } +} diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json index 0461bed26a9..009f2334be6 100644 --- a/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json +++ b/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json @@ -1,6 +1,6 @@ { "schema": "paperclip.pi-paid-qualification-progress.v1", - "recordedAt": "2026-09-28T20:39:38.907308+00:00", + "recordedAt": "2026-09-28T21:12:21.059145+00:00", "qualificationStatus": "pending", "environment": { "platform": "darwin", @@ -17,8 +17,21 @@ "budget": { "authorizedMaximumUsd": 25, "reservedPerBatchUsd": 2, - "measuredSpendUsd": 0.006755083, - "measurement": "exclusive OpenRouter key billing delta; model catalog estimates are not provider billing" + "measuredSpendUsd": 0.017750126, + "measurement": "exclusive OpenRouter key billing delta; model catalog estimates are not provider billing", + "billingGroups": [ + { + "id": "pi-denial-file-01", + "attemptIds": [ + "pi-native-denial-01", + "pi-product-file-01" + ], + "measuredDeltaUsd": 0.010995043, + "attribution": "Exclusive key aggregate; delayed charge crossed attempt baselines, so per-generation attribution is unavailable.", + "delayedDeltaBetweenAttemptWindowsUsd": 0.000105107, + "billingObservedThrough": "2026-09-28T20:55:09Z" + } + ] }, "attempts": [ { @@ -354,6 +367,67 @@ "automaticRetries": 0 }, "evidence": "product-hello-proof.darwin-arm64.json" + }, + { + "attemptId": "pi-native-denial-01", + "family": "native-acp", + "caseId": "restrictive-native-write-denial", + "startedAt": "2026-09-28T20:47:09.907468+00:00", + "finishedAt": "2026-09-28T20:47:30.345865+00:00", + "durationSeconds": 20.438, + "exitCode": 0, + "sourceRevision": "dd78df1ef8b279c30c710c9b7a7f9fda22e321d6", + "providerPackDigest": "sha256:f75d3de7814276508f3706edb6e4510ce1dcecbc3e8fa674991fd554e5a75273", + "profileDigest": "sha256:0f687e38cb3c607a01fab80f03e19bbbf5cb53a8afb1fc40d71f9ab54551cff1", + "submittedProviderTurns": 1, + "providerRequestCount": null, + "billingDeltaUsd": null, + "billingCoverage": "accounted in aggregate pi-denial-file-01; no per-generation attribution", + "billingGroupId": "pi-denial-file-01", + "automaticRetries": 0, + "retainedStatus": "passed", + "retainedMachineDisposition": null, + "cleanup": "passed", + "terminalUsage": { + "inputTokens": 1735, + "outputTokens": 125, + "cachedReadTokens": 1792, + "cachedWriteTokens": 0, + "totalTokens": 3652, + "_meta": { + "paperclipPi": { + "provenance": "assistant_message_receipts", + "costUsd": 0.00032807600000000006, + "costSource": "pi_pricing_estimate" + } + } + }, + "evidence": "native-denial-proof.darwin-arm64.json", + "diagnosis": "Original offered deny choice persisted before response; matching native write failed, prompt settled, and independent marker stat proved no write occurred." + }, + { + "attemptId": "pi-product-file-01", + "family": "product-e2e", + "caseId": "extended-harnesses.runner-acpx-pi.local.file-edit-validate", + "startedAt": "2026-09-28T20:49:00.303832+00:00", + "finishedAt": "2026-09-28T20:51:41.406899+00:00", + "durationSeconds": 161.103, + "exitCode": 1, + "sourceRevision": "dd78df1ef8b279c30c710c9b7a7f9fda22e321d6", + "providerPackDigest": "sha256:f75d3de7814276508f3706edb6e4510ce1dcecbc3e8fa674991fd554e5a75273", + "profileDigest": "sha256:0f687e38cb3c607a01fab80f03e19bbbf5cb53a8afb1fc40d71f9ab54551cff1", + "submittedProviderTurns": 1, + "providerRequestCount": null, + "billingDeltaUsd": null, + "billingCoverage": "accounted in aggregate pi-denial-file-01; no per-generation attribution", + "billingGroupId": "pi-denial-file-01", + "automaticRetries": 0, + "retainedStatus": "failed", + "retainedMachineDisposition": "candidate_failure", + "cleanup": "passed", + "terminalUsage": null, + "evidence": "product-file-failure.darwin-arm64.json", + "diagnosis": "Five semantic finish requests failed with generic bridge error; active deadline expired without file oracle or terminal receipt. Version 3 retains bounded validation details and native Bash output." } ], "observedSemanticOperations": { @@ -401,7 +475,8 @@ "deployedPackCredentialFreeProbe": "passed", "fullCredentialFreeRunnerStartup": "terminal rejection within30s deadline; not authenticated success", "cleanDependencyResolution": "two independent runs from tracked manifest/config/patch inputs with pnpm9.15.4 --resolution-only --ignore-scripts --no-frozen-lockfile", - "cleanDependencyLockSha256": "2c46a68811ba1d504a41b6f4d3f642bc93f4a1c615097754c9c6486881dba8e6" + "cleanDependencyLockSha256": "2c46a68811ba1d504a41b6f4d3f642bc93f4a1c615097754c9c6486881dba8e6", + "scope": "historical version 2 verification, before version 3 bridge/output changes" }, "startupDiagnosis": "startup-diagnostic.darwin-arm64.json", "authenticatedModelNoPrompt": { @@ -420,7 +495,7 @@ "remaining": [ "Successful canonical Runner protocol completion", "File edit, typed question, plan and restart Product journeys", - "Native permissions and active steering/follow-up delivery", + "Version 3 native permission regression, durable restart replay and active steering/follow-up delivery", "Complete price/billing coverage, separate from catalog estimates", "Linux x64 Daytona qualification" ] diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/product-file-failure.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/product-file-failure.darwin-arm64.json new file mode 100644 index 00000000000..962f55425c3 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/product-file-failure.darwin-arm64.json @@ -0,0 +1,90 @@ +{ + "schema": "paperclip.pi-product-file-failure.v1", + "sourceRevision": "dd78df1ef8b279c30c710c9b7a7f9fda22e321d6", + "repositoryRevision": "06e3bae33be97d014f4d9add8d3a46a995568050", + "providerPackDigest": "sha256:f75d3de7814276508f3706edb6e4510ce1dcecbc3e8fa674991fd554e5a75273", + "profileDigest": "sha256:0f687e38cb3c607a01fab80f03e19bbbf5cb53a8afb1fc40d71f9ab54551cff1", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "environment": { + "platform": "darwin", + "architecture": "arm64", + "execution": "local" + }, + "attemptId": "pi-product-file-01", + "caseId": "extended-harnesses.runner-acpx-pi.local.file-edit-validate", + "result": { + "status": "failed", + "failureClass": "candidate_failure", + "errorCode": "native_session_interrupted", + "activeDeadlineMs": 120000, + "canonicalDurationMs": 138060, + "supervisorDurationSeconds": 161.103, + "terminalUsage": null, + "matcherResults": [], + "cleanup": "passed" + }, + "observations": { + "nativeToolCalls": 20, + "failedFinishCalls": 5, + "frozenCompletionContract": { + "revision": "1", + "criterionId": "objective" + }, + "finishErrorText": "Paperclip tool request was rejected", + "fileOracleCaptured": false, + "finalIssueState": "in_progress" + }, + "evidenceLimitations": [ + "The original five finish argument objects did not cross the privacy projection and are not retained.", + "The canonical fixture removed its workspace and provider session during cleanup; file bytes cannot be independently reconstructed.", + "No terminal token receipt exists. Canonical billing has unavailable status; numeric default fields are not proof of zero inference.", + "Private reasoning content is excluded." + ], + "defects": { + "semanticValidationDetails": "Owned MCP bridge discarded authenticated isError content; fixed in version 3.", + "nativeBashOutput": "Upstream private terminal metadata was not preserved in standard ACP fields; fixed in version 3.", + "canonicalExitCode": "Shared projection still reports unknown; no inference from prose or status." + }, + "cost": { + "aggregateBillingGroup": "pi-denial-file-01", + "aggregateDenialAndFileDeltaUsd": 0.010995043, + "attributableFileGenerationCostUsd": null, + "catalogEstimateUsd": null + }, + "supervision": { + "reservationUsd": 2, + "combinedReservationUsd": 4, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 300, + "deadlineClocks": [ + "wall", + "monotonic" + ], + "sleepAssertion": "caffeinate -i", + "automaticRetries": 0 + }, + "cleanup": { + "canonicalCleanupPassed": true, + "residualAttemptProcesses": 0, + "fixtureWorkspaceRemoved": true, + "providerSnapshotRemoved": true, + "providerScratchRemoved": true + }, + "executionTrees": { + "packages/paperclip-runner/src": "a8123cb4396c77ce0127d5ccf49d3584b63a9a58", + "packages/paperclip-runner/scripts": "9ede97769bb3f2bbfa8e02be220b5ce06e438a99", + "packages/paperclip-runner/runner": "2cc84e150aecbd5faf2c8bcecc4b6352c24dbca8", + "packages/paperclip-runner/protocol": "1d50458fb572d0c75f9a398f419209c52c23ba6f", + "patches": "1e2f846b8b2dda202b79ee83baeb0860a3b9ab64", + "server/src": "b267adc1e0ad36955b68314829cfd4256ab88d37", + "tests/runner-e2e": "5ff4b8a85a14e3e005e3e240c145899c0bc742b5" + }, + "qualificationStatus": "pending", + "publicationBoundary": { + "credentialsIncluded": false, + "hiddenReasoningIncluded": false, + "rawTranscriptsIncluded": false, + "providerSessionIdsIncluded": false + } +}