diff --git a/doc/architecture/runner-pi-capabilities.md b/doc/architecture/runner-pi-capabilities.md index bb14b9828ce..00111e02f86 100644 --- a/doc/architecture/runner-pi-capabilities.md +++ b/doc/architecture/runner-pi-capabilities.md @@ -1,9 +1,11 @@ # Pi rich ACP runtime -Status: implementation candidate, 2026-09-28. The local Product hello-complete -journey passes with real Pi, semantic MCP completion, terminal tokens and measured -billing. Deterministic protocol and immutable launch checks also pass. The earlier -Runner protocol timeout and Product infrastructure failures remain retained. +Status: implementation candidate, 2026-09-28. Profile version 3 passes local +Product file delivery and typed question continuation, plus a native steering, +queued follow-up and denied-write probe. The semantic plan journey found a shared +MCP identity projection defect and remains failed. Version 2 hello completion and +all earlier failures remain retained. Deterministic protocol and immutable launch +checks also pass. The wider local and Linux x64 Daytona matrix remains pending; this document does not promote the candidate to a qualified production runtime. @@ -27,7 +29,7 @@ boundaries explicit. | Questions | Pi `select`, `confirm`, `input`, and `editor` map to ACP form elicitation with typed schemas. Decline, cancellation, timeout, malformed replies, and duplicate/late replies cannot become accepted answers. | | Semantic tools | Runner-bound HTTP MCP catalogs (numeric loopback HTTP or assigned HTTPS gateways) register under exact `mcp____` names. Calls retain the native tool call ID and cancellation signal. Authenticated PRP tool handling owns semantic authorization and durable interactions. Only the exact session-assigned gateway URL and credential are used, with redirects disabled. Ambient and unassigned MCP servers are not admitted. | | Plans and artifacts | Pi has no native structured plan or artifact channel. Paperclip plan and artifact semantic tools remain available through the MCP bridge; native file edits retain bounded, workspace-confined ACP diff projection. Tool text/image results are preserved, and resource blocks are recorded without following URLs. | -| Steering | Capability-negotiated `pi/steer` issues native RPC `steer` during an active turn. `pi/follow_up` explicitly queues native RPC `follow_up`. Neither is inferred from a second ACP prompt. Each takes `{sessionId, message}` and returns `{accepted: true}`. | +| Steering | Capability-negotiated `pi/steer` issues native RPC `steer` during an active turn. `pi/follow_up` explicitly queues native RPC `follow_up`. Neither is inferred from a second ACP prompt. Each takes `{sessionId, message}` and returns `{accepted: true, sessionId, kind}` with the matching control kind. | | Usage | Prompt results sum actual assistant message usage receipts across continuations. Input, output, cache reads/writes, total tokens and Pi-reported pricing estimates have provenance. Context-window occupancy is not billed usage. No receipt means no usage assertion; absent cache or cost fields remain unknown. Pi calculates cost from its model catalog rates, so this is not an authoritative provider bill. | | Retry and compaction | Upstream retry/compaction notices are retained. `agent_settled`, rather than a transient `agent_end`, settles a prompt. A final provider error remains a failed prompt and does not become successful completion. | | Images | Upstream ACP image prompt blocks are passed to native Pi RPC. Model-specific image support still requires live qualification. | @@ -332,18 +334,22 @@ every field in Pi's native event stream. These are explicit follow-ups: ## Authenticated local qualification progress The [sanitized attempt ledger](../../packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json) -retains all twelve attempts, including seven failures before a model prompt. One +retains all sixteen attempts, including seven failures before a model prompt. One Runner protocol turn reached the exact model and successfully called `get_task_context`, `get_task_history`, `list_documents`, and `read_document`. The canonical case permits those extra orientation reads, but also requires a completed turn; it timed out after 120s without terminal usage. The later Product hello journey passes; the canonical Runner protocol case has not yet passed. -Exclusive OpenRouter key billing increased by **$0.017750126**, including +Exclusive OpenRouter key billing increased by **$0.025720049**, including $0.000351509 from the sleep-interrupted Product attempt and $0.000654767 from the successful hello journey, plus $0.010995043 across the restrictive-denial and file-edit batch. A delayed charge crossed the latter attempt baselines, so -that batch is accounted as an aggregate without per-generation attribution. The seven pre-prompt failures had two post-attempt +that batch is accounted as an aggregate without per-generation attribution. +Version 3 file, question and failed plan cases add $0.007828938. The version 3 +native controls probe adds $0.000140985 after a later key reading resolves its +initially unchanged observations; those early reads did not prove free inference. +The seven pre-prompt failures had two post-attempt observations with zero delta. This is measured provider spend, separate from Pi's stale catalog pricing estimate. A missing terminal receipt remains unknown, not free execution. Each attempt had no automatic retries, a $2 reservation and a $0.50 billing watchdog; @@ -433,9 +439,9 @@ The Product path enforces this comparison before prompting, but its retained identity. The separate proof closes model-selection evidence without pretending it came from that Product transcript. -The current pack, model and hello proof JSON files retain Git tree hashes for +The historical version 2 pack, model and hello proof JSON files retain Git tree hashes for Runner source, scripts, Rust, protocol, patches, server source and Product E2E. -The pack runtime stays pinned to `dd78df1e` when later commits change only these +Those version 2 observations stay pinned to `dd78df1e` when later commits change only these reports and fixtures; an execution-source change requires a fresh build. | Execution source path | Git tree at `dd78df1e` | @@ -491,3 +497,56 @@ The [file-edit failure](../../packages/paperclip-runner/test-fixtures/pi-acp/pro retains the candidate timeout, five generic finish errors, missing file oracle, unknown terminal usage and successful process cleanup. These failures are kept alongside the version 3 corrections rather than overwritten by a future retry. +## Version 3 live evidence + +The [file delivery proof](../../packages/paperclip-runner/test-fixtures/pi-acp/product-file-proof.darwin-arm64.json) +passes all seven canonical matchers. The independent oracle reads the exact +24-byte workspace file before cleanup. The first finish request fails because a +workspace-only file is not downloadable; the retained validation message tells +Pi to register the deliverable. Pi does so and then finishes successfully. The +browser shows the download card and Done status. All five Bash results are +non-null; canonical exit code remains unknown. The case measured $0.003324737. +No separate raw copy of the workspace file was retained. + +The [typed question proof](../../packages/paperclip-runner/test-fixtures/pi-acp/product-question-proof.darwin-arm64.json) +passes all six matchers. The browser submits option ID `cobalt` for question ID +`verification-word`; the same durable interaction becomes answered and a distinct +warm continuation completes. The pending form and final answer were visually +inspected. This is continuation without server restart. The waiting run has no +usage receipt, while the continuation has one; total token coverage remains +partial. The measured key delta is $0.001413697. + +The [plan failure](../../packages/paperclip-runner/test-fixtures/pi-acp/product-plan-failure.darwin-arm64.json) +retains a complete two-step Plan at revision 1 and an unanswered confirmation +bound to that exact revision. The provider run succeeds and the task remains in +review. The real browser trace shows `task-chat-plan-preview-fallback` inside a +settled turn. Rust had retained `mcp__paperclip__write_document` as a builtin tool +with no namespace, so the UI could not embed the Plan at its `write_document` +boundary. The meaningful canonical matcher remains unchanged. No approval was +submitted, terminal usage is unavailable, and the measured key delta is +$0.003090504. Shared repair `5aa02662b` restores MCP name/namespace/transport while preserving +operation authority. Its negative fixture retains the observed fallback, and the +positive settled-turn UI regression passes. A fresh daemon and new source-pinned +live run are still required. + +The [native controls proof](../../packages/paperclip-runner/test-fixtures/pi-acp/native-controls-proof.darwin-arm64.json) +submits one ACP prompt to the actual selected model. While its native write waits +for permission, both explicit controls acknowledge the same active session. The +original offered denial ID is persisted and fsynced before the pipe response. +Visible output is exactly `STEERED_CURRENT` followed by `QUEUED_NEXT`; a matching +failed tool update and independent absent-file check prove the denied write. +The prompt settles and a later steering request is rejected as stale. The native +receipt reports 1,941 input, 134 output and 3,584 cache-read tokens. Its +$0.000409612 catalog estimate differs from the later measured $0.000140985 key +delta. This proves consumed +controls on a live process, not durable queue replay after process replacement. + +All four observations use immutable pack `eb31cada…` at source `7710736ca…`, +with runtime source `06cf356a8…`; they predate the combined provider integration +`7ab463697…`. Execution tree hashes are retained in the Product proofs. The +[Linux x64 proof](../../packages/paperclip-runner/test-fixtures/pi-acp/offline-native-proof.linux-x64.v3.json) +uses profile version 3 at source `06cf356a8…`, image +`sha256:c5fa7976bba92a186a2f70dc8b8ddc58ab86606b80a819c89bf550d2d057c832`. +It launches the verified native wrapper, initializes ACP protocol 1 and exits +cleanly on EOF with network disabled and no credentials or model prompts. This +proves Linux executable admission; authenticated Daytona behavior remains pending. diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/native-controls-proof.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/native-controls-proof.darwin-arm64.json new file mode 100644 index 00000000000..ff8e9c07267 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/native-controls-proof.darwin-arm64.json @@ -0,0 +1,123 @@ +{ + "schema": "paperclip.pi-native-controls-proof.v1", + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "actualSelectedModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "modelPromptRequests": 1, + "visibleText": "STEERED_CURRENTQUEUED_NEXT", + "staleSteerRejected": true, + "permissionRequests": 1, + "settlement": { + "stopReason": "end_turn", + "usage": { + "inputTokens": 1941, + "outputTokens": 134, + "cachedReadTokens": 3584, + "cachedWriteTokens": 0, + "totalTokens": 5659, + "_meta": { + "paperclipPi": { + "provenance": "assistant_message_receipts", + "costUsd": 0.00040961200000000006, + "costSource": "pi_pricing_estimate" + } + } + }, + "metadata": null + }, + "oracle": { + "markerExistedBefore": false, + "markerExistsAfter": false, + "method": "independent filesystem stat before cleanup" + }, + "limits": [ + "Direct ACP active steering, queued native follow-up and owned extension denial; durable PRP restart replay is not exercised.", + "Pipe write acknowledgement is followed by the matching failed tool update and settled prompt." + ], + "attemptId": "pi-native-controls-01", + "runtimeCodeRevision": "06cf356a8bc94f709fcd15606fe05e17933fe3b2", + "profileVersion": 3, + "environment": { + "platform": "darwin", + "architecture": "arm64", + "execution": "direct-native-acp" + }, + "qualificationStatus": "pending", + "controls": [ + { + "method": "pi/steer", + "kind": "steer", + "accepted": true, + "activePromptPending": true, + "ackBeforePermissionResponse": true, + "activeSessionIdentityMatched": true + }, + { + "method": "pi/follow_up", + "kind": "follow_up", + "accepted": true, + "activePromptPending": true, + "ackBeforePermissionResponse": true, + "activeSessionIdentityMatched": true + } + ], + "decision": { + "count": 1, + "toolTitle": "Pi write", + "originalOfferedOptionIds": [ + "allow_once", + "allow_always", + "reject_once" + ], + "selectedOriginalOptionId": "reject_once", + "persistedAndFsyncedBeforeResponse": true, + "pipeWriteAcknowledged": true, + "matchingFailedToolUpdate": true, + "rawRequestIdsRetainedPrivately": true + }, + "result": { + "status": "passed", + "durationSeconds": 21.342, + "exitCode": 0, + "stopReason": null + }, + "cost": { + "catalogEstimateUsd": 0.00040961200000000006, + "costSource": "pi_pricing_estimate", + "providerReportedUsd": null, + "observedKeyDeltaUsd": 0.000140985, + "attributedProviderSpendUsd": 0.000140985, + "billingReconciliation": "Later exclusive-key observation increased after unchanged initial reads; early and later observations retained separately.", + "cumulativeObservedExclusiveKeyDeltaUsd": 0.025720049, + "earlyObservedKeyDeltaUsd": 0 + }, + "supervision": { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "source": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "packDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeDeadlineSeconds": 120, + "outerDeadlineSeconds": 180, + "deadlineClocks": [ + "wall", + "monotonic" + ], + "sleepAssertion": "caffeinate -i", + "automaticRetries": 0, + "providerPromptMaximum": 1 + }, + "cleanup": { + "childExited": true, + "commandLeaseClosed": true, + "temporaryWorkspaceRemoved": true, + "residualAttemptProcesses": 0 + }, + "publicationBoundary": { + "hiddenReasoningIncluded": false, + "rawTranscriptsIncluded": false, + "providerSessionIdsIncluded": false, + "credentialsIncluded": false + } +} diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/offline-native-proof.linux-x64.v3.json b/packages/paperclip-runner/test-fixtures/pi-acp/offline-native-proof.linux-x64.v3.json new file mode 100644 index 00000000000..41029c4b0b4 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/offline-native-proof.linux-x64.v3.json @@ -0,0 +1,42 @@ +{ + "provider": "pi", + "target": { + "platform": "linux", + "architecture": "x64" + }, + "sourceSha": "06cf356a8bc94f709fcd15606fe05e17933fe3b2", + "packDigest": "sha256:5d4cf665692088c34b90f46fc37d943e63dbf81cdfb0bed890b626d145aeebf0", + "candidate": { + "version": "0.84.2", + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "closureDigest": "sha256:002812b62463ffae84b8e8b58567a6f4ba5025dd830528c5ed3b4c50c2a0ee29", + "qualification": "pending", + "path": "provider-assets/pi/linux-x64", + "sha256": "sha256:1122af73027d3824b3cc964e4d85f80dd6c28216300be7f12842786732d15b72" + }, + "commandDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "runnerdSha256": "7935dd1ab411c0e4a577fbad0d7dc432b5bc396cc349f2fd597fa97c28a039a7", + "protocolVersion": 1, + "nativeInfo": { + "name": "pi-acp", + "title": "pi ACP adapter", + "version": "0.0.33" + }, + "exitCode": 0, + "eofSettled": true, + "prompts": 0, + "network": "none", + "credentials": false, + "schema": "paperclip.pi-linux-native-admission-proof.v1", + "imageDigest": "sha256:c5fa7976bba92a186a2f70dc8b8ddc58ab86606b80a819c89bf550d2d057c832", + "imageName": "ghcr.io/paperclipai/paperclip-daytona-runner:e2e-content-14a042557b61ea6e83046aa67c2d10028ed6a21101444ee6adc9d1c69b3d27db", + "profileVersion": 3, + "sourceBoundary": "Pi-only committed source06cf, before combined provider stack. No authenticated session or Daytona allocation.", + "qualificationStatus": "pending", + "publicationBoundary": { + "hiddenReasoningIncluded": false, + "rawTranscriptsIncluded": false, + "providerSessionIdsIncluded": false, + "credentialsIncluded": false + } +} diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json index 009f2334be6..d7738b868ff 100644 --- a/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json +++ b/packages/paperclip-runner/test-fixtures/pi-acp/paid-qualification-progress.darwin-arm64.json @@ -1,6 +1,6 @@ { "schema": "paperclip.pi-paid-qualification-progress.v1", - "recordedAt": "2026-09-28T21:12:21.059145+00:00", + "recordedAt": "2026-09-28T21:34:16.545526+00:00", "qualificationStatus": "pending", "environment": { "platform": "darwin", @@ -17,7 +17,7 @@ "budget": { "authorizedMaximumUsd": 25, "reservedPerBatchUsd": 2, - "measuredSpendUsd": 0.017750126, + "measuredSpendUsd": 0.025720049, "measurement": "exclusive OpenRouter key billing delta; model catalog estimates are not provider billing", "billingGroups": [ { @@ -31,7 +31,8 @@ "delayedDeltaBetweenAttemptWindowsUsd": 0.000105107, "billingObservedThrough": "2026-09-28T20:55:09Z" } - ] + ], + "pendingBillingAttemptIds": [] }, "attempts": [ { @@ -428,6 +429,189 @@ "terminalUsage": null, "evidence": "product-file-failure.darwin-arm64.json", "diagnosis": "Five semantic finish requests failed with generic bridge error; active deadline expired without file oracle or terminal receipt. Version 3 retains bounded validation details and native Bash output." + }, + { + "attemptId": "pi-product-file-02", + "family": "product-e2e", + "caseId": "extended-harnesses.runner-acpx-pi.local.file-edit-validate", + "startedAt": "2026-09-28T21:17:12.291736+00:00", + "finishedAt": "2026-09-28T21:18:19.632350+00:00", + "durationSeconds": 67.34, + "exitCode": 0, + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "submittedProviderTurns": null, + "providerRequestCount": null, + "billingDeltaUsd": 0.003324737, + "billingCoverage": "exclusive OpenRouter key delta, two spaced post-attempt observations", + "retainedStatus": "passed", + "retainedMachineDisposition": null, + "cleanup": "passed", + "terminalUsage": { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "biller": "openrouter", + "provider": "openrouter", + "costStatus": "unpriced", + "billingType": "unknown", + "inputTokens": 14461, + "outputTokens": 3604, + "cachedInputTokens": 116736, + "usageSource": "per_run" + }, + "evidence": "product-file-proof.darwin-arm64.json", + "diagnosis": "File oracle passed; retained MCP validation enabled deliverable registration and successful finish.", + "supervision": { + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 300, + "sleepAssertion": "caffeinate -i", + "deadlineClocks": [ + "wall", + "monotonic" + ], + "automaticRetries": 0 + } + }, + { + "attemptId": "pi-product-question-01", + "family": "product-e2e", + "caseId": "extended-harnesses.runner-acpx-pi.local.question-resume-complete", + "startedAt": "2026-09-28T21:20:00.032734+00:00", + "finishedAt": "2026-09-28T21:22:55.721708+00:00", + "durationSeconds": 175.689, + "exitCode": 0, + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "submittedProviderTurns": null, + "providerRequestCount": null, + "billingDeltaUsd": 0.001413697, + "billingCoverage": "exclusive OpenRouter key delta, two spaced post-attempt observations", + "retainedStatus": "passed", + "retainedMachineDisposition": null, + "cleanup": "passed", + "terminalUsage": { + "runCount": 2, + "byRun": [ + null, + { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "biller": "openrouter", + "provider": "openrouter", + "costStatus": "unpriced", + "billingType": "unknown", + "inputTokens": 17894, + "outputTokens": 1891, + "cachedInputTokens": 19200, + "usageSource": "per_run", + "sessionReused": true + } + ], + "coverage": "partial: waiting run receipt unavailable; resumed run receipt retained" + }, + "evidence": "product-question-proof.darwin-arm64.json", + "diagnosis": "Typed semantic question answered with original question and option identity; warm continuation finished. Waiting-run usage remains missing.", + "supervision": { + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 300, + "sleepAssertion": "caffeinate -i", + "deadlineClocks": [ + "wall", + "monotonic" + ], + "automaticRetries": 0 + } + }, + { + "attemptId": "pi-product-plan-01", + "family": "product-e2e", + "caseId": "extended-harnesses.runner-acpx-pi.local.plan-approve-complete", + "startedAt": "2026-09-28T21:23:40.821883+00:00", + "finishedAt": "2026-09-28T21:25:59.888902+00:00", + "durationSeconds": 139.067, + "exitCode": 1, + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "submittedProviderTurns": null, + "providerRequestCount": null, + "billingDeltaUsd": 0.003090504, + "billingCoverage": "exclusive OpenRouter key delta, two spaced post-attempt observations", + "retainedStatus": "failed", + "retainedMachineDisposition": "candidate_failure", + "cleanup": "passed", + "terminalUsage": null, + "evidence": "product-plan-failure.darwin-arm64.json", + "diagnosis": "Plan and exact revision confirmation persisted, but Rust MCP identity loss placed its UI card in fallback; unchanged meaningful oracle failed.", + "supervision": { + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 300, + "sleepAssertion": "caffeinate -i", + "deadlineClocks": [ + "wall", + "monotonic" + ], + "automaticRetries": 0 + } + }, + { + "attemptId": "pi-native-controls-01", + "family": "native-acp", + "caseId": "active-steer-queued-follow-up-and-denied-write", + "startedAt": "2026-09-28T21:32:45.580333+00:00", + "finishedAt": "2026-09-28T21:33:06.922601+00:00", + "durationSeconds": 21.342, + "exitCode": 0, + "automaticRetries": 0, + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "submittedProviderTurns": 1, + "providerRequestCount": null, + "billingDeltaUsd": 0.000140985, + "billingCoverage": "exclusive OpenRouter key delta; delayed charge observed after unchanged initial readings", + "retainedStatus": "passed", + "retainedMachineDisposition": null, + "cleanup": "passed", + "terminalUsage": { + "inputTokens": 1941, + "outputTokens": 134, + "cachedReadTokens": 3584, + "cachedWriteTokens": 0, + "totalTokens": 5659, + "_meta": { + "paperclipPi": { + "provenance": "assistant_message_receipts", + "costUsd": 0.00040961200000000006, + "costSource": "pi_pricing_estimate" + } + } + }, + "evidence": "native-controls-proof.darwin-arm64.json", + "diagnosis": "Active steer and native queued follow-up were acknowledged and consumed in order; original native write denied, no marker, terminal settlement, stale steer rejected.", + "supervision": { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "source": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "packDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeDeadlineSeconds": 120, + "outerDeadlineSeconds": 180, + "deadlineClocks": [ + "wall", + "monotonic" + ], + "sleepAssertion": "caffeinate -i", + "automaticRetries": 0, + "providerPromptMaximum": 1 + }, + "earlyObservedKeyDeltaUsd": 0 } ], "observedSemanticOperations": { @@ -484,6 +668,17 @@ "actualSelectedCurrentModelId": "openrouter/deepseek/deepseek-v4-flash-0731", "promptRequests": 0, "billingDeltaUsd": 0 + }, + "version3": { + "source": "06cf356a8bc94f709fcd15606fe05e17933fe3b2", + "packSource": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "typescriptBuild": "passed", + "releaseRunnerBuild": "passed", + "vitestPassed": 95, + "installedPackageChecksPassed": 25, + "nativeRustTestsPassed": 16, + "offlineProof": "offline-provider-pack-proof.darwin-arm64.json", + "linuxInitializeProof": "offline-native-proof.linux-x64.v3.json" } }, "publicationBoundary": { @@ -494,9 +689,9 @@ }, "remaining": [ "Successful canonical Runner protocol completion", - "File edit, typed question, plan and restart Product journeys", - "Version 3 native permission regression, durable restart replay and active steering/follow-up delivery", - "Complete price/billing coverage, separate from catalog estimates", - "Linux x64 Daytona qualification" + "Semantic plan approval with canonical write boundary, pending-input restart, and refreshed version3 Product hello", + "Durable native pending-request reconnect/recovery beyond direct ACP controls", + "Complete token and price/billing coverage, separate from catalog estimates", + "Linux x64 Daytona authenticated qualification" ] } diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/product-file-proof.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/product-file-proof.darwin-arm64.json new file mode 100644 index 00000000000..2ca25f6644d --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/product-file-proof.darwin-arm64.json @@ -0,0 +1,182 @@ +{ + "schema": "paperclip.pi-product-file-proof.v1", + "attemptId": "pi-product-file-02", + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "repositoryRevision": "9cd42f6df9356cb673cd12accd94dc3e171dc13d", + "runtimeCodeRevision": "06cf356a8bc94f709fcd15606fe05e17933fe3b2", + "profileVersion": 3, + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "runnerdDigest": "sha256:2c8efdc99f988b1c8ab0e270677a3321af0fd6d5789dbf1b671faa921031b42f", + "executionTrees": { + "packages/paperclip-runner/src": "7df6b47574e6838e63a7d13c403c615a749702db", + "packages/paperclip-runner/scripts": "f845a2dd5b1ac2f1a4ef8d95db8fba2daea9999f", + "packages/paperclip-runner/runner": "9c595f7eae18197bb5cfd61c1af463df0a8741cd", + "packages/paperclip-runner/protocol": "1d50458fb572d0c75f9a398f419209c52c23ba6f", + "patches": "784152935b091eed6ea1795d39c23cfbbd154ab7", + "server/src": "b267adc1e0ad36955b68314829cfd4256ab88d37", + "tests/runner-e2e": "5ff4b8a85a14e3e005e3e240c145899c0bc742b5" + }, + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "environment": { + "platform": "darwin", + "architecture": "arm64", + "execution": "local" + }, + "result": { + "status": "passed", + "durationMs": 49427, + "supervisorDurationSeconds": 67.34, + "matcherResults": [ + { + "matcher": { + "kind": "message_exact", + "expected": "EXTENDED-FILE-ede290404669-1" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "message_occurrences", + "expected": "EXTENDED-FILE-ede290404669-1", + "count": 1 + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "issue_status", + "expected": "done" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "run_status", + "expected": "succeeded" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "runtime_mode", + "expected": "native" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "environment", + "expected": "local" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "file_exact", + "path": "extended-ede290404669-1.txt", + "expected": "verified-ede290404669-1\n" + }, + "passed": true, + "detail": "matched" + } + ], + "cleanup": "passed" + }, + "fileOracle": { + "path": "extended-ede290404669-1.txt", + "expected": "verified-ede290404669-1\n", + "expectedBytesSha256": "0987a044fc5ad449b2b96210c1b7dcb1205431c353cb87c34366618c7438ece7", + "byteLength": 24, + "method": "Canonical matcher read actual workspace file before cleanup and compared exact UTF-8 contents.", + "rawArtifactCopyRetained": false + }, + "semanticCorrection": { + "failedFinishCalls": 1, + "retainedValidationError": "Paperclip tool request was rejected: Completion cites a workspace-only file that the user cannot download. Before finishing, use register_deliverable for requested file outputs and cite deliverable: from the receipt, with /api/attachments//content as the download link. For repository changes, cite an accessible PR or registered work product instead. No human completion approval was created.", + "registeredDeliverableCalls": 1, + "registrationDisposition": "applied", + "filePreparationState": "prepared", + "downloadPathShape": "/api/attachments//content?download=1", + "externalChatDeliveryConfirmed": false, + "successfulFinishCalls": 1 + }, + "bashProjection": { + "terminalToolCalls": 5, + "allResultsRetained": true, + "canonicalExitCode": "unknown", + "statuses": [ + "completed", + "completed", + "completed", + "failed", + "completed" + ] + }, + "terminal": { + "schema": "paperclip.prp.terminal.v1", + "runTerminalState": "succeeded", + "turnTerminalState": "completed", + "reportedWorkDisposition": "done" + }, + "usage": { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "biller": "openrouter", + "provider": "openrouter", + "costStatus": "unpriced", + "billingType": "unknown", + "inputTokens": 14461, + "outputTokens": 3604, + "cachedInputTokens": 116736, + "usageSource": "per_run" + }, + "cost": { + "catalogEstimateUsd": null, + "providerReportedUsd": null, + "costStatus": "unpriced", + "billingComplete": false, + "exclusiveKeyDeltaUsd": 0.003324737, + "measurement": "Exclusive key baseline and two spaced post-attempt observations, separate from model catalog estimates." + }, + "screenshot": { + "sha256": "741695cf889df6755c2b02c70ec71b445723a3289953bfe9dd3333caa91c7cf0", + "visuallyInspected": true, + "observed": [ + "Done status", + "one final completion marker", + "24 B file download card and file attachment", + "no error banner" + ], + "imageRetainedPrivately": true + }, + "supervision": { + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 300, + "sleepAssertion": "caffeinate -i", + "deadlineClocks": [ + "wall", + "monotonic" + ], + "automaticRetries": 0 + }, + "cleanup": { + "canonicalCleanupPassed": true, + "fixtureRootRemoved": true, + "residualAttemptProcesses": 0 + }, + "qualificationStatus": "pending", + "publicationBoundary": { + "credentialsIncluded": false, + "hiddenReasoningIncluded": false, + "rawTranscriptsIncluded": false, + "providerSessionIdsIncluded": false + } +} diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/product-plan-failure.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/product-plan-failure.darwin-arm64.json new file mode 100644 index 00000000000..bb1f7caa450 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/product-plan-failure.darwin-arm64.json @@ -0,0 +1,123 @@ +{ + "schema": "paperclip.pi-product-plan-failure.v1", + "attemptId": "pi-product-plan-01", + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "runtimeCodeRevision": "06cf356a8bc94f709fcd15606fe05e17933fe3b2", + "repositoryRevision": "9cd42f6df9356cb673cd12accd94dc3e171dc13d", + "profileVersion": 3, + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "runnerdDigest": "sha256:2c8efdc99f988b1c8ab0e270677a3321af0fd6d5789dbf1b671faa921031b42f", + "executionTrees": { + "packages/paperclip-runner/src": "7df6b47574e6838e63a7d13c403c615a749702db", + "packages/paperclip-runner/scripts": "f845a2dd5b1ac2f1a4ef8d95db8fba2daea9999f", + "packages/paperclip-runner/runner": "9c595f7eae18197bb5cfd61c1af463df0a8741cd", + "packages/paperclip-runner/protocol": "1d50458fb572d0c75f9a398f419209c52c23ba6f", + "patches": "784152935b091eed6ea1795d39c23cfbbd154ab7", + "server/src": "b267adc1e0ad36955b68314829cfd4256ab88d37", + "tests/runner-e2e": "5ff4b8a85a14e3e005e3e240c145899c0bc742b5" + }, + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "environment": { + "platform": "darwin", + "architecture": "arm64", + "execution": "local" + }, + "result": { + "status": "failed", + "failureClass": "candidate_failure", + "durationMs": 119553, + "supervisorDurationSeconds": 139.067, + "failedBoundary": "Native saved Plan did not appear at its canonical write_document boundary within30s.", + "matcherResults": [], + "cleanup": "passed" + }, + "semanticState": { + "runStatus": "succeeded", + "nativePhase": "committed", + "issueStatus": "in_review", + "plan": { + "key": "plan", + "revision": 1, + "body": "# OpenRouter breadth plan bed573ffa86f-1\n\n1. Define the OpenRouter breadth scope and success criteria, and record the ready marker PC_P_READY_bed573ffa86f-1 in this canonical Plan.\n2. Submit this Plan for approval via a confirmation request bound to this exact Plan revision; do not implement anything before that revision is accepted.", + "revisionIdSha256": "a0b4b8055d87bf25d8e44aedab42b4439416dfd5e13af8d6e0e34d600412dccf" + }, + "interaction": { + "kind": "request_confirmation", + "status": "pending", + "targetType": "issue_document", + "targetKey": "plan", + "targetRevisionMatchesWriteReceipt": true, + "decisionSubmitted": false + } + }, + "canonicalToolEvent": { + "name": "mcp__paperclip__write_document", + "status": "completed", + "transport": "builtin", + "namespace": null, + "outputDigest": "sha256:1bf0af100a60fbf97b9e97e4e946be3ae2ff1dc0ff56e9f69e0a0d96a30d9425" + }, + "retainedDom": { + "matchingSnapshots": 4, + "planTestId": "task-chat-plan-preview-fallback", + "ariaLabel": "Open Plan revision 1", + "ancestorTurnTestId": "task-chat-turn", + "ancestorSettled": "true", + "expectedPlanTestId": "task-chat-plan-preview", + "oracleRemainsUnchanged": true + }, + "diagnosis": { + "cause": "Rust ACPX projection hardcodes builtin transport and null namespace while preserving mcp__paperclip__write_document as the name; UI exact write_document matching therefore creates a fallback Plan.", + "productionSeam": "packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs:normalize_acpx_tool_call", + "presentationSeam": "ui/src/components/task-chat/transcript-adapter.ts:embedPlanDocumentAtWriteBoundary", + "selectorDefect": false, + "followupFix": { + "commit": "5aa02662b34fc70f5fe9ffd5ac8f7ae81b5fe3bd", + "scope": "shared Rust MCP identity normalization plus negative/positive settled UI boundary regression", + "verification": "43 ACP boundary tests, 12 provider units and 158 TaskChatThread tests reported by owning agent", + "liveRetry": "pending; no new prompt" + } + }, + "usage": null, + "cost": { + "providerReportedUsd": null, + "terminalUsageAvailable": false, + "exclusiveKeyDeltaUsd": 0.003090504, + "measurement": "Exclusive key baseline and two spaced post-attempt observations; not inferred from the missing usage receipt." + }, + "screenshot": { + "sha256": "9d1e8b5892f620f0ff94895eefba22c2d3cc57ff5ec5a79d20c419ebbef38e13", + "visuallyInspected": true, + "observed": [ + "Plan card visible", + "Complete two-step Plan visible in side pane", + "Pending Confirm and Request changes controls" + ], + "imageRetainedPrivately": true + }, + "supervision": { + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 300, + "sleepAssertion": "caffeinate -i", + "deadlineClocks": [ + "wall", + "monotonic" + ], + "automaticRetries": 0 + }, + "cleanup": { + "canonicalCleanupPassed": true, + "fixtureRootRemoved": true, + "residualAttemptProcesses": 0 + }, + "qualificationStatus": "pending", + "publicationBoundary": { + "credentialsIncluded": false, + "hiddenReasoningIncluded": false, + "rawTranscriptsIncluded": false, + "providerSessionIdsIncluded": false + } +} diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/product-question-proof.darwin-arm64.json b/packages/paperclip-runner/test-fixtures/pi-acp/product-question-proof.darwin-arm64.json new file mode 100644 index 00000000000..5ab8fc6d694 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/product-question-proof.darwin-arm64.json @@ -0,0 +1,196 @@ +{ + "schema": "paperclip.pi-product-question-proof.v1", + "attemptId": "pi-product-question-01", + "sourceRevision": "7710736ca3924c655c5b0efd172cfd3c0173766a", + "runtimeCodeRevision": "06cf356a8bc94f709fcd15606fe05e17933fe3b2", + "repositoryRevision": "9cd42f6df9356cb673cd12accd94dc3e171dc13d", + "profileVersion": 3, + "profileDigest": "sha256:72cb225288376f733b9ed3afa5e13565eb4152f0de509bc1181382fa44bee472", + "providerPackDigest": "sha256:eb31cadae93805b901d2565912121fca6e79024c0120b1bd7982e05215b5aa5a", + "runnerdDigest": "sha256:2c8efdc99f988b1c8ab0e270677a3321af0fd6d5789dbf1b671faa921031b42f", + "executionTrees": { + "packages/paperclip-runner/src": "7df6b47574e6838e63a7d13c403c615a749702db", + "packages/paperclip-runner/scripts": "f845a2dd5b1ac2f1a4ef8d95db8fba2daea9999f", + "packages/paperclip-runner/runner": "9c595f7eae18197bb5cfd61c1af463df0a8741cd", + "packages/paperclip-runner/protocol": "1d50458fb572d0c75f9a398f419209c52c23ba6f", + "patches": "784152935b091eed6ea1795d39c23cfbbd154ab7", + "server/src": "b267adc1e0ad36955b68314829cfd4256ab88d37", + "tests/runner-e2e": "5ff4b8a85a14e3e005e3e240c145899c0bc742b5" + }, + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "environment": { + "platform": "darwin", + "architecture": "arm64", + "execution": "local" + }, + "result": { + "status": "passed", + "durationMs": 158388, + "supervisorDurationSeconds": 175.689, + "matcherResults": [ + { + "matcher": { + "kind": "message_exact", + "expected": "PC_Q_C_05746e362d67-1" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "message_occurrences", + "expected": "PC_Q_C_05746e362d67-1", + "count": 1 + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "issue_status", + "expected": "done" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "run_status", + "expected": "succeeded" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "runtime_mode", + "expected": "native" + }, + "passed": true, + "detail": "matched" + }, + { + "matcher": { + "kind": "environment", + "expected": "local" + }, + "passed": true, + "detail": "matched" + } + ], + "cleanup": "passed" + }, + "interaction": { + "kind": "ask_user_questions", + "title": "Verification word", + "initialStatus": "pending", + "resolvedStatus": "answered", + "continuationPolicy": "wake_assignee", + "questions": [ + { + "id": "verification-word", + "prompt": "Choose the verification word.", + "selectionMode": "single", + "required": true, + "options": [ + { + "id": "cobalt", + "label": "Cobalt" + }, + { + "id": "amber", + "label": "Amber" + } + ] + } + ], + "result": { + "version": 1, + "answers": [ + { + "questionId": "verification-word", + "optionIds": [ + "cobalt" + ] + } + ], + "summaryMarkdown": null + }, + "requestIdentityPreserved": true, + "distinctContinuationRun": true, + "serverRestartedBeforeAnswer": false + }, + "usage": { + "runCount": 2, + "byRun": [ + null, + { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "biller": "openrouter", + "provider": "openrouter", + "costStatus": "unpriced", + "billingType": "unknown", + "inputTokens": 17894, + "outputTokens": 1891, + "cachedInputTokens": 19200, + "usageSource": "per_run", + "sessionReused": true + } + ], + "coverage": "partial: waiting run receipt unavailable; resumed run receipt retained" + }, + "cost": { + "providerReportedUsd": null, + "costStatus": "unpriced", + "billingComplete": false, + "exclusiveKeyDeltaUsd": 0.001413697, + "measurement": "Exclusive key baseline and two spaced post-attempt observations; separate from missing terminal usage." + }, + "screenshot": { + "sha256": "35138e6ef99fe566c2aadce7594f1e582b00b71207214d6d576633e33f1741ee", + "visuallyInspected": true, + "observed": [ + "Done status", + "Questions answered", + "Cobalt answer bubble", + "one completion marker" + ], + "imageRetainedPrivately": true + }, + "supervision": { + "reservationUsd": 2, + "billingWatchdogUsd": 0.5, + "activeTimeoutSeconds": 120, + "outerTimeoutSeconds": 300, + "sleepAssertion": "caffeinate -i", + "deadlineClocks": [ + "wall", + "monotonic" + ], + "automaticRetries": 0 + }, + "cleanup": { + "canonicalCleanupPassed": true, + "fixtureRootRemoved": true, + "residualAttemptProcesses": 0 + }, + "qualificationStatus": "pending", + "publicationBoundary": { + "credentialsIncluded": false, + "hiddenReasoningIncluded": false, + "rawTranscriptsIncluded": false, + "providerSessionIdsIncluded": false + }, + "pendingScreenshot": { + "sha256": "7e94f6db31750b64c5d7a9e60c6ab22364f8af627738db3a7a21cffbb9a85a1b", + "visuallyInspected": true, + "observed": [ + "In Review status", + "Verification word form", + "Cobalt and Amber choices plus freeform Other", + "Submit answers disabled until required selection", + "Cancel action visible" + ], + "imageRetainedPrivately": true + } +}