From 2663e4af7c54fd9d664770983f70c7bbfc853eb0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 28 Sep 2026 14:51:27 -0500 Subject: [PATCH] docs(runner): retain Copilot Product proof and accounting defect Co-Authored-By: Paperclip --- .../runner-copilot-capabilities.md | 37 ++++++++ ...copilot-product-live-proof-2026-09-28.json | 93 +++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 packages/paperclip-runner/test/fixtures/copilot-product-live-proof-2026-09-28.json diff --git a/doc/architecture/runner-copilot-capabilities.md b/doc/architecture/runner-copilot-capabilities.md index 59085255094..cb5678eb358 100644 --- a/doc/architecture/runner-copilot-capabilities.md +++ b/doc/architecture/runner-copilot-capabilities.md @@ -371,3 +371,40 @@ as unavailable through this pinned integration (P1 follow-up: upstream ACP limit support). The 60-second deadline, $0.50 declared envelope and $2 reservation do not become a per-response hard cap. All profiles remain pending until the full local and Daytona qualification succeeds. + + +## First local Product result and accounting defect (2026-09-28) + +The [retained local Product proof](../../packages/paperclip-runner/test/fixtures/copilot-product-live-proof-2026-09-28.json) +records `extended-harnesses.runner-acpx-copilot.local.hello-complete` from committed +source `bcc9c638a25b91b84065f12633f083bd4f7a689f`. The first attempt passed all six +behavior matchers in 38.436 seconds, with one provider turn, no automatic retry, +and successful cleanup. Browser evidence shows the exact completion marker once, +the task marked Done, the Copilot agent, and expandable tool activity. The active +turn deadline was 120 seconds within the existing $2 reservation. This is one +basic Product case, not full local or Daytona qualification. + +The original result is retained unchanged with digest +`sha256:92c8aff3960e443be0c009c891d49d285476c3d6b67999fe97bb323076c4dc8b`. +It exposed a shared accounting defect: model-family inference labeled the biller +`openai`, while missing provider cost became `costUsd: 0`, `costStatus: reported` +and `billing.complete: true`. Those fields are invalid accounting evidence. The +correct biller is GitHub, provider USD cost is unknown, and the retained behavioral +pass must not be interpreted as accounting qualification. A shared server fix is +in progress; paid follow-ups are paused pending that correction. The original +result also lacks source SHA fields; the independent launcher manifest records +the exact committed source above. + +After this case, GitHub displayed 1 of 1,500 included AI credits consumed across +the successful protocol and Product turns together. Exact per-run credit use +remains unknown. Additional usage is disabled, with $0 of the $0 cash budget +spent; included-credit consumption is reported separately. The Product receipt +contains 31,332 input, 15,451 cached input and 337 output tokens, without a verified +USD receipt. + +The shared eval CLI now preserves completed provider outcomes while returning a +separate nonzero accounting failure when cost coverage is unknown or its bound +is exceeded. The sibling scorer retains semantic assertions under +`accounting_failure`; roster and campaign orchestration stop subsequent queued +cases. External dashboard reconciliation does not override the CLI budget +result. Profiles remain pending until the outstanding qualification matrix passes. diff --git a/packages/paperclip-runner/test/fixtures/copilot-product-live-proof-2026-09-28.json b/packages/paperclip-runner/test/fixtures/copilot-product-live-proof-2026-09-28.json new file mode 100644 index 00000000000..2c36acffee3 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-product-live-proof-2026-09-28.json @@ -0,0 +1,93 @@ +{ + "schema": "paperclip.copilot-product-live-proof/v1", + "providerVersion": "1.0.88", + "model": "gpt-5.6-luna", + "modelProvider": "github", + "qualificationStatus": "pending", + "sourceRevision": "bcc9c638a25b91b84065f12633f083bd4f7a689f", + "runnerdSourceRevision": "f77ae83aeb6d802c73f1c955760017ae272eba29", + "runnerdSha256": "sha256:986060810ba7377c6ddd64a5d89e322d0a434e1a9c80400e6d4acf5934bfc421", + "executionMode": "source-mode Product E2E with build-owned verified native assets", + "caseId": "extended-harnesses.runner-acpx-copilot.local.hello-complete", + "environment": "local", + "startedAt": "2026-09-28T19:41:44.170Z", + "finishedAt": "2026-09-28T19:42:22.606Z", + "durationMs": 38436, + "attempt": 1, + "automaticRetries": 0, + "promptRequestsSent": 1, + "upstreamModelRequestCount": null, + "activeTurnDeadlineSeconds": 120, + "outerDeadlineSeconds": 600, + "reservationUsd": 2, + "behaviorStatus": "passed", + "matchers": [ + { + "kind": "message_exact", + "passed": true + }, + { + "kind": "message_occurrences", + "passed": true + }, + { + "kind": "issue_status", + "passed": true + }, + { + "kind": "run_status", + "passed": true + }, + { + "kind": "runtime_mode", + "passed": true + }, + { + "kind": "environment", + "passed": true + } + ], + "cleanup": "passed", + "visualReview": "Exact single completion marker visible; task Done; Copilot agent attribution and tool activity visible.", + "originalResultSha256": "sha256:92c8aff3960e443be0c009c891d49d285476c3d6b67999fe97bb323076c4dc8b", + "finalScreenshotSha256": "sha256:d5715c79b9c00ee1a87872999ab24f38e7d642c10e52f31ba183d3af2cdc74b9", + "sourceProvenanceNote": "Original Product result source.sha is null; exact committed source is retained in the independent private launcher manifest.", + "tokens": { + "inputTokens": 31332, + "cachedInputTokens": 15451, + "outputTokens": 337 + }, + "accounting": { + "providerReportedCostUsd": null, + "costCoverage": "unknown", + "originalResultUnchanged": true, + "originalErroneousFields": { + "usage.biller": "openai", + "usage.provider": "openai", + "usage.costUsd": 0, + "usage.costStatus": "reported", + "billing.complete": true + }, + "defect": "The shared server path classified the model family as its biller and converted absent cost to a reported zero. Behavior passes do not qualify accounting. Shared fix and verification are required." + }, + "externalBilling": { + "source": "parent verified GitHub billing dashboard after Product hello", + "displayedIncludedAiCreditsUsed": 1, + "displayedIncludedAiCreditsTotal": 1500, + "baselineDisplayedIncludedAiCreditsUsed": 0, + "combinedDisplayedCreditDelta": 1, + "exactPerRunCreditDelta": null, + "additionalUsageEnabled": false, + "additionalCashBudgetUsd": 0, + "additionalCashChargesUsd": 0, + "note": "Combined attribution covers the successful canonical protocol and Product hello turns; per-run and finer-grained credit usage remain unknown." + }, + "pendingQualification": [ + "Product local remaining semantic/edit/interactive cases", + "Product Daytona cases", + "restrictive permission denial before side effects", + "native background command settlement", + "warm continuation and restart recovery", + "post-fix accounting provenance and exact per-run credit reconciliation" + ] +}