diff --git a/doc/architecture/runner-copilot-capabilities.md b/doc/architecture/runner-copilot-capabilities.md index 8294856b486..59085255094 100644 --- a/doc/architecture/runner-copilot-capabilities.md +++ b/doc/architecture/runner-copilot-capabilities.md @@ -2,8 +2,9 @@ Audited 2026-09-28 against repository base `c65fc9e3c81c41aafe421aa90a00514b84343285`. Status: **candidate, not qualified**. Authenticated model discovery and exact model -selection now pass; real inference and Daytona execution remain unverified. -Real executable offline probes cost $0; no model prompt has been sent. The allocated live budget remains $25, subject to +selection now pass. One canonical local protocol case has passed through the real +runner with authenticated semantic tools. Product E2E and Daytona remain pending. +Real executable offline probes cost $0; the live token receipt has no verified USD charge. The allocated live budget remains $25, subject to the shared $100 hard stop and verifiable spend. Do not expose this profile as supported until the required local and Daytona qualification passes. @@ -344,6 +345,29 @@ The initial paid qualification reservation is $2 within the provider's $25 allocation and shared $100 budget. The account dashboard baseline is 0 of 1,500 included AI credits; additional paid usage is disabled with a $0 cash budget. Included credit consumption must still be reconciled and reported separately -from cash charges. One canonical `get-task-context` case with a 60-second turn, -zero retries, and a $0.50 declared envelope is next. Neither that envelope nor -Copilot's optional soft credit limit is a guaranteed per-response hard cap. +from cash charges. The [sanitized first live proof](../../packages/paperclip-runner/test/fixtures/copilot-canonical-live-proof-2026-09-28.json) +records one `get-task-context` turn completed in 33.383 seconds: one authenticated +semantic call and successful result, completed terminal, complete transcript and +mock-only boundary all pass the unchanged canonical scorer. The first attempt +failed before prompting because a stale Rust binary omitted credential binding; +the fresh source-built release binary fixed admission. The live attempt's +post-turn package-provenance lookup failed, so the same retained artifact was +scored offline after regenerating the exact source tarball. Both original +failure records remain inspectable; recovery sent no additional model prompt. +The sibling launcher now validates that tarball before paid work. + +The receipt reports 24,258 input tokens, 11,781 cached input tokens and 441 output +tokens. Its $0.00326022 catalog estimate is not a GitHub charge. The runner's +`providerRequests: 1` is a terminal receipt count, not an observed count of +upstream HTTP requests. GitHub still displayed 0/1,500 credits after this small +turn; lag or rounding prevents exact credit attribution. Additional paid usage remains disabled with a $0 budget; that billing boundary +does not imply zero consumption of included credits. + +The pinned CLI documents `--max-ai-credits` as a soft cap, minimum 30 credits. +Its ACP startup/session implementation does not propagate that option into +`sessionLimits`, unlike the native interactive/server paths; ACP exposes no +budget configuration option. Treat provider-enforced per-session credit limits +as unavailable through this pinned integration (P1 follow-up: upstream ACP limit +support). The 60-second deadline, $0.50 declared envelope and $2 reservation do +not become a per-response hard cap. All profiles remain pending until the full +local and Daytona qualification succeeds. diff --git a/packages/paperclip-runner/test/fixtures/copilot-canonical-live-proof-2026-09-28.json b/packages/paperclip-runner/test/fixtures/copilot-canonical-live-proof-2026-09-28.json new file mode 100644 index 00000000000..7e9f398e6bc --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-canonical-live-proof-2026-09-28.json @@ -0,0 +1,79 @@ +{ + "schema": "paperclip.copilot-canonical-live-proof/v1", + "providerVersion": "1.0.88", + "model": "gpt-5.6-luna", + "modelProvider": "github", + "qualificationStatus": "pending", + "sourceRevision": "92fcaf0c721a7adb3bc4dcf7a9ee6e6784b81e71", + "runnerdSourceRevision": "f77ae83aeb6d802c73f1c955760017ae272eba29", + "runnerdSha256": "sha256:986060810ba7377c6ddd64a5d89e322d0a434e1a9c80400e6d4acf5934bfc421", + "providerPackDigest": "sha256:172199bd72d7cf171344dd1e4b9eca240f8bbe2969c5e7df30c38c717820f890", + "evalRevision": "b01676fdb7a2783f7b3635a90ba9583041e86338", + "caseId": "get-task-context", + "startedAt": "2026-09-28T19:29:25.165Z", + "durationMs": 33383, + "promptRequestsSent": 1, + "upstreamModelRequestCount": null, + "semanticOperations": [ + { + "operationId": "get_task_context", + "calls": 1, + "successfulResults": 1 + } + ], + "terminalStatus": "completed", + "runnerExited": true, + "credentialLeakDetected": false, + "usage": { + "agentTurns": 1, + "providerRequests": 1, + "inputTokens": 24258, + "outputTokens": 441, + "cachedInputTokens": 11781, + "reasoningTokens": 0, + "providerReportedCostNanodollars": null, + "providerReportedCostProvenance": "unavailable", + "estimatedCostNanodollars": 3260220, + "pricingVersion": "provider-list-prices-2026-09-07-openrouter", + "ratesUsdPerMillionTokens": { + "input": 0.2, + "cachedInput": 0.02, + "output": 1.2 + }, + "costCoverage": "estimated" + }, + "usageCountProvenance": "providerRequests=1 is the runner terminal receipt count; actual provider HTTP request count is unverified", + "costReconciliation": { + "dashboardCreditsUsed": 0, + "dashboardCreditsTotal": 1500, + "additionalUsageEnabled": false, + "cashBudgetUsd": 0, + "creditDelta": null, + "note": "Dashboard still shows zero after this small call; billing lag or rounding prevents exact credit attribution. The catalog estimate is not a GitHub USD receipt." + }, + "originalArtifactSha256": "sha256:49690c56fe0cb956e7fc6c946601d345667d1f8e7f91b551a7547df93203c75f", + "originalRosterExitCode": 1, + "offlineScore": { + "passed": true, + "checksPassed": 4, + "checksTotal": 4, + "artifactSha256": "sha256:17e3148fc1526574c59cb14377e5c28bec6305cb85847e61f5cdff43d0cbb644", + "semanticDigest": "sha256:19da8536da1f821121e15b7dc328c0f7557b19bfc52828c1c4bb4b3e16cff037", + "additionalProviderCalls": 0, + "reason": "Post-turn provenance packaging failed; exact source tarball rebuilt, same retained runtime artifact scored offline. Original attempt and recovery metadata preserved." + }, + "priorAttempt": { + "stage": "session.open", + "result": "infrastructure_failure_before_prompt", + "cause": "stale Rust artifact lacked explicit credential binding propagation", + "promptRequestsSent": 0 + }, + "pendingQualification": [ + "Product E2E local and Daytona", + "restrictive permissions denial with no side effects", + "interactive questions and applicable plan decisions", + "warm continuation and recovery", + "background settlement", + "exact credit reconciliation" + ] +}