Retain first real Copilot semantic qualification evidence

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-28 15:17:09 -05:00
1 parent 08cabb1f2e
commit 986af2854d
2 files changed
+108 -5

No files matched your search

@@ -2,8 +2,9 @@
Audited 2026-09-28 against repository base `c65fc9e3c81c41aafe421aa90a00514b84343285`.
Status: **candidate, not qualified**. Authenticated model discovery and exact model
selection now pass; real inference and Daytona execution remain unverified.
Real executable offline probes cost $0; no model prompt has been sent. The allocated live budget remains $25, subject to
selection now pass. One canonical local protocol case has passed through the real
runner with authenticated semantic tools. Product E2E and Daytona remain pending.
Real executable offline probes cost $0; the live token receipt has no verified USD charge. The allocated live budget remains $25, subject to
the shared $100 hard stop and verifiable spend. Do not expose this profile as
supported until the required local and Daytona qualification passes.
@@ -344,6 +345,29 @@ The initial paid qualification reservation is $2 within the provider's $25
allocation and shared $100 budget. The account dashboard baseline is 0 of 1,500
included AI credits; additional paid usage is disabled with a $0 cash budget.
Included credit consumption must still be reconciled and reported separately
from cash charges. One canonical `get-task-context` case with a 60-second turn,
zero retries, and a $0.50 declared envelope is next. Neither that envelope nor
Copilot's optional soft credit limit is a guaranteed per-response hard cap.
from cash charges. The [sanitized first live proof](../../packages/paperclip-runner/test/fixtures/copilot-canonical-live-proof-2026-09-28.json)
records one `get-task-context` turn completed in 33.383 seconds: one authenticated
semantic call and successful result, completed terminal, complete transcript and
mock-only boundary all pass the unchanged canonical scorer. The first attempt
failed before prompting because a stale Rust binary omitted credential binding;
the fresh source-built release binary fixed admission. The live attempt's
post-turn package-provenance lookup failed, so the same retained artifact was
scored offline after regenerating the exact source tarball. Both original
failure records remain inspectable; recovery sent no additional model prompt.
The sibling launcher now validates that tarball before paid work.
The receipt reports 24,258 input tokens, 11,781 cached input tokens and 441 output
tokens. Its $0.00326022 catalog estimate is not a GitHub charge. The runner's
`providerRequests: 1` is a terminal receipt count, not an observed count of
upstream HTTP requests. GitHub still displayed 0/1,500 credits after this small
turn; lag or rounding prevents exact credit attribution. Additional paid usage remains disabled with a $0 budget; that billing boundary
does not imply zero consumption of included credits.
The pinned CLI documents `--max-ai-credits` as a soft cap, minimum 30 credits.
Its ACP startup/session implementation does not propagate that option into
`sessionLimits`, unlike the native interactive/server paths; ACP exposes no
budget configuration option. Treat provider-enforced per-session credit limits
as unavailable through this pinned integration (P1 follow-up: upstream ACP limit
support). The 60-second deadline, $0.50 declared envelope and $2 reservation do
not become a per-response hard cap. All profiles remain pending until the full
local and Daytona qualification succeeds.
@@ -0,0 +1,79 @@
{
"schema": "paperclip.copilot-canonical-live-proof/v1",
"providerVersion": "1.0.88",
"model": "gpt-5.6-luna",
"modelProvider": "github",
"qualificationStatus": "pending",
"sourceRevision": "92fcaf0c721a7adb3bc4dcf7a9ee6e6784b81e71",
"runnerdSourceRevision": "f77ae83aeb6d802c73f1c955760017ae272eba29",
"runnerdSha256": "sha256:986060810ba7377c6ddd64a5d89e322d0a434e1a9c80400e6d4acf5934bfc421",
"providerPackDigest": "sha256:172199bd72d7cf171344dd1e4b9eca240f8bbe2969c5e7df30c38c717820f890",
"evalRevision": "b01676fdb7a2783f7b3635a90ba9583041e86338",
"caseId": "get-task-context",
"startedAt": "2026-09-28T19:29:25.165Z",
"durationMs": 33383,
"promptRequestsSent": 1,
"upstreamModelRequestCount": null,
"semanticOperations": [
{
"operationId": "get_task_context",
"calls": 1,
"successfulResults": 1
}
],
"terminalStatus": "completed",
"runnerExited": true,
"credentialLeakDetected": false,
"usage": {
"agentTurns": 1,
"providerRequests": 1,
"inputTokens": 24258,
"outputTokens": 441,
"cachedInputTokens": 11781,
"reasoningTokens": 0,
"providerReportedCostNanodollars": null,
"providerReportedCostProvenance": "unavailable",
"estimatedCostNanodollars": 3260220,
"pricingVersion": "provider-list-prices-2026-09-07-openrouter",
"ratesUsdPerMillionTokens": {
"input": 0.2,
"cachedInput": 0.02,
"output": 1.2
},
"costCoverage": "estimated"
},
"usageCountProvenance": "providerRequests=1 is the runner terminal receipt count; actual provider HTTP request count is unverified",
"costReconciliation": {
"dashboardCreditsUsed": 0,
"dashboardCreditsTotal": 1500,
"additionalUsageEnabled": false,
"cashBudgetUsd": 0,
"creditDelta": null,
"note": "Dashboard still shows zero after this small call; billing lag or rounding prevents exact credit attribution. The catalog estimate is not a GitHub USD receipt."
},
"originalArtifactSha256": "sha256:49690c56fe0cb956e7fc6c946601d345667d1f8e7f91b551a7547df93203c75f",
"originalRosterExitCode": 1,
"offlineScore": {
"passed": true,
"checksPassed": 4,
"checksTotal": 4,
"artifactSha256": "sha256:17e3148fc1526574c59cb14377e5c28bec6305cb85847e61f5cdff43d0cbb644",
"semanticDigest": "sha256:19da8536da1f821121e15b7dc328c0f7557b19bfc52828c1c4bb4b3e16cff037",
"additionalProviderCalls": 0,
"reason": "Post-turn provenance packaging failed; exact source tarball rebuilt, same retained runtime artifact scored offline. Original attempt and recovery metadata preserved."
},
"priorAttempt": {
"stage": "session.open",
"result": "infrastructure_failure_before_prompt",
"cause": "stale Rust artifact lacked explicit credential binding propagation",
"promptRequestsSent": 0
},
"pendingQualification": [
"Product E2E local and Daytona",
"restrictive permissions denial with no side effects",
"interactive questions and applicable plan decisions",
"warm continuation and recovery",
"background settlement",
"exact credit reconciliation"
]
}