diff --git a/doc/plans/2026-10-02-native-completion-calibrated-manifest.json b/doc/plans/2026-10-02-native-completion-calibrated-manifest.json index 00ef3c5687c..c16d5a5dbd1 100644 --- a/doc/plans/2026-10-02-native-completion-calibrated-manifest.json +++ b/doc/plans/2026-10-02-native-completion-calibrated-manifest.json @@ -266,8 +266,8 @@ "fixtureIdentityVerified": true, "onlyDeclaredElevenNativeSourcesDiffer": true, "fixedMasterContextVerified": true, - "providerCallsSoFar": null, - "liveStatus": "both_dispatched_no_graded_cells", + "providerCallsSoFar": 12, + "liveStatus": "all_original_cell_results_retained_final_audit_pending", "workflowRuns": { "candidate": { "id": 37098728980, @@ -304,5 +304,9 @@ "classification": "pre_provider_calibration_mock" } ], - "workflowSourceSha": "215586d127e97c9301d86e769a39a15c13298ca2" + "workflowSourceSha": "215586d127e97c9301d86e769a39a15c13298ca2", + "gradedPairs": 6, + "candidatePassedCells": 6, + "baselinePassedCells": 5, + "comparisonPath": "doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json" } diff --git a/doc/plans/2026-10-02-native-completion-calibrated-results/baseline-audit.json b/doc/plans/2026-10-02-native-completion-calibrated-results/baseline-audit.json new file mode 100644 index 00000000000..8d6f6af7dd2 --- /dev/null +++ b/doc/plans/2026-10-02-native-completion-calibrated-results/baseline-audit.json @@ -0,0 +1,1568 @@ +{ + "schema": "paperclip.native-completion.baseline-audit.v1", + "asOf": "2026-10-03T05:23:16.220052+00:00", + "workflowRunId": 37098815696, + "workflowSourceSha": "215586d127e97c9301d86e769a39a15c13298ca2", + "targetSourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "expectedCells": 6, + "completedCells": 6, + "pendingExecutionIds": [], + "rows": [ + { + "executionId": "native-completion.runner-acpx-claude.local.assigned-skill-explicit-invocation", + "profileId": "runner-acpx-claude", + "caseId": "assigned-skill-explicit-invocation", + "status": "passed", + "provider": "acpx", + "model": "claude-sonnet-5", + "attempt": 1, + "durationMs": 69791, + "cleanup": "passed", + "runtimeUsage": { + "provider": "local", + "agentRunDurationMs": 27643, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 10, + "outputTokens": 1338, + "cachedInputTokens": 162274, + "totalTokens": 163622, + "reportedCostUsd": 0, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 27643, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0, + "complete": true + }, + "failureClass": null, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "resultSha256": "6933dcb354f517653824b2c166bceef0f5c524ae30f915a9f4423276e53ebd15", + "sourceMatches": true, + "configMatches": true, + "expectedOneAttempt": true, + "originalFailedMatchers": [], + "originalMatcherCount": 14, + "persistedConfiguration": { + "provider": "acpx", + "model": "claude-sonnet-5", + "acpxAgent": "claude", + "lifecycleMode": "per_turn", + "codexPermissionMode": "never", + "acpxPermissionMode": "approve-all" + }, + "declaredModelQualification": { + "source": "qualified_runner_profile", + "qualificationId": "acpx:claude" + }, + "admission": { + "schema": "paperclip.native-completion-preflight.v1", + "passed": true, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "variant": "historical", + "providerCalls": 0, + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "fixtureFingerprint": "a50581c04f3d87843d69f92050712977f8e114e4c2f23f96f243fd5386119f27", + "manifestFingerprint": "b92f3ab7d561188bc09214433b264f2e7810e8ccdc237f68caa82e1aecc1811e", + "runnerdProvenance": { + "schema": "paperclip.native-completion-runnerd-provenance.v1", + "mode": "trusted_hosted_archive", + "passed": true, + "selectedPath": "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + "binarySha256": "9a007971dd5079bbc4d7c5bf6d385560c7e3fdedefb4990188836bbd5ae016a2", + "binaryBytes": 394443976, + "archiveMemberBytes": 394443976, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "archiveSha256": "d83acb6c73b9cf2256669bcf875d593aa18cfa941f07ad93ca92f30906047190", + "workflowRunId": "37098815696", + "workflowRunAttempt": "1", + "artifactName": "runner-e2e-build-e74ed61a69fbdd8b3a8f15dd6456bc3140246e33-37098815696-1", + "errors": [] + }, + "preflightSha256": "b323fe95681e946a7f1604f2a73c0b39197b400964d8da731337c0f7807a67f8", + "executedTypeScriptAssertions": 127, + "rustCalibrationExecutedInCell": false, + "failedGates": [], + "checksMatch": true + }, + "defaults": { + "passed": true, + "entryFile": "AGENTS.md", + "files": [ + { + "path": "AGENTS.md", + "sha256": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f", + "bytes": 4249 + } + ], + "budgets": { + "company": 1000, + "agent": 1000 + }, + "independentMatch": true + }, + "nativeAudit": { + "evidenceSource": "original-native-observation", + "originalNativeGradeAvailable": true, + "originalGradePassed": true, + "originalChecks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + } + ], + "originalIntegrity": [], + "runCount": 1, + "runStatus": "succeeded", + "nativeRun": true, + "issueStatus": "done", + "reportedDisposition": "done", + "runOwnershipMatches": true, + "retryOrContinuationObserved": false, + "expectedNamedTool": "paperclip_finish", + "namedStartCount": 1, + "matchingCompletionCount": 1, + "proposalSeq": [ + 75 + ], + "completionSeqs": [ + 77 + ], + "finalSeqs": [ + 99 + ], + "acceptedSeq": [ + 103 + ], + "terminalSeq": [ + 104 + ], + "independentNamedSequencePass": true, + "persistedProviderFinalMatches": true, + "finalMarkerCount": 1, + "finalMentionsOwner": false, + "finalMentionsAction": false, + "toolInvocationsAfterProposal": 0, + "blocker": null, + "workspaceChanged": false, + "documentCount": 1, + "interactionCount": 0, + "eventCount": 112 + }, + "perRunLedger": [ + { + "model": "claude-sonnet-5", + "provider": "openai", + "biller": "openai", + "billingType": "unknown", + "usageSource": "per_run", + "costStatus": "reported", + "costUsd": 0, + "cacheAdjustedCostUsd": 0, + "inputTokens": 10, + "outputTokens": 1338, + "cachedInputTokens": 162274 + } + ], + "runLedgerCostMatchesAggregate": true, + "documentAudit": { + "originalAllChecksPassed": true, + "originalChecks": [ + { + "id": "assigned-skill-present", + "passed": true, + "detail": "The task run must receive one pinned skill version through the public assignment state." + }, + { + "id": "skill-request-explicit", + "passed": true, + "detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference." + }, + { + "id": "skill-source-marker", + "passed": true, + "detail": "The assigned pinned skill source must contain the output marker." + }, + { + "id": "marker-not-in-request", + "passed": true, + "detail": "The output marker must originate from the assigned skill, not the task request or comments." + }, + { + "id": "single-durable-output", + "passed": true, + "detail": "Exactly one durable task document must contain the skill's marker." + }, + { + "id": "completed-task", + "passed": true, + "detail": "Final task status: done." + }, + { + "id": "successful-runs", + "passed": true, + "detail": "All recorded context-integrity runs must succeed." + } + ], + "documentCount": 1, + "savedDocuments": [ + { + "key": "context-integrity-output", + "hasRevision": true, + "containsMarker": true, + "bodySha256": "18c62978a266f6706ad41eb6054e42f6ddc2de09098d7b42da9861ac3525bf11" + } + ] + }, + "ledgerAudit": { + "runCount": 1, + "tokenReceiptCount": 1, + "reportedCostReceiptCount": 1, + "reportedCostUsd": 0, + "allOneRunReceipts": true, + "externalActualBillingKnown": false, + "runtimeMetered": false + }, + "evidenceHashes": { + "blob-report/report.zip": "c5b837c26c65c79bcbd5c1f67d8ee9fa67f2ec5862c59c99eaa75f233ab9486f", + "evidence-manifest.json": "47446c71c349c1f8982e2fe47d178d5df01328247550c1059d8a180e61c71e27", + "final-state.png": "3b227e1d8f403ac3c6ac66601e1bf5693c0afe82a785e43c836f23b71a4dbbea", + "html-report/data/a4f76fc714d2652b2df78f4838aa0019931091d4.png": "3b227e1d8f403ac3c6ac66601e1bf5693c0afe82a785e43c836f23b71a4dbbea", + "html-report/data/c9d42daa8139d3ae3fbb9ecdcee5d01dcc8a2ad3.json": "6933dcb354f517653824b2c166bceef0f5c524ae30f915a9f4423276e53ebd15", + "playwright-output/runner-native-completion-r-a812c-the-browser-and-public-APIs/attachments/final-state-58b43ab4b4432af5af042b396bc9729f665406ff.png": "3b227e1d8f403ac3c6ac66601e1bf5693c0afe82a785e43c836f23b71a4dbbea", + "playwright-output/runner-native-completion-r-a812c-the-browser-and-public-APIs/attachments/runner-e2e-result-976acf95b669a494f6b53b87655331d225de7a7b.json": "6933dcb354f517653824b2c166bceef0f5c524ae30f915a9f4423276e53ebd15", + "result.json": "6933dcb354f517653824b2c166bceef0f5c524ae30f915a9f4423276e53ebd15", + "snapshots/api-state.json": "1e15edc3c80caf8f0d4d233f416b0538792f15c5b13f7cedd36843e31082c8c8", + "snapshots/browser-diagnostics.json": "38b37eaf79a152d205e4b4d74cf598078d09ad14cd9e9e3a6e693f81b2f6c2d6", + "snapshots/chat-final-run-ledger.json": "e080fadf0574161e6a0cf0f2091dfb163f8806a3d6928223fa68ecf88ad1d107", + "snapshots/context-integrity.json": "4e36c40590094d428dacdc097dca65eab22ed829e4af93dd87b9b623486d02da", + "snapshots/fixtures.json": "5e73383bcc87014325b0c05fdfd0ef05bebd57272e79897f9b2eab26db4c537d", + "snapshots/native-completion.json": "c64b3d3da9edc3b2e1aa137ccbae821071c322f9bc4c1fd72adf1d5998b7e88a", + "snapshots/native-default-before-execution.json": "67b30b37debfa1d3b7ec4a34e306a2c78bd52eed7bffd173994f93fafe34cc9f" + } + }, + { + "executionId": "native-completion.runner-acpx-claude.local.native-blocked-report", + "profileId": "runner-acpx-claude", + "caseId": "native-blocked-report", + "status": "passed", + "provider": "acpx", + "model": "claude-sonnet-5", + "attempt": 1, + "durationMs": 69608, + "cleanup": "passed", + "runtimeUsage": { + "provider": "local", + "agentRunDurationMs": 28241, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 6, + "outputTokens": 1365, + "cachedInputTokens": 89624, + "totalTokens": 90995, + "reportedCostUsd": 0, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 28241, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0, + "complete": true + }, + "failureClass": null, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "resultSha256": "cdbd533c56af2b70ca9ef7650356396a45a64a6e8b9a6b370a3a50b8ea50e3fd", + "sourceMatches": true, + "configMatches": true, + "expectedOneAttempt": true, + "originalFailedMatchers": [], + "originalMatcherCount": 21, + "persistedConfiguration": { + "provider": "acpx", + "model": "claude-sonnet-5", + "acpxAgent": "claude", + "lifecycleMode": "per_turn", + "codexPermissionMode": "never", + "acpxPermissionMode": "approve-all" + }, + "declaredModelQualification": { + "source": "qualified_runner_profile", + "qualificationId": "acpx:claude" + }, + "admission": { + "schema": "paperclip.native-completion-preflight.v1", + "passed": true, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "variant": "historical", + "providerCalls": 0, + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "fixtureFingerprint": "a50581c04f3d87843d69f92050712977f8e114e4c2f23f96f243fd5386119f27", + "manifestFingerprint": "b92f3ab7d561188bc09214433b264f2e7810e8ccdc237f68caa82e1aecc1811e", + "runnerdProvenance": { + "schema": "paperclip.native-completion-runnerd-provenance.v1", + "mode": "trusted_hosted_archive", + "passed": true, + "selectedPath": "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + "binarySha256": "9a007971dd5079bbc4d7c5bf6d385560c7e3fdedefb4990188836bbd5ae016a2", + "binaryBytes": 394443976, + "archiveMemberBytes": 394443976, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "archiveSha256": "d83acb6c73b9cf2256669bcf875d593aa18cfa941f07ad93ca92f30906047190", + "workflowRunId": "37098815696", + "workflowRunAttempt": "1", + "artifactName": "runner-e2e-build-e74ed61a69fbdd8b3a8f15dd6456bc3140246e33-37098815696-1", + "errors": [] + }, + "preflightSha256": "9171ff4d7bd4caa3a73947ed1b024caf0f442834564a1bfbe6ff1f58211a389a", + "executedTypeScriptAssertions": 127, + "rustCalibrationExecutedInCell": false, + "failedGates": [], + "checksMatch": true + }, + "defaults": { + "passed": true, + "entryFile": "AGENTS.md", + "files": [ + { + "path": "AGENTS.md", + "sha256": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f", + "bytes": 4249 + } + ], + "budgets": { + "company": 1000, + "agent": 1000 + }, + "independentMatch": true + }, + "nativeAudit": { + "evidenceSource": "original-native-observation", + "originalNativeGradeAvailable": true, + "originalGradePassed": true, + "originalChecks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "exact-blocker", + "passed": true, + "detail": "The structured owner, exact action and whole-task scope are independently checked." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "visible-blocker-content", + "passed": true, + "detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + }, + { + "id": "no-deployment-or-file-work", + "passed": true, + "detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed." + } + ], + "originalIntegrity": [], + "runCount": 1, + "runStatus": "succeeded", + "nativeRun": true, + "issueStatus": "blocked", + "reportedDisposition": "blocked", + "runOwnershipMatches": true, + "retryOrContinuationObserved": false, + "expectedNamedTool": "paperclip_block", + "namedStartCount": 1, + "matchingCompletionCount": 1, + "proposalSeq": [ + 52 + ], + "completionSeqs": [ + 54 + ], + "finalSeqs": [ + 79 + ], + "acceptedSeq": [ + 83 + ], + "terminalSeq": [ + 84 + ], + "independentNamedSequencePass": true, + "persistedProviderFinalMatches": true, + "finalMarkerCount": 1, + "finalMentionsOwner": true, + "finalMentionsAction": true, + "toolInvocationsAfterProposal": 0, + "blocker": { + "owner": { + "kind": "external", + "name": "Release Owner" + }, + "unblockAction": "Grant deployment access", + "scope": "task_wide" + }, + "workspaceChanged": false, + "documentCount": 0, + "interactionCount": 0, + "eventCount": 92 + }, + "perRunLedger": [ + { + "model": "claude-sonnet-5", + "provider": "openai", + "biller": "openai", + "billingType": "unknown", + "usageSource": "per_run", + "costStatus": "reported", + "costUsd": 0, + "cacheAdjustedCostUsd": 0, + "inputTokens": 6, + "outputTokens": 1365, + "cachedInputTokens": 89624 + } + ], + "runLedgerCostMatchesAggregate": true, + "ledgerAudit": { + "runCount": 1, + "tokenReceiptCount": 1, + "reportedCostReceiptCount": 1, + "reportedCostUsd": 0, + "allOneRunReceipts": true, + "externalActualBillingKnown": false, + "runtimeMetered": false + }, + "evidenceHashes": { + "blob-report/report.zip": "11eeaa524f504ffd7e6b61e57b87fc30b5b9ded9132314379727bf98d21c2e10", + "evidence-manifest.json": "22908c3a9fd1d8fedd27b73ac920adebf0aaa900c403bc60681f416f0b7ba6b7", + "final-state.png": "3c9811d15b3c8c092f47600c1174a7788edeb3133c8ed60f6787f1ecab20c969", + "html-report/data/45d129d15dc8cff7da9b56d7904a9931b69ba7ad.png": "3c9811d15b3c8c092f47600c1174a7788edeb3133c8ed60f6787f1ecab20c969", + "html-report/data/7fef77446378c87f080beca69f1600275399dbcd.json": "cdbd533c56af2b70ca9ef7650356396a45a64a6e8b9a6b370a3a50b8ea50e3fd", + "playwright-output/runner-native-completion-r-141db-the-browser-and-public-APIs/attachments/final-state-76d2855631cee8e29995eb1fb96f41a39fe1e924.png": "3c9811d15b3c8c092f47600c1174a7788edeb3133c8ed60f6787f1ecab20c969", + "playwright-output/runner-native-completion-r-141db-the-browser-and-public-APIs/attachments/runner-e2e-result-634e7fe4a283dd542979c028382100d1270d9d02.json": "cdbd533c56af2b70ca9ef7650356396a45a64a6e8b9a6b370a3a50b8ea50e3fd", + "result.json": "cdbd533c56af2b70ca9ef7650356396a45a64a6e8b9a6b370a3a50b8ea50e3fd", + "snapshots/api-state.json": "0ff529820e1e17d741a3e7c41d3f36fda92352f02e65ba24cc3d06c13565810c", + "snapshots/browser-diagnostics.json": "e623949e9bf51664630b7243ea35a9de351d98a7bdbbc0b3f8509e59fdb5e8d0", + "snapshots/fixtures.json": "a6a709c5549326a0ab5ae4910521a5be92389c34c08c55a77efc11866225e52b", + "snapshots/native-completion.json": "c2aee8b83b78f9fdb0672748b82074c4e3b852a61ef6d8844d8c2512da60496c", + "snapshots/native-default-before-execution.json": "0e86a3ef333821923975722d4785454d70780c9a6c798281903fe994615b9fbc" + } + }, + { + "executionId": "native-completion.runner-codex.local.assigned-skill-explicit-invocation", + "profileId": "runner-codex", + "caseId": "assigned-skill-explicit-invocation", + "status": "passed", + "provider": "codex", + "model": "gpt-5.6-sol", + "attempt": 1, + "durationMs": 71995, + "cleanup": "passed", + "runtimeUsage": { + "provider": "local", + "agentRunDurationMs": 31884, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 209614, + "outputTokens": 1700, + "cachedInputTokens": 184860, + "totalTokens": 396174, + "reportedCostUsd": 0, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 31884, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0, + "complete": true + }, + "failureClass": null, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "resultSha256": "22043de2aba3bac6ad94ee6d31a242746bad34a11790976d48ee0027b82c81b5", + "sourceMatches": true, + "configMatches": true, + "expectedOneAttempt": true, + "originalFailedMatchers": [], + "originalMatcherCount": 14, + "persistedConfiguration": { + "provider": "codex", + "model": "gpt-5.6-sol", + "lifecycleMode": "per_turn", + "codexPermissionMode": "never" + }, + "declaredModelQualification": { + "source": "adapter_constant", + "qualificationId": "codex:qualified-model" + }, + "admission": { + "schema": "paperclip.native-completion-preflight.v1", + "passed": true, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "variant": "historical", + "providerCalls": 0, + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "fixtureFingerprint": "a50581c04f3d87843d69f92050712977f8e114e4c2f23f96f243fd5386119f27", + "manifestFingerprint": "b92f3ab7d561188bc09214433b264f2e7810e8ccdc237f68caa82e1aecc1811e", + "runnerdProvenance": { + "schema": "paperclip.native-completion-runnerd-provenance.v1", + "mode": "trusted_hosted_archive", + "passed": true, + "selectedPath": "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + "binarySha256": "9a007971dd5079bbc4d7c5bf6d385560c7e3fdedefb4990188836bbd5ae016a2", + "binaryBytes": 394443976, + "archiveMemberBytes": 394443976, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "archiveSha256": "d83acb6c73b9cf2256669bcf875d593aa18cfa941f07ad93ca92f30906047190", + "workflowRunId": "37098815696", + "workflowRunAttempt": "1", + "artifactName": "runner-e2e-build-e74ed61a69fbdd8b3a8f15dd6456bc3140246e33-37098815696-1", + "errors": [] + }, + "preflightSha256": "934a06d0cd0fb576234ab8f4bc00f7c970b79a2f2577e9db8a017f37e26fc07b", + "executedTypeScriptAssertions": 127, + "rustCalibrationExecutedInCell": false, + "failedGates": [], + "checksMatch": true + }, + "defaults": { + "passed": true, + "entryFile": "AGENTS.md", + "files": [ + { + "path": "AGENTS.md", + "sha256": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f", + "bytes": 4249 + } + ], + "budgets": { + "company": 1000, + "agent": 1000 + }, + "independentMatch": true + }, + "nativeAudit": { + "evidenceSource": "original-native-observation", + "originalNativeGradeAvailable": true, + "originalGradePassed": true, + "originalChecks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + } + ], + "originalIntegrity": [], + "runCount": 1, + "runStatus": "succeeded", + "nativeRun": true, + "issueStatus": "done", + "reportedDisposition": "done", + "runOwnershipMatches": true, + "retryOrContinuationObserved": false, + "expectedNamedTool": "paperclip_finish", + "namedStartCount": 2, + "matchingCompletionCount": 1, + "proposalSeq": [ + 175 + ], + "completionSeqs": [ + 176 + ], + "finalSeqs": [ + 203 + ], + "acceptedSeq": [ + 208 + ], + "terminalSeq": [ + 209 + ], + "independentNamedSequencePass": true, + "persistedProviderFinalMatches": true, + "finalMarkerCount": 1, + "finalMentionsOwner": false, + "finalMentionsAction": false, + "toolInvocationsAfterProposal": 0, + "blocker": null, + "workspaceChanged": false, + "documentCount": 1, + "interactionCount": 0, + "eventCount": 217 + }, + "perRunLedger": [ + { + "model": "gpt-5.6-sol", + "provider": "openai", + "biller": "openai", + "billingType": "unknown", + "usageSource": "per_run", + "costStatus": "reported", + "costUsd": 0, + "cacheAdjustedCostUsd": 0, + "inputTokens": 209614, + "outputTokens": 1700, + "cachedInputTokens": 184860 + } + ], + "runLedgerCostMatchesAggregate": true, + "documentAudit": { + "originalAllChecksPassed": true, + "originalChecks": [ + { + "id": "assigned-skill-present", + "passed": true, + "detail": "The task run must receive one pinned skill version through the public assignment state." + }, + { + "id": "skill-request-explicit", + "passed": true, + "detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference." + }, + { + "id": "skill-source-marker", + "passed": true, + "detail": "The assigned pinned skill source must contain the output marker." + }, + { + "id": "marker-not-in-request", + "passed": true, + "detail": "The output marker must originate from the assigned skill, not the task request or comments." + }, + { + "id": "single-durable-output", + "passed": true, + "detail": "Exactly one durable task document must contain the skill's marker." + }, + { + "id": "completed-task", + "passed": true, + "detail": "Final task status: done." + }, + { + "id": "successful-runs", + "passed": true, + "detail": "All recorded context-integrity runs must succeed." + } + ], + "documentCount": 1, + "savedDocuments": [ + { + "key": "context-integrity-output", + "hasRevision": true, + "containsMarker": true, + "bodySha256": "2bfa9cf9875f4e244b388adb3ae9fd9fa4959e031a3424a78ca1e34f0c23f885" + } + ] + }, + "ledgerAudit": { + "runCount": 1, + "tokenReceiptCount": 1, + "reportedCostReceiptCount": 1, + "reportedCostUsd": 0, + "allOneRunReceipts": true, + "externalActualBillingKnown": false, + "runtimeMetered": false + }, + "evidenceHashes": { + "blob-report/report.zip": "b1825fc7e5edcf1d65bc0b510237561110dc9c91f0e3749844aa66115d602183", + "evidence-manifest.json": "0bdca2fba590b30162921b0b2c11fe5ab964c2aff32687b7566738b58d8880da", + "final-state.png": "f1b72104509399d3d9a04e7c11451a1712f9ba7444a23992e65c34a8df97fcfd", + "html-report/data/8e6fdfa895db69c6265de156033d935ce29cdbff.json": "22043de2aba3bac6ad94ee6d31a242746bad34a11790976d48ee0027b82c81b5", + "html-report/data/ff1807fd3e2c54f1f52483b4341e8a0a0b20639c.png": "f1b72104509399d3d9a04e7c11451a1712f9ba7444a23992e65c34a8df97fcfd", + "playwright-output/runner-native-completion-r-25dfe-the-browser-and-public-APIs/attachments/final-state-6ddaed261f99039773a7751ffd4f2969c86fbc57.png": "f1b72104509399d3d9a04e7c11451a1712f9ba7444a23992e65c34a8df97fcfd", + "playwright-output/runner-native-completion-r-25dfe-the-browser-and-public-APIs/attachments/runner-e2e-result-201710ca5c9181dab99f6979efddd0c63fdf2941.json": "22043de2aba3bac6ad94ee6d31a242746bad34a11790976d48ee0027b82c81b5", + "result.json": "22043de2aba3bac6ad94ee6d31a242746bad34a11790976d48ee0027b82c81b5", + "snapshots/api-state.json": "9075b25a0237c62824cc99f6cd3e1e087cee6b37c8c5d992f32fac7d8c1d4ad4", + "snapshots/browser-diagnostics.json": "e726e3bcdf57913f86621a3826d3c1492981d3ad54afdf2897c4e11e3e5ee4fa", + "snapshots/chat-final-run-ledger.json": "4c1416c8f47612af541fd44922f97b931844731fc50b3b72e489b9e9e1efb649", + "snapshots/context-integrity.json": "fac884ff4ae3697bfe85ded5a9df6ffcbaf207e44c3892d15b09cee2169e8cf1", + "snapshots/fixtures.json": "93d45a26f4eaa14c874d52a0abd2ffa16ba1d7e5b2018da4cc941455945dea90", + "snapshots/native-completion.json": "8604e2bf01bf9aaece2b3ca0b83f220b1f00911673d8eded1b1e9216910fb238", + "snapshots/native-default-before-execution.json": "a24381e105d2dad18100c31b3c0b1fbced8707be20574f67b1cd394131e43a95" + } + }, + { + "executionId": "native-completion.runner-codex.local.native-blocked-report", + "profileId": "runner-codex", + "caseId": "native-blocked-report", + "status": "failed", + "provider": "codex", + "model": "gpt-5.6-sol", + "attempt": 1, + "durationMs": 94105, + "cleanup": "passed", + "runtimeUsage": { + "provider": "local", + "agentRunDurationMs": 17560, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 44092, + "outputTokens": 521, + "cachedInputTokens": 21752, + "totalTokens": 66365, + "reportedCostUsd": 0, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 17560, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0, + "complete": true + }, + "failureClass": "candidate_failure", + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "resultSha256": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f", + "sourceMatches": true, + "configMatches": true, + "expectedOneAttempt": true, + "originalFailedMatchers": [ + { + "kind": "message_contains", + "path": null, + "expected": "BLOCKED_548de9e6b5fd1", + "detail": "expected {\"kind\":\"message_contains\",\"expected\":\"BLOCKED_548de9e6b5fd1\"}; observed \"Task blocked. Owner: Release Owner. Unblock action: Grant deployment access.\"" + } + ], + "originalMatcherCount": 11, + "persistedConfiguration": { + "provider": "codex", + "model": "gpt-5.6-sol", + "lifecycleMode": "per_turn", + "codexPermissionMode": "never" + }, + "declaredModelQualification": { + "source": "adapter_constant", + "qualificationId": "codex:qualified-model" + }, + "admission": { + "schema": "paperclip.native-completion-preflight.v1", + "passed": true, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "variant": "historical", + "providerCalls": 0, + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "fixtureFingerprint": "a50581c04f3d87843d69f92050712977f8e114e4c2f23f96f243fd5386119f27", + "manifestFingerprint": "b92f3ab7d561188bc09214433b264f2e7810e8ccdc237f68caa82e1aecc1811e", + "runnerdProvenance": { + "schema": "paperclip.native-completion-runnerd-provenance.v1", + "mode": "trusted_hosted_archive", + "passed": true, + "selectedPath": "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + "binarySha256": "9a007971dd5079bbc4d7c5bf6d385560c7e3fdedefb4990188836bbd5ae016a2", + "binaryBytes": 394443976, + "archiveMemberBytes": 394443976, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "archiveSha256": "d83acb6c73b9cf2256669bcf875d593aa18cfa941f07ad93ca92f30906047190", + "workflowRunId": "37098815696", + "workflowRunAttempt": "1", + "artifactName": "runner-e2e-build-e74ed61a69fbdd8b3a8f15dd6456bc3140246e33-37098815696-1", + "errors": [] + }, + "preflightSha256": "f8c3d913ac601a2ffaf10e76df87d15d9158a4bc2c83f56a96267c78bbc851ec", + "executedTypeScriptAssertions": 127, + "rustCalibrationExecutedInCell": false, + "failedGates": [], + "checksMatch": true + }, + "defaults": { + "passed": true, + "entryFile": "AGENTS.md", + "files": [ + { + "path": "AGENTS.md", + "sha256": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f", + "bytes": 4249 + } + ], + "budgets": { + "company": 1000, + "agent": 1000 + }, + "independentMatch": true + }, + "nativeAudit": { + "evidenceSource": "retained-failure-api-state", + "originalNativeGradeAvailable": false, + "originalGradePassed": null, + "originalChecks": null, + "originalIntegrity": null, + "runCount": 1, + "runStatus": "succeeded", + "nativeRun": true, + "issueStatus": "blocked", + "reportedDisposition": "blocked", + "runOwnershipMatches": true, + "retryOrContinuationObserved": false, + "expectedNamedTool": "paperclip_block", + "namedStartCount": 1, + "matchingCompletionCount": 1, + "proposalSeq": [ + 61 + ], + "completionSeqs": [ + 62 + ], + "finalSeqs": [ + 81 + ], + "acceptedSeq": [ + 86 + ], + "terminalSeq": [ + 87 + ], + "independentNamedSequencePass": true, + "persistedProviderFinalMatches": true, + "finalMarkerCount": 0, + "finalMentionsOwner": true, + "finalMentionsAction": true, + "toolInvocationsAfterProposal": 0, + "blocker": { + "owner": { + "kind": "external", + "name": "Release Owner" + }, + "unblockAction": "Grant deployment access", + "scope": "task_wide" + }, + "workspaceChanged": null, + "documentCount": 0, + "interactionCount": 0, + "eventCount": 95 + }, + "perRunLedger": [ + { + "model": "gpt-5.6-sol", + "provider": "openai", + "biller": "openai", + "billingType": "unknown", + "usageSource": "per_run", + "costStatus": "reported", + "costUsd": 0, + "cacheAdjustedCostUsd": 0, + "inputTokens": 44092, + "outputTokens": 521, + "cachedInputTokens": 21752 + } + ], + "runLedgerCostMatchesAggregate": true, + "ledgerAudit": { + "runCount": 1, + "tokenReceiptCount": 1, + "reportedCostReceiptCount": 1, + "reportedCostUsd": 0, + "allOneRunReceipts": true, + "externalActualBillingKnown": false, + "runtimeMetered": false + }, + "evidenceHashes": { + "blob-report/report.zip": "e577c9c897fc92074b265ac8b68d719a0b110d46afd28ec230f16e8a88ebadc8", + "evidence-manifest.json": "789ee2fa73629c6dbe7603996a8d2ec5fb9b9a4f2071aa996125b9476d68d65a", + "failure.png": "a45aa92e0826f9efaa84f5f4603eab9cbfba8bfa3d98cadab9c6db64fd3824a0", + "html-report/data/340fc3360b8fe6107dd0b897e1b8b2886e292647.json": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f", + "html-report/data/3e3911b774df45117d72142c8da34797ff93570d.zip": "1fe92cf097af72ddb6f819fd5eab9ea76b994fdf439358dd45a0c72d83e42320", + "html-report/data/f2a317ca2b8cd26e062d42fdd2d773c640e583f0.png": "a45aa92e0826f9efaa84f5f4603eab9cbfba8bfa3d98cadab9c6db64fd3824a0", + "playwright-output/runner-native-completion-r-aa60d-the-browser-and-public-APIs/attachments/failure-89945ece535001f5341b6d3de607d820e3f69463.png": "a45aa92e0826f9efaa84f5f4603eab9cbfba8bfa3d98cadab9c6db64fd3824a0", + "playwright-output/runner-native-completion-r-aa60d-the-browser-and-public-APIs/attachments/runner-e2e-result-b86746251a773f8b4eb781a6a4f7b622952a297d.json": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f", + "playwright-output/runner-native-completion-r-aa60d-the-browser-and-public-APIs/test-failed-1.png": "a45aa92e0826f9efaa84f5f4603eab9cbfba8bfa3d98cadab9c6db64fd3824a0", + "playwright-output/runner-native-completion-r-aa60d-the-browser-and-public-APIs/trace.zip": "1fe92cf097af72ddb6f819fd5eab9ea76b994fdf439358dd45a0c72d83e42320", + "result.json": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f", + "snapshots/api-state.json": "161ae90f28b181cc041ec41dc3892a65b906f1174cf9ad2d0222f759164bbad0", + "snapshots/browser-diagnostics.json": "294fa7303fe819a228f61aac5977912c6d8de4f9408fe3a25342a7ecb6c24ebf", + "snapshots/fixtures.json": "123fb7c906f0c3b6e23b420d930824b65f805402c64825986b759566c6204f78", + "snapshots/native-default-before-execution.json": "9e9a2fd1d35965fa686410ad678e57159d63b761bec0d6f8f404b7e56f01ca94" + } + }, + { + "executionId": "native-completion.runner-opencode.local.assigned-skill-explicit-invocation", + "profileId": "runner-opencode", + "caseId": "assigned-skill-explicit-invocation", + "status": "passed", + "provider": "opencode", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "attempt": 1, + "durationMs": 71062, + "cleanup": "passed", + "runtimeUsage": { + "provider": "local", + "agentRunDurationMs": 27842, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 23125, + "outputTokens": 915, + "cachedInputTokens": 88320, + "totalTokens": 112360, + "reportedCostUsd": 0.0029267655, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 27842, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.0029267655, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.0029267655, + "complete": true + }, + "failureClass": null, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "resultSha256": "5f084208b3211b0caccc47705c08fad3ccba72fa4568882c78f8946ac26e4be6", + "sourceMatches": true, + "configMatches": true, + "expectedOneAttempt": true, + "originalFailedMatchers": [], + "originalMatcherCount": 14, + "persistedConfiguration": { + "provider": "opencode", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "lifecycleMode": "per_turn", + "codexPermissionMode": "never", + "opencodePermissionMode": "allow" + }, + "declaredModelQualification": { + "source": "adapter_constant", + "qualificationId": "opencode:qualified-model" + }, + "admission": { + "schema": "paperclip.native-completion-preflight.v1", + "passed": true, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "variant": "historical", + "providerCalls": 0, + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "fixtureFingerprint": "a50581c04f3d87843d69f92050712977f8e114e4c2f23f96f243fd5386119f27", + "manifestFingerprint": "b92f3ab7d561188bc09214433b264f2e7810e8ccdc237f68caa82e1aecc1811e", + "runnerdProvenance": { + "schema": "paperclip.native-completion-runnerd-provenance.v1", + "mode": "trusted_hosted_archive", + "passed": true, + "selectedPath": "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + "binarySha256": "9a007971dd5079bbc4d7c5bf6d385560c7e3fdedefb4990188836bbd5ae016a2", + "binaryBytes": 394443976, + "archiveMemberBytes": 394443976, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "archiveSha256": "d83acb6c73b9cf2256669bcf875d593aa18cfa941f07ad93ca92f30906047190", + "workflowRunId": "37098815696", + "workflowRunAttempt": "1", + "artifactName": "runner-e2e-build-e74ed61a69fbdd8b3a8f15dd6456bc3140246e33-37098815696-1", + "errors": [] + }, + "preflightSha256": "72d8d77b18f722b535a49c5630f879ca91a8f58f40783811b15899ca295769bf", + "executedTypeScriptAssertions": 127, + "rustCalibrationExecutedInCell": false, + "failedGates": [], + "checksMatch": true + }, + "defaults": { + "passed": true, + "entryFile": "AGENTS.md", + "files": [ + { + "path": "AGENTS.md", + "sha256": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f", + "bytes": 4249 + } + ], + "budgets": { + "company": 1000, + "agent": 1000 + }, + "independentMatch": true + }, + "nativeAudit": { + "evidenceSource": "original-native-observation", + "originalNativeGradeAvailable": true, + "originalGradePassed": true, + "originalChecks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + } + ], + "originalIntegrity": [], + "runCount": 1, + "runStatus": "succeeded", + "nativeRun": true, + "issueStatus": "done", + "reportedDisposition": "done", + "runOwnershipMatches": true, + "retryOrContinuationObserved": false, + "expectedNamedTool": "paperclip_finish", + "namedStartCount": 1, + "matchingCompletionCount": 1, + "proposalSeq": [ + 50 + ], + "completionSeqs": [ + 51 + ], + "finalSeqs": [ + 57 + ], + "acceptedSeq": [ + 61 + ], + "terminalSeq": [ + 62 + ], + "independentNamedSequencePass": true, + "persistedProviderFinalMatches": true, + "finalMarkerCount": 1, + "finalMentionsOwner": false, + "finalMentionsAction": false, + "toolInvocationsAfterProposal": 0, + "blocker": null, + "workspaceChanged": false, + "documentCount": 1, + "interactionCount": 0, + "eventCount": 70 + }, + "perRunLedger": [ + { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "provider": "openai", + "biller": "openai", + "billingType": "unknown", + "usageSource": "per_run", + "costStatus": "reported", + "costUsd": 0.0029267655, + "cacheAdjustedCostUsd": 0.0029267655, + "inputTokens": 23125, + "outputTokens": 915, + "cachedInputTokens": 88320 + } + ], + "runLedgerCostMatchesAggregate": true, + "documentAudit": { + "originalAllChecksPassed": true, + "originalChecks": [ + { + "id": "assigned-skill-present", + "passed": true, + "detail": "The task run must receive one pinned skill version through the public assignment state." + }, + { + "id": "skill-request-explicit", + "passed": true, + "detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference." + }, + { + "id": "skill-source-marker", + "passed": true, + "detail": "The assigned pinned skill source must contain the output marker." + }, + { + "id": "marker-not-in-request", + "passed": true, + "detail": "The output marker must originate from the assigned skill, not the task request or comments." + }, + { + "id": "single-durable-output", + "passed": true, + "detail": "Exactly one durable task document must contain the skill's marker." + }, + { + "id": "completed-task", + "passed": true, + "detail": "Final task status: done." + }, + { + "id": "successful-runs", + "passed": true, + "detail": "All recorded context-integrity runs must succeed." + } + ], + "documentCount": 1, + "savedDocuments": [ + { + "key": "context-integrity-output-eac4e7a674271", + "hasRevision": true, + "containsMarker": true, + "bodySha256": "5cb4e7fe93bc3159f6862ba14bcfb42a1eec48be87c1afb592b5ec906ed0b132" + } + ] + }, + "ledgerAudit": { + "runCount": 1, + "tokenReceiptCount": 1, + "reportedCostReceiptCount": 1, + "reportedCostUsd": 0.0029267655, + "allOneRunReceipts": true, + "externalActualBillingKnown": false, + "runtimeMetered": false + }, + "evidenceHashes": { + "blob-report/report.zip": "786633c5e6017f51c910c98cbfc74f62af3e033bf193dae867f4d70e74f98570", + "evidence-manifest.json": "6c23192583065f6392f48bb17e8ce917c3550e8742009d11c9330999065fa7a9", + "final-state.png": "fef8214feca49249941e158c7816488e8722e951156c46c83a371997936811b7", + "html-report/data/bfbc6e192fb74c93ad3c8575e271074105a2f080.json": "5f084208b3211b0caccc47705c08fad3ccba72fa4568882c78f8946ac26e4be6", + "html-report/data/dc61925a79bf5575b06535d1862806ca2a9c5ec5.png": "fef8214feca49249941e158c7816488e8722e951156c46c83a371997936811b7", + "playwright-output/runner-native-completion-r-39fd3-the-browser-and-public-APIs/attachments/final-state-bb2545d482a1244a26bebcbd67dbdeac7a76ea6b.png": "fef8214feca49249941e158c7816488e8722e951156c46c83a371997936811b7", + "playwright-output/runner-native-completion-r-39fd3-the-browser-and-public-APIs/attachments/runner-e2e-result-409b6bb773c0f7a749f7fe3250fe29c2547fb54a.json": "5f084208b3211b0caccc47705c08fad3ccba72fa4568882c78f8946ac26e4be6", + "result.json": "5f084208b3211b0caccc47705c08fad3ccba72fa4568882c78f8946ac26e4be6", + "snapshots/api-state.json": "a8b45b2b46462d2d27f7c225cdc1746b1272876f1bb46c4829062d5382c23d84", + "snapshots/browser-diagnostics.json": "712dd147bff6276d6a9695678adf476e1b8c27e79679e649379882ec816e97f7", + "snapshots/chat-final-run-ledger.json": "b9790a85d832ff2225b211f1f146fb11a0b77c5d0cfbabb0827be536f8654f8e", + "snapshots/context-integrity.json": "d0faa11a4aecd5928c950c43aeb2caa686f1031a8612f9588ff8481fe3a651eb", + "snapshots/fixtures.json": "7d8dacec7e1738d6bf86ee916f7abb01d07cbbca9f2052e97b897dc812eb2cbc", + "snapshots/native-completion.json": "33bbff87b4bf6b5f526563c3bbbbbedb5de8bb6500e61165f385c9f7e329de6c", + "snapshots/native-default-before-execution.json": "6bdef94a36b2b999dcb3ff7b80c69325ea2c3b28e0f01c3172fa1b40cfb63283" + } + }, + { + "executionId": "native-completion.runner-opencode.local.native-blocked-report", + "profileId": "runner-opencode", + "caseId": "native-blocked-report", + "status": "passed", + "provider": "opencode", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "attempt": 1, + "durationMs": 78989, + "cleanup": "passed", + "runtimeUsage": { + "provider": "local", + "agentRunDurationMs": 35963, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 42988, + "outputTokens": 432, + "cachedInputTokens": 0, + "totalTokens": 43420, + "reportedCostUsd": 0.0012880548, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 35963, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.0012880548, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.0012880548, + "complete": true + }, + "failureClass": null, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "resultSha256": "d8e19a8cf7770ea6138cd4f816892a7a04e6160063bbcbb6f7693efe28278826", + "sourceMatches": true, + "configMatches": true, + "expectedOneAttempt": true, + "originalFailedMatchers": [], + "originalMatcherCount": 21, + "persistedConfiguration": { + "provider": "opencode", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "lifecycleMode": "per_turn", + "codexPermissionMode": "never", + "opencodePermissionMode": "allow" + }, + "declaredModelQualification": { + "source": "adapter_constant", + "qualificationId": "opencode:qualified-model" + }, + "admission": { + "schema": "paperclip.native-completion-preflight.v1", + "passed": true, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "variant": "historical", + "providerCalls": 0, + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "fixtureFingerprint": "a50581c04f3d87843d69f92050712977f8e114e4c2f23f96f243fd5386119f27", + "manifestFingerprint": "b92f3ab7d561188bc09214433b264f2e7810e8ccdc237f68caa82e1aecc1811e", + "runnerdProvenance": { + "schema": "paperclip.native-completion-runnerd-provenance.v1", + "mode": "trusted_hosted_archive", + "passed": true, + "selectedPath": "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + "binarySha256": "9a007971dd5079bbc4d7c5bf6d385560c7e3fdedefb4990188836bbd5ae016a2", + "binaryBytes": 394443976, + "archiveMemberBytes": 394443976, + "sourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", + "sourceFingerprint": "a01593363d6512e30d8149fcf4f34adfff8b07aeef7f845f0c9bbed3cb7b979f", + "archiveSha256": "d83acb6c73b9cf2256669bcf875d593aa18cfa941f07ad93ca92f30906047190", + "workflowRunId": "37098815696", + "workflowRunAttempt": "1", + "artifactName": "runner-e2e-build-e74ed61a69fbdd8b3a8f15dd6456bc3140246e33-37098815696-1", + "errors": [] + }, + "preflightSha256": "c03255709fd525a05532e4941172f99df3451c4c92264142b95b77613125f005", + "executedTypeScriptAssertions": 127, + "rustCalibrationExecutedInCell": false, + "failedGates": [], + "checksMatch": true + }, + "defaults": { + "passed": true, + "entryFile": "AGENTS.md", + "files": [ + { + "path": "AGENTS.md", + "sha256": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f", + "bytes": 4249 + } + ], + "budgets": { + "company": 1000, + "agent": 1000 + }, + "independentMatch": true + }, + "nativeAudit": { + "evidenceSource": "original-native-observation", + "originalNativeGradeAvailable": true, + "originalGradePassed": true, + "originalChecks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "exact-blocker", + "passed": true, + "detail": "The structured owner, exact action and whole-task scope are independently checked." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "visible-blocker-content", + "passed": true, + "detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + }, + { + "id": "no-deployment-or-file-work", + "passed": true, + "detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed." + } + ], + "originalIntegrity": [], + "runCount": 1, + "runStatus": "succeeded", + "nativeRun": true, + "issueStatus": "blocked", + "reportedDisposition": "blocked", + "runOwnershipMatches": true, + "retryOrContinuationObserved": false, + "expectedNamedTool": "paperclip_block", + "namedStartCount": 1, + "matchingCompletionCount": 1, + "proposalSeq": [ + 35 + ], + "completionSeqs": [ + 36 + ], + "finalSeqs": [ + 44 + ], + "acceptedSeq": [ + 48 + ], + "terminalSeq": [ + 49 + ], + "independentNamedSequencePass": true, + "persistedProviderFinalMatches": true, + "finalMarkerCount": 1, + "finalMentionsOwner": true, + "finalMentionsAction": true, + "toolInvocationsAfterProposal": 0, + "blocker": { + "owner": { + "kind": "external", + "name": "Release Owner" + }, + "unblockAction": "Grant deployment access", + "scope": "task_wide" + }, + "workspaceChanged": false, + "documentCount": 0, + "interactionCount": 0, + "eventCount": 57 + }, + "perRunLedger": [ + { + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "provider": "openai", + "biller": "openai", + "billingType": "unknown", + "usageSource": "per_run", + "costStatus": "reported", + "costUsd": 0.0012880548, + "cacheAdjustedCostUsd": 0.0012880548, + "inputTokens": 42988, + "outputTokens": 432, + "cachedInputTokens": 0 + } + ], + "runLedgerCostMatchesAggregate": true, + "ledgerAudit": { + "runCount": 1, + "tokenReceiptCount": 1, + "reportedCostReceiptCount": 1, + "reportedCostUsd": 0.0012880548, + "allOneRunReceipts": true, + "externalActualBillingKnown": false, + "runtimeMetered": false + }, + "evidenceHashes": { + "blob-report/report.zip": "fcc0d066b282de5c18b6ee1628c2b420d57d0a94f54e091cd8d105072f96a68e", + "evidence-manifest.json": "7e05f6f15f8a0bb9bcc9667b9b71d1a8556bf8cc1d64867eff216acbda74db5d", + "final-state.png": "490baa12e6d4a0b7eed7febc659a8922167f54fee9695cdc96c5b773823b3f5e", + "html-report/data/5daf64c61ab3533cea05ee059d5bff8247931e8b.png": "490baa12e6d4a0b7eed7febc659a8922167f54fee9695cdc96c5b773823b3f5e", + "html-report/data/ba1e1a0915714914158c5369d4b534ae27bcc316.json": "d8e19a8cf7770ea6138cd4f816892a7a04e6160063bbcbb6f7693efe28278826", + "playwright-output/runner-native-completion-r-c73dc-the-browser-and-public-APIs/attachments/final-state-2d96ecf321c00be1c480f2f7b4be7ca9b2fdd25d.png": "490baa12e6d4a0b7eed7febc659a8922167f54fee9695cdc96c5b773823b3f5e", + "playwright-output/runner-native-completion-r-c73dc-the-browser-and-public-APIs/attachments/runner-e2e-result-7649dedcc0061aec00b730ce3236af39bb7e8088.json": "d8e19a8cf7770ea6138cd4f816892a7a04e6160063bbcbb6f7693efe28278826", + "result.json": "d8e19a8cf7770ea6138cd4f816892a7a04e6160063bbcbb6f7693efe28278826", + "snapshots/api-state.json": "4180020c15489673c5dd25677cfd5daeab9743d3a27dcef003234dfed003a94e", + "snapshots/browser-diagnostics.json": "e46c22102670f04ffd904dc8c7716f07d5e45fedd8274011f9fd3d3bc201a9ce", + "snapshots/fixtures.json": "962345aed62dd41e84add9168eecffb14e517d646b416bbde92fc1d2162f4631", + "snapshots/native-completion.json": "cef17203858e394411e964338e62e3e421c9d5595c700ee161c5e736775388b8", + "snapshots/native-default-before-execution.json": "574f448b18d40fff600784b6678bba300c6425fa75f4b2c1d0e49cd42206a5d5" + } + } + ], + "limitations": [ + "Original machine verdicts are preserved; this is a separate retained-evidence audit.", + "Reported provider cost is not actual external billing, and local runtime is unmetered.", + "Named-call sequencing is observable; provider consumption of feedback is not proven." + ] +} diff --git a/doc/plans/2026-10-02-native-completion-calibrated-results/candidate-audit.json b/doc/plans/2026-10-02-native-completion-calibrated-results/candidate-audit.json new file mode 100644 index 00000000000..d939e282e18 --- /dev/null +++ b/doc/plans/2026-10-02-native-completion-calibrated-results/candidate-audit.json @@ -0,0 +1,886 @@ +{ + "schema": "paperclip.native-completion-retained-audit.v1", + "originalResultsChanged": false, + "providerRuns": 6, + "rows": [ + { + "executionId": "native-completion.runner-codex.local.native-blocked-report", + "variant": "candidate", + "source": { + "sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21", + "ref": "refs/heads/codex/native-completion-master-calibrated-candidate", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980" + }, + "suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d", + "artifactId": 11265785208, + "artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11265785208", + "zipSha256": "53b1125e7b5d183462e1e1e536cd7674cacfec9995ee8a47e32116740564916f", + "resultSha256": "60c8a4f3e41b8ca9a80c10f73d804c30d6e6ede9ace5ce3e93d2c4a053503f32", + "nativeSnapshotSha256": "ad5cc611524fc7ac98dd8a65201b90eacb75d76309031f4e39090768fca1a909", + "defaultSnapshotSha256": "c5bb4c9b1ed1a9f13c90320d0081fc22f04ae4cf40af751864a8de1c5cc04d7a", + "contextSnapshotSha256": null, + "status": "passed", + "attempt": 1, + "runCount": 1, + "runStatus": "succeeded", + "issueStatus": "blocked", + "model": "gpt-5.6-sol", + "provider": "codex", + "grade": { + "schema": "paperclip.native-completion-observation.v2", + "passed": true, + "checks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "exact-blocker", + "passed": true, + "detail": "The structured owner, exact action and whole-task scope are independently checked." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "visible-blocker-content", + "passed": true, + "detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + }, + { + "id": "no-deployment-or-file-work", + "passed": true, + "detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed." + } + ], + "limitations": [ + "Exact provider feedback identity/consumption is not measured by the public sequence." + ] + }, + "integrityFailures": [], + "defaultGrade": { + "passed": true, + "checks": [ + { + "id": "production-default-bundle", + "passed": true, + "detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." + }, + { + "id": "bounded-company-and-agent", + "passed": true, + "detail": "Both public budgets enforce the declared 1,000-cent hard stop." + } + ] + }, + "contextChecks": null, + "durableState": { + "documentCount": 0, + "interactionCount": 0, + "additionalIssueCount": 1, + "additionalAgentCount": 0 + }, + "sequencing": { + "proposalSeq": [ + 60 + ], + "acceptedSeq": [ + 72 + ], + "terminalSeq": [ + 73 + ], + "providerFinalSeq": [ + 67 + ] + }, + "durationMs": 61424, + "agentRunDurationMs": 18794, + "tokens": { + "input": 44226, + "output": 577, + "cachedInput": 21814 + }, + "reportedCostUsd": 0, + "costStatus": "reported", + "billingType": "unknown", + "actualBillingVerified": false, + "runtimeCostStatus": "not_metered", + "cleanup": "passed", + "matcherCount": 21, + "failedMatchers": [], + "independentRetainedReplayPassed": true + }, + { + "executionId": "native-completion.runner-codex.local.assigned-skill-explicit-invocation", + "variant": "candidate", + "source": { + "sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21", + "ref": "refs/heads/codex/native-completion-master-calibrated-candidate", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980" + }, + "suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d", + "artifactId": 11265632369, + "artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11265632369", + "zipSha256": "afc06b58cb048774274bd856b25cb8359b28f3c07f104428a92219148f60fca4", + "resultSha256": "09077172094e8f63861739884ae42e1fb5755f113e9a18a53b21697055d25607", + "nativeSnapshotSha256": "efc7af6363ffda141ed4d7648f48db79069ceb7bcc96475beb30be1c10b35dca", + "defaultSnapshotSha256": "b4af7764ab4a84b541f2045468889296538a3ef420d7f8898835df0d12faaf4e", + "contextSnapshotSha256": "4f221d6c03955759889c978521cab9d617a8477cce8f1276f55457b5dc71b988", + "status": "passed", + "attempt": 1, + "runCount": 1, + "runStatus": "succeeded", + "issueStatus": "done", + "model": "gpt-5.6-sol", + "provider": "codex", + "grade": { + "schema": "paperclip.native-completion-observation.v2", + "passed": true, + "checks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + } + ], + "limitations": [ + "Exact provider feedback identity/consumption is not measured by the public sequence." + ] + }, + "integrityFailures": [], + "defaultGrade": { + "passed": true, + "checks": [ + { + "id": "production-default-bundle", + "passed": true, + "detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." + }, + { + "id": "bounded-company-and-agent", + "passed": true, + "detail": "Both public budgets enforce the declared 1,000-cent hard stop." + } + ] + }, + "contextChecks": [ + { + "id": "assigned-skill-present", + "passed": true, + "detail": "The task run must receive one pinned skill version through the public assignment state." + }, + { + "id": "skill-request-explicit", + "passed": true, + "detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference." + }, + { + "id": "skill-source-marker", + "passed": true, + "detail": "The assigned pinned skill source must contain the output marker." + }, + { + "id": "marker-not-in-request", + "passed": true, + "detail": "The output marker must originate from the assigned skill, not the task request or comments." + }, + { + "id": "single-durable-output", + "passed": true, + "detail": "Exactly one durable task document must contain the skill's marker." + }, + { + "id": "completed-task", + "passed": true, + "detail": "Final task status: done." + }, + { + "id": "successful-runs", + "passed": true, + "detail": "All recorded context-integrity runs must succeed." + } + ], + "durableState": { + "documentCount": 1, + "interactionCount": 0, + "additionalIssueCount": 1, + "additionalAgentCount": 0 + }, + "sequencing": { + "proposalSeq": [ + 158 + ], + "acceptedSeq": [ + 191 + ], + "terminalSeq": [ + 192 + ], + "providerFinalSeq": [ + 186 + ] + }, + "durationMs": 78188, + "agentRunDurationMs": 34018, + "tokens": { + "input": 234514, + "output": 1598, + "cachedInput": 209637 + }, + "reportedCostUsd": 0, + "costStatus": "reported", + "billingType": "unknown", + "actualBillingVerified": false, + "runtimeCostStatus": "not_metered", + "cleanup": "passed", + "matcherCount": 14, + "failedMatchers": [], + "independentRetainedReplayPassed": true + }, + { + "executionId": "native-completion.runner-opencode.local.assigned-skill-explicit-invocation", + "variant": "candidate", + "source": { + "sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21", + "ref": "refs/heads/codex/native-completion-master-calibrated-candidate", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980" + }, + "suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d", + "artifactId": 11265222862, + "artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11265222862", + "zipSha256": "0037c195e310ec8dbe5d54f4c23d00bbc3edf1bd812f75f33bd4fa0eb3f3ba4c", + "resultSha256": "f70d358b361e425fb4992b2101745a9912b2682134e780203ae5f79845dd5ee1", + "nativeSnapshotSha256": "f7273fd6716e145f2136211be32e7a2f1e9b18501eac32d9217a02d2abc10ce1", + "defaultSnapshotSha256": "015a77243ee0298d729d893aa941fed93c8f8252eaa4a12a63673bcaf909edb4", + "contextSnapshotSha256": "0291e48dc4873d5d20c0d1daeef05c9536445ee3d24a6b51af9d787c324ca4e6", + "status": "passed", + "attempt": 1, + "runCount": 1, + "runStatus": "succeeded", + "issueStatus": "done", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "provider": "opencode", + "grade": { + "schema": "paperclip.native-completion-observation.v2", + "passed": true, + "checks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + } + ], + "limitations": [ + "Exact provider feedback identity/consumption is not measured by the public sequence." + ] + }, + "integrityFailures": [], + "defaultGrade": { + "passed": true, + "checks": [ + { + "id": "production-default-bundle", + "passed": true, + "detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." + }, + { + "id": "bounded-company-and-agent", + "passed": true, + "detail": "Both public budgets enforce the declared 1,000-cent hard stop." + } + ] + }, + "contextChecks": [ + { + "id": "assigned-skill-present", + "passed": true, + "detail": "The task run must receive one pinned skill version through the public assignment state." + }, + { + "id": "skill-request-explicit", + "passed": true, + "detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference." + }, + { + "id": "skill-source-marker", + "passed": true, + "detail": "The assigned pinned skill source must contain the output marker." + }, + { + "id": "marker-not-in-request", + "passed": true, + "detail": "The output marker must originate from the assigned skill, not the task request or comments." + }, + { + "id": "single-durable-output", + "passed": true, + "detail": "Exactly one durable task document must contain the skill's marker." + }, + { + "id": "completed-task", + "passed": true, + "detail": "Final task status: done." + }, + { + "id": "successful-runs", + "passed": true, + "detail": "All recorded context-integrity runs must succeed." + } + ], + "durableState": { + "documentCount": 1, + "interactionCount": 0, + "additionalIssueCount": 1, + "additionalAgentCount": 0 + }, + "sequencing": { + "proposalSeq": [ + 56 + ], + "acceptedSeq": [ + 68 + ], + "terminalSeq": [ + 69 + ], + "providerFinalSeq": [ + 64 + ] + }, + "durationMs": 85536, + "agentRunDurationMs": 41645, + "tokens": { + "input": 45592, + "output": 731, + "cachedInput": 65536 + }, + "reportedCostUsd": 0.0027245575999999996, + "costStatus": "reported", + "billingType": "unknown", + "actualBillingVerified": false, + "runtimeCostStatus": "not_metered", + "cleanup": "passed", + "matcherCount": 14, + "failedMatchers": [], + "independentRetainedReplayPassed": true + }, + { + "executionId": "native-completion.runner-opencode.local.native-blocked-report", + "variant": "candidate", + "source": { + "sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21", + "ref": "refs/heads/codex/native-completion-master-calibrated-candidate", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980" + }, + "suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d", + "artifactId": 11264929195, + "artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11264929195", + "zipSha256": "33014bc90997444b1954bc465fd86ce1d93b3cfc4079a88545c110bc3896787b", + "resultSha256": "8c943b38d480fcb3513fe6a458e75d0606e4669ba047ecd3a779b35a65ec2a30", + "nativeSnapshotSha256": "b7bed5ee5d4942b3092516c790c2f527ca0859fa451ffc6b9423853ba70fea5a", + "defaultSnapshotSha256": "3489c5facf99991f25ba864b99c12628e3f87442cc43a251beab60e0d0fe2992", + "contextSnapshotSha256": null, + "status": "passed", + "attempt": 1, + "runCount": 1, + "runStatus": "succeeded", + "issueStatus": "blocked", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "provider": "opencode", + "grade": { + "schema": "paperclip.native-completion-observation.v2", + "passed": true, + "checks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "exact-blocker", + "passed": true, + "detail": "The structured owner, exact action and whole-task scope are independently checked." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "visible-blocker-content", + "passed": true, + "detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + }, + { + "id": "no-deployment-or-file-work", + "passed": true, + "detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed." + } + ], + "limitations": [ + "Exact provider feedback identity/consumption is not measured by the public sequence." + ] + }, + "integrityFailures": [], + "defaultGrade": { + "passed": true, + "checks": [ + { + "id": "production-default-bundle", + "passed": true, + "detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." + }, + { + "id": "bounded-company-and-agent", + "passed": true, + "detail": "Both public budgets enforce the declared 1,000-cent hard stop." + } + ] + }, + "contextChecks": null, + "durableState": { + "documentCount": 0, + "interactionCount": 0, + "additionalIssueCount": 1, + "additionalAgentCount": 0 + }, + "sequencing": { + "proposalSeq": [ + 36 + ], + "acceptedSeq": [ + 49 + ], + "terminalSeq": [ + 50 + ], + "providerFinalSeq": [ + 45 + ] + }, + "durationMs": 81864, + "agentRunDurationMs": 39947, + "tokens": { + "input": 22300, + "output": 735, + "cachedInput": 21248 + }, + "reportedCostUsd": 0.0016493492000000001, + "costStatus": "reported", + "billingType": "unknown", + "actualBillingVerified": false, + "runtimeCostStatus": "not_metered", + "cleanup": "passed", + "matcherCount": 21, + "failedMatchers": [], + "independentRetainedReplayPassed": true + }, + { + "executionId": "native-completion.runner-acpx-claude.local.native-blocked-report", + "variant": "candidate", + "source": { + "sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21", + "ref": "refs/heads/codex/native-completion-master-calibrated-candidate", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980" + }, + "suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d", + "artifactId": 11264674612, + "artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11264674612", + "zipSha256": "309843a997c07414b999c3712cdea7592e5010e70a2dceff34a1c70b397954a9", + "resultSha256": "4cdce8a05751fbbc6d3711cd40dc7a4d807c5e93dd5b8008b100692517afaa52", + "nativeSnapshotSha256": "6ef4cf8878fbb4e5d530f846b73f749d3dc3d4d19ae64ace48d35f34a9b4143c", + "defaultSnapshotSha256": "2f176d472827e84dc7e7d417080c6fdcdc413a6caec088164c4ffbf38b8cce91", + "contextSnapshotSha256": null, + "status": "passed", + "attempt": 1, + "runCount": 1, + "runStatus": "succeeded", + "issueStatus": "blocked", + "model": "claude-sonnet-5", + "provider": "acpx", + "grade": { + "schema": "paperclip.native-completion-observation.v2", + "passed": true, + "checks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "exact-blocker", + "passed": true, + "detail": "The structured owner, exact action and whole-task scope are independently checked." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "visible-blocker-content", + "passed": true, + "detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + }, + { + "id": "no-deployment-or-file-work", + "passed": true, + "detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed." + } + ], + "limitations": [ + "Exact provider feedback identity/consumption is not measured by the public sequence." + ] + }, + "integrityFailures": [], + "defaultGrade": { + "passed": true, + "checks": [ + { + "id": "production-default-bundle", + "passed": true, + "detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." + }, + { + "id": "bounded-company-and-agent", + "passed": true, + "detail": "Both public budgets enforce the declared 1,000-cent hard stop." + } + ] + }, + "contextChecks": null, + "durableState": { + "documentCount": 0, + "interactionCount": 0, + "additionalIssueCount": 1, + "additionalAgentCount": 0 + }, + "sequencing": { + "proposalSeq": [ + 52 + ], + "acceptedSeq": [ + 81 + ], + "terminalSeq": [ + 82 + ], + "providerFinalSeq": [ + 77 + ] + }, + "durationMs": 69422, + "agentRunDurationMs": 29059, + "tokens": { + "input": 6, + "output": 1420, + "cachedInput": 69575 + }, + "reportedCostUsd": 0, + "costStatus": "reported", + "billingType": "unknown", + "actualBillingVerified": false, + "runtimeCostStatus": "not_metered", + "cleanup": "passed", + "matcherCount": 21, + "failedMatchers": [], + "independentRetainedReplayPassed": true + }, + { + "executionId": "native-completion.runner-acpx-claude.local.assigned-skill-explicit-invocation", + "variant": "candidate", + "source": { + "sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21", + "ref": "refs/heads/codex/native-completion-master-calibrated-candidate", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980" + }, + "suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d", + "artifactId": 11264634829, + "artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11264634829", + "zipSha256": "7d4e0b3733bfea937167cac4ec0c18a5d9892e52b33a62f9a67fe7b55e039987", + "resultSha256": "97a7d6cd5eb5bb1568b9f8ada8ef21b83323e0644d5aa7a741eefa946f6389c6", + "nativeSnapshotSha256": "40b64742374ea90fada7fe1f8f484f1eebeaed424b12be87746efa9a60bc1e5f", + "defaultSnapshotSha256": "ae2ae47d2f5dac9d2bcb243ea8d62c828a4c5081acb4538ca3ee4c8eb7f871a1", + "contextSnapshotSha256": "2f451ae80966bb3d5512e1012048431c4e3216a97933ac222c51cdb602a11588", + "status": "passed", + "attempt": 1, + "runCount": 1, + "runStatus": "succeeded", + "issueStatus": "done", + "model": "claude-sonnet-5", + "provider": "acpx", + "grade": { + "schema": "paperclip.native-completion-observation.v2", + "passed": true, + "checks": [ + { + "id": "one-native-attempt", + "passed": true, + "detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work." + }, + { + "id": "durable-disposition", + "passed": true, + "detail": "Public durable issue status and the native semantic result agree with the requested disposition." + }, + { + "id": "complete-bound-events", + "passed": true, + "detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity." + }, + { + "id": "authoritative-native-result", + "passed": true, + "detail": "One runner proposal is independently accepted and terminated by the control plane." + }, + { + "id": "final-after-tool-result", + "passed": true, + "detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently." + }, + { + "id": "persisted-provider-final", + "passed": true, + "detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient." + }, + { + "id": "bounded-durable-work", + "passed": true, + "detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none." + } + ], + "limitations": [ + "Exact provider feedback identity/consumption is not measured by the public sequence." + ] + }, + "integrityFailures": [], + "defaultGrade": { + "passed": true, + "checks": [ + { + "id": "production-default-bundle", + "passed": true, + "detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." + }, + { + "id": "bounded-company-and-agent", + "passed": true, + "detail": "Both public budgets enforce the declared 1,000-cent hard stop." + } + ] + }, + "contextChecks": [ + { + "id": "assigned-skill-present", + "passed": true, + "detail": "The task run must receive one pinned skill version through the public assignment state." + }, + { + "id": "skill-request-explicit", + "passed": true, + "detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference." + }, + { + "id": "skill-source-marker", + "passed": true, + "detail": "The assigned pinned skill source must contain the output marker." + }, + { + "id": "marker-not-in-request", + "passed": true, + "detail": "The output marker must originate from the assigned skill, not the task request or comments." + }, + { + "id": "single-durable-output", + "passed": true, + "detail": "Exactly one durable task document must contain the skill's marker." + }, + { + "id": "completed-task", + "passed": true, + "detail": "Final task status: done." + }, + { + "id": "successful-runs", + "passed": true, + "detail": "All recorded context-integrity runs must succeed." + } + ], + "durableState": { + "documentCount": 1, + "interactionCount": 0, + "additionalIssueCount": 1, + "additionalAgentCount": 0 + }, + "sequencing": { + "proposalSeq": [ + 76 + ], + "acceptedSeq": [ + 104 + ], + "terminalSeq": [ + 105 + ], + "providerFinalSeq": [ + 100 + ] + }, + "durationMs": 79552, + "agentRunDurationMs": 27682, + "tokens": { + "input": 10, + "output": 1096, + "cachedInput": 161787 + }, + "reportedCostUsd": 0, + "costStatus": "reported", + "billingType": "unknown", + "actualBillingVerified": false, + "runtimeCostStatus": "not_metered", + "cleanup": "passed", + "matcherCount": 14, + "failedMatchers": [], + "independentRetainedReplayPassed": true + } + ] +} diff --git a/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json b/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json new file mode 100644 index 00000000000..1b93f6127d0 --- /dev/null +++ b/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json @@ -0,0 +1,94 @@ +{ + "schema": "paperclip.native-completion-calibrated-comparison.v1", + "sources": { + "candidate": "d6e59e4712a3158ab4cd7d58deff1389b4578c21", + "baseline": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33" + }, + "gradedPairs": 6, + "pendingPairs": 0, + "newFailures": [], + "newPasses": [ + "native-completion.runner-codex.local.native-blocked-report" + ], + "unchangedFailures": [], + "unchangedPasses": [ + "native-completion.runner-acpx-claude.local.assigned-skill-explicit-invocation", + "native-completion.runner-acpx-claude.local.native-blocked-report", + "native-completion.runner-codex.local.assigned-skill-explicit-invocation", + "native-completion.runner-opencode.local.assigned-skill-explicit-invocation", + "native-completion.runner-opencode.local.native-blocked-report" + ], + "providerRuns": 12, + "maximumAttemptsObserved": 1, + "pairs": [ + { + "executionId": "native-completion.runner-acpx-claude.local.assigned-skill-explicit-invocation", + "baselineStatus": "passed", + "candidateStatus": "passed", + "baselineAgentMs": 27643, + "candidateAgentMs": 27682, + "baselineReportedUsd": 0, + "candidateReportedUsd": 0, + "baselineResultSha256": "6933dcb354f517653824b2c166bceef0f5c524ae30f915a9f4423276e53ebd15", + "candidateResultSha256": "97a7d6cd5eb5bb1568b9f8ada8ef21b83323e0644d5aa7a741eefa946f6389c6" + }, + { + "executionId": "native-completion.runner-acpx-claude.local.native-blocked-report", + "baselineStatus": "passed", + "candidateStatus": "passed", + "baselineAgentMs": 28241, + "candidateAgentMs": 29059, + "baselineReportedUsd": 0, + "candidateReportedUsd": 0, + "baselineResultSha256": "cdbd533c56af2b70ca9ef7650356396a45a64a6e8b9a6b370a3a50b8ea50e3fd", + "candidateResultSha256": "4cdce8a05751fbbc6d3711cd40dc7a4d807c5e93dd5b8008b100692517afaa52" + }, + { + "executionId": "native-completion.runner-codex.local.assigned-skill-explicit-invocation", + "baselineStatus": "passed", + "candidateStatus": "passed", + "baselineAgentMs": 31884, + "candidateAgentMs": 34018, + "baselineReportedUsd": 0, + "candidateReportedUsd": 0, + "baselineResultSha256": "22043de2aba3bac6ad94ee6d31a242746bad34a11790976d48ee0027b82c81b5", + "candidateResultSha256": "09077172094e8f63861739884ae42e1fb5755f113e9a18a53b21697055d25607" + }, + { + "executionId": "native-completion.runner-codex.local.native-blocked-report", + "baselineStatus": "failed", + "candidateStatus": "passed", + "baselineAgentMs": 17560, + "candidateAgentMs": 18794, + "baselineReportedUsd": 0, + "candidateReportedUsd": 0, + "baselineResultSha256": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f", + "candidateResultSha256": "60c8a4f3e41b8ca9a80c10f73d804c30d6e6ede9ace5ce3e93d2c4a053503f32" + }, + { + "executionId": "native-completion.runner-opencode.local.assigned-skill-explicit-invocation", + "baselineStatus": "passed", + "candidateStatus": "passed", + "baselineAgentMs": 27842, + "candidateAgentMs": 41645, + "baselineReportedUsd": 0.0029267655, + "candidateReportedUsd": 0.0027245575999999996, + "baselineResultSha256": "5f084208b3211b0caccc47705c08fad3ccba72fa4568882c78f8946ac26e4be6", + "candidateResultSha256": "f70d358b361e425fb4992b2101745a9912b2682134e780203ae5f79845dd5ee1" + }, + { + "executionId": "native-completion.runner-opencode.local.native-blocked-report", + "baselineStatus": "passed", + "candidateStatus": "passed", + "baselineAgentMs": 35963, + "candidateAgentMs": 39947, + "baselineReportedUsd": 0.0012880548, + "candidateReportedUsd": 0.0016493492000000001, + "baselineResultSha256": "d8e19a8cf7770ea6138cd4f816892a7a04e6160063bbcbb6f7693efe28278826", + "candidateResultSha256": "8c943b38d480fcb3513fe6a458e75d0606e4669ba047ecd3a779b35a65ec2a30" + } + ], + "detailedBaselineFailureAudit": "in_progress", + "actualBillingVerified": false, + "generalCodingQualityMeasured": false +} diff --git a/doc/plans/2026-10-02-native-completion-master-qualification.md b/doc/plans/2026-10-02-native-completion-master-qualification.md index d0e4b51b4aa..85665d8b737 100644 --- a/doc/plans/2026-10-02-native-completion-master-qualification.md +++ b/doc/plans/2026-10-02-native-completion-master-qualification.md @@ -1,3 +1,27 @@ +# Native completion qualification: retained paired task outcomes + +**TL;DR: no extra failing task outcomes were observed in the six matched pairs.** Candidate passes all six cases; historical descriptions pass five. Codex whole-task blocking improves Fail→Pass; the other five remain Pass→Pass. There are zero newly failing cells, zero unchanged failures and zero pending pairs. This is one bounded trial, not proof of general coding quality. + +The historical Codex blocker durably records the correct whole-task blocker and succeeds as a native run, but its visible reply omits the required marker. Its original executable failure is retained. The detailed provider-final versus semantic-fallback diagnosis is still being inspected; no cause is inferred from the grade alone. + +| Profile | Case | Historical | Candidate | Native run seconds: historical → candidate | +|---|---|---|---|---| +| runner-acpx-claude | assigned-skill-explicit-invocation | Passed | Passed | 27.6 → 27.7 | +| runner-acpx-claude | native-blocked-report | Passed | Passed | 28.2 → 29.1 | +| runner-codex | assigned-skill-explicit-invocation | Passed | Passed | 31.9 → 34.0 | +| runner-codex | native-blocked-report | Failed | Passed | 17.6 → 18.8 | +| runner-opencode | assigned-skill-explicit-invocation | Passed | Passed | 27.8 → 41.6 | +| runner-opencode | native-blocked-report | Passed | Passed | 36.0 → 39.9 | + +[Closed retained-result audit and hashes](2026-10-02-native-completion-calibrated-results/comparison.json), [candidate audit](2026-10-02-native-completion-calibrated-results/candidate-audit.json), and [historical audit](2026-10-02-native-completion-calibrated-results/baseline-audit.json) are inspectable. Original access-controlled artifact ZIPs/results remain unchanged. All twelve attempts are attempt 1 and cleanup passes. Candidate source is `d6e59e4712a3158ab4cd7d58deff1389b4578c21`; historical source is `e74ed61a69fbdd8b3a8f15dd6456bc3140246e33`. Both use suite hash `96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d` and the frozen matched configurations. + +Independent candidate retained-input replay agrees with all original grades: 51 strict native checks, 12 production-default/budget checks and 21 original assigned-skill document checks pass. The three requested documents are durably saved; the three blockers retain exact owner/action/scope and visible explanation, with no commanded file/deployment work. Named same-call finishing results precede provider finals with authoritative native acceptance/termination. Historical passed cases also retain their named sequence and document evidence; the failed Codex blocker exits before its additional native snapshot is written. + +Recorded model-cost subtotal is $0.00421482 historical and $0.00437391 candidate. Only OpenCode reports positive costs; Codex/Claude report zeros with unknown billing type. These are recorded subtotals, **not verified invoices or evidence of free calls**. Hosted execution cost is unmetered. Per-run token/count/timing coverage is retained, and every actual run is counted. Single-run timing differences are descriptive; they do not establish a performance trend. + +Exact-head normal CI and fresh Greptile 5/5 are green with no unresolved threads. The PR remains draft and unmerged while final evidence audit/publication completes. Fixed native prompt removal, resume behavior under paid models, provider feedback consumption and general coding quality are not measured by these single-turn cases. Native resume/catalog behavior has separate deterministic coverage. + +## Preserved admission and prior zero-provider cohorts # Native completion qualification: calibrated hosted boundary **TL;DR: model performance remains unmeasured; six task pairs are pending.** The candidate `d6e59e4712a3158ab4cd7d58deff1389b4578c21` and historical-description baseline `e74ed61a69fbdd8b3a8f15dd6456bc3140246e33` now pass complete actual launcher prepare→verify in both fresh local and explicitly synthetic hosted modes. The latter reproduces verified archive hydration, public source/run metadata and captured provenance framing without credentials or providers. It is a provider-free calibration, **not an actual trusted GitHub run**. diff --git a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md index 957014ab515..852e266cb0d 100644 --- a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md +++ b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md @@ -510,3 +510,5 @@ The selected #14961 `0a9c5a7` / `00a761b` hosted cohort also stopped before prov Selected #14961 now freezes `d6e59e471` / `e74ed61a6` after the approved five-file setup-only repair. Both exact refs pass actual launcher local and explicitly synthetic hosted prepare→verify (no credentials/providers): 132/127 selected TypeScript, 128 Node each, one local Rust calibration each; synthetic mode truthfully records no Rust execution. [The calibrated immutable manifest](2026-10-02-native-completion-calibrated-manifest.json) keeps exactly six pairs/twelve initial expected provider turns, strict oracles and single attempts. Prior twelve-cell zero-provider setup cohorts remain intact; model outcomes are pending. The first actual-provider #14961 comparison is dispatched once on separate immutable targets: [candidate37098728980](https://github.com/paperclipai/paperclip/actions/runs/37098728980) and [historical37098815696](https://github.com/paperclipai/paperclip/actions/runs/37098815696), same trusted workflow215586d. Builds overlap; zero graded pairs at dispatch, usage not yet available. Normal review resolved the publication/baseline approval provenance gates. No paid behavior retry or merge occurred. + +Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; detailed fallback/final diagnosis is pending. CI/review is green, PR stays draft/unmerged pending final audit. Actual invoiced costs and general coding quality remain unmeasured.