diff --git a/doc/plans/2026-10-02-native-completion-calibrated-manifest.json b/doc/plans/2026-10-02-native-completion-calibrated-manifest.json index c16d5a5dbd1..bc1c522f2b7 100644 --- a/doc/plans/2026-10-02-native-completion-calibrated-manifest.json +++ b/doc/plans/2026-10-02-native-completion-calibrated-manifest.json @@ -266,18 +266,19 @@ "fixtureIdentityVerified": true, "onlyDeclaredElevenNativeSourcesDiffer": true, "fixedMasterContextVerified": true, - "providerCallsSoFar": 12, - "liveStatus": "all_original_cell_results_retained_final_audit_pending", + "liveStatus": "complete_original_results_and_independent_audit", "workflowRuns": { "candidate": { "id": 37098728980, "url": "https://github.com/paperclipai/paperclip/actions/runs/37098728980", - "createdAt": "2026-10-03T05:06:09Z" + "createdAt": "2026-10-03T05:06:09Z", + "conclusion": "success" }, "baseline": { "id": 37098815696, "url": "https://github.com/paperclipai/paperclip/actions/runs/37098815696", - "createdAt": "2026-10-03T05:07:44Z" + "createdAt": "2026-10-03T05:07:44Z", + "conclusion": "failure" } }, "preservedZeroProviderCohorts": [ @@ -308,5 +309,17 @@ "gradedPairs": 6, "candidatePassedCells": 6, "baselinePassedCells": 5, - "comparisonPath": "doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json" + "comparisonPath": "doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json", + "nativeRunsObserved": 12, + "actualVendorHttpCallCount": null, + "pendingPairs": 0, + "newFailures": 0, + "newPasses": 1, + "unchangedPasses": 5, + "publicReports": { + "candidate": "https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098728980-1/index.html", + "baseline": "https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098815696-1/index.html" + }, + "originalResultFilesChanged": false, + "behaviorRetries": 0 } diff --git a/doc/plans/2026-10-02-native-completion-calibrated-results/baseline-audit.json b/doc/plans/2026-10-02-native-completion-calibrated-results/baseline-audit.json index 8d6f6af7dd2..461c56aac58 100644 --- a/doc/plans/2026-10-02-native-completion-calibrated-results/baseline-audit.json +++ b/doc/plans/2026-10-02-native-completion-calibrated-results/baseline-audit.json @@ -1,6 +1,6 @@ { "schema": "paperclip.native-completion.baseline-audit.v1", - "asOf": "2026-10-03T05:23:16.220052+00:00", + "asOf": "2026-10-03T05:27:34.593717+00:00", "workflowRunId": 37098815696, "workflowSourceSha": "215586d127e97c9301d86e769a39a15c13298ca2", "targetSourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33", @@ -185,6 +185,7 @@ "terminalSeq": [ 104 ], + "failedNamedToolResultSeqs": [], "independentNamedSequencePass": true, "persistedProviderFinalMatches": true, "finalMarkerCount": 1, @@ -195,7 +196,8 @@ "workspaceChanged": false, "documentCount": 1, "interactionCount": 0, - "eventCount": 112 + "eventCount": 112, + "independentSingleRunPass": true }, "perRunLedger": [ { @@ -271,6 +273,15 @@ "externalActualBillingKnown": false, "runtimeMetered": false }, + "screenshotsAudit": [ + { + "file": "final-state.png", + "publication": "public-runner-fixture", + "sha256": "3b227e1d8f403ac3c6ac66601e1bf5693c0afe82a785e43c836f23b71a4dbbea", + "declaredHashMatches": true, + "visuallyInspectedByAgent": true + } + ], "evidenceHashes": { "blob-report/report.zip": "c5b837c26c65c79bcbd5c1f67d8ee9fa67f2ec5862c59c99eaa75f233ab9486f", "evidence-manifest.json": "47446c71c349c1f8982e2fe47d178d5df01328247550c1059d8a180e61c71e27", @@ -481,6 +492,7 @@ "terminalSeq": [ 84 ], + "failedNamedToolResultSeqs": [], "independentNamedSequencePass": true, "persistedProviderFinalMatches": true, "finalMarkerCount": 1, @@ -498,7 +510,10 @@ "workspaceChanged": false, "documentCount": 0, "interactionCount": 0, - "eventCount": 92 + "eventCount": 92, + "independentSingleRunPass": true, + "independentDurableBlockerPass": true, + "independentBlockedContentPass": true }, "perRunLedger": [ { @@ -525,6 +540,15 @@ "externalActualBillingKnown": false, "runtimeMetered": false }, + "screenshotsAudit": [ + { + "file": "final-state.png", + "publication": "public-runner-fixture", + "sha256": "3c9811d15b3c8c092f47600c1174a7788edeb3133c8ed60f6787f1ecab20c969", + "declaredHashMatches": true, + "visuallyInspectedByAgent": true + } + ], "evidenceHashes": { "blob-report/report.zip": "11eeaa524f504ffd7e6b61e57b87fc30b5b9ded9132314379727bf98d21c2e10", "evidence-manifest.json": "22908c3a9fd1d8fedd27b73ac920adebf0aaa900c403bc60681f416f0b7ba6b7", @@ -716,6 +740,9 @@ "terminalSeq": [ 209 ], + "failedNamedToolResultSeqs": [ + 161 + ], "independentNamedSequencePass": true, "persistedProviderFinalMatches": true, "finalMarkerCount": 1, @@ -726,7 +753,8 @@ "workspaceChanged": false, "documentCount": 1, "interactionCount": 0, - "eventCount": 217 + "eventCount": 217, + "independentSingleRunPass": true }, "perRunLedger": [ { @@ -802,6 +830,15 @@ "externalActualBillingKnown": false, "runtimeMetered": false }, + "screenshotsAudit": [ + { + "file": "final-state.png", + "publication": "public-runner-fixture", + "sha256": "f1b72104509399d3d9a04e7c11451a1712f9ba7444a23992e65c34a8df97fcfd", + "declaredHashMatches": true, + "visuallyInspectedByAgent": true + } + ], "evidenceHashes": { "blob-report/report.zip": "b1825fc7e5edcf1d65bc0b510237561110dc9c91f0e3749844aa66115d602183", "evidence-manifest.json": "0bdca2fba590b30162921b0b2c11fe5ab964c2aff32687b7566738b58d8880da", @@ -966,6 +1003,7 @@ "terminalSeq": [ 87 ], + "failedNamedToolResultSeqs": [], "independentNamedSequencePass": true, "persistedProviderFinalMatches": true, "finalMarkerCount": 0, @@ -983,7 +1021,10 @@ "workspaceChanged": null, "documentCount": 0, "interactionCount": 0, - "eventCount": 95 + "eventCount": 95, + "independentSingleRunPass": true, + "independentDurableBlockerPass": true, + "independentBlockedContentPass": false }, "perRunLedger": [ { @@ -1010,6 +1051,15 @@ "externalActualBillingKnown": false, "runtimeMetered": false }, + "screenshotsAudit": [ + { + "file": "failure.png", + "publication": "public-runner-fixture", + "sha256": "a45aa92e0826f9efaa84f5f4603eab9cbfba8bfa3d98cadab9c6db64fd3824a0", + "declaredHashMatches": true, + "visuallyInspectedByAgent": true + } + ], "evidenceHashes": { "blob-report/report.zip": "e577c9c897fc92074b265ac8b68d719a0b110d46afd28ec230f16e8a88ebadc8", "evidence-manifest.json": "789ee2fa73629c6dbe7603996a8d2ec5fb9b9a4f2071aa996125b9476d68d65a", @@ -1204,6 +1254,7 @@ "terminalSeq": [ 62 ], + "failedNamedToolResultSeqs": [], "independentNamedSequencePass": true, "persistedProviderFinalMatches": true, "finalMarkerCount": 1, @@ -1214,7 +1265,8 @@ "workspaceChanged": false, "documentCount": 1, "interactionCount": 0, - "eventCount": 70 + "eventCount": 70, + "independentSingleRunPass": true }, "perRunLedger": [ { @@ -1290,6 +1342,15 @@ "externalActualBillingKnown": false, "runtimeMetered": false }, + "screenshotsAudit": [ + { + "file": "final-state.png", + "publication": "public-runner-fixture", + "sha256": "fef8214feca49249941e158c7816488e8722e951156c46c83a371997936811b7", + "declaredHashMatches": true, + "visuallyInspectedByAgent": true + } + ], "evidenceHashes": { "blob-report/report.zip": "786633c5e6017f51c910c98cbfc74f62af3e033bf193dae867f4d70e74f98570", "evidence-manifest.json": "6c23192583065f6392f48bb17e8ce917c3550e8742009d11c9330999065fa7a9", @@ -1499,6 +1560,7 @@ "terminalSeq": [ 49 ], + "failedNamedToolResultSeqs": [], "independentNamedSequencePass": true, "persistedProviderFinalMatches": true, "finalMarkerCount": 1, @@ -1516,7 +1578,10 @@ "workspaceChanged": false, "documentCount": 0, "interactionCount": 0, - "eventCount": 57 + "eventCount": 57, + "independentSingleRunPass": true, + "independentDurableBlockerPass": true, + "independentBlockedContentPass": true }, "perRunLedger": [ { @@ -1543,6 +1608,15 @@ "externalActualBillingKnown": false, "runtimeMetered": false }, + "screenshotsAudit": [ + { + "file": "final-state.png", + "publication": "public-runner-fixture", + "sha256": "490baa12e6d4a0b7eed7febc659a8922167f54fee9695cdc96c5b773823b3f5e", + "declaredHashMatches": true, + "visuallyInspectedByAgent": true + } + ], "evidenceHashes": { "blob-report/report.zip": "fcc0d066b282de5c18b6ee1628c2b420d57d0a94f54e091cd8d105072f96a68e", "evidence-manifest.json": "7e05f6f15f8a0bb9bcc9667b9b71d1a8556bf8cc1d64867eff216acbda74db5d", diff --git a/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json b/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json index 1b93f6127d0..8bf5977cd43 100644 --- a/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json +++ b/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json @@ -18,7 +18,6 @@ "native-completion.runner-opencode.local.assigned-skill-explicit-invocation", "native-completion.runner-opencode.local.native-blocked-report" ], - "providerRuns": 12, "maximumAttemptsObserved": 1, "pairs": [ { @@ -88,7 +87,54 @@ "candidateResultSha256": "8c943b38d480fcb3513fe6a458e75d0606e4669ba047ecd3a779b35a65ec2a30" } ], - "detailedBaselineFailureAudit": "in_progress", + "detailedBaselineFailureAudit": { + "status": "complete", + "executionId": "native-completion.runner-codex.local.native-blocked-report", + "evidenceSource": "retained-failure-api-state", + "originalResultSha256": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f", + "apiStateSha256": "161ae90f28b181cc041ec41dc3892a65b906f1174cf9ad2d0222f759164bbad0", + "originalMachineStatus": "failed", + "originalFailureClass": "candidate_failure", + "originalNativeGradeAvailable": false, + "originalNativeObservationAvailable": false, + "actualProviderFinal": "Task blocked. Owner: Release Owner. Unblock action: Grant deployment access.", + "presentationSource": "final_agent_message", + "finalIsSemanticSummaryFallback": false, + "requiredMarkerCountInProviderFinal": 0, + "nativeSemanticSummaryIncludesRequiredMarker": true, + "persistedReplyMatchesProviderFinal": true, + "independentDurableBlockerPass": true, + "independentNamedSequencePass": true, + "finishingStartSeq": 59, + "proposalSeq": 61, + "finishingResultSeq": 62, + "providerFinalSeq": 81, + "acceptedSeq": 86, + "terminalSeq": 87, + "workspaceBeforeAfterEvidenceAvailable": false, + "unexecutedOriginalNativeAssertionsNotRegraded": true, + "causationEstablished": false + }, "actualBillingVerified": false, - "generalCodingQualityMeasured": false + "generalCodingQualityMeasured": false, + "withinRunFeedbackCorrection": { + "executionId": "native-completion.runner-codex.local.assigned-skill-explicit-invocation", + "variant": "baseline", + "namedFinishingStarts": 2, + "failedFinishingResultSeq": 161, + "successfulFinishingResultSeq": 176, + "nativeRunCount": 1, + "attempt": 1, + "isCampaignRetry": false, + "allUsageRetained": true + }, + "nativeRunsObserved": 12, + "actualVendorHttpCallCount": null, + "finalAuditComplete": true, + "originalResultsChanged": false, + "readinessWithinMeasuredScope": true, + "auditSha256": { + "candidate-audit.json": "4032c133114bed52c63ec083d966afa70f260013277f416f7bdae7aaa4650d97", + "baseline-audit.json": "bd6bed38b6289b6473762ad6a761a8efb31fbd403bf813974fe3d5a1762b2ad0" + } } diff --git a/doc/plans/2026-10-02-native-completion-master-qualification.md b/doc/plans/2026-10-02-native-completion-master-qualification.md index 85665d8b737..d1888eafc94 100644 --- a/doc/plans/2026-10-02-native-completion-master-qualification.md +++ b/doc/plans/2026-10-02-native-completion-master-qualification.md @@ -2,7 +2,7 @@ **TL;DR: no extra failing task outcomes were observed in the six matched pairs.** Candidate passes all six cases; historical descriptions pass five. Codex whole-task blocking improves Fail→Pass; the other five remain Pass→Pass. There are zero newly failing cells, zero unchanged failures and zero pending pairs. This is one bounded trial, not proof of general coding quality. -The historical Codex blocker durably records the correct whole-task blocker and succeeds as a native run, but its visible reply omits the required marker. Its original executable failure is retained. The detailed provider-final versus semantic-fallback diagnosis is still being inspected; no cause is inferred from the grade alone. +The historical Codex blocker durably records the correct whole-task blocker and succeeds as a native run, but its **actual provider final** omits the required marker: “Task blocked. Owner: Release Owner. Unblock action: Grant deployment access.” The run-attributed saved reply is byte-identical to that final; presentation selects `final_agent_message`. Its native semantic summary includes the marker, so this is an observed final-output defect, not semantic-summary fallback or absence of a final. The original matcher/browser failure remains unchanged. One trial does not establish that the description change caused the improvement. | Profile | Case | Historical | Candidate | Native run seconds: historical → candidate | |---|---|---|---|---| @@ -15,11 +15,15 @@ The historical Codex blocker durably records the correct whole-task blocker and [Closed retained-result audit and hashes](2026-10-02-native-completion-calibrated-results/comparison.json), [candidate audit](2026-10-02-native-completion-calibrated-results/candidate-audit.json), and [historical audit](2026-10-02-native-completion-calibrated-results/baseline-audit.json) are inspectable. Original access-controlled artifact ZIPs/results remain unchanged. All twelve attempts are attempt 1 and cleanup passes. Candidate source is `d6e59e4712a3158ab4cd7d58deff1389b4578c21`; historical source is `e74ed61a69fbdd8b3a8f15dd6456bc3140246e33`. Both use suite hash `96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d` and the frozen matched configurations. -Independent candidate retained-input replay agrees with all original grades: 51 strict native checks, 12 production-default/budget checks and 21 original assigned-skill document checks pass. The three requested documents are durably saved; the three blockers retain exact owner/action/scope and visible explanation, with no commanded file/deployment work. Named same-call finishing results precede provider finals with authoritative native acceptance/termination. Historical passed cases also retain their named sequence and document evidence; the failed Codex blocker exits before its additional native snapshot is written. +Independent candidate retained-input replay agrees with all original grades: 51 strict native checks, 12 production-default/budget checks and 21 original assigned-skill document checks pass. The three requested documents are durably saved; the three blockers retain exact owner/action/scope and visible explanation, with no commanded file/deployment work. Named same-call finishing results precede provider finals with authoritative native acceptance/termination. Historical passed cases also retain their named sequence and document evidence. The failed historical Codex blocker exits before its additional native snapshot/grade is written. A separately labeled audit of its retained API/PRP stream confirms the named start (seq59), proposal (61), matching result (62), actual provider final (81), control-plane acceptance (86) and terminal (87), the correct durable owner/action/scope, and the missing final marker. It does **not** manufacture the unexecuted original native grade or a before/after workspace digest. Public state captures zero documents/interactions for that failed blocker. Recorded model-cost subtotal is $0.00421482 historical and $0.00437391 candidate. Only OpenCode reports positive costs; Codex/Claude report zeros with unknown billing type. These are recorded subtotals, **not verified invoices or evidence of free calls**. Hosted execution cost is unmetered. Per-run token/count/timing coverage is retained, and every actual run is counted. Single-run timing differences are descriptive; they do not establish a performance trend. -Exact-head normal CI and fresh Greptile 5/5 are green with no unresolved threads. The PR remains draft and unmerged while final evidence audit/publication completes. Fixed native prompt removal, resume behavior under paid models, provider feedback consumption and general coding quality are not measured by these single-turn cases. Native resume/catalog behavior has separate deterministic coverage. +The historical Codex completion makes one rejected `paperclip_finish` call, then corrects it successfully within the **same native run** (failed result seq161; successful result seq176). Both calls and their usage are retained. This is tool-feedback correction within attempt 1, not an extra harness run or campaign retry. The candidate has no corresponding failed terminal-call correction in the inspected six streams. + +[Candidate public HTML report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098728980-1/index.html) and [historical public HTML report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098815696-1/index.html) include the trusted declared screenshots. The candidate workflow completes successfully; the historical workflow fails solely on its preserved Codex blocker cell. Their trusted workflow source is `215586d127e97c9301d86e769a39a15c13298ca2`, separate from the measured source revisions. + +**Merge recommendation within this scope:** PR #14961 has six candidate passes, no extra failing outcomes in these six pairs, unchanged master defaults, complete retained audit, green exact-head CI and fresh Greptile 5/5 with zero unresolved threads. It is ready for review/merge consideration; merging remains the user's decision. Source head `d6e59e471` and both measured refs are unchanged. Fixed native prompt removal, resume behavior under paid models, provider feedback consumption and general coding quality are not measured by these single-turn cases. Native resume/catalog behavior has separate deterministic coverage. ## Preserved admission and prior zero-provider cohorts # Native completion qualification: calibrated hosted boundary diff --git a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md index 852e266cb0d..8ea0da8be91 100644 --- a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md +++ b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md @@ -511,4 +511,4 @@ Selected #14961 now freezes `d6e59e471` / `e74ed61a6` after the approved five-fi The first actual-provider #14961 comparison is dispatched once on separate immutable targets: [candidate37098728980](https://github.com/paperclipai/paperclip/actions/runs/37098728980) and [historical37098815696](https://github.com/paperclipai/paperclip/actions/runs/37098815696), same trusted workflow215586d. Builds overlap; zero graded pairs at dispatch, usage not yet available. Normal review resolved the publication/baseline approval provenance gates. No paid behavior retry or merge occurred. -Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; detailed fallback/final diagnosis is pending. CI/review is green, PR stays draft/unmerged pending final audit. Actual invoiced costs and general coding quality remain unmeasured. +Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; independent final audit confirms the historical failure is a missing marker in the actual provider final and identical saved reply, not summary fallback. Its additional native grade/workspace digest was never written after the original browser failure; those missing assertions are not fabricated. The historical Codex completion also corrects a rejected finish call within the same run; all twelve runs/costs and original failures remain retained. Candidate six passes, no extra failing outcomes in six matched pairs, exact-head full CI/fresh 5/5 and zero unresolved threads make #14961 ready for review/merge consideration within this scope; no merge is performed here. Actual invoiced costs and general coding quality remain unmeasured.