mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-12 06:10:44 +01:00
docs: complete native matched qualification audit
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
dfa0e2cc9b
commit
a72856d101
5 files changed
+156
-19
No files matched your search
@@ -266,18 +266,19 @@
|
||||
"fixtureIdentityVerified": true,
|
||||
"onlyDeclaredElevenNativeSourcesDiffer": true,
|
||||
"fixedMasterContextVerified": true,
|
||||
"providerCallsSoFar": 12,
|
||||
"liveStatus": "all_original_cell_results_retained_final_audit_pending",
|
||||
"liveStatus": "complete_original_results_and_independent_audit",
|
||||
"workflowRuns": {
|
||||
"candidate": {
|
||||
"id": 37098728980,
|
||||
"url": "https://github.com/paperclipai/paperclip/actions/runs/37098728980",
|
||||
"createdAt": "2026-10-03T05:06:09Z"
|
||||
"createdAt": "2026-10-03T05:06:09Z",
|
||||
"conclusion": "success"
|
||||
},
|
||||
"baseline": {
|
||||
"id": 37098815696,
|
||||
"url": "https://github.com/paperclipai/paperclip/actions/runs/37098815696",
|
||||
"createdAt": "2026-10-03T05:07:44Z"
|
||||
"createdAt": "2026-10-03T05:07:44Z",
|
||||
"conclusion": "failure"
|
||||
}
|
||||
},
|
||||
"preservedZeroProviderCohorts": [
|
||||
@@ -308,5 +309,17 @@
|
||||
"gradedPairs": 6,
|
||||
"candidatePassedCells": 6,
|
||||
"baselinePassedCells": 5,
|
||||
"comparisonPath": "doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json"
|
||||
"comparisonPath": "doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json",
|
||||
"nativeRunsObserved": 12,
|
||||
"actualVendorHttpCallCount": null,
|
||||
"pendingPairs": 0,
|
||||
"newFailures": 0,
|
||||
"newPasses": 1,
|
||||
"unchangedPasses": 5,
|
||||
"publicReports": {
|
||||
"candidate": "https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098728980-1/index.html",
|
||||
"baseline": "https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098815696-1/index.html"
|
||||
},
|
||||
"originalResultFilesChanged": false,
|
||||
"behaviorRetries": 0
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema": "paperclip.native-completion.baseline-audit.v1",
|
||||
"asOf": "2026-10-03T05:23:16.220052+00:00",
|
||||
"asOf": "2026-10-03T05:27:34.593717+00:00",
|
||||
"workflowRunId": 37098815696,
|
||||
"workflowSourceSha": "215586d127e97c9301d86e769a39a15c13298ca2",
|
||||
"targetSourceSha": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33",
|
||||
@@ -185,6 +185,7 @@
|
||||
"terminalSeq": [
|
||||
104
|
||||
],
|
||||
"failedNamedToolResultSeqs": [],
|
||||
"independentNamedSequencePass": true,
|
||||
"persistedProviderFinalMatches": true,
|
||||
"finalMarkerCount": 1,
|
||||
@@ -195,7 +196,8 @@
|
||||
"workspaceChanged": false,
|
||||
"documentCount": 1,
|
||||
"interactionCount": 0,
|
||||
"eventCount": 112
|
||||
"eventCount": 112,
|
||||
"independentSingleRunPass": true
|
||||
},
|
||||
"perRunLedger": [
|
||||
{
|
||||
@@ -271,6 +273,15 @@
|
||||
"externalActualBillingKnown": false,
|
||||
"runtimeMetered": false
|
||||
},
|
||||
"screenshotsAudit": [
|
||||
{
|
||||
"file": "final-state.png",
|
||||
"publication": "public-runner-fixture",
|
||||
"sha256": "3b227e1d8f403ac3c6ac66601e1bf5693c0afe82a785e43c836f23b71a4dbbea",
|
||||
"declaredHashMatches": true,
|
||||
"visuallyInspectedByAgent": true
|
||||
}
|
||||
],
|
||||
"evidenceHashes": {
|
||||
"blob-report/report.zip": "c5b837c26c65c79bcbd5c1f67d8ee9fa67f2ec5862c59c99eaa75f233ab9486f",
|
||||
"evidence-manifest.json": "47446c71c349c1f8982e2fe47d178d5df01328247550c1059d8a180e61c71e27",
|
||||
@@ -481,6 +492,7 @@
|
||||
"terminalSeq": [
|
||||
84
|
||||
],
|
||||
"failedNamedToolResultSeqs": [],
|
||||
"independentNamedSequencePass": true,
|
||||
"persistedProviderFinalMatches": true,
|
||||
"finalMarkerCount": 1,
|
||||
@@ -498,7 +510,10 @@
|
||||
"workspaceChanged": false,
|
||||
"documentCount": 0,
|
||||
"interactionCount": 0,
|
||||
"eventCount": 92
|
||||
"eventCount": 92,
|
||||
"independentSingleRunPass": true,
|
||||
"independentDurableBlockerPass": true,
|
||||
"independentBlockedContentPass": true
|
||||
},
|
||||
"perRunLedger": [
|
||||
{
|
||||
@@ -525,6 +540,15 @@
|
||||
"externalActualBillingKnown": false,
|
||||
"runtimeMetered": false
|
||||
},
|
||||
"screenshotsAudit": [
|
||||
{
|
||||
"file": "final-state.png",
|
||||
"publication": "public-runner-fixture",
|
||||
"sha256": "3c9811d15b3c8c092f47600c1174a7788edeb3133c8ed60f6787f1ecab20c969",
|
||||
"declaredHashMatches": true,
|
||||
"visuallyInspectedByAgent": true
|
||||
}
|
||||
],
|
||||
"evidenceHashes": {
|
||||
"blob-report/report.zip": "11eeaa524f504ffd7e6b61e57b87fc30b5b9ded9132314379727bf98d21c2e10",
|
||||
"evidence-manifest.json": "22908c3a9fd1d8fedd27b73ac920adebf0aaa900c403bc60681f416f0b7ba6b7",
|
||||
@@ -716,6 +740,9 @@
|
||||
"terminalSeq": [
|
||||
209
|
||||
],
|
||||
"failedNamedToolResultSeqs": [
|
||||
161
|
||||
],
|
||||
"independentNamedSequencePass": true,
|
||||
"persistedProviderFinalMatches": true,
|
||||
"finalMarkerCount": 1,
|
||||
@@ -726,7 +753,8 @@
|
||||
"workspaceChanged": false,
|
||||
"documentCount": 1,
|
||||
"interactionCount": 0,
|
||||
"eventCount": 217
|
||||
"eventCount": 217,
|
||||
"independentSingleRunPass": true
|
||||
},
|
||||
"perRunLedger": [
|
||||
{
|
||||
@@ -802,6 +830,15 @@
|
||||
"externalActualBillingKnown": false,
|
||||
"runtimeMetered": false
|
||||
},
|
||||
"screenshotsAudit": [
|
||||
{
|
||||
"file": "final-state.png",
|
||||
"publication": "public-runner-fixture",
|
||||
"sha256": "f1b72104509399d3d9a04e7c11451a1712f9ba7444a23992e65c34a8df97fcfd",
|
||||
"declaredHashMatches": true,
|
||||
"visuallyInspectedByAgent": true
|
||||
}
|
||||
],
|
||||
"evidenceHashes": {
|
||||
"blob-report/report.zip": "b1825fc7e5edcf1d65bc0b510237561110dc9c91f0e3749844aa66115d602183",
|
||||
"evidence-manifest.json": "0bdca2fba590b30162921b0b2c11fe5ab964c2aff32687b7566738b58d8880da",
|
||||
@@ -966,6 +1003,7 @@
|
||||
"terminalSeq": [
|
||||
87
|
||||
],
|
||||
"failedNamedToolResultSeqs": [],
|
||||
"independentNamedSequencePass": true,
|
||||
"persistedProviderFinalMatches": true,
|
||||
"finalMarkerCount": 0,
|
||||
@@ -983,7 +1021,10 @@
|
||||
"workspaceChanged": null,
|
||||
"documentCount": 0,
|
||||
"interactionCount": 0,
|
||||
"eventCount": 95
|
||||
"eventCount": 95,
|
||||
"independentSingleRunPass": true,
|
||||
"independentDurableBlockerPass": true,
|
||||
"independentBlockedContentPass": false
|
||||
},
|
||||
"perRunLedger": [
|
||||
{
|
||||
@@ -1010,6 +1051,15 @@
|
||||
"externalActualBillingKnown": false,
|
||||
"runtimeMetered": false
|
||||
},
|
||||
"screenshotsAudit": [
|
||||
{
|
||||
"file": "failure.png",
|
||||
"publication": "public-runner-fixture",
|
||||
"sha256": "a45aa92e0826f9efaa84f5f4603eab9cbfba8bfa3d98cadab9c6db64fd3824a0",
|
||||
"declaredHashMatches": true,
|
||||
"visuallyInspectedByAgent": true
|
||||
}
|
||||
],
|
||||
"evidenceHashes": {
|
||||
"blob-report/report.zip": "e577c9c897fc92074b265ac8b68d719a0b110d46afd28ec230f16e8a88ebadc8",
|
||||
"evidence-manifest.json": "789ee2fa73629c6dbe7603996a8d2ec5fb9b9a4f2071aa996125b9476d68d65a",
|
||||
@@ -1204,6 +1254,7 @@
|
||||
"terminalSeq": [
|
||||
62
|
||||
],
|
||||
"failedNamedToolResultSeqs": [],
|
||||
"independentNamedSequencePass": true,
|
||||
"persistedProviderFinalMatches": true,
|
||||
"finalMarkerCount": 1,
|
||||
@@ -1214,7 +1265,8 @@
|
||||
"workspaceChanged": false,
|
||||
"documentCount": 1,
|
||||
"interactionCount": 0,
|
||||
"eventCount": 70
|
||||
"eventCount": 70,
|
||||
"independentSingleRunPass": true
|
||||
},
|
||||
"perRunLedger": [
|
||||
{
|
||||
@@ -1290,6 +1342,15 @@
|
||||
"externalActualBillingKnown": false,
|
||||
"runtimeMetered": false
|
||||
},
|
||||
"screenshotsAudit": [
|
||||
{
|
||||
"file": "final-state.png",
|
||||
"publication": "public-runner-fixture",
|
||||
"sha256": "fef8214feca49249941e158c7816488e8722e951156c46c83a371997936811b7",
|
||||
"declaredHashMatches": true,
|
||||
"visuallyInspectedByAgent": true
|
||||
}
|
||||
],
|
||||
"evidenceHashes": {
|
||||
"blob-report/report.zip": "786633c5e6017f51c910c98cbfc74f62af3e033bf193dae867f4d70e74f98570",
|
||||
"evidence-manifest.json": "6c23192583065f6392f48bb17e8ce917c3550e8742009d11c9330999065fa7a9",
|
||||
@@ -1499,6 +1560,7 @@
|
||||
"terminalSeq": [
|
||||
49
|
||||
],
|
||||
"failedNamedToolResultSeqs": [],
|
||||
"independentNamedSequencePass": true,
|
||||
"persistedProviderFinalMatches": true,
|
||||
"finalMarkerCount": 1,
|
||||
@@ -1516,7 +1578,10 @@
|
||||
"workspaceChanged": false,
|
||||
"documentCount": 0,
|
||||
"interactionCount": 0,
|
||||
"eventCount": 57
|
||||
"eventCount": 57,
|
||||
"independentSingleRunPass": true,
|
||||
"independentDurableBlockerPass": true,
|
||||
"independentBlockedContentPass": true
|
||||
},
|
||||
"perRunLedger": [
|
||||
{
|
||||
@@ -1543,6 +1608,15 @@
|
||||
"externalActualBillingKnown": false,
|
||||
"runtimeMetered": false
|
||||
},
|
||||
"screenshotsAudit": [
|
||||
{
|
||||
"file": "final-state.png",
|
||||
"publication": "public-runner-fixture",
|
||||
"sha256": "490baa12e6d4a0b7eed7febc659a8922167f54fee9695cdc96c5b773823b3f5e",
|
||||
"declaredHashMatches": true,
|
||||
"visuallyInspectedByAgent": true
|
||||
}
|
||||
],
|
||||
"evidenceHashes": {
|
||||
"blob-report/report.zip": "fcc0d066b282de5c18b6ee1628c2b420d57d0a94f54e091cd8d105072f96a68e",
|
||||
"evidence-manifest.json": "7e05f6f15f8a0bb9bcc9667b9b71d1a8556bf8cc1d64867eff216acbda74db5d",
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
"native-completion.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"native-completion.runner-opencode.local.native-blocked-report"
|
||||
],
|
||||
"providerRuns": 12,
|
||||
"maximumAttemptsObserved": 1,
|
||||
"pairs": [
|
||||
{
|
||||
@@ -88,7 +87,54 @@
|
||||
"candidateResultSha256": "8c943b38d480fcb3513fe6a458e75d0606e4669ba047ecd3a779b35a65ec2a30"
|
||||
}
|
||||
],
|
||||
"detailedBaselineFailureAudit": "in_progress",
|
||||
"detailedBaselineFailureAudit": {
|
||||
"status": "complete",
|
||||
"executionId": "native-completion.runner-codex.local.native-blocked-report",
|
||||
"evidenceSource": "retained-failure-api-state",
|
||||
"originalResultSha256": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f",
|
||||
"apiStateSha256": "161ae90f28b181cc041ec41dc3892a65b906f1174cf9ad2d0222f759164bbad0",
|
||||
"originalMachineStatus": "failed",
|
||||
"originalFailureClass": "candidate_failure",
|
||||
"originalNativeGradeAvailable": false,
|
||||
"originalNativeObservationAvailable": false,
|
||||
"actualProviderFinal": "Task blocked. Owner: Release Owner. Unblock action: Grant deployment access.",
|
||||
"presentationSource": "final_agent_message",
|
||||
"finalIsSemanticSummaryFallback": false,
|
||||
"requiredMarkerCountInProviderFinal": 0,
|
||||
"nativeSemanticSummaryIncludesRequiredMarker": true,
|
||||
"persistedReplyMatchesProviderFinal": true,
|
||||
"independentDurableBlockerPass": true,
|
||||
"independentNamedSequencePass": true,
|
||||
"finishingStartSeq": 59,
|
||||
"proposalSeq": 61,
|
||||
"finishingResultSeq": 62,
|
||||
"providerFinalSeq": 81,
|
||||
"acceptedSeq": 86,
|
||||
"terminalSeq": 87,
|
||||
"workspaceBeforeAfterEvidenceAvailable": false,
|
||||
"unexecutedOriginalNativeAssertionsNotRegraded": true,
|
||||
"causationEstablished": false
|
||||
},
|
||||
"actualBillingVerified": false,
|
||||
"generalCodingQualityMeasured": false
|
||||
"generalCodingQualityMeasured": false,
|
||||
"withinRunFeedbackCorrection": {
|
||||
"executionId": "native-completion.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
"variant": "baseline",
|
||||
"namedFinishingStarts": 2,
|
||||
"failedFinishingResultSeq": 161,
|
||||
"successfulFinishingResultSeq": 176,
|
||||
"nativeRunCount": 1,
|
||||
"attempt": 1,
|
||||
"isCampaignRetry": false,
|
||||
"allUsageRetained": true
|
||||
},
|
||||
"nativeRunsObserved": 12,
|
||||
"actualVendorHttpCallCount": null,
|
||||
"finalAuditComplete": true,
|
||||
"originalResultsChanged": false,
|
||||
"readinessWithinMeasuredScope": true,
|
||||
"auditSha256": {
|
||||
"candidate-audit.json": "4032c133114bed52c63ec083d966afa70f260013277f416f7bdae7aaa4650d97",
|
||||
"baseline-audit.json": "bd6bed38b6289b6473762ad6a761a8efb31fbd403bf813974fe3d5a1762b2ad0"
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
**TL;DR: no extra failing task outcomes were observed in the six matched pairs.** Candidate passes all six cases; historical descriptions pass five. Codex whole-task blocking improves Fail→Pass; the other five remain Pass→Pass. There are zero newly failing cells, zero unchanged failures and zero pending pairs. This is one bounded trial, not proof of general coding quality.
|
||||
|
||||
The historical Codex blocker durably records the correct whole-task blocker and succeeds as a native run, but its visible reply omits the required marker. Its original executable failure is retained. The detailed provider-final versus semantic-fallback diagnosis is still being inspected; no cause is inferred from the grade alone.
|
||||
The historical Codex blocker durably records the correct whole-task blocker and succeeds as a native run, but its **actual provider final** omits the required marker: “Task blocked. Owner: Release Owner. Unblock action: Grant deployment access.” The run-attributed saved reply is byte-identical to that final; presentation selects `final_agent_message`. Its native semantic summary includes the marker, so this is an observed final-output defect, not semantic-summary fallback or absence of a final. The original matcher/browser failure remains unchanged. One trial does not establish that the description change caused the improvement.
|
||||
|
||||
| Profile | Case | Historical | Candidate | Native run seconds: historical → candidate |
|
||||
|---|---|---|---|---|
|
||||
@@ -15,11 +15,15 @@ The historical Codex blocker durably records the correct whole-task blocker and
|
||||
|
||||
[Closed retained-result audit and hashes](2026-10-02-native-completion-calibrated-results/comparison.json), [candidate audit](2026-10-02-native-completion-calibrated-results/candidate-audit.json), and [historical audit](2026-10-02-native-completion-calibrated-results/baseline-audit.json) are inspectable. Original access-controlled artifact ZIPs/results remain unchanged. All twelve attempts are attempt 1 and cleanup passes. Candidate source is `d6e59e4712a3158ab4cd7d58deff1389b4578c21`; historical source is `e74ed61a69fbdd8b3a8f15dd6456bc3140246e33`. Both use suite hash `96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d` and the frozen matched configurations.
|
||||
|
||||
Independent candidate retained-input replay agrees with all original grades: 51 strict native checks, 12 production-default/budget checks and 21 original assigned-skill document checks pass. The three requested documents are durably saved; the three blockers retain exact owner/action/scope and visible explanation, with no commanded file/deployment work. Named same-call finishing results precede provider finals with authoritative native acceptance/termination. Historical passed cases also retain their named sequence and document evidence; the failed Codex blocker exits before its additional native snapshot is written.
|
||||
Independent candidate retained-input replay agrees with all original grades: 51 strict native checks, 12 production-default/budget checks and 21 original assigned-skill document checks pass. The three requested documents are durably saved; the three blockers retain exact owner/action/scope and visible explanation, with no commanded file/deployment work. Named same-call finishing results precede provider finals with authoritative native acceptance/termination. Historical passed cases also retain their named sequence and document evidence. The failed historical Codex blocker exits before its additional native snapshot/grade is written. A separately labeled audit of its retained API/PRP stream confirms the named start (seq59), proposal (61), matching result (62), actual provider final (81), control-plane acceptance (86) and terminal (87), the correct durable owner/action/scope, and the missing final marker. It does **not** manufacture the unexecuted original native grade or a before/after workspace digest. Public state captures zero documents/interactions for that failed blocker.
|
||||
|
||||
Recorded model-cost subtotal is $0.00421482 historical and $0.00437391 candidate. Only OpenCode reports positive costs; Codex/Claude report zeros with unknown billing type. These are recorded subtotals, **not verified invoices or evidence of free calls**. Hosted execution cost is unmetered. Per-run token/count/timing coverage is retained, and every actual run is counted. Single-run timing differences are descriptive; they do not establish a performance trend.
|
||||
|
||||
Exact-head normal CI and fresh Greptile 5/5 are green with no unresolved threads. The PR remains draft and unmerged while final evidence audit/publication completes. Fixed native prompt removal, resume behavior under paid models, provider feedback consumption and general coding quality are not measured by these single-turn cases. Native resume/catalog behavior has separate deterministic coverage.
|
||||
The historical Codex completion makes one rejected `paperclip_finish` call, then corrects it successfully within the **same native run** (failed result seq161; successful result seq176). Both calls and their usage are retained. This is tool-feedback correction within attempt 1, not an extra harness run or campaign retry. The candidate has no corresponding failed terminal-call correction in the inspected six streams.
|
||||
|
||||
[Candidate public HTML report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098728980-1/index.html) and [historical public HTML report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098815696-1/index.html) include the trusted declared screenshots. The candidate workflow completes successfully; the historical workflow fails solely on its preserved Codex blocker cell. Their trusted workflow source is `215586d127e97c9301d86e769a39a15c13298ca2`, separate from the measured source revisions.
|
||||
|
||||
**Merge recommendation within this scope:** PR #14961 has six candidate passes, no extra failing outcomes in these six pairs, unchanged master defaults, complete retained audit, green exact-head CI and fresh Greptile 5/5 with zero unresolved threads. It is ready for review/merge consideration; merging remains the user's decision. Source head `d6e59e471` and both measured refs are unchanged. Fixed native prompt removal, resume behavior under paid models, provider feedback consumption and general coding quality are not measured by these single-turn cases. Native resume/catalog behavior has separate deterministic coverage.
|
||||
|
||||
## Preserved admission and prior zero-provider cohorts
|
||||
# Native completion qualification: calibrated hosted boundary
|
||||
|
||||
@@ -511,4 +511,4 @@ Selected #14961 now freezes `d6e59e471` / `e74ed61a6` after the approved five-fi
|
||||
|
||||
The first actual-provider #14961 comparison is dispatched once on separate immutable targets: [candidate37098728980](https://github.com/paperclipai/paperclip/actions/runs/37098728980) and [historical37098815696](https://github.com/paperclipai/paperclip/actions/runs/37098815696), same trusted workflow215586d. Builds overlap; zero graded pairs at dispatch, usage not yet available. Normal review resolved the publication/baseline approval provenance gates. No paid behavior retry or merge occurred.
|
||||
|
||||
Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; detailed fallback/final diagnosis is pending. CI/review is green, PR stays draft/unmerged pending final audit. Actual invoiced costs and general coding quality remain unmeasured.
|
||||
Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; independent final audit confirms the historical failure is a missing marker in the actual provider final and identical saved reply, not summary fallback. Its additional native grade/workspace digest was never written after the original browser failure; those missing assertions are not fabricated. The historical Codex completion also corrects a rejected finish call within the same run; all twelve runs/costs and original failures remain retained. Candidate six passes, no extra failing outcomes in six matched pairs, exact-head full CI/fresh 5/5 and zero unresolved threads make #14961 ready for review/merge consideration within this scope; no merge is performed here. Actual invoiced costs and general coding quality remain unmeasured.
|
||||
Reference in new issue
Block a user