From 532066620b88e8731a5211fbe1cbc48ce8c7dd1a Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 3 Oct 2026 00:33:17 -0500 Subject: [PATCH] docs: clarify retained failed finish evidence limit Co-Authored-By: Paperclip --- .../comparison.json | 6 +++++- .../2026-10-02-native-completion-master-qualification.md | 2 +- doc/plans/2026-10-02-stock-harness-paperclip-checklist.md | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json b/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json index 8bf5977cd43..a38c37f928f 100644 --- a/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json +++ b/doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json @@ -126,7 +126,11 @@ "nativeRunCount": 1, "attempt": 1, "isCampaignRetry": false, - "allUsageRetained": true + "allUsageRetained": true, + "failureReasonObserved": false, + "firstFailedResultOutputBytes": 0, + "firstFailedResultOutputTruncated": false, + "nativeObservationSha256": "8604e2bf01bf9aaece2b3ca0b83f220b1f00911673d8eded1b1e9216910fb238" }, "nativeRunsObserved": 12, "actualVendorHttpCallCount": null, diff --git a/doc/plans/2026-10-02-native-completion-master-qualification.md b/doc/plans/2026-10-02-native-completion-master-qualification.md index d1888eafc94..c906996405b 100644 --- a/doc/plans/2026-10-02-native-completion-master-qualification.md +++ b/doc/plans/2026-10-02-native-completion-master-qualification.md @@ -19,7 +19,7 @@ Independent candidate retained-input replay agrees with all original grades: 51 Recorded model-cost subtotal is $0.00421482 historical and $0.00437391 candidate. Only OpenCode reports positive costs; Codex/Claude report zeros with unknown billing type. These are recorded subtotals, **not verified invoices or evidence of free calls**. Hosted execution cost is unmetered. Per-run token/count/timing coverage is retained, and every actual run is counted. Single-run timing differences are descriptive; they do not establish a performance trend. -The historical Codex completion makes one rejected `paperclip_finish` call, then corrects it successfully within the **same native run** (failed result seq161; successful result seq176). Both calls and their usage are retained. This is tool-feedback correction within attempt 1, not an extra harness run or campaign retry. The candidate has no corresponding failed terminal-call correction in the inspected six streams. +The historical Codex completion has one failed `paperclip_finish` call followed by a successful one within the **same native run** (failed result seq161; successful result seq176). The first public receipt has empty output and records no failure reason; a validation cause is not inferred. Both calls and their usage are retained. This is tool-feedback correction within attempt 1, not an extra harness run or campaign retry. The candidate has no corresponding failed terminal-call correction in the inspected six streams. [Candidate public HTML report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098728980-1/index.html) and [historical public HTML report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37098815696-1/index.html) include the trusted declared screenshots. The candidate workflow completes successfully; the historical workflow fails solely on its preserved Codex blocker cell. Their trusted workflow source is `215586d127e97c9301d86e769a39a15c13298ca2`, separate from the measured source revisions. diff --git a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md index 8ea0da8be91..e38b3b68c27 100644 --- a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md +++ b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md @@ -511,4 +511,4 @@ Selected #14961 now freezes `d6e59e471` / `e74ed61a6` after the approved five-fi The first actual-provider #14961 comparison is dispatched once on separate immutable targets: [candidate37098728980](https://github.com/paperclipai/paperclip/actions/runs/37098728980) and [historical37098815696](https://github.com/paperclipai/paperclip/actions/runs/37098815696), same trusted workflow215586d. Builds overlap; zero graded pairs at dispatch, usage not yet available. Normal review resolved the publication/baseline approval provenance gates. No paid behavior retry or merge occurred. -Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; independent final audit confirms the historical failure is a missing marker in the actual provider final and identical saved reply, not summary fallback. Its additional native grade/workspace digest was never written after the original browser failure; those missing assertions are not fabricated. The historical Codex completion also corrects a rejected finish call within the same run; all twelve runs/costs and original failures remain retained. Candidate six passes, no extra failing outcomes in six matched pairs, exact-head full CI/fresh 5/5 and zero unresolved threads make #14961 ready for review/merge consideration within this scope; no merge is performed here. Actual invoiced costs and general coding quality remain unmeasured. +Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; independent final audit confirms the historical failure is a missing marker in the actual provider final and identical saved reply, not summary fallback. Its additional native grade/workspace digest was never written after the original browser failure; those missing assertions are not fabricated. The historical Codex completion also has a failed finish call followed by success within the same run (failure reason not retained); all twelve runs/costs and original failures remain retained. Candidate six passes, no extra failing outcomes in six matched pairs, exact-head full CI/fresh 5/5 and zero unresolved threads make #14961 ready for review/merge consideration within this scope; no merge is performed here. Actual invoiced costs and general coding quality remain unmeasured.