mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
docs(evals): close the unchanged-source baseline recovery
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
1f54c3b75a
commit
9e0df9098c
4 files changed
+34
-25
No files matched your search
@@ -52,6 +52,7 @@
|
||||
{"executionId":"stock-harness.legacy-codex.local.ordered-comment-continuation","profile":"legacy-codex","case":"ordered-comment-continuation","sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceRef":"refs/heads/codex/stock-harness-previous-instructions","workflowRunUrl":"https://github.com/paperclipai/paperclip/actions/runs/37042864888","cellAttempt":1,"status":"passed","failureClass":null,"causeCodes":[],"failedMatcherPaths":[],"model":"gpt-5.6-sol","runtimeMode":"legacy","durationMs":74114,"providerDurationMs":57150,"cleanup":"passed","billing":{"llm":{"runCount":2,"runsWithTokenUsage":2,"runsWithReportedCost":0,"inputTokens":628626,"outputTokens":7098,"cachedInputTokens":563211,"totalTokens":1198935,"reportedCostUsd":0,"costStatus":"unpriced"},"runtime":{"provider":"local","agentRunDurationMs":57150,"leaseDurationMs":null,"leaseCount":0,"costStatus":"not_metered","costSource":"local_not_metered"},"reportedCostUsd":0,"estimatedRuntimeCostUsd":0,"observedAndEstimatedCostUsd":0,"complete":false},"billingTypes":["unavailable"],"stockChecks":{"budget-hard-stops":true,"default-hire-bundle":true,"provider-runs-present":true,"invocation-evidence-complete":true,"historical-generic-procedures-observed":true,"fresh-default-delivered":true},"invocationMetrics":[{"capturedPromptChars":9811,"reportedPromptChars":9785,"freshPromptChars":0,"retrievalClipped":false},{"capturedPromptChars":12746,"reportedPromptChars":12746,"freshPromptChars":4683,"retrievalClipped":false}],"prerequisite":{"sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceFingerprint":"2e5f22848450179ac2d573662c718300f86a935d09d15204c9d97e98f65b53f4","passed":true,"providerCallsBeforeAdmission":0,"passedTests":536},"evidenceHashes":{"result.json":"56a457fcc7a36809eb4bfcedeff2fa079f5272754ff52fc453e8bf8d813e52f3","evidence-manifest.json":"bf17fb7d2a05f7e99e0723907bdc57f0a88bf6838d68f31f4b84dc6ebc62d3c7","snapshots/context-integrity.json":"a9156b810e32fa1d9d07826276a00c62ebe4f25ac3dc284ada157f2b2d72d800","snapshots/stock-harness-hire.json":"5fc8abd51d401c96d701b0862ac2dc674eb36244fdc25c49b67ba3947ba5a2fb","snapshots/stock-harness.json":"c72b8c53e1e50a82ab6ff815b363942936440cfff75f8f8ca5f54185c723889d","snapshots/stock-harness-preflight.json":"b14d304196cdc3146436c66199243c6da04784c13e45c9bd31e0ef83e97a29bc"},"taskDocumentCount":1,"behaviorCheckCounts":{"passed":12,"failed":0}},
|
||||
{"executionId":"stock-harness.legacy-opencode.local.assigned-skill-explicit-invocation","profile":"legacy-opencode","case":"assigned-skill-explicit-invocation","sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceRef":"refs/heads/codex/stock-harness-previous-instructions","workflowRunUrl":"https://github.com/paperclipai/paperclip/actions/runs/37042864888","cellAttempt":1,"status":"passed","failureClass":null,"causeCodes":[],"failedMatcherPaths":[],"model":"openrouter/deepseek/deepseek-v4-flash-0731","runtimeMode":"legacy","durationMs":86384,"providerDurationMs":68956,"cleanup":"passed","billing":{"llm":{"runCount":1,"runsWithTokenUsage":1,"runsWithReportedCost":1,"inputTokens":57002,"outputTokens":2365,"cachedInputTokens":223232,"totalTokens":282599,"reportedCostUsd":0.0044563934000000005,"costStatus":"reported"},"runtime":{"provider":"local","agentRunDurationMs":68956,"leaseDurationMs":null,"leaseCount":0,"costStatus":"not_metered","costSource":"local_not_metered"},"reportedCostUsd":0.0044563934000000005,"estimatedRuntimeCostUsd":0,"observedAndEstimatedCostUsd":0.0044563934000000005,"complete":true},"billingTypes":["unknown"],"stockChecks":{"budget-hard-stops":true,"default-hire-bundle":true,"provider-runs-present":true,"invocation-evidence-complete":true,"historical-generic-procedures-observed":true,"fresh-default-delivered":true},"invocationMetrics":[{"capturedPromptChars":12335,"reportedPromptChars":12335,"freshPromptChars":4686,"retrievalClipped":false}],"prerequisite":{"sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceFingerprint":"2e5f22848450179ac2d573662c718300f86a935d09d15204c9d97e98f65b53f4","passed":true,"providerCallsBeforeAdmission":0,"passedTests":536},"evidenceHashes":{"result.json":"20dbb34bb975f73c3ac2695c047b6cbc8f6382757939ae6210b08a0f13ba9e9d","evidence-manifest.json":"67c3e042be8e0a5a0cd7cca82d1b0887e1392a2806c3b4db67baa3e99365b6be","snapshots/context-integrity.json":"decf2d93769acce2e2990de27c81ee32fd805ca53d3c2be68c56f22a013a7ac0","snapshots/stock-harness-hire.json":"187598ed72837719be8d6f0b77c5f8d35cfc871f8cf41d4fdc5b259f88171025","snapshots/stock-harness.json":"98424edd880cd05e0cbf8c32973fe9fbfc055037a74e76c20f893c3ee263f6a0","snapshots/stock-harness-preflight.json":"86d9f4080a2045ae3fa68468d0ffaf68baa26f27f0528274f13762f093b4c113"},"taskDocumentCount":1,"behaviorCheckCounts":{"passed":7,"failed":0}},
|
||||
{"executionId":"stock-harness.legacy-opencode.local.continuity-restart","profile":"legacy-opencode","case":"continuity-restart","sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceRef":"refs/heads/codex/stock-harness-previous-instructions","workflowRunUrl":"https://github.com/paperclipai/paperclip/actions/runs/37042864888","cellAttempt":1,"status":"passed","failureClass":null,"causeCodes":[],"failedMatcherPaths":[],"model":"openrouter/deepseek/deepseek-v4-flash-0731","runtimeMode":"legacy","durationMs":126589,"providerDurationMs":78910,"cleanup":"passed","billing":{"llm":{"runCount":3,"runsWithTokenUsage":3,"runsWithReportedCost":3,"inputTokens":64286,"outputTokens":1046,"cachedInputTokens":28672,"totalTokens":94004,"reportedCostUsd":0.0022020907,"costStatus":"reported"},"runtime":{"provider":"local","agentRunDurationMs":78910,"leaseDurationMs":null,"leaseCount":0,"costStatus":"not_metered","costSource":"local_not_metered"},"reportedCostUsd":0.0022020907,"estimatedRuntimeCostUsd":0,"observedAndEstimatedCostUsd":0.0022020907,"complete":true},"billingTypes":["unavailable"],"stockChecks":{"budget-hard-stops":true,"default-hire-bundle":true,"provider-runs-present":true,"invocation-evidence-complete":true,"historical-generic-procedures-observed":true,"fresh-default-delivered":true},"invocationMetrics":[{"capturedPromptChars":7611,"reportedPromptChars":7611,"freshPromptChars":0,"retrievalClipped":false},{"capturedPromptChars":7554,"reportedPromptChars":7554,"freshPromptChars":0,"retrievalClipped":false},{"capturedPromptChars":16335,"reportedPromptChars":16335,"freshPromptChars":1460,"retrievalClipped":false}],"prerequisite":{"sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceFingerprint":"2e5f22848450179ac2d573662c718300f86a935d09d15204c9d97e98f65b53f4","passed":true,"providerCallsBeforeAdmission":0,"passedTests":536},"evidenceHashes":{"result.json":"29dce6418aa35bb8070b07433046bbf39c9a8367a198cdb0aa28a0bea480032c","evidence-manifest.json":"439dc46130b607b7488e9df7736a711b267efa53689fe9fdad2f822525677215","snapshots/context-integrity.json":null,"snapshots/stock-harness-hire.json":"7f7ad64827bbe430e06a7e01b1377443653793584264c0fa00ac88a742bd0f49","snapshots/stock-harness.json":"4dd536951a950e7507710a9eb44cdbcd354907397f1e95b8297b28a98fd75dbc","snapshots/stock-harness-preflight.json":"8fa2ec342ad583fb20fed4db88890fc31eab3ef5898a952a85371e53dd433aaf"}},
|
||||
{"executionId":"stock-harness.legacy-opencode.local.ordered-comment-continuation","profile":"legacy-opencode","case":"ordered-comment-continuation","sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceRef":"refs/heads/codex/stock-harness-previous-instructions","workflowRunUrl":"https://github.com/paperclipai/paperclip/actions/runs/37048838402","cellAttempt":1,"status":"failed","failureClass":"candidate_failure","causeCodes":["no_single_paperclip_document","deadline_or_run_start_timeout"],"failedMatcherPaths":["contextIntegrity.comments-queued-as-batch","contextIntegrity.initial-run-started","contextIntegrity.ordered-comments","contextIntegrity.changed-scope-preserved","contextIntegrity.packing-report-order","contextIntegrity.final-scope-applied","contextIntegrity.continuation-run-count","contextIntegrity.continuation-wake-comment-ids","contextIntegrity.single-durable-output","contextIntegrity.completed-task","contextIntegrity.successful-runs"],"model":"openrouter/deepseek/deepseek-v4-flash-0731","runtimeMode":"legacy","durationMs":720616,"providerDurationMs":442902,"cleanup":"passed","billing":{"llm":{"runCount":3,"runsWithTokenUsage":2,"runsWithReportedCost":2,"inputTokens":51446,"outputTokens":8911,"cachedInputTokens":967936,"totalTokens":1028293,"reportedCostUsd":0.030657645200000007,"costStatus":"partial"},"runtime":{"provider":"local","agentRunDurationMs":442902,"leaseDurationMs":null,"leaseCount":0,"costStatus":"not_metered","costSource":"local_not_metered"},"reportedCostUsd":0.030657645200000007,"estimatedRuntimeCostUsd":0,"observedAndEstimatedCostUsd":0.030657645200000007,"complete":false},"billingTypes":["unavailable"],"stockChecks":{"budget-hard-stops":true,"default-hire-bundle":true,"provider-runs-present":true,"invocation-evidence-complete":true,"historical-generic-procedures-observed":true,"fresh-default-delivered":true},"invocationMetrics":[{"capturedPromptChars":16392,"reportedPromptChars":16461,"freshPromptChars":0,"retrievalClipped":true},{"capturedPromptChars":16406,"reportedPromptChars":16879,"freshPromptChars":4686,"retrievalClipped":true},{"capturedPromptChars":12749,"reportedPromptChars":12749,"freshPromptChars":4686,"retrievalClipped":false}],"prerequisite":{"sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceFingerprint":"2e5f22848450179ac2d573662c718300f86a935d09d15204c9d97e98f65b53f4","passed":true,"providerCallsBeforeAdmission":0,"passedTests":536},"evidenceHashes":{"result.json":"6d0e5d54b1742b02eb37346747510c66cb5cd0b80262a6c9590fd2a28861dcb2","evidence-manifest.json":"758931a4d30213d1b320f58cd2aa338e8a6a8aed579b878e14c5e52848bd5f13","snapshots/context-integrity.json":"bcaa4ee278361a90347625044aa49086b27093356abc0f3700b0363d28c8653c","snapshots/stock-harness-hire.json":"21431fa5e7cb6486037c670fc1e4aa64fb24716ad8ab325601ff294ada40263f","snapshots/stock-harness.json":"29c55e52f4adbbc9259cb9e221c08bfbbd7ad4c5f8247c24b3f029362e9000de","snapshots/stock-harness-preflight.json":"f341723074c996c79d8a6c30b9f12270634b20b40da452b13e4ab9ce91b47cac"},"taskDocumentCount":null,"behaviorCheckCounts":{"passed":1,"failed":11}},
|
||||
{"executionId":"stock-harness.runner-acpx-claude.local.assigned-skill-explicit-invocation","profile":"runner-acpx-claude","case":"assigned-skill-explicit-invocation","sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceRef":"refs/heads/codex/stock-harness-previous-instructions","workflowRunUrl":"https://github.com/paperclipai/paperclip/actions/runs/37042864888","cellAttempt":1,"status":"passed","failureClass":null,"causeCodes":[],"failedMatcherPaths":[],"model":"claude-sonnet-5","runtimeMode":"native","durationMs":46264,"providerDurationMs":25372,"cleanup":"passed","billing":{"llm":{"runCount":1,"runsWithTokenUsage":1,"runsWithReportedCost":1,"inputTokens":10,"outputTokens":1042,"cachedInputTokens":141254,"totalTokens":142306,"reportedCostUsd":0,"costStatus":"reported"},"runtime":{"provider":"local","agentRunDurationMs":25372,"leaseDurationMs":null,"leaseCount":0,"costStatus":"not_metered","costSource":"local_not_metered"},"reportedCostUsd":0,"estimatedRuntimeCostUsd":0,"observedAndEstimatedCostUsd":0,"complete":true},"billingTypes":["unknown"],"stockChecks":{"budget-hard-stops":true,"default-hire-bundle":true,"provider-runs-present":true},"invocationMetrics":[],"prerequisite":{"sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceFingerprint":"2e5f22848450179ac2d573662c718300f86a935d09d15204c9d97e98f65b53f4","passed":true,"providerCallsBeforeAdmission":0,"passedTests":536},"evidenceHashes":{"result.json":"073085f804a58d191bd44972edfb08d914fe622a86f236d0c4dc1b2bf8a03322","evidence-manifest.json":"4fad428c1df72abbfee24a4f7338e25284fc874cff1220e6b631fa86eb2d0bed","snapshots/context-integrity.json":"77e27b8ea01a83e0b65f3d5410e0787fbf84a7f6ea02f4401bd77488999dd4a2","snapshots/stock-harness-hire.json":"8c6c5d56fd068c15f1e19ce9a0b62204443c894d204311bd949d5e22c624fd3d","snapshots/stock-harness.json":"2a6860d7c29ad0d03240bda0a779b83d0455b8ce9fcfb1cf9e6ad559c4dec112","snapshots/stock-harness-preflight.json":"79b11e544d2821d1025476f5fd0d737e822ae5ab731ab32047d1fbf6b76015e4"},"taskDocumentCount":1,"behaviorCheckCounts":{"passed":7,"failed":0}},
|
||||
{"executionId":"stock-harness.runner-acpx-claude.local.continuity-restart","profile":"runner-acpx-claude","case":"continuity-restart","sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceRef":"refs/heads/codex/stock-harness-previous-instructions","workflowRunUrl":"https://github.com/paperclipai/paperclip/actions/runs/37042864888","cellAttempt":1,"status":"passed","failureClass":null,"causeCodes":[],"failedMatcherPaths":[],"model":"claude-sonnet-5","runtimeMode":"native","durationMs":77424,"providerDurationMs":36820,"cleanup":"passed","billing":{"llm":{"runCount":3,"runsWithTokenUsage":3,"runsWithReportedCost":3,"inputTokens":6,"outputTokens":94,"cachedInputTokens":94997,"totalTokens":95097,"reportedCostUsd":0,"costStatus":"reported"},"runtime":{"provider":"local","agentRunDurationMs":36820,"leaseDurationMs":null,"leaseCount":0,"costStatus":"not_metered","costSource":"local_not_metered"},"reportedCostUsd":0,"estimatedRuntimeCostUsd":0,"observedAndEstimatedCostUsd":0,"complete":true},"billingTypes":["unavailable"],"stockChecks":{"budget-hard-stops":true,"default-hire-bundle":true,"provider-runs-present":true},"invocationMetrics":[],"prerequisite":{"sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceFingerprint":"2e5f22848450179ac2d573662c718300f86a935d09d15204c9d97e98f65b53f4","passed":true,"providerCallsBeforeAdmission":0,"passedTests":536},"evidenceHashes":{"result.json":"be582fe8af5f19543c16c8590697c5aa9dc688fdb653b0550aba0b55174becd1","evidence-manifest.json":"cfbd1cb13917a96b08dd46aae1fcc63e2b27af5d1d22f4952974d3415c74ffb8","snapshots/context-integrity.json":null,"snapshots/stock-harness-hire.json":"254f18a8e8e1d6a5a2e7731728756671c1374cd4d1210469172bed9c2cd6e4c2","snapshots/stock-harness.json":"8efa84d6907de220dd7b9fc2c384c61aae0606e7f5311f0e926e26ae1617d869","snapshots/stock-harness-preflight.json":"c7e3f2342ef9fc824b28e3f22ffbbb326d2ba1ff84e36c4deedc56a7abefbe07"}},
|
||||
{"executionId":"stock-harness.runner-acpx-claude.local.ordered-comment-continuation","profile":"runner-acpx-claude","case":"ordered-comment-continuation","sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceRef":"refs/heads/codex/stock-harness-previous-instructions","workflowRunUrl":"https://github.com/paperclipai/paperclip/actions/runs/37042864888","cellAttempt":1,"status":"passed","failureClass":null,"causeCodes":[],"failedMatcherPaths":[],"model":"claude-sonnet-5","runtimeMode":"native","durationMs":132587,"providerDurationMs":108895,"cleanup":"passed","billing":{"llm":{"runCount":2,"runsWithTokenUsage":2,"runsWithReportedCost":2,"inputTokens":26,"outputTokens":7996,"cachedInputTokens":565030,"totalTokens":573052,"reportedCostUsd":0,"costStatus":"reported"},"runtime":{"provider":"local","agentRunDurationMs":108895,"leaseDurationMs":null,"leaseCount":0,"costStatus":"not_metered","costSource":"local_not_metered"},"reportedCostUsd":0,"estimatedRuntimeCostUsd":0,"observedAndEstimatedCostUsd":0,"complete":true},"billingTypes":["unavailable"],"stockChecks":{"budget-hard-stops":true,"default-hire-bundle":true,"provider-runs-present":true},"invocationMetrics":[],"prerequisite":{"sourceSha":"12c5433c67dc2e62916b879349c7ba2b6e0431f0","sourceFingerprint":"2e5f22848450179ac2d573662c718300f86a935d09d15204c9d97e98f65b53f4","passed":true,"providerCallsBeforeAdmission":0,"passedTests":536},"evidenceHashes":{"result.json":"62c6c55047813b0b5aa5334e29dbe3f90225c83ac436eab14afd97d78ed9ec65","evidence-manifest.json":"7d53ad7f80b65a1c7af3fad4fa3d0c652ae25cdaeae3d41cbee1bb888b853fb5","snapshots/context-integrity.json":"2081459b761a84469399bec377d7380c172b64fddae48735c5329771656e28a7","snapshots/stock-harness-hire.json":"f11c90fdae215949458d61a6e1ca6565fe277ea30d52fd922c05cffdf4867e5c","snapshots/stock-harness.json":"f303a3fb5c5e55cefb696d46204aafe6f1cefc5b0a6b07a728f2bf0dbf1dfdcf","snapshots/stock-harness-preflight.json":"79768aa58996d28ee80c8a9e8d5c4ba1a42eba58a408163cd8a6037ac6005b26"},"taskDocumentCount":1,"behaviorCheckCounts":{"passed":12,"failed":0}},
|
||||
@@ -90,21 +91,21 @@
|
||||
],
|
||||
"totals": {
|
||||
"baseline": {
|
||||
"results": 23,
|
||||
"results": 24,
|
||||
"passed": 15,
|
||||
"failed": 8,
|
||||
"missing": 1,
|
||||
"cleanupPassed": 23,
|
||||
"durationMs": 2529579,
|
||||
"providerDurationMs": 1665132,
|
||||
"failed": 9,
|
||||
"missing": 0,
|
||||
"cleanupPassed": 24,
|
||||
"durationMs": 3250195,
|
||||
"providerDurationMs": 2108034,
|
||||
"llm": {
|
||||
"runCount": 55,
|
||||
"runsWithTokenUsage": 50,
|
||||
"runsWithReportedCost": 41,
|
||||
"inputTokens": 2727135,
|
||||
"outputTokens": 59104,
|
||||
"cachedInputTokens": 5940981,
|
||||
"reportedCostUsd": 1.7187676166
|
||||
"runCount": 58,
|
||||
"runsWithTokenUsage": 52,
|
||||
"runsWithReportedCost": 43,
|
||||
"inputTokens": 2778581,
|
||||
"outputTokens": 68015,
|
||||
"cachedInputTokens": 6908917,
|
||||
"reportedCostUsd": 1.7494252618000001
|
||||
}
|
||||
},
|
||||
"candidate": {
|
||||
@@ -138,9 +139,7 @@
|
||||
"medianProviderDurationMs": 43862
|
||||
}
|
||||
},
|
||||
"missingBaseline": [
|
||||
"stock-harness.legacy-opencode.local.ordered-comment-continuation"
|
||||
],
|
||||
"missingBaseline": [],
|
||||
"cancellationRecovery": {
|
||||
"sourceSha": "f02d8d0df327abb43b20c7e7beb86798239abbf5",
|
||||
"originalRun": 37042856368,
|
||||
@@ -187,6 +186,10 @@
|
||||
{
|
||||
"campaign": "candidate-recovery",
|
||||
"sha256": "7788f879d0cf1d140d452cfa78fe085b34bc23d298d7c31cac9d630b12b2572f"
|
||||
},
|
||||
{
|
||||
"campaign": "baseline-recovery-final",
|
||||
"sha256": "e39d62dd60f092e76f8776e844041f2458fca1989854d57ab13932a36c937ceb"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -344,6 +347,11 @@
|
||||
"run": 37044967981,
|
||||
"workflowSha": "7d59de6113caf5219d34d65e1fb5f0c1a7fbdc2a",
|
||||
"workflowUrl": "https://github.com/paperclipai/paperclip/actions/runs/37044967981"
|
||||
},
|
||||
{
|
||||
"run": 37048838402,
|
||||
"workflowSha": "7d59de6113caf5219d34d65e1fb5f0c1a7fbdc2a",
|
||||
"workflowUrl": "https://github.com/paperclipai/paperclip/actions/runs/37048838402"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -25,7 +25,7 @@ These are recorded overall qualifications, including security and receipt failur
|
||||
| `legacy-claude` | `claude-sonnet-4-6` | `ordered-comment-continuation` | Pass | Pass |
|
||||
| `legacy-claude` | `claude-sonnet-4-6` | `continuity-restart` | Fail: chat memory | Fail: chat memory |
|
||||
| `legacy-opencode` | `openrouter/deepseek/deepseek-v4-flash-0731` | `assigned-skill-explicit-invocation` | Pass | Fail: no Paperclip document |
|
||||
| `legacy-opencode` | `openrouter/deepseek/deepseek-v4-flash-0731` | `ordered-comment-continuation` | Missing: runner shutdown | Pass |
|
||||
| `legacy-opencode` | `openrouter/deepseek/deepseek-v4-flash-0731` | `ordered-comment-continuation` | Fail: no Paperclip document, deadline/run-start timeout | Pass |
|
||||
| `legacy-opencode` | `openrouter/deepseek/deepseek-v4-flash-0731` | `continuity-restart` | Pass | Pass |
|
||||
| `legacy-acp-codex` | `gpt-5.6-sol` | `assigned-skill-explicit-invocation` | Fail: credential guard | Fail: credential guard |
|
||||
| `legacy-acp-codex` | `gpt-5.6-sol` | `ordered-comment-continuation` | Fail: credential guard, no Paperclip document, receipt incomplete | Fail: credential guard, no Paperclip document |
|
||||
@@ -51,7 +51,7 @@ Keep the agreed tiny manual. Next, verify that legacy agents discover and follow
|
||||
|
||||
Classic Claude chat fails the memory assertion in both variants. Six legacy ACP cells per variant fail the persisted-credential guard: the scanner finds a scoped provider credential in persisted ACP session state. Raw session files and credential values are not published. These are existing qualification failures requiring their own investigation; they cannot be waived or interpreted as instruction-reduction quality evidence.
|
||||
|
||||
Historical native OpenCode ordered comments hit the case deadline without the required continuation outcome. The initial historical classic OpenCode ordered cell lost its AWS runner before result upload; only that missing cell is recovered. Completed failures are never rerun to select a better result. A candidate pass against a missing/interrupted baseline is not an improvement claim.
|
||||
Historical native OpenCode ordered comments hit the case deadline without the required continuation outcome. The initial historical classic OpenCode ordered cell lost its AWS runner before result upload; its one unchanged-source recovery also hits the 12-minute deadline. Both OpenCode ordered cases pass in the candidate. Completed failures are never rerun to select a better result. Those candidate passes against historical timeouts are observed differences, not proof of a general improvement.
|
||||
|
||||
Historical public prompt retrieval is clipped in three legacy ACP cells (four invocations), including identity/connection suffix text. Those structural receipt failures cannot establish that the provider omitted instructions. The candidate ACP Codex chat run also lacks a complete per-run invocation receipt. Instruction presence/absence claims are bounded by the retained public receipts.
|
||||
|
||||
@@ -59,7 +59,7 @@ Historical public prompt retrieval is clipped in three legacy ACP cells (four in
|
||||
|
||||
| Variant | Retained cells | Pass / fail / missing | Cell duration sum | Provider duration sum | Recorded runs with tokens / with reported cost / total | Reported LLM subtotal |
|
||||
| --- | ---: | --- | ---: | ---: | --- | ---: |
|
||||
| baseline | 23 | 15 / 8 / 1 | 2529.579s | 1665.132s | 50 / 41 / 55 | $1.718768 |
|
||||
| baseline | 24 | 15 / 9 / 0 | 3250.195s | 2108.034s | 52 / 43 / 58 | $1.749425 |
|
||||
| candidate | 24 | 15 / 9 / 0 | 2804.913s | 1540.860s | 45 / 36 / 47 | $1.743151 |
|
||||
|
||||
For the 13 cells that pass both variants, median cell time is 85.550s historical versus 69.761s reduced; median provider time is 57.150s versus 43.862s. This excludes failures and is descriptive, not a reliable speedup estimate.
|
||||
@@ -78,12 +78,12 @@ Provider billing is incomplete. Legacy Codex/ACP usage is unpriced in several ru
|
||||
| [Initial candidate matrix](https://github.com/paperclipai/paperclip/actions/runs/37042856368) | `f02d8d0df` | 4 pass, 7 fail, 13 cancelled when a same-target-branch pilot superseded it. Original attempts retained. |
|
||||
| [Candidate cancelled-cell recovery](https://github.com/paperclipai/paperclip/actions/runs/37045368302) | `f02d8d0df` | Only the 13 cancelled cells: 11 pass, 2 fail. Combined candidate has 24 results, 15 pass / 9 fail. |
|
||||
| [Historical matrix](https://github.com/paperclipai/paperclip/actions/runs/37042864888) | `12c5433c6` | 15 pass, 8 recorded failures, one AWS-runner shutdown without result upload. |
|
||||
| [Historical interrupted-cell recovery](https://github.com/paperclipai/paperclip/actions/runs/37048838402) | `12c5433c6` | Only legacy OpenCode ordered comments; final result shown above when available. |
|
||||
| [Historical interrupted-cell recovery](https://github.com/paperclipai/paperclip/actions/runs/37048838402) | `12c5433c6` | Only legacy OpenCode ordered comments: 720.616s deadline failure; evidence and cleanup valid. Historical cohort now has 24 results, 15 pass / 9 fail. |
|
||||
| [Current packaging pilot](https://github.com/paperclipai/paperclip/actions/runs/37044967981) | `1eb5ba420` | Attempt 1 stopped before cell/provider execution; attempt 2 passes, evidence valid and cleanup pass. |
|
||||
|
||||
The same-target-branch workflow concurrency rule caused the candidate interruption; that orchestration mistake was acknowledged and corrected with a separate unchanged-source recovery branch. No completed model failure was retried. The historical missing cell is recovered once for a runner shutdown. Partial/failed attempts remain part of the history and unknown-spend accounting.
|
||||
|
||||
The measured matrix packages contain a prerequisite folder beside the exact campaign root. The trusted selector correctly rejects this layout, so their original published reports synthesize missing-result infrastructure failures. A separate local reconstruction nests that prerequisite folder under the campaign root and verifies byte identity for every copied file; result, scorer, usage, screenshot and prerequisite content are unchanged. The unchanged trusted selector then selects 11/24 original candidate, 13/13 recovery and 23/24 historical packages; recovered normalization validates every retained result’s evidence. Original missing packages stay missing. This is declared directory-layout recovery, not a canonical successful publication or a change to the oracle. The safe JSON records result/receipt/scoring hashes and recovery-manifest hashes.
|
||||
The measured matrix packages contain a prerequisite folder beside the exact campaign root. The trusted selector correctly rejects this layout, so their original published reports synthesize missing-result infrastructure failures. A separate local reconstruction nests that prerequisite folder under the campaign root and verifies byte identity for every copied file; result, scorer, usage, screenshot and prerequisite content are unchanged. The unchanged trusted selector then selects 11/24 original candidate, 13/13 recovery 23/24 original historical packages and 1/1 interrupted-cell recovery; recovered normalization validates every retained result’s evidence. Original missing packages stay missing in their original campaign; the separate recovery result has its own provenance. This is declared directory-layout recovery, not a canonical successful publication or a change to the oracle. The safe JSON records result/receipt/scoring hashes and recovery-manifest hashes.
|
||||
|
||||
The fix at `1eb5ba420` writes prerequisites beneath the exact campaign root. Its single selected Codex skill cell passes through the protected report and publication pipeline: [interactive pilot report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37044967981-2/index.html), [GitHub evidence](https://github.com/paperclipai/paperclip/actions/runs/37044967981#artifacts). It has `evidenceValid=true`, cleanup pass, 27.149s provider / 49.523s cell, and a reported zero subtotal with unknown billing type. It does not replace the full matched cohort or qualify untested providers.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ Created: 2026-10-02. Status: item 1 merged for native Codex app-server in
|
||||
hire manual is reduced to identity only, and common legacy startup/resume
|
||||
instructions are in [PR #14948](https://github.com/paperclipai/paperclip/pull/14948).
|
||||
GitHub live qualification has measured document-delivery failures; PR #14948 remains
|
||||
draft. A matched prior-instruction comparison and its final interrupted-cell
|
||||
draft. A matched prior-instruction comparison and its completed interrupted-cell
|
||||
recovery are recorded in the [live report](2026-10-02-stock-harness-live-comparison.md).
|
||||
Additional carriers and native instructions remain open.
|
||||
|
||||
@@ -418,5 +418,6 @@ will address them. Record intentional behavior explicitly rather than as a bug.
|
||||
For each completed item, add the chosen behavior, changed paths, verification
|
||||
results, remaining exceptions, and follow-ups here before checking it off.
|
||||
| 2026-10-02 | Dotta deferred 2.2 additional legacy carriers and approved the first 2.3 native tool-description slice separately. | Keep wrappers open; native finish/block documentation must not be supplied to legacy skill/API completion paths. |
|
||||
| 2026-10-02 | Matched default-manual/shared-prompt campaigns measured failures, with #14920 constant. | Candidate `f02d8d0df`: 15/24 pass. Historical `12c5433c6`: 15 pass, eight recorded failures, one interrupted cell recovering. Classic Claude/OpenCode Paperclip document delivery regressed in observed trials; keep PR #14948 draft. [Live report](2026-10-02-stock-harness-live-comparison.md). |
|
||||
| 2026-10-02 | Matched default-manual/shared-prompt campaigns measured failures, with #14920 constant. | Candidate `f02d8d0df`: 15/24 pass. Historical `12c5433c6`: 15/24 pass after the one runner-shutdown recovery also timed out. Classic Claude/OpenCode Paperclip document delivery regressed in observed trials; keep PR #14948 draft. [Live report](2026-10-02-stock-harness-live-comparison.md). |
|
||||
| 2026-10-02 | Cold prerequisite and packaging faults were repaired without weakening admission or behavioral graders. | Three setup attempts stopped before providers. Current `1eb5ba420` pilot passes 557 prerequisite checks and protected report publication; source/hash/cost evidence retained. Candidate cancellation and historical runner shutdown recovered only for missing cells. |
|
||||
| 2026-10-02 | Final bounded recovery completed; no further model reruns. | Both matched cohorts have all 24 results, 15 pass and nine fail. Two classic skill deliveries regress; two OpenCode ordered cases pass only with reduced instructions. Overall parity is not behavioral equivalence. All 48 retained result/receipt projections are hashed and sanitized; original interruptions and partial unknown spend remain recorded. |
|
||||
@@ -206,8 +206,8 @@ and its safe JSON projection contain the per-profile/case comparison, exact
|
||||
source hashes, all campaign and recovery links, timing/usage, cost coverage,
|
||||
security failures, clipping limits and publication-layout recovery. Candidate
|
||||
`f02d8d0df` has 24 retained results: 15 pass and nine fail. Historical
|
||||
`12c5433c6` has 23 retained results plus one bounded AWS-runner recovery in
|
||||
progress. Classic Claude/OpenCode skill runs save no Paperclip task document
|
||||
`12c5433c6` also has 24 retained results: 15 pass and nine fail. Its single
|
||||
AWS-runner recovery timed out; original missing evidence remains recorded. Classic Claude/OpenCode skill runs save no Paperclip task document
|
||||
where historical runs save one; the original oracle remains failed.
|
||||
|
||||
The merged native Codex change is held constant. Same aggregate success counts
|
||||
|
||||
Reference in new issue
Block a user