mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-12 06:10:44 +01:00
docs: publish retained native paired task outcomes
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
516eb910d4
commit
dfa0e2cc9b
6 files changed
+2581
-3
No files matched your search
@@ -266,8 +266,8 @@
|
||||
"fixtureIdentityVerified": true,
|
||||
"onlyDeclaredElevenNativeSourcesDiffer": true,
|
||||
"fixedMasterContextVerified": true,
|
||||
"providerCallsSoFar": null,
|
||||
"liveStatus": "both_dispatched_no_graded_cells",
|
||||
"providerCallsSoFar": 12,
|
||||
"liveStatus": "all_original_cell_results_retained_final_audit_pending",
|
||||
"workflowRuns": {
|
||||
"candidate": {
|
||||
"id": 37098728980,
|
||||
@@ -304,5 +304,9 @@
|
||||
"classification": "pre_provider_calibration_mock"
|
||||
}
|
||||
],
|
||||
"workflowSourceSha": "215586d127e97c9301d86e769a39a15c13298ca2"
|
||||
"workflowSourceSha": "215586d127e97c9301d86e769a39a15c13298ca2",
|
||||
"gradedPairs": 6,
|
||||
"candidatePassedCells": 6,
|
||||
"baselinePassedCells": 5,
|
||||
"comparisonPath": "doc/plans/2026-10-02-native-completion-calibrated-results/comparison.json"
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,886 @@
|
||||
{
|
||||
"schema": "paperclip.native-completion-retained-audit.v1",
|
||||
"originalResultsChanged": false,
|
||||
"providerRuns": 6,
|
||||
"rows": [
|
||||
{
|
||||
"executionId": "native-completion.runner-codex.local.native-blocked-report",
|
||||
"variant": "candidate",
|
||||
"source": {
|
||||
"sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21",
|
||||
"ref": "refs/heads/codex/native-completion-master-calibrated-candidate",
|
||||
"workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980"
|
||||
},
|
||||
"suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d",
|
||||
"artifactId": 11265785208,
|
||||
"artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11265785208",
|
||||
"zipSha256": "53b1125e7b5d183462e1e1e536cd7674cacfec9995ee8a47e32116740564916f",
|
||||
"resultSha256": "60c8a4f3e41b8ca9a80c10f73d804c30d6e6ede9ace5ce3e93d2c4a053503f32",
|
||||
"nativeSnapshotSha256": "ad5cc611524fc7ac98dd8a65201b90eacb75d76309031f4e39090768fca1a909",
|
||||
"defaultSnapshotSha256": "c5bb4c9b1ed1a9f13c90320d0081fc22f04ae4cf40af751864a8de1c5cc04d7a",
|
||||
"contextSnapshotSha256": null,
|
||||
"status": "passed",
|
||||
"attempt": 1,
|
||||
"runCount": 1,
|
||||
"runStatus": "succeeded",
|
||||
"issueStatus": "blocked",
|
||||
"model": "gpt-5.6-sol",
|
||||
"provider": "codex",
|
||||
"grade": {
|
||||
"schema": "paperclip.native-completion-observation.v2",
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "one-native-attempt",
|
||||
"passed": true,
|
||||
"detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work."
|
||||
},
|
||||
{
|
||||
"id": "durable-disposition",
|
||||
"passed": true,
|
||||
"detail": "Public durable issue status and the native semantic result agree with the requested disposition."
|
||||
},
|
||||
{
|
||||
"id": "exact-blocker",
|
||||
"passed": true,
|
||||
"detail": "The structured owner, exact action and whole-task scope are independently checked."
|
||||
},
|
||||
{
|
||||
"id": "complete-bound-events",
|
||||
"passed": true,
|
||||
"detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity."
|
||||
},
|
||||
{
|
||||
"id": "authoritative-native-result",
|
||||
"passed": true,
|
||||
"detail": "One runner proposal is independently accepted and terminated by the control plane."
|
||||
},
|
||||
{
|
||||
"id": "final-after-tool-result",
|
||||
"passed": true,
|
||||
"detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently."
|
||||
},
|
||||
{
|
||||
"id": "persisted-provider-final",
|
||||
"passed": true,
|
||||
"detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient."
|
||||
},
|
||||
{
|
||||
"id": "visible-blocker-content",
|
||||
"passed": true,
|
||||
"detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker."
|
||||
},
|
||||
{
|
||||
"id": "bounded-durable-work",
|
||||
"passed": true,
|
||||
"detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none."
|
||||
},
|
||||
{
|
||||
"id": "no-deployment-or-file-work",
|
||||
"passed": true,
|
||||
"detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed."
|
||||
}
|
||||
],
|
||||
"limitations": [
|
||||
"Exact provider feedback identity/consumption is not measured by the public sequence."
|
||||
]
|
||||
},
|
||||
"integrityFailures": [],
|
||||
"defaultGrade": {
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "production-default-bundle",
|
||||
"passed": true,
|
||||
"detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection."
|
||||
},
|
||||
{
|
||||
"id": "bounded-company-and-agent",
|
||||
"passed": true,
|
||||
"detail": "Both public budgets enforce the declared 1,000-cent hard stop."
|
||||
}
|
||||
]
|
||||
},
|
||||
"contextChecks": null,
|
||||
"durableState": {
|
||||
"documentCount": 0,
|
||||
"interactionCount": 0,
|
||||
"additionalIssueCount": 1,
|
||||
"additionalAgentCount": 0
|
||||
},
|
||||
"sequencing": {
|
||||
"proposalSeq": [
|
||||
60
|
||||
],
|
||||
"acceptedSeq": [
|
||||
72
|
||||
],
|
||||
"terminalSeq": [
|
||||
73
|
||||
],
|
||||
"providerFinalSeq": [
|
||||
67
|
||||
]
|
||||
},
|
||||
"durationMs": 61424,
|
||||
"agentRunDurationMs": 18794,
|
||||
"tokens": {
|
||||
"input": 44226,
|
||||
"output": 577,
|
||||
"cachedInput": 21814
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported",
|
||||
"billingType": "unknown",
|
||||
"actualBillingVerified": false,
|
||||
"runtimeCostStatus": "not_metered",
|
||||
"cleanup": "passed",
|
||||
"matcherCount": 21,
|
||||
"failedMatchers": [],
|
||||
"independentRetainedReplayPassed": true
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
"variant": "candidate",
|
||||
"source": {
|
||||
"sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21",
|
||||
"ref": "refs/heads/codex/native-completion-master-calibrated-candidate",
|
||||
"workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980"
|
||||
},
|
||||
"suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d",
|
||||
"artifactId": 11265632369,
|
||||
"artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11265632369",
|
||||
"zipSha256": "afc06b58cb048774274bd856b25cb8359b28f3c07f104428a92219148f60fca4",
|
||||
"resultSha256": "09077172094e8f63861739884ae42e1fb5755f113e9a18a53b21697055d25607",
|
||||
"nativeSnapshotSha256": "efc7af6363ffda141ed4d7648f48db79069ceb7bcc96475beb30be1c10b35dca",
|
||||
"defaultSnapshotSha256": "b4af7764ab4a84b541f2045468889296538a3ef420d7f8898835df0d12faaf4e",
|
||||
"contextSnapshotSha256": "4f221d6c03955759889c978521cab9d617a8477cce8f1276f55457b5dc71b988",
|
||||
"status": "passed",
|
||||
"attempt": 1,
|
||||
"runCount": 1,
|
||||
"runStatus": "succeeded",
|
||||
"issueStatus": "done",
|
||||
"model": "gpt-5.6-sol",
|
||||
"provider": "codex",
|
||||
"grade": {
|
||||
"schema": "paperclip.native-completion-observation.v2",
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "one-native-attempt",
|
||||
"passed": true,
|
||||
"detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work."
|
||||
},
|
||||
{
|
||||
"id": "durable-disposition",
|
||||
"passed": true,
|
||||
"detail": "Public durable issue status and the native semantic result agree with the requested disposition."
|
||||
},
|
||||
{
|
||||
"id": "complete-bound-events",
|
||||
"passed": true,
|
||||
"detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity."
|
||||
},
|
||||
{
|
||||
"id": "authoritative-native-result",
|
||||
"passed": true,
|
||||
"detail": "One runner proposal is independently accepted and terminated by the control plane."
|
||||
},
|
||||
{
|
||||
"id": "final-after-tool-result",
|
||||
"passed": true,
|
||||
"detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently."
|
||||
},
|
||||
{
|
||||
"id": "persisted-provider-final",
|
||||
"passed": true,
|
||||
"detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient."
|
||||
},
|
||||
{
|
||||
"id": "bounded-durable-work",
|
||||
"passed": true,
|
||||
"detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none."
|
||||
}
|
||||
],
|
||||
"limitations": [
|
||||
"Exact provider feedback identity/consumption is not measured by the public sequence."
|
||||
]
|
||||
},
|
||||
"integrityFailures": [],
|
||||
"defaultGrade": {
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "production-default-bundle",
|
||||
"passed": true,
|
||||
"detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection."
|
||||
},
|
||||
{
|
||||
"id": "bounded-company-and-agent",
|
||||
"passed": true,
|
||||
"detail": "Both public budgets enforce the declared 1,000-cent hard stop."
|
||||
}
|
||||
]
|
||||
},
|
||||
"contextChecks": [
|
||||
{
|
||||
"id": "assigned-skill-present",
|
||||
"passed": true,
|
||||
"detail": "The task run must receive one pinned skill version through the public assignment state."
|
||||
},
|
||||
{
|
||||
"id": "skill-request-explicit",
|
||||
"passed": true,
|
||||
"detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference."
|
||||
},
|
||||
{
|
||||
"id": "skill-source-marker",
|
||||
"passed": true,
|
||||
"detail": "The assigned pinned skill source must contain the output marker."
|
||||
},
|
||||
{
|
||||
"id": "marker-not-in-request",
|
||||
"passed": true,
|
||||
"detail": "The output marker must originate from the assigned skill, not the task request or comments."
|
||||
},
|
||||
{
|
||||
"id": "single-durable-output",
|
||||
"passed": true,
|
||||
"detail": "Exactly one durable task document must contain the skill's marker."
|
||||
},
|
||||
{
|
||||
"id": "completed-task",
|
||||
"passed": true,
|
||||
"detail": "Final task status: done."
|
||||
},
|
||||
{
|
||||
"id": "successful-runs",
|
||||
"passed": true,
|
||||
"detail": "All recorded context-integrity runs must succeed."
|
||||
}
|
||||
],
|
||||
"durableState": {
|
||||
"documentCount": 1,
|
||||
"interactionCount": 0,
|
||||
"additionalIssueCount": 1,
|
||||
"additionalAgentCount": 0
|
||||
},
|
||||
"sequencing": {
|
||||
"proposalSeq": [
|
||||
158
|
||||
],
|
||||
"acceptedSeq": [
|
||||
191
|
||||
],
|
||||
"terminalSeq": [
|
||||
192
|
||||
],
|
||||
"providerFinalSeq": [
|
||||
186
|
||||
]
|
||||
},
|
||||
"durationMs": 78188,
|
||||
"agentRunDurationMs": 34018,
|
||||
"tokens": {
|
||||
"input": 234514,
|
||||
"output": 1598,
|
||||
"cachedInput": 209637
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported",
|
||||
"billingType": "unknown",
|
||||
"actualBillingVerified": false,
|
||||
"runtimeCostStatus": "not_metered",
|
||||
"cleanup": "passed",
|
||||
"matcherCount": 14,
|
||||
"failedMatchers": [],
|
||||
"independentRetainedReplayPassed": true
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"variant": "candidate",
|
||||
"source": {
|
||||
"sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21",
|
||||
"ref": "refs/heads/codex/native-completion-master-calibrated-candidate",
|
||||
"workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980"
|
||||
},
|
||||
"suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d",
|
||||
"artifactId": 11265222862,
|
||||
"artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11265222862",
|
||||
"zipSha256": "0037c195e310ec8dbe5d54f4c23d00bbc3edf1bd812f75f33bd4fa0eb3f3ba4c",
|
||||
"resultSha256": "f70d358b361e425fb4992b2101745a9912b2682134e780203ae5f79845dd5ee1",
|
||||
"nativeSnapshotSha256": "f7273fd6716e145f2136211be32e7a2f1e9b18501eac32d9217a02d2abc10ce1",
|
||||
"defaultSnapshotSha256": "015a77243ee0298d729d893aa941fed93c8f8252eaa4a12a63673bcaf909edb4",
|
||||
"contextSnapshotSha256": "0291e48dc4873d5d20c0d1daeef05c9536445ee3d24a6b51af9d787c324ca4e6",
|
||||
"status": "passed",
|
||||
"attempt": 1,
|
||||
"runCount": 1,
|
||||
"runStatus": "succeeded",
|
||||
"issueStatus": "done",
|
||||
"model": "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
"provider": "opencode",
|
||||
"grade": {
|
||||
"schema": "paperclip.native-completion-observation.v2",
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "one-native-attempt",
|
||||
"passed": true,
|
||||
"detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work."
|
||||
},
|
||||
{
|
||||
"id": "durable-disposition",
|
||||
"passed": true,
|
||||
"detail": "Public durable issue status and the native semantic result agree with the requested disposition."
|
||||
},
|
||||
{
|
||||
"id": "complete-bound-events",
|
||||
"passed": true,
|
||||
"detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity."
|
||||
},
|
||||
{
|
||||
"id": "authoritative-native-result",
|
||||
"passed": true,
|
||||
"detail": "One runner proposal is independently accepted and terminated by the control plane."
|
||||
},
|
||||
{
|
||||
"id": "final-after-tool-result",
|
||||
"passed": true,
|
||||
"detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently."
|
||||
},
|
||||
{
|
||||
"id": "persisted-provider-final",
|
||||
"passed": true,
|
||||
"detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient."
|
||||
},
|
||||
{
|
||||
"id": "bounded-durable-work",
|
||||
"passed": true,
|
||||
"detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none."
|
||||
}
|
||||
],
|
||||
"limitations": [
|
||||
"Exact provider feedback identity/consumption is not measured by the public sequence."
|
||||
]
|
||||
},
|
||||
"integrityFailures": [],
|
||||
"defaultGrade": {
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "production-default-bundle",
|
||||
"passed": true,
|
||||
"detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection."
|
||||
},
|
||||
{
|
||||
"id": "bounded-company-and-agent",
|
||||
"passed": true,
|
||||
"detail": "Both public budgets enforce the declared 1,000-cent hard stop."
|
||||
}
|
||||
]
|
||||
},
|
||||
"contextChecks": [
|
||||
{
|
||||
"id": "assigned-skill-present",
|
||||
"passed": true,
|
||||
"detail": "The task run must receive one pinned skill version through the public assignment state."
|
||||
},
|
||||
{
|
||||
"id": "skill-request-explicit",
|
||||
"passed": true,
|
||||
"detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference."
|
||||
},
|
||||
{
|
||||
"id": "skill-source-marker",
|
||||
"passed": true,
|
||||
"detail": "The assigned pinned skill source must contain the output marker."
|
||||
},
|
||||
{
|
||||
"id": "marker-not-in-request",
|
||||
"passed": true,
|
||||
"detail": "The output marker must originate from the assigned skill, not the task request or comments."
|
||||
},
|
||||
{
|
||||
"id": "single-durable-output",
|
||||
"passed": true,
|
||||
"detail": "Exactly one durable task document must contain the skill's marker."
|
||||
},
|
||||
{
|
||||
"id": "completed-task",
|
||||
"passed": true,
|
||||
"detail": "Final task status: done."
|
||||
},
|
||||
{
|
||||
"id": "successful-runs",
|
||||
"passed": true,
|
||||
"detail": "All recorded context-integrity runs must succeed."
|
||||
}
|
||||
],
|
||||
"durableState": {
|
||||
"documentCount": 1,
|
||||
"interactionCount": 0,
|
||||
"additionalIssueCount": 1,
|
||||
"additionalAgentCount": 0
|
||||
},
|
||||
"sequencing": {
|
||||
"proposalSeq": [
|
||||
56
|
||||
],
|
||||
"acceptedSeq": [
|
||||
68
|
||||
],
|
||||
"terminalSeq": [
|
||||
69
|
||||
],
|
||||
"providerFinalSeq": [
|
||||
64
|
||||
]
|
||||
},
|
||||
"durationMs": 85536,
|
||||
"agentRunDurationMs": 41645,
|
||||
"tokens": {
|
||||
"input": 45592,
|
||||
"output": 731,
|
||||
"cachedInput": 65536
|
||||
},
|
||||
"reportedCostUsd": 0.0027245575999999996,
|
||||
"costStatus": "reported",
|
||||
"billingType": "unknown",
|
||||
"actualBillingVerified": false,
|
||||
"runtimeCostStatus": "not_metered",
|
||||
"cleanup": "passed",
|
||||
"matcherCount": 14,
|
||||
"failedMatchers": [],
|
||||
"independentRetainedReplayPassed": true
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-opencode.local.native-blocked-report",
|
||||
"variant": "candidate",
|
||||
"source": {
|
||||
"sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21",
|
||||
"ref": "refs/heads/codex/native-completion-master-calibrated-candidate",
|
||||
"workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980"
|
||||
},
|
||||
"suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d",
|
||||
"artifactId": 11264929195,
|
||||
"artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11264929195",
|
||||
"zipSha256": "33014bc90997444b1954bc465fd86ce1d93b3cfc4079a88545c110bc3896787b",
|
||||
"resultSha256": "8c943b38d480fcb3513fe6a458e75d0606e4669ba047ecd3a779b35a65ec2a30",
|
||||
"nativeSnapshotSha256": "b7bed5ee5d4942b3092516c790c2f527ca0859fa451ffc6b9423853ba70fea5a",
|
||||
"defaultSnapshotSha256": "3489c5facf99991f25ba864b99c12628e3f87442cc43a251beab60e0d0fe2992",
|
||||
"contextSnapshotSha256": null,
|
||||
"status": "passed",
|
||||
"attempt": 1,
|
||||
"runCount": 1,
|
||||
"runStatus": "succeeded",
|
||||
"issueStatus": "blocked",
|
||||
"model": "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
"provider": "opencode",
|
||||
"grade": {
|
||||
"schema": "paperclip.native-completion-observation.v2",
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "one-native-attempt",
|
||||
"passed": true,
|
||||
"detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work."
|
||||
},
|
||||
{
|
||||
"id": "durable-disposition",
|
||||
"passed": true,
|
||||
"detail": "Public durable issue status and the native semantic result agree with the requested disposition."
|
||||
},
|
||||
{
|
||||
"id": "exact-blocker",
|
||||
"passed": true,
|
||||
"detail": "The structured owner, exact action and whole-task scope are independently checked."
|
||||
},
|
||||
{
|
||||
"id": "complete-bound-events",
|
||||
"passed": true,
|
||||
"detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity."
|
||||
},
|
||||
{
|
||||
"id": "authoritative-native-result",
|
||||
"passed": true,
|
||||
"detail": "One runner proposal is independently accepted and terminated by the control plane."
|
||||
},
|
||||
{
|
||||
"id": "final-after-tool-result",
|
||||
"passed": true,
|
||||
"detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently."
|
||||
},
|
||||
{
|
||||
"id": "persisted-provider-final",
|
||||
"passed": true,
|
||||
"detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient."
|
||||
},
|
||||
{
|
||||
"id": "visible-blocker-content",
|
||||
"passed": true,
|
||||
"detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker."
|
||||
},
|
||||
{
|
||||
"id": "bounded-durable-work",
|
||||
"passed": true,
|
||||
"detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none."
|
||||
},
|
||||
{
|
||||
"id": "no-deployment-or-file-work",
|
||||
"passed": true,
|
||||
"detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed."
|
||||
}
|
||||
],
|
||||
"limitations": [
|
||||
"Exact provider feedback identity/consumption is not measured by the public sequence."
|
||||
]
|
||||
},
|
||||
"integrityFailures": [],
|
||||
"defaultGrade": {
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "production-default-bundle",
|
||||
"passed": true,
|
||||
"detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection."
|
||||
},
|
||||
{
|
||||
"id": "bounded-company-and-agent",
|
||||
"passed": true,
|
||||
"detail": "Both public budgets enforce the declared 1,000-cent hard stop."
|
||||
}
|
||||
]
|
||||
},
|
||||
"contextChecks": null,
|
||||
"durableState": {
|
||||
"documentCount": 0,
|
||||
"interactionCount": 0,
|
||||
"additionalIssueCount": 1,
|
||||
"additionalAgentCount": 0
|
||||
},
|
||||
"sequencing": {
|
||||
"proposalSeq": [
|
||||
36
|
||||
],
|
||||
"acceptedSeq": [
|
||||
49
|
||||
],
|
||||
"terminalSeq": [
|
||||
50
|
||||
],
|
||||
"providerFinalSeq": [
|
||||
45
|
||||
]
|
||||
},
|
||||
"durationMs": 81864,
|
||||
"agentRunDurationMs": 39947,
|
||||
"tokens": {
|
||||
"input": 22300,
|
||||
"output": 735,
|
||||
"cachedInput": 21248
|
||||
},
|
||||
"reportedCostUsd": 0.0016493492000000001,
|
||||
"costStatus": "reported",
|
||||
"billingType": "unknown",
|
||||
"actualBillingVerified": false,
|
||||
"runtimeCostStatus": "not_metered",
|
||||
"cleanup": "passed",
|
||||
"matcherCount": 21,
|
||||
"failedMatchers": [],
|
||||
"independentRetainedReplayPassed": true
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-acpx-claude.local.native-blocked-report",
|
||||
"variant": "candidate",
|
||||
"source": {
|
||||
"sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21",
|
||||
"ref": "refs/heads/codex/native-completion-master-calibrated-candidate",
|
||||
"workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980"
|
||||
},
|
||||
"suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d",
|
||||
"artifactId": 11264674612,
|
||||
"artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11264674612",
|
||||
"zipSha256": "309843a997c07414b999c3712cdea7592e5010e70a2dceff34a1c70b397954a9",
|
||||
"resultSha256": "4cdce8a05751fbbc6d3711cd40dc7a4d807c5e93dd5b8008b100692517afaa52",
|
||||
"nativeSnapshotSha256": "6ef4cf8878fbb4e5d530f846b73f749d3dc3d4d19ae64ace48d35f34a9b4143c",
|
||||
"defaultSnapshotSha256": "2f176d472827e84dc7e7d417080c6fdcdc413a6caec088164c4ffbf38b8cce91",
|
||||
"contextSnapshotSha256": null,
|
||||
"status": "passed",
|
||||
"attempt": 1,
|
||||
"runCount": 1,
|
||||
"runStatus": "succeeded",
|
||||
"issueStatus": "blocked",
|
||||
"model": "claude-sonnet-5",
|
||||
"provider": "acpx",
|
||||
"grade": {
|
||||
"schema": "paperclip.native-completion-observation.v2",
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "one-native-attempt",
|
||||
"passed": true,
|
||||
"detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work."
|
||||
},
|
||||
{
|
||||
"id": "durable-disposition",
|
||||
"passed": true,
|
||||
"detail": "Public durable issue status and the native semantic result agree with the requested disposition."
|
||||
},
|
||||
{
|
||||
"id": "exact-blocker",
|
||||
"passed": true,
|
||||
"detail": "The structured owner, exact action and whole-task scope are independently checked."
|
||||
},
|
||||
{
|
||||
"id": "complete-bound-events",
|
||||
"passed": true,
|
||||
"detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity."
|
||||
},
|
||||
{
|
||||
"id": "authoritative-native-result",
|
||||
"passed": true,
|
||||
"detail": "One runner proposal is independently accepted and terminated by the control plane."
|
||||
},
|
||||
{
|
||||
"id": "final-after-tool-result",
|
||||
"passed": true,
|
||||
"detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently."
|
||||
},
|
||||
{
|
||||
"id": "persisted-provider-final",
|
||||
"passed": true,
|
||||
"detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient."
|
||||
},
|
||||
{
|
||||
"id": "visible-blocker-content",
|
||||
"passed": true,
|
||||
"detail": "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker."
|
||||
},
|
||||
{
|
||||
"id": "bounded-durable-work",
|
||||
"passed": true,
|
||||
"detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none."
|
||||
},
|
||||
{
|
||||
"id": "no-deployment-or-file-work",
|
||||
"passed": true,
|
||||
"detail": "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed."
|
||||
}
|
||||
],
|
||||
"limitations": [
|
||||
"Exact provider feedback identity/consumption is not measured by the public sequence."
|
||||
]
|
||||
},
|
||||
"integrityFailures": [],
|
||||
"defaultGrade": {
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "production-default-bundle",
|
||||
"passed": true,
|
||||
"detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection."
|
||||
},
|
||||
{
|
||||
"id": "bounded-company-and-agent",
|
||||
"passed": true,
|
||||
"detail": "Both public budgets enforce the declared 1,000-cent hard stop."
|
||||
}
|
||||
]
|
||||
},
|
||||
"contextChecks": null,
|
||||
"durableState": {
|
||||
"documentCount": 0,
|
||||
"interactionCount": 0,
|
||||
"additionalIssueCount": 1,
|
||||
"additionalAgentCount": 0
|
||||
},
|
||||
"sequencing": {
|
||||
"proposalSeq": [
|
||||
52
|
||||
],
|
||||
"acceptedSeq": [
|
||||
81
|
||||
],
|
||||
"terminalSeq": [
|
||||
82
|
||||
],
|
||||
"providerFinalSeq": [
|
||||
77
|
||||
]
|
||||
},
|
||||
"durationMs": 69422,
|
||||
"agentRunDurationMs": 29059,
|
||||
"tokens": {
|
||||
"input": 6,
|
||||
"output": 1420,
|
||||
"cachedInput": 69575
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported",
|
||||
"billingType": "unknown",
|
||||
"actualBillingVerified": false,
|
||||
"runtimeCostStatus": "not_metered",
|
||||
"cleanup": "passed",
|
||||
"matcherCount": 21,
|
||||
"failedMatchers": [],
|
||||
"independentRetainedReplayPassed": true
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-acpx-claude.local.assigned-skill-explicit-invocation",
|
||||
"variant": "candidate",
|
||||
"source": {
|
||||
"sha": "d6e59e4712a3158ab4cd7d58deff1389b4578c21",
|
||||
"ref": "refs/heads/codex/native-completion-master-calibrated-candidate",
|
||||
"workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980"
|
||||
},
|
||||
"suiteDefinitionHash": "96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d",
|
||||
"artifactId": 11264634829,
|
||||
"artifactUrl": "https://github.com/paperclipai/paperclip/actions/runs/37098728980/artifacts/11264634829",
|
||||
"zipSha256": "7d4e0b3733bfea937167cac4ec0c18a5d9892e52b33a62f9a67fe7b55e039987",
|
||||
"resultSha256": "97a7d6cd5eb5bb1568b9f8ada8ef21b83323e0644d5aa7a741eefa946f6389c6",
|
||||
"nativeSnapshotSha256": "40b64742374ea90fada7fe1f8f484f1eebeaed424b12be87746efa9a60bc1e5f",
|
||||
"defaultSnapshotSha256": "ae2ae47d2f5dac9d2bcb243ea8d62c828a4c5081acb4538ca3ee4c8eb7f871a1",
|
||||
"contextSnapshotSha256": "2f451ae80966bb3d5512e1012048431c4e3216a97933ac222c51cdb602a11588",
|
||||
"status": "passed",
|
||||
"attempt": 1,
|
||||
"runCount": 1,
|
||||
"runStatus": "succeeded",
|
||||
"issueStatus": "done",
|
||||
"model": "claude-sonnet-5",
|
||||
"provider": "acpx",
|
||||
"grade": {
|
||||
"schema": "paperclip.native-completion-observation.v2",
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "one-native-attempt",
|
||||
"passed": true,
|
||||
"detail": "Exactly one succeeded native run, with no continuation/retry or uncounted extra work."
|
||||
},
|
||||
{
|
||||
"id": "durable-disposition",
|
||||
"passed": true,
|
||||
"detail": "Public durable issue status and the native semantic result agree with the requested disposition."
|
||||
},
|
||||
{
|
||||
"id": "complete-bound-events",
|
||||
"passed": true,
|
||||
"detail": "A complete contiguous public PRP stream is bound to this run and persisted source identity."
|
||||
},
|
||||
{
|
||||
"id": "authoritative-native-result",
|
||||
"passed": true,
|
||||
"detail": "One runner proposal is independently accepted and terminated by the control plane."
|
||||
},
|
||||
{
|
||||
"id": "final-after-tool-result",
|
||||
"passed": true,
|
||||
"detail": "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently."
|
||||
},
|
||||
{
|
||||
"id": "persisted-provider-final",
|
||||
"passed": true,
|
||||
"detail": "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient."
|
||||
},
|
||||
{
|
||||
"id": "bounded-durable-work",
|
||||
"passed": true,
|
||||
"detail": "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none."
|
||||
}
|
||||
],
|
||||
"limitations": [
|
||||
"Exact provider feedback identity/consumption is not measured by the public sequence."
|
||||
]
|
||||
},
|
||||
"integrityFailures": [],
|
||||
"defaultGrade": {
|
||||
"passed": true,
|
||||
"checks": [
|
||||
{
|
||||
"id": "production-default-bundle",
|
||||
"passed": true,
|
||||
"detail": "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection."
|
||||
},
|
||||
{
|
||||
"id": "bounded-company-and-agent",
|
||||
"passed": true,
|
||||
"detail": "Both public budgets enforce the declared 1,000-cent hard stop."
|
||||
}
|
||||
]
|
||||
},
|
||||
"contextChecks": [
|
||||
{
|
||||
"id": "assigned-skill-present",
|
||||
"passed": true,
|
||||
"detail": "The task run must receive one pinned skill version through the public assignment state."
|
||||
},
|
||||
{
|
||||
"id": "skill-request-explicit",
|
||||
"passed": true,
|
||||
"detail": "The task request must explicitly name the assigned skill with a supported slash or dollar reference."
|
||||
},
|
||||
{
|
||||
"id": "skill-source-marker",
|
||||
"passed": true,
|
||||
"detail": "The assigned pinned skill source must contain the output marker."
|
||||
},
|
||||
{
|
||||
"id": "marker-not-in-request",
|
||||
"passed": true,
|
||||
"detail": "The output marker must originate from the assigned skill, not the task request or comments."
|
||||
},
|
||||
{
|
||||
"id": "single-durable-output",
|
||||
"passed": true,
|
||||
"detail": "Exactly one durable task document must contain the skill's marker."
|
||||
},
|
||||
{
|
||||
"id": "completed-task",
|
||||
"passed": true,
|
||||
"detail": "Final task status: done."
|
||||
},
|
||||
{
|
||||
"id": "successful-runs",
|
||||
"passed": true,
|
||||
"detail": "All recorded context-integrity runs must succeed."
|
||||
}
|
||||
],
|
||||
"durableState": {
|
||||
"documentCount": 1,
|
||||
"interactionCount": 0,
|
||||
"additionalIssueCount": 1,
|
||||
"additionalAgentCount": 0
|
||||
},
|
||||
"sequencing": {
|
||||
"proposalSeq": [
|
||||
76
|
||||
],
|
||||
"acceptedSeq": [
|
||||
104
|
||||
],
|
||||
"terminalSeq": [
|
||||
105
|
||||
],
|
||||
"providerFinalSeq": [
|
||||
100
|
||||
]
|
||||
},
|
||||
"durationMs": 79552,
|
||||
"agentRunDurationMs": 27682,
|
||||
"tokens": {
|
||||
"input": 10,
|
||||
"output": 1096,
|
||||
"cachedInput": 161787
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported",
|
||||
"billingType": "unknown",
|
||||
"actualBillingVerified": false,
|
||||
"runtimeCostStatus": "not_metered",
|
||||
"cleanup": "passed",
|
||||
"matcherCount": 14,
|
||||
"failedMatchers": [],
|
||||
"independentRetainedReplayPassed": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"schema": "paperclip.native-completion-calibrated-comparison.v1",
|
||||
"sources": {
|
||||
"candidate": "d6e59e4712a3158ab4cd7d58deff1389b4578c21",
|
||||
"baseline": "e74ed61a69fbdd8b3a8f15dd6456bc3140246e33"
|
||||
},
|
||||
"gradedPairs": 6,
|
||||
"pendingPairs": 0,
|
||||
"newFailures": [],
|
||||
"newPasses": [
|
||||
"native-completion.runner-codex.local.native-blocked-report"
|
||||
],
|
||||
"unchangedFailures": [],
|
||||
"unchangedPasses": [
|
||||
"native-completion.runner-acpx-claude.local.assigned-skill-explicit-invocation",
|
||||
"native-completion.runner-acpx-claude.local.native-blocked-report",
|
||||
"native-completion.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
"native-completion.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"native-completion.runner-opencode.local.native-blocked-report"
|
||||
],
|
||||
"providerRuns": 12,
|
||||
"maximumAttemptsObserved": 1,
|
||||
"pairs": [
|
||||
{
|
||||
"executionId": "native-completion.runner-acpx-claude.local.assigned-skill-explicit-invocation",
|
||||
"baselineStatus": "passed",
|
||||
"candidateStatus": "passed",
|
||||
"baselineAgentMs": 27643,
|
||||
"candidateAgentMs": 27682,
|
||||
"baselineReportedUsd": 0,
|
||||
"candidateReportedUsd": 0,
|
||||
"baselineResultSha256": "6933dcb354f517653824b2c166bceef0f5c524ae30f915a9f4423276e53ebd15",
|
||||
"candidateResultSha256": "97a7d6cd5eb5bb1568b9f8ada8ef21b83323e0644d5aa7a741eefa946f6389c6"
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-acpx-claude.local.native-blocked-report",
|
||||
"baselineStatus": "passed",
|
||||
"candidateStatus": "passed",
|
||||
"baselineAgentMs": 28241,
|
||||
"candidateAgentMs": 29059,
|
||||
"baselineReportedUsd": 0,
|
||||
"candidateReportedUsd": 0,
|
||||
"baselineResultSha256": "cdbd533c56af2b70ca9ef7650356396a45a64a6e8b9a6b370a3a50b8ea50e3fd",
|
||||
"candidateResultSha256": "4cdce8a05751fbbc6d3711cd40dc7a4d807c5e93dd5b8008b100692517afaa52"
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
"baselineStatus": "passed",
|
||||
"candidateStatus": "passed",
|
||||
"baselineAgentMs": 31884,
|
||||
"candidateAgentMs": 34018,
|
||||
"baselineReportedUsd": 0,
|
||||
"candidateReportedUsd": 0,
|
||||
"baselineResultSha256": "22043de2aba3bac6ad94ee6d31a242746bad34a11790976d48ee0027b82c81b5",
|
||||
"candidateResultSha256": "09077172094e8f63861739884ae42e1fb5755f113e9a18a53b21697055d25607"
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-codex.local.native-blocked-report",
|
||||
"baselineStatus": "failed",
|
||||
"candidateStatus": "passed",
|
||||
"baselineAgentMs": 17560,
|
||||
"candidateAgentMs": 18794,
|
||||
"baselineReportedUsd": 0,
|
||||
"candidateReportedUsd": 0,
|
||||
"baselineResultSha256": "b80d822f6a4c6c66ee41d7d50e135cfa039a727aab8d304a54111ee839a7490f",
|
||||
"candidateResultSha256": "60c8a4f3e41b8ca9a80c10f73d804c30d6e6ede9ace5ce3e93d2c4a053503f32"
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"baselineStatus": "passed",
|
||||
"candidateStatus": "passed",
|
||||
"baselineAgentMs": 27842,
|
||||
"candidateAgentMs": 41645,
|
||||
"baselineReportedUsd": 0.0029267655,
|
||||
"candidateReportedUsd": 0.0027245575999999996,
|
||||
"baselineResultSha256": "5f084208b3211b0caccc47705c08fad3ccba72fa4568882c78f8946ac26e4be6",
|
||||
"candidateResultSha256": "f70d358b361e425fb4992b2101745a9912b2682134e780203ae5f79845dd5ee1"
|
||||
},
|
||||
{
|
||||
"executionId": "native-completion.runner-opencode.local.native-blocked-report",
|
||||
"baselineStatus": "passed",
|
||||
"candidateStatus": "passed",
|
||||
"baselineAgentMs": 35963,
|
||||
"candidateAgentMs": 39947,
|
||||
"baselineReportedUsd": 0.0012880548,
|
||||
"candidateReportedUsd": 0.0016493492000000001,
|
||||
"baselineResultSha256": "d8e19a8cf7770ea6138cd4f816892a7a04e6160063bbcbb6f7693efe28278826",
|
||||
"candidateResultSha256": "8c943b38d480fcb3513fe6a458e75d0606e4669ba047ecd3a779b35a65ec2a30"
|
||||
}
|
||||
],
|
||||
"detailedBaselineFailureAudit": "in_progress",
|
||||
"actualBillingVerified": false,
|
||||
"generalCodingQualityMeasured": false
|
||||
}
|
||||
@@ -1,3 +1,27 @@
|
||||
# Native completion qualification: retained paired task outcomes
|
||||
|
||||
**TL;DR: no extra failing task outcomes were observed in the six matched pairs.** Candidate passes all six cases; historical descriptions pass five. Codex whole-task blocking improves Fail→Pass; the other five remain Pass→Pass. There are zero newly failing cells, zero unchanged failures and zero pending pairs. This is one bounded trial, not proof of general coding quality.
|
||||
|
||||
The historical Codex blocker durably records the correct whole-task blocker and succeeds as a native run, but its visible reply omits the required marker. Its original executable failure is retained. The detailed provider-final versus semantic-fallback diagnosis is still being inspected; no cause is inferred from the grade alone.
|
||||
|
||||
| Profile | Case | Historical | Candidate | Native run seconds: historical → candidate |
|
||||
|---|---|---|---|---|
|
||||
| runner-acpx-claude | assigned-skill-explicit-invocation | Passed | Passed | 27.6 → 27.7 |
|
||||
| runner-acpx-claude | native-blocked-report | Passed | Passed | 28.2 → 29.1 |
|
||||
| runner-codex | assigned-skill-explicit-invocation | Passed | Passed | 31.9 → 34.0 |
|
||||
| runner-codex | native-blocked-report | Failed | Passed | 17.6 → 18.8 |
|
||||
| runner-opencode | assigned-skill-explicit-invocation | Passed | Passed | 27.8 → 41.6 |
|
||||
| runner-opencode | native-blocked-report | Passed | Passed | 36.0 → 39.9 |
|
||||
|
||||
[Closed retained-result audit and hashes](2026-10-02-native-completion-calibrated-results/comparison.json), [candidate audit](2026-10-02-native-completion-calibrated-results/candidate-audit.json), and [historical audit](2026-10-02-native-completion-calibrated-results/baseline-audit.json) are inspectable. Original access-controlled artifact ZIPs/results remain unchanged. All twelve attempts are attempt 1 and cleanup passes. Candidate source is `d6e59e4712a3158ab4cd7d58deff1389b4578c21`; historical source is `e74ed61a69fbdd8b3a8f15dd6456bc3140246e33`. Both use suite hash `96e18a7bcde391304cbdb2e811276e95f7c6a99c01b183facd7c5e1e4a98363d` and the frozen matched configurations.
|
||||
|
||||
Independent candidate retained-input replay agrees with all original grades: 51 strict native checks, 12 production-default/budget checks and 21 original assigned-skill document checks pass. The three requested documents are durably saved; the three blockers retain exact owner/action/scope and visible explanation, with no commanded file/deployment work. Named same-call finishing results precede provider finals with authoritative native acceptance/termination. Historical passed cases also retain their named sequence and document evidence; the failed Codex blocker exits before its additional native snapshot is written.
|
||||
|
||||
Recorded model-cost subtotal is $0.00421482 historical and $0.00437391 candidate. Only OpenCode reports positive costs; Codex/Claude report zeros with unknown billing type. These are recorded subtotals, **not verified invoices or evidence of free calls**. Hosted execution cost is unmetered. Per-run token/count/timing coverage is retained, and every actual run is counted. Single-run timing differences are descriptive; they do not establish a performance trend.
|
||||
|
||||
Exact-head normal CI and fresh Greptile 5/5 are green with no unresolved threads. The PR remains draft and unmerged while final evidence audit/publication completes. Fixed native prompt removal, resume behavior under paid models, provider feedback consumption and general coding quality are not measured by these single-turn cases. Native resume/catalog behavior has separate deterministic coverage.
|
||||
|
||||
## Preserved admission and prior zero-provider cohorts
|
||||
# Native completion qualification: calibrated hosted boundary
|
||||
|
||||
**TL;DR: model performance remains unmeasured; six task pairs are pending.** The candidate `d6e59e4712a3158ab4cd7d58deff1389b4578c21` and historical-description baseline `e74ed61a69fbdd8b3a8f15dd6456bc3140246e33` now pass complete actual launcher prepare→verify in both fresh local and explicitly synthetic hosted modes. The latter reproduces verified archive hydration, public source/run metadata and captured provenance framing without credentials or providers. It is a provider-free calibration, **not an actual trusted GitHub run**.
|
||||
|
||||
@@ -510,3 +510,5 @@ The selected #14961 `0a9c5a7` / `00a761b` hosted cohort also stopped before prov
|
||||
Selected #14961 now freezes `d6e59e471` / `e74ed61a6` after the approved five-file setup-only repair. Both exact refs pass actual launcher local and explicitly synthetic hosted prepare→verify (no credentials/providers): 132/127 selected TypeScript, 128 Node each, one local Rust calibration each; synthetic mode truthfully records no Rust execution. [The calibrated immutable manifest](2026-10-02-native-completion-calibrated-manifest.json) keeps exactly six pairs/twelve initial expected provider turns, strict oracles and single attempts. Prior twelve-cell zero-provider setup cohorts remain intact; model outcomes are pending.
|
||||
|
||||
The first actual-provider #14961 comparison is dispatched once on separate immutable targets: [candidate37098728980](https://github.com/paperclipai/paperclip/actions/runs/37098728980) and [historical37098815696](https://github.com/paperclipai/paperclip/actions/runs/37098815696), same trusted workflow215586d. Builds overlap; zero graded pairs at dispatch, usage not yet available. Normal review resolved the publication/baseline approval provenance gates. No paid behavior retry or merge occurred.
|
||||
|
||||
Selected #14961 first actual cohort now has all twelve original results retained: six candidate passes; historical five passes plus Codex blocker failure for a missing visible marker. Six pairs: zero new failures, one Fail→Pass, five Pass→Pass; no pending pairs. All are single attempts and cleanup passes. [The paired report](2026-10-02-native-completion-master-qualification.md) preserves original failure and exact hashes; detailed fallback/final diagnosis is pending. CI/review is green, PR stays draft/unmerged pending final audit. Actual invoiced costs and general coding quality remain unmeasured.
|
||||
Reference in new issue
Block a user