mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
Fix blocker browser grading and retain the complete native comparison
Accept the requested marker plus concrete blocker explanation, calibrate contradictory and future-condition replies, and clarify future fixture punctuation. Preserve every original paid failure and label retained-DOM replay separately from live qualification. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
be63fb52b3
commit
de487b20d3
11 files changed
+783
-109
No files matched your search
@@ -1,20 +1,127 @@
|
||||
{
|
||||
"schema": "paperclip.native-completion.safe-comparison/v1",
|
||||
"asOf": "2026-10-02T19:41:00Z",
|
||||
"partial": true,
|
||||
"schema": "paperclip.native-completion.safe-comparison/v2",
|
||||
"asOf": "2026-10-02T20:30:00Z",
|
||||
"partial": false,
|
||||
"expectedCellsPerVariant": 6,
|
||||
"expectedProviderTurnsPerVariant": 6,
|
||||
"candidateSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"candidatePilotSha": "55ce97b675e1ed68cc171fe44b729147209a7e24",
|
||||
"baselineSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"originalPairedDelta": {
|
||||
"newFailures": 0,
|
||||
"newPasses": 0,
|
||||
"unchangedPasses": 3,
|
||||
"unchangedFailures": 3,
|
||||
"pending": 0
|
||||
},
|
||||
"candidate": [
|
||||
{
|
||||
"executionId": "stock-harness.runner-acpx-claude.local.assigned-skill-explicit-invocation",
|
||||
"profileId": "runner-acpx-claude",
|
||||
"caseId": "assigned-skill-explicit-invocation",
|
||||
"status": "passed",
|
||||
"model": "claude-sonnet-5",
|
||||
"durationMs": 45214,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 10,
|
||||
"outputTokens": 1012,
|
||||
"cachedInputTokens": 156332,
|
||||
"totalTokens": 157354,
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 25013,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"providerDurationMs": 25013,
|
||||
"originalFailureClass": null,
|
||||
"originalMarkerOnlyUiFailure": false,
|
||||
"independentChecks": 10,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"sourceFingerprint": "f4ab12423213e122604d34bdad0234db389478aa4a8ab883e307c1dbbc8ecff8",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 589
|
||||
},
|
||||
"resultSha256": "41070facb1177c865c2ae3b1c52ea2233bcc68e1e22aa53b44cad983e06e3f26",
|
||||
"publicTaskDocuments": 1,
|
||||
"behaviorChecks": 7,
|
||||
"allBehaviorChecksPass": true
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-acpx-claude.local.native-blocked-report",
|
||||
"profileId": "runner-acpx-claude",
|
||||
"caseId": "native-blocked-report",
|
||||
"status": "failed",
|
||||
"model": "claude-sonnet-5",
|
||||
"durationMs": 73954,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 6,
|
||||
"outputTokens": 1238,
|
||||
"cachedInputTokens": 86721,
|
||||
"totalTokens": 87965,
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 26599,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"providerDurationMs": 26599,
|
||||
"originalFailureClass": "candidate_failure",
|
||||
"originalMarkerOnlyUiFailure": true,
|
||||
"independentChecks": 14,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"sourceFingerprint": "f4ab12423213e122604d34bdad0234db389478aa4a8ab883e307c1dbbc8ecff8",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 589
|
||||
},
|
||||
"resultSha256": "9d10096be385cce395e6ebe365a83a912cb92390b1e47b392003fac698aadb60",
|
||||
"retainedDiagnosis": "marker-only UI grader defect; all persisted/visible API matchers pass"
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
"sourceSha": "55ce97b675e1ed68cc171fe44b729147209a7e24",
|
||||
"profileId": "runner-codex",
|
||||
"caseId": "assigned-skill-explicit-invocation",
|
||||
"status": "passed",
|
||||
"model": "gpt-5.6-sol",
|
||||
"durationMs": 46812,
|
||||
"providerDurationMs": 25483,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
@@ -41,35 +148,482 @@
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"checks": 7,
|
||||
"allBehaviorChecksPass": true,
|
||||
"publicTaskDocuments": 1,
|
||||
"sourceSha": "55ce97b675e1ed68cc171fe44b729147209a7e24",
|
||||
"providerDurationMs": 25483,
|
||||
"originalFailureClass": null,
|
||||
"originalMarkerOnlyUiFailure": false,
|
||||
"independentChecks": 10,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "55ce97b675e1ed68cc171fe44b729147209a7e24",
|
||||
"sourceFingerprint": "ca46783b9d6529260667716a45d47c7fec43b36d6c813549212f6dbd4d2a25d0",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"tests": 585
|
||||
"executedChecks": 585
|
||||
},
|
||||
"resultSha256": "3a59532dd61292e551e1de678b5f84df49c01b760ae03ea4a784a039ba0e55d1"
|
||||
"resultSha256": "3a59532dd61292e551e1de678b5f84df49c01b760ae03ea4a784a039ba0e55d1",
|
||||
"publicTaskDocuments": 1,
|
||||
"behaviorChecks": 7,
|
||||
"allBehaviorChecksPass": true
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-codex.local.native-blocked-report",
|
||||
"profileId": "runner-codex",
|
||||
"caseId": "native-blocked-report",
|
||||
"status": "failed",
|
||||
"model": "gpt-5.6-sol",
|
||||
"durationMs": 62782,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 42772,
|
||||
"outputTokens": 382,
|
||||
"cachedInputTokens": 21169,
|
||||
"totalTokens": 64323,
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 16475,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"providerDurationMs": 16475,
|
||||
"originalFailureClass": "candidate_failure",
|
||||
"originalMarkerOnlyUiFailure": true,
|
||||
"independentChecks": 14,
|
||||
"failedIndependentPaths": [
|
||||
"run.resultJson.nativeResult.blocker.unblockAction"
|
||||
],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"sourceFingerprint": "f4ab12423213e122604d34bdad0234db389478aa4a8ab883e307c1dbbc8ecff8",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 589
|
||||
},
|
||||
"resultSha256": "fc1b6262c51f9d8b92fb4c82c9de4977ce8259ae0b63597a0c1a3cb406e130f9",
|
||||
"retainedDiagnosis": "marker-only UI grader defect plus exact-action punctuation mismatch"
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"profileId": "runner-opencode",
|
||||
"caseId": "assigned-skill-explicit-invocation",
|
||||
"status": "passed",
|
||||
"model": "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
"durationMs": 66835,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 43262,
|
||||
"outputTokens": 1072,
|
||||
"cachedInputTokens": 85248,
|
||||
"totalTokens": 129582,
|
||||
"reportedCostUsd": 0.0028500250000000004,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 44963,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0.0028500250000000004,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0.0028500250000000004,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"providerDurationMs": 44963,
|
||||
"originalFailureClass": null,
|
||||
"originalMarkerOnlyUiFailure": false,
|
||||
"independentChecks": 10,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"sourceFingerprint": "f4ab12423213e122604d34bdad0234db389478aa4a8ab883e307c1dbbc8ecff8",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 589
|
||||
},
|
||||
"resultSha256": "e02ba645c4d938ec6e106f245504df3610b8b0d4eaaadec11c66553bb01615d9",
|
||||
"publicTaskDocuments": 1,
|
||||
"behaviorChecks": 7,
|
||||
"allBehaviorChecksPass": true
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-opencode.local.native-blocked-report",
|
||||
"profileId": "runner-opencode",
|
||||
"caseId": "native-blocked-report",
|
||||
"status": "failed",
|
||||
"model": "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
"durationMs": 92115,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 21456,
|
||||
"outputTokens": 823,
|
||||
"cachedInputTokens": 20224,
|
||||
"totalTokens": 42503,
|
||||
"reportedCostUsd": 0.00153276,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 44633,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0.00153276,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0.00153276,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"providerDurationMs": 44633,
|
||||
"originalFailureClass": "candidate_failure",
|
||||
"originalMarkerOnlyUiFailure": true,
|
||||
"independentChecks": 14,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"sourceFingerprint": "f4ab12423213e122604d34bdad0234db389478aa4a8ab883e307c1dbbc8ecff8",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 589
|
||||
},
|
||||
"resultSha256": "255b5aea2c938baf9875fb0c98198bfabf3aa9a0845af7ee89ef3a6a2e9adf83",
|
||||
"retainedDiagnosis": "marker-only UI grader defect; all persisted/visible API matchers pass"
|
||||
}
|
||||
],
|
||||
"baseline": [],
|
||||
"pendingCandidate": [
|
||||
"stock-harness.runner-codex.local.native-blocked-report",
|
||||
"stock-harness.runner-acpx-claude.local.assigned-skill-explicit-invocation",
|
||||
"stock-harness.runner-acpx-claude.local.native-blocked-report",
|
||||
"stock-harness.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"stock-harness.runner-opencode.local.native-blocked-report"
|
||||
],
|
||||
"pendingBaseline": [
|
||||
"stock-harness.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
"stock-harness.runner-codex.local.native-blocked-report",
|
||||
"stock-harness.runner-acpx-claude.local.assigned-skill-explicit-invocation",
|
||||
"stock-harness.runner-acpx-claude.local.native-blocked-report",
|
||||
"stock-harness.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"stock-harness.runner-opencode.local.native-blocked-report"
|
||||
"baseline": [
|
||||
{
|
||||
"executionId": "stock-harness.runner-acpx-claude.local.assigned-skill-explicit-invocation",
|
||||
"profileId": "runner-acpx-claude",
|
||||
"caseId": "assigned-skill-explicit-invocation",
|
||||
"status": "passed",
|
||||
"model": "claude-sonnet-5",
|
||||
"durationMs": 51820,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 10,
|
||||
"outputTokens": 1015,
|
||||
"cachedInputTokens": 155959,
|
||||
"totalTokens": 156984,
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 30896,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"providerDurationMs": 30896,
|
||||
"originalFailureClass": null,
|
||||
"originalMarkerOnlyUiFailure": false,
|
||||
"independentChecks": 10,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"sourceFingerprint": "07aae3c4b57106fa9c6a6f218f6fa16aa42d998f65d8f9be9abea6120019b401",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 588
|
||||
},
|
||||
"resultSha256": "64a4599ce7dc230d5fe586af8c0f000e87d27a96c9d7fcb24969ee04b6069bee",
|
||||
"publicTaskDocuments": 1,
|
||||
"behaviorChecks": 7,
|
||||
"allBehaviorChecksPass": true
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-acpx-claude.local.native-blocked-report",
|
||||
"profileId": "runner-acpx-claude",
|
||||
"caseId": "native-blocked-report",
|
||||
"status": "failed",
|
||||
"model": "claude-sonnet-5",
|
||||
"durationMs": 77209,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 6,
|
||||
"outputTokens": 909,
|
||||
"cachedInputTokens": 66538,
|
||||
"totalTokens": 67453,
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 29743,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"providerDurationMs": 29743,
|
||||
"originalFailureClass": "candidate_failure",
|
||||
"originalMarkerOnlyUiFailure": true,
|
||||
"independentChecks": 14,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"sourceFingerprint": "07aae3c4b57106fa9c6a6f218f6fa16aa42d998f65d8f9be9abea6120019b401",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 588
|
||||
},
|
||||
"resultSha256": "091e358cfd955d7b9a00d2c6b5a7a9cd01e16a21b8b320ff5e80326c7b75b42f",
|
||||
"retainedDiagnosis": "marker-only UI grader defect; all persisted/visible API matchers pass"
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
"profileId": "runner-codex",
|
||||
"caseId": "assigned-skill-explicit-invocation",
|
||||
"status": "passed",
|
||||
"model": "gpt-5.6-sol",
|
||||
"durationMs": 51075,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 156433,
|
||||
"outputTokens": 1495,
|
||||
"cachedInputTokens": 132956,
|
||||
"totalTokens": 290884,
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 30332,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"providerDurationMs": 30332,
|
||||
"originalFailureClass": null,
|
||||
"originalMarkerOnlyUiFailure": false,
|
||||
"independentChecks": 10,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"sourceFingerprint": "07aae3c4b57106fa9c6a6f218f6fa16aa42d998f65d8f9be9abea6120019b401",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 588
|
||||
},
|
||||
"resultSha256": "ca3e16bf9d94a88071a62d9eaecf9f3aa115e7594913b9f7bcbd55001d4366fc",
|
||||
"publicTaskDocuments": 1,
|
||||
"behaviorChecks": 7,
|
||||
"allBehaviorChecksPass": true
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-codex.local.native-blocked-report",
|
||||
"profileId": "runner-codex",
|
||||
"caseId": "native-blocked-report",
|
||||
"status": "failed",
|
||||
"model": "gpt-5.6-sol",
|
||||
"durationMs": 64524,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 42597,
|
||||
"outputTokens": 464,
|
||||
"cachedInputTokens": 21040,
|
||||
"totalTokens": 64101,
|
||||
"reportedCostUsd": 0,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 16644,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"providerDurationMs": 16644,
|
||||
"originalFailureClass": "candidate_failure",
|
||||
"originalMarkerOnlyUiFailure": true,
|
||||
"independentChecks": 14,
|
||||
"failedIndependentPaths": [
|
||||
"run.resultJson.nativeResult.blocker.unblockAction"
|
||||
],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"sourceFingerprint": "07aae3c4b57106fa9c6a6f218f6fa16aa42d998f65d8f9be9abea6120019b401",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 588
|
||||
},
|
||||
"resultSha256": "e28c005b1f44a4fc09fb8f9169f50a40d759af2d33b377b737bd44fa7cc4d5bb",
|
||||
"retainedDiagnosis": "marker-only UI grader defect plus exact-action punctuation mismatch"
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-opencode.local.assigned-skill-explicit-invocation",
|
||||
"profileId": "runner-opencode",
|
||||
"caseId": "assigned-skill-explicit-invocation",
|
||||
"status": "passed",
|
||||
"model": "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
"durationMs": 90664,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 44977,
|
||||
"outputTokens": 1584,
|
||||
"cachedInputTokens": 105984,
|
||||
"totalTokens": 152545,
|
||||
"reportedCostUsd": 0.0037635715,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 66281,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0.0037635715,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0.0037635715,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"providerDurationMs": 66281,
|
||||
"originalFailureClass": null,
|
||||
"originalMarkerOnlyUiFailure": false,
|
||||
"independentChecks": 10,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"sourceFingerprint": "07aae3c4b57106fa9c6a6f218f6fa16aa42d998f65d8f9be9abea6120019b401",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 588
|
||||
},
|
||||
"resultSha256": "63d94f6b94affe09c4d52031b12904e10a4b685b1b576af13f23d9880dcf8a37",
|
||||
"publicTaskDocuments": 1,
|
||||
"behaviorChecks": 7,
|
||||
"allBehaviorChecksPass": true
|
||||
},
|
||||
{
|
||||
"executionId": "stock-harness.runner-opencode.local.native-blocked-report",
|
||||
"profileId": "runner-opencode",
|
||||
"caseId": "native-blocked-report",
|
||||
"status": "failed",
|
||||
"model": "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
"durationMs": 71434,
|
||||
"cleanup": "passed",
|
||||
"billing": {
|
||||
"llm": {
|
||||
"runCount": 1,
|
||||
"runsWithTokenUsage": 1,
|
||||
"runsWithReportedCost": 1,
|
||||
"inputTokens": 20857,
|
||||
"outputTokens": 488,
|
||||
"cachedInputTokens": 20224,
|
||||
"totalTokens": 41569,
|
||||
"reportedCostUsd": 0.0010970715,
|
||||
"costStatus": "reported"
|
||||
},
|
||||
"runtime": {
|
||||
"provider": "local",
|
||||
"agentRunDurationMs": 23477,
|
||||
"leaseDurationMs": null,
|
||||
"leaseCount": 0,
|
||||
"costStatus": "not_metered",
|
||||
"costSource": "local_not_metered"
|
||||
},
|
||||
"reportedCostUsd": 0.0010970715,
|
||||
"estimatedRuntimeCostUsd": 0,
|
||||
"observedAndEstimatedCostUsd": 0.0010970715,
|
||||
"complete": true
|
||||
},
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"providerDurationMs": 23477,
|
||||
"originalFailureClass": "candidate_failure",
|
||||
"originalMarkerOnlyUiFailure": true,
|
||||
"independentChecks": 14,
|
||||
"failedIndependentPaths": [],
|
||||
"prerequisites": {
|
||||
"passed": true,
|
||||
"sourceSha": "8792aed8ac8e9d004406afcb4b9a4ffddccd72c8",
|
||||
"sourceFingerprint": "07aae3c4b57106fa9c6a6f218f6fa16aa42d998f65d8f9be9abea6120019b401",
|
||||
"providerCallsBeforeAdmission": 0,
|
||||
"executedChecks": 588
|
||||
},
|
||||
"resultSha256": "475f9bb10c3397b6d3d4826262f38873ac7d6f9491f9db63bb07be2b3c85f8df",
|
||||
"retainedDiagnosis": "marker-only UI grader defect; all persisted/visible API matchers pass"
|
||||
}
|
||||
],
|
||||
"pendingCandidate": [],
|
||||
"pendingBaseline": [],
|
||||
"fixtureMatchProof": {
|
||||
"schema": "paperclip.native-completion.fixture-match-proof/v1",
|
||||
"cells": [
|
||||
@@ -105,10 +659,109 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"retainedDomRegrade": {
|
||||
"schema": "paperclip.native-blocker.retained-dom-regrade/v1",
|
||||
"providerCalls": 0,
|
||||
"method": "Retained final Playwright DOM text replays through the corrected assertion; original visibility separately inspected in the retained failure screenshot. This does not replace original live verdicts or claim a new live run.",
|
||||
"rows": [
|
||||
{
|
||||
"variant": "baseline",
|
||||
"profileId": "runner-acpx-claude",
|
||||
"retainedMarkerExplanationPass": true,
|
||||
"traceSha256": "85764e148f85aee75fe8ff31a7084e844b544e33e5b18b90b8b8e91b05f1cc73",
|
||||
"resultSha256": "091e358cfd955d7b9a00d2c6b5a7a9cd01e16a21b8b320ff5e80326c7b75b42f",
|
||||
"snapshotWallTime": 1790970574509,
|
||||
"retainedBubbleCount": 1,
|
||||
"markerBubbleCount": 1,
|
||||
"ownerPresent": true,
|
||||
"actionPresent": true,
|
||||
"contradictoryClaim": false,
|
||||
"htmlSha256": "b81a25b3dfdaf7dc67a7d6950cab880ac08e89e61e78672fb0af177aa127cd5e",
|
||||
"screenshotVisuallyInspected": true
|
||||
},
|
||||
{
|
||||
"variant": "baseline",
|
||||
"profileId": "runner-codex",
|
||||
"retainedMarkerExplanationPass": true,
|
||||
"traceSha256": "91a8630edfc00378b24cf9426ced8f2001252a603759efc3b28ea292d6ce53d2",
|
||||
"resultSha256": "e28c005b1f44a4fc09fb8f9169f50a40d759af2d33b377b737bd44fa7cc4d5bb",
|
||||
"snapshotWallTime": 1790970933943,
|
||||
"retainedBubbleCount": 1,
|
||||
"markerBubbleCount": 1,
|
||||
"ownerPresent": true,
|
||||
"actionPresent": true,
|
||||
"contradictoryClaim": false,
|
||||
"htmlSha256": "eb5ce83ec4c1044a8e1ec1c3c9444e0ceef8f6e074260f3b3e1b1daddd47b43c",
|
||||
"screenshotVisuallyInspected": true
|
||||
},
|
||||
{
|
||||
"variant": "baseline",
|
||||
"profileId": "runner-opencode",
|
||||
"retainedMarkerExplanationPass": true,
|
||||
"traceSha256": "f3d2e76581cb5090ab16e7110062ecd5be25bd06bf58eba6a2aa10976eeabee1",
|
||||
"resultSha256": "475f9bb10c3397b6d3d4826262f38873ac7d6f9491f9db63bb07be2b3c85f8df",
|
||||
"snapshotWallTime": 1790970990379,
|
||||
"retainedBubbleCount": 1,
|
||||
"markerBubbleCount": 1,
|
||||
"ownerPresent": true,
|
||||
"actionPresent": true,
|
||||
"contradictoryClaim": false,
|
||||
"htmlSha256": "1ff8be1dff61e629e2b4ef5a25ba1bca4c9d35b4bfb22720c78a9eebbe378901",
|
||||
"screenshotVisuallyInspected": true
|
||||
},
|
||||
{
|
||||
"variant": "candidate",
|
||||
"profileId": "runner-acpx-claude",
|
||||
"retainedMarkerExplanationPass": true,
|
||||
"traceSha256": "97799f078293c15c2e976ba744fb89688b9a0da1e43045d7fbad9be3ce326d84",
|
||||
"resultSha256": "9d10096be385cce395e6ebe365a83a912cb92390b1e47b392003fac698aadb60",
|
||||
"snapshotWallTime": 1790970677775,
|
||||
"retainedBubbleCount": 1,
|
||||
"markerBubbleCount": 1,
|
||||
"ownerPresent": true,
|
||||
"actionPresent": true,
|
||||
"contradictoryClaim": false,
|
||||
"htmlSha256": "b51ab457bda29867f790073762aa089e15ed85885253a95b4fa099579a6c2709",
|
||||
"screenshotVisuallyInspected": true
|
||||
},
|
||||
{
|
||||
"variant": "candidate",
|
||||
"profileId": "runner-codex",
|
||||
"retainedMarkerExplanationPass": true,
|
||||
"traceSha256": "f8ee2b43e5f2face66eeff696772bf733d0657056db7a0affe180ed763d3681a",
|
||||
"resultSha256": "fc1b6262c51f9d8b92fb4c82c9de4977ce8259ae0b63597a0c1a3cb406e130f9",
|
||||
"snapshotWallTime": 1790970650513,
|
||||
"retainedBubbleCount": 1,
|
||||
"markerBubbleCount": 1,
|
||||
"ownerPresent": true,
|
||||
"actionPresent": true,
|
||||
"contradictoryClaim": false,
|
||||
"htmlSha256": "fb0a8ee2730d29f9aa174db973c66af924a99f45029f9fb3d92089e58a269010",
|
||||
"screenshotVisuallyInspected": true
|
||||
},
|
||||
{
|
||||
"variant": "candidate",
|
||||
"profileId": "runner-opencode",
|
||||
"retainedMarkerExplanationPass": true,
|
||||
"traceSha256": "042b6d741b8cc42dc8d6584c0f9ec9eae795eaf03d155ddf305f0c6768017bd1",
|
||||
"resultSha256": "255b5aea2c938baf9875fb0c98198bfabf3aa9a0845af7ee89ef3a6a2e9adf83",
|
||||
"snapshotWallTime": 1790971103735,
|
||||
"retainedBubbleCount": 1,
|
||||
"markerBubbleCount": 1,
|
||||
"ownerPresent": true,
|
||||
"actionPresent": true,
|
||||
"contradictoryClaim": false,
|
||||
"htmlSha256": "54192526b4f2c99d9d9177a8ae4b41089d1cf5eb97bb2e19af57d072e8045d72",
|
||||
"screenshotVisuallyInspected": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"limitations": [
|
||||
"One graded candidate cell, no baseline or blocker results yet.",
|
||||
"Native reported zero cost has unknown actual provider billing; local runtime is unmetered.",
|
||||
"Pilot source precedes blocker-only grader correction; its completion fixture and production guidance are unchanged.",
|
||||
"Single trials do not establish general coding quality, reliability, speed or cost improvement."
|
||||
"Measures shared/additional native completion tool descriptions, not removal of native fixed prompts.",
|
||||
"Original 3/6 machine passes per variant are preserved. Four Claude/OpenCode blocker verdicts are grader defects. Codex strict action mismatch remains a failure in both.",
|
||||
"Provider-free retained DOM replay validates recorded reply text; original visibility is separately inspected in retained screenshots. It is not a new live execution.",
|
||||
"Future clarified fixture punctuation does not retroactively pass the original exact-action failures.",
|
||||
"Native reported zero cost does not establish zero actual billing; local runtime is unmetered.",
|
||||
"Single trials do not establish general coding quality, broad reliability, timing or cost improvement."
|
||||
]
|
||||
}
|
||||
@@ -1,86 +1,45 @@
|
||||
# Native completion guidance qualification — 2026-10-02
|
||||
|
||||
**Partial report, updated 19:41 UTC.** One of six candidate results is available:
|
||||
native Codex completion passes. No baseline or blocker result is available yet.
|
||||
The selected scope is three native profiles, each with one
|
||||
completion and one concrete blocker case: six cells per variant. Missing cells
|
||||
are pending, not passing. [Draft PR #14961](https://github.com/paperclipai/paperclip/pull/14961)
|
||||
remains stacked on [draft PR #14948](https://github.com/paperclipai/paperclip/pull/14948).
|
||||
**TL;DR: zero newly failing cases, zero newly passing cases, three unchanged completion passes and three unchanged blocker failures in six paired cases.** All 12 results are available; none are pending. Claude/OpenCode blocker failures come from a shared marker-only browser grader defect. Codex has the same additional exact-action punctuation mismatch in both variants. This measures added/shared native finish/block descriptions, not removal of native fixed prompts. The grading defects and single trials limit non-regression claims.
|
||||
|
||||
## Results available now
|
||||
Draft [PR #14961](https://github.com/paperclipai/paperclip/pull/14961) remains stacked on draft [PR #14948](https://github.com/paperclipai/paperclip/pull/14948). Original machine verdicts and failure classifications are preserved in the [safe evidence projection](2026-10-02-native-completion-live-comparison.json).
|
||||
|
||||
The earlier [manual/shared-prompt comparison](2026-10-02-stock-harness-live-comparison.md)
|
||||
has all 48 results, including failures. Both variants pass 15 of 24 cases, with
|
||||
different outcomes. Classic Claude/OpenCode document delivery fails with the
|
||||
reduced instructions. That draft remains unqualified for merge. Those legacy
|
||||
results do not measure this native completion-tool change.
|
||||
| Native profile | Completion baseline → candidate | Original blocker baseline → candidate | Separate retained-evidence assessment |
|
||||
| --- | --- | --- | --- |
|
||||
| Codex | Pass → Pass | Fail → Fail | Visible reply passes corrected UI check in both; exact stored action adds a period in both and remains a strict failure. |
|
||||
| ACPX Claude | Pass → Pass | Fail → Fail | All 14 original API matchers pass in both; corrected retained-DOM check passes in both. Original failures are UI grader defects. |
|
||||
| OpenCode | Pass → Pass | Fail → Fail | All 14 original API matchers pass in both; corrected retained-DOM check passes in both. Original failures are UI grader defects. |
|
||||
|
||||
| Native profile | Completion baseline | Completion candidate | Blocker baseline | Blocker candidate |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Codex | Pending | Pass at `55ce97b675e1ed68cc171fe44b729147209a7e24` | Pending | Pending |
|
||||
| ACPX Claude | Pending | Pending | Pending | Pending |
|
||||
| OpenCode | Pending | Pending | Pending | Pending |
|
||||
## Sources and public reports
|
||||
|
||||
## Attempts and provenance
|
||||
- [Candidate Codex completion pilot](https://github.com/paperclipai/paperclip/actions/runs/37053897522), source `55ce97b675e1ed68cc171fe44b729147209a7e24`: one selected cell passes all seven task/skill checks, saves one public document, passes 585 exact-head prerequisites and cleanup. [Published pilot report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37053897522-1/index.html).
|
||||
- [Remaining candidate campaign](https://github.com/paperclipai/paperclip/actions/runs/37055582470), frozen source `ae34843731ac338bd806a4106cb447134bff402c`: two completion passes and three blocker failures. Every cell passes 589 exact-head prerequisites before provider access. [Published candidate report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37055582470-1/index.html).
|
||||
- [Matched historical campaign](https://github.com/paperclipai/paperclip/actions/runs/37055273989), source `8792aed8ac8e9d004406afcb4b9a4ffddccd72c8`: three completion passes and three blocker failures. Every cell passes 588 exact-head prerequisites, including prior v13 compatibility rather than candidate v13-to-v14 rotation. [Published baseline report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37055273989-1/index.html).
|
||||
- All publishers completed successfully, including failed-campaign publication. Each public dashboard describes its selected scope; none alone represents the full catalog. The combined comparison has all six intended cells per variant.
|
||||
- The comparison restores only five native production sources containing tool descriptions and the session fingerprint. The tiny manual/shared prompts and merged Codex base fix #14920 remain constant. Twelve behavioral/fixture sources and all six model/effort/auth/permission/environment configurations match. Production and the completion fixture are unchanged between the two candidate revisions.
|
||||
- The earlier [cancelled historical setup](https://github.com/paperclipai/paperclip/actions/runs/37054642871) at `9060f7ee4` stopped before any provider cell after review found an inadequate visible-text oracle. The strengthened API/visible oracle was then applied symmetrically before the measured campaigns. All original attempts remain retained.
|
||||
|
||||
- The protected [candidate completion pilot](https://github.com/paperclipai/paperclip/actions/runs/37053897522)
|
||||
runs from the trusted default-branch workflow against immutable candidate
|
||||
`55ce97b675e1ed68cc171fe44b729147209a7e24`. It selects only native Codex's skill
|
||||
completion case. All 585 exact-head prerequisites pass before provider admission.
|
||||
All seven independent task/skill checks pass; one Paperclip task document is
|
||||
saved, public hire/budget receipts pass, and cleanup passes. Provider time is
|
||||
25.483 seconds; cell time is 46.812 seconds. Its reported zero cost does not
|
||||
establish zero actual billing. Result SHA-256:
|
||||
`3a59532dd61292e551e1de678b5f84df49c01b760ae03ea4a784a039ba0e55d1`.
|
||||
The completion fixture and production guidance are unchanged by the subsequent
|
||||
blocker-grader correction.
|
||||
- The [remaining five candidate cells](https://github.com/paperclipai/paperclip/actions/runs/37055582470)
|
||||
run at `ae34843731ac338bd806a4106cb447134bff402c` on the frozen
|
||||
`codex/native-completion-qualified-cells` branch. This separate target avoids
|
||||
superseding the pilot while its report publication finishes.
|
||||
- The [corrected six-cell historical comparison](https://github.com/paperclipai/paperclip/actions/runs/37055273989)
|
||||
runs at `8792aed8ac8e9d004406afcb4b9a4ffddccd72c8` on
|
||||
`codex/native-completion-previous-guidance`, concurrently with candidate setup.
|
||||
- The first [historical setup](https://github.com/paperclipai/paperclip/actions/runs/37054642871)
|
||||
selected six cells at `9060f7ee4` on `codex/native-completion-previous-guidance`.
|
||||
It was cancelled before provider cells started after review identified a gap
|
||||
in the blocker grader. This partial setup remains recorded. Both variants
|
||||
will receive the same stronger visible-explanation checks before that case
|
||||
runs. The completion fixture is unchanged.
|
||||
- The comparison restores only five native production sources containing tool
|
||||
descriptions and their session fingerprint. The tiny hire manual, reduced
|
||||
shared prompts, and merged Codex base fix #14920 remain constant. Twelve
|
||||
behavioral/fixture sources are byte-identical between the corrected baseline
|
||||
and `ae34843731`; all six profile/model/auth/effort fixture configurations match.
|
||||
- Historical structural assertions expect the prior descriptions in actual
|
||||
authenticated/wire/provider catalogs. The historical v13 branch tests its
|
||||
prior compatibility cases; the candidate separately requires v13-to-v14
|
||||
rotation. Structural differences are not behavioral failures.
|
||||
## Grader correction and retained regrading
|
||||
|
||||
## Verification and review
|
||||
The blocker request requires a final explanation naming the owner and action. The generic browser assertion instead required the agent reply to contain only the marker, so every correct explanation failed that assertion. The separate API oracle already confirmed blocked issue state, succeeded native run, task-wide scope, owner, requested action, and visible explanation. It passes all 14 checks for Claude/OpenCode; Codex fails only the exact stored action check in each variant.
|
||||
|
||||
At the pilot source, full repository typecheck and build pass locally. All 909
|
||||
E2E support tests pass; typecheck and 27-cell discovery pass. Exact-head
|
||||
credential-free prerequisites pass all 585 executed checks (584 TypeScript and
|
||||
one Rust), with retained source/binary hashes and
|
||||
zero provider calls. The receipt is retained under
|
||||
`tests/runner-e2e/results/stock-harness-preflight-2026-10-02T19-29-43.267Z/`.
|
||||
The corrected browser helper requires exactly one visible marker-containing reply with the owner, action and blocking reason. Ten credential-free browser calibrations and 919 E2E support tests pass. Calibrations reject marker-only, missing fields, contradictory/resolved claims and duplicate replies, while accepting a future access condition. An initial future-condition calibration exposed an overly broad denial rule; the corrected rule distinguishes "already granted" from "blocked until access is granted". Initial failures are retained. E2E typecheck passes.
|
||||
|
||||
Greptile found that the blocker oracle could accept a visible marker while only
|
||||
checking owner/action in backend state. The strengthened oracle now also checks
|
||||
the visible owner, exact requested action, and a blocking explanation; its 12
|
||||
positive/plausible-negative calibrations pass. All 913 current support tests and
|
||||
E2E typecheck pass after that fix. The first added calibration caught
|
||||
the marker itself matching the word "blocked"; word boundaries corrected that
|
||||
grader defect. The initial failed calibration remains retained. Candidate
|
||||
production instructions and independent completion graders were not changed.
|
||||
A separate zero-provider analytical replay reconstructs only the final agent reply DOM from each original Playwright trace, preserving trace/result/DOM hashes. All six retained replies pass the corrected text assertion. All six original failure screenshots were inspected separately and show a visible blocked explanation with owner/action. This assessment does not replace original live results or claim that the new UI path ran in a fresh full-stack execution.
|
||||
|
||||
Fresh CI/review for the corrected fixture and paid outcomes remain pending.
|
||||
At corrected source `ae34843731`, all 589 credential-free prerequisite checks pass
|
||||
locally (588 TypeScript and one Rust), with zero provider calls. Its receipt is
|
||||
retained under `tests/runner-e2e/results/stock-harness-preflight-2026-10-02T19-37-13.267Z/`.
|
||||
The safe [partial evidence projection](2026-10-02-native-completion-live-comparison.json)
|
||||
records the graded result and explicitly pending cells. Actual cost remains
|
||||
unknown; no speed, spending, coding-quality,
|
||||
or broad reliability improvement is claimed. Native finish/block documentation
|
||||
must remain separate from legacy Paperclip skill/API completion guidance.
|
||||
Both Codex replies store `Grant deployment access.` rather than the strict expected `Grant deployment access`. The original fixture ends that requested phrase with a period, making punctuation ambiguous. Both exact-action failures remain failed. Future fixture wording quotes the phrase and explicitly excludes trailing punctuation; the expected action and plausible-negative punctuation calibration stay strict. No extra native provider run was launched to improve the report.
|
||||
|
||||
## Timing, cost and limitations
|
||||
|
||||
| Profile | Journey | Baseline provider seconds | Candidate provider seconds |
|
||||
| --- | --- | ---: | ---: |
|
||||
| runner-codex | Completion | 30.332 | 25.483 |
|
||||
| runner-codex | Blocker | 16.644 | 16.475 |
|
||||
| runner-acpx-claude | Completion | 30.896 | 25.013 |
|
||||
| runner-acpx-claude | Blocker | 29.743 | 26.599 |
|
||||
| runner-opencode | Completion | 66.281 | 44.963 |
|
||||
| runner-opencode | Blocker | 23.477 | 44.633 |
|
||||
|
||||
Each variant records six runs with token usage and reported cost on all six. Reported totals are $0.0048606430 baseline and $0.0043827850 candidate, principally OpenCode; Codex/Claude reported zeros have unknown actual billing. Local runtime is unmetered. All 12 retained cells cleaned up successfully. Browser-defect waits inflate cell durations, and one trial per cell cannot establish a timing or spending improvement.
|
||||
|
||||
The earlier [legacy prompt-removal comparison](2026-10-02-stock-harness-live-comparison.md) has two observed new document-delivery failures and two OpenCode ordered-case improvements; equal aggregate totals do not show equivalence. Its approved skill repair is measured separately in a narrow follow-up. Native finish/block documentation remains separate from legacy skill/API completion guidance. Both PRs remain draft, with user-controlled merging.
|
||||
@@ -0,0 +1,28 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { assertNativeBlockerReply } from "./native-blocker-visible.js";
|
||||
|
||||
test("accepts a visible blocker explanation rather than requiring a marker-only reply", async ({ page }) => {
|
||||
await page.setContent('<div data-testid="reply">Deployment is <strong>blocked</strong>.<br>Release Owner must Grant deployment access. BLOCKED_probe</div>');
|
||||
await assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100);
|
||||
});
|
||||
|
||||
test("accepts a future access condition while the task remains blocked", async ({ page }) => {
|
||||
await page.setContent('<div data-testid="reply">Deployment remains blocked until access is granted. Release Owner must Grant deployment access. BLOCKED_probe</div>');
|
||||
await assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100);
|
||||
});
|
||||
|
||||
for (const [name, body] of [
|
||||
["marker only", "BLOCKED_probe"],
|
||||
["missing owner", "Blocked. Someone must Grant deployment access. BLOCKED_probe"],
|
||||
["missing action", "Blocked. Release Owner must take action. BLOCKED_probe"],
|
||||
["missing reason", "Release Owner: Grant deployment access. BLOCKED_probe"],
|
||||
["contradictory disposition", "Deployment is not blocked. Release Owner completed Grant deployment access. BLOCKED_probe"],
|
||||
["resolved blocker", "Deployment is no longer blocked. Release Owner must Grant deployment access. BLOCKED_probe"],
|
||||
["granted access", "Blocked. Access already granted. Release Owner must Grant deployment access. BLOCKED_probe"],
|
||||
["duplicate replies", "Blocked. Release Owner must Grant deployment access. BLOCKED_probe</div><div data-testid=reply>BLOCKED_probe"],
|
||||
]) {
|
||||
test(`rejects ${name}`, async ({ page }) => {
|
||||
await page.setContent(`<div data-testid="reply">${body}</div>`);
|
||||
await expect(assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100)).rejects.toThrow();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { expect, type Locator } from "@playwright/test";
|
||||
|
||||
/** Blocker replies include an explanation; ordinary marker-only replies do not. */
|
||||
export async function assertNativeBlockerReply(agentReplies: Locator, marker: string, timeout = 30_000): Promise<void> {
|
||||
const reply = agentReplies.filter({ hasText: marker });
|
||||
await expect(reply).toHaveCount(1, { timeout });
|
||||
await expect(reply).toBeVisible({ timeout });
|
||||
await expect(reply).toContainText("Release Owner", { timeout });
|
||||
await expect(reply).toContainText("Grant deployment access", { timeout });
|
||||
await expect(reply).not.toContainText(
|
||||
/\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\b/i,
|
||||
{ timeout },
|
||||
);
|
||||
await expect(reply).toContainText(
|
||||
/\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\b/i,
|
||||
{ timeout },
|
||||
);
|
||||
}
|
||||
@@ -15,9 +15,16 @@ describe("native completion blocker oracle", () => {
|
||||
it("accepts the persisted blocker, owner and exact requested unblock action", async () => {
|
||||
expect((await evaluateMatchers(matchers, observation())).every(row => row.passed)).toBe(true);
|
||||
expect(nativeBlockedReportTask.buildPrompt("probe")).not.toMatch(/paperclip_(?:finish|block)|reportedWorkDisposition|json/i);
|
||||
expect(nativeBlockedReportTask.buildPrompt("probe")).toContain('"Grant deployment access" (without quotation marks or trailing punctuation)');
|
||||
});
|
||||
it("accepts a future access condition while the task remains blocked", async () => {
|
||||
const current = observation();
|
||||
current.message = "Deployment remains blocked until access is granted. Release Owner must Grant deployment access. BLOCKED_probe";
|
||||
expect((await evaluateMatchers(matchers, current)).every(row => row.passed)).toBe(true);
|
||||
});
|
||||
it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed",
|
||||
"visible-marker-only", "visible-owner-missing", "visible-action-missing", "visible-reason-missing"])(
|
||||
"visible-marker-only", "visible-owner-missing", "visible-action-missing", "visible-reason-missing",
|
||||
"visible-not-blocked", "visible-resolved", "visible-access-granted", "action-punctuation"])(
|
||||
"rejects %s without trusting the visible completion claim", async variant => {
|
||||
const current = observation();
|
||||
const result = (current.json as { run: { resultJson: { nativeResult?: { reportedWorkDisposition: string;
|
||||
@@ -33,6 +40,10 @@ describe("native completion blocker oracle", () => {
|
||||
if (variant === "visible-owner-missing") current.message = current.message!.replace("Release Owner", "Someone");
|
||||
if (variant === "visible-action-missing") current.message = current.message!.replace("Grant deployment access", "Take action");
|
||||
if (variant === "visible-reason-missing") current.message = "BLOCKED_probe. Release Owner: Grant deployment access.";
|
||||
if (variant === "visible-not-blocked") current.message = "Deployment is not blocked. Release Owner completed Grant deployment access. BLOCKED_probe";
|
||||
if (variant === "visible-resolved") current.message = "Deployment is no longer blocked. Release Owner must Grant deployment access. BLOCKED_probe";
|
||||
if (variant === "visible-access-granted") current.message = "Blocked. Access already granted. Release Owner must Grant deployment access. BLOCKED_probe";
|
||||
if (variant === "action-punctuation") result.nativeResult!.blocker.unblockAction += ".";
|
||||
expect((await evaluateMatchers(matchers, current)).some(row => !row.passed)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -15,7 +15,7 @@ export const nativeBlockedReportTask: RunnerTaskFixture = {
|
||||
buildPrompt: nonce => [
|
||||
"Deployment cannot proceed because required release access has not been granted.",
|
||||
"Release Owner is the external owner of that grant. Do not attempt deployment, create subtasks, or claim the access exists.",
|
||||
"Record the whole task as blocked, identify the owner as Release Owner, and state the unblock action exactly: Grant deployment access.",
|
||||
'Record the whole task as blocked and identify the owner as Release Owner. The exact unblock action is "Grant deployment access" (without quotation marks or trailing punctuation).',
|
||||
`Include BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")} in your final explanation.`,
|
||||
].join("\n"),
|
||||
buildMatchers: nonce => [
|
||||
@@ -25,7 +25,7 @@ export const nativeBlockedReportTask: RunnerTaskFixture = {
|
||||
{ kind: "message_contains", expected: `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}` },
|
||||
{ kind: "message_contains", expected: "Release Owner" },
|
||||
{ kind: "message_contains", expected: "Grant deployment access" },
|
||||
{ kind: "message_regex", pattern: "\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" },
|
||||
{ kind: "message_regex", pattern: "^(?![\\s\\S]*\\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\\b)[\\s\\S]*\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.reportedWorkDisposition", expected: "blocked" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.blocker.owner.name", expected: "Release Owner" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.blocker.unblockAction", expected: "Grant deployment access" },
|
||||
|
||||
@@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test";
|
||||
/** Browser-only harness regressions: no Paperclip instance or provider credentials. */
|
||||
export default defineConfig({
|
||||
testDir: ".",
|
||||
testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts"],
|
||||
testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts"],
|
||||
workers: 1,
|
||||
retries: 0,
|
||||
timeout: 10_000,
|
||||
|
||||
@@ -5,6 +5,7 @@ import { completionDelivery, type CompletionObservation } from "./completion-upd
|
||||
import { runInstructionPersistenceFlow } from "./instruction-persistence.js";
|
||||
import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js";
|
||||
import { runBlockerFlow } from "./blocker-flow.js";
|
||||
import { assertNativeBlockerReply } from "./native-blocker-visible.js";
|
||||
import { largeJournalEvidence } from "./journal-evidence.js";
|
||||
import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js";
|
||||
import { runAccountingFlow } from "./accounting-flow.js";
|
||||
@@ -2597,6 +2598,8 @@ for (const execution of executions) {
|
||||
expect(finalRunMessage.trim()).not.toBe("");
|
||||
await expect(visibleAgentReplies.filter({ hasText: finalRunMessage }).last())
|
||||
.toBeVisible({ timeout: 30_000 });
|
||||
} else if (execution.suite.id === "stock-harness" && execution.task.id === "native-blocked-report") {
|
||||
await assertNativeBlockerReply(visibleAgentReplies, marker);
|
||||
} else {
|
||||
const escapedMarker = marker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const terminalAgentReplies = visibleAgentReplies.filter({
|
||||
|
||||
@@ -96,6 +96,7 @@ export function sourceFingerprint() {
|
||||
...["capabilities.yaml", "mcp-tool-map.yaml", "eval-traceability.yaml", "capability-contract.md", "downstream-handoff.md"]
|
||||
.map(file => `packages/paperclip-runner/generated/capability/${file}`),
|
||||
"tests/runner-e2e/native-completion-case.ts",
|
||||
"tests/runner-e2e/native-blocker-visible.ts", "tests/runner-e2e/native-blocker-visible.spec.ts",
|
||||
"packages/adapter-utils/src/server-utils.ts", "packages/shared/src/connection-intent-guidance.ts",
|
||||
"server/src/onboarding-assets/default/AGENTS.md", "server/src/routes/agents.ts", "scripts/ensure-plugin-build-deps.mjs",
|
||||
"packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts",
|
||||
|
||||
@@ -14,6 +14,7 @@ describe("stock harness instruction revision", () => {
|
||||
"packages/paperclip-runner/src/drivers/runner-tool-bridge.ts", "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts",
|
||||
"server/src/services/native-runtime/native-session-resume.ts",
|
||||
"tests/runner-e2e/native-completion-case.ts",
|
||||
"tests/runner-e2e/native-blocker-visible.ts",
|
||||
])(
|
||||
"changes when the evaluated %s changes", source => {
|
||||
vi.mocked(readFileSync).mockImplementation(() => Buffer.from("unchanged"));
|
||||
|
||||
@@ -121,7 +121,7 @@ export async function captureStockHarness(input: {
|
||||
export function stockHarnessSourceDigest() {
|
||||
const hash = createHash("sha256");
|
||||
for (const source of [
|
||||
"stock-harness.ts", "native-completion-case.ts", "context-integrity-cases.ts", "context-integrity-scoring.ts",
|
||||
"stock-harness.ts", "native-completion-case.ts", "native-blocker-visible.ts", "context-integrity-cases.ts", "context-integrity-scoring.ts",
|
||||
"context-integrity-flow.ts", "chat-cases.ts", "chat-flow.ts", "live-fixtures.ts", "runner.spec.ts",
|
||||
"stock-harness-checks.mjs", "stock-harness-admission.ts", "stock-harness-manifest.ts",
|
||||
"../../packages/paperclip-runner/scripts/generate-capability-contract.mjs",
|
||||
|
||||
Reference in new issue
Block a user