diff --git a/doc/plans/2026-10-05-native-procedure-comparison.json b/doc/plans/2026-10-05-native-procedure-comparison.json new file mode 100644 index 00000000000..c78f6b90037 --- /dev/null +++ b/doc/plans/2026-10-05-native-procedure-comparison.json @@ -0,0 +1,5433 @@ +{ + "schema": "paperclip.native-procedure-public-comparison.v1", + "disposition": "hold_draft_no_regression_gate_failed", + "originalGradesPreserved": true, + "sources": { + "originalBaseline": "64cd2194a1d058356679b653c181f16404aefc80", + "originalCandidate": "eb01d073c35134bf0d373947b55a79f9e443f8ee", + "correctedBaseline": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "correctedCandidate": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "integratedCandidate": "ee096aa23fc88a001e7666215bcd338b5863f377", + "integratedProviderFreeBaseline": "ce31f5efd0d416205c3994c55dd02b645f8b8423" + }, + "controls": { + "trustedWorkflowSha": "a386a599983519eb1d399f8b770bfccdb2a74762", + "commonFixtureFilesChecked": 256, + "sourceDelta": [ + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json", + "packages/paperclip-runner/generated/semantic-action-catalog.json", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json", + "packages/paperclip-runner/protocol/manifest.json", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts", + "packages/paperclip-runner/src/contracts/runtime-context.ts", + "packages/paperclip-runner/src/protocol-actions/create-task.ts", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts", + "server/src/services/native-runtime/native-session-resume.ts", + "server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts" + ], + "cellsPerVariant": 6, + "maximumAttemptsPerCell": 1, + "companyHardStopCents": 1000, + "leadHardStopCents": 1000, + "workerIndividualHardStop": "not asserted; company hard stop applies", + "cellDeadlineMs": 720000, + "maximumStoryRunRecordsPerCell": 12 + }, + "commonFixtureHashesCanonicalJsonSha256": "2c1113b8995e5d004960905668d935ab77d463fa27ab979c824e3402210d1f27", + "original": { + "totals": { + "baseline": { + "pass": 4, + "fail": 2, + "retainedStartedRuns": 25 + }, + "candidate": { + "pass": 2, + "fail": 4, + "retainedStartedRuns": 15 + } + }, + "pairs": [ + { + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "baseline": "passed", + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "baseline": "passed", + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "baseline": "passed", + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "baseline": "passed", + "candidate": "passed" + }, + { + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "baseline": "failed", + "candidate": "passed" + }, + { + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "baseline": "failed", + "candidate": "failed" + } + ], + "cells": [ + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "source": { + "sha": "64cd2194a1d058356679b653c181f16404aefc80", + "ref": "refs/heads/codex/native-procedure-baseline", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325870695" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "b08ea732990fd75bea0f17df7ca06a87fc5ebe4a59c3113e810b777b7087d6a9", + "evidenceHashes": { + "evidence-manifest.json": "e37c96ffe166426f4c99e398142b1072d53421992cde8c7aada6303145fcdff3", + "html-report/data/5d2572f36528e8fddbc2153c632730c4f013b79a.json": "b08ea732990fd75bea0f17df7ca06a87fc5ebe4a59c3113e810b777b7087d6a9", + "playwright-output/runner-everyday-workflows--fdae5-the-browser-and-public-APIs/attachments/runner-e2e-result-eb06dc1fa3ae54172129187047897c5f1b2adb19.json": "b08ea732990fd75bea0f17df7ca06a87fc5ebe4a59c3113e810b777b7087d6a9", + "result.json": "b08ea732990fd75bea0f17df7ca06a87fc5ebe4a59c3113e810b777b7087d6a9", + "snapshots/api-state.json": "fa464c86756c6d01517cf486965a0266e9ec874fae8248ae547c58d39c6c7cea", + "snapshots/browser-diagnostics.json": "0bdbe36772ce52f8a4140cdf061596614d347229901be18b5cc17325cd0ea781", + "snapshots/chat-final-run-ledger.json": "1b4925e0c32d5369ddd23317f0a72d4060c99fc5f411d129c3db3b32a6a829dd", + "snapshots/everyday-workflow.json": "85a6509a51869f944d52b34d26a42ac48e83c455770140a55f0db58ba6aa1f0b", + "snapshots/fixtures.json": "140546a07e4fa11c45fea99741682eb5dc0d7905a2d20df22a45c4449aa07684" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "source": { + "sha": "64cd2194a1d058356679b653c181f16404aefc80", + "ref": "refs/heads/codex/native-procedure-baseline", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325870695" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 6, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "85935339f572d46cf3dfa351bad4376bb537bafe1ada9a7d798b75dbb099941a", + "evidenceHashes": { + "evidence-manifest.json": "5c935a3744afc62c64fd29a664c33026aef235a92ec0500c9912d3507052e697", + "html-report/data/8bbe2058a3abe580b4ccb871d532ba41350da7aa.json": "85935339f572d46cf3dfa351bad4376bb537bafe1ada9a7d798b75dbb099941a", + "playwright-output/runner-everyday-workflows--9a7d2-the-browser-and-public-APIs/attachments/runner-e2e-result-89ae8a684c0bb5d6bb61e001e54f638ef8240960.json": "85935339f572d46cf3dfa351bad4376bb537bafe1ada9a7d798b75dbb099941a", + "result.json": "85935339f572d46cf3dfa351bad4376bb537bafe1ada9a7d798b75dbb099941a", + "snapshots/api-state.json": "7073c56ae937235b6d25f830d8da2dc1c1740f9295b737d361c31bc5c21b2302", + "snapshots/browser-diagnostics.json": "695c94cb21dc497359abc12568fe2270509df2327368590067b4eeeceb8f6784", + "snapshots/chat-final-run-ledger.json": "1437a3268c85532e7c64dec8c8b09ebb8c9b94a8213ea13e1795214df04888c7", + "snapshots/everyday-workflow.json": "acedad4621258df94eeb405fb43a180d17a449b4edc3ba4a1ace5b97ded79b63", + "snapshots/fixtures.json": "32736c391b9c292c81d7d7aeb8b3ab678784d68ed169727430ed9a429f97d5a2" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "source": { + "sha": "64cd2194a1d058356679b653c181f16404aefc80", + "ref": "refs/heads/codex/native-procedure-baseline", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325870695" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 5, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "2b601d5736f3756ffb890a50821a655a49490a113922e15c199fef1fe0954e39", + "evidenceHashes": { + "evidence-manifest.json": "eb30cdd784087c1a34c4ecbaa7a43f051eddd95d3805fc89ab1054af70a044c2", + "html-report/data/cdbcafe148f5698ecd0df0cdf30809758fd418e8.json": "2b601d5736f3756ffb890a50821a655a49490a113922e15c199fef1fe0954e39", + "playwright-output/runner-everyday-workflows--a6fcb-the-browser-and-public-APIs/attachments/runner-e2e-result-b59bcc2342e97bbc85432cee261db26c76f3c80a.json": "2b601d5736f3756ffb890a50821a655a49490a113922e15c199fef1fe0954e39", + "result.json": "2b601d5736f3756ffb890a50821a655a49490a113922e15c199fef1fe0954e39", + "snapshots/api-state.json": "0b6a89bc7cdff9fa19c241eff210031468d7d500d9dd427526a888d740e8454a", + "snapshots/browser-diagnostics.json": "c520523de99f806d8cbe9fad497639ef813839b7dce1bbfaddaa7a1b77f66995", + "snapshots/chat-final-run-ledger.json": "ebae3fb5b1c5a87451af70817f9a6a35087b7606aed4c15437d453500675a3c6", + "snapshots/everyday-workflow.json": "691a281397c434341784cbbeabdf53d7ffbbe34cf6fccf05e05b365fffc2d2db", + "snapshots/fixtures.json": "93b762a930101110ad1953551cb5ba0094ccb609626f9ae2767d5d8c8d610424" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "source": { + "sha": "64cd2194a1d058356679b653c181f16404aefc80", + "ref": "refs/heads/codex/native-procedure-baseline", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325870695" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 6, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "200c06ab5dadf615bac0ec66b9a9ce851c75832ddaad5c854ed7fbbac16f90a7", + "evidenceHashes": { + "evidence-manifest.json": "5ff552d77ebbe35f2a123fcfe60f55008fd221b9c890ef92733b427e52634e28", + "html-report/data/cba00c1f6256ff612b9b89e78005f1a7460ae138.json": "200c06ab5dadf615bac0ec66b9a9ce851c75832ddaad5c854ed7fbbac16f90a7", + "playwright-output/runner-everyday-workflows--d934c-the-browser-and-public-APIs/attachments/runner-e2e-result-04508984e2fc501021db8f215efa4abf738af42d.json": "200c06ab5dadf615bac0ec66b9a9ce851c75832ddaad5c854ed7fbbac16f90a7", + "result.json": "200c06ab5dadf615bac0ec66b9a9ce851c75832ddaad5c854ed7fbbac16f90a7", + "snapshots/api-state.json": "e802749b93f083dbd37eae5a6cb4d63726bfa880fdad7a0cb9033a9e5132e5d1", + "snapshots/browser-diagnostics.json": "f9fd5484cc80f9fce274ba98be946d6f2826c20cf454ac5f0f09d5f29f4d4c9d", + "snapshots/chat-final-run-ledger.json": "44af7eed470338dcc0ad99eda29b65f4493e5ab48456e3196be70671550edf3e", + "snapshots/everyday-workflow.json": "36a50e25841b6a3827d7dc0c227fee06a07a5dfdfc6cdcef45e87006e2c49aec", + "snapshots/fixtures.json": "8e01cdc83c06cd80f9a7aa45844bd165c8152b40c551578f702dcd1facfe49fd" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "source": { + "sha": "64cd2194a1d058356679b653c181f16404aefc80", + "ref": "refs/heads/codex/native-procedure-baseline", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325870695" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": false + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "d9e4207efd71b1fde7b626100905f486ea758ccf149b1f8708d6f47e1b314f96", + "evidenceHashes": { + "evidence-manifest.json": "f12b13167c11896a77e2313295f8a4740a82d38e11179a92fdf9694af41a0014", + "html-report/data/1de72a5e503228c98ef17641b6a7e8e5520dd6d9.json": "d9e4207efd71b1fde7b626100905f486ea758ccf149b1f8708d6f47e1b314f96", + "playwright-output/runner-everyday-workflows--616ad-the-browser-and-public-APIs/attachments/runner-e2e-result-cbaa721581d55eb04872c27fef1564ad1093b5db.json": "d9e4207efd71b1fde7b626100905f486ea758ccf149b1f8708d6f47e1b314f96", + "result.json": "d9e4207efd71b1fde7b626100905f486ea758ccf149b1f8708d6f47e1b314f96", + "snapshots/api-state.json": "a88d15d5d1ea5cb4864f0ca86a9104207f90cb57424a0642c98bc63cd90ac5d5", + "snapshots/browser-diagnostics.json": "2ae30c24e8081099a72f8cb3fa5e86a633e525956cee65c09f36e6353c42efc1", + "snapshots/chat-final-run-ledger.json": "3674c59de912197327ccb8ba1d659f120b1cdc51b1f88b1bc3859485c31f1422", + "snapshots/everyday-workflow.json": "43730e83f96c51dda1cb39a7ab0509955af7bfcc18dd939a84cc1cef3c739d87", + "snapshots/fixtures.json": "9b2ef4cebf077c3ed74e55dc4faf5e5b547ba7be990214757cd479368c71717e" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "source": { + "sha": "64cd2194a1d058356679b653c181f16404aefc80", + "ref": "refs/heads/codex/native-procedure-baseline", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325870695" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "permanent_infrastructure", + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 0, + "resultLedgerRunSetEqual": true, + "originalChecks": [], + "resultSha256": "f8ef0e87c24ecacb8dd18ebc158771a74b5cbb8c0b673edd00c80e60a1932832", + "evidenceHashes": { + "evidence-manifest.json": "33b1383486d18fcb0a01b7e3488704fcfa760114958554469016725f79212508", + "html-report/data/26ec75604543a922ef760e2cfb41ed3f340a019b.json": "f8ef0e87c24ecacb8dd18ebc158771a74b5cbb8c0b673edd00c80e60a1932832", + "playwright-output/runner-everyday-workflows--90607-the-browser-and-public-APIs/attachments/runner-e2e-result-b9a31955fc83e0239705bc49c9a7cb13939d91ed.json": "f8ef0e87c24ecacb8dd18ebc158771a74b5cbb8c0b673edd00c80e60a1932832", + "result.json": "f8ef0e87c24ecacb8dd18ebc158771a74b5cbb8c0b673edd00c80e60a1932832", + "snapshots/browser-diagnostics.json": "94d2792cae9e2a1628779f7349538e0ce00418a6e448c54fcbba978b55a1b8ed" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "source": { + "sha": "eb01d073c35134bf0d373947b55a79f9e443f8ee", + "ref": "refs/heads/codex/native-procedure-guidance", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325777217" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "transient_infrastructure", + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 1, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "887c612dedbc4319e8ddf443d238b9ea6ded27e701748982a5b6f24ffeead10e", + "evidenceHashes": { + "evidence-manifest.json": "b1f020afba6828a685735014a221db1ef141b9958f4e42b075dfa14f2a574d5f", + "html-report/data/1c84a0af56116198fe02730dc6e07c03e7708a91.json": "887c612dedbc4319e8ddf443d238b9ea6ded27e701748982a5b6f24ffeead10e", + "playwright-output/runner-everyday-workflows--fdae5-the-browser-and-public-APIs/attachments/runner-e2e-result-3c37bc94f4716823d58a0e836c85db647f971bc7.json": "887c612dedbc4319e8ddf443d238b9ea6ded27e701748982a5b6f24ffeead10e", + "result.json": "887c612dedbc4319e8ddf443d238b9ea6ded27e701748982a5b6f24ffeead10e", + "snapshots/api-state.json": "a09940179b96e90991043e5305e3820c0e23b227b7d77ec93edb6c9a42e664a0", + "snapshots/browser-diagnostics.json": "44550ff6356f15e46b961ed53c4c5130c16b59360a55c2b7ffd57eda5bcf1987", + "snapshots/chat-final-run-ledger.json": "5650a472f29e83dde20802f25d913d403fd5ef590d4dedabf6da3fd0aeb9c3a1", + "snapshots/everyday-workflow.json": "495455927af91cd9686bf7441aaf078b895bdf5dbf4b0f392bcd0da930a5862b", + "snapshots/fixtures.json": "a566dedac5d1b16487d0e4d064905797a2e65a19a0c068c5f3aece0130c88f39" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "source": { + "sha": "eb01d073c35134bf0d373947b55a79f9e443f8ee", + "ref": "refs/heads/codex/native-procedure-guidance", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325777217" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 2, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "33995933e0ba31d829c5e52a5a5169aa1bf5bd3531a33e2762c5e2f021b65db3", + "evidenceHashes": { + "evidence-manifest.json": "c253edc0ea3e1a215a04bdddc07b420b90ed75ae9581b299a93653939ac48149", + "html-report/data/34410e200327d4861b2fa8863ac1446eb3daaf4e.json": "33995933e0ba31d829c5e52a5a5169aa1bf5bd3531a33e2762c5e2f021b65db3", + "playwright-output/runner-everyday-workflows--9a7d2-the-browser-and-public-APIs/attachments/runner-e2e-result-a19b320e711940bd3128319ae068af3a6e41fa43.json": "33995933e0ba31d829c5e52a5a5169aa1bf5bd3531a33e2762c5e2f021b65db3", + "result.json": "33995933e0ba31d829c5e52a5a5169aa1bf5bd3531a33e2762c5e2f021b65db3", + "snapshots/api-state.json": "150a6ecc15a5a64abfd8742289301eca792b406cf51dfc95df22cd66531d09bb", + "snapshots/browser-diagnostics.json": "37fe5436927fd28b49859eca1751a2ff4b6f8683ec96274273724dd03b2a9f30", + "snapshots/chat-final-run-ledger.json": "5397a9497d49879c09dab2c9b0a27d108792a8f36c6643593f848a259868829e", + "snapshots/everyday-workflow.json": "f0dbaaa800c3c3808e62370a69bb6b8dca1d43bd51b187617a2242a12e18a2d3", + "snapshots/fixtures.json": "91da96a65a7196b1a2b3e3d359e164063e6c5ef95096641a8789f2944fff825e" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "source": { + "sha": "eb01d073c35134bf0d373947b55a79f9e443f8ee", + "ref": "refs/heads/codex/native-procedure-guidance", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325777217" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 3, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "833df00d30db77e3aad32e091ed73e8161ad38ff23d98eff7f0bc4f2cc8fa8f0", + "evidenceHashes": { + "evidence-manifest.json": "446cae83c521fe16af543ddac6b4816b8aa11fb24aa9462442b57fc8a16ad9fa", + "html-report/data/1adabfb6931e1bbba0e3717037ae9a4cfa8cd37e.json": "833df00d30db77e3aad32e091ed73e8161ad38ff23d98eff7f0bc4f2cc8fa8f0", + "playwright-output/runner-everyday-workflows--a6fcb-the-browser-and-public-APIs/attachments/runner-e2e-result-8d7203080e1991ce5b786549e01e4580160cfcee.json": "833df00d30db77e3aad32e091ed73e8161ad38ff23d98eff7f0bc4f2cc8fa8f0", + "result.json": "833df00d30db77e3aad32e091ed73e8161ad38ff23d98eff7f0bc4f2cc8fa8f0", + "snapshots/api-state.json": "8d02b400e815af4adc73f9295a013841b5b24744ab17aace60b85240a08ecbd3", + "snapshots/browser-diagnostics.json": "c5ad3d5ac16cb2d57635abf7dffb7ee9db66b3ed9af02fa93e53fd015b3da435", + "snapshots/chat-final-run-ledger.json": "855e8c256db5cfc1bfd28a79325e7b14d205a1a7096d0fe2d95b16b3d6b3bddc", + "snapshots/everyday-workflow.json": "a86aafa04eba8193a6acd8ed927568b7fd82d2ffc69894e43c67361de3e60d33", + "snapshots/fixtures.json": "87a46bfe1470d96dbba3c61a02f82c389e7fa765c0631ae3af148e9bd9ea5bf7" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "source": { + "sha": "eb01d073c35134bf0d373947b55a79f9e443f8ee", + "ref": "refs/heads/codex/native-procedure-guidance", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325777217" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 6, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "94c594173473b2b40e6f8635d97f7e70e71e856cf99a84400fcdf3185782b81a", + "evidenceHashes": { + "evidence-manifest.json": "63babab4e851eedd8dc00474d0815c507d60343e121e770ce0f05c4aefa740dd", + "html-report/data/f84e6632cdf019513a84a0b08672da740648a5e7.json": "94c594173473b2b40e6f8635d97f7e70e71e856cf99a84400fcdf3185782b81a", + "playwright-output/runner-everyday-workflows--d934c-the-browser-and-public-APIs/attachments/runner-e2e-result-f768e9d7e6ca953c7c34ef967667dbac637d037e.json": "94c594173473b2b40e6f8635d97f7e70e71e856cf99a84400fcdf3185782b81a", + "result.json": "94c594173473b2b40e6f8635d97f7e70e71e856cf99a84400fcdf3185782b81a", + "snapshots/api-state.json": "d6b685e3fedec4993160abbb9436dfbf3f819c82f78b2acf912d9fea24150fdb", + "snapshots/browser-diagnostics.json": "21d2288d0a3b6db142c294c1a11657309147493d5295ed4f00a74c76eef34660", + "snapshots/chat-final-run-ledger.json": "667c2d841eb10e43176fe54e9344619bdebe15c7404d85f921be009e5a028984", + "snapshots/everyday-workflow.json": "5e5375d89bad4c311b4eecb0551c12d3aaad05337ab32b0dd3807c2b76fd3997", + "snapshots/fixtures.json": "9cc5dfc3ae89c49634c051c5f33be0ea1a8acc4932d2877225eac25021b1bdf8" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "source": { + "sha": "eb01d073c35134bf0d373947b55a79f9e443f8ee", + "ref": "refs/heads/codex/native-procedure-guidance", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325777217" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 3, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "41f0ad5604813feb32fa37fb6f57a4f413ef1a57dbeba2619449cfbc8a50871f", + "evidenceHashes": { + "evidence-manifest.json": "c3f408e84ab9a36f7abe3a58dd3b7df514f4c5f48ceed3efcf42caa36fec39c7", + "html-report/data/a829fecb87471d91b31c772c54d90f475156f3e8.json": "41f0ad5604813feb32fa37fb6f57a4f413ef1a57dbeba2619449cfbc8a50871f", + "playwright-output/runner-everyday-workflows--616ad-the-browser-and-public-APIs/attachments/runner-e2e-result-f10cd915174a4203d526eb0a386078715e179a96.json": "41f0ad5604813feb32fa37fb6f57a4f413ef1a57dbeba2619449cfbc8a50871f", + "result.json": "41f0ad5604813feb32fa37fb6f57a4f413ef1a57dbeba2619449cfbc8a50871f", + "snapshots/api-state.json": "9ef26183b198e896e7aa009caae51a43485c8eec8882e6b87fc0bb01eb86d38e", + "snapshots/browser-diagnostics.json": "4cd36a195c3a38b669c2c2be7c3724fceabf43229b27178664dee30755e97cdb", + "snapshots/chat-final-run-ledger.json": "10634af18a8d28e77c905e5688ea7289bd846a26d2dd9ab1b2a123830aff476a", + "snapshots/everyday-workflow.json": "21b9add74a177fb9433bcfeec3c41fabded5449713da1a1eeff188f973a54808", + "snapshots/fixtures.json": "1a3dedfabfc9e1114b14eedaf4725d8a3b4f81d5945cfea1bd61e34f49fdfcf1" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "source": { + "sha": "eb01d073c35134bf0d373947b55a79f9e443f8ee", + "ref": "refs/heads/codex/native-procedure-guidance", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37325777217" + }, + "suiteDefinitionHash": "c0b25d634e19cb18f647058f357a83cb5a14845b6fada1764cf21258d00abf54", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "permanent_infrastructure", + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 0, + "resultLedgerRunSetEqual": true, + "originalChecks": [], + "resultSha256": "73d68db5221120745f90c411d265faa13f3b99bd4e22735e4ceab320ec2f8c6a", + "evidenceHashes": { + "evidence-manifest.json": "2607859e543f14494e344ceca6896cda6ccb7810c1fcc8c62d89009add2652bd", + "html-report/data/f2c1f121afe0207d99ecf59a23882f023b2296e0.json": "73d68db5221120745f90c411d265faa13f3b99bd4e22735e4ceab320ec2f8c6a", + "playwright-output/runner-everyday-workflows--90607-the-browser-and-public-APIs/attachments/runner-e2e-result-ae68640cfa3c5759281518d751c2991e889aaa73.json": "73d68db5221120745f90c411d265faa13f3b99bd4e22735e4ceab320ec2f8c6a", + "result.json": "73d68db5221120745f90c411d265faa13f3b99bd4e22735e4ceab320ec2f8c6a", + "snapshots/browser-diagnostics.json": "94d2792cae9e2a1628779f7349538e0ce00418a6e448c54fcbba978b55a1b8ed" + } + } + ], + "evidenceHashEntryCount": 100 + }, + "corrected": { + "totals": { + "baseline": { + "pass": 5, + "fail": 1, + "retainedStartedRuns": 27 + }, + "candidate": { + "pass": 3, + "fail": 3, + "retainedStartedRuns": 33 + } + }, + "pairs": [ + { + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "baseline": "passed", + "candidate": "passed" + }, + { + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "baseline": "passed", + "candidate": "passed" + }, + { + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "baseline": "passed", + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "baseline": "passed", + "candidate": "passed" + }, + { + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "baseline": "failed", + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "baseline": "passed", + "candidate": "failed" + } + ], + "cells": [ + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "source": { + "sha": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "ref": "refs/heads/codex/native-procedure-baseline-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331642168" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "689e5154888c5ea3697b7577587c2aa8630dddb2100cb1b7786bc995c13c8097", + "evidenceHashes": { + "evidence-manifest.json": "07f77f464ba2e896decfb2db87db62b4c19ac37303f4b9e0cb3b89afcf707af6", + "html-report/data/580f383fcce38922446d0dd2f23a50283699a0b1.json": "689e5154888c5ea3697b7577587c2aa8630dddb2100cb1b7786bc995c13c8097", + "playwright-output/runner-everyday-workflows--fdae5-the-browser-and-public-APIs/attachments/runner-e2e-result-1a68e97dada87da3688de2deb3e9c9f12df25ad9.json": "689e5154888c5ea3697b7577587c2aa8630dddb2100cb1b7786bc995c13c8097", + "result.json": "689e5154888c5ea3697b7577587c2aa8630dddb2100cb1b7786bc995c13c8097", + "snapshots/api-state.json": "984d171f9e501c3b700e7c60e0abbed70f4abe3012dcfb33ad35ad0fa823c486", + "snapshots/browser-diagnostics.json": "26865f180e7e1f9d174b83fabcf13ed8912731f7533e257ad85bb01fb84d9d1e", + "snapshots/chat-final-run-ledger.json": "0740de3c3c6f91c79b7fe8b88ee75176d24d5fd07e12597f26d5504c682e1ee4", + "snapshots/everyday-workflow.json": "d8ee30954306a973cf4e1c412e648b87429961b1ba485e5fe76735d22e8b9b9e", + "snapshots/fixtures.json": "34b5db8164c2149e57b3bad0c249c7c738b114ccf11897533c85644d22700159" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "source": { + "sha": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "ref": "refs/heads/codex/native-procedure-baseline-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331642168" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 6, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "08ad50bd253f9551f531bd4ce0a15ee200bd4c544a171a6540b8f39bd526f3cd", + "evidenceHashes": { + "evidence-manifest.json": "ad4ae5c149d9a288f09510a9f40d751a0e16056a15c0e3e5fc22673f4fe89b5a", + "html-report/data/f1b2ae22fca1509f0c62f23f8d6df4ac70ed88ea.json": "08ad50bd253f9551f531bd4ce0a15ee200bd4c544a171a6540b8f39bd526f3cd", + "playwright-output/runner-everyday-workflows--9a7d2-the-browser-and-public-APIs/attachments/runner-e2e-result-345f227fa1bde20467d1efcdba85e9a90b9d2e06.json": "08ad50bd253f9551f531bd4ce0a15ee200bd4c544a171a6540b8f39bd526f3cd", + "result.json": "08ad50bd253f9551f531bd4ce0a15ee200bd4c544a171a6540b8f39bd526f3cd", + "snapshots/api-state.json": "748fcd21ef5cd800eec08adc76f8713475adb3c54605eec179001c79272347a8", + "snapshots/browser-diagnostics.json": "9cac545625ca2970a8b3dba8c8d3ab50473dfba21fd057d7d4d6fe59724f0a8e", + "snapshots/chat-final-run-ledger.json": "b8e78dbd52dff17b77f1dfcf18bf469a691e9bf30cff3dcc91e6d3d761334502", + "snapshots/everyday-workflow.json": "10552e78b7e70a050e43003419a37f5c215665e890cc928e188806c30e464589", + "snapshots/fixtures.json": "667ce63d133703653093d3217dd537ac50d5ada0bd3c6d0e15e4cc3607305314" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "source": { + "sha": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "ref": "refs/heads/codex/native-procedure-baseline-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331642168" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "d0535b600235580ada1b0227b84ae5f621e1a3932a7a94e3285bfebfe7dd716a", + "evidenceHashes": { + "evidence-manifest.json": "8317f0026d88709a1fcd716b9373786bee39b9f61902a3c1cc48c4ff64b1b59e", + "html-report/data/0192f8929945cb662a6241d65fdd9e43994b876e.json": "d0535b600235580ada1b0227b84ae5f621e1a3932a7a94e3285bfebfe7dd716a", + "playwright-output/runner-everyday-workflows--a6fcb-the-browser-and-public-APIs/attachments/runner-e2e-result-be735800e00ca4152df495a405c0f0024010d799.json": "d0535b600235580ada1b0227b84ae5f621e1a3932a7a94e3285bfebfe7dd716a", + "result.json": "d0535b600235580ada1b0227b84ae5f621e1a3932a7a94e3285bfebfe7dd716a", + "snapshots/api-state.json": "4239640e901c755f81484e852b092a4ee99caf7421ca4ed364b418a59f6def51", + "snapshots/browser-diagnostics.json": "5b90904dacfbd32aecbe9cbe8ad59962c09242ee61ddfcb5feac326030b2e2d4", + "snapshots/chat-final-run-ledger.json": "8bb0946ed361a28a841c7d209ca18619436d087ec7f88fa7e1c1b1f326e35aae", + "snapshots/everyday-workflow.json": "668dfef6034aed2079cb9e8023fef8198074dbd08ebe3521b6c70b77de783ca6", + "snapshots/fixtures.json": "59d95bfccd4f60d1ff990483fba59210d0a8eaf63a8f0197c328e8e6c7bfb654" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "source": { + "sha": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "ref": "refs/heads/codex/native-procedure-baseline-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331642168" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 6, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "a23a9e92d7ff47999ee6483f5a927f41c725c0cce69d91c150cf4e9ab2cbe5eb", + "evidenceHashes": { + "evidence-manifest.json": "0a8bef60505c30179b11c36c764ba991e3e2a56be3b61a6fa7c3848786a5ebe9", + "html-report/data/71d82d27caa117b7d6102f8fe8ff41d42c222c7f.json": "a23a9e92d7ff47999ee6483f5a927f41c725c0cce69d91c150cf4e9ab2cbe5eb", + "playwright-output/runner-everyday-workflows--d934c-the-browser-and-public-APIs/attachments/runner-e2e-result-0c090b31617607484975dfd0adaed62b9f8ea755.json": "a23a9e92d7ff47999ee6483f5a927f41c725c0cce69d91c150cf4e9ab2cbe5eb", + "result.json": "a23a9e92d7ff47999ee6483f5a927f41c725c0cce69d91c150cf4e9ab2cbe5eb", + "snapshots/api-state.json": "9df20d0d9a4a319a87271c66eb38ea57dbc649464574c10e7cf62734378a29a6", + "snapshots/browser-diagnostics.json": "467a514913b0d01e876353bf5a76813fc7fb717dd8cbc567ee13eb17fbca8f72", + "snapshots/chat-final-run-ledger.json": "215358fb7caa14e656a15464b693940b9c202b0aefee817fe5375eb7912b0df2", + "snapshots/everyday-workflow.json": "b70f39ea8147669a795deffe59ce6fcc024f9d39a0e7a9ef8a809f8a8636f4e8", + "snapshots/fixtures.json": "7d104f51b41eee886b810b583b90a242652aba9e0596d9dddab5e2b4d764da42" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "source": { + "sha": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "ref": "refs/heads/codex/native-procedure-baseline-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331642168" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 3, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": false + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "a51a5345df8812be3f754d3f0e5df30a1f826150a6c7778f0dc0726d642d7f4b", + "evidenceHashes": { + "evidence-manifest.json": "71573393fe99cd451ce4fdc90010c72572b769b2b0bbdfa11bc3f28bd8655d2a", + "html-report/data/0c3bfda38c3dc2d51f9f55d711970b533bf283fa.json": "a51a5345df8812be3f754d3f0e5df30a1f826150a6c7778f0dc0726d642d7f4b", + "playwright-output/runner-everyday-workflows--616ad-the-browser-and-public-APIs/attachments/runner-e2e-result-b4cc93b23776a0ae821207b52c43a3f464f99977.json": "a51a5345df8812be3f754d3f0e5df30a1f826150a6c7778f0dc0726d642d7f4b", + "result.json": "a51a5345df8812be3f754d3f0e5df30a1f826150a6c7778f0dc0726d642d7f4b", + "snapshots/api-state.json": "bab60c326f4ad0971748cf30f2de276eb7f69c985a38733d923c78b3fec659aa", + "snapshots/browser-diagnostics.json": "d164e04291112ffe4efc8d248d61608e944d7b1ae6d3fb14c10c5a655eb06790", + "snapshots/chat-final-run-ledger.json": "de0b3b353dd606474591946c3bdb42aac2084a0de01ec83b5faae0004c250003", + "snapshots/everyday-workflow.json": "6505cc96cb7ff2f371c73eeef2318e105fb8b8aaa14f7b3252ef8a432f048345", + "snapshots/fixtures.json": "46fdccbefe0f5b14de8d26798e04bf042c1b9ea9f5155cde523d292fa32c4e85" + } + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "source": { + "sha": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "ref": "refs/heads/codex/native-procedure-baseline-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331642168" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "741f7e0deee7f7b7b8c8498e998100ad86e3670340a116bba7e3d7f19b139ff4", + "evidenceHashes": { + "evidence-manifest.json": "70ca2a5525ce594a58d9e88ceae252e03918fe9c5af5f478bd345e7623085aff", + "html-report/data/b4ef23d5b811da0b4c40ac32d7109b15b2ed3eea.json": "741f7e0deee7f7b7b8c8498e998100ad86e3670340a116bba7e3d7f19b139ff4", + "playwright-output/runner-everyday-workflows--90607-the-browser-and-public-APIs/attachments/runner-e2e-result-a3041877169260aa9a9b8c577933a9aaffdc90d6.json": "741f7e0deee7f7b7b8c8498e998100ad86e3670340a116bba7e3d7f19b139ff4", + "result.json": "741f7e0deee7f7b7b8c8498e998100ad86e3670340a116bba7e3d7f19b139ff4", + "snapshots/api-state.json": "50a969cb558df7ca61bad975ace655421f6afc9e894a344834b79a8e760ce5e4", + "snapshots/browser-diagnostics.json": "f5d63620b23461a6ea972734fe519a3e1e08cd6d4301e4520255b552d20862a8", + "snapshots/chat-final-run-ledger.json": "2688ded68ad348d62bfe745b4a26c55f1108a6c8c2bcc35e5cb0fcd421130e4b", + "snapshots/everyday-workflow.json": "ca3bf543901083b817c9e08e336a54a28a61d7ca4af5a03fbed5579ce372adce", + "snapshots/fixtures.json": "b04cca5e54ad156e911331ae8e2d6af3b8a3e31feedbd3ecab464414ca7d9a75" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "source": { + "sha": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "ref": "refs/heads/codex/native-procedure-candidate-v2-recovery", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37334061104" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "1d0a1e688b1f003491c27497edeeff963a5459ed99acfd32ada06fb9decd93b1", + "evidenceHashes": { + "evidence-manifest.json": "977cf3c0ac2c93190b71f6bb766ce44107e78f1d9b726db79d7e4c7e1efe7ed0", + "html-report/data/8e6146fc86993655e3a364a37eff7cf125570318.json": "1d0a1e688b1f003491c27497edeeff963a5459ed99acfd32ada06fb9decd93b1", + "playwright-output/runner-everyday-workflows--fdae5-the-browser-and-public-APIs/attachments/runner-e2e-result-c75e8d1cd375ccf9308dc1626ec8d9959eae7c13.json": "1d0a1e688b1f003491c27497edeeff963a5459ed99acfd32ada06fb9decd93b1", + "result.json": "1d0a1e688b1f003491c27497edeeff963a5459ed99acfd32ada06fb9decd93b1", + "snapshots/api-state.json": "e5ce82339b356c8125b32949371049d6b1ba85305ecff321e2b30ee022029b18", + "snapshots/browser-diagnostics.json": "72af20421dd49d3362dc6b5688cd96d960bf41b34ed58dd8833fea2ca9d58386", + "snapshots/chat-final-run-ledger.json": "b3d15cabdbe0819da45f08621fba6df2e29f6d2655ea3f6a51a55daed14b15ef", + "snapshots/everyday-workflow.json": "d2074d8f9c302d383ea3e25b81a5d844173725b70ba7b1fecca3bfa8f75687e6", + "snapshots/fixtures.json": "7ad274c380095f860f5bf95a31dd2e9bd772b64c7230f30288aa171560c43f11" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "source": { + "sha": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "ref": "refs/heads/codex/native-procedure-candidate-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331580879" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "claude-sonnet-5", + "cleanup": "passed", + "startedRunCount": 6, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "6f4ef2c3760227fc504de1030e30788afe54168b91db72f5088c5eb634295a47", + "evidenceHashes": { + "evidence-manifest.json": "7c04512304d32ac249741243f3081f87a6c9d3530b29f2fc31af2619ae79ba1a", + "html-report/data/e789dbe6a578c12c42b3ab46cc3c657158c41129.json": "6f4ef2c3760227fc504de1030e30788afe54168b91db72f5088c5eb634295a47", + "playwright-output/runner-everyday-workflows--9a7d2-the-browser-and-public-APIs/attachments/runner-e2e-result-02e9625499f3a6da62e679ca785ff0a376465eef.json": "6f4ef2c3760227fc504de1030e30788afe54168b91db72f5088c5eb634295a47", + "result.json": "6f4ef2c3760227fc504de1030e30788afe54168b91db72f5088c5eb634295a47", + "snapshots/api-state.json": "83a3295d9563dcbbaccbdcf5278a277af41cd4a24b723af1c28f5f6795e895d7", + "snapshots/browser-diagnostics.json": "85188804dee7b70a86a96808b1625e54b421663f3a47f8662845c44e8f675bea", + "snapshots/chat-final-run-ledger.json": "042fb89f01a7b790147efc55d0879c8fc39090518cf6bf9c3ed4a3772acf6ac6", + "snapshots/everyday-workflow.json": "063028cb1f58cf86b2c13ac2ae17a5d54f0780c5f1883a489f00904ce2e3696c", + "snapshots/fixtures.json": "366e91cf2365f156fa370ac1fb816f920ebceba5d6978b5fd6e773c2717a0601" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "source": { + "sha": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "ref": "refs/heads/codex/native-procedure-candidate-v2-recovery", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37334061104" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "2b4c8af68619aca7ffc2ac40f2569b1595ba21bdabf2481c6073b7186d220adf", + "evidenceHashes": { + "evidence-manifest.json": "008abe42e978f9224955e623d237b53a0dfaefafec767635aef09dbf20874c77", + "html-report/data/247c57b22f37607373563c94a9554ba3c8f6a4f0.json": "2b4c8af68619aca7ffc2ac40f2569b1595ba21bdabf2481c6073b7186d220adf", + "playwright-output/runner-everyday-workflows--a6fcb-the-browser-and-public-APIs/attachments/runner-e2e-result-146fc7aa673f1bd55ea9e51ad19afdbd163cc99b.json": "2b4c8af68619aca7ffc2ac40f2569b1595ba21bdabf2481c6073b7186d220adf", + "result.json": "2b4c8af68619aca7ffc2ac40f2569b1595ba21bdabf2481c6073b7186d220adf", + "snapshots/api-state.json": "d4d98e4fa703535b9d7a02eab49f3ac69e32836c1a556285327e0c51a88ea6db", + "snapshots/browser-diagnostics.json": "791cb4bc513bbbfbd7342f09d827520a8aed93bb14b1749e911dba31be0f0a94", + "snapshots/chat-final-run-ledger.json": "3c97659721c812cc3ab632f532982c8d1889133cad37ca324aece54eb0d78152", + "snapshots/everyday-workflow.json": "5e9907a320ace5ddbdc695a846ea711bd887aaddc793e56c9e20af9a6ec386b4", + "snapshots/fixtures.json": "96d4acbe3a2da0c05d8f5d41447302e805f76e6bf66996d8d8ab26f6db99c4e5" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "source": { + "sha": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "ref": "refs/heads/codex/native-procedure-candidate-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331580879" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "configuredModel": "gpt-5.6-sol", + "cleanup": "passed", + "startedRunCount": 6, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ], + "resultSha256": "266e8d35f8d4ef3a570ea6940c64b6a9892eb6ab1d14324625da2337b4e7b99c", + "evidenceHashes": { + "evidence-manifest.json": "4b60c8a396e229b271fb377c44148e7b922286093824c48ca86782ed19490c0b", + "html-report/data/7c7b49ba9c9fcdcbde63035d3af25ba0bdade045.json": "266e8d35f8d4ef3a570ea6940c64b6a9892eb6ab1d14324625da2337b4e7b99c", + "playwright-output/runner-everyday-workflows--d934c-the-browser-and-public-APIs/attachments/runner-e2e-result-2d7949d3b25761e104f66bbe3f3f56425e20a3ab.json": "266e8d35f8d4ef3a570ea6940c64b6a9892eb6ab1d14324625da2337b4e7b99c", + "result.json": "266e8d35f8d4ef3a570ea6940c64b6a9892eb6ab1d14324625da2337b4e7b99c", + "snapshots/api-state.json": "87cc674d10f53be4beecaf60e8c50813e76a2c774bf5c1e9173733be23979615", + "snapshots/browser-diagnostics.json": "139fedd6a92bed9ff5ae218d8b012da508b97cf4494eaa102472b59a230d952d", + "snapshots/chat-final-run-ledger.json": "692d37db951fe5b2852a1748f72392cb14977d07cfee009961535e8170830712", + "snapshots/everyday-workflow.json": "68fbc3745b7aa60145479bd4cbd8646e3faac56b96bd501c438491d04b881964", + "snapshots/fixtures.json": "396dcdb89dd9656c748c5a5a1e34cf774bcb2e178124bd46e8d051dba963fd10" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "source": { + "sha": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "ref": "refs/heads/codex/native-procedure-candidate-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331580879" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 4, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": false + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "17c1145948d1f444211d9b319f36d7e9562bb4c99e804024cfbe0bf6f780354b", + "evidenceHashes": { + "evidence-manifest.json": "4911bab65a4f2e24573870946b894d3f5ca4030819ae023881644f7c35e443fc", + "html-report/data/ef9ff8e5b5a82f998ba0c465bdce4ec2c19294d2.json": "17c1145948d1f444211d9b319f36d7e9562bb4c99e804024cfbe0bf6f780354b", + "playwright-output/runner-everyday-workflows--616ad-the-browser-and-public-APIs/attachments/runner-e2e-result-1f3520dbb09fae78c011e89c25533003bd91a93a.json": "17c1145948d1f444211d9b319f36d7e9562bb4c99e804024cfbe0bf6f780354b", + "result.json": "17c1145948d1f444211d9b319f36d7e9562bb4c99e804024cfbe0bf6f780354b", + "snapshots/api-state.json": "7b0631c60b459355779aa75d83554df845192bd19ee748c5ef8acda3a15a97ff", + "snapshots/browser-diagnostics.json": "cf9da33e8f81b874be572e7bce6103fc71bb0add1ecc9a48d1805379bdb1430d", + "snapshots/chat-final-run-ledger.json": "3f87d37dd18a1311f34d75ab3e07f1e7cf674d3b1180ca219e241ae8decea5dc", + "snapshots/everyday-workflow.json": "e67b30bbf111cd16281f2b6bed2a99343a62924493149b461716dce96c578efb", + "snapshots/fixtures.json": "f1b5d61b9cb04ca16642614b53a5f676a527466d31c835ecdd417ea9bda60aed" + } + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "source": { + "sha": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "ref": "refs/heads/codex/native-procedure-candidate-v2", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37331580879" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "configuredModel": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "startedRunCount": 9, + "resultLedgerRunSetEqual": true, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ], + "resultSha256": "b001ce9d1741adc087dab22643a3f65e5dabc88bcbde432dcebe9c1f793f6a8e", + "evidenceHashes": { + "evidence-manifest.json": "1426f8127ca46828ef5320c3bf5ab2134d60172361bb1c43a2b415fe61c766ff", + "html-report/data/9777bc5b0460c2689ba53533aad6b65d6879e30b.json": "b001ce9d1741adc087dab22643a3f65e5dabc88bcbde432dcebe9c1f793f6a8e", + "playwright-output/runner-everyday-workflows--90607-the-browser-and-public-APIs/attachments/runner-e2e-result-e406f7ff0925c2cfeaed8101a63cb789dac7ec22.json": "b001ce9d1741adc087dab22643a3f65e5dabc88bcbde432dcebe9c1f793f6a8e", + "result.json": "b001ce9d1741adc087dab22643a3f65e5dabc88bcbde432dcebe9c1f793f6a8e", + "snapshots/api-state.json": "fa9852b1155d3f3753bfe711e30e8fff6847cf30d82f58089db6592365719c59", + "snapshots/browser-diagnostics.json": "0ae03cb7af8cb2e405f6a1d78f3fb86fe9193de6bc07c5b7a69f44f4956c664d", + "snapshots/chat-final-run-ledger.json": "ad8bed7e06561967f88ea2b0beb12728257db8b579f3547df55729225fbb2889", + "snapshots/everyday-workflow.json": "b5b0b3e3f07eaa9be98b1c335585fa1ba80bdda1acab3f95b81e512b376f375f", + "snapshots/fixtures.json": "1b310f2e030acbf3997ffd87f648f0106e7a06e475671c16a8164f7b1400cb4c" + } + } + ], + "evidenceHashEntryCount": 108 + }, + "correctedPairCounts": { + "newFailures": 2, + "newPasses": 0, + "unchangedPasses": 3, + "unchangedFailures": 1, + "pending": 0 + }, + "sourceModelSuiteAttemptLeadBudgetCleanupAndRunSetAssertions": true, + "interruptions": { + "campaign": 37331580879, + "source": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "recoveryPolicy": "One bounded replacement cell each on identical source, distinct workflow target; preserve original interruption, unknown run/cost totals, and both attempt identities. Do not repeat successful baselines.", + "recoveryTrustedWorkflowSha": "a65ca0950834a85bb93bcc4b4042ecacdebfef53", + "workflowByteIdenticalToOriginal": true, + "workflowFileSha256": "eb570267e66043c1196338bb386b07ab8e8e8306ba04b6223f13b73b04fdc8f4", + "recoveryCampaign": 37334061104, + "cells": [ + { + "profile": "claude", + "jobId": 111838521502, + "logSha256": "ad84bbdc76f9a48a16886f676335ec5fe9ba2190c47f912f34619d010c36bc1a", + "originalWorkflowConclusion": "failure", + "behavioralGrade": null, + "finalLedger": null, + "cleanupReceipt": null, + "actualRunCount": null, + "actualCharges": null, + "observedPartialRunRecordCount": 4 + }, + { + "profile": "codex", + "jobId": 111838521694, + "logSha256": "3f7b36ae28d0cb254a6199f548ce6c347e91a39dcc456eb7e8078290211008ea", + "originalWorkflowConclusion": "failure", + "behavioralGrade": null, + "finalLedger": null, + "cleanupReceipt": null, + "actualRunCount": null, + "actualCharges": null, + "observedPartialRunRecordCount": 4 + } + ] + }, + "accounting": { + "originalRetainedStartedRuns": 40, + "correctedCompletedOrRecoveredRetainedStartedRuns": 60, + "allRetainedStartedRuns": 100, + "additionalInterruptedActualRuns": null, + "actualCharges": null, + "completeTotalRunCount": false, + "infrastructureRecoveryCellCount": 2, + "limit": "Completed/recovered cell ledgers match their result run sets. Two earlier interrupted candidate jobs lack final result/ledger/cleanup receipts; their actual runs/charges are additional and incompletely known. No claim of total known cost or first-attempt success." + }, + "measurement": { + "corrected-baseline": { + "schema": "paperclip.native-procedure-measurement.v2", + "sourceSha": "aecfa3b6a5cab110a9b926e93705311bbcf4db47", + "sourceDirty": false, + "measuredInputsDirty": false, + "sourceHashes": { + "server/src/onboarding-assets/default/AGENTS.md": "c6318bf7e425cc0dd7acd4ff2905dab5c46ffddedef979afdf06720c82f6e95a", + "packages/paperclip-runner/src/contracts/runtime-context.ts": "a17e30a69f3cb38fba00c37e1a06753d723651a004e6050c9e77ae202a61264b", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts": "3b786405fa48e6ba9cf63789c03bbbb370fc2962a4a5c8bd80ccca57759bf912", + "packages/paperclip-runner/src/protocol-actions/create-task.ts": "4efe71c1693dd65d8087b2fdbaf5831b397130d036afb837ece20d4791631805", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts": "53c65776ce4984f177d261c1e53957a97e1a236ce58e731119cb8ee26f141da2", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts": "170a1e97f6402d87041b944fc45a7b3a41cf5d07dc158b3cc494ac38208dbd6d", + "packages/paperclip-runner/generated/semantic-action-catalog.json": "6b398dbf14c2cb239333321387e3978b0ef21b2c1c6a12882105d077eb0214a0", + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json": "c31cb3b21aa81c46847e6827865bc17d100a2f135822b7580bec7fc1893454a2", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json": "b4e488f922dd328a3967b2560951c2377f4c1411fb2c4c9eef7a44195a8f7514", + "packages/paperclip-runner/protocol/manifest.json": "9cebc584dce7b42ed6800bef9b3e1c83880a67798a7bdceb9d9482b246f66a18", + "packages/paperclip-runner/src/backends/codex-native-backend.ts": "affecc515a623e0dfeb338f553ea53ebb7d18baa3d13e4395d17b21000dbee93", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "ec1cd94c17be22d0556d4586fa6407b28e2309ed7c2437d95a32beeb0755cc4f", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "d2360cdaa63902cbf0c6a6109da452ba7e0b2a5aaa76ee5cab9c63b4bf12e584", + "server/src/services/native-runtime/native-session-resume.ts": "5b27465e1a328412770fc27c4bc637e88b62022adc6a67709bb74187eadc465b", + "server/src/__tests__/native-procedure-measurement.test.ts": "4e5725c0d86442c0214d96e6ab489547b919d45f8f237d9ec14d02437965268d" + }, + "boundary": "scripted runnerd RPC with production standard-mode authority, API tools enabled, local workspace, no assigned external apps", + "byteAccounting": "utf8Bytes counts the unmodified JSON serialization of each extracted component/projection; normalizedComparison substitutes only the fixture workspace path. Neither is the entire transport request.", + "providerCalls": 0, + "tokenCount": null, + "upstreamLoadingOrTruncation": "unverified", + "vendorStockPrompt": "unavailable", + "receipts": [ + { + "provider": "codex", + "phase": "start", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "a336ca46d52cada364ad79d6072332acf4c3920a5f1f6d595ab5c8e1b2be6117", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48949, + "characters": 48939, + "sha256": "696a27c9d8dd92d5c22a94eca82f214786f7fb12a7a00712685990450d14fc6f", + "normalizedComparison": { + "utf8Bytes": 48781, + "characters": 48771, + "sha256": "1678ae17dcdfedd17de242b5fce9a27d287bfd9c4203611678043a99342a7754" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "resume", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "a336ca46d52cada364ad79d6072332acf4c3920a5f1f6d595ab5c8e1b2be6117", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48949, + "characters": 48939, + "sha256": "696a27c9d8dd92d5c22a94eca82f214786f7fb12a7a00712685990450d14fc6f", + "normalizedComparison": { + "utf8Bytes": 48781, + "characters": 48771, + "sha256": "1678ae17dcdfedd17de242b5fce9a27d287bfd9c4203611678043a99342a7754" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47087, + "characters": 47077, + "sha256": "bb7057ad43526b62652d752d504b41b9113254ebae72d461859ea12080905ca2", + "normalizedComparison": { + "utf8Bytes": 47003, + "characters": 46993, + "sha256": "9888af76927545d1e986c9f810975e6a1c4c3e395d0b25550e74d7f75146397c" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "start", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "a336ca46d52cada364ad79d6072332acf4c3920a5f1f6d595ab5c8e1b2be6117", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48949, + "characters": 48939, + "sha256": "696a27c9d8dd92d5c22a94eca82f214786f7fb12a7a00712685990450d14fc6f", + "normalizedComparison": { + "utf8Bytes": 48781, + "characters": 48771, + "sha256": "1678ae17dcdfedd17de242b5fce9a27d287bfd9c4203611678043a99342a7754" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "resume", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "a336ca46d52cada364ad79d6072332acf4c3920a5f1f6d595ab5c8e1b2be6117", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48949, + "characters": 48939, + "sha256": "696a27c9d8dd92d5c22a94eca82f214786f7fb12a7a00712685990450d14fc6f", + "normalizedComparison": { + "utf8Bytes": 48781, + "characters": 48771, + "sha256": "1678ae17dcdfedd17de242b5fce9a27d287bfd9c4203611678043a99342a7754" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47087, + "characters": 47077, + "sha256": "bb7057ad43526b62652d752d504b41b9113254ebae72d461859ea12080905ca2", + "normalizedComparison": { + "utf8Bytes": 47003, + "characters": 46993, + "sha256": "9888af76927545d1e986c9f810975e6a1c4c3e395d0b25550e74d7f75146397c" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "start", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "a336ca46d52cada364ad79d6072332acf4c3920a5f1f6d595ab5c8e1b2be6117", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48949, + "characters": 48939, + "sha256": "696a27c9d8dd92d5c22a94eca82f214786f7fb12a7a00712685990450d14fc6f", + "normalizedComparison": { + "utf8Bytes": 48781, + "characters": 48771, + "sha256": "1678ae17dcdfedd17de242b5fce9a27d287bfd9c4203611678043a99342a7754" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "resume", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "a336ca46d52cada364ad79d6072332acf4c3920a5f1f6d595ab5c8e1b2be6117", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48949, + "characters": 48939, + "sha256": "696a27c9d8dd92d5c22a94eca82f214786f7fb12a7a00712685990450d14fc6f", + "normalizedComparison": { + "utf8Bytes": 48781, + "characters": 48771, + "sha256": "1678ae17dcdfedd17de242b5fce9a27d287bfd9c4203611678043a99342a7754" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "f89d1d862203db2f03023c0144139925f6784c0fbeae1ac0c2957962c4d60cc4", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293", + "normalizedComparison": { + "utf8Bytes": 44944, + "characters": 44934, + "sha256": "c8f98376b8e7e5c7f92bce1c0d36ca2fb82fe0e2bd398a6b2750a9a25043f293" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47087, + "characters": 47077, + "sha256": "bb7057ad43526b62652d752d504b41b9113254ebae72d461859ea12080905ca2", + "normalizedComparison": { + "utf8Bytes": 47003, + "characters": 46993, + "sha256": "9888af76927545d1e986c9f810975e6a1c4c3e395d0b25550e74d7f75146397c" + } + }, + "deliveredToolCount": 39 + } + ], + "mcpReceipt": { + "boundary": "authenticated loopback MCP tools/list; no OpenCode/provider process", + "serverAnnotationsForwarded": false, + "tools": { + "utf8Bytes": 44249, + "characters": 44239, + "sha256": "07d02ed46f9055af77756edc004dd3b30ec796afbbabe21623709f810d7ba1d1", + "normalizedComparison": { + "utf8Bytes": 44249, + "characters": 44239, + "sha256": "07d02ed46f9055af77756edc004dd3b30ec796afbbabe21623709f810d7ba1d1" + } + }, + "count": 39 + }, + "originalReceiptSha256": "2158cc185b2d7c5dd6082737836a7d1c5d3e20bf4539a897e41ed4e99080aadd" + }, + "corrected-candidate": { + "schema": "paperclip.native-procedure-measurement.v2", + "sourceSha": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "sourceDirty": true, + "measuredInputsDirty": false, + "sourceHashes": { + "server/src/onboarding-assets/default/AGENTS.md": "c6318bf7e425cc0dd7acd4ff2905dab5c46ffddedef979afdf06720c82f6e95a", + "packages/paperclip-runner/src/contracts/runtime-context.ts": "3d2ee39cf7cc2210104e155213f04cec82981f76cd7c38384cdc11fa3ce4b013", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts": "ed8d270e2f3a8357bde5f2b180616d09f45218cd90bad22609d2e2e3d02d0319", + "packages/paperclip-runner/src/protocol-actions/create-task.ts": "8557d4030c97aaf99e8ef45dc608e4e0117c764eb459b221ce6739499d79b8aa", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts": "89819f6741dc14009f2daf54717f5e23d583c6f5685c30459fe8a0d672a2720b", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts": "786b597ab2199f3d943e499911fb58cfd014e78ff60923b9f44b9e2d80d73d4d", + "packages/paperclip-runner/generated/semantic-action-catalog.json": "795c5c8b98b1d7d60a2097760ffcb5a72ffa27713ca75b086ec741bbb0260f3a", + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json": "383ca779a4a4e98b9cf739ccfbfd072e470b57d8a340dbf3e86c7d502285f5ba", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json": "044d8532b9a51d5ad254618a32f2d9d39227b70110d916e1a05b8072f4ee4e22", + "packages/paperclip-runner/protocol/manifest.json": "420055c0f98b46f57096ac32bdfb6a0d53ae40ac2c3345dc1fea7cfdab2addb0", + "packages/paperclip-runner/src/backends/codex-native-backend.ts": "affecc515a623e0dfeb338f553ea53ebb7d18baa3d13e4395d17b21000dbee93", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "ec1cd94c17be22d0556d4586fa6407b28e2309ed7c2437d95a32beeb0755cc4f", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "ae5cd9c6d076cf44471b42c583f69eea9776aaad41ccbc8319d416aa88f78b44", + "server/src/services/native-runtime/native-session-resume.ts": "d9d4d7fde805c150e8278fd26f36d5e27d77c49690a0371f20e83b542f9cb2db", + "server/src/__tests__/native-procedure-measurement.test.ts": "4e5725c0d86442c0214d96e6ab489547b919d45f8f237d9ec14d02437965268d" + }, + "boundary": "scripted runnerd RPC with production standard-mode authority, API tools enabled, local workspace, no assigned external apps", + "byteAccounting": "utf8Bytes counts the unmodified JSON serialization of each extracted component/projection; normalizedComparison substitutes only the fixture workspace path. Neither is the entire transport request.", + "providerCalls": 0, + "tokenCount": null, + "upstreamLoadingOrTruncation": "unverified", + "vendorStockPrompt": "unavailable", + "receipts": [ + { + "provider": "codex", + "phase": "start", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "28e674540ba45792d6a31a1acef0a5b99fe8aa60c66df2d4aeda12dbe7bba3b5", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48489, + "characters": 48479, + "sha256": "911dace61347be9b8515ffd249ca84bfb67df487c5a1a2cecebe6c5b822f34ce", + "normalizedComparison": { + "utf8Bytes": 48321, + "characters": 48311, + "sha256": "31bc2475becb447e0f2722ad61a32938693258cca07c3b1e3f7162b8be0d8319" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "resume", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "28e674540ba45792d6a31a1acef0a5b99fe8aa60c66df2d4aeda12dbe7bba3b5", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48489, + "characters": 48479, + "sha256": "911dace61347be9b8515ffd249ca84bfb67df487c5a1a2cecebe6c5b822f34ce", + "normalizedComparison": { + "utf8Bytes": 48321, + "characters": 48311, + "sha256": "31bc2475becb447e0f2722ad61a32938693258cca07c3b1e3f7162b8be0d8319" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 46627, + "characters": 46617, + "sha256": "8e8e003648345a706d3eb28b3e0762e8accf51291101eed78be98b0e876cd445", + "normalizedComparison": { + "utf8Bytes": 46543, + "characters": 46533, + "sha256": "20b7f03d24078f192aa8935c269f63e06b50dc2a7176f417d64ca8c1ace92591" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "start", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "28e674540ba45792d6a31a1acef0a5b99fe8aa60c66df2d4aeda12dbe7bba3b5", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48489, + "characters": 48479, + "sha256": "911dace61347be9b8515ffd249ca84bfb67df487c5a1a2cecebe6c5b822f34ce", + "normalizedComparison": { + "utf8Bytes": 48321, + "characters": 48311, + "sha256": "31bc2475becb447e0f2722ad61a32938693258cca07c3b1e3f7162b8be0d8319" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "resume", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "28e674540ba45792d6a31a1acef0a5b99fe8aa60c66df2d4aeda12dbe7bba3b5", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48489, + "characters": 48479, + "sha256": "911dace61347be9b8515ffd249ca84bfb67df487c5a1a2cecebe6c5b822f34ce", + "normalizedComparison": { + "utf8Bytes": 48321, + "characters": 48311, + "sha256": "31bc2475becb447e0f2722ad61a32938693258cca07c3b1e3f7162b8be0d8319" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 46627, + "characters": 46617, + "sha256": "8e8e003648345a706d3eb28b3e0762e8accf51291101eed78be98b0e876cd445", + "normalizedComparison": { + "utf8Bytes": 46543, + "characters": 46533, + "sha256": "20b7f03d24078f192aa8935c269f63e06b50dc2a7176f417d64ca8c1ace92591" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "start", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "28e674540ba45792d6a31a1acef0a5b99fe8aa60c66df2d4aeda12dbe7bba3b5", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48489, + "characters": 48479, + "sha256": "911dace61347be9b8515ffd249ca84bfb67df487c5a1a2cecebe6c5b822f34ce", + "normalizedComparison": { + "utf8Bytes": 48321, + "characters": 48311, + "sha256": "31bc2475becb447e0f2722ad61a32938693258cca07c3b1e3f7162b8be0d8319" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "resume", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "28e674540ba45792d6a31a1acef0a5b99fe8aa60c66df2d4aeda12dbe7bba3b5", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48489, + "characters": 48479, + "sha256": "911dace61347be9b8515ffd249ca84bfb67df487c5a1a2cecebe6c5b822f34ce", + "normalizedComparison": { + "utf8Bytes": 48321, + "characters": 48311, + "sha256": "31bc2475becb447e0f2722ad61a32938693258cca07c3b1e3f7162b8be0d8319" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "a415c39912f9b7c8a15b24dc27702229e2458adcef143511d77d67bd125d854b", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c", + "normalizedComparison": { + "utf8Bytes": 45037, + "characters": 45027, + "sha256": "6830d63d19141fa7efcc7b53f0f6f7e044352d006c3e146ddc0073d502646e5c" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 46627, + "characters": 46617, + "sha256": "8e8e003648345a706d3eb28b3e0762e8accf51291101eed78be98b0e876cd445", + "normalizedComparison": { + "utf8Bytes": 46543, + "characters": 46533, + "sha256": "20b7f03d24078f192aa8935c269f63e06b50dc2a7176f417d64ca8c1ace92591" + } + }, + "deliveredToolCount": 39 + } + ], + "mcpReceipt": { + "boundary": "authenticated loopback MCP tools/list; no OpenCode/provider process", + "serverAnnotationsForwarded": false, + "tools": { + "utf8Bytes": 44342, + "characters": 44332, + "sha256": "4241b7d6404f286c693d8760fb41d9dfdf22170271bdf49a0674cc6717fe689b", + "normalizedComparison": { + "utf8Bytes": 44342, + "characters": 44332, + "sha256": "4241b7d6404f286c693d8760fb41d9dfdf22170271bdf49a0674cc6717fe689b" + } + }, + "count": 39 + }, + "originalReceiptSha256": "8486d4c2897ceca5124500899e83eb9754230e80b992c971e586354253ec547e" + }, + "integrated-baseline": { + "schema": "paperclip.native-procedure-measurement.v2", + "sourceSha": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "sourceDirty": false, + "measuredInputsDirty": false, + "sourceHashes": { + "server/src/onboarding-assets/default/AGENTS.md": "c6318bf7e425cc0dd7acd4ff2905dab5c46ffddedef979afdf06720c82f6e95a", + "packages/paperclip-runner/src/contracts/runtime-context.ts": "a17e30a69f3cb38fba00c37e1a06753d723651a004e6050c9e77ae202a61264b", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts": "3b786405fa48e6ba9cf63789c03bbbb370fc2962a4a5c8bd80ccca57759bf912", + "packages/paperclip-runner/src/protocol-actions/create-task.ts": "4efe71c1693dd65d8087b2fdbaf5831b397130d036afb837ece20d4791631805", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts": "53c65776ce4984f177d261c1e53957a97e1a236ce58e731119cb8ee26f141da2", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts": "b8f723ef39f31218d143e284ce6d379d3ba252d4f3086ce05c20db9daf56f848", + "packages/paperclip-runner/generated/semantic-action-catalog.json": "d287caff473a61800135866148f3a2a38ea46cf532343a390dfbf2557f6adc08", + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json": "3d490614baafe423d77d1ad6d46504d47c79373b33ed639614a5f67e9d9bde93", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json": "a53e3d8eae39899111ff1a0110135a9cfafdd12f7843cf6872a3de8fae915a76", + "packages/paperclip-runner/protocol/manifest.json": "9cd783f47dd2b950a89404421a3e76446249693e61c934a91d6f4e529ec18f9b", + "packages/paperclip-runner/src/backends/codex-native-backend.ts": "affecc515a623e0dfeb338f553ea53ebb7d18baa3d13e4395d17b21000dbee93", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "ec1cd94c17be22d0556d4586fa6407b28e2309ed7c2437d95a32beeb0755cc4f", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "d2360cdaa63902cbf0c6a6109da452ba7e0b2a5aaa76ee5cab9c63b4bf12e584", + "server/src/services/native-runtime/native-session-resume.ts": "5b27465e1a328412770fc27c4bc637e88b62022adc6a67709bb74187eadc465b", + "server/src/__tests__/native-procedure-measurement.test.ts": "4e5725c0d86442c0214d96e6ab489547b919d45f8f237d9ec14d02437965268d" + }, + "boundary": "scripted runnerd RPC with production standard-mode authority, API tools enabled, local workspace, no assigned external apps", + "byteAccounting": "utf8Bytes counts the unmodified JSON serialization of each extracted component/projection; normalizedComparison substitutes only the fixture workspace path. Neither is the entire transport request.", + "providerCalls": 0, + "tokenCount": null, + "upstreamLoadingOrTruncation": "unverified", + "vendorStockPrompt": "unavailable", + "receipts": [ + { + "provider": "codex", + "phase": "start", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "3c83a00a3403aa8b20b47ecb9320b1006e0e823116e7947743c4adc6b3ec5803", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 49368, + "characters": 49358, + "sha256": "b89d8d78a06fb98b17022a1f36644c2b309fc8bbab9bf82ddc9a547d77cfa479", + "normalizedComparison": { + "utf8Bytes": 49200, + "characters": 49190, + "sha256": "721d0f637736336c3c8a470b6c80bf40208b09526076c565a98cab364324b63f" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "resume", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "3c83a00a3403aa8b20b47ecb9320b1006e0e823116e7947743c4adc6b3ec5803", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 49368, + "characters": 49358, + "sha256": "b89d8d78a06fb98b17022a1f36644c2b309fc8bbab9bf82ddc9a547d77cfa479", + "normalizedComparison": { + "utf8Bytes": 49200, + "characters": 49190, + "sha256": "721d0f637736336c3c8a470b6c80bf40208b09526076c565a98cab364324b63f" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47506, + "characters": 47496, + "sha256": "73d1774b566e6aaeca24a60a05410362609c7547c94413809bbb2cefe687477f", + "normalizedComparison": { + "utf8Bytes": 47422, + "characters": 47412, + "sha256": "bdb47770fd8f092e6f34ecb85768ed24f1a8613962ea3461252c60911dba5ed6" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "start", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "3c83a00a3403aa8b20b47ecb9320b1006e0e823116e7947743c4adc6b3ec5803", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 49368, + "characters": 49358, + "sha256": "b89d8d78a06fb98b17022a1f36644c2b309fc8bbab9bf82ddc9a547d77cfa479", + "normalizedComparison": { + "utf8Bytes": 49200, + "characters": 49190, + "sha256": "721d0f637736336c3c8a470b6c80bf40208b09526076c565a98cab364324b63f" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "resume", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "3c83a00a3403aa8b20b47ecb9320b1006e0e823116e7947743c4adc6b3ec5803", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 49368, + "characters": 49358, + "sha256": "b89d8d78a06fb98b17022a1f36644c2b309fc8bbab9bf82ddc9a547d77cfa479", + "normalizedComparison": { + "utf8Bytes": 49200, + "characters": 49190, + "sha256": "721d0f637736336c3c8a470b6c80bf40208b09526076c565a98cab364324b63f" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47506, + "characters": 47496, + "sha256": "73d1774b566e6aaeca24a60a05410362609c7547c94413809bbb2cefe687477f", + "normalizedComparison": { + "utf8Bytes": 47422, + "characters": 47412, + "sha256": "bdb47770fd8f092e6f34ecb85768ed24f1a8613962ea3461252c60911dba5ed6" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "start", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "3c83a00a3403aa8b20b47ecb9320b1006e0e823116e7947743c4adc6b3ec5803", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 49368, + "characters": 49358, + "sha256": "b89d8d78a06fb98b17022a1f36644c2b309fc8bbab9bf82ddc9a547d77cfa479", + "normalizedComparison": { + "utf8Bytes": 49200, + "characters": 49190, + "sha256": "721d0f637736336c3c8a470b6c80bf40208b09526076c565a98cab364324b63f" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "resume", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "3c83a00a3403aa8b20b47ecb9320b1006e0e823116e7947743c4adc6b3ec5803", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 49368, + "characters": 49358, + "sha256": "b89d8d78a06fb98b17022a1f36644c2b309fc8bbab9bf82ddc9a547d77cfa479", + "normalizedComparison": { + "utf8Bytes": 49200, + "characters": 49190, + "sha256": "721d0f637736336c3c8a470b6c80bf40208b09526076c565a98cab364324b63f" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1893, + "characters": 1893, + "sha256": "fd2c444ee5d225b4ec24fc46cb66b4c1e11e1f86745b50a297ce94da152b0be5", + "normalizedComparison": { + "utf8Bytes": 1809, + "characters": 1809, + "sha256": "d3618f4e545ed8279a6d71e0f0e214bb713786d91df9a57a2359c8ee076df80a" + } + }, + "tools": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd", + "normalizedComparison": { + "utf8Bytes": 45363, + "characters": 45353, + "sha256": "ab3c8469ca02e07bd99d0a475ac7c10d8bff5b12da3a788896f4f79c675248cd" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47506, + "characters": 47496, + "sha256": "73d1774b566e6aaeca24a60a05410362609c7547c94413809bbb2cefe687477f", + "normalizedComparison": { + "utf8Bytes": 47422, + "characters": 47412, + "sha256": "bdb47770fd8f092e6f34ecb85768ed24f1a8613962ea3461252c60911dba5ed6" + } + }, + "deliveredToolCount": 39 + } + ], + "mcpReceipt": { + "boundary": "authenticated loopback MCP tools/list; no OpenCode/provider process", + "serverAnnotationsForwarded": false, + "tools": { + "utf8Bytes": 44668, + "characters": 44658, + "sha256": "9c6e59816c0a49d675a4afd18f6d05a9d3315da5af803519373bf19fd81ea8dd", + "normalizedComparison": { + "utf8Bytes": 44668, + "characters": 44658, + "sha256": "9c6e59816c0a49d675a4afd18f6d05a9d3315da5af803519373bf19fd81ea8dd" + } + }, + "count": 39 + }, + "originalReceiptSha256": "356693e0c75a86b7e53b397822399b0f4a09652b3d88be280b7a4569fd40efb6" + }, + "integrated-candidate": { + "schema": "paperclip.native-procedure-measurement.v2", + "sourceSha": "ee096aa23fc88a001e7666215bcd338b5863f377", + "sourceDirty": true, + "measuredInputsDirty": false, + "sourceHashes": { + "server/src/onboarding-assets/default/AGENTS.md": "c6318bf7e425cc0dd7acd4ff2905dab5c46ffddedef979afdf06720c82f6e95a", + "packages/paperclip-runner/src/contracts/runtime-context.ts": "3d2ee39cf7cc2210104e155213f04cec82981f76cd7c38384cdc11fa3ce4b013", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts": "ed8d270e2f3a8357bde5f2b180616d09f45218cd90bad22609d2e2e3d02d0319", + "packages/paperclip-runner/src/protocol-actions/create-task.ts": "8557d4030c97aaf99e8ef45dc608e4e0117c764eb459b221ce6739499d79b8aa", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts": "89819f6741dc14009f2daf54717f5e23d583c6f5685c30459fe8a0d672a2720b", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts": "315674491b8e2656b5d9ad77ed41cafd9647f356ed4c46f0dc2de08eb82b646d", + "packages/paperclip-runner/generated/semantic-action-catalog.json": "7238896278a78e307b7eeae1b02d86d785694d10cfdf8dbb4402f1f756e9ebf5", + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json": "4a641cd79e9a75b8247f4e756b6ef0711a76348816b17eb267d284832ed59d37", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json": "1ef1b62506d7c044d68dd9ed8033bd7901f8dba47aea70188c85878d448f6a0a", + "packages/paperclip-runner/protocol/manifest.json": "4c6e1bd289db4be0d669fe79439d7adf3c1a56fc1c762e663299bcd431557597", + "packages/paperclip-runner/src/backends/codex-native-backend.ts": "affecc515a623e0dfeb338f553ea53ebb7d18baa3d13e4395d17b21000dbee93", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "ec1cd94c17be22d0556d4586fa6407b28e2309ed7c2437d95a32beeb0755cc4f", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "ae5cd9c6d076cf44471b42c583f69eea9776aaad41ccbc8319d416aa88f78b44", + "server/src/services/native-runtime/native-session-resume.ts": "d9d4d7fde805c150e8278fd26f36d5e27d77c49690a0371f20e83b542f9cb2db", + "server/src/__tests__/native-procedure-measurement.test.ts": "4e5725c0d86442c0214d96e6ab489547b919d45f8f237d9ec14d02437965268d" + }, + "boundary": "scripted runnerd RPC with production standard-mode authority, API tools enabled, local workspace, no assigned external apps", + "byteAccounting": "utf8Bytes counts the unmodified JSON serialization of each extracted component/projection; normalizedComparison substitutes only the fixture workspace path. Neither is the entire transport request.", + "providerCalls": 0, + "tokenCount": null, + "upstreamLoadingOrTruncation": "unverified", + "vendorStockPrompt": "unavailable", + "receipts": [ + { + "provider": "codex", + "phase": "start", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "20af9131c3b9db8aef690c86b254aab8595f0abac36bb4e2253dd3ab59a6cb25", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48908, + "characters": 48898, + "sha256": "5137d332b7cde806c4276c725dbdb3cb558061517f5fb38f9e9140427f3fe69a", + "normalizedComparison": { + "utf8Bytes": 48740, + "characters": 48730, + "sha256": "120a6211bc0b3a5e141e33e8ac21ee22948940f7698d75f60b6713829ebaac22" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "resume", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "20af9131c3b9db8aef690c86b254aab8595f0abac36bb4e2253dd3ab59a6cb25", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48908, + "characters": 48898, + "sha256": "5137d332b7cde806c4276c725dbdb3cb558061517f5fb38f9e9140427f3fe69a", + "normalizedComparison": { + "utf8Bytes": 48740, + "characters": 48730, + "sha256": "120a6211bc0b3a5e141e33e8ac21ee22948940f7698d75f60b6713829ebaac22" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "codex", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47046, + "characters": 47036, + "sha256": "5defafb88865b1d19aa776323faab1081d84e92c9e832d6f4f5980e3c3d70a06", + "normalizedComparison": { + "utf8Bytes": 46962, + "characters": 46952, + "sha256": "5729e7faaff14fa6de4211022fb8f43fab471dfd2d9eec2c8c950ee56bd55976" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "start", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "20af9131c3b9db8aef690c86b254aab8595f0abac36bb4e2253dd3ab59a6cb25", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48908, + "characters": 48898, + "sha256": "5137d332b7cde806c4276c725dbdb3cb558061517f5fb38f9e9140427f3fe69a", + "normalizedComparison": { + "utf8Bytes": 48740, + "characters": 48730, + "sha256": "120a6211bc0b3a5e141e33e8ac21ee22948940f7698d75f60b6713829ebaac22" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "resume", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "20af9131c3b9db8aef690c86b254aab8595f0abac36bb4e2253dd3ab59a6cb25", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48908, + "characters": 48898, + "sha256": "5137d332b7cde806c4276c725dbdb3cb558061517f5fb38f9e9140427f3fe69a", + "normalizedComparison": { + "utf8Bytes": 48740, + "characters": 48730, + "sha256": "120a6211bc0b3a5e141e33e8ac21ee22948940f7698d75f60b6713829ebaac22" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "acpx", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47046, + "characters": 47036, + "sha256": "5defafb88865b1d19aa776323faab1081d84e92c9e832d6f4f5980e3c3d70a06", + "normalizedComparison": { + "utf8Bytes": 46962, + "characters": 46952, + "sha256": "5729e7faaff14fa6de4211022fb8f43fab471dfd2d9eec2c8c950ee56bd55976" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "start", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "20af9131c3b9db8aef690c86b254aab8595f0abac36bb4e2253dd3ab59a6cb25", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48908, + "characters": 48898, + "sha256": "5137d332b7cde806c4276c725dbdb3cb558061517f5fb38f9e9140427f3fe69a", + "normalizedComparison": { + "utf8Bytes": 48740, + "characters": 48730, + "sha256": "120a6211bc0b3a5e141e33e8ac21ee22948940f7698d75f60b6713829ebaac22" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "resume", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 2077, + "characters": 2077, + "sha256": "20af9131c3b9db8aef690c86b254aab8595f0abac36bb4e2253dd3ab59a6cb25", + "normalizedComparison": { + "utf8Bytes": 1993, + "characters": 1993, + "sha256": "f4a263fa69d0ca3a5ed13095b76da6b5d588540f8777708ee9e08104d066f2ff" + } + }, + "fullProjection": { + "utf8Bytes": 48908, + "characters": 48898, + "sha256": "5137d332b7cde806c4276c725dbdb3cb558061517f5fb38f9e9140427f3fe69a", + "normalizedComparison": { + "utf8Bytes": 48740, + "characters": 48730, + "sha256": "120a6211bc0b3a5e141e33e8ac21ee22948940f7698d75f60b6713829ebaac22" + } + }, + "deliveredToolCount": 39 + }, + { + "provider": "opencode", + "phase": "continuation", + "instructions": { + "utf8Bytes": 1340, + "characters": 1340, + "sha256": "32cebc711830c7db05593c29c6ff59b7e643a0bf0d9434ae51b779bdf3ee18ec", + "normalizedComparison": { + "utf8Bytes": 1256, + "characters": 1256, + "sha256": "1a623b330bb5ae167dacc32a5ea5d714c716ec1ef7f7e3be78a7bc8033fd42be" + } + }, + "tools": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd", + "normalizedComparison": { + "utf8Bytes": 45456, + "characters": 45446, + "sha256": "8f93bd4543a64025b161fe52a98f32491e023379b3216be0ced70cd4a72be5fd" + } + }, + "input": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0", + "normalizedComparison": { + "utf8Bytes": 215, + "characters": 215, + "sha256": "a85a9583b37a072e1707791482d5aa2a5e2cc1e4649b26e009a7abcc488643f0" + } + }, + "fullProjection": { + "utf8Bytes": 47046, + "characters": 47036, + "sha256": "5defafb88865b1d19aa776323faab1081d84e92c9e832d6f4f5980e3c3d70a06", + "normalizedComparison": { + "utf8Bytes": 46962, + "characters": 46952, + "sha256": "5729e7faaff14fa6de4211022fb8f43fab471dfd2d9eec2c8c950ee56bd55976" + } + }, + "deliveredToolCount": 39 + } + ], + "mcpReceipt": { + "boundary": "authenticated loopback MCP tools/list; no OpenCode/provider process", + "serverAnnotationsForwarded": false, + "tools": { + "utf8Bytes": 44761, + "characters": 44751, + "sha256": "ddcc592d0acc330d6fe5783473b7e35a698ef6d4030b0311dee563b9d60aeda2", + "normalizedComparison": { + "utf8Bytes": 44761, + "characters": 44751, + "sha256": "ddcc592d0acc330d6fe5783473b7e35a698ef6d4030b0311dee563b9d60aeda2" + } + }, + "count": 39 + }, + "originalReceiptSha256": "63348f5a79226c2b24991812f9331c671c3559812852ef69d7d1826ee309a8cb" + } + }, + "integration": { + "integratedHead": "ee096aa23fc88a001e7666215bcd338b5863f377", + "paidCandidate": "ffe847fabaebbf24bc93924a4b6c8922d46a1fb6", + "master": "a65ca0950834a85bb93bcc4b4042ecacdebfef53", + "paidSourcesUnchanged": true, + "resolution": "Regenerated three conflicted contract files from combined source; retained master project discovery and finalization changes.", + "liveOnIntegratedHead": false, + "integratedBaseline": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "intendedSourceDeltaIdentical": true, + "commonFixtureFilesChecked": 256, + "sourceDelta": [ + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json", + "packages/paperclip-runner/generated/semantic-action-catalog.json", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json", + "packages/paperclip-runner/protocol/manifest.json", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts", + "packages/paperclip-runner/src/contracts/runtime-context.ts", + "packages/paperclip-runner/src/protocol-actions/create-task.ts", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts", + "server/src/services/native-runtime/native-session-resume.ts", + "server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts" + ] + }, + "limitations": [ + "Single trials do not establish causation, general equivalence, coding quality, speed or cost trends.", + "All graded results retain their original measured sources. No paid result measured the integrated PR head.", + "Original setup used a company cap but left the lead budget at zero. Corrected setup applied both company and lead caps.", + "Unknown total activity and missing final result/ledger/cleanup for two interrupted jobs remain missing, not silently passed.", + "Configured model identity is not independent proof of provider-side execution identity.", + "ACPX provider-execution IDs are not proven mapped to host-call IDs. Never join by name, order or count.", + "Measurement is an extracted delivery projection; vendor private prompts, tokenization, loading and truncation remain unverified.", + "Numeric billing subtotals and raw sessions are excluded from this public summary; no complete billing or savings claim is made." + ] +} diff --git a/doc/plans/2026-10-05-native-procedure-guidance.md b/doc/plans/2026-10-05-native-procedure-guidance.md new file mode 100644 index 00000000000..f531b238ce7 --- /dev/null +++ b/doc/plans/2026-10-05-native-procedure-guidance.md @@ -0,0 +1,382 @@ +# Native hiring and dependency guidance + +Status: **ship measurement and eval coverage only. Do not ship the procedure +relocation or the unqualified runtime repairs.** + +The final PR keeps production agent behavior byte-identical to master +`a65ca0950834a85bb93bcc4b4042ecacdebfef53`. Its changes are limited to the +measurement test, explicit eval selection, correct hiring credential selection, +company/lead budget stops, test-runner configuration, and retained reports. +No production prompt, tool description, tool schema, session catalog, dependency +mutation or status-transition code changes are included in the final diff. + +The experiment did not pass the no-regression gate. The first corrected +comparison had baseline 5 PASS / 1 FAIL and candidate 3 PASS / 3 FAIL: two new +failures, no new passes, three unchanged passes and one unchanged failure. +The attempted repair also had two new failures among five comparable pairs. +Moving text out of the fixed prompt did not justify shipping worse outcomes. +Single trials do not establish that the changed instructions caused a failure. + +The original candidate reduced the complete standing projection by 460 bytes, +about 0.9%. Restoring important instructions reduced the saving to 125 bytes, +about 0.25%. A later, unqualified dependency guard made the projection 127 bytes +larger than baseline. None of those changes is included in the final production +tree. The final measurement records the unchanged full catalog. Bytes are not +tokens, billing, or proof of upstream loading or truncation. + +The [original sanitized comparison](2026-10-05-native-procedure-comparison.json) +and [readiness comparison v4](2026-10-05-native-procedure-readiness-v4.json) +retain exact sources, original grades and evidence hashes. Their old source +revisions remain historical experiments; they do not describe production changes +in this measurement-only PR. No completed behavioral failure was rerolled to +obtain a pass. Earlier infrastructure recovery and missing evidence remain +explicit below. This follows the merged completion slice in #15151. + +## Final scope and verification + +The server measurement test instantiates the actual standard-mode tool authority +with all advertised tools and input schemas. It captures scripted native start, +resume and continuation payloads across Codex, ACPX Claude and OpenCode and the +OpenCode MCP declaration list. These are provider-free measurements. + +The two original Everyday Workflows stories remain explicit-only and keep their +independent artifact and ordering oracles. OpenCode is added only to the local +hiring/reuse and delegation/feedback cells. These stories use one attempt per +cell and actual company and lead-agent budget stops. Managed hiring selects +the credential provider from the chosen profile, including OpenRouter. Existing +Node test files run under Node's test runner instead of being misclassified as +empty Vitest suites. + +The final production identity check compares `packages/` and production +`server/` files against the fixed master context. A fresh CI and review must +qualify the final tree. The full local database suite is unavailable while this +host's PostgreSQL shared-memory slots are exhausted; hosted CI provides those +database checks. No new paid campaign is needed to compare unchanged production +bytes. This does not claim that the existing product is free of the failures +observed in the retained trials. + +## Unqualified runtime repairs retained for follow-up + +The local branch `codex/native-procedure-controller-repairs-unqualified` at +`7e270a3d3` preserves the attempted controller repairs. It is not part of this +PR's final diff and has not passed live qualification. It scopes completion +wakes to each committed transition and rejects silent removal of unfinished +dependencies, with explicit handling for obsolete dependencies. Its regression +tests still require database execution. Source typecheck, catalog generation, +and the non-database measurement/session checks passed locally. + +The remaining issues need separate bounded investigation: + +- Reopened child completion can reuse a previously consumed parent wake key. +- A parent can discard a still-running required dependency and finish early. +- OpenCode's v4 hiring candidate failed the exact-credential persistence guard + in its local SQLite database. Cleanup removed that database. The credential + identity and exact persistence mechanism are not retained. No cause or fix + is claimed, and no credential check is weakened. + +## Readiness comparison v4 — historical failed experiment + +Candidate `aba7ec219b588de02cf323b4ebca559a105e6d01` and baseline +`ce31f5efd0d416205c3994c55dd02b645f8b8423` use common master +`a65ca0950834a85bb93bcc4b4042ecacdebfef53`. Candidate +[campaign 37348723829](https://github.com/paperclipai/paperclip/actions/runs/37348723829) +and baseline [campaign 37348764875](https://github.com/paperclipai/paperclip/actions/runs/37348764875) +use trusted workflow `59015846ae02f935411afc620e0867ce812378fb`, distinct +from both measured trees. Eleven result/API/ledger sets and 99 evidence hashes +are retained. Their final ledgers contain 56 started runs, in addition to the +earlier cohorts. All eleven cleanup results passed. + +Candidate: 3 PASS / 3 FAIL. Baseline: 3 PASS / 2 FAIL / 1 setup cell without a +behavioral grade. Comparable pairs: two new failures, two new passes and one +unchanged pass. The remaining Codex delegation pair is uncomparable because +baseline Docker setup timed out before its paid step; no behavioral grade or +final result artifact exists for that cell. + +| Story and harness | Baseline | Candidate | Observed missing behavior | +|---|---|---|---| +| Codex hiring/reuse | PASS | PASS | None observed by the original oracle | +| Codex delegation/feedback | No behavioral grade | FAIL | Parent remains blocked after the revised child finishes | +| Claude hiring/reuse | FAIL | PASS | Baseline did not deliver the revised ZIP | +| Claude delegation/feedback | PASS | FAIL | Candidate finishes the parent before the child revision | +| OpenCode hiring/reuse | PASS | FAIL | Candidate credential-persistence guard fails | +| OpenCode delegation/feedback | FAIL | PASS | Baseline finishes the parent before the child revision | + +The two new passes do not offset the new failures. The `aba7ec2` CI passed on +attempt 2 after five infrastructure-interrupted jobs were recovered and its +fresh review was 5/5; source checks did not clear these behavioral failures. + +## Historical experiment scope + +The original paid comparison used master +`a386a599983519eb1d399f8b770bfccdb2a74762` as common context. It moved +hiring/reuse and assigned-worker guidance to `hire_agent` and `create_task`, +kept dependency discovery in the fixed prompt, and returned waiting instructions +from `set_dependencies`. It advanced prompt and session-catalog revisions. +Those production changes have been removed from the final PR. Connection +procedures and legacy skill/API instructions remain unchanged. + +## Measurement boundary + +Capture the actual standard-mode server tool authority with API tools enabled, +a local workspace, and no assigned external apps. Measure fixed instructions, +all advertised tool descriptions and input schemas, and the serialized task +input together. Scripted native start/resume/continuation checks cover Codex, +ACPX Claude and OpenCode; authenticated MCP tools/list checks the OpenCode +bridge separately. Export source identity, file hashes, bytes and tool counts. + +These receipts have zero provider calls. Bytes are not model tokens or invoice +costs. This boundary cannot establish the upstream SDK's private instructions, +lazy loading, truncation or cognitive use of returned guidance. The previous +completion-only capture selected a partial catalog and is not a full-tool +baseline. Preserve its historical evidence unchanged. + +## Historical matched live comparison plan + +Use the existing explicit-only `everyday-workflows` suite with original +`hire-reuse` and `delegate-feedback` requests and independent artifact oracles. +Select only native Codex `gpt-5.6-sol`, ACPX Claude `claude-sonnet-5`, and OpenCode +`openrouter/deepseek/deepseek-v4-flash-0731`, in local environments. + +Freeze common measurement, harness and test-runner repairs in both branches +before the candidate production changes. The historical branch keeps the +current production prompt and tool contracts. Record exact measured source +SHAs separately from the trusted master workflow definition and selected +runtime/artifact hashes. Never label a later source revision as the measured +revision without showing its delta. + +The plan has six cells per variant, one attempt per cell, a 12-minute cell +deadline, at most 12 story run records per cell, and a 1,000-cent company and +lead-agent hard stop per cell. The original campaign applied the company cap +but mistakenly left the lead budget at 0; the corrected common setup verifies +both actual API payloads. All worker runs count toward the company budget; +individual worker caps are not asserted. +Count every actual run, including notifications and failed attempts. No broad +matrix, automatic retries or baseline rerolls are part of this comparison. + +Inspect original grades and retained content for persistent teammate identity, +reuse on revision, worker ownership, independently tested delivered artifacts, +parent review, dependency release, and final ordering. Check per-pair changes; +equal totals are insufficient. Preserve every original failure and missing +evidence. Model-authored tests cannot replace the existing independent oracle. + +These two stories do not qualify arbitrary live resume, every existing-blocker +combination, general coding quality, or cost/speed trends. Explicitly report +missing evidence. Do not loosen the original oracle to create a pass. + +## Historical complete-catalog measurement + +The corrected source `ffe847fabaebbf24bc93924a4b6c8922d46a1fb6` is compared with +baseline `aecfa3b6a5cab110a9b926e93705311bbcf4db47`. Both include the same +measurement fixture and common eval repairs. Exactly 12 source/generated/unit +paths differ; 256 common fixture files have identical bytes. + +| Normalized extracted component | Baseline bytes | Corrected candidate bytes | Change | +|---|---:|---:|---:| +| Fixed instructions, start/resume | 1,809 | 1,256 | −553 | +| All 39 tool declarations including schemas | 44,944 | 45,037 | +93 | +| Task input, start/resume | 1,993 | 1,993 | 0 | +| Combined standing projection, start/resume | 48,781 | 48,321 | −460 | +| OpenCode MCP tools/list declarations | 44,249 | 44,342 | +93 | + +Scripted Codex, ACPX Claude and OpenCode deliveries give the same comparative +change. The serialization of the extracted components is measured separately +from its path-normalized comparison. The fixture's real temporary paths add +168 bytes to the combined start/resume projection in each variant. Neither +projection is the entire transport request. Returned dependency guidance adds +input when used, so this is not an overall token, latency or cost saving claim. +The measurement produces zero provider calls and does not prove upstream +loading or truncation behavior. + +## Original comparison — failures retained + +Original sources were candidate `eb01d073c35134bf0d373947b55a79f9e443f8ee` and +baseline `64cd2194a1d058356679b653c181f16404aefc80`, using trusted master workflow +`a386a599983519eb1d399f8b770bfccdb2a74762`. Source identity remains distinct from +workflow identity and the later measurement-only correction. + +- [Original candidate campaign](https://github.com/paperclipai/paperclip/actions/runs/37325777217) +- [Original baseline campaign](https://github.com/paperclipai/paperclip/actions/runs/37325870695) + +| Profile | Hire/reuse baseline → candidate | Delegate/feedback baseline → candidate | +|---|---|---| +| Codex | PASS → PASS | PASS → FAIL | +| ACPX Claude | PASS → FAIL | PASS → FAIL | +| OpenCode | FAIL → FAIL | FAIL → PASS | + +There are three new overall failures, one new pass, one unchanged pass and one +unchanged failure. The candidate did not qualify. Final ledgers match the +result run-ID sets: 25 started baseline runs and 15 candidate runs, zero retry +links. All twelve cell cleanup receipts passed. Missing/zero billing amounts +and unmetered host runtime prevent any claim about actual cost savings. + +The candidate Claude hire created Morgan and a child task, then ended with a +waiting message without recording dependencies or using a terminal tool. The +lead failed with `native_finalization_missing`. The corrected v7 prompt keeps +explicit `set_dependencies` and `paperclip_block` discovery. This is a concrete +missing behavior; a single trial does not establish causation. + +The candidate Codex delegate failed in Riley's worker turn with the provider +error `Selected model is at capacity` / `serverOverloaded`. Its parent had +received the dependency guidance. Candidate Claude delegation failed opening +an ACPX session (`session_ensure_failed`); its underlying cause is unknown. +Both original grades and classifiers remain unchanged. + +Both OpenCode hire cells failed common setup requesting an OpenAI credential +although OpenRouter was selected, before any agent run. Shared setup now uses +the profile's actual credential. A separate nonce/cell-ID bug left original +lead budgets at 0; six nonce-based setup tests now verify company and lead caps +using only the selected provider credential. These common control changes are +why the corrected campaign uses a newly matched baseline. + +The baseline OpenCode delegation delivered an artifact passing the independent +oracle but completed the parent before the child's final revision. Candidate +passed both the artifact and ordering checks. Correct output alone did not +qualify that baseline outcome. + +## Corrected comparison + +- [Candidate ffe847f](https://github.com/paperclipai/paperclip/actions/runs/37331580879) +- [Baseline aecfa3b](https://github.com/paperclipai/paperclip/actions/runs/37331642168) +- [Candidate recovery for two interrupted cells, same ffe847f source](https://github.com/paperclipai/paperclip/actions/runs/37334061104) + +These use the same corrected fixtures, original requests and artifact graders, +with one attempt per campaign cell. The original campaign is preserved and is +not relabeled as this revision. The common suite hash is +`acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334`. + +| Profile | Hire/reuse baseline → candidate | Delegate/feedback baseline → candidate | +|---|---|---| +| Codex | PASS → PASS | PASS → FAIL (recovery) | +| ACPX Claude | PASS → PASS | PASS → PASS (recovery) | +| OpenCode | PASS → FAIL | FAIL → FAIL | + +There are **two new failures, zero new passes, three unchanged passes, one +unchanged failure and zero pending pairs**. Workflow completion is not the +behavioral grade. These are the original result grades; no regrading was done. + +### Missing and recovered behavior + +**Codex delegation regressed in the observed pair.** The first parent turn +recorded its dependency and blocked correctly. After feedback, the parent +reviewed a stale ZIP and requested a replacement. An API comment operation was +rejected with `API comments cannot change runner execution state`. The child +finished the replacement at 15:45:48.284 UTC. The parent then recorded the +dependency at 15:45:51.801–15:45:59.295 UTC; its receipt scheduled no wake. The +parent blocked and finalized at 15:46:19.909 UTC. The child was Done, but the +parent remained Blocked with no active continuation. All four native runs had +successful run status; that does not make the task outcome successful. This +sequence does not by itself establish a prompt, race or platform root cause. + +**OpenCode hiring regressed in the observed pair.** It hired the expected +teammate and produced an initial artifact that passed the independent checks. +It reused the same teammate for the separator revision, then requested further +CLI argument-handling changes. After nine started runs, the story reached the +12-minute deadline with parent and revision work still in progress. Cleanup +cancelled the remaining work. The final revised-ZIP and settled-outcome checks +were not completed; they are not assumed to pass. The baseline passed after +four runs. This single trial does not establish a speed or cost trend. + +**OpenCode delegation failed in both variants.** The artifacts passed the +independent output checks, but the parents finished before the final child +revision. Candidate parent/child completion times were 15:33:43.628 / +15:35:03.222 UTC; baseline times were 15:35:21.429 / 15:36:03.043 UTC. Both +parents lacked review of that final revision. Equal FAIL totals conceal neither +this missing behavior nor the passing artifact checks. + +**Claude hiring now passes all 45 original checks.** Retained semantic receipts +show dependency recording, returned guidance and an accepted current-track +block with the child agent as owner. The parent later resumed and finished. +This repairs the original missing dependency/finalization observation in this +trial. It does not prove cognitive use of the guidance or every hiring tool +path; API hiring remained available. + +### Interruptions and accounting + +Candidate Codex and Claude delegation jobs received runner shutdown signals at +15:27:38 UTC and were cancelled at 15:27:50 UTC. Jobs `111838521694` and +`111838521502` retained no result, final ledger or cleanup receipt. Their +behavioral grades and total actual runs/charges are unknown. Eight distinct +run-record IDs were visible in partial logs; this is not a complete started-run +ledger. Preserve the workflow failures and missing evidence. + +Only those two missing candidate cells received one bounded recovery each, on +the identical measured source. No completed baseline or other candidate cell +was repeated. The original workflow revision was `a386a599`; recovery used +`a65ca0950834a85bb93bcc4b4042ecacdebfef53` after master advanced. Workflow file +bytes were identical, SHA-256 +`eb570267e66043c1196338bb386b07ab8e8e8306ba04b6223f13b73b04fdc8f4`. +The measured source stayed `ffe847f`, distinct from both workflow revisions. + +All twelve completed/recovered cells have matching result/final-ledger run +sets and passing cleanup receipts. They contain 27 baseline and 33 candidate +started runs. The original campaign contains 40 more: **100 retained started +runs, plus incompletely known activity from the two interrupted jobs**. Zero +retry links within the completed ledgers do not erase the two infrastructure +recoveries. The audits retain 100 original and 108 corrected JSON evidence +hashes. Actual charges remain unknown because usage and interruption records +are incomplete and host runtime is unmetered. No free-call or savings claim is +made. + +ACPX provider execution IDs are not proven mapped to host call IDs. No strict +no-extra-work claim or feedback-consumption inference is made from matching +names, ordering or counts. These evidence limits remain even for a passing +original story grade. + +## Historical integration and repository checks + +After the paid sources were frozen, the PR merged master +`a65ca0950834a85bb93bcc4b4042ecacdebfef53` as +`ee096aa23fc88a001e7666215bcd338b5863f377`. Three generated contract conflicts +were resolved by normal generation from the combined sources. The paired +provider-free baseline is `ce31f5efd0d416205c3994c55dd02b645f8b8423`. +Their twelve intended differing paths are identical to the paid source delta; +all 256 common fixture files remain byte-identical. **No live campaign measured +the integrated head.** Later documentation-only commits do not change that. + +The integrated full-catalog projection is 49,200 → 48,740 normalized bytes at +start/resume and 47,422 → 46,962 during continuation. The 460-byte reduction is +unchanged. Master's project-discovery schema adds 419 bytes to both tool sets. +The actual extracted start/resume projection is 49,368 → 48,908 bytes. All 39 +declarations remain present; this still does not measure private vendor prompts, +model tokens, lazy loading or upstream truncation limits. + +Before integration, 74 focused tool/session/measurement checks, 1,252 eval-support +Vitest checks, 128 Node checks and the eval TypeScript check passed. Repository +typecheck and build passed before the prompt-only v7 correction. The corrected +source CI then passed typecheck, build, tests and all eight browser shards. An +interrupted canary job was retried once; that is separate from paid eval recovery. + +At integrated `ee096aa23`, local measurement/session checks passed 48 tests; +23 authority tests could not start because PostgreSQL initialization failed, +and 21 project-tool tests were skipped by the unsupported database helper. +The integrated baseline passed six measurement tests. The older full local +suite was stopped after source revisions changed during its run. Its failures +remain retained; it is not reported as green. + +[Integrated CI server shard 5](https://github.com/paperclipai/paperclip/actions/runs/37335450901/job/111849219339) +failed `workspace-git-snapshot-streaming.test.ts`: the four Git filename lanes +over 32 MiB exceeded the 180-second timeout (707 tests passed, two skipped, +one failed in that shard). The aggregate verify job therefore failed. The cause +has not been established; this is not reclassified as an infrastructure pass. +The [review summary](https://github.com/paperclipai/paperclip/pull/15218#issuecomment-5996723170) +gave integrated `ee096aa23` 5/5 but noted project-discovery authorization work +for actors with sparse visibility. Both inline measurement findings are +resolved; the summary's separate concern is not erased by that count. + +At this historical checkpoint the failed behavior comparison required keeping +the candidate draft. The final scope separates measurement from the rejected +reduction. Future behavior changes need their own diagnosis and qualification; +completed paid failures must not be rerolled simply to obtain passing grades. + +## Historical behavior-change gates (not met; change removed) + +- [x] Freeze common fixtures and both source revisions. +- [x] Export comparable full-catalog delivery projections. +- [x] Pass 74 focused checks, 1,252 eval-support Vitest checks and 128 Node checks. +- [ ] Complete repository typecheck, tests and build. +- [x] Run and inspect the bounded matched campaign, including original failures. +- [ ] Meet the no-regression gate: **failed, two newly failing corrected pairs**. +- [ ] Verify current-head CI, review and conflicts before readiness. +- [x] Preserve original and corrected grades, hashes and incomplete accounting. + +No merge is authorized. diff --git a/doc/plans/2026-10-05-native-procedure-readiness-v4.json b/doc/plans/2026-10-05-native-procedure-readiness-v4.json new file mode 100644 index 00000000000..ec15020f87e --- /dev/null +++ b/doc/plans/2026-10-05-native-procedure-readiness-v4.json @@ -0,0 +1,2019 @@ +{ + "schema": "paperclip.native-procedure-readiness-public.v1", + "state": "no_regression_gate_failed_one_setup_cell_uncomparable", + "sources": { + "candidate": "aba7ec219b588de02cf323b4ebca559a105e6d01", + "baseline": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "candidateBranch": "codex/native-procedure-candidate-v4", + "baselineBranch": "codex/native-procedure-baseline-v3" + }, + "campaigns": { + "candidate": 37348723829, + "baseline": 37348764875 + }, + "workflowSha": "59015846ae02f935411afc620e0867ce812378fb", + "originalGradesPreserved": true, + "pairCounts": { + "newFailures": 2, + "newPasses": 2, + "unchangedPasses": 1, + "unchangedFailures": 0, + "pending": 1 + }, + "pairs": [ + { + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "baseline": "passed", + "candidate": "passed" + }, + { + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "baseline": null, + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "baseline": "failed", + "candidate": "passed" + }, + { + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "baseline": "passed", + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "baseline": "passed", + "candidate": "failed" + }, + { + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "baseline": "failed", + "candidate": "passed" + } + ], + "cells": [ + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "source": { + "sha": "aba7ec219b588de02cf323b4ebca559a105e6d01", + "ref": "refs/heads/codex/native-procedure-candidate-v4", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348723829" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "model": "claude-sonnet-5", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 4, + "startedRunCount": 4, + "runStatuses": { + "succeeded": 4 + }, + "resultSha256": "2052ff9fd63a10bba7940034fe7c1362b7d9e3bfa00cbae490371226e3769789", + "evidenceHashes": { + "evidence-manifest.json": "e10659910f5060bcf07bf22ac92d776f39d8dd97ff0caba3e5029db6c27f3160", + "html-report/data/5c30415881df8ec08d97ca7a6bede282562ed011.json": "2052ff9fd63a10bba7940034fe7c1362b7d9e3bfa00cbae490371226e3769789", + "playwright-output/runner-everyday-workflows--fdae5-the-browser-and-public-APIs/attachments/runner-e2e-result-86dec9e27afb28b2e2187e13cb5c2f9fc0e27da7.json": "2052ff9fd63a10bba7940034fe7c1362b7d9e3bfa00cbae490371226e3769789", + "result.json": "2052ff9fd63a10bba7940034fe7c1362b7d9e3bfa00cbae490371226e3769789", + "snapshots/api-state.json": "79629476404aacd118880ca3fac7807e36384714a1d0665f39a912e6a5ba4596", + "snapshots/browser-diagnostics.json": "09425086c198b70cb77ab0445b05ef49dfc07d448e89504a35e073a8d1fe9b5e", + "snapshots/chat-final-run-ledger.json": "1c71ffedd9a955323eee0094120248e9bcc288f62936b903b5ed39086a5f867c", + "snapshots/everyday-workflow.json": "08614a740e941a2b6898490d09e16034c0c2a35093a6281c0d085fa9a0b3de96", + "snapshots/fixtures.json": "49ea2e205256f8c6ffda07c568ab82f4588e9e6f98d831ba0b2a224d621d5299" + }, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": false + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ] + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "source": { + "sha": "aba7ec219b588de02cf323b4ebca559a105e6d01", + "ref": "refs/heads/codex/native-procedure-candidate-v4", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348723829" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "model": "claude-sonnet-5", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 6, + "startedRunCount": 6, + "runStatuses": { + "succeeded": 6 + }, + "resultSha256": "89cf913d14d8539ac5b57e9f4e25de320a747c88afefa0eada25e06e3b12623f", + "evidenceHashes": { + "evidence-manifest.json": "105c12346c49a688db40e18e02d6efc50d14c84910916e553cc5d0a8b346afff", + "html-report/data/cb46b8428a23af11c95ce3d137f6335b810bd844.json": "89cf913d14d8539ac5b57e9f4e25de320a747c88afefa0eada25e06e3b12623f", + "playwright-output/runner-everyday-workflows--9a7d2-the-browser-and-public-APIs/attachments/runner-e2e-result-8789ecd32f82f4b761fb24a7725ca7ec6cb73ac0.json": "89cf913d14d8539ac5b57e9f4e25de320a747c88afefa0eada25e06e3b12623f", + "result.json": "89cf913d14d8539ac5b57e9f4e25de320a747c88afefa0eada25e06e3b12623f", + "snapshots/api-state.json": "29e710b168e8af6ccc61834c26651d2000b04460d01bd66a70288d22b33f56eb", + "snapshots/browser-diagnostics.json": "526698e7c5252f6a3977fbc39f9378100ec085828ef0496393555770a1d6a013", + "snapshots/chat-final-run-ledger.json": "765feb06ac615078708fb8ac606d1a1ab97906fe9a13a7e879c257062321a90a", + "snapshots/everyday-workflow.json": "d611e433959e70022e99e4dd52df71203c74e3a82dd901c4f0d1f72fc5fb5739", + "snapshots/fixtures.json": "a10c0fcdbf6f630c57ee74e882ddac14b842ecdf74bdcd358ca4d29ae3370897" + }, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ] + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "source": { + "sha": "aba7ec219b588de02cf323b4ebca559a105e6d01", + "ref": "refs/heads/codex/native-procedure-candidate-v4", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348723829" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "model": "gpt-5.6-sol", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 4, + "startedRunCount": 4, + "runStatuses": { + "succeeded": 4 + }, + "resultSha256": "7bc74633dc89cea37a3da9808649a9a0e8594ced95e43a01df1d0db55e3443aa", + "evidenceHashes": { + "evidence-manifest.json": "7cebcff0de1037aeffbcf2e23d2bdf5a21a362210c553ec3fc3b0afb9f3b7ddc", + "html-report/data/9cf046f592d24ccff4f9df9a989cbeb14d66185e.json": "7bc74633dc89cea37a3da9808649a9a0e8594ced95e43a01df1d0db55e3443aa", + "playwright-output/runner-everyday-workflows--a6fcb-the-browser-and-public-APIs/attachments/runner-e2e-result-f13a90c839b7c34d9d295c72bec11dc94b31ff3d.json": "7bc74633dc89cea37a3da9808649a9a0e8594ced95e43a01df1d0db55e3443aa", + "result.json": "7bc74633dc89cea37a3da9808649a9a0e8594ced95e43a01df1d0db55e3443aa", + "snapshots/api-state.json": "7c68b1ae4cfb624f59d975a6ebb645ee648270805235a4e3603504b7d09977a5", + "snapshots/browser-diagnostics.json": "dec21a43696a72270af7bc10ab45e1d635d231f10e7ae4cd1a31e4476123bc7f", + "snapshots/chat-final-run-ledger.json": "2ac35bc9eb6c6f3d29452f287745d1fbb9a49b9e8e72bd241da519455ef86d64", + "snapshots/everyday-workflow.json": "d16fcfc9a2b0ce8bcedf8b2c6d0fecbe55813400dae7e407a166fdbb3b6f4960", + "snapshots/fixtures.json": "5094477bb71791879b257bb6f341e5f7804b7bf13304f62d4cb65554d6ba7e79" + }, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ] + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "source": { + "sha": "aba7ec219b588de02cf323b4ebca559a105e6d01", + "ref": "refs/heads/codex/native-procedure-candidate-v4", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348723829" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "model": "gpt-5.6-sol", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 6, + "startedRunCount": 6, + "runStatuses": { + "succeeded": 6 + }, + "resultSha256": "f4e0c4dcb62b9d1ce7a63aa5c9f94123f201b9534f90d924fd1a51828ac705cb", + "evidenceHashes": { + "evidence-manifest.json": "3e4e2408709036e1f79c70d50edb91230a905706fc204e963a0ee1e2f38887e7", + "html-report/data/e07424ec1b7a5523ba0b6982a1886f1bff8f008e.json": "f4e0c4dcb62b9d1ce7a63aa5c9f94123f201b9534f90d924fd1a51828ac705cb", + "playwright-output/runner-everyday-workflows--d934c-the-browser-and-public-APIs/attachments/runner-e2e-result-2af08a5b86fffd7530895a7502aaf2acbb80f798.json": "f4e0c4dcb62b9d1ce7a63aa5c9f94123f201b9534f90d924fd1a51828ac705cb", + "result.json": "f4e0c4dcb62b9d1ce7a63aa5c9f94123f201b9534f90d924fd1a51828ac705cb", + "snapshots/api-state.json": "0626b20467a3e403cb1e43372e96fc704807a8742396c4847c29440a4e68df3d", + "snapshots/browser-diagnostics.json": "bf94315df08454b2c1361c5553f46eb82af593db78d125e9e6c195cc13dc125c", + "snapshots/chat-final-run-ledger.json": "349026e3638cb2d2b16279b7b98ee45fedb85e070fb780c44392c01e6bc56740", + "snapshots/everyday-workflow.json": "abbbfabaff06ad00f6ce6c8f0a1fd43e40398b800c76bf479365c1cbd45f3a2b", + "snapshots/fixtures.json": "1195bf03b48915c8d0f55157bad3d0b33e1b3a888dc5868c33aec2874af599d9" + }, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ] + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "source": { + "sha": "aba7ec219b588de02cf323b4ebca559a105e6d01", + "ref": "refs/heads/codex/native-procedure-candidate-v4", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348723829" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 4, + "startedRunCount": 4, + "runStatuses": { + "succeeded": 4 + }, + "resultSha256": "b08a55f91bbdce0cf0558a78f25fd6d5de4459a1c96d5f0559dbb13834b5b738", + "evidenceHashes": { + "evidence-manifest.json": "db3dd6cdbf826a9bae0d8ed97ebe4edc51f8c0f71022d7addc1718a3066ed2d2", + "html-report/data/c1b9063fd22a68f64bf4d6e5d4c3426ef6cac4dd.json": "b08a55f91bbdce0cf0558a78f25fd6d5de4459a1c96d5f0559dbb13834b5b738", + "playwright-output/runner-everyday-workflows--616ad-the-browser-and-public-APIs/attachments/runner-e2e-result-de3bc5087d02916426e722e1a0fbd2c23d867550.json": "b08a55f91bbdce0cf0558a78f25fd6d5de4459a1c96d5f0559dbb13834b5b738", + "result.json": "b08a55f91bbdce0cf0558a78f25fd6d5de4459a1c96d5f0559dbb13834b5b738", + "snapshots/api-state.json": "57257f9b63b63b34d049b3755f4428eeee1b71656e0797399174b503925be7cb", + "snapshots/browser-diagnostics.json": "ec80144d5efb3c0fcd5d8de6d847559b47f1c982ce89a4b2b6be7e4ff2745c56", + "snapshots/chat-final-run-ledger.json": "05648c596135076994e3e729927ed1ef75ba1caa7b063e26db175536c1f15f27", + "snapshots/everyday-workflow.json": "5029b09ed242ca9526f217d61031b264bb21980a0872ba914a6c077f45ea5307", + "snapshots/fixtures.json": "d359e7e9fab7f6d7479962dcd54ec82a7015ea7f0a2af75d1267716a2227227e" + }, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ] + }, + { + "variant": "candidate", + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "source": { + "sha": "aba7ec219b588de02cf323b4ebca559a105e6d01", + "ref": "refs/heads/codex/native-procedure-candidate-v4", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348723829" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "secret_leak", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 6, + "startedRunCount": 6, + "runStatuses": { + "succeeded": 6 + }, + "resultSha256": "c3e7b3ccfcd8d40e8212d7cc48bc0e12aaf8340ec3e6e76ee0f93cf3502dc8b7", + "evidenceHashes": { + "evidence-manifest.json": "fb6d9c63a3dabfc2f9a4fc882fc8ab0eb4c3f185d9c01a86e735fb5c04430aaa", + "html-report/data/e2a9234f30c494f82aac6de3a85691ca9e0002db.json": "98cfdcc136287d080efeb8e0b2ce6ab8c91169a99a3da627e38591cfe9187588", + "playwright-output/runner-everyday-workflows--90607-the-browser-and-public-APIs/attachments/runner-e2e-result-5b5c108dfa523e2f2e72dd426cbcce5fa8a4f16f.json": "98cfdcc136287d080efeb8e0b2ce6ab8c91169a99a3da627e38591cfe9187588", + "result.json": "c3e7b3ccfcd8d40e8212d7cc48bc0e12aaf8340ec3e6e76ee0f93cf3502dc8b7", + "snapshots/api-state.json": "b2ca1ef09614cb4cf2dffdd5aa2fb5bd574eff3c316a689f4dd242427428b5c4", + "snapshots/browser-diagnostics.json": "8744b2d708c86d0c3b3595d910b6d831444fc2e217b09aa90c618dddf7326933", + "snapshots/chat-final-run-ledger.json": "6ee7305e3dff3192e613c6c38701b8c4b63f79aecfc7d25e40ecc5c14122cf0d", + "snapshots/everyday-workflow.json": "8e82f0643eda9224c7ae6c5cc941f09d1fefe6da0ea8df31b65ce1fa1bfdba16", + "snapshots/fixtures.json": "3800139b64c63a0cdf6109ad45f738b8cc52a36c33b085b32acc1cc78f583a37" + }, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ] + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-acpx-claude.local.delegate-feedback", + "source": { + "sha": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "ref": "refs/heads/codex/native-procedure-baseline-v3", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348764875" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "model": "claude-sonnet-5", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 4, + "startedRunCount": 4, + "runStatuses": { + "succeeded": 4 + }, + "resultSha256": "0c241c2bf4aa78b2e53c9a81965f2af5cd0a65914e02a9f33d7f9905446a6aa7", + "evidenceHashes": { + "evidence-manifest.json": "dce0a724dbeefc0578d07a5e4955a8ddfe9e5dd78a3e72472f48d2022864a29f", + "html-report/data/e247a98e2ef3ab1585b500ddeb1083f5c62eebd2.json": "0c241c2bf4aa78b2e53c9a81965f2af5cd0a65914e02a9f33d7f9905446a6aa7", + "playwright-output/runner-everyday-workflows--fdae5-the-browser-and-public-APIs/attachments/runner-e2e-result-b7cfd2b7fd8d7cbe59d1960795561610ea681b41.json": "0c241c2bf4aa78b2e53c9a81965f2af5cd0a65914e02a9f33d7f9905446a6aa7", + "result.json": "0c241c2bf4aa78b2e53c9a81965f2af5cd0a65914e02a9f33d7f9905446a6aa7", + "snapshots/api-state.json": "a2684af703be5f88a2c7e87632dc5db8c835586930b210c6514d6e3e979ded8c", + "snapshots/browser-diagnostics.json": "9fdc8a931f20fcc121b9aef71a34753bc762caa20fea0b6754b1de20fe9be298", + "snapshots/chat-final-run-ledger.json": "747524776dfbd2c11582bba85552d5599b00ff31ae86350ea1f570772215be7e", + "snapshots/everyday-workflow.json": "20cdb9a0d00ed734fdcdb750f8e3490146b75ba481563729df3afeb7dd7e6b49", + "snapshots/fixtures.json": "beaa1bb97dd431fbcf35989d3ec464312dffe206f961e39ada5abb042dd4dd0d" + }, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ] + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-acpx-claude.local.hire-reuse", + "source": { + "sha": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "ref": "refs/heads/codex/native-procedure-baseline-v3", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348764875" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "model": "claude-sonnet-5", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 6, + "startedRunCount": 6, + "runStatuses": { + "succeeded": 6 + }, + "resultSha256": "cd6a2a4dfadf970ed0bc8067bbd4352fe512b290e97dbb0234e804a980c44013", + "evidenceHashes": { + "evidence-manifest.json": "0f42e325c18e5ab1caffed574ef8c7ff064238b590376624023cbfb1c0444c69", + "html-report/data/25e2cab109cbcb2f719700b2acf37c71a11af214.json": "cd6a2a4dfadf970ed0bc8067bbd4352fe512b290e97dbb0234e804a980c44013", + "playwright-output/runner-everyday-workflows--9a7d2-the-browser-and-public-APIs/attachments/runner-e2e-result-e98664c4029e78829752908076817b85e6db5d67.json": "cd6a2a4dfadf970ed0bc8067bbd4352fe512b290e97dbb0234e804a980c44013", + "result.json": "cd6a2a4dfadf970ed0bc8067bbd4352fe512b290e97dbb0234e804a980c44013", + "snapshots/api-state.json": "daa7f727dc624e21f5f7168617271a71364f7f0f039da8e0a607d420f329d67b", + "snapshots/browser-diagnostics.json": "effa05d63cbf9b8a92d4df8308a8e555cd630847ad58c71406a55ece0d16d08d", + "snapshots/chat-final-run-ledger.json": "cf892a73243bd2da5276f87c8d5a8ab6b42dceae76145c91e759f21073f12b44", + "snapshots/everyday-workflow.json": "a54d3acefb569e385ae1c656a4d1a0ea142479efef35b5b83c4d9139b2f94181", + "snapshots/fixtures.json": "68f592252222a8a70d3253103840ad6a282f3c68a26f5cf6360a9f03bb8c20b6" + }, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.zip-delivered", + "passed": false + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ] + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-codex.local.hire-reuse", + "source": { + "sha": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "ref": "refs/heads/codex/native-procedure-baseline-v3", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348764875" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "model": "gpt-5.6-sol", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 6, + "startedRunCount": 6, + "runStatuses": { + "succeeded": 6 + }, + "resultSha256": "553c84d9cac87670d5ccac940733f9480791955f4e257f82467bd2295802e8d8", + "evidenceHashes": { + "evidence-manifest.json": "2064b2faa0a710fcffdde7eef98231698bcac82f3b07cc829688c976abd588b7", + "html-report/data/b0521708080dfd9e14c867175d841965fce30834.json": "553c84d9cac87670d5ccac940733f9480791955f4e257f82467bd2295802e8d8", + "playwright-output/runner-everyday-workflows--d934c-the-browser-and-public-APIs/attachments/runner-e2e-result-175a197d38cae1451e9c0d0edc257c25f52b813a.json": "553c84d9cac87670d5ccac940733f9480791955f4e257f82467bd2295802e8d8", + "result.json": "553c84d9cac87670d5ccac940733f9480791955f4e257f82467bd2295802e8d8", + "snapshots/api-state.json": "de1d7d8ebbfb7087c6d74ed0256db7ad7c4a59e64431fd7e53bb097c61dd647e", + "snapshots/browser-diagnostics.json": "a1e96e38e345962123df2513785399aa9675a585717d716be574e2221dbcdcea", + "snapshots/chat-final-run-ledger.json": "d96faa781e4131a912333486d66527d9dc0acd3b959260aadde20c1a7d55535e", + "snapshots/everyday-workflow.json": "f0800a8e169f6b6f4af6e214cea0010d326b6053954125753467a4b70fbd4b61", + "snapshots/fixtures.json": "7e0d747ae4117918ab879363f592ce940773f2aa612f4dcd7b4421395ad6d18e" + }, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ] + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-opencode.local.delegate-feedback", + "source": { + "sha": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "ref": "refs/heads/codex/native-procedure-baseline-v3", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348764875" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "failed", + "originalFailureClass": "candidate_failure", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 4, + "startedRunCount": 4, + "runStatuses": { + "succeeded": 4 + }, + "resultSha256": "622c78df06de7c9c17e5d875d6a7ce5eab07d24691fa8cc07e5e998202bcc410", + "evidenceHashes": { + "evidence-manifest.json": "ef77cb6f49ced2b1192156b098bf4c90c90b4ad008b059bb5c333d01321d9b22", + "html-report/data/921db0ab223237bb50570635a7a6652b5424ff63.json": "622c78df06de7c9c17e5d875d6a7ce5eab07d24691fa8cc07e5e998202bcc410", + "playwright-output/runner-everyday-workflows--616ad-the-browser-and-public-APIs/attachments/runner-e2e-result-5df85a6ef79eb3010b42b9aa7bdd5d68e2d5ae69.json": "622c78df06de7c9c17e5d875d6a7ce5eab07d24691fa8cc07e5e998202bcc410", + "result.json": "622c78df06de7c9c17e5d875d6a7ce5eab07d24691fa8cc07e5e998202bcc410", + "snapshots/api-state.json": "89f5399ae3d320c23ae7aa08650dbe596524ea2de9e98ff18f92ee6d0802d7d4", + "snapshots/browser-diagnostics.json": "bf82e258e3391b191b500cd10105b1f5b7a7895959c5d848e69a24b4f2e3b04f", + "snapshots/chat-final-run-ledger.json": "853e65b05cf46057363b99a7da7fdfa49eb51c30831a51647728f7e14244e7da", + "snapshots/everyday-workflow.json": "ab5ae69afa478a2ae622387acea63bc2f8a1740de89c03f5a91494ef280fa88c", + "snapshots/fixtures.json": "c6f60fd9fcb2fedce2be5f50b6429db062956d352267209ff128dc4aee76d9fd" + }, + "originalChecks": [ + { + "id": "reply-3-stored-once", + "passed": true + }, + { + "id": "one-child", + "passed": true + }, + { + "id": "child-consumed-feedback", + "passed": true + }, + { + "id": "delegated-delivery.one-slugify-source", + "passed": true + }, + { + "id": "delegated-delivery.readme-present", + "passed": true + }, + { + "id": "delegated-delivery.project-tests-present", + "passed": true + }, + { + "id": "delegated-delivery.base-0", + "passed": true + }, + { + "id": "delegated-delivery.base-1", + "passed": true + }, + { + "id": "delegated-delivery.base-2", + "passed": true + }, + { + "id": "delegated-delivery.base-3", + "passed": true + }, + { + "id": "delegated-delivery.base-4", + "passed": true + }, + { + "id": "delegated-delivery.base-5", + "passed": true + }, + { + "id": "delegated-delivery.base-6", + "passed": true + }, + { + "id": "delegated-delivery.base-7", + "passed": true + }, + { + "id": "delegated-delivery.importable-function", + "passed": true + }, + { + "id": "delegated-delivery.length-7", + "passed": true + }, + { + "id": "delegated-delivery.length-6", + "passed": true + }, + { + "id": "delegated-delivery.length-1", + "passed": true + }, + { + "id": "delegated-delivery.reject-zero-length", + "passed": true + }, + { + "id": "feedback-delivered-to-child", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": false + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + }, + { + "id": "workflow-completed", + "passed": false + } + ] + }, + { + "variant": "baseline", + "executionId": "everyday-workflows.runner-opencode.local.hire-reuse", + "source": { + "sha": "ce31f5efd0d416205c3994c55dd02b645f8b8423", + "ref": "refs/heads/codex/native-procedure-baseline-v3", + "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/37348764875" + }, + "suiteDefinitionHash": "acd17b4acbd7051070531f7d7587131139900f54245b992eb7a62faacda8e334", + "harnessDigest": "7fc184b06de3af473980ef1a0ad7124342cfd8d36ca721fe452298e775154cf5", + "attempt": 1, + "originalStatus": "passed", + "originalFailureClass": null, + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "cleanup": "passed", + "resultLedgerRunSetEqual": true, + "apiRunSetEqual": true, + "ledgerCount": 6, + "startedRunCount": 6, + "runStatuses": { + "succeeded": 6 + }, + "resultSha256": "2e2f3d418169af50464e4963b0d8b24b536da50b5fb74196a5ed404799a2a482", + "evidenceHashes": { + "evidence-manifest.json": "1f1c22b1fd6ec09bf20a7a4b02e9dcb80bffff35374b8b7f252caf2d80245713", + "html-report/data/76284c18712671e245f76266f0a2275337bf287b.json": "2e2f3d418169af50464e4963b0d8b24b536da50b5fb74196a5ed404799a2a482", + "playwright-output/runner-everyday-workflows--90607-the-browser-and-public-APIs/attachments/runner-e2e-result-c90198d41e9925e738230e788c65a93ed8d33a09.json": "2e2f3d418169af50464e4963b0d8b24b536da50b5fb74196a5ed404799a2a482", + "result.json": "2e2f3d418169af50464e4963b0d8b24b536da50b5fb74196a5ed404799a2a482", + "snapshots/api-state.json": "310afec9ef9767b36d3968cbf013e75c3a892a0169226e6fc243c2552337da95", + "snapshots/browser-diagnostics.json": "b08a66f0f71cf2fc46c79b2b0ff182d9bb36831daffffef587039ce8418de392", + "snapshots/chat-final-run-ledger.json": "4d2b9a76c2c44b2251dfde036a6107bc95f607d95b534b77b562f696b47cafe2", + "snapshots/everyday-workflow.json": "551c6a37b971e6bd56930aa4d226edf7e4a985d33b1213fd5b914019d5e0409e", + "snapshots/fixtures.json": "207a9015bb726a98e4bca348e7e3048e2c94e2fed69ae6694a6d63977e318d23" + }, + "originalChecks": [ + { + "id": "exactly-one-hire", + "passed": true + }, + { + "id": "manager-correct", + "passed": true + }, + { + "id": "hire-native-connection", + "passed": true + }, + { + "id": "hired-agent-executed", + "passed": true + }, + { + "id": "hired-delivery.one-slugify-source", + "passed": true + }, + { + "id": "hired-delivery.readme-present", + "passed": true + }, + { + "id": "hired-delivery.project-tests-present", + "passed": true + }, + { + "id": "hired-delivery.base-0", + "passed": true + }, + { + "id": "hired-delivery.base-1", + "passed": true + }, + { + "id": "hired-delivery.base-2", + "passed": true + }, + { + "id": "hired-delivery.base-3", + "passed": true + }, + { + "id": "hired-delivery.base-4", + "passed": true + }, + { + "id": "hired-delivery.base-5", + "passed": true + }, + { + "id": "hired-delivery.base-6", + "passed": true + }, + { + "id": "hired-delivery.base-7", + "passed": true + }, + { + "id": "hired-delivery.importable-function", + "passed": true + }, + { + "id": "reply-5-stored-once", + "passed": true + }, + { + "id": "hire-reused", + "passed": true + }, + { + "id": "hired-agent-executed-revision", + "passed": true + }, + { + "id": "reused-delivery.one-slugify-source", + "passed": true + }, + { + "id": "reused-delivery.readme-present", + "passed": true + }, + { + "id": "reused-delivery.project-tests-present", + "passed": true + }, + { + "id": "reused-delivery.base-0", + "passed": true + }, + { + "id": "reused-delivery.base-1", + "passed": true + }, + { + "id": "reused-delivery.base-2", + "passed": true + }, + { + "id": "reused-delivery.base-3", + "passed": true + }, + { + "id": "reused-delivery.base-4", + "passed": true + }, + { + "id": "reused-delivery.base-5", + "passed": true + }, + { + "id": "reused-delivery.base-6", + "passed": true + }, + { + "id": "reused-delivery.base-7", + "passed": true + }, + { + "id": "reused-delivery.importable-function", + "passed": true + }, + { + "id": "reused-delivery.separator-_", + "passed": true + }, + { + "id": "reused-delivery.separator--", + "passed": true + }, + { + "id": "reused-delivery.reject-invalid-separator", + "passed": true + }, + { + "id": "submitted-replies-consumed", + "passed": true + }, + { + "id": "parent-finishes-after-child", + "passed": true + }, + { + "id": "native-model-config", + "passed": true + }, + { + "id": "native-terminal-contract", + "passed": true + }, + { + "id": "tasks-done", + "passed": true + }, + { + "id": "settled", + "passed": true + }, + { + "id": "native-runtime", + "passed": true + }, + { + "id": "successful-runs", + "passed": true + }, + { + "id": "bounded-work", + "passed": true + }, + { + "id": "parent-owned-by-lead", + "passed": true + }, + { + "id": "no-pending-bookkeeping", + "passed": true + } + ] + } + ], + "retainedStartedRuns": 56, + "missingCell": { + "variant": "baseline", + "executionId": "everyday-workflows.runner-codex.local.delegate-feedback", + "jobId": 111896338920, + "observedFailure": "Pinned Python oracle preparation: docker version timed out after 30 seconds (exit 124); Run paid cell was skipped; no result artifact was produced.", + "behavioralGrade": null, + "setupLogSha256": "b2c2502f9aff687facdff805834ffd1d6ad5fb0e1b9840749891dbab1583553b" + }, + "limits": [ + "Single trials do not establish causation, equivalence, speed or cost trends.", + "The OpenCode candidate hiring credential-persistence guard failed. Its original provider database was removed by cleanup; the credential type and exact persistence cause are not retained.", + "Only eleven final result/API/ledger sets are available. The missing setup cell is not passed or behaviorally failed.", + "Raw sessions, reasoning, run identifiers, and numeric billing subtotals are excluded. Actual charges remain unknown." + ] +} diff --git a/package.json b/package.json index 1d7fe58f88d..a54a7f4456b 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "test:runner-recovery": "vitest run server/src/services/native-runtime/native-replacement-evidence.test.ts server/src/services/native-runtime/stopped-codex-turn.test.ts server/src/services/native-runtime/native-safe-replacement.test.ts", "test:e2e:browser-context": "playwright test --config tests/e2e/playwright-company-context.config.ts", "test:e2e:runner:browser-support": "playwright test --config tests/runner-e2e/playwright-support.config.ts", - "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts", + "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts && node --test tests/runner-e2e/native-completion-checks.test.mjs tests/runner-e2e/native-completion-git-source.test.mjs tests/runner-e2e/native-completion-source-contract.test.mjs", "test:e2e:runner:stock-harness": "node tests/runner-e2e/stock-harness-checks.mjs", "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", diff --git a/server/src/__tests__/native-procedure-measurement.test.ts b/server/src/__tests__/native-procedure-measurement.test.ts new file mode 100644 index 00000000000..09c0b2308b3 --- /dev/null +++ b/server/src/__tests__/native-procedure-measurement.test.ts @@ -0,0 +1,161 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { afterAll, describe, expect, it } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; +import { buildNativeModelEnvelope, parseNativeExecutionInput, type NativeExecutionInput } from "../../../packages/paperclip-runner/src/contracts/native-execution.js"; +import type { PersistedNativeSession } from "../../../packages/paperclip-runner/src/contracts/native-session-backend.js"; +import { NATIVE_RUNTIME_ASSET_SCHEMA, PAPERCLIP_EXECUTION_PROMPT, PAPERCLIP_EXECUTION_PROMPT_REVISION, canonicalNativeRuntimeContextDigest, nativeRuntimePromptDigest } from "../../../packages/paperclip-runner/src/contracts/runtime-context.js"; +import { createRunnerdNativeSessionBackend } from "../../../packages/paperclip-runner/src/backends/codex-native-backend.js"; +import { resolveQualifiedAcpxProfile } from "../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; +import { FakeCodexTransport, WORKSPACE } from "../../../packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.js"; +import { startOpenCodeMcpBridge } from "../../../packages/paperclip-runner/src/drivers/opencode/mcp-bridge.js"; + +// Production catalog selection, scripted provider transport: this measures what +// Paperclip delivers, not the vendor's private prompt, tokenizer or billing. +const root = new URL("../../../", import.meta.url); +const sha = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); +const identity = { companyId: "company", agentId: "agent", issueId: "issue", runId: "run", sessionId: "session" }; +const tools = new PaperclipRunnerToolAuthority({} as Db, { + ...identity, apiToolsEnabled: true, workspaceRoot: WORKSPACE, +}).definitions(); +const { qualificationModel: _model, reportedModelId: _reportedModel, permissionPolicy: _policy, ...profile } = resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"); +const providers: NativeExecutionInput["provider"][] = [ + { kind: "codex", model: "gpt-5.6-sol", approvalPolicy: "never" }, + { kind: "acpx", agent: "claude", model: "claude-sonnet-5", permissionMode: "approve-all", profile }, + { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", permissionMode: "allow" }, +]; +const receipts: unknown[] = []; +let mcpReceipt: unknown; +function measure(value: unknown) { + const json = JSON.stringify(value); + const normalized = json.replaceAll(WORKSPACE, "/workspace"); + return { + utf8Bytes: Buffer.byteLength(json), characters: json.length, sha256: sha(json), + normalizedComparison: { utf8Bytes: Buffer.byteLength(normalized), characters: normalized.length, sha256: sha(normalized) }, + }; +} +function execution(provider: NativeExecutionInput["provider"]): NativeExecutionInput { + const entry = readFileSync(new URL("server/src/onboarding-assets/default/AGENTS.md", root), "utf8"); + writeFileSync(`${WORKSPACE}/AGENTS.md`, entry); + const context = { + prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() }, + instructions: { entryPath: "AGENTS.md", bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, rootPath: WORKSPACE, digest: sha(entry), manifestDigest: sha(entry), fileCount: 1, totalBytes: Buffer.byteLength(entry) } }, + skills: [], mcp: { assignmentSetId: "none", digest: "0".repeat(64), bindingId: null }, + }; + return parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v5", binding: { companyId: identity.companyId, agentId: identity.agentId, issueId: identity.issueId, runId: identity.runId, executionWorkspaceId: "workspace" }, + task: { identifier: "TASK-1", title: "Review delegated work", description: "Review delegated work", prompt: "Review delegated work.", workMode: "standard" }, + workspace: { cwd: WORKSPACE, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: "session", driverKind: provider.kind === "acpx" ? "acpx_runtime" : provider.kind === "opencode" ? "opencode_server" : "codex_app_server", protocolVersion: 1 }, + provider, executionMode: "default", planningContext: null, + completionContract: { id: "contract", sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.completion-contract.v1", contract: { revision: "1", objective: "Review delegated work", criteria: [{ id: "objective", requirement: "Review complete" }] } }, + interactionResponses: [], credentialBindings: [], + runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) }, + }); +} +function transport(provider: NativeExecutionInput["provider"]) { + return new FakeCodexTransport("thread-1", "provider-session-1", provider.kind === "acpx" ? { + kind: "acpx", normalizedSessionId: "session", acpxRecordId: "fixture-record", backendSessionId: "fixture-backend", + agentSessionId: "fixture-agent", profileDigest: `sha256:${"a".repeat(64)}`, workspaceDigest: `sha256:${"b".repeat(64)}`, + requestedModel: provider.model, effectiveModel: provider.model, permissionMode: "approve-all", + providerLifetimeFenceCandidates: [60001, 60002, 60003], + } : undefined); +} + +describe("production native procedure instruction measurement", () => { + it("retains delivered serialized bytes separately from path-normalized comparison", () => { + const value = { cwd: WORKSPACE, text: "λ" }; + expect(measure(value).utf8Bytes).toBe(Buffer.byteLength(JSON.stringify(value))); + expect(measure(value).normalizedComparison.utf8Bytes).toBe(Buffer.byteLength(JSON.stringify({ ...value, cwd: "/workspace" }))); + expect(measure(value).sha256).not.toBe(measure(value).normalizedComparison.sha256); + }); + + it("uses the real standard-mode authority, including optional and connection tools", () => { + expect(tools.map(tool => tool.name)).toEqual(expect.arrayContaining([ + "hire_agent", "list_agents", "create_task", "set_dependencies", "connections_search", "connection_request", "register_deliverable", + ])); + expect(new Set(tools.map(tool => tool.name)).size).toBe(tools.length); + const withoutApi = new PaperclipRunnerToolAuthority({} as Db, { ...identity, apiToolsEnabled: false }).definitions(); + expect(withoutApi.some(tool => tool.name === "hire_agent")).toBe(false); + }); + + for (const provider of providers) it(`captures the full ${provider.kind} start/resume/continuation payload`, async () => { + const input = execution(provider); + let checkpoint: PersistedNativeSession | undefined; + for (const phase of ["start", "resume", "continuation"] as const) { + const captured = transport(provider); + captured.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, turns: [] } }; + const current = phase === "continuation" ? parseNativeExecutionInput({ ...input, continuationPrompt: "Review the teammate's saved result." }) : input; + const backend = createRunnerdNativeSessionBackend(current, { transportFactory: () => captured, dynamicTools: tools, environment: { PAPERCLIP_WORKSPACE_CWD: WORKSPACE } }); + const recovery = checkpoint ? await backend.recoverSession!(checkpoint, { signal: new AbortController().signal }) : null; + if (recovery) expect(recovery.recovered).toBe(true); + const session = recovery?.session ?? await backend.openSession({ identity, workingDirectory: WORKSPACE }); + try { + checkpoint ??= await session.snapshot(); + const envelope = buildNativeModelEnvelope(current, phase === "continuation" ? { resumedSession: true } : undefined); + if ("requestedSkills" in envelope && backend.preparedTaskConstraints) envelope.constraints = [...backend.preparedTaskConstraints]; + await session.startTurn({ message: { role: "user", text: JSON.stringify(envelope) } }); + const setup = captured.calls.find(call => call.method === (phase === "start" ? "thread/start" : "thread/resume"))!.params; + const turn = captured.calls.find(call => call.method === "turn/start")!.params; + const instructions = setup.developerInstructions ?? setup.baseInstructions; + const delivered = setup.dynamicTools as Record[]; + expect(delivered.filter(tool => !["paperclip_finish", "paperclip_block"].includes(String(tool.name)))).toEqual(tools); + expect(delivered).toHaveLength(tools.length + 2); + expect(instructions).toContain(PAPERCLIP_EXECUTION_PROMPT); + expect(JSON.parse((turn.input as {text: string}[])[0]!.text)).toEqual(envelope); + receipts.push({ provider: provider.kind, phase, instructions: measure(instructions), tools: measure(delivered), input: measure(turn.input), + fullProjection: measure({ instructions, tools: delivered, input: turn.input }), deliveredToolCount: delivered.length }); + } finally { await session.close({ reason: "measurement" }); } + } + }); + + it("preserves the full catalog through the actual OpenCode MCP tools/list bridge", async () => { + const bridge = await startOpenCodeMcpBridge({ tools, handler: async () => { throw new Error("No tool execution in measurement"); } }); + try { + const response = await fetch(bridge.url, { method: "POST", headers: { authorization: `Bearer ${bridge.secret}`, "content-type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }) }); + expect(response.ok).toBe(true); + const result = await response.json() as { result: { tools: Record[] } }; + expect(result.result.tools.filter(tool => !["paperclip_finish", "paperclip_block"].includes(String(tool.name)))).toEqual( + tools.map(({ name, description, inputSchema }) => ({ name, description, inputSchema })), + ); + mcpReceipt = { boundary: "authenticated loopback MCP tools/list; no OpenCode/provider process", serverAnnotationsForwarded: false, tools: measure(result.result.tools), count: result.result.tools.length }; + } finally { await bridge.close(); } + }); +}); + +afterAll(() => { + const output = process.env.PAPERCLIP_NATIVE_PROCEDURE_MEASUREMENT; + if (!output) return; + if (receipts.length !== 9 || !mcpReceipt) throw new Error("Refusing incomplete instruction measurement"); + const paths = [ + "server/src/onboarding-assets/default/AGENTS.md", + "packages/paperclip-runner/src/contracts/runtime-context.ts", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts", + "packages/paperclip-runner/src/protocol-actions/create-task.ts", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts", + "packages/paperclip-runner/generated/semantic-action-catalog.json", + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json", + "packages/paperclip-runner/protocol/manifest.json", + "packages/paperclip-runner/src/backends/codex-native-backend.ts", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts", + "server/src/services/native-runtime/native-session-resume.ts", + "server/src/__tests__/native-procedure-measurement.test.ts", + ]; + writeFileSync(output, `${JSON.stringify({ schema: "paperclip.native-procedure-measurement.v2", + sourceSha: execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(), + sourceDirty: execFileSync("git", ["status", "--porcelain"], { cwd: root, encoding: "utf8" }).trim().length > 0, + measuredInputsDirty: execFileSync("git", ["status", "--porcelain", "--", ...paths], { cwd: root, encoding: "utf8" }).trim().length > 0, + sourceHashes: Object.fromEntries(paths.map(path => [path, sha(readFileSync(new URL(path, root)))])), + boundary: "scripted runnerd RPC with production standard-mode authority, API tools enabled, local workspace, no assigned external apps", + byteAccounting: "utf8Bytes counts the unmodified JSON serialization of each extracted component/projection; normalizedComparison substitutes only the fixture workspace path. Neither is the entire transport request.", + providerCalls: 0, tokenCount: null, upstreamLoadingOrTruncation: "unverified", vendorStockPrompt: "unavailable", + toolDescriptions: tools.map(tool => ({ name: tool.name, characters: String(tool.description).length, utf8Bytes: Buffer.byteLength(String(tool.description)), inputSchema: measure(tool.inputSchema) })), + receipts, mcpReceipt, + }, null, 2)}\n`); +}); diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index cec2fe6fb5e..675c4ed8db8 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -40,7 +40,9 @@ See [controlled recovery tests](../runner-recovery/README.md). The local matrix has fourteen cases on native Codex `gpt-5.6-sol`, native ACPX Claude `claude-sonnet-5`, and native Codex `gpt-5.4-mini`: 42 cells. The two core profiles also declare build/revise, delegation, controller-restart, and skill-creation cases -on Daytona: eight cells. Remote runner-process killing is not supported. For remote controller +on Daytona: eight cells. OpenCode adds only local hiring/reuse and delegation, +for 52 cells total. Hiring/reuse and delegation have one attempt and a +1,000-cent company and lead-agent hard stop. Remote runner-process killing is not supported. For remote controller restart, a verified first download supplies the persistence checkpoint; the controller is interrupted during a subsequent revision with another queued requirement. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 397f33da086..5feba8909d3 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -12,6 +12,31 @@ scheduled execution gets a fresh Paperclip home, embedded Postgres database, instance configuration, port, workspace, company, encrypted secrets, environment, and agent. +## Native procedure guidance comparison (explicit only) + +Select `--suite everyday-workflows --environment local --case hire-reuse --case delegate-feedback +--profile runner-codex --profile runner-acpx-claude --profile runner-opencode` for the six comparison +cells on native Codex, ACPX Claude, and OpenCode. +Both variants use the same Studio Lead persona, original user requests, +artifact oracle, lifecycle checks, models and permissions. Each cell permits +one attempt, a 12-minute deadline, at most 12 story run records, and a +1,000-cent company hard stop; the lead also has a 1,000-cent hard stop. Worker +runs count toward the company budget. The suite is excluded from `--all`. + +Compare frozen branches on the same master with identical fixture sources. +Retain each original grade, source SHA, harness digest, actual run inventory, +downloaded artifacts and partial cost evidence. Check hiring identity/reuse, +worker ownership, delivered revisions, dependency release and parent completion +ordering independently of aggregate grades. These bounded stories do not +qualify every existing-blocker combination or arbitrary provider resume. + +The provider-free `native-procedure-measurement.test.ts` uses the server's real +tool authority and captures scripted start/resume/continuation delivery plus +the OpenCode MCP catalog. It includes descriptions and argument schemas. Its +byte counts are not model token counts or proof of an upstream harness's lazy +loading/truncation. The older completion measurement used a partial catalog; +do not use it as a full production tool-payload baseline. + The vocabulary is: a **campaign** is one workflow invocation against one SHA; a **suite** is a durable testing purpose; a **matrix** is that suite's profiles × environments × cases; an **execution/cell** is one parallel job; and an diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 8f44a3a160b..8e764f9f015 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,10 +136,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, + 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(476); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(476); + expect(validateRunnerCatalog()).toHaveLength(478); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(478); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index cc37562702b..2943c23efe6 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1191,12 +1191,21 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ { id: "everyday-workflows", label: "Everyday Paperclip Work", manualOnly: true, description: "Real user requests, useful downloaded work, and durable continuation using production instructions.", - groups: ["native"], profiles: everydayProfiles, environments: [localEnvironment, daytonaWarmEnvironment], - tasks: everydayTasks, expectedMatrixSize: 50, + groups: ["native"], profiles: [...everydayProfiles, + productionStoryProfile(runnerProfiles.find(profile => profile.id === "runner-opencode")!), + ], environments: [localEnvironment, daytonaWarmEnvironment], + tasks: everydayTasks.map(task => ["hire-reuse", "delegate-feedback"].includes(task.id) + ? { ...task, automaticRetryPolicy: "single_attempt" as const } : task), expectedMatrixSize: 52, excludedExecutionIds: [...everydayProfiles.flatMap(profile => everydayTasks .filter(task => !["build-revise", "delegate-feedback", "recover-controller", "create-skill-studio"].includes(task.id)) - .map(task => `everyday-workflows.${profile.id}.daytona.${task.id}`))], - definitionMetadata: { version: 4, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only" }, + .map(task => `everyday-workflows.${profile.id}.daytona.${task.id}`)), + ...everydayTasks.flatMap(task => [ + `everyday-workflows.runner-opencode.daytona.${task.id}`, + ...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]), + ]), + ], + definitionMetadata: { version: 6, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } }, }, { id: NATIVE_INSTRUCTION_SUITE, label: "Native completion instruction consolidation", manualOnly: true, diff --git a/tests/runner-e2e/everyday.test.ts b/tests/runner-e2e/everyday.test.ts index 3afd8735fa9..88cbecddd72 100644 --- a/tests/runner-e2e/everyday.test.ts +++ b/tests/runner-e2e/everyday.test.ts @@ -212,7 +212,7 @@ describe("manual everyday workflow catalog", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "everyday-workflows"]), ); - expect(selected).toHaveLength(50); + expect(selected).toHaveLength(52); expect(selected.every((e) => e.profile.generation === "native")).toBe(true); expect( selectRunnerExecutions( diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index 04be71d847f..a2c231dc3a3 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -4,6 +4,29 @@ import { runnerMatrix } from "./catalog.js"; import { setupLiveFixtures } from "./live-fixtures.js"; describe("live runner fixtures", () => { + it.each(["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile => + ["hire-reuse", "delegate-feedback"].map(task => [profile, task]))) ( + "applies %s %s budget stops in actual setup with a nonce and only its selected credential", async (profile, task) => { + const execution = runnerMatrix.find(row => row.id === `everyday-workflows.${profile}.local.${task}`)!; + let agentBody: any; + let companyBody: any; + const api = { + async get() { return [{ id: "local", driver: "local" }]; }, + async postSensitive(url: string) { return url.endsWith("/ai-connections") ? { connectionId: "account" } : { id: "secret" }; }, + async post(url: string, data: any) { + if (url === "/api/companies") { companyBody = data; return { id: "company", name: "Test" }; } + if (url.endsWith("/agents")) { agentBody = data; return { id: "lead", ...data }; } + throw new Error(`Unexpected POST ${url}`); + }, + } as unknown as RunnerApi; + const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "random-nonce", workspacePath: "/tmp/test", + credentials: { [execution.profile.credential]: "test-value" } }); + expect(companyBody.budgetMonthlyCents).toBe(1_000); + expect(agentBody.budgetMonthlyCents).toBe(1_000); + await fixtures.teardown(); + }, + ); + it.each(["runner-codex", "legacy-codex", "runner-acpx-claude", "legacy-opencode"])( "creates a production-default %s hire with company and agent budget stops", async (profile) => { const execution = runnerMatrix.find(row => row.suite.id === "stock-harness" && row.profile.id === profile)!; @@ -51,6 +74,7 @@ describe("live runner fixtures", () => { ["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"], ["runner-codex", "everyday-workflows", "hire-reuse"], ["runner-acpx-claude", "everyday-workflows", "hire-reuse"], + ["runner-opencode", "everyday-workflows", "hire-reuse"], ["runner-codex", "agent-chat-hardening", "hire-delegate-reuse"], ["runner-acpx-claude", "agent-chat-hardening", "hire-delegate-reuse"], ])( @@ -64,7 +88,7 @@ describe("live runner fixtures", () => { e.environment.id === "local", )!; const provider = - profile === "runner-acpx-claude" ? "anthropic" : "openai"; + profile === "runner-acpx-claude" ? "anthropic" : profile === "runner-opencode" ? "openrouter" : "openai"; let connected = false; let agentBody: any; const api = { @@ -101,8 +125,7 @@ describe("live runner fixtures", () => { executionNonce: "nonce", workspacePath: "/tmp/test", credentials: { - OPENAI_API_KEY: "test-value", - ANTHROPIC_API_KEY: "test-value", + [execution.profile.credential]: "test-value", }, }); expect(connected).toBe(true); diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index ac5ce1bdc61..96ccc5025e7 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -39,7 +39,7 @@ interface AgentRecord { interface ManagedAccountFixture { connectionId: string; binding: { - provider: "openai" | "anthropic"; + provider: "openai" | "anthropic" | "openrouter"; method: "api_key"; mode: "responsible_user"; }; @@ -136,7 +136,8 @@ export async function setupLiveFixtures(input: { return api.post("/api/companies", { name: `Runner E2E ${execution.id} ${input.executionNonce}`, description: "Ephemeral paid full-stack runner acceptance fixture", - budgetMonthlyCents: ["native-completion", "native-instruction-consolidation"].includes(execution.suite.id) ? NATIVE_COMPLETION_BUDGET_CENTS + budgetMonthlyCents: ["native-completion", "native-instruction-consolidation"].includes(execution.suite.id) + || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS : execution.suite.id === "stock-harness" ? 1_000 : 0, }); @@ -244,10 +245,11 @@ export async function setupLiveFixtures(input: { dependencies: ["company"], async setup(resolved) { const company = value(resolved, "company"); - const provider = - execution.profile.provider === "acpx" ? "anthropic" : "openai"; - const key = - provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; + const key = execution.profile.credential; + const provider = key === "ANTHROPIC_API_KEY" ? "anthropic" + : key === "OPENROUTER_API_KEY" ? "openrouter" + : key === "OPENAI_API_KEY" ? "openai" : null; + if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`); const apiKey = input.credentials[key]; if (!apiKey) throw new Error(`Missing credential ${key}`); const account = await api.postSensitive<{ connectionId: string }>( @@ -290,7 +292,10 @@ export async function setupLiveFixtures(input: { secretRefs, executionId: input.executionNonce, }); - if (execution.suite.id === "stock-harness") agent.budgetMonthlyCents = 1_000; + if (execution.suite.id === "stock-harness" + || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) { + agent.budgetMonthlyCents = 1_000; + } if (managedHiring) { const account = value(resolved, "ai-connection"); const config = agent.adapterConfig as Record; diff --git a/tests/runner-e2e/native-instruction-consolidation.test.ts b/tests/runner-e2e/native-instruction-consolidation.test.ts index 6ba5ea2819f..407ed385ed1 100644 --- a/tests/runner-e2e/native-instruction-consolidation.test.ts +++ b/tests/runner-e2e/native-instruction-consolidation.test.ts @@ -69,7 +69,10 @@ describe("native instruction comparison admission", () => { const files = Object.keys(NATIVE_INSTRUCTION_VARIANTS.baseline); const baseline = new Map(files.map(file => [file, execFileSync("git", ["show", `${NATIVE_INSTRUCTION_BASE_SHA}:${file}`])])); expect(nativeInstructionVariant(file => baseline.get(file)!)).toBe("baseline"); - const candidate = new Map(files.map(file => [file, readFileSync(new URL(`../../${file}`, import.meta.url))])); + // This historical admission intentionally rejects later production changes. + // Exercise the retained qualified tree rather than requiring HEAD to remain + // one of this completed comparison's frozen variants forever. + const candidate = new Map(files.map(file => [file, execFileSync("git", ["show", `16ef1a5744b23bb043a8b286e6288921da1de98f:${file}`])])); const currentVariant = nativeInstructionVariant(file => candidate.get(file)!); expect(["baseline", "candidate", "corrected", "feedback", "opencodeFeedback", "opencodeFeedbackSettlement", "opencodeFeedbackResponses"]).toContain(currentVariant); candidate.set(files[0]!, currentVariant === "candidate" ? baseline.get(files[0]!)! : Buffer.from("unknown source")); diff --git a/tests/runner-e2e/native-procedure-guidance.test.ts b/tests/runner-e2e/native-procedure-guidance.test.ts new file mode 100644 index 00000000000..f84db225a10 --- /dev/null +++ b/tests/runner-e2e/native-procedure-guidance.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix, runnerSuites } from "./catalog.js"; +import { everydayTasks } from "./everyday-cases.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; + +describe("bounded native procedure comparison", () => { + const suite = runnerSuites.find(suite => suite.id === "everyday-workflows")!; + it("selects only two original journeys on three native profiles, with no automatic retry", () => { + expect(suite.manualOnly).toBe(true); + const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", suite.id, + "--profile", "runner-codex", "--profile", "runner-acpx-claude", "--profile", "runner-opencode", + "--case", "hire-reuse", "--case", "delegate-feedback", "--environment", "local"])); + expect(selected).toHaveLength(6); + for (const { task } of selected) { + const original = everydayTasks.find(candidate => candidate.id === task.id)!; + expect(task).toEqual({ ...original, automaticRetryPolicy: "single_attempt" }); + expect(task.buildPrompt("nonce")).toBe(original.buildPrompt("nonce")); + expect(task.attemptTimeoutMs.local).toBe(720_000); + } + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"]), runnerMatrix).some(cell => cell.suite.id === suite.id)).toBe(false); + }); + it("keeps procedure and completion directions out of the fixture persona", () => { + for (const profile of suite.profiles) { + const input = { executionId: "random-nonce", environmentId: "env", environmentFixtureId: "local" as const, workspacePath: "/workspace", secretRefs: { + [profile.credential]: { type: "secret_ref" as const, secretId: "00000000-0000-4000-8000-000000000001", version: "latest" as const }, + } }; + const agent = profile.buildAgent(input); + expect(JSON.stringify(agent.instructionsBundle)).not.toMatch(/paperclip_finish|paperclip_block|set_dependencies|hire_agent/); + } + }); +}); diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index fd13ba8fd34..7d0dcdbc0fe 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -1,10 +1,12 @@ -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { name: "runner-e2e-support", include: ["**/*.test.ts", "**/*.test.mjs"], + // These use node:test and run separately in test:e2e:runner:unit. + exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs"], testTimeout: 30_000, }, });