diff --git a/doc/plans/2026-10-02-hiring-template-live-comparison.json b/doc/plans/2026-10-02-hiring-template-live-comparison.json new file mode 100644 index 00000000000..d0906af81be --- /dev/null +++ b/doc/plans/2026-10-02-hiring-template-live-comparison.json @@ -0,0 +1,1028 @@ +{ + "schema": "paperclip.hiring-templates.live-comparison.v1", + "state": "dispatched_no_graded_cells", + "measuredAt": "2026-10-02T23:02:00Z", + "mergedHiringPr": "https://github.com/paperclipai/paperclip/pull/14985", + "mergedHiringSha": "862a5758ba0e88a33232c1f1fa645e85c38a3113", + "sourceProof": { + "schema": "paperclip.hiring-templates.tracked-source-match.v1", + "candidateSha": "9f5404ad3aacbe76777952759414d34fd381e674", + "baselineSha": "296a4df85e8bcc97a160fc78c291b17adb828196", + "commonTrackedFiles": 8242, + "commonGitBlobManifestSha256": "7a45cb7541c9deb26152662747563df9b0d5021f369e52861ae7bb50bdf208dd", + "differingFiles": [ + "doc/plans/2026-10-02-hiring-matched-baseline-source.json", + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/TEAM.md", + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/agents/ceo/AGENTS.md", + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/agents/cto/AGENTS.md", + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/agents/qa/AGENTS.md", + "packages/teams-catalog/catalog/bundled/product/product-design/TEAM.md", + "packages/teams-catalog/catalog/bundled/product/product-design/agents/ux-designer/AGENTS.md", + "packages/teams-catalog/catalog/bundled/software-development/product-engineering/agents/cto/AGENTS.md", + "packages/teams-catalog/catalog/bundled/software-development/product-engineering/agents/qa/AGENTS.md", + "packages/teams-catalog/catalog/bundled/software-development/product-engineering/agents/senior-coder/AGENTS.md", + "packages/teams-catalog/generated/catalog.json", + "server/src/onboarding-assets/ceo/AGENTS.md", + "server/src/onboarding-assets/first-task/chief-of-staff/AGENTS.md", + "server/src/services/default-agent-instructions.ts", + "skills/paperclip-create-agent/SKILL.md", + "skills/paperclip-create-agent/references/agent-instruction-templates.md", + "skills/paperclip-create-agent/references/agents/coder.md", + "skills/paperclip-create-agent/references/agents/qa.md", + "skills/paperclip-create-agent/references/agents/securityengineer.md", + "skills/paperclip-create-agent/references/agents/uxdesigner.md", + "skills/paperclip-create-agent/references/baseline-role-guide.md", + "skills/paperclip-create-agent/references/draft-review-checklist.md" + ], + "fixtureDigest": "4d7f18420889325af89aa48eaaf021e8178aa6d26d504b92041635a15741f004", + "intentionalProductionHashes": { + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/TEAM.md": { + "candidate": "e0c38e9af1a5dd1888fb81a72feba75a9ab1be4c78d8ceb658e2e5cd621c5096", + "baseline": "4ab4f57f147e89a56a6753755d7ef142d91741b3c0159f2e53fb333cd2779eaf" + }, + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/agents/ceo/AGENTS.md": { + "candidate": "75848c6f794646a93e23bc73b80d31ef0eb8b83c105423eb98ea29e7685d2fd0", + "baseline": "50f580c51807762265f09c6fe7bf32d7701d64caeb30595bfd577b2f5087c21b" + }, + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/agents/cto/AGENTS.md": { + "candidate": "d33bfdecdf2edcedde25599c72c383445365c595430109469a3919bf9cb12a3d", + "baseline": "596e2e9fe08d7368b3b0cfe72c2635ea80444d92a2c0414157bfb0a17773fcaf" + }, + "packages/teams-catalog/catalog/bundled/company-defaults/core-exec-team/agents/qa/AGENTS.md": { + "candidate": "8b7fbdd93c0ecebddc1c87fb87d201340bd5f90297015da6cacd1681e7be70b8", + "baseline": "2a139b88b4b98516048395c569c1bc74fd31cdc2da65db410e9779b8d92c0343" + }, + "packages/teams-catalog/catalog/bundled/product/product-design/TEAM.md": { + "candidate": "f950f4f77a26afa458ca0696994b51353a6bcfac81471fdc00c05f63950fcac5", + "baseline": "205be8d3ef6c1f5e73729207913c387eee4cc9379191c3ce6d943fcf9ca2545d" + }, + "packages/teams-catalog/catalog/bundled/product/product-design/agents/ux-designer/AGENTS.md": { + "candidate": "301f2c7ed92bdd98036c06530588c77d74c3a3c6a8b9c724a3d35fee1142af6f", + "baseline": "f5a4b5696d8249e9f9d808a4373c2a1e6b07faa463e72f83fd9d5bed98d2022f" + }, + "packages/teams-catalog/catalog/bundled/software-development/product-engineering/agents/cto/AGENTS.md": { + "candidate": "28849cb94cb506c4437fa4f9fa19dccb3e29b16175a9b99204a006dac721da48", + "baseline": "0a1a73ec9e5008056bca0a26325131ded2fcf4c4680a5014ba1e13400b32c173" + }, + "packages/teams-catalog/catalog/bundled/software-development/product-engineering/agents/qa/AGENTS.md": { + "candidate": "577f585f6db5271cb7335861debe91fc3201fba9e1565e32611e957ddfab8e05", + "baseline": "2d5738f831aa772befa3471c844a0c9e611584be668e3ec82580ca9e92dc13fc" + }, + "packages/teams-catalog/catalog/bundled/software-development/product-engineering/agents/senior-coder/AGENTS.md": { + "candidate": "e03e0eef79adc04b4ee558ac920bf2ef0581e9494510b7a12d5f1aea70858f18", + "baseline": "b61071518755d9e584c099ae2082c7a78448d09586093ee91393082fe10872b5" + }, + "packages/teams-catalog/generated/catalog.json": { + "candidate": "e924b338b5bf24004eb57ca6a60daa7ae7e9c06b277e64ac8887e1379f16d492", + "baseline": "28384fe47f2a24eee2f24f662bc3ac9725307df9c3b4f3106155c51a82e6db1c" + }, + "server/src/onboarding-assets/ceo/AGENTS.md": { + "candidate": "780fc699350b54471d8ff071716973b5cda184fbd202185f030db6fdc259c879", + "baseline": "089ef9fc2f12934c882910ae35df048597729ee84d437a75d88ce8100c9f1862" + }, + "server/src/onboarding-assets/first-task/chief-of-staff/AGENTS.md": { + "candidate": "d568332c7209f19d03c89faba5c5405dd2da2c4edc9a1b87c41ed90aea75859d", + "baseline": "986e1607ccd1faa80e2269afee1fc92e5b26eb89c8e76e38ab13b80ecd93f062" + }, + "server/src/services/default-agent-instructions.ts": { + "candidate": "8bd3b59f900c132db5ed89b96524f35ba7c8dafb9d58db1dac888430f08c4d63", + "baseline": "5350ea86587087135b2ef46962d07a23a88078e03d67b234ecbfbefd7aa0e023" + }, + "skills/paperclip-create-agent/SKILL.md": { + "candidate": "00e3063a40bbe32592171fa613956bc1b503fe0a41c4973c8dae7b8cdf8c148a", + "baseline": "66b935b273e2fd2f0abedc960b72c2063656c457633ac1bcc943b5fb274539f4" + }, + "skills/paperclip-create-agent/references/agent-instruction-templates.md": { + "candidate": "9ce87fc420b41e951cab664655886d0ee50fa253cb5b8647b4a7afa4a6082d2d", + "baseline": "45787eaf0b7b751c5684518b73219b154af32b9cc23f99cbab539b365b83da6c" + }, + "skills/paperclip-create-agent/references/agents/coder.md": { + "candidate": "e04a25fbc8bb29910b2f4a3719730b8840553b0524ce1ec570617701aca20650", + "baseline": "766c6f5db907f3dc2e317d540feb80e77358202c59d759449228a32450fa6202" + }, + "skills/paperclip-create-agent/references/agents/qa.md": { + "candidate": "2f1fc8f4c082ccac9f0a0428c8fba8e12fc50405dc39f3d3976da0afd2f25676", + "baseline": "405119037d216deaa6476b0f21b69329115dfee9d10064b6820973a91c05bf65" + }, + "skills/paperclip-create-agent/references/agents/securityengineer.md": { + "candidate": "9eecf981844ab1f5896dd1bae6b76acce8471c448841608c5655497d691ba1a0", + "baseline": "cd4f34687e873de4c9f0c6c43f268f8d95a43e283b20092c6e6216e0ec68dee4" + }, + "skills/paperclip-create-agent/references/agents/uxdesigner.md": { + "candidate": "f6341c7dd4ce19b7f90e613ad1b46b3f4ba2b5ac19bde1b7b3b4a60ec0376b43", + "baseline": "d44fe590bf20e743cdf7f3de5a365d679e61335165d1da02d78e926a928adc3d" + }, + "skills/paperclip-create-agent/references/baseline-role-guide.md": { + "candidate": "0841c5c483526d8d64c7217df7164bca7f3f03edcf4578f2baefcca01cd156b0", + "baseline": "9b86775beb3ea62a821a1593abe0e1a280b64944fc68fe76388ec29f8f1a9932" + }, + "skills/paperclip-create-agent/references/draft-review-checklist.md": { + "candidate": "1cbe0577a9888f31467f5c7beea01d0aa6435da2a0f577b5fdd3b5fe773984ad", + "baseline": "517eb8d0033e2713053574c7a7ce685e2229fa9f61d094db70271c321dbfe397" + } + }, + "hiringFixtureSourceSha": "57dcee147ed0b2d2e3cc657cd9e50fb16bf9ec25", + "commonReducedNativeSourceSha": "f46492a45354eff21b83946407f5a7642cbaa2d6" + }, + "fixtureProof": { + "schema": "paperclip.hiring-templates.fixture-match-proof.v1", + "substitutions": "Campaign nonces, company/agent/task IDs and their rendered names vary normally; matched-placeholder profile/prompt semantics are byte-identical.", + "cells": [ + { + "executionId": "hiring-templates.runner-codex.local.hire-coder-template-reuse", + "fixtureConfigSha256": "a984a9971fd91082e201fb0f59bb4018d3fb2471244aaa207cb2d61a794a3674", + "model": "gpt-5.6-sol", + "expectedTurns": 5, + "deadline": { + "local": 900000, + "daytona": 900000 + }, + "maxAutomaticRetries": 0 + }, + { + "executionId": "hiring-templates.runner-acpx-claude.local.hire-coder-template-reuse", + "fixtureConfigSha256": "59a6167124c351025dbcf99bb83211d5de6f351ef1f533720f4d1e2ab9b30476", + "model": "claude-sonnet-5", + "expectedTurns": 5, + "deadline": { + "local": 900000, + "daytona": 900000 + }, + "maxAutomaticRetries": 0 + } + ] + }, + "admission": { + "candidate": { + "sourceSha": "9f5404ad3aacbe76777952759414d34fd381e674", + "fingerprint": "b40293a7b2aaed4681acbc697cc26720b2ed2975833f6330b6280d6c5d6a790a", + "receiptSha256": "cd54d25e3de354440cf35c98e1b0abcc1b9abc33d79b096daa353222707edd68", + "passed": true, + "providerCalls": 0, + "passedAssertions": 705, + "gateResults": [ + { + "id": "SH-1", + "name": "Native Codex additive instructions", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "src/drivers/codex/codex-app-server-driver.test.ts", + "passed": true + }, + { + "file": "src/drivers/codex/codex-app-server-driver.lifecycle.test.ts", + "passed": true + }, + { + "file": "src/live/runnerd-codex-transport.test.ts", + "passed": true + }, + { + "file": "src/live/live-session.test.ts", + "passed": true + }, + { + "file": "src/cli/eval-provider-runtime.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "adds Paperclip developer instructions", + "passed": true + }, + { + "name": "preserves stock Codex instructions on task recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on prepared recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on direct recovery", + "passed": true + }, + { + "name": "captures exact provider frames and correlates", + "passed": true + }, + { + "name": "exposes native completion consistently on fresh and resumed sessions", + "passed": true + }, + { + "name": "passes caller-supplied native system instructions", + "passed": true + } + ], + "total": 329, + "passedTests": 11, + "failedTests": 0, + "pendingTests": 318 + }, + { + "id": "SH-2", + "name": "Production-default hire bundle", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "server/src/__tests__/agent-skills-routes.test.ts", + "passed": true + }, + { + "file": "server/src/services/onboarding-first-task-assets.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "materializes minimal default instructions for non-CEO agents with no prompt template", + "passed": true + } + ], + "total": 71, + "passedTests": 67, + "failedTests": 0, + "pendingTests": 4 + }, + { + "id": "SH-3", + "name": "Shared legacy startup and continuation", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "packages/adapter-utils/src/server-utils.test.ts", + "passed": true + }, + { + "file": "packages/adapter-utils/src/prompt-sections.test.ts", + "passed": true + }, + { + "file": "packages/adapter-utils/src/acpx-engine/execute.test.ts", + "passed": true + }, + { + "file": "packages/adapters/pi-local/src/server/execute.remote.test.ts", + "passed": true + }, + { + "file": "packages/adapters/opencode-local/src/server/execute.test.ts", + "passed": true + }, + { + "file": "packages/adapters/cursor-cloud/src/server/execute.test.ts", + "passed": true + }, + { + "file": "server/src/__tests__/codex-local-execute.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "keeps task and chat defaults to identity and connection guidance", + "passed": true + }, + { + "name": "does not restore generic procedures on resume or with the legacy opt-in", + "passed": true + } + ], + "total": 396, + "passedTests": 396, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-3-hermes", + "name": "Hermes shared-prompt delivery", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "src/server/prompt-rendering.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "renders standard assignment wake with task authority", + "passed": true + }, + { + "name": "renders scoped planning wake authority", + "passed": true + } + ], + "total": 13, + "passedTests": 13, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-4-tools", + "name": "Native completion tool delivery", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "src/contracts/completion-result.test.ts", + "passed": true + }, + { + "file": "src/drivers/runner-tool-bridge.test.ts", + "passed": true + }, + { + "file": "src/drivers/opencode/mcp-bridge.test.ts", + "passed": true + }, + { + "file": "src/drivers/codex/codex-app-server-driver.lifecycle.test.ts", + "passed": true + }, + { + "file": "src/live/runnerd-codex-transport.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "propagates answer and wait descriptions", + "passed": true + }, + { + "name": "runner semantic MCP bridge binds to loopback", + "passed": true + }, + { + "name": "OpenCode MCP bridge binds to loopback", + "passed": true + }, + { + "name": "preserves stock Codex instructions on task recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on prepared recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on direct recovery", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native codex tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native opencode tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native claude_managed tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native aws_agentcore tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native acpx tool catalog", + "passed": true + } + ], + "total": 239, + "passedTests": 11, + "failedTests": 0, + "pendingTests": 228 + }, + { + "id": "SH-4-resume", + "name": "Refresh retained native completion guidance", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "server/src/services/native-runtime/native-session-resume.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "native completion tool guidance", + "passed": true + } + ], + "total": 45, + "passedTests": 3, + "failedTests": 0, + "pendingTests": 42 + }, + { + "id": "SH-eval", + "name": "Independent oracle and qualification admission", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "tests/runner-e2e/stock-harness-manifest.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/paperclip-document.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness-checks.test.mjs", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness-admission.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness-digest.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/select-rerun-artifacts.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/native-completion-case.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "requires generated capability manifests for the current skill sources", + "passed": true + }, + { + "name": "the shipped recipe delivers the current", + "passed": true + }, + { + "name": "the shipped recipe supports an unnumbered issue", + "passed": true + }, + { + "name": "rejects old SHA before providers", + "passed": true + }, + { + "name": "allows toolchain paths and excludes every present or future credential", + "passed": true + }, + { + "name": "changes when the evaluated server/src/onboarding-assets/default/AGENTS.md changes", + "passed": true + }, + { + "name": "changes when the evaluated packages/adapter-utils/src/server-utils.ts changes", + "passed": true + }, + { + "name": "changes when the evaluated packages/shared/src/connection-intent-guidance.ts changes", + "passed": true + }, + { + "name": "changes when the evaluated tests/runner-e2e/native-completion-case.ts changes", + "passed": true + }, + { + "name": "accepts the persisted blocker, owner and exact requested unblock action", + "passed": true + }, + { + "name": "retains credential-free prerequisites inside the exact campaign root", + "passed": true + } + ], + "total": 133, + "passedTests": 133, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-hiring", + "name": "Matched production hiring oracle and source coverage", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "tests/runner-e2e/hiring-template.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/catalog.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/live-fixtures.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/hiring-comparison-context.test.mjs", + "passed": true + } + ], + "requirements": [ + { + "name": "compares each revision's bundle and example without imposing candidate length on baseline", + "passed": true + }, + { + "name": "accepts only supported direct read arguments", + "passed": true + }, + { + "name": "reads the actual default CEO selection and served hiring files", + "passed": true + }, + { + "name": "validates the selected CEO files against this variant's loader", + "passed": true + }, + { + "name": "matches the canonical team catalog to this variant's production sources", + "passed": true + } + ], + "total": 70, + "passedTests": 70, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-1-rust", + "passed": true, + "exitCode": 0 + } + ], + "workflowUrl": "https://github.com/paperclipai/paperclip/actions/runs/37075466208" + }, + "baseline": { + "sourceSha": "296a4df85e8bcc97a160fc78c291b17adb828196", + "fingerprint": "3124e05caa711aa050595bb02c4085bde30a7466f5174a2b071946a3e57e6d0d", + "receiptSha256": "2a29b3cff539c55a1eb06dd129c0da9491883389854ea1bc0392bcef28dcea58", + "passed": true, + "providerCalls": 0, + "passedAssertions": 705, + "gateResults": [ + { + "id": "SH-1", + "name": "Native Codex additive instructions", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "src/drivers/codex/codex-app-server-driver.test.ts", + "passed": true + }, + { + "file": "src/drivers/codex/codex-app-server-driver.lifecycle.test.ts", + "passed": true + }, + { + "file": "src/live/runnerd-codex-transport.test.ts", + "passed": true + }, + { + "file": "src/live/live-session.test.ts", + "passed": true + }, + { + "file": "src/cli/eval-provider-runtime.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "adds Paperclip developer instructions", + "passed": true + }, + { + "name": "preserves stock Codex instructions on task recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on prepared recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on direct recovery", + "passed": true + }, + { + "name": "captures exact provider frames and correlates", + "passed": true + }, + { + "name": "exposes native completion consistently on fresh and resumed sessions", + "passed": true + }, + { + "name": "passes caller-supplied native system instructions", + "passed": true + } + ], + "total": 329, + "passedTests": 11, + "failedTests": 0, + "pendingTests": 318 + }, + { + "id": "SH-2", + "name": "Production-default hire bundle", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "server/src/__tests__/agent-skills-routes.test.ts", + "passed": true + }, + { + "file": "server/src/services/onboarding-first-task-assets.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "materializes minimal default instructions for non-CEO agents with no prompt template", + "passed": true + } + ], + "total": 71, + "passedTests": 67, + "failedTests": 0, + "pendingTests": 4 + }, + { + "id": "SH-3", + "name": "Shared legacy startup and continuation", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "packages/adapter-utils/src/server-utils.test.ts", + "passed": true + }, + { + "file": "packages/adapter-utils/src/prompt-sections.test.ts", + "passed": true + }, + { + "file": "packages/adapter-utils/src/acpx-engine/execute.test.ts", + "passed": true + }, + { + "file": "packages/adapters/pi-local/src/server/execute.remote.test.ts", + "passed": true + }, + { + "file": "packages/adapters/opencode-local/src/server/execute.test.ts", + "passed": true + }, + { + "file": "packages/adapters/cursor-cloud/src/server/execute.test.ts", + "passed": true + }, + { + "file": "server/src/__tests__/codex-local-execute.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "keeps task and chat defaults to identity and connection guidance", + "passed": true + }, + { + "name": "does not restore generic procedures on resume or with the legacy opt-in", + "passed": true + } + ], + "total": 396, + "passedTests": 396, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-3-hermes", + "name": "Hermes shared-prompt delivery", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "src/server/prompt-rendering.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "renders standard assignment wake with task authority", + "passed": true + }, + { + "name": "renders scoped planning wake authority", + "passed": true + } + ], + "total": 13, + "passedTests": 13, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-4-tools", + "name": "Native completion tool delivery", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "src/contracts/completion-result.test.ts", + "passed": true + }, + { + "file": "src/drivers/runner-tool-bridge.test.ts", + "passed": true + }, + { + "file": "src/drivers/opencode/mcp-bridge.test.ts", + "passed": true + }, + { + "file": "src/drivers/codex/codex-app-server-driver.lifecycle.test.ts", + "passed": true + }, + { + "file": "src/live/runnerd-codex-transport.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "propagates answer and wait descriptions", + "passed": true + }, + { + "name": "runner semantic MCP bridge binds to loopback", + "passed": true + }, + { + "name": "OpenCode MCP bridge binds to loopback", + "passed": true + }, + { + "name": "preserves stock Codex instructions on task recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on prepared recovery", + "passed": true + }, + { + "name": "preserves stock Codex instructions on direct recovery", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native codex tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native opencode tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native claude_managed tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native aws_agentcore tool catalog", + "passed": true + }, + { + "name": "preserves answer and internal wait descriptions in the serialized native acpx tool catalog", + "passed": true + } + ], + "total": 239, + "passedTests": 11, + "failedTests": 0, + "pendingTests": 228 + }, + { + "id": "SH-4-resume", + "name": "Refresh retained native completion guidance", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "server/src/services/native-runtime/native-session-resume.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "native completion tool guidance", + "passed": true + } + ], + "total": 45, + "passedTests": 3, + "failedTests": 0, + "pendingTests": 42 + }, + { + "id": "SH-eval", + "name": "Independent oracle and qualification admission", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "tests/runner-e2e/stock-harness-manifest.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/paperclip-document.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness-checks.test.mjs", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness-admission.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/stock-harness-digest.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/select-rerun-artifacts.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/native-completion-case.test.ts", + "passed": true + } + ], + "requirements": [ + { + "name": "requires generated capability manifests for the current skill sources", + "passed": true + }, + { + "name": "the shipped recipe delivers the current", + "passed": true + }, + { + "name": "the shipped recipe supports an unnumbered issue", + "passed": true + }, + { + "name": "rejects old SHA before providers", + "passed": true + }, + { + "name": "allows toolchain paths and excludes every present or future credential", + "passed": true + }, + { + "name": "changes when the evaluated server/src/onboarding-assets/default/AGENTS.md changes", + "passed": true + }, + { + "name": "changes when the evaluated packages/adapter-utils/src/server-utils.ts changes", + "passed": true + }, + { + "name": "changes when the evaluated packages/shared/src/connection-intent-guidance.ts changes", + "passed": true + }, + { + "name": "changes when the evaluated tests/runner-e2e/native-completion-case.ts changes", + "passed": true + }, + { + "name": "accepts the persisted blocker, owner and exact requested unblock action", + "passed": true + }, + { + "name": "retains credential-free prerequisites inside the exact campaign root", + "passed": true + } + ], + "total": 133, + "passedTests": 133, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-hiring", + "name": "Matched production hiring oracle and source coverage", + "passed": true, + "exitCode": 0, + "files": [ + { + "file": "tests/runner-e2e/hiring-template.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/catalog.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/live-fixtures.test.ts", + "passed": true + }, + { + "file": "tests/runner-e2e/hiring-comparison-context.test.mjs", + "passed": true + } + ], + "requirements": [ + { + "name": "compares each revision's bundle and example without imposing candidate length on baseline", + "passed": true + }, + { + "name": "accepts only supported direct read arguments", + "passed": true + }, + { + "name": "reads the actual default CEO selection and served hiring files", + "passed": true + }, + { + "name": "validates the selected CEO files against this variant's loader", + "passed": true + }, + { + "name": "matches the canonical team catalog to this variant's production sources", + "passed": true + } + ], + "total": 70, + "passedTests": 70, + "failedTests": 0, + "pendingTests": 0 + }, + { + "id": "SH-1-rust", + "passed": true, + "exitCode": 0 + } + ], + "workflowUrl": "https://github.com/paperclipai/paperclip/actions/runs/37075469463" + } + }, + "pairedResults": [ + { + "profile": "runner-codex", + "candidate": "pending", + "baseline": "pending", + "coverage": "pending", + "newRegression": "unknown" + }, + { + "profile": "runner-acpx-claude", + "candidate": "pending", + "baseline": "pending", + "coverage": "pending", + "newRegression": "unknown" + } + ], + "providerScope": { + "selectedCells": 4, + "expectedTurns": 20, + "perCellDeadlineMs": 900000, + "maxAutomaticRetries": 0, + "companyBudgetMonthlyCents": 1000, + "leadBudgetMonthlyCents": 1000 + }, + "limitations": [ + "No live cell has been graded. Pending or missing coverage cannot establish non-regression.", + "The PR is master-based; these measured unions hold the reduced generic manual/shared prompts, operational skill and native completion descriptions constant.", + "Only historical hiring production sources and derived team catalog differ; unrelated scenario, fixture, oracle, model, effort, auth, tools and permissions are fixed.", + "Case demonstrates a requested default CEO hire, coder instruction-template use and two saved JSON artifacts with reuse. It does not qualify every role or overall coding quality.", + "Outcome and source/read coverage are independently reported; missing/redacted read receipts are uncomparable, not a pass." + ] +} diff --git a/doc/plans/2026-10-02-hiring-template-live-comparison.md b/doc/plans/2026-10-02-hiring-template-live-comparison.md new file mode 100644 index 00000000000..cd98176b392 --- /dev/null +++ b/doc/plans/2026-10-02-hiring-template-live-comparison.md @@ -0,0 +1,38 @@ +# Matched hiring template qualification + +TL;DR: No live outcomes are graded yet. There are **0 established new failures, 0 established new passes, 0 established unchanged outcomes, and 2 pending profile pairs**. Pending pairs cannot establish whether the reduced hiring templates made task outcomes worse. PR #14985 was merged after current-head CI and 5/5 review; live qualification remains separate and in progress. + +## Frozen comparison + +| Variant | Exact measured source | Protected GitHub run | Credential-free admission | +| --- | --- | --- | --- | +| candidate | `9f5404ad3aacbe76777952759414d34fd381e674` | [workflow](https://github.com/paperclipai/paperclip/actions/runs/37075466208) | 705 assertions; zero providers; retained receipt verified | +| baseline | `296a4df85e8bcc97a160fc78c291b17adb828196` | [workflow](https://github.com/paperclipai/paperclip/actions/runs/37075469463) | 705 assertions; zero providers; retained receipt verified | + +The candidate incorporates corrected hiring source `57dcee147ed0b2d2e3cc657cd9e50fb16bf9ec25` and common reduced/shared/operational-skill/native context `f46492a45354eff21b83946407f5a7642cbaa2d6`. The baseline restores the 21 production/derived files listed in [the common source contract](../../tests/runner-e2e/hiring-comparison-context.json) from `d7bdfc422cadcc407f2945e211833a8382b80117`. The baseline adds one provenance receipt. **8,242 other tracked files are byte-identical**, including fixture/oracle/admission glue, native completion guidance, tiny generic manual, shared prompts and operational skill. + +The hiring fixture digest is `4d7f18420889325af89aa48eaaf021e8178aa6d26d504b92041635a15741f004`. Matched-placeholder configuration hashes prove identical models, effort, credential-binding structure, permissions, environment and initial prompt. Campaign nonces, company/agent/task IDs and rendered names vary normally. Neither frozen ref changes after dispatch. + +## Results available so far + +| Profile / model | Historical outcome | Reduced-template outcome | Source/read coverage | Paired conclusion | +| --- | --- | --- | --- | --- | +| Native Codex / gpt-5.6-sol | Pending | Pending | Pending | Unknown | +| Native ACPX Claude / claude-sonnet-5 | Pending | Pending | Pending | Unknown | + +Each cell uses the actual production default CEO bundle with no custom manual and explicitly requests the production hiring skill and coder reference. It should create exactly one reusable coder, preserve that coder’s managed template and skill selection, produce two independently scored saved JSON documents in separate tasks, preserve the first, and report ownership/status. Expected scope is 5 turns per cell, **4 cells / 20 turns total**, 15 minutes per cell, **zero automatic retries**, with $10 company and lead monthly hard stops. Costs, actual run count, timings and cleanup are pending retained evidence; they are not assumed zero. + +## Admission boundaries + +Ordinary PR configuration tests continue to require the candidate’s one-file CEO default. The comparison alone symmetrically filters the four candidate-specific assertions below because the historical variant deliberately loads four files. Independent gates validate each variant’s actual selected files, exact production hashes and canonical derived team catalog. All other custom-bundle, generic-manual, skill/auth/permissions and native/legacy boundaries remain admitted. + +- `materializes only the CEO entry file for claude_local via agents` +- `materializes only the CEO entry file for claude_local via agent-hires` +- `materializes only the CEO entry file for paperclip_runner via agents` +- `materializes only the CEO entry file for paperclip_runner via agent-hires` + +The same executed historical/candidate oracle calibrations require source-derived CEO/coder content, reject fake/help/version read receipts and independently grade durable document contents and task ownership/status. Missing or redacted operational read coverage is reported as **uncomparable** even if outcomes pass. The original failures from other stock-harness comparisons are preserved; this hiring-only comparison does not regrade them or measure the earlier native fixed-prompt removal. + +## Evidence + +The [JSON report](2026-10-02-hiring-template-live-comparison.json) includes exact source/hash proofs, gate receipts, selected cells and limits. The [hiring PR](https://github.com/paperclipai/paperclip/pull/14985) merged at `862a5758ba0e88a33232c1f1fa645e85c38a3113`. Its green unit/CI checks do not substitute for the pending model outcomes. Further updates will retain failed/partial results and separate outcome from instruction coverage.