From 9b2ab0d3fb6f430a15d14dcacdc1ba504cfa38c9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 11:25:07 -0500 Subject: [PATCH] test(evals): cover production default hires across stock harnesses Co-Authored-By: Paperclip --- doc/evals.md | 8 + ...10-02-stock-harness-paperclip-checklist.md | 205 ++++++++++++++++-- package.json | 1 + tests/runner-e2e/FIXTURES.md | 11 + tests/runner-e2e/README.md | 8 + tests/runner-e2e/STOCK-HARNESS.md | 110 ++++++++++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 21 ++ tests/runner-e2e/live-fixtures.test.ts | 24 ++ tests/runner-e2e/live-fixtures.ts | 5 +- tests/runner-e2e/runner.spec.ts | 31 +++ tests/runner-e2e/stock-harness-checks.mjs | 109 ++++++++++ .../runner-e2e/stock-harness-checks.test.mjs | 40 ++++ tests/runner-e2e/stock-harness.test.ts | 111 ++++++++++ tests/runner-e2e/stock-harness.ts | 115 ++++++++++ tests/runner-e2e/vitest.config.ts | 2 +- 16 files changed, 785 insertions(+), 22 deletions(-) create mode 100644 tests/runner-e2e/STOCK-HARNESS.md create mode 100644 tests/runner-e2e/stock-harness-checks.mjs create mode 100644 tests/runner-e2e/stock-harness-checks.test.mjs create mode 100644 tests/runner-e2e/stock-harness.test.ts create mode 100644 tests/runner-e2e/stock-harness.ts diff --git a/doc/evals.md b/doc/evals.md index f936899203b..c529972e76d 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -95,6 +95,14 @@ execution ID because `--all` excludes explicit-only suites. Each cell applies a 1,000-cent company and agent budget hard stop before task creation and records both limits in its evidence. +The explicit-only [stock-harness suite](../tests/runner-e2e/STOCK-HARNESS.md) +reuses skill, ordered-continuation, and chat-restart journeys across eight local +legacy/native profiles with production-default hires. It closes the custom QA +manual coverage gap. Its required credential-free prerequisite maps vendor +instruction layering, the tiny hire bundle, and shared startup/resume reductions +to executable checks. The 24 live cells are configured; no live qualification is +claimed from their setup or unit calibration. + The explicit-only `agent-chat-stories` suite covers the experimental settings lifecycle for a configured native agent and follow-ups during active work. Its fixture-driven file wait and persisted-plan oracle are documented in the diff --git a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md index f71067ff070..c333ae53856 100644 --- a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md +++ b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md @@ -1,7 +1,10 @@ # Stock harness, with Paperclip: working checklist -Created: 2026-10-02. Status: item 1 implemented for native Codex app-server; -PR validation in progress. Other implementation items remain open. +Created: 2026-10-02. Status: item 1 merged for native Codex app-server in +[PR #14920](https://github.com/paperclipai/paperclip/pull/14920). Item 2's default +hire manual is reduced to identity only, and common legacy startup/resume +instructions are reduced locally. Additional carriers and native instructions +remain open. Goal: keep the agent's stock harness behavior and add only what it needs to work with Paperclip. Apply this across legacy adapters, the new Runner, and their @@ -12,7 +15,11 @@ record the proposed behavior and Dotta's direction, make a bounded change, and verify the affected paths before moving on. New findings get stable IDs in the ledger below so they do not disappear into conversation history. -**Current item: 1 — validate and review the native Codex instruction fix.** +For every change, record its executable test/eval coverage before continuing. +Distinguish coverage setup, deterministic results, and measured live results; +configured cells alone do not qualify behavior. + +**Current item: 2 — reduce the default operating manual and shared prompt layers.** ## Agreed direction and boundaries @@ -43,7 +50,7 @@ ledger below so they do not disappear into conversation history. - [x] Remove default replacement of stock base instructions across those paths. - [x] Verify the actual app-server request and retained session instructions, including resume; checking only a prompt builder is insufficient. -- [ ] Verify Paperclip task context, tools, auth, assigned skills, and completion +- [x] Verify Paperclip task context, tools, auth, assigned skills, and completion still work. Record applicable regressions and eval results. Starting points: [Codex backend](../../packages/paperclip-runner/src/backends/codex-native-backend.ts), @@ -61,14 +68,20 @@ and need a provider session reset. Do not reset active sessions automatically. The separate Codex-through-ACP dependency patch remains a coverage follow-up under item 6; this change covers the native app-server path. +Verification: 412 focused TypeScript/Rust tests, repository typecheck/build, +55 passing PR checks, and Greptile 5/5. The original full local test command +was stopped after setup failures; affected suites passed on rerun. No paid +live campaign was run, and no task-quality improvement is claimed. + ## 2. Reduce the default operating manual and shared prompt layers - [ ] Inventory what an agent actually receives: hire instructions, shared prompt template, wake context, runtime prompt, bootstrap, and loaded skills. Separate always-present text from content loaded on demand. -- [ ] Agree on the tiny common contract and the coordination details that each - runtime still requires. Legacy API coordination and native semantic tools - need appropriate instructions for their respective interfaces. +- [x] Agree on the tiny default: identity only, with no skill or native-tool + pointers. The harness supplies coordination instructions. +- [x] Reduce the generic default hire `AGENTS.md` to the agreed sentence and + update the existing creation test to expect the minimal bundle. - [ ] Remove repeated workflow rules and stock coding/style/autonomy guidance. - [ ] Move detailed planning, hiring, artifacts, and exceptional procedures to discoverable references or tools where feasible. @@ -80,7 +93,116 @@ Starting points: [default hire instructions](../../server/src/onboarding-assets/ [native runtime contract](../../packages/paperclip-runner/src/contracts/runtime-context.ts), [Paperclip operational skill](../../skills/paperclip/SKILL.md). -Decision: exact retained paragraph and on-demand boundaries pending. +Decision: Dotta confirmed the harness already handles skill and tool delivery; +the default hire manual is now only "You are an agent in a Paperclip company." +This replaces 602 words with eight. The shared loader supplies this default +across instruction-bundle-capable adapters for non-CEO hires without explicit +instructions. Existing saved bundles retain their content; CEO, first-agent, +and role/team templates remain separate work under item 3. The common +prompt/wake reduction is complete locally below; additional carriers, native +instructions, and skill/reference corrections remain open. + +Verification: the existing agent-skills route suite passed all 54 tests, +including default creation, custom bundles, and CEO/first-agent paths. The +onboarding asset suite passed all 10 tests. This initially had only deterministic +coverage. Dotta subsequently requested behavioral eval coverage for every change; +the dedicated production-default-hire suite below closes the custom QA manual +gap. Full repository typecheck/build/test were not rerun for this narrow asset +and existing-test update. + +### Shared prompt follow-ups + +- [x] **2.1 Reduce common legacy startup/resume instructions.** Keep + identity and connection guidance in the shared task/chat defaults; remove the + generic resumed-wake execution contract. Preserve current task/event data, + specialized wake contracts, custom templates, skills, and auth. +- [ ] **2.2 Review additional legacy carriers.** Reduce Hermes local/gateway + wrappers, review Pi system delivery, and check OpenClaw fresh-wake framing. + Preserve transport facts and user configuration. +- [ ] **2.3 Reduce native Runner instructions and constraints.** Improve + discoverable tool documentation first, then shorten fixed guidance and + consolidate completion rules. Verify prompt revisions, digests, and session + compatibility across native Codex, ACPX, and OpenCode. + +### Follow-up 2.1 implementation and verification + +The common task and conversation defaults now share the same identity sentence +and unchanged connection guidance: 113 words each, down from 661 and 205. +Connection guidance accounts for 108 of those words. The ordinary resumed-wake +execution contract is removed (172 words), including the old opt-in used by +OpenClaw on fresh turns. `includeExecutionContract` remains accepted as a +deprecated no-op for adapter/plugin source compatibility. + +Current task facts, ordered comments, work modes, approval/review state, +checkout, holds/blockers, recovery/watchdog roles, and external-chat contracts +remain in their existing owners. Skill delivery, runtime authentication, +custom `promptTemplate`, and `bootstrapPromptTemplate` mechanics are unchanged. +Hermes's own wrappers and Pi's system carrier remain follow-up 2.2; native +fixed instructions and full-turn constraints remain follow-up 2.3. + +The new defaults apply when prompts are assembled after deployment. Existing +provider sessions can retain earlier startup instructions in their history +until reset; no active session or saved custom template is rewritten here. + +Verification: 563 distinct focused tests passed across 18 suites: + +- Shared prompt selection/rendering, operational-skill selection, and retained + specialized wake contracts (130 tests). +- ACPX, Pi, OpenCode, Cursor Cloud, and Codex local execution, including + task/chat fresh/resumed/reset/fallback delivery, custom prompts, skill + mounting, and authentication (259 tests). +- Hermes local prompt rendering and gateway execution (41 tests). +- Claude, Gemini, Cursor local, Grok, Kimi, OpenClaw, and Claude/Codex ACP + fallback execution (113 tests). +- Native execution input, preserving shared wake compatibility (20 tests). + +`@paperclipai/adapter-utils` typecheck and build passed; `git diff --check` +passed. Full repository typecheck/build/test and live behavioral evals were not +run for this local bounded change. These checks prove prompt and runtime +mechanics, not an improvement in coding-task quality. + +### Remaining shared layers inspected (2026-10-02) + +The table records the initial inspection before follow-up 2.1. Word counts +cover fixed source text, not complete assembled prompts or token counts. They +exclude task data, assigned agent instructions, and skills. This inspection +establishes instruction mechanics, not a performance result. + +| Layer | Delivery at initial inspection | Disposition | +| --- | --- | --- | +| Shared legacy task/chat templates | `DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE` was 661 words and the conversation template was 205, including connection guidance. Used by Claude, Codex, Cursor local/cloud, Gemini, Grok, Kimi, OpenCode, Pi, and Hermes local. | Completed in 2.1: both defaults now 113 words. Repeated procedures removed, connection guidance retained, no replacement skill pointer. | +| Generic resumed wake contract | `renderPaperclipWakePrompt` added a 172-word execution contract on ordinary resumed turns. OpenClaw requested it on fresh turns too because it has no shared template. | Completed in 2.1: generic contract removed, including legacy opt-ins. Task/event data and conditional runtime contracts retained. | +| Additional legacy carriers | Hermes local adds a 171-word identity/API/curl wrapper before the shared task template; Hermes gateway supplies its own four-rule execution contract. Pi carries shared defaults in its system extension and suppresses the wake copy when that extension owns policy. | Separate follow-up 2.2; changing the shared constants alone does not remove every wrapper. Preserve transport facts and custom configuration. | +| Native fixed prompt | The 262-word `paperclip-execution.v5` prompt carries hiring, delegation/dependencies, connection setup, and finalization guidance across native Codex, ACPX, and OpenCode backends. | Keep one short native completion/dependency contract. Relocate uncommon procedures to the relevant tool documentation, improving that documentation where needed before removing instructions. Any change needs prompt revision/digest/session-compatibility verification. | +| Native full-turn constraints | `nativeTaskConstraints` adds assigned-skill limits, current agent-file paths, document/file delivery, answered-question handling, and accepted completion/final-response sequencing. Codex/OpenCode add another completion reminder. Prepared constraints reach the model envelope through `native-session-runtime`; compact continuations avoid replaying prior task constraints. | Consolidate duplicate completion instructions. Keep current paths, answer scope, and the required result protocol. Move detailed file/document procedure to discoverable tool descriptions where adequately covered. | +| Task/event-specific framing | Server task Markdown and wake rendering supply work mode, ordered comments, approved revisions, checkout/holds/blockers, recovery/review/watchdog roles, external-chat bookkeeping/delivery, and conversation handoff. | Retain the facts and mode boundaries in the first reduction. Use one authoritative owner for shared mode directives; review specialized wording separately. | + +Custom `promptTemplate` and fresh-only `bootstrapPromptTemplate` configuration +are distinct from shipped defaults and should not be silently rewritten. +The built-in process adapter invokes a configured command; HTTP passes context +as JSON. They do not use the shared text template. External adapter plugins and +hosted configuration variants still require the broader item 6 audit. + +An accepted-plan discrepancy needs resolution: task Markdown permits cohesive +implementation on the current issue, while the wake renderer's planning-mode +accepted-confirmation branch requires children and prohibits implementation on +the source issue. Existing tests assert both versions. Production reachability +after the server changes work mode has not yet been established; do not claim +this proves a live plan-continuation failure. + +Dotta selected follow-up 2.1 first: reduce the common legacy startup/resume +copies, preserving conditional context. Additional carriers and native fixed +instructions remain distinct follow-ups 2.2 and 2.3. No shared runtime text was +changed by the initial inspection. + +Coverage identified during inspection includes shared prompt selection/rendering, +adapter execution tests, native backend/runtime context tests, and server +task-context/native-input tests. Follow-up 2.1 extended the shared and adapter +checks; its results are above. Follow-ups 2.2 and 2.3 need their corresponding +carrier/protocol assertions. Product E2E context-integrity, completion-updates, +and direct blocker guidance are candidate lifecycle checks; their current scope +does not establish coding-task quality. The initial inspection itself ran no +tests or live evals. ## 3. Review every hiring and role template @@ -154,7 +276,7 @@ does not apply. | Legacy Codex / Claude | Local adapters, managed auth and isolated configuration | Initial inspection only | | Other legacy local adapters | ACPX, OpenCode, Pi, Cursor, Gemini, Grok, Kimi, Hermes | Pending | | Other adapter transports | Cursor Cloud, Hermes/OpenClaw gateways, process, HTTP, external adapter plugins | Pending | -| Runner Codex | App-server driver, runnerd bridge, Rust provider, direct/fallback paths | Additive instruction fix implemented; PR validation pending | +| Runner Codex | App-server driver, runnerd bridge, Rust provider, direct/fallback paths | Additive instruction fix merged in PR #14920 | | Runner ACPX | Enabled profiles, especially Claude/Grok; declared or pending profiles tracked separately | Claude initial inspection; remaining audit pending | | Runner OpenCode | Native provider and configuration paths | Pending | | Hosted/remote providers | Claude Managed and AWS AgentCore; identify their own baseline rather than assuming CLI semantics | Pending | @@ -172,15 +294,18 @@ does not apply. ## Verification and evals — apply to each item -- [ ] Map existing coverage before adding cases; use [doc/evals.md](../evals.md). +- [x] Map existing coverage for all implemented changes before adding cases; + see the SH-1–SH-3 map below and [doc/evals.md](../evals.md). Keep Runner protocol evals and Product E2E evals distinct. -- [ ] Start with narrow deterministic checks for instruction layering, effective +- [x] Set up narrow deterministic checks for the implemented instruction layering, + hire, and shared-prompt changes. Future changes still need their own map for effective configuration, skill/auth delivery, and session behavior where appropriate. -- [ ] Use Runner evals for provider/session/tool protocol changes; use Product - E2E for real hiring, repository context, task lifecycle, and artifact delivery. -- [ ] Review existing context-integrity, hiring, completion-updates, and blocker - suites for reusable coverage; record gaps rather than claiming coverage from - a similarly named case. +- [x] Reuse existing protocol tests for native Codex request layering and add Product + E2E for production-default hiring, skill delivery, task lifecycle, and chat + continuity. Repository context and additional artifact cases remain future work. +- [x] Review existing suites for reusable coverage. Their custom QA manuals did + not exercise the tiny default hire; the new suite deliberately omits those + bundles and reuses independently graded skill, continuation, and chat journeys. - [ ] Compare task quality as well as Paperclip protocol compliance when claiming that fewer instructions improve agent performance. Keep model, effort, permissions, tools, and fixture comparable. @@ -189,19 +314,61 @@ does not apply. - [ ] Run the relevant checks for each change and the repository's required full verification before a PR-ready handoff. Record unrun checks and their reasons. +### Coverage for changes implemented so far + +The [stock-harness runbook](../../tests/runner-e2e/STOCK-HARNESS.md) defines a +credential-free prerequisite plus 24 explicit local Product E2E cells across +eight legacy/native profiles. Each profile runs assigned-skill invocation, +ordered comment continuation, and chat continuity across restart. Public receipts +verify the identity-only managed bundle before provider execution; final receipts +check actual legacy prompts and both budget hard stops. The existing lifecycle +oracles still own task/chat success. Models, auth, skills, permissions, and +secret-reference plumbing are inherited from existing profiles. + +| Coverage ID | Implemented change | Executable coverage | Live status | +| --- | --- | --- | --- | +| SH-1 | Native Codex additive developer instructions, including start/resume/recovery | TypeScript driver, runnerd transport, Runner Lab/live-session, and Rust provider tests in `pnpm test:e2e:runner:stock-harness`; native Codex cells exercise real hires. | `not_run`; task success is not vendor-base proof. | +| SH-2 | Eight-word default hire `AGENTS.md` | Public creation/onboarding tests; exact independent public bundle oracle before and after provider execution for every stock-harness cell. | 24 cells configured, `not_run`. | +| SH-3 | Reduced shared task/chat defaults and removed generic resume contract | Shared renderer and ACPX/Codex/OpenCode/Pi/Hermes/Cursor Cloud regressions; actual legacy invocation prompts checked in the new suite. | Legacy cells configured, `not_run`; additional carriers remain item 2.2. | + +The suite is explicit-only and excluded from `--all`; it does not add paid work +to ordinary campaigns. Negative calibration covers manual regrowth, missing or +malformed prompt receipts, old startup/resume procedures, missing connection +guidance, wrong budgets, and skipped prerequisite assertions. Source revisions, +attempts, cost, partial failures, cleanup, and sanitized evidence use the existing +report pipeline. No paid providers were launched during setup. No coding-quality +improvement or live qualification is claimed. Unrepresented harnesses remain +unqualified; Codex-through-ACP vendor-base preservation is still F7. + +Local setup verification: 478 prerequisite tests passed (477 TypeScript plus one +Rust), 860 Product E2E support tests passed, E2E typecheck passed, and all 24 cells +were discovered. The 313 unrelated native tests filtered by the gate are not +passing coverage. Final prerequisite evidence is retained at +`tests/runner-e2e/results/stock-harness-preflight-2026-10-02T16-15-39.064Z/preflight.json`; +earlier interrupted, missing-Hermes-discovery, and setup/test-timeout attempts are +retained as failures. Hermes is checked through its package config because the +root Vitest project list omits it. Full repository checks have not been rerun for +this eval setup; they remain required before a PR-ready handoff. + ## Findings ledger Confirmed mechanics below do not by themselves establish an effect on task quality. | ID | Finding | Work item / disposition | | --- | --- | --- | -| F1 | Native Codex sent Paperclip text as `baseInstructions`. Probes on codex-cli 0.153.4 showed replacement; additive `developerInstructions` retained the stock base on start and cold resume. | 1; app-server fix implemented, PR validation pending | +| F1 | Native Codex sent Paperclip text as `baseInstructions`. Probes on codex-cli 0.153.4 showed replacement; additive `developerInstructions` retained the stock base on start and cold resume. | 1; app-server fix merged in PR #14920 | | F2 | Default hires and role templates prescribe substantial operating procedures; common prompt and wake layers add further coordination text. | 2–3; mechanics confirmed, performance effect unmeasured | | F3 | Hiring references require legacy Paperclip skill/comment procedures, while native Runner intentionally omits that operational skill and uses semantic tools. | 2–3, 5; reconcile runtime contracts | | F4 | Local Claude appends instructions; Runner Claude preserves the Claude Code preset. Runner isolation excludes project/local settings, which can also exclude repository instruction discovery. | 4; selective context fix to design | | F5 | Some Codex capability settings differ between the direct driver and daemon path; an intermediate configuration does not prove the final provider behavior. | 6; effective-path audit pending | | F6 | Omitting or nulling `baseInstructions` on an old Codex thread's resume preserves its saved replacement; an empty string produces an empty base. | 1; document the required provider session reset; no automatic migration in this PR | | F7 | The isolated Codex-through-ACP dependency patch also sets `baseInstructions` on start/resume. It is a separate path from the native app-server backend. | 6; follow-up patch/profile audit pending | +| F8 | The operational skill says target-bound confirmations default `supersedeOnUserComment` to true; the default manual and server normalizer say false. The server uses false. | 2; correct stale skill/reference guidance in a follow-up | +| F9 | The default manual required a comment on every task, while the operational skill's verified external-chat shortcut delegates comments and lifecycle bookkeeping to the harness. | 2; unconditional manual rule removed; shared layers still need review | +| F10 | Removing the default manual left the 661-word legacy task template and its 172-word resumed-wake execution contract. Hermes and Pi have additional policy carriers. | 2.1 complete locally: task/chat defaults 113 words, no generic resume contract; 2.2 wrappers pending | +| F11 | Task Markdown and the wake renderer prescribe different accepted-plan behavior for planning-mode accepted-confirmation payloads; tests currently expect both. | 2; unify the directive owner; production reachability still to trace | +| F12 | Native fixed instructions and full-turn constraints repeat completion and uncommon procedures, while reserved finish/block tool descriptions are only one sentence each. | 2; improve tool documentation before removing needed native protocol guidance | +| F13 | Existing context-integrity/chat fixtures injected a QA manual, so their green results did not qualify the production tiny hire default. | Dedicated stock-harness suite configured with public default-bundle and delivered-prompt receipts; live measurement pending. | Append new findings with evidence, affected paths, and the numbered item that will address them. Record intentional behavior explicitly rather than as a bug. @@ -214,6 +381,10 @@ will address them. Record intentional behavior explicitly rather than as a bug. | 2026-10-02 | Paperclip-owned MCP isolation and configuration changes/session resets are acceptable. | Preserve Paperclip auth and assigned skills while fixing repository context. | | 2026-10-02 | Dotta requested implementation and a PR for item 1. Native app-server paths now use additive developer instructions. | 139 targeted TypeScript tests and 91 Rust provider tests passed; repository typecheck/build passed. Remaining test/review results to record. | | 2026-10-02 | Verified actual Codex instruction layering using a localhost Responses stub, without paid inference. | codex-cli 0.153.4 sent identical 14,732-character stock base instructions on start and cold resume, with the Paperclip marker retained in developer input. This is protocol evidence, not a task-quality eval. | +| 2026-10-02 | Dotta requested and confirmed the merge of item 1. | PR #14920 merged at `408f70e69f9c5e49cb4377f4886ac2001bfa67a2`, with all 55 checks passing and Greptile 5/5. | +| 2026-10-02 | Dotta directed an identity-only default manual with no skill or runtime pointers; the harness already handles coordination. | Reduced the default to eight words. Existing hire and onboarding suites passed all 64 tests. Shared prompts and role templates remain pending. | +| 2026-10-02 | Dotta requested three explicit shared-prompt follow-ups and selected common legacy startup/resume reduction first. | Follow-ups 2.1–2.3 recorded. 2.1 complete locally: both defaults 113 words; generic resume contract removed. 563 focused tests and shared utility typecheck/build passed. Extra carriers and native instruction reduction remain pending. | +| 2026-10-02 | Dotta requested executable eval coverage for everything implemented so far and all subsequent changes before continuing. | Added SH-1–SH-3 coverage map, credential-free prerequisite, and 24 production-default-hire cells. 478 prerequisite and 860 support tests passed; E2E typecheck/discovery passed. Live provider results remain `not_run`; item 2.2/2.3 unchanged. | For each completed item, add the chosen behavior, changed paths, verification results, remaining exceptions, and follow-ups here before checking it off. diff --git a/package.json b/package.json index 2fbd9d95a99..29223ed74ec 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "test:e2e:browser-context": "playwright test --config tests/e2e/playwright-company-context.config.ts", "test:e2e:runner:browser-support": "playwright test --config tests/runner-e2e/playwright-support.config.ts", "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts", + "test:e2e:runner:stock-harness": "node tests/runner-e2e/stock-harness-checks.mjs", "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", "test:runner-workflow-evals": "pnpm --filter @paperclipai/paperclip-eval-kernel build && pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals", diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 2ff1eace509..d65997921c0 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -17,6 +17,17 @@ profile, model qualification, environment, task, or ranking-snapshot change must change that fingerprint automatically so the dashboard can annotate the boundary instead of silently joining unlike totals. +The explicit [stock-harness suite](STOCK-HARNESS.md) wraps existing profiles with +`productionDefaultHireProfile`: omit only `instructionsBundle` so the public +hire route loads the shipped default, while preserving runtime, permissions, +auth, skills, and managed secret references. Do not replace this with a fixture +copy of the default manual. Public receipts check the exact independently +specified bundle before provider execution and again during cleanup, along with +both budget hard stops and actual legacy invocation prompts. Missing evidence +fails closed. The definition fingerprint includes the helper, graders, journey +sources, live fixture, and execution integration; editing those sources changes +the suite revision automatically. + ## Agent profiles Add `RunnerProfileFixture` entries in `catalog.ts`. A profile declares: diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 0da021312dc..b652b810ab7 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -444,6 +444,14 @@ pnpm test:e2e:runner -- --list --suite context-integrity pnpm test:e2e:runner -- --id context-integrity.runner-codex.local.ordered-comment-continuation ``` +`stock-harness` reuses ordered continuation, assigned-skill invocation, and chat +restart journeys with production-default hires instead of the custom QA manual. +Its 24 explicit local cells cover eight legacy/native profiles and are excluded +from `--all`. Run `pnpm test:e2e:runner:stock-harness` for the credential-free +instruction-layering, hire, and shared-prompt prerequisites. The +[suite contract](STOCK-HARNESS.md) maps each change to its graders, budgets, +evidence, and remaining qualification limits. + Each hardening oracle has positive and plausible-negative calibration tests. The review grader parses the worker's saved JSON and compares both source values and the consistency verdict. Hiring requires one identity, correct reporting diff --git a/tests/runner-e2e/STOCK-HARNESS.md b/tests/runner-e2e/STOCK-HARNESS.md new file mode 100644 index 00000000000..605ab5b13ce --- /dev/null +++ b/tests/runner-e2e/STOCK-HARNESS.md @@ -0,0 +1,110 @@ +# Stock harness with Paperclip + +This suite covers the instruction reductions tracked in the +[working checklist](../../doc/plans/2026-10-02-stock-harness-paperclip-checklist.md). +Existing context-integrity and chat evals supplied a custom QA instruction +bundle. Their passing results therefore did not qualify a hire with the tiny +production default. The new suite omits that fixture bundle and lets the public +agent-creation route materialize the shipped default. + +## Coverage contract + +| Change | Required deterministic evidence | Product E2E evidence | +| --- | --- | --- | +| SH-1: native Codex preserves vendor base instructions | Serialized start/resume requests from the TypeScript driver, recovery paths, runnerd transport, Runner Lab/live sessions, and Rust provider use additive developer instructions. | Real new native Codex hires execute the three journeys below. Task success alone cannot prove vendor base preservation. | +| SH-2: identity-only default hire manual | Existing public agent-creation and onboarding-asset tests cover default, custom, and CEO exceptions. | The public bundle is exactly one `AGENTS.md` containing the eight-word shipped identity, checked before provider execution and again during cleanup. | +| SH-3: reduced shared legacy task/chat defaults and resume delta | Shared prompt tests and ACPX, Codex, OpenCode, Pi, Hermes, and Cursor Cloud adapter regressions retain runtime context and exclude removed generic procedures. | Actual legacy `adapter.invoke` prompts retain fresh identity/connection guidance, omit the removed procedures, and have complete receipts for every observed run. | + +`pnpm test:e2e:runner:stock-harness` runs these credential-free prerequisites, +including the Rust test. It writes JSON reports, the source SHA, a working-source +fingerprint, selected checks, test counts, and `providerCalls: 0` under +`results/stock-harness-preflight-/`. Missing reports or required skipped +assertions fail. Unrelated native tests filtered by the name selector remain +explicitly skipped; they are not counted as executed coverage. Run this gate +before qualifying live cells; the live launcher does not run Cargo implicitly. + +## Live matrix + +There are 24 explicit local cells: these eight existing profiles each run three +existing journeys with independently calibrated graders. + +- Legacy: `legacy-codex`, `legacy-claude`, `legacy-opencode`, + `legacy-acp-codex`, `legacy-acp-claude`. +- Native: `runner-codex`, `runner-acpx-claude`, `runner-opencode`. + +| Journey | Expected provider turns | Observable outcome | Cell deadline | +| --- | --- | --- | --- | +| `assigned-skill-explicit-invocation` | 1 | Public skill creation/pinning, an explicit skill request, and saved output containing the marker available only in the skill body. | 12 minutes | +| `ordered-comment-continuation` | 2 | Initial report followed by three ordered public comments, including repeated wording and a changed scope; final saved report preserves the ledger and requested scope. | 12 minutes | +| `continuity-restart` | 3 | Task-backed chat retains the requested context across a server restart and subsequent replies. | 15 minutes | + +A full matrix expects 48 provider turns. Models, credentials, effort, +permissions, assigned skills, environment, and managed secret references are +inherited from the existing profile; only its QA manual is omitted. Both company +and agent receive a 1,000-cent monthly hard stop before provider execution, +verified through public records. These limits do not predict final spend: +attempts, retries, partial runs, unknown billing, and cleanup remain in the +existing campaign accounting and qualification rules. + +`stock-harness` is excluded from `--all`; select it explicitly. It has no Daytona +cells. Pending or unrepresented harnesses are not live-qualified by this matrix. +Pi/Hermes/Cursor Cloud rendering coverage is deterministic here. The separate +Codex-through-ACP base-instruction patch is still an open checklist item: its +legacy cells qualify the common prompt reduction, not vendor base preservation. + +## Run and inspect + +```sh +# No credentials or paid providers: +pnpm test:e2e:runner:stock-harness --list +pnpm test:e2e:runner:stock-harness +pnpm test:e2e:runner:typecheck +pnpm test:e2e:runner:unit +pnpm test:e2e:runner -- --list --suite stock-harness + +# With authorization and the selected profile's required provider credential: +pnpm test:e2e:runner -- --id stock-harness.runner-codex.local.assigned-skill-explicit-invocation +``` + +Use an exact cell first, then expand profile/journey selections when its evidence +is understood. The ordinary launcher, isolated instance, fixture cleanup, +screenshots, attempt history, usage/cost reporting, and dashboard publisher are +unchanged. `snapshots/stock-harness-hire.json` captures the public bundle and +budgets before paid execution. `snapshots/stock-harness.json` captures the final +bundle, budgets, run IDs, actual legacy prompts, and matcher verdicts. Evidence +API failures retain an error snapshot and cannot pass. All snapshots use the +existing secret sanitizer; screenshots remain the original captured pixels. + +The oracle's identity is independent of the implementation constant, so changing +the shipped manual cannot silently change the expected result. The suite +definition digest incorporates the fixture, journeys, grader, and execution +integration sources. Positive and plausible-negative support tests exercise +missing/malformed receipts, manual regrowth, removed startup/resume procedures, +absent connection guidance, and budget drift. Existing calibrated lifecycle +graders still own task/chat success. + +The skill oracle proves the requested pinned skill's output marker reached the +saved result without appearing in the task request or follow-up comments. Skill +tool/read event detection is retained as supporting evidence; it is not a +required cross-provider tool-trace assertion. + +## Qualification status + +Setup was validated locally on 2026-10-02 with the deterministic prerequisites, +Product E2E support tests, typecheck, and discovery: 478 executed prerequisite +tests passed (477 TypeScript plus one Rust), along with 860 support tests and +discovery of all 24 cells. The 313 unrelated native tests filtered by the gate +are not counted as passing coverage. Local evidence is retained under +`results/stock-harness-preflight-2026-10-02T16-15-39.064Z/preflight.json`. +Earlier interrupted, discovery-failure, and setup/test-timeout attempts remain +retained; the final unchanged-assertion retry passed. No live cells or paid +providers were run. This establishes executable coverage, not a live reliability +result or improved coding quality. A quality claim needs comparable tasks, +models, effort, tools, and independently graded before/after results. + +The suite exercises fresh isolated hires and their continuations. Existing saved +manuals and old Codex sessions are not automatically migrated. The latter still +need a provider-session reset to restore a previously replaced vendor base. +Private Runner protocol definitions remain separate: they use mock control-plane +operations and cannot substitute for this public hiring and assembled-prompt +coverage. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 49748c8a6f6..cecd20e6afd 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,10 +136,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 6, 20, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, + 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 6, 20, 24, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(444); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(444); + expect(validateRunnerCatalog()).toHaveLength(468); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(468); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 89301b7f111..1b575f5a319 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -9,6 +9,7 @@ import { blockerTasks, blockerProfile } from "./blocker-cases.js"; import { accountingTasks } from "./accounting-cases.js"; import { continuationTasks } from "./continuation-cases.js"; import { contextIntegrityTasks } from "./context-integrity-cases.js"; +import { productionDefaultHireProfile, stockHarnessSourceDigest } from "./stock-harness.js"; import { lifecycleLiveTasks, lifecycleLiveDefinitionDigest } from "./lifecycle-live-cases.js"; import { everydayTasks, productionStoryProfile } from "./everyday-cases.js"; @@ -54,6 +55,7 @@ const SELECTABLE_GROUPS = [ "chat", "onboarding", "context-integrity", + "stock-harness", ] as const; const SAMPLE_UUID = "11111111-1111-4111-8111-111111111111"; @@ -1224,6 +1226,25 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ prerequisiteGate: PENDING_PROFILE_PREREQUISITES, }, }, + { + id: "stock-harness", + label: "Stock harness with Paperclip", + manualOnly: true, + description: "Production-default hires, reduced shared prompts, assigned skills, continuation, and persistent chat.", + groups: ["stock-harness", "native", "legacy"], + profiles: [...contextIntegrityProfiles.filter(profile => !pendingContextIntegrityProfiles.includes(profile)), + runnerProfiles.find(profile => profile.id === "legacy-opencode")!].map(productionDefaultHireProfile), + environments: [localEnvironment], + tasks: [...contextIntegrityTasks, chatTasks.find(task => task.id === "continuity-restart")!], + expectedMatrixSize: 24, + definitionMetadata: { + version: 1, instructions: "production-default-hire", scheduling: "explicit-only", + sourceDigest: stockHarnessSourceDigest(), + grading: "public-default-bundle-and-delivered-prompts-plus-independent-lifecycle-oracles", + vendorBaseEvidence: "required deterministic Codex driver/runnerd/Rust gate; task success is not vendor-base proof", + paidCalls: "one skill turn, two ordered-comment turns, three chat turns per profile", + }, + }, { id: "first-task", label: "First-task onboarding", description: "Production onboarding, first replies, approval, and durable task execution.", diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index cf99914f16b..04be71d847f 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -4,6 +4,30 @@ import { runnerMatrix } from "./catalog.js"; import { setupLiveFixtures } from "./live-fixtures.js"; describe("live runner fixtures", () => { + it.each(["runner-codex", "legacy-codex", "runner-acpx-claude", "legacy-opencode"])( + "creates a production-default %s hire with company and agent budget stops", async (profile) => { + const execution = runnerMatrix.find(row => row.suite.id === "stock-harness" && row.profile.id === profile)!; + let agentBody: any; + const api = { + async get() { return [{ id: "local", driver: "local" }]; }, + async postSensitive() { return { id: "secret" }; }, + async post(url: string, data: any) { + if (url === "/api/companies") { + expect(data.budgetMonthlyCents).toBe(1_000); + return { id: "company", name: "Fixture" }; + } + if (url.endsWith("/agents")) { agentBody = data; return { id: "agent", ...data }; } + throw new Error(`Unexpected POST ${url}`); + }, + } as unknown as RunnerApi; + const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture", + credentials: { [execution.profile.credential]: "fixture-key" } }); + expect(agentBody).not.toHaveProperty("instructionsBundle"); + expect(agentBody.budgetMonthlyCents).toBe(1_000); + expect(agentBody.adapterConfig.env[execution.profile.credential]).toEqual({ type: "secret_ref", secretId: "secret", version: "latest" }); + await fixtures.teardown(); + }, + ); it("anchors extended file validation to a public project workspace for local and remote copy-back", async () => { const execution = runnerMatrix.find(e => e.id === "extended-harnesses.runner-acpx-pi.local.file-edit-validate")!; let projectBody: any; diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index da0aa8e19c9..391bdc6448f 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -136,7 +136,9 @@ export async function setupLiveFixtures(input: { return api.post("/api/companies", { name: `Runner E2E ${execution.id} ${input.executionNonce}`, description: "Ephemeral paid full-stack runner acceptance fixture", - budgetMonthlyCents: execution.suite.id === "native-completion" ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS : 0, + budgetMonthlyCents: execution.suite.id === "native-completion" ? NATIVE_COMPLETION_BUDGET_CENTS + : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS + : execution.suite.id === "stock-harness" ? 1_000 : 0, }); }, async teardown() { @@ -288,6 +290,7 @@ export async function setupLiveFixtures(input: { secretRefs, executionId: input.executionNonce, }); + if (execution.suite.id === "stock-harness") agent.budgetMonthlyCents = 1_000; if (managedHiring) { const account = value(resolved, "ai-connection"); const config = agent.adapterConfig as Record; diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 46cc8e96aaa..3026329710e 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -18,6 +18,7 @@ import { runContinuationFlow } from "./continuation-flow.js"; import { runEverydayFlow } from "./everyday-flow.js"; import { gradeTaskTitle } from "./task-titles.js"; import { runContextIntegrityFlow } from "./context-integrity-flow.js"; +import { captureStockHarness, gradeStockHarness, gradeStockHire } from "./stock-harness.js"; import { createTaskThroughUi, submitTaskReply } from "./user-actions.js"; import { runFirstTaskFlow, setupFirstTaskFixtures } from "./first-task-flow.js"; @@ -865,6 +866,18 @@ for (const execution of executions) { nativeInitial = { issueIds: issues.map(value => value.id), agentIds: agents.map(value => value.id), workspaceDigest }; } + if (execution.suite.id === "stock-harness") { + const hire = await captureStockHarness({ api, companyId: fixtures.company.id, + agentId: fixtures.agent.id, generation: execution.profile.generation, runIds: [] }); + const checks = gradeStockHire(hire); + await writeSanitizedJson(snapshotsDir, "stock-harness-hire.json", { + capturePhase: "before-provider", ...hire, checks, + }, secrets); + const failed = checks.filter(check => !check.passed); + if (failed.length) throw new Error(`Stock hire matcher failures: ${failed.map(check => check.id).join(", ")}`); + + } + if (execution.suite.id === "api-response-reading") { const source = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/issues`, { title: `Synthetic diagnostic evidence ${nonce}`, @@ -2764,6 +2777,24 @@ for (const execution of executions) { selectedRuns = await Promise.all(companyRuns.map(run => api.get(`/api/heartbeat-runs/${run.id}`))); await writeSanitizedJson(snapshotsDir, execution.task.flow === "first_task" ? "first-task-final-run-ledger.json" : "chat-final-run-ledger.json", selectedRuns, secrets); } + if (execution.suite.id === "stock-harness") { + try { + const stock = await captureStockHarness({ api, companyId: fixtures.company.id, + agentId: fixtures.agent.id, generation: execution.profile.generation, + runIds: selectedRuns.map(run => run.id) }); + const checks = gradeStockHarness(stock); + matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const, + path: `stockHarness.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }))); + await writeSanitizedJson(snapshotsDir, "stock-harness.json", { ...stock, checks }, secrets); + const failures = checks.filter(check => !check.passed); + if (failures.length && !primaryError) primaryError = new Error(`Stock harness matcher failures: ${failures.map(check => check.id).join(", ")}`); + } catch (error) { + await writeSanitizedJson(snapshotsDir, "stock-harness-evidence-error.json", { + error: error instanceof Error ? error.message : String(error), + }, secrets); + if (!primaryError) primaryError = error; + } + } await fixtures.teardown(); cleanup = "passed"; } catch (error) { diff --git a/tests/runner-e2e/stock-harness-checks.mjs b/tests/runner-e2e/stock-harness-checks.mjs new file mode 100644 index 00000000000..d3013151a74 --- /dev/null +++ b/tests/runner-e2e/stock-harness-checks.mjs @@ -0,0 +1,109 @@ +// Deterministic prerequisite for the stock-harness Product E2E suite. No model +// calls or credentials. Reuse the existing protocol assertions, not model +// self-reports about what its system instructions contain. +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +const root = resolve(import.meta.dirname, "../.."); +export const stockHarnessGates = [ + { id: "SH-1", name: "Native Codex additive instructions", cwd: "packages/paperclip-runner", files: [ + "src/drivers/codex/codex-app-server-driver.test.ts", + "src/drivers/codex/codex-app-server-driver.lifecycle.test.ts", + "src/live/runnerd-codex-transport.test.ts", + "src/live/live-session.test.ts", + "src/cli/eval-provider-runtime.test.ts", + ], testPattern: "adds Paperclip developer instructions|preserves stock Codex instructions|captures exact provider frames and correlates|direct eval provider runtime|exposes native completion consistently on fresh and resumed sessions|passes caller-supplied native system instructions", + required: ["adds Paperclip developer instructions", "preserves stock Codex instructions on task recovery", + "preserves stock Codex instructions on prepared recovery", "preserves stock Codex instructions on direct recovery", + "captures exact provider frames and correlates", "exposes native completion consistently on fresh and resumed sessions", + "passes caller-supplied native system instructions"] }, + { id: "SH-2", name: "Production-default hire bundle", cwd: ".", files: [ + "server/src/__tests__/agent-skills-routes.test.ts", + "server/src/services/onboarding-first-task-assets.test.ts", + ], required: ["materializes minimal default instructions for non-CEO agents with no prompt template"] }, + { id: "SH-3", name: "Shared legacy startup and continuation", cwd: ".", files: [ + "packages/adapter-utils/src/server-utils.test.ts", + "packages/adapter-utils/src/prompt-sections.test.ts", + "packages/adapter-utils/src/acpx-engine/execute.test.ts", + "packages/adapters/pi-local/src/server/execute.remote.test.ts", + "packages/adapters/opencode-local/src/server/execute.test.ts", + "packages/adapters/cursor-cloud/src/server/execute.test.ts", + "server/src/__tests__/codex-local-execute.test.ts", + ], required: ["keeps task and chat defaults to identity and connection guidance", + "does not restore generic procedures on resume or with the legacy opt-in"] }, + // Hermes is not in the root Vitest project list. Run its package config so + // the requested file cannot silently disappear from discovery. + { id: "SH-3-hermes", name: "Hermes shared-prompt delivery", cwd: "packages/adapters/hermes", + files: ["src/server/prompt-rendering.test.ts"], + required: ["renders standard assignment wake with task authority", "renders scoped planning wake authority"] }, +]; + +export function gradeGate(gate, report, exitCode) { + const assertions = (report?.testResults ?? []).flatMap(file => file.assertionResults ?? []); + const requirements = (gate.required ?? []).map(name => ({ name, + passed: assertions.some(assertion => assertion.fullName?.includes(name) && assertion.status === "passed") })); + const files = gate.files.map(file => ({ file, + passed: (report?.testResults ?? []).some(result => result.name?.endsWith(file) && result.status === "passed" && + result.assertionResults?.some(assertion => assertion.status === "passed") && + result.assertionResults.every(assertion => assertion.status === "passed" || + (gate.testPattern && assertion.status === "skipped" && !(gate.required ?? []).some(name => assertion.fullName?.includes(name))))) })); + return { id: gate.id, name: gate.name, passed: exitCode === 0 && requirements.every(row => row.passed) && files.every(row => row.passed), + exitCode, files, requirements, total: report?.numTotalTests ?? 0, passedTests: report?.numPassedTests ?? 0, + failedTests: report?.numFailedTests ?? 0, pendingTests: report?.numPendingTests ?? 0 }; +} + +export function main(args = process.argv.slice(2)) { + if (args.includes("--list")) { + console.log(JSON.stringify({ gates: stockHarnessGates, rust: "runtime_instructions_are_additive_for_codex_on_start_and_resume", live: "not invoked" }, null, 2)); + return; + } + if (args.length) throw new Error("Use --list or no arguments; this command never runs paid providers."); + const output = join(root, "tests/runner-e2e/results", `stock-harness-preflight-${new Date().toISOString().replaceAll(":", "-")}`); + mkdirSync(output, { recursive: true }); + const results = []; + for (const gate of stockHarnessGates) { + console.log(`Checking ${gate.id}: ${gate.name}`); + const file = join(output, `${gate.id}.json`); + const run = spawnSync(process.execPath, [join(root, "node_modules/vitest/vitest.mjs"), "run", ...gate.files, + ...(gate.testPattern ? ["--testNamePattern", gate.testPattern] : []), + "--reporter=default", "--reporter=json", `--outputFile.json=${file}`], + { cwd: resolve(root, gate.cwd), stdio: "inherit", timeout: 10 * 60_000 }); + let report; + try { report = JSON.parse(readFileSync(file, "utf8")); } catch { /* missing evidence fails closed below */ } + results.push(gradeGate(gate, report, run.status)); + } + const rust = spawnSync("cargo", ["test", "--offline", "-p", "paperclip-runner-core", "--test", "codex_provider", + "runtime_instructions_are_additive_for_codex_on_start_and_resume", "--", "--exact"], + { cwd: join(root, "packages/paperclip-runner/runner"), encoding: "utf8", timeout: 10 * 60_000 }); + const rustOutput = `${rust.stdout ?? ""}\n${rust.stderr ?? ""}`; + writeFileSync(join(output, "rust.txt"), rustOutput); + results.push({ id: "SH-1-rust", passed: rust.status === 0 && /test runtime_instructions_are_additive_for_codex_on_start_and_resume \.\.\. ok/.test(rustOutput), exitCode: rust.status }); + const git = spawnSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }); + const hash = createHash("sha256"); + const sources = new Set([ + ...stockHarnessGates.flatMap(gate => gate.files.map(file => join(gate.cwd, file))), + "tests/runner-e2e/stock-harness.ts", "tests/runner-e2e/stock-harness-checks.mjs", "tests/runner-e2e/catalog.ts", + "packages/adapter-utils/src/server-utils.ts", "server/src/onboarding-assets/default/AGENTS.md", "server/src/routes/agents.ts", + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts", + "packages/paperclip-runner/src/live/runnerd-codex-transport.ts", + "packages/paperclip-runner/src/live/live-session.ts", + "packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs", + "packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs", + ]); + const sourceErrors = []; + for (const source of [...sources].sort()) { + hash.update(source); + try { hash.update(readFileSync(join(root, source))); } + catch { sourceErrors.push(source); } + } + const report = { schema: "paperclip.stock-harness-preflight.v1", sourceSha: git.stdout?.trim() || null, + sourceFingerprint: hash.digest("hex"), measuredAt: new Date().toISOString(), providerCalls: 0, + live: "not_run", passed: sourceErrors.length === 0 && results.every(row => row.passed), sourceErrors, gates: results }; + writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n"); + console.log(`Stock harness prerequisite evidence: ${output}/preflight.json`); + if (!report.passed) process.exitCode = 1; +} + +if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) main(); diff --git a/tests/runner-e2e/stock-harness-checks.test.mjs b/tests/runner-e2e/stock-harness-checks.test.mjs new file mode 100644 index 00000000000..3397cae78c6 --- /dev/null +++ b/tests/runner-e2e/stock-harness-checks.test.mjs @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { gradeGate, stockHarnessGates } from "./stock-harness-checks.mjs"; + +const gate = { id: "SH-test", name: "Boundary", files: ["boundary.test.ts"], required: ["must preserve instructions"] }; +const report = () => ({ testResults: [{ name: "/repo/boundary.test.ts", status: "passed", + assertionResults: [{ fullName: "Boundary must preserve instructions", status: "passed" }] }], +numTotalTests: 1, numPassedTests: 1, numFailedTests: 0, numPendingTests: 0 }); +describe("stock harness prerequisite coverage", () => { + it("maps every implemented change to an executable gate", () => { + expect(stockHarnessGates.map(gate => gate.id)).toEqual(["SH-1", "SH-2", "SH-3", "SH-3-hermes"]); + expect(stockHarnessGates.every(gate => gate.files.length > 0 && gate.required.length > 0)).toBe(true); + }); + it("accepts an executed passing boundary", () => expect(gradeGate(gate, report(), 0).passed).toBe(true)); + it.each([null, undefined, { testResults: [] }])("rejects unavailable evidence %s", (report) => { + expect(gradeGate(gate, report, 0).passed).toBe(false); + }); + it.each(["pending", "skipped", "failed", "todo"])("rejects a %s required assertion", (status) => { + const observation = report(); + observation.testResults[0].assertionResults[0].status = status; + expect(gradeGate(gate, observation, 0).passed).toBe(false); + }); + it("rejects a passing different assertion and a nonzero launcher exit", () => { + const different = report(); + different.testResults[0].assertionResults[0].fullName = "Unrelated pass"; + expect(gradeGate(gate, different, 0).passed).toBe(false); + expect(gradeGate(gate, report(), 1).passed).toBe(false); + }); + it("allows explicitly filtered unrelated tests while rejecting a skipped required test", () => { + const filtered = { ...gate, testPattern: "must preserve instructions" }; + const observation = report(); + observation.testResults[0].assertionResults.push({ fullName: "Unrelated", status: "skipped" }); + expect(gradeGate(filtered, observation, 0).passed).toBe(true); + observation.testResults[0].assertionResults[0].status = "skipped"; + expect(gradeGate(filtered, observation, 0).passed).toBe(false); + }); + it("rejects a requested file absent from Vitest discovery even when required names pass", () => { + expect(gradeGate({ ...gate, files: [...gate.files, "undiscovered.test.ts"] }, report(), 0).passed).toBe(false); + expect(stockHarnessGates.find(gate => gate.id === "SH-3-hermes").cwd).toBe("packages/adapters/hermes"); + }); +}); diff --git a/tests/runner-e2e/stock-harness.test.ts b/tests/runner-e2e/stock-harness.test.ts new file mode 100644 index 00000000000..37776355e84 --- /dev/null +++ b/tests/runner-e2e/stock-harness.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix, runnerProfiles, suiteDefinitionHash } from "./catalog.js"; +import { captureStockHarness, gradeStockHarness, gradeStockHire, STOCK_HIRE_IDENTITY, type StockHarnessEvidence } from "./stock-harness.js"; + +function recording(generation: "legacy" | "native" = "legacy"): StockHarnessEvidence { + return { + schema: "paperclip.stock-harness.v1", generation, agentId: "agent", + budgets: { companyMonthlyCents: 1_000, agentMonthlyCents: 1_000 }, + bundle: { entryFile: "AGENTS.md", files: [{ path: "AGENTS.md", content: STOCK_HIRE_IDENTITY }] }, + runIds: ["fresh", "resumed"], + invocations: generation === "native" ? [] : [ + { runId: "fresh", prompt: "You are agent agent (QA).\nConnection tools:\nUse connections_search.\nCurrent assignment.", promptMetrics: { heartbeatPromptChars: 113 } }, + { runId: "resumed", prompt: "Paperclip Resume Delta\nCurrent ordered comments.", promptMetrics: { heartbeatPromptChars: 0 } }, + ], + }; +} + +describe("stock harness Product E2E", () => { + it("registers 24 explicit local cells with independent skill, continuation and chat oracles", () => { + const cells = runnerMatrix.filter(row => row.suite.id === "stock-harness"); + expect(cells).toHaveLength(24); + expect(new Set(cells.map(row => row.profile.id))).toEqual(new Set([ + "legacy-codex", "legacy-claude", "legacy-opencode", "legacy-acp-codex", "legacy-acp-claude", + "runner-codex", "runner-acpx-claude", "runner-opencode", + ])); + expect(new Set(cells.map(row => row.task.id))).toEqual(new Set([ + "ordered-comment-continuation", "assigned-skill-explicit-invocation", "continuity-restart", + ])); + expect(cells.every(row => row.suite.manualOnly && row.environment.id === "local")).toBe(true); + expect(cells.reduce((turns, row) => turns + row.task.expectedRunCount, 0)).toBe(48); + expect(cells[0]!.suite.definitionMetadata?.sourceDigest).toMatch(/^[a-f0-9]{64}$/); + }); + + it.each(["legacy-codex", "legacy-claude", "legacy-opencode", "runner-codex", "runner-acpx-claude", "runner-opencode"])( + "omits the custom QA bundle for %s while preserving runtime, auth and skills configuration", (profileId) => { + const profile = runnerMatrix.find(row => row.suite.id === "stock-harness" && row.profile.id === profileId)!.profile; + const original = runnerProfiles.find(row => row.id === profileId)!; + const input = { executionId: "test", workspacePath: "/workspace", environmentId: "environment", environmentFixtureId: "local" as const, + secretRefs: { [profile.credential]: { type: "secret_ref" as const, secretId: "secret", version: "latest" as const } } }; + const expected = original.buildAgent(input); + const { instructionsBundle: _manual, ...withoutManual } = expected; + expect(profile.buildAgent(input)).toEqual(withoutManual); + expect(profile.buildAgent(input)).not.toHaveProperty("instructionsBundle"); + }, + ); + + it.each(["legacy", "native"] as const)("accepts complete %s evidence without interpreting task success as vendor-base proof", (generation) => { + expect(gradeStockHarness(recording(generation)).every(check => check.passed)).toBe(true); + expect(gradeStockHarness(recording(generation)).some(check => /vendor|base/.test(check.id))).toBe(false); + }); + + it("checks the hire before paid execution without treating that receipt as provider evidence", () => { + const hire = recording(); + hire.runIds = []; + hire.invocations = []; + expect(gradeStockHire(hire).every(check => check.passed)).toBe(true); + expect(gradeStockHarness(hire)).toContainEqual(expect.objectContaining({ id: "provider-runs-present", passed: false })); + hire.bundle.files[0]!.content += "Old operating manual."; + expect(gradeStockHire(hire)).toContainEqual(expect.objectContaining({ id: "default-hire-bundle", passed: false })); + }); + + it.each([ + ["manual-regrew", (e: StockHarnessEvidence) => { e.bundle.files[0]!.content += "Always comment and delegate."; }, "default-hire-bundle"], + ["extra-file", (e: StockHarnessEvidence) => { e.bundle.files.push({ path: "HEARTBEAT.md", content: "Do everything again." }); }, "default-hire-bundle"], + ["wrong-entry", (e: StockHarnessEvidence) => { e.bundle.entryFile = "OTHER.md"; }, "default-hire-bundle"], + ["missing-runs", (e: StockHarnessEvidence) => { e.runIds = []; }, "provider-runs-present"], + ["missing-invocation", (e: StockHarnessEvidence) => { e.invocations.pop(); }, "invocation-evidence-complete"], + ["empty-prompt", (e: StockHarnessEvidence) => { e.invocations[1]!.prompt = ""; }, "invocation-evidence-complete"], + ["malformed-prompt", (e: StockHarnessEvidence) => { e.invocations[1]!.prompt = { text: "hidden" }; }, "invocation-evidence-complete"], + ["old-startup", (e: StockHarnessEvidence) => { e.invocations[0]!.prompt += "\nExecution contract:"; }, "generic-procedures-absent"], + ["old-resume", (e: StockHarnessEvidence) => { e.invocations[1]!.prompt += "\na successful process exit or final response is not sufficient"; }, "generic-procedures-absent"], + ["missing-connections", (e: StockHarnessEvidence) => { e.invocations[0]!.prompt = "You are agent agent."; }, "fresh-default-delivered"], + ["missing-fresh-evidence", (e: StockHarnessEvidence) => { e.invocations[0]!.promptMetrics = {}; }, "fresh-default-delivered"], + ["wrong-budget", (e: StockHarnessEvidence) => { e.budgets.agentMonthlyCents = 0; }, "budget-hard-stops"], + ] as const)("fails closed for %s", (_name, mutate, failedId) => { + const evidence = recording(); + mutate(evidence); + expect(gradeStockHarness(evidence)).toContainEqual(expect.objectContaining({ id: failedId, passed: false })); + }); + + it("captures actual public bundle and invocation receipts, ignoring unrelated events", async () => { + const paths: string[] = []; + const api = { async get(url: string): Promise { + paths.push(url); + if (url.endsWith("/instructions-bundle")) return { entryFile: "AGENTS.md", files: [{ path: "AGENTS.md" }] } as T; + if (url.includes("/file?")) return { content: STOCK_HIRE_IDENTITY } as T; + if (url.includes("/events?")) return [ + { eventType: "adapter.invoke", payload: { prompt: "Actual prompt", promptMetrics: { heartbeatPromptChars: 10 } } }, + { eventType: "assistant", payload: { prompt: "Model claims about instructions" } }, + ] as T; + return { budgetMonthlyCents: 1_000 } as T; + } }; + const evidence = await captureStockHarness({ api, companyId: "company", agentId: "agent", generation: "legacy", runIds: ["run"] }); + expect(evidence.bundle.files[0]?.content).toBe(STOCK_HIRE_IDENTITY); + expect(evidence.invocations).toHaveLength(1); + expect(evidence.invocations[0]).toMatchObject({ runId: "run", prompt: "Actual prompt" }); + expect(paths).toContain("/api/heartbeat-runs/run/events?limit=1000"); + }); + + it("does not hide evidence API failures", async () => { + const api = { async get(): Promise { throw new Error("Public API unavailable"); } }; + await expect(captureStockHarness({ api, companyId: "company", agentId: "agent", generation: "legacy", runIds: ["run"] })) + .rejects.toThrow("Public API unavailable"); + }); + + it("changes the definition hash when the grader digest changes", () => { + const suite = runnerMatrix.find(row => row.suite.id === "stock-harness")!.suite; + expect(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, sourceDigest: "changed" } })) + .not.toBe(suiteDefinitionHash(suite)); + }); +}); diff --git a/tests/runner-e2e/stock-harness.ts b/tests/runner-e2e/stock-harness.ts new file mode 100644 index 00000000000..5993e06cb17 --- /dev/null +++ b/tests/runner-e2e/stock-harness.ts @@ -0,0 +1,115 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerProfileFixture } from "./types.js"; + +// These are an independent product contract, not imports of the implementation +// constants. Otherwise a larger shipped manual could silently update the oracle. +export const STOCK_HIRE_IDENTITY = "You are an agent in a Paperclip company.\n"; +export const STOCK_TEMPLATE_IDENTITY = "You are agent"; +const REMOVED_PROCEDURES = [ + "Execution contract:", + "Start actionable work in this heartbeat", + "clear final disposition", + "After 2 consecutive failures of the same control-plane write", + "Use child issues for parallel or long delegated work", + "a successful process exit or final response is not sufficient", +]; + +export function productionDefaultHireProfile(profile: RunnerProfileFixture): RunnerProfileFixture { + return { + ...profile, + buildAgent(input) { + const { instructionsBundle: _fixtureManual, ...agent } = profile.buildAgent(input); + return agent; + }, + }; +} + +export interface StockHarnessEvidence { + schema: "paperclip.stock-harness.v1"; + generation: "legacy" | "native"; + agentId: string; + budgets: { companyMonthlyCents: unknown; agentMonthlyCents: unknown }; + bundle: { entryFile?: string; files: Array<{ path: string; content: string }> }; + invocations: Array<{ runId: string; prompt: unknown; promptMetrics?: Record }>; + runIds: string[]; +} + +export function gradeStockHire(evidence: Pick) { + const checks: Array<{ id: string; passed: boolean; detail: string }> = []; + const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail }); + check("budget-hard-stops", evidence.budgets.companyMonthlyCents === 1_000 && evidence.budgets.agentMonthlyCents === 1_000, + "Public company and agent records must both retain the 1,000-cent monthly hard stops."); + check("default-hire-bundle", evidence.bundle.entryFile === "AGENTS.md" && + evidence.bundle.files.length === 1 && evidence.bundle.files[0]?.path === "AGENTS.md" && + evidence.bundle.files[0]?.content === STOCK_HIRE_IDENTITY, + "The public managed bundle must contain only the eight-word shipped identity, without an injected QA manual."); + return checks; +} + +export function gradeStockHarness(evidence: StockHarnessEvidence) { + const checks = gradeStockHire(evidence); + const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail }); + check("provider-runs-present", evidence.runIds.length > 0 && evidence.runIds.every(Boolean), + "The lifecycle oracle must reach actual provider runs; missing runs cannot pass instruction delivery."); + if (evidence.generation === "legacy") { + const prompts = evidence.invocations.filter(row => typeof row.prompt === "string" && row.prompt.length > 0); + check("invocation-evidence-complete", evidence.runIds.length > 0 && evidence.runIds.every(runId => + prompts.some(row => row.runId === runId)), + "Every legacy run must have a public adapter.invoke event with its actual nonempty prompt."); + check("generic-procedures-absent", prompts.length > 0 && prompts.every(row => + REMOVED_PROCEDURES.every(procedure => !String(row.prompt).includes(procedure))), + "Neither startup nor continuation may reintroduce the removed generic manual."); + const fresh = prompts.filter(row => Number(row.promptMetrics?.heartbeatPromptChars) > 0); + check("fresh-default-delivered", fresh.length > 0 && fresh.every(row => + String(row.prompt).includes(STOCK_TEMPLATE_IDENTITY) && String(row.prompt).includes("Connection tools:") && + String(row.prompt).includes("connections_search")), + "At least one fresh invocation must carry default identity and connection guidance."); + } + return checks; +} + +/** Public API receipts only; no provider-memory claim or private runner hook. */ +export async function captureStockHarness(input: { + api: { get(path: string): Promise }; + agentId: string; + companyId: string; + generation: "legacy" | "native"; + runIds: string[]; +}): Promise { + const bundle = await input.api.get<{ entryFile: string; files: Array<{ path: string }> }>( + `/api/agents/${input.agentId}/instructions-bundle`, + ); + const [company, agent] = await Promise.all([ + input.api.get<{ budgetMonthlyCents: unknown }>(`/api/companies/${input.companyId}`), + input.api.get<{ budgetMonthlyCents: unknown }>(`/api/agents/${input.agentId}`), + ]); + const files = await Promise.all(bundle.files.map(async file => { + const detail = await input.api.get<{ content: string }>( + `/api/agents/${input.agentId}/instructions-bundle/file?path=${encodeURIComponent(file.path)}`, + ); + return { path: file.path, content: detail.content }; + })); + const events = await Promise.all(input.runIds.map(async runId => ({ + runId, + rows: await input.api.get }>>( + `/api/heartbeat-runs/${runId}/events?limit=1000`, + ), + }))); + return { + schema: "paperclip.stock-harness.v1", generation: input.generation, + budgets: { companyMonthlyCents: company.budgetMonthlyCents, agentMonthlyCents: agent.budgetMonthlyCents }, + agentId: input.agentId, bundle: { entryFile: bundle.entryFile, files }, runIds: input.runIds, + invocations: events.flatMap(({ runId, rows }) => rows.filter(row => row.eventType === "adapter.invoke") + .map(row => ({ runId, prompt: row.payload?.prompt, promptMetrics: row.payload?.promptMetrics as Record | undefined }))), + }; +} + +export function stockHarnessSourceDigest() { + const hash = createHash("sha256"); + for (const source of [ + "stock-harness.ts", "context-integrity-cases.ts", "context-integrity-scoring.ts", + "context-integrity-flow.ts", "chat-cases.ts", "chat-flow.ts", "live-fixtures.ts", "runner.spec.ts", + ]) hash.update(source).update(readFileSync(new URL(source, import.meta.url))); + return hash.digest("hex"); +} diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index 38dcf1b369a..fd13ba8fd34 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ root: import.meta.dirname, test: { name: "runner-e2e-support", - include: ["**/*.test.ts"], + include: ["**/*.test.ts", "**/*.test.mjs"], testTimeout: 30_000, }, });