From 34e3910b30cee8b7d938f953d1ff720b0f9fc798 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 11:54:05 -0500 Subject: [PATCH] test(evals): enforce matched historical qualification prerequisites Restore only the prior instruction sources and their structural tests. Keep behavioral grading, fixtures, models, auth, tools, and native Codex additive instructions identical to candidate 36e987246. Co-Authored-By: Paperclip --- ...10-02-stock-harness-paperclip-checklist.md | 18 ++- .../src/acpx-engine/execute.test.ts | 17 +-- .../adapter-utils/src/prompt-sections.test.ts | 13 +- .../adapter-utils/src/server-utils.test.ts | 134 ++++++++++++++---- .../cursor-cloud/src/server/execute.test.ts | 5 +- .../src/server/prompt-rendering.test.ts | 6 +- .../opencode-local/src/server/execute.test.ts | 5 +- .../src/server/execute.remote.test.ts | 12 +- .../src/__tests__/agent-skills-routes.test.ts | 32 ++++- .../src/__tests__/codex-local-execute.test.ts | 3 +- tests/runner-e2e/STOCK-HARNESS.md | 56 +++++++- tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/launch.ts | 4 + tests/runner-e2e/runner.spec.ts | 5 + .../stock-harness-admission.test.ts | 43 ++++++ tests/runner-e2e/stock-harness-admission.ts | 36 +++++ tests/runner-e2e/stock-harness-checks.mjs | 112 ++++++++++----- .../runner-e2e/stock-harness-checks.test.mjs | 24 +++- .../runner-e2e/stock-harness-comparison.json | 6 +- tests/runner-e2e/stock-harness-digest.test.ts | 15 ++ tests/runner-e2e/stock-harness.ts | 7 +- 21 files changed, 436 insertions(+), 119 deletions(-) create mode 100644 tests/runner-e2e/stock-harness-admission.test.ts create mode 100644 tests/runner-e2e/stock-harness-admission.ts create mode 100644 tests/runner-e2e/stock-harness-digest.test.ts diff --git a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md index c333ae53856..1455e78acf4 100644 --- a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md +++ b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md @@ -3,8 +3,9 @@ Created: 2026-10-02. Status: item 1 merged for native Codex app-server in [PR #14920](https://github.com/paperclipai/paperclip/pull/14920). Item 2's default hire manual is reduced to identity only, and common legacy startup/resume -instructions are reduced locally. Additional carriers and native instructions -remain open. +instructions are in [PR #14948](https://github.com/paperclipai/paperclip/pull/14948). +GitHub live qualification and a matched prior-instruction comparison are in progress. +Additional carriers and native instructions remain open. Goal: keep the agent's stock harness behavior and add only what it needs to work with Paperclip. Apply this across legacy adapters, the new Runner, and their @@ -118,7 +119,8 @@ and existing-test update. specialized wake contracts, custom templates, skills, and auth. - [ ] **2.2 Review additional legacy carriers.** Reduce Hermes local/gateway wrappers, review Pi system delivery, and check OpenClaw fresh-wake framing. - Preserve transport facts and user configuration. + Preserve transport facts and user configuration. Dotta deferred this item on + 2026-10-02 for a later revisit; it remains open. - [ ] **2.3 Reduce native Runner instructions and constraints.** Improve discoverable tool documentation first, then shorten fixed guidance and consolidate completion rules. Verify prompt revisions, digests, and session @@ -140,6 +142,16 @@ custom `promptTemplate`, and `bootstrapPromptTemplate` mechanics are unchanged. Hermes's own wrappers and Pi's system carrier remain follow-up 2.2; native fixed instructions and full-turn constraints remain follow-up 2.3. +The later read-only 2.2 audit found another OpenClaw-owned HTTP identity, +checkout/status, delegation, plan-approval and task-discovery wrapper in +`buildWakeText`. Its short conversation branch is selected when optional task +Markdown is present, including on ordinary task dispatch. Existing dispatch +coverage does not assert the absence of conversation waiting language. This is +a framing concern to verify, not a measured failure. Pi already uses additive +`--append-system-prompt` delivery and suppresses the duplicate user-prompt copy; +its next step is a delivery audit with minimal changes. These findings are +deferred with 2.2 and are not implemented in PR #14948. + The new defaults apply when prompts are assembled after deployment. Existing provider sessions can retain earlier startup instructions in their history until reset; no active session or saved custom template is rewritten here. diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index c3d42b7ade4..07d6070d409 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -596,9 +596,6 @@ describe("shared ACPX engine runtime behavior", () => { const fresh = await runExecutor(config, { context }); expect(fresh.turnInputs).toHaveLength(1); const prompt = String(fresh.turnInputs[0]?.text ?? ""); - expect(prompt).toContain("You are agent agent-1"); - expect(prompt).toContain("Connection tools:"); - expect(prompt).not.toContain("Execution contract:"); expect(prompt).toContain(context.paperclipTaskMarkdownAssignment); expect(prompt).toContain(context.paperclipTaskCommunicationGuidance); expect(prompt).not.toContain('"objective":'); @@ -612,18 +609,10 @@ describe("shared ACPX engine runtime behavior", () => { const resumed = await runExecutor(config, { context, runtime: { sessionParams: fresh.result.sessionParams } }); expect(resumed.sessionInputs[0]?.resumeSessionId).toBe(fresh.result.sessionId); const resumedPrompt = String(resumed.turnInputs[0]?.text ?? ""); - expect(resumedPrompt).not.toContain("Execution contract:"); expect(resumedPrompt).toContain(context.paperclipTaskMarkdownAssignmentCompact); expect(resumedPrompt).not.toContain(context.paperclipTaskCommunicationGuidance); expect(resumedPrompt).not.toContain('"id":"comment-first"'); expect(resumedPrompt).toContain('"id":"comment-second"'); - const reset = await runExecutor(config, { context }); - expect(reset.sessionInputs[0]?.resumeSessionId).toBeUndefined(); - const resetPrompt = String(reset.turnInputs[0]?.text ?? ""); - expect(resetPrompt).toContain("You are agent agent-1"); - expect(resetPrompt).toContain("Connection tools:"); - expect(resetPrompt).toContain(context.paperclipTaskMarkdownAssignment); - expect(resetPrompt).not.toContain("Execution contract:"); }); it("includes Paperclip env and API access notes in the ACPX prompt without leaking the token", async () => { @@ -757,10 +746,10 @@ describe("shared ACPX engine runtime behavior", () => { expect(prompt).not.toContain("Create child issues"); expect(prompt).not.toContain("Use child issues"); } - expect(String(fresh.meta[0]?.prompt)).toContain(custom ? "Custom agent instructions." : "You are agent agent-1"); - expect(String(reset.meta[0]?.prompt)).toContain(custom ? "Custom agent instructions." : "You are agent agent-1"); + expect(String(fresh.meta[0]?.prompt)).toContain(custom ? "Custom agent instructions." : "Continue your Paperclip conversation"); + expect(String(reset.meta[0]?.prompt)).toContain(custom ? "Custom agent instructions." : "Continue your Paperclip conversation"); const ordinary = await runExecutor({ ...config, promptTemplate: "" }, { context: { ...context, conversationMode: false } }); - expect(String(ordinary.meta[0]?.prompt)).not.toContain("Execution contract:"); + expect(String(ordinary.meta[0]?.prompt)).toContain("Execution contract:"); expect(String(ordinary.meta[0]?.prompt)).toContain("Create child issues from the approved plan"); }); diff --git a/packages/adapter-utils/src/prompt-sections.test.ts b/packages/adapter-utils/src/prompt-sections.test.ts index bc660ccfde8..6c8e67192a6 100644 --- a/packages/adapter-utils/src/prompt-sections.test.ts +++ b/packages/adapter-utils/src/prompt-sections.test.ts @@ -3,14 +3,13 @@ import { selectPaperclipPromptSections as selectSections } from "./server-utils. import { createPromptContextFixture } from "./test-fixtures/prompt-context.js"; describe("task and event section ownership", () => { - it("preserves resumed wake data without reintroducing generic procedures", () => { + it("lets a separate instruction carrier own the execution contract on a resumed turn", () => { const context = createPromptContextFixture(); - for (const includeExecutionContract of [undefined, false, true]) { - const sections = selectSections(context, { resumedSession: true, includeExecutionContract }); - expect(sections.taskContextNote).toBe(context.paperclipTaskMarkdownAssignmentCompact); - expect(sections.wakePrompt).toContain('"id":"comment-second"'); - expect(sections.wakePrompt).not.toContain("Execution contract:"); - } + const sections = selectSections(context, { resumedSession: true, includeExecutionContract: false }); + expect(sections.taskContextNote).toBe(context.paperclipTaskMarkdownAssignmentCompact); + expect(sections.wakePrompt).toContain('"id":"comment-second"'); + expect(sections.wakePrompt).not.toContain("Execution contract:"); + expect(selectSections(context, { resumedSession: true }).wakePrompt).toContain("Execution contract:"); }); it("preserves user repetition and distinct same-body comments under their source owners", () => { diff --git a/packages/adapter-utils/src/server-utils.test.ts b/packages/adapter-utils/src/server-utils.test.ts index 17d776f27d1..b0e1125c672 100644 --- a/packages/adapter-utils/src/server-utils.test.ts +++ b/packages/adapter-utils/src/server-utils.test.ts @@ -924,7 +924,7 @@ describe("renderPaperclipWakePrompt", () => { fallbackFetchNeeded: false, }; const ordinary = renderPaperclipWakePrompt(payload, { resumedSession: true }); - expect(ordinary).not.toContain("Execution contract:"); + expect(ordinary).toContain("Execution contract:"); expect(ordinary).toContain("Create child issues from the approved plan"); for (const resumedSession of [false, true]) { const chat = renderPaperclipWakePrompt(payload, { @@ -1170,7 +1170,7 @@ describe("renderPaperclipWakePrompt", () => { expect(incompleteResumePrompt).toContain( "[continuation summary truncated]", ); - expect(incompleteResumePrompt).not.toContain( + expect(incompleteResumePrompt).toContain( "a successful process exit or final response is not sufficient", ); }); @@ -1409,18 +1409,76 @@ describe("renderPaperclipWakePrompt", () => { ); }); - it("keeps task and chat defaults to identity and connection guidance", () => { - for (const template of [ - DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE, - DEFAULT_PAPERCLIP_CONVERSATION_PROMPT_TEMPLATE, - ]) { - expect(template).toBe( - `You are agent {{agent.id}} ({{agent.name}}).\n\n${CONNECTION_INTENT_AGENT_GUIDANCE}`, - ); - } + it("keeps the default local-agent prompt action-oriented", () => { + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Start actionable work in this heartbeat", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "do not stop at a plan", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "clear final disposition", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "evidence, not valid liveness paths by themselves", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "keep `in_progress` only when a live continuation path exists", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Prefer the smallest verification that proves the change", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "After 2 consecutive failures of the same control-plane write", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "adapter/runtime status channel as the sanctioned fallback", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Use child issues", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "instead of polling agents, sessions, or processes", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Create child issues directly when you know what needs to be done", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "POST /api/issues/$PAPERCLIP_TASK_ID/interactions", + ); + // URL paths in prompt text carry real ids or env vars, never brace + // placeholders: agents paste these lines verbatim, and a literal {issueId} + // reaches the server as /api/issues/%7BissueId%7D and 404s. + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).not.toContain( + "/api/issues/{issueId}", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).not.toContain( + "/api/issues/{id}", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "kind suggest_tasks, ask_user_questions, or request_confirmation", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Use continuationPolicy wake_assignee when you need to resume after a response (it wakes on acceptance and rejection alike; only expiry does not wake); use wake_assignee_on_accept when you want to resume only after acceptance", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).not.toContain( + "for request_confirmation this resumes only after acceptance", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Never create probe or throwaway issue-thread interactions to discover the interactions API shape or your permissions", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "confirmation:{issueId}:plan:{revisionId}", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Wait for acceptance before creating implementation subtasks", + ); + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( + "Respect budget, pause/cancel, approval gates, and company boundaries", + ); }); - it("keeps current task data without generic procedures on fresh scoped wake prompts", () => { + it("leaves the execution contract to the heartbeat template on fresh scoped wake prompts", () => { const prompt = renderPaperclipWakePrompt({ reason: "issue_assigned", issue: { @@ -1440,12 +1498,12 @@ describe("renderPaperclipWakePrompt", () => { expect(prompt).toContain("## Paperclip Wake Payload"); expect(prompt).not.toContain("Execution contract:"); - expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).not.toContain( + expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain( "Execution contract:", ); }); - it("does not restore generic procedures on resume or with the legacy opt-in", () => { + it("adds the execution contract to resume delta prompts and opted-in fresh prompts", () => { const payload = { reason: "issue_assigned", issue: { @@ -1463,19 +1521,36 @@ describe("renderPaperclipWakePrompt", () => { fallbackFetchNeeded: false, }; - for (const resumedSession of [false, true]) { - for (const includeExecutionContract of [undefined, false, true]) { - const prompt = renderPaperclipWakePrompt(payload, { - resumedSession, includeExecutionContract, - }); - expect(prompt).toContain(resumedSession ? "## Paperclip Resume Delta" : "## Paperclip Wake Payload"); - expect(prompt).toContain("- reason: issue_assigned"); - expect(prompt).toContain("- issue: PAP-1580 Update prompts"); - expect(prompt).toContain("- issue status: in_progress"); - expect(prompt).not.toContain("Execution contract:"); - expect(prompt).not.toContain("clear final disposition"); - expect(prompt).not.toContain("do not end the run"); - } + for (const prompt of [ + renderPaperclipWakePrompt(payload, { resumedSession: true }), + renderPaperclipWakePrompt(payload, { includeExecutionContract: true }), + ]) { + expect(prompt).toContain( + "Execution contract: take concrete action in this heartbeat", + ); + expect(prompt).toContain("clear final disposition"); + expect(prompt).toContain( + "Immediately before returning, verify that Paperclip records one of those dispositions", + ); + expect(prompt).toContain( + "a successful process exit or final response is not sufficient", + ); + expect(prompt).toContain( + "If no valid disposition is recorded, record it now and do not end the run", + ); + expect(prompt).toContain( + "After 2 consecutive failures of the same control-plane write", + ); + expect(prompt).toContain( + "adapter/runtime status channel as the sanctioned fallback", + ); + expect(prompt).toContain( + "evidence, not valid liveness paths by themselves", + ); + expect(prompt).toContain( + "Use child issues for long or parallel delegated work instead of polling", + ); + expect(prompt).toContain("named unblock owner/action"); } }); @@ -1871,7 +1946,7 @@ describe("renderPaperclipWakePrompt", () => { ); }); - it("keeps connection guidance without a generic manual in a composed fresh prompt", () => { + it("keeps exactly one execution contract in a composed fresh heartbeat prompt", () => { const wakePrompt = renderPaperclipWakePrompt({ reason: "issue_assigned", issue: { @@ -1891,8 +1966,7 @@ describe("renderPaperclipWakePrompt", () => { const composed = [wakePrompt, DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE].join( "\n\n", ); - expect(composed).toContain(CONNECTION_INTENT_AGENT_GUIDANCE); - expect(composed).not.toContain("Execution contract:"); + expect(composed.match(/Execution contract/g)).toHaveLength(1); }); it("trims comment-batch boilerplate on fresh wakes with zero pending comments", () => { diff --git a/packages/adapters/cursor-cloud/src/server/execute.test.ts b/packages/adapters/cursor-cloud/src/server/execute.test.ts index b57f8207814..db4617fa816 100644 --- a/packages/adapters/cursor-cloud/src/server/execute.test.ts +++ b/packages/adapters/cursor-cloud/src/server/execute.test.ts @@ -164,7 +164,7 @@ describe("cursor_cloud execute", () => { expect(result.exitCode).toBe(0); const prompt = String(sdkAgent.send.mock.calls[0]?.[0]); expect(prompt).toContain(directive); - expect(prompt).toContain(custom ? "Do the work for" : "You are agent agent-1"); + expect(prompt).toContain(custom ? "Do the work for" : "Continue your Paperclip conversation"); expect(prompt).not.toContain("Execution contract:"); expect(prompt).not.toContain("Create child issues"); }); @@ -178,9 +178,6 @@ describe("cursor_cloud execute", () => { expect(result.exitCode).toBe(0); const prompt = String(sdkAgent.send.mock.calls[0]?.[0]); expect(prompt).toContain("## Owned assignment"); - expect(prompt).toContain("You are agent agent-1 (Cursor Cloud Agent)."); - expect(prompt).toContain("Connection tools:"); - expect(prompt).not.toContain("Execution contract:"); expect(prompt.indexOf("Append the same ledger entry.")).toBeLessThan(prompt.indexOf("Change the final scope to the launch checklist.")); expect(prompt.split("Append the same ledger entry.")).toHaveLength(3); }); diff --git a/packages/adapters/hermes/src/server/prompt-rendering.test.ts b/packages/adapters/hermes/src/server/prompt-rendering.test.ts index b47dca03619..d12da187587 100644 --- a/packages/adapters/hermes/src/server/prompt-rendering.test.ts +++ b/packages/adapters/hermes/src/server/prompt-rendering.test.ts @@ -82,7 +82,7 @@ test("renders standard assignment wake with task authority and no backlog discov expect(prompt).toContain("Paperclip task context:"); expect(prompt).toContain("Add focused unit tests for assignment wake and custom prompt rendering."); expect(prompt).toContain("The harness already checked out this issue for the current run."); - expect(prompt).not.toContain("clear final disposition"); + expect(prompt).toContain("clear final disposition"); expect(prompt).not.toContain("check for unassigned issues"); expect(prompt).not.toContain("status=backlog"); }); @@ -130,8 +130,8 @@ test("renders scoped planning wake authority before the Hermes default workflow" expect(prompt).toContain("- checkout: already claimed by the harness for this run"); expect(prompt).toContain("The harness already checked out this issue for the current run."); expect(prompt).toContain("Issue description:\n```text\nUse the wake payload as runtime authority.\n```"); - expect(prompt).not.toContain("clear final disposition"); - expect(prompt).not.toContain("keep `in_progress` only when a live continuation path exists"); + expect(prompt).toContain("clear final disposition"); + expect(prompt).toContain("keep `in_progress` only when a live continuation path exists"); expect(prompt).not.toContain("check for unassigned issues"); expect(prompt).not.toContain("status=backlog"); }); diff --git a/packages/adapters/opencode-local/src/server/execute.test.ts b/packages/adapters/opencode-local/src/server/execute.test.ts index a754b9d4b97..d1480d95b2d 100644 --- a/packages/adapters/opencode-local/src/server/execute.test.ts +++ b/packages/adapters/opencode-local/src/server/execute.test.ts @@ -77,7 +77,7 @@ describe("OpenCode local skill injection", () => { }); expect(result.exitCode).toBe(0); expect(prompt).toContain(directive); - expect(prompt).toContain(custom ? "Custom agent instruction." : "You are agent agent-1"); + expect(prompt).toContain(custom ? "Custom agent instruction." : "Continue your Paperclip conversation"); expect(prompt).not.toContain("Execution contract:"); expect(prompt).not.toContain("Create child issues"); }); @@ -102,9 +102,6 @@ describe("OpenCode local skill injection", () => { expect(prompts).toHaveLength(2); expect(prompts[0]).toContain("## Compact assignment"); expect(prompts[1]).toContain("## Owned assignment"); - for (const prompt of prompts) expect(prompt).not.toContain("Execution contract:"); - expect(prompts[1]).toContain("You are agent agent-1 (OpenCode)."); - expect(prompts[1]).toContain("Connection tools:"); }); it("injects runtime skills into the configured child HOME", async () => { diff --git a/packages/adapters/pi-local/src/server/execute.remote.test.ts b/packages/adapters/pi-local/src/server/execute.remote.test.ts index 6f443902ff3..99cd81345c0 100644 --- a/packages/adapters/pi-local/src/server/execute.remote.test.ts +++ b/packages/adapters/pi-local/src/server/execute.remote.test.ts @@ -696,7 +696,7 @@ describe("pi remote execution", () => { expect(userPrompt).toBe("CUSTOM POLICY run-custom-policy"); }); - it("keeps resumed default identity and connection guidance in the system carrier", async () => { + it("keeps the resumed default execution contract in the system carrier only", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pi-resumed-policy-")); cleanupDirs.push(rootDir); const sessionPath = path.join(rootDir, "session.jsonl"); @@ -726,13 +726,11 @@ describe("pi remote execution", () => { const args = call?.[2] ?? []; const systemPrompt = args[args.indexOf("--append-system-prompt") + 1] ?? ""; const userPrompt = args.at(-1) ?? ""; - expect(systemPrompt).toContain("You are agent agent-1 (Pi Builder)."); - expect(systemPrompt).toContain("Connection tools:"); - expect(systemPrompt).not.toContain("Execution contract:"); + expect(systemPrompt).toContain("Execution contract:"); expect(userPrompt).not.toContain("Execution contract:"); }); - it("keeps default identity in system input when custom prompt uses loaded instructions", async () => { + it("keeps the default contract in system input when custom prompt uses loaded instructions", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pi-instructions-policy-")); cleanupDirs.push(rootDir); const sessionPath = path.join(rootDir, "session.jsonl"); @@ -771,9 +769,7 @@ describe("pi remote execution", () => { const systemPrompt = args[args.indexOf("--append-system-prompt") + 1] ?? ""; const userPrompt = args.at(-1) ?? ""; expect(systemPrompt).toContain("Loaded instructions for this run."); - expect(systemPrompt).toContain("You are agent agent-1 (Pi Builder)."); - expect(systemPrompt).toContain("Connection tools:"); - expect(systemPrompt).not.toContain("Execution contract:"); + expect(systemPrompt).toContain("Execution contract:"); expect(userPrompt).not.toContain("CUSTOM POLICY run-instructions-policy"); expect(userPrompt).not.toContain("Execution contract:"); }); diff --git a/server/src/__tests__/agent-skills-routes.test.ts b/server/src/__tests__/agent-skills-routes.test.ts index fd958478067..ff8fd049ac2 100644 --- a/server/src/__tests__/agent-skills-routes.test.ts +++ b/server/src/__tests__/agent-skills-routes.test.ts @@ -1145,7 +1145,7 @@ describe.sequential("agent skill routes", () => { ); }); - it("materializes minimal default instructions for non-CEO agents with no prompt template", async () => { + it("materializes the bundled default instruction set for non-CEO agents with no prompt template", async () => { const res = await requestApp(await createApp(), (baseUrl) => request(baseUrl) .post("/api/companies/company-1/agents") .send({ @@ -1165,10 +1165,38 @@ describe.sequential("agent skill routes", () => { adapterType: "claude_local", }), expect.objectContaining({ - "AGENTS.md": "You are an agent in a Paperclip company.\n", + "AGENTS.md": expect.stringMatching(/Start actionable work in the same heartbeat\.[\s\S]*Keep the work moving until it is done\./), }), { entryFile: "AGENTS.md", replaceExisting: false }, ); + expect(mockAgentInstructionsService.materializeManagedBundle).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + "AGENTS.md": expect.stringContaining('kind: "request_confirmation"'), + }), + expect.any(Object), + ); + expect(mockAgentInstructionsService.materializeManagedBundle).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + "AGENTS.md": expect.stringContaining("confirmation:{issueId}:plan:{revisionId}"), + }), + expect.any(Object), + ); + expect(mockAgentInstructionsService.materializeManagedBundle).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + "AGENTS.md": expect.stringMatching(/PUT \/issues\/\{id\}\/documents\/plan[\s\S]*Re-`GET \/documents\/plan`, assert it returns `200`[\s\S]*latestRevisionId[\s\S]*target=\{ type: 'issue_document', key: 'plan', revisionId: latestRevisionId \}[\s\S]*Never present a plan only in a thread comment or through `ask_user_questions`/), + }), + expect.any(Object), + ); + expect(mockAgentInstructionsService.materializeManagedBundle).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + "AGENTS.md": expect.stringContaining("skills/paperclip/scripts/paperclip-upload-artifact.sh"), + }), + expect.any(Object), + ); }); }); diff --git a/server/src/__tests__/codex-local-execute.test.ts b/server/src/__tests__/codex-local-execute.test.ts index 29a3a2b2255..983fa6e87ef 100644 --- a/server/src/__tests__/codex-local-execute.test.ts +++ b/server/src/__tests__/codex-local-execute.test.ts @@ -1443,8 +1443,9 @@ process.exit(1); expect(invocationPrompt).toContain("baseRevisionId set to that latestRevisionId"); expect(capture.prompt).not.toContain("Execution contract:"); expect(capture.prompt).not.toContain("Use child issues"); + } else { + expect(capture.prompt).toContain("Execution contract:"); } - expect(capture.prompt).not.toContain("Execution contract:"); expect(capture.prompt).not.toContain("Follow the paperclip heartbeat."); if (resumedSession) { expect(capture.prompt).not.toContain("You are managed instructions."); diff --git a/tests/runner-e2e/STOCK-HARNESS.md b/tests/runner-e2e/STOCK-HARNESS.md index 605ab5b13ce..bfa2156939e 100644 --- a/tests/runner-e2e/STOCK-HARNESS.md +++ b/tests/runner-e2e/STOCK-HARNESS.md @@ -20,8 +20,17 @@ including the Rust test. It writes JSON reports, the source SHA, a working-sourc fingerprint, selected checks, test counts, and `providerCalls: 0` under `results/stock-harness-preflight-/`. Missing reports or required skipped assertions fail. Unrelated native tests filtered by the name selector remain -explicitly skipped; they are not counted as executed coverage. Run this gate -before qualifying live cells; the live launcher does not run Cargo implicitly. +explicitly skipped; they are not counted as executed coverage. The live launcher +runs the prerequisites automatically before loading local credentials or starting +an isolated provider instance. Its subprocess receives only allowlisted toolchain +and operating-system variables. Disposable GitHub runners may resolve Cargo +dependencies; local prerequisites retain offline Cargo execution. + +The direct Playwright path also requires the retained prerequisite receipt. It +verifies the exact checkout SHA, evaluated-source fingerprint, all requested +Vitest reports and required assertions, and the Rust result before creating a +company. Missing, stale, partial, or failed prerequisites cannot qualify a cell. +Oracle/admission calibration is itself included in the prerequisite gate. ## Live matrix @@ -77,8 +86,9 @@ existing secret sanitizer; screenshots remain the original captured pixels. The oracle's identity is independent of the implementation constant, so changing the shipped manual cannot silently change the expected result. The suite -definition digest incorporates the fixture, journeys, grader, and execution -integration sources. Positive and plausible-negative support tests exercise +definition digest incorporates the evaluated default manual and shared prompt +implementation, fixture, journeys, grader, prerequisite, and execution integration +sources. Positive and plausible-negative support tests exercise missing/malformed receipts, manual regrowth, removed startup/resume procedures, absent connection guidance, and budget drift. Existing calibrated lifecycle graders still own task/chat success. @@ -108,3 +118,41 @@ need a provider-session reset to restore a previously replaced vendor base. Private Runner protocol definitions remain separate: they use mock control-plane operations and cannot substitute for this public hiring and assembled-prompt coverage. + +## GitHub qualification and matched comparison + +The instruction reductions and coverage are under review in +[PR #14948](https://github.com/paperclipai/paperclip/pull/14948). +The first diagnostic native Codex skill cell +[passed on GitHub](https://github.com/paperclipai/paperclip/actions/runs/37034213743) +at `a63437069de58d22ee5adbcb6a6202c007dcf037`. All seven independent skill/task +checks passed; the provider run lasted 34.745 seconds and the cell 56.660 seconds. +Its tiny public hire bundle and budget receipts passed, and cleanup passed. +This diagnostic predates enforced prerequisite admission and the expanded source +digest, so it is retained separately from the final qualification matrix. + +Dispatch the trusted workflow from `master`, with `target_branch` naming the +same-repository candidate and an exact cell selector first. The workflow resolves +that target once to an immutable SHA. Never dispatch target-controlled workflow +definitions with protected credentials. Protected environments, scoped provider +keys, frozen target dependencies, report sanitization, publication, and existing +bounded retry/cleanup policies retain their existing owners. + +A temporary `codex/stock-harness-previous-instructions` branch compares the same +24 cells with the previous default manual and shared startup/resume prompts. +It holds merged native Codex fix #14920 constant. Only those two production +instruction sources differ. Its explicit historical structural oracle expects +the old manual and records old generic procedures; the candidate's reduction +assertions remain mandatory. Historical tests verify that prior contract. +The independent skill/context/chat journeys, behavioral graders, fixtures, +models, effort, tools, permissions, and credentials are identical. The retained +comparison manifest records their hashes and the restored instruction revision. + +One full campaign per variant initially expects 48 provider turns, plus any +existing bounded automatic retries; the earlier one-cell diagnostic remains +separate. Compare behavioral results by profile and journey, with missing +evidence unqualified. Keep structural instruction differences separate from task +success. Report every attempt, failure attribution, provider timing, token usage, +reported costs and unknown spend. A reported zero subtotal is not proof of zero +provider spending. This small single-trial matrix cannot establish general coding +quality or broad performance equivalence, and does not compare #14920 before/after. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index ed7a0932de6..71e6aebe406 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1226,7 +1226,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ expectedMatrixSize: 24, definitionMetadata: { version: 1, instructions: "historical-default-hire-comparison", scheduling: "explicit-only", - comparison: { variant: "previous-instructions", candidateSha: "a63437069de58d22ee5adbcb6a6202c007dcf037", restoredFrom: "e00d10d5d5594f6e2d1e8cf4e0ec81c074b0bd88", nativeCodexFix: "held-constant-14920" }, + comparison: { variant: "previous-instructions", candidateSha: "36e987246b649927e96ce1184cd616c4e490106e", restoredFrom: "e00d10d5d5594f6e2d1e8cf4e0ec81c074b0bd88", nativeCodexFix: "held-constant-14920" }, sourceDigest: stockHarnessSourceDigest(), grading: "public-default-bundle-and-delivered-prompts-plus-independent-lifecycle-oracles", vendorBaseEvidence: "required deterministic Codex driver/runnerd/Rust gate; task success is not vendor-base proof", diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index ceb5927ec79..33d709a71ba 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -50,6 +50,7 @@ import { type RunnerE2EResult, } from "./types.js"; import { assertRunnerE2EPrerequisites } from "./prerequisites.js"; +import { prepareStockHarnessPreflight, STOCK_PREFLIGHT_ENV } from "./stock-harness-admission.js"; import { reapNewDetachedDarwinSharedMemory, snapshotDarwinSharedMemory, @@ -1098,6 +1099,9 @@ async function main() { // Keep admission before local-env loading and credential checks. Pending // profiles remain discoverable, but cannot reach a provider. assertRunnerE2EPrerequisites(executions); + if (executions.some(execution => execution.suite.id === "stock-harness")) { + process.env[STOCK_PREFLIGHT_ENV] = prepareStockHarnessPreflight(); + } await loadLocalEnvironment(process.env); const missingCredentials = [ diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 430ed59eec0..13d1905018b 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -15,6 +15,7 @@ import { runEverydayFlow } from "./everyday-flow.js"; import { gradeTaskTitle } from "./task-titles.js"; import { runContextIntegrityFlow } from "./context-integrity-flow.js"; import { captureStockHarness, gradeStockHarness, gradeStockHire } from "./stock-harness.js"; +import { verifyStockHarnessPreflight, STOCK_PREFLIGHT_ENV } from "./stock-harness-admission.js"; import { createTaskThroughUi, submitTaskReply } from "./user-actions.js"; import { runFirstTaskFlow, setupFirstTaskFixtures } from "./first-task-flow.js"; @@ -815,6 +816,10 @@ for (const execution of executions) { }); try { + if (execution.suite.id === "stock-harness") { + const receipt = verifyStockHarnessPreflight(process.env[STOCK_PREFLIGHT_ENV]); + await writeSanitizedJson(snapshotsDir, "stock-harness-preflight.json", receipt, secrets); + } const experimental = await api.patch<{ enableNativeRunner: boolean; }>("/api/instance/settings/experimental", { diff --git a/tests/runner-e2e/stock-harness-admission.test.ts b/tests/runner-e2e/stock-harness-admission.test.ts new file mode 100644 index 00000000000..0872ae55e58 --- /dev/null +++ b/tests/runner-e2e/stock-harness-admission.test.ts @@ -0,0 +1,43 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { spawnSync } from "node:child_process"; +import { prepareStockHarnessPreflight, stockPreflightEnvironment, verifyStockHarnessPreflight } from "./stock-harness-admission.js"; +import { CREDENTIAL_NAMES } from "./types.js"; + +vi.mock("node:child_process", () => ({ spawnSync: vi.fn() })); +const spawn = vi.mocked(spawnSync); +beforeEach(() => { vi.resetAllMocks(); vi.unstubAllEnvs(); }); + +describe("stock harness credential-free admission", () => { + it("allows toolchain paths and excludes every present or future credential", () => { + const source = { PATH: "/bin", HOME: "/home/fixture", CARGO_HOME: "/cargo", CI: "true", + GH_TOKEN: "secret", FUTURE_PROVIDER_API_KEY: "secret", ...Object.fromEntries(CREDENTIAL_NAMES.map(name => [name, "secret"])) }; + expect(stockPreflightEnvironment(source)).toEqual({ PATH: "/bin", HOME: "/home/fixture", CARGO_HOME: "/cargo", CI: "true" }); + }); + it("rejects missing receipts without invoking a process", () => { + expect(() => verifyStockHarnessPreflight(undefined)).toThrow("no prerequisite receipt"); + expect(spawn).not.toHaveBeenCalled(); + }); + it("fails before provider execution when the prerequisite subprocess fails", () => { + spawn.mockReturnValue({ status: 1 } as ReturnType); + expect(() => prepareStockHarnessPreflight()).toThrow("before provider execution"); + expect(spawn).toHaveBeenCalledTimes(1); + }); + it("verifies retained evidence after a passing prerequisite subprocess", () => { + spawn.mockReturnValueOnce({ status: 0 } as ReturnType); + spawn.mockReturnValueOnce({ status: 0, stdout: '{"passed":true}' } as ReturnType); + const receipt = prepareStockHarnessPreflight(); + expect(receipt).toMatch(/stock-harness-preflight-.*\/preflight.json$/); + expect(spawn.mock.calls[1]?.[1]).toContain(`--verify=${receipt}`); + }); + it("allows Cargo dependency resolution only on the disposable GitHub runner", () => { + vi.stubEnv("GITHUB_ACTIONS", "true"); + spawn.mockReturnValueOnce({ status: 0 } as ReturnType); + spawn.mockReturnValueOnce({ status: 0, stdout: '{}' } as ReturnType); + prepareStockHarnessPreflight(); + expect(spawn.mock.calls[0]?.[1]).toContain("--allow-rust-network"); + }); + it("refuses failed receipt verification", () => { + spawn.mockReturnValue({ status: 1, stderr: "stale source" } as ReturnType); + expect(() => verifyStockHarnessPreflight("/fixture/preflight.json")).toThrow("stale source"); + }); +}); diff --git a/tests/runner-e2e/stock-harness-admission.ts b/tests/runner-e2e/stock-harness-admission.ts new file mode 100644 index 00000000000..09ff72e8fc7 --- /dev/null +++ b/tests/runner-e2e/stock-harness-admission.ts @@ -0,0 +1,36 @@ +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import path from "node:path"; + +const root = path.resolve(import.meta.dirname, "../.."); +export const STOCK_PREFLIGHT_ENV = "PAPERCLIP_RUNNER_E2E_STOCK_PREFLIGHT"; + +// An allowlist keeps every provider credential, ambient auth override, and GH +// token out of the prerequisite subprocess, including secrets added later. +export function stockPreflightEnvironment(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return Object.fromEntries([ + "PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL", + "CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS", + ].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]])); +} + +export function verifyStockHarnessPreflight(receiptPath: string | undefined) { + if (!receiptPath) throw new Error("Stock harness has no prerequisite receipt; use the live launcher."); + const run = spawnSync(process.execPath, [path.join(root, "tests/runner-e2e/stock-harness-checks.mjs"), `--verify=${receiptPath}`], { + cwd: root, env: stockPreflightEnvironment(process.env), encoding: "utf8", timeout: 30_000, + }); + if (run.status !== 0) throw new Error(`Stock harness prerequisite verification failed: ${run.stderr || run.error?.message || run.status}`); + return JSON.parse(run.stdout) as Record; +} + +export function prepareStockHarnessPreflight() { + const output = path.join(root, "tests/runner-e2e/results", `stock-harness-preflight-${randomUUID()}`); + const receipt = path.join(output, "preflight.json"); + const run = spawnSync(process.execPath, [path.join(root, "tests/runner-e2e/stock-harness-checks.mjs"), + `--output-dir=${output}`, ...(process.env.GITHUB_ACTIONS === "true" ? ["--allow-rust-network"] : [])], { + cwd: root, env: stockPreflightEnvironment(process.env), stdio: "inherit", timeout: 50 * 60_000, + }); + if (run.status !== 0) throw new Error(`Stock harness prerequisites failed before provider execution. Retained receipt: ${receipt}`); + verifyStockHarnessPreflight(receipt); + return receipt; +} diff --git a/tests/runner-e2e/stock-harness-checks.mjs b/tests/runner-e2e/stock-harness-checks.mjs index d3013151a74..a689e3317e5 100644 --- a/tests/runner-e2e/stock-harness-checks.mjs +++ b/tests/runner-e2e/stock-harness-checks.mjs @@ -22,7 +22,7 @@ export const stockHarnessGates = [ { id: "SH-2", name: "Production-default hire bundle", cwd: ".", files: [ "server/src/__tests__/agent-skills-routes.test.ts", "server/src/services/onboarding-first-task-assets.test.ts", - ], required: ["materializes minimal default instructions for non-CEO agents with no prompt template"] }, + ], required: ["materializes the bundled default instruction set for non-CEO agents with no prompt template"] }, { id: "SH-3", name: "Shared legacy startup and continuation", cwd: ".", files: [ "packages/adapter-utils/src/server-utils.test.ts", "packages/adapter-utils/src/prompt-sections.test.ts", @@ -31,13 +31,19 @@ export const stockHarnessGates = [ "packages/adapters/opencode-local/src/server/execute.test.ts", "packages/adapters/cursor-cloud/src/server/execute.test.ts", "server/src/__tests__/codex-local-execute.test.ts", - ], required: ["keeps task and chat defaults to identity and connection guidance", - "does not restore generic procedures on resume or with the legacy opt-in"] }, + ], required: ["adds the execution contract to resume delta prompts and opted-in fresh prompts"] }, // Hermes is not in the root Vitest project list. Run its package config so // the requested file cannot silently disappear from discovery. { id: "SH-3-hermes", name: "Hermes shared-prompt delivery", cwd: "packages/adapters/hermes", files: ["src/server/prompt-rendering.test.ts"], required: ["renders standard assignment wake with task authority", "renders scoped planning wake authority"] }, + { id: "SH-eval", name: "Independent oracle and qualification admission", cwd: ".", + config: "tests/runner-e2e/vitest.config.ts", + files: ["tests/runner-e2e/stock-harness.test.ts", "tests/runner-e2e/stock-harness-checks.test.mjs", + "tests/runner-e2e/stock-harness-admission.test.ts", "tests/runner-e2e/stock-harness-digest.test.ts"], + required: ["rejects old SHA before providers", "allows toolchain paths and excludes every present or future credential", + "changes when the evaluated server/src/onboarding-assets/default/AGENTS.md changes", + "changes when the evaluated packages/adapter-utils/src/server-utils.ts changes"] }, ]; export function gradeGate(gate, report, exitCode) { @@ -54,37 +60,12 @@ export function gradeGate(gate, report, exitCode) { failedTests: report?.numFailedTests ?? 0, pendingTests: report?.numPendingTests ?? 0 }; } -export function main(args = process.argv.slice(2)) { - if (args.includes("--list")) { - console.log(JSON.stringify({ gates: stockHarnessGates, rust: "runtime_instructions_are_additive_for_codex_on_start_and_resume", live: "not invoked" }, null, 2)); - return; - } - if (args.length) throw new Error("Use --list or no arguments; this command never runs paid providers."); - const output = join(root, "tests/runner-e2e/results", `stock-harness-preflight-${new Date().toISOString().replaceAll(":", "-")}`); - mkdirSync(output, { recursive: true }); - const results = []; - for (const gate of stockHarnessGates) { - console.log(`Checking ${gate.id}: ${gate.name}`); - const file = join(output, `${gate.id}.json`); - const run = spawnSync(process.execPath, [join(root, "node_modules/vitest/vitest.mjs"), "run", ...gate.files, - ...(gate.testPattern ? ["--testNamePattern", gate.testPattern] : []), - "--reporter=default", "--reporter=json", `--outputFile.json=${file}`], - { cwd: resolve(root, gate.cwd), stdio: "inherit", timeout: 10 * 60_000 }); - let report; - try { report = JSON.parse(readFileSync(file, "utf8")); } catch { /* missing evidence fails closed below */ } - results.push(gradeGate(gate, report, run.status)); - } - const rust = spawnSync("cargo", ["test", "--offline", "-p", "paperclip-runner-core", "--test", "codex_provider", - "runtime_instructions_are_additive_for_codex_on_start_and_resume", "--", "--exact"], - { cwd: join(root, "packages/paperclip-runner/runner"), encoding: "utf8", timeout: 10 * 60_000 }); - const rustOutput = `${rust.stdout ?? ""}\n${rust.stderr ?? ""}`; - writeFileSync(join(output, "rust.txt"), rustOutput); - results.push({ id: "SH-1-rust", passed: rust.status === 0 && /test runtime_instructions_are_additive_for_codex_on_start_and_resume \.\.\. ok/.test(rustOutput), exitCode: rust.status }); - const git = spawnSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }); +export function sourceFingerprint() { const hash = createHash("sha256"); const sources = new Set([ ...stockHarnessGates.flatMap(gate => gate.files.map(file => join(gate.cwd, file))), "tests/runner-e2e/stock-harness.ts", "tests/runner-e2e/stock-harness-checks.mjs", "tests/runner-e2e/catalog.ts", + "tests/runner-e2e/stock-harness-admission.ts", "tests/runner-e2e/launch.ts", "tests/runner-e2e/runner.spec.ts", "packages/adapter-utils/src/server-utils.ts", "server/src/onboarding-assets/default/AGENTS.md", "server/src/routes/agents.ts", "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts", "packages/paperclip-runner/src/live/runnerd-codex-transport.ts", @@ -98,9 +79,76 @@ export function main(args = process.argv.slice(2)) { try { hash.update(readFileSync(join(root, source))); } catch { sourceErrors.push(source); } } + return { fingerprint: hash.digest("hex"), sourceErrors }; +} + +export function assertPreflightReceipt(report, current) { + const expected = [...stockHarnessGates.map(gate => gate.id), "SH-1-rust"]; + if (report?.schema !== "paperclip.stock-harness-preflight.v1" || report.passed !== true || + report.providerCalls !== 0 || report.sourceSha !== current.sha || + report.sourceFingerprint !== current.fingerprint || report.sourceErrors?.length !== 0 || + !Array.isArray(report.gates) || report.gates.length !== expected.length || + expected.some(id => report.gates.filter(gate => gate.id === id && gate.passed === true && gate.exitCode === 0).length !== 1)) { + throw new Error("Stock harness requires passing prerequisites for this exact source SHA and fingerprint."); + } + return report; +} + +export function main(args = process.argv.slice(2)) { + if (args.includes("--list")) { + console.log(JSON.stringify({ gates: stockHarnessGates, rust: "runtime_instructions_are_additive_for_codex_on_start_and_resume", live: "not invoked" }, null, 2)); + return; + } + if (args.some(arg => !arg.startsWith("--output-dir=") && !arg.startsWith("--verify=") && arg !== "--allow-rust-network")) + throw new Error("Use --list, --output-dir=, --verify=, or --allow-rust-network; never paid providers."); + const git = spawnSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }); + const verify = args.find(arg => arg.startsWith("--verify="))?.slice("--verify=".length); + if (verify) { + const source = sourceFingerprint(); + if (git.status !== 0 || source.sourceErrors.length) throw new Error("Cannot verify stock harness source provenance."); + const report = assertPreflightReceipt(JSON.parse(readFileSync(verify, "utf8")), { + sha: git.stdout.trim(), fingerprint: source.fingerprint, + }); + const output = resolve(verify, ".."); + for (const gate of stockHarnessGates) { + const grade = gradeGate(gate, JSON.parse(readFileSync(join(output, `${gate.id}.json`), "utf8")), 0); + if (!grade.passed) throw new Error(`Missing or failed retained prerequisite assertions: ${gate.id}`); + } + if (!/test runtime_instructions_are_additive_for_codex_on_start_and_resume \.\.\. ok/.test(readFileSync(join(output, "rust.txt"), "utf8"))) + throw new Error("Missing passing retained Rust prerequisite."); + console.log(JSON.stringify(report)); + return report; + } + const output = args.find(arg => arg.startsWith("--output-dir="))?.slice("--output-dir=".length) ?? + join(root, "tests/runner-e2e/results", `stock-harness-preflight-${new Date().toISOString().replaceAll(":", "-")}`); + mkdirSync(output, { recursive: true }); + const env = Object.fromEntries([ + "PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL", + "CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS", + ].flatMap(name => process.env[name] === undefined ? [] : [[name, process.env[name]]])); + const results = []; + for (const gate of stockHarnessGates) { + console.log(`Checking ${gate.id}: ${gate.name}`); + const file = join(output, `${gate.id}.json`); + const run = spawnSync(process.execPath, [join(root, "node_modules/vitest/vitest.mjs"), "run", ...gate.files, + ...(gate.config ? ["--config", gate.config] : []), + ...(gate.testPattern ? ["--testNamePattern", gate.testPattern] : []), + "--reporter=default", "--reporter=json", `--outputFile.json=${file}`], + { cwd: resolve(root, gate.cwd), env, stdio: "inherit", timeout: 10 * 60_000 }); + let report; + try { report = JSON.parse(readFileSync(file, "utf8")); } catch { /* missing evidence fails closed below */ } + results.push(gradeGate(gate, report, run.status)); + } + const rust = spawnSync("cargo", ["test", ...(args.includes("--allow-rust-network") ? [] : ["--offline"]), "-p", "paperclip-runner-core", "--test", "codex_provider", + "runtime_instructions_are_additive_for_codex_on_start_and_resume", "--", "--exact"], + { cwd: join(root, "packages/paperclip-runner/runner"), env, encoding: "utf8", timeout: 10 * 60_000 }); + const rustOutput = `${rust.stdout ?? ""}\n${rust.stderr ?? ""}`; + writeFileSync(join(output, "rust.txt"), rustOutput); + results.push({ id: "SH-1-rust", passed: rust.status === 0 && /test runtime_instructions_are_additive_for_codex_on_start_and_resume \.\.\. ok/.test(rustOutput), exitCode: rust.status }); + const { fingerprint, sourceErrors } = sourceFingerprint(); const report = { schema: "paperclip.stock-harness-preflight.v1", sourceSha: git.stdout?.trim() || null, - sourceFingerprint: hash.digest("hex"), measuredAt: new Date().toISOString(), providerCalls: 0, - live: "not_run", passed: sourceErrors.length === 0 && results.every(row => row.passed), sourceErrors, gates: results }; + sourceFingerprint: fingerprint, measuredAt: new Date().toISOString(), providerCalls: 0, + live: "not_run", passed: git.status === 0 && sourceErrors.length === 0 && results.every(row => row.passed), sourceErrors, gates: results }; writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n"); console.log(`Stock harness prerequisite evidence: ${output}/preflight.json`); if (!report.passed) process.exitCode = 1; diff --git a/tests/runner-e2e/stock-harness-checks.test.mjs b/tests/runner-e2e/stock-harness-checks.test.mjs index 3397cae78c6..2af6a8fee66 100644 --- a/tests/runner-e2e/stock-harness-checks.test.mjs +++ b/tests/runner-e2e/stock-harness-checks.test.mjs @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { gradeGate, stockHarnessGates } from "./stock-harness-checks.mjs"; +import { assertPreflightReceipt, gradeGate, stockHarnessGates } from "./stock-harness-checks.mjs"; const gate = { id: "SH-test", name: "Boundary", files: ["boundary.test.ts"], required: ["must preserve instructions"] }; const report = () => ({ testResults: [{ name: "/repo/boundary.test.ts", status: "passed", @@ -7,7 +7,7 @@ const report = () => ({ testResults: [{ name: "/repo/boundary.test.ts", status: numTotalTests: 1, numPassedTests: 1, numFailedTests: 0, numPendingTests: 0 }); describe("stock harness prerequisite coverage", () => { it("maps every implemented change to an executable gate", () => { - expect(stockHarnessGates.map(gate => gate.id)).toEqual(["SH-1", "SH-2", "SH-3", "SH-3-hermes"]); + expect(stockHarnessGates.map(gate => gate.id)).toEqual(["SH-1", "SH-2", "SH-3", "SH-3-hermes", "SH-eval"]); expect(stockHarnessGates.every(gate => gate.files.length > 0 && gate.required.length > 0)).toBe(true); }); it("accepts an executed passing boundary", () => expect(gradeGate(gate, report(), 0).passed).toBe(true)); @@ -38,3 +38,23 @@ describe("stock harness prerequisite coverage", () => { expect(stockHarnessGates.find(gate => gate.id === "SH-3-hermes").cwd).toBe("packages/adapters/hermes"); }); }); + +describe("stock harness prerequisite admission", () => { + const current = { sha: "a".repeat(40), fingerprint: "b".repeat(64) }; + const receipt = () => ({ schema: "paperclip.stock-harness-preflight.v1", passed: true, + providerCalls: 0, sourceSha: current.sha, sourceFingerprint: current.fingerprint, + sourceErrors: [], gates: [...stockHarnessGates.map(g => g.id), "SH-1-rust"].map(id => ({ id, passed: true, exitCode: 0 })) }); + it("admits the same passing source revision", () => expect(assertPreflightReceipt(receipt(), current).passed).toBe(true)); + it.each([ + ["old SHA", r => { r.sourceSha = "c".repeat(40); }], + ["changed source", r => { r.sourceFingerprint = "d".repeat(64); }], + ["failed boundary", r => { r.gates[0].passed = false; }], + ["nonzero exit", r => { r.gates[0].exitCode = 1; }], + ["missing Rust", r => { r.gates.pop(); }], + ["duplicate boundary", r => { r.gates[1] = r.gates[0]; }], + ["provider calls", r => { r.providerCalls = 1; }], + ["missing source", r => { r.sourceErrors.push("missing.ts"); }], + ])("rejects %s before providers", (_name, mutate) => { + const r = receipt(); mutate(r); expect(() => assertPreflightReceipt(r, current)).toThrow("exact source SHA and fingerprint"); + }); +}); diff --git a/tests/runner-e2e/stock-harness-comparison.json b/tests/runner-e2e/stock-harness-comparison.json index 03e66157010..a847b321035 100644 --- a/tests/runner-e2e/stock-harness-comparison.json +++ b/tests/runner-e2e/stock-harness-comparison.json @@ -1,7 +1,7 @@ { "schema": "paperclip.stock-harness-comparison.v1", "variant": "previous-instructions", - "candidateSha": "a63437069de58d22ee5adbcb6a6202c007dcf037", + "candidateSha": "36e987246b649927e96ce1184cd616c4e490106e", "restoredInstructionsFrom": "e00d10d5d5594f6e2d1e8cf4e0ec81c074b0bd88", "nativeCodex14920": "held-constant; no before/after performance claim", "productionDifferences": [ @@ -19,7 +19,9 @@ "chat-cases.ts": "754b251bdd42605e147da9081b663329234260d2b13df0df3d44562e362b14ca", "chat-flow.ts": "11fc506559da61315ce04a877f6b38a055e71e532fecfcf62c749a1d3bf985e7", "live-fixtures.ts": "1ef80209cc53cc514a8e58af4c2225e38ec15b20acf17ca7564506ca6617ac3e", - "runner.spec.ts": "41090ca983607edb2a6c21975dccf6ee84dde723ad31f60b7f39c42b65446fa1", + "runner.spec.ts": "14d57e0f9131b3f1981ac593464738d01362008eefe85ea836075c794681fa85", + "launch.ts": "5ada6ca5484542f028b29fc603dc7992213d4c06fa9046834bd7d23c4511b847", + "stock-harness-admission.ts": "c9775d77d48f8d59ba1197a7e5769f14db0c7d33f5e2a2968a04466e083d63ff", "types.ts": "c45dc3b2366457449ec9e7d413f930605dec88ea93a14ce5c5a5f445a27d148c" }, "instructionSha256": { diff --git a/tests/runner-e2e/stock-harness-digest.test.ts b/tests/runner-e2e/stock-harness-digest.test.ts new file mode 100644 index 00000000000..05c08b4f5d2 --- /dev/null +++ b/tests/runner-e2e/stock-harness-digest.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it, vi } from "vitest"; +import { readFileSync } from "node:fs"; +import { stockHarnessSourceDigest } from "./stock-harness.js"; + +vi.mock("node:fs", async importOriginal => ({ ...await importOriginal(), readFileSync: vi.fn() })); + +describe("stock harness instruction revision", () => { + it.each(["server/src/onboarding-assets/default/AGENTS.md", "packages/adapter-utils/src/server-utils.ts"])( + "changes when the evaluated %s changes", source => { + vi.mocked(readFileSync).mockImplementation(() => Buffer.from("unchanged")); + const original = stockHarnessSourceDigest(); + vi.mocked(readFileSync).mockImplementation(file => Buffer.from(String(file).endsWith(source) ? "changed instructions" : "unchanged")); + expect(stockHarnessSourceDigest()).not.toBe(original); + }); +}); diff --git a/tests/runner-e2e/stock-harness.ts b/tests/runner-e2e/stock-harness.ts index 2e9dcf5933f..78318042c2b 100644 --- a/tests/runner-e2e/stock-harness.ts +++ b/tests/runner-e2e/stock-harness.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import type { RunnerProfileFixture } from "./types.js"; -// This temporary comparison revision independently pins the historical manual. +// The comparison revision independently pins the historical manual. // Candidate instruction assertions remain unchanged on the candidate branch. export const STOCK_HIRE_IDENTITY = "You are an agent at Paperclip company.\n\n## Execution Contract\n\n- Start actionable work in the same heartbeat. Do not stop at a plan unless the issue explicitly asks for planning.\n- Keep the work moving until it is done. If you need QA to review it, ask them. If you need your boss to review it, ask them.\n- Leave durable progress in task comments, documents, or work products, then update the issue to a clear final disposition before you exit.\n- When your work produces a user-inspectable deliverable file, follow the Paperclip skill's \"Generated Artifacts and Work Products\" workflow before final disposition. Use `skills/paperclip/scripts/paperclip-upload-artifact.sh` when working in this repo, create/update an artifact work product when the file is the deliverable, and link the uploaded attachment in the final comment. Do not rely on local filesystem paths as the only access path. If an important file intentionally remains workspace-only, create/update a work product with `metadata.resourceRef.kind: \"workspace_file\"` and a workspace-relative path, then name that work product and path in the final comment. Treat browse/search as a fallback for recovering workspace files, not the preferred deliverable path.\n- When your work produces or updates an operator-facing engineering output, create/update the matching work product: `pull_request` for opened PRs, `preview_url` for published previews, `runtime_service` for managed preview/dev services, `commit` for notable pushed commits, and `branch` when the branch itself is the handoff. A comment is not a substitute for the work product access path.\n- Comments, documents, screenshots, work products, and `Remaining` bullets are evidence, not valid liveness paths by themselves.\n- Final disposition checklist: mark `done` when complete and verified; use `in_review` only with a real reviewer, approval, interaction, or monitor path; use `blocked` only with first-class blockers or a named unblock owner/action; create delegated follow-up issues with blockers when another agent owns the next step; keep `in_progress` only when a live continuation path exists.\n- Use child issues for parallel or long delegated work instead of polling agents, sessions, or processes.\n- Create child issues directly when you know what needs to be done. If the board/user needs to choose suggested tasks, answer structured questions, or confirm a proposal first, create an issue-thread interaction on the current issue with `POST /api/issues/{issueId}/interactions` using `kind: \"suggest_tasks\"`, `kind: \"ask_user_questions\"`, or `kind: \"request_confirmation\"`.\n- Use `request_confirmation` instead of asking for yes/no decisions in markdown. Before presenting a plan for review, you MUST complete this publish contract:\n 1. `PUT /issues/{id}/documents/plan` with `{ format: 'markdown', body, changeSummary }`.\n 2. Re-`GET /documents/plan`, assert it returns `200`, and capture its `latestRevisionId`.\n 3. Only then create `request_confirmation` with `target={ type: 'issue_document', key: 'plan', revisionId: latestRevisionId }` and `idempotencyKey=confirmation:{issueId}:plan:{revisionId}`.\n 4. Wait for acceptance before creating implementation subtasks.\n Never present a plan only in a thread comment or through `ask_user_questions`; comments are supporting context and questions are for gathering input, not plan review.\n- `ask_user_questions` and confirmations default `supersedeOnUserComment` to `false`, so a later board/user comment keeps the pending card open while discussion continues. Set it to `true` when a new comment should replace the pending request. If you wake up from a superseding comment, revise the artifact, question set, or proposal and create a fresh interaction if input is still needed.\n- For human input, save a pending question/confirmation interaction and set `in_review`; prose alone does not create a waiting path. Use `blockedByIssueIds` for issue dependencies. An agent may set an `unblockDescriptor` only for itself (`owner: { \"agentId\": \"\" }` plus `action`), not for the board/user or another agent.\n- Respect budget, pause/cancel, approval gates, and company boundaries.\n\nDo not let work sit here. You must always update your task with a comment.\n"; export const STOCK_TEMPLATE_IDENTITY = "You are agent"; @@ -59,7 +59,7 @@ export function gradeStockHarness(evidence: StockHarnessEvidence) { "Every legacy run must have a public adapter.invoke event with its actual nonempty prompt."); check("historical-generic-procedures-observed", prompts.length > 0 && prompts.some(row => REMOVED_PROCEDURES.some(procedure => String(row.prompt).includes(procedure))), - "Baseline structural receipt observes the historical procedures; identical lifecycle graders own task performance."); + "Historical structural receipt observes prior procedures; identical lifecycle graders own task performance."); const fresh = prompts.filter(row => Number(row.promptMetrics?.heartbeatPromptChars) > 0); check("fresh-default-delivered", fresh.length > 0 && fresh.every(row => String(row.prompt).includes(STOCK_TEMPLATE_IDENTITY) && String(row.prompt).includes("Connection tools:") && @@ -110,6 +110,9 @@ export function stockHarnessSourceDigest() { for (const source of [ "stock-harness.ts", "context-integrity-cases.ts", "context-integrity-scoring.ts", "context-integrity-flow.ts", "chat-cases.ts", "chat-flow.ts", "live-fixtures.ts", "runner.spec.ts", + "stock-harness-checks.mjs", "stock-harness-admission.ts", + "../../server/src/onboarding-assets/default/AGENTS.md", + "../../packages/adapter-utils/src/server-utils.ts", ]) hash.update(source).update(readFileSync(new URL(source, import.meta.url))); return hash.digest("hex"); }