From 3866c9773f110498fc3e9e99c961bef45ec2c1c0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 23 Jun 2026 17:28:49 +0000 Subject: [PATCH] Add Hermes prompt rendering regression tests --- .../hermes-local/src/server/execute.ts | 1 + .../src/server/prompt-rendering.test.ts | 120 ++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/packages/adapters/hermes-local/src/server/execute.ts b/packages/adapters/hermes-local/src/server/execute.ts index 614ae83481e..187f48c7d7f 100644 --- a/packages/adapters/hermes-local/src/server/execute.ts +++ b/packages/adapters/hermes-local/src/server/execute.ts @@ -90,6 +90,7 @@ const HERMES_DEFAULT_PROMPT_TEMPLATE = [ "Paperclip API guidance:", "- Use `curl` from the terminal for Paperclip API calls; browser/web extraction tools may not reach localhost.", "- Use `$PAPERCLIP_API_URL`, `$PAPERCLIP_API_KEY`, and `$PAPERCLIP_RUN_ID`; do not hard-code local ports or copy secrets into comments.", + "- Displayed command logs may redact secrets; rely on environment variables instead of printed token values.", "- Include `-H \"Authorization: Bearer $PAPERCLIP_API_KEY\"` on API requests.", "- Include `-H \"X-Paperclip-Run-Id: $PAPERCLIP_RUN_ID\"` on mutating issue requests.", "- For multiline comments or status updates, preserve newlines with `jq --arg` or a heredoc-fed helper rather than hand-escaping JSON.", diff --git a/packages/adapters/hermes-local/src/server/prompt-rendering.test.ts b/packages/adapters/hermes-local/src/server/prompt-rendering.test.ts index 95cb2cda729..1a5d1c9a8ab 100644 --- a/packages/adapters/hermes-local/src/server/prompt-rendering.test.ts +++ b/packages/adapters/hermes-local/src/server/prompt-rendering.test.ts @@ -46,6 +46,47 @@ function baseContext(overrides: Record = {}) { } as any; } +test("renders standard assignment wake with task authority and no backlog discovery guidance", () => { + const prompt = buildPrompt(baseContext({ + paperclipWake: { + reason: "issue_assigned", + issue: { + id: "issue-1", + identifier: "PAP-11750", + title: "Add Hermes prompt rendering regression tests", + status: "in_progress", + priority: "medium", + workMode: "standard", + }, + checkedOutByHarness: true, + commentWindow: { requestedCount: 0, includedCount: 0, missingCount: 0 }, + comments: [], + fallbackFetchNeeded: false, + }, + paperclipTaskMarkdown: [ + "Paperclip task context:", + '- Issue: "PAP-11750"', + '- Title: "Add Hermes prompt rendering regression tests"', + "", + "Issue description:", + "```text", + "Add focused unit tests for assignment wake and custom prompt rendering.", + "```", + ].join("\n"), + }), {}); + + expect(prompt).toContain("## Paperclip Wake Payload"); + expect(prompt).toContain("- reason: issue_assigned"); + expect(prompt).toContain("- issue: PAP-11750 Add Hermes prompt rendering regression tests"); + expect(prompt).toContain("- issue work mode: standard"); + expect(prompt).toContain("Paperclip task context:"); + expect(prompt).toContain("Add focused unit tests for assignment wake and custom prompt rendering."); + expect(prompt).toContain("The harness already checked out this issue for the current run."); + expect(prompt).toContain("clear final disposition"); + expect(prompt).not.toContain("check for unassigned issues"); + expect(prompt).not.toContain("status=backlog"); +}); + test("renders scoped planning wake authority before the Hermes default workflow", () => { const prompt = buildPrompt(baseContext(), { paperclipApiUrl: "http://127.0.0.1:3101/api", @@ -63,6 +104,33 @@ test("renders scoped planning wake authority before the Hermes default workflow" expect(prompt).not.toContain("status=backlog"); }); +test("renders resume deltas instead of full scoped-wake boilerplate when continuing a session", () => { + const prompt = buildPrompt(baseContext({ + paperclipWake: { + reason: "issue_commented", + issue: { + id: "issue-1", + identifier: "PAP-11750", + title: "Add Hermes prompt rendering regression tests", + status: "in_progress", + priority: "medium", + workMode: "standard", + }, + latestCommentId: "comment-2", + commentWindow: { requestedCount: 1, includedCount: 1, missingCount: 0 }, + comments: [{ id: "comment-2", body: "Please add the resume-delta case.", createdAt: "2026-06-23T00:00:00.000Z" }], + fallbackFetchNeeded: false, + }, + }), {}, { resumedSession: true }); + + expect(prompt).toContain("## Paperclip Resume Delta"); + expect(prompt).toContain("You are resuming an existing Paperclip session."); + expect(prompt).toContain("Focus on the new wake delta below"); + expect(prompt).toContain("Please add the resume-delta case."); + expect(prompt).toContain("- fallback fetch needed: no"); + expect(prompt).not.toContain("Before generic repo exploration or boilerplate heartbeat updates"); +}); + test("renders comment wake batch guidance without defaulting to a full-thread refetch", () => { const prompt = buildPrompt(baseContext({ wakeCommentId: "comment-1", @@ -90,6 +158,33 @@ test("renders comment wake batch guidance without defaulting to a full-thread re expect(prompt).toContain("- fallback fetch needed: no"); }); +test("renders accepted-plan continuation without authorizing implementation on the planning issue", () => { + const prompt = buildPrompt(baseContext({ + paperclipWake: { + reason: "issue_commented", + issue: { + id: "issue-1", + identifier: "PAP-3404", + title: "Plan the Hermes prompt update", + status: "in_progress", + priority: "medium", + workMode: "planning", + }, + interactionKind: "request_confirmation", + interactionStatus: "accepted", + commentWindow: { requestedCount: 0, includedCount: 0, missingCount: 0 }, + comments: [], + fallbackFetchNeeded: false, + }, + }), {}); + + expect(prompt).toContain("- planning directive: Create child issues from the approved plan only. Do not write code or perform implementation work on the planning issue."); + expect(prompt).toContain("- accepted-plan continuation: you may create child implementation issues from the approved plan"); + expect(prompt).toContain("must not start implementation work on the planning issue itself"); + expect(prompt).not.toContain("- planning directive: Make the plan only."); + expect(prompt).not.toContain("Update the plan only"); +}); + test("keeps authoritative parent and ancestor context from task markdown", () => { const prompt = buildPrompt(baseContext({ paperclipTaskMarkdown: [ @@ -107,6 +202,21 @@ test("keeps authoritative parent and ancestor context from task markdown", () => expect(prompt).not.toContain("check the issue body or comments for references"); }); +test("renders safe Paperclip API examples from environment variables with multiline update preservation", () => { + const prompt = buildPrompt(baseContext(), { + paperclipApiUrl: "http://paperclip.local/api", + }); + + expect(prompt).toContain("Use `$PAPERCLIP_API_URL`, `$PAPERCLIP_API_KEY`, and `$PAPERCLIP_RUN_ID`"); + expect(prompt).toContain("Displayed command logs may redact secrets"); + expect(prompt).toContain('-H "Authorization: Bearer $PAPERCLIP_API_KEY"'); + expect(prompt).toContain('-H "X-Paperclip-Run-Id: $PAPERCLIP_RUN_ID"'); + expect(prompt).toContain("body=$(cat <<'MD'"); + expect(prompt).toContain("jq -n --arg status done --arg comment \"$body\""); + expect(prompt).toContain("--data-binary @-"); + expect(prompt).not.toContain("Authorization: Bearer <"); +}); + test("preserves custom prompt templates while exposing runtime and wake variables", () => { const prompt = buildPrompt(baseContext(), { paperclipApiUrl: "http://paperclip.local/api", @@ -114,7 +224,11 @@ test("preserves custom prompt templates while exposing runtime and wake variable "CUSTOM TEMPLATE", "agent={{agent.name}}", "api={{paperclipApiUrl}}", + "keyEnv={{paperclipApiKeyEnv}}", + "runEnv={{paperclipRunIdEnv}}", + "wakePrompt={{paperclipWakePrompt}}", "task={{paperclipTaskMarkdown}}", + "wakeJson={{paperclipWakeJson}}", "wake={{wakePayloadJson}}", ].join("\n"), }); @@ -122,7 +236,13 @@ test("preserves custom prompt templates while exposing runtime and wake variable expect(prompt).toContain("CUSTOM TEMPLATE"); expect(prompt).toContain("agent=Hermes Engineer"); expect(prompt).toContain("api=http://paperclip.local/api"); + expect(prompt).toContain("keyEnv=PAPERCLIP_API_KEY"); + expect(prompt).toContain("runEnv=PAPERCLIP_RUN_ID"); + expect(prompt).toContain("wakePrompt=## Paperclip Wake Payload"); expect(prompt).toContain("task=Paperclip task context:"); + expect(prompt).toContain("wakeJson={\"reason\":\"issue_assigned\""); expect(prompt).toContain('"reason":"issue_assigned"'); expect(prompt).toContain("## Paperclip Wake Payload"); + expect(prompt).toContain("Issue description:\n```text\nUse the wake payload as runtime authority.\n```"); + expect(prompt).not.toContain("Paperclip runtime identity:"); });