From c471504099ae582c7f2630a6bf486e68ae6bea22 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 17:36:08 -0500 Subject: [PATCH] test(evals): correct hiring template comparison evidence Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 7 +- .../hiring-templates/coder.d7bdfc4.md | 64 +++++++++++++++++++ tests/runner-e2e/hiring-template-cases.ts | 2 +- tests/runner-e2e/hiring-template-flow.ts | 8 ++- tests/runner-e2e/hiring-template-scoring.ts | 38 ++++++++--- tests/runner-e2e/hiring-template.test.ts | 37 +++++++++-- 6 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 tests/runner-e2e/fixtures/hiring-templates/coder.d7bdfc4.md diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index f538b99baf8..2e4da82b8c6 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1508,7 +1508,8 @@ The fixture creates a CEO through the public API without an instructions bundle override, using production permission defaults and a personal managed AI connection. Chromium sends the same user request on candidate and baseline: use `paperclip-create-agent`, read its skill, drafting guide, review checklist -and coder example, hire one permanent coder with that example, and delegate a +and coder example, fill its name/company/manager/issue-prefix placeholders, +hire one permanent coder with that example, and delegate a saved JSON label-normalization fixture. Reading the optional references is an explicit fixture user request. It is not an additional production requirement. A follow-up delegates a second fixture to the same coder with underscore @@ -1551,7 +1552,9 @@ pair, not presented as a workflow behavior regression or template equivalence. The new marked screenshot shows only the synthetic chat/task state; private snapshots follow the existing publication boundary. -Read receipt support deliberately recognizes bounded direct shell reads and +Read receipt support deliberately recognizes direct `cat`, positive-count +`head`/`tail`, and printing-only `sed -n` argument forms. Help, version, zero-count, +editing and unknown arguments do not count. It also recognizes canonical file-read events with a preserved relative skill path and completed output. ACPX redacts absolute file locations from canonical events; a read whose path no longer survives is unprovable and remains uncomparable. Echoing or diff --git a/tests/runner-e2e/fixtures/hiring-templates/coder.d7bdfc4.md b/tests/runner-e2e/fixtures/hiring-templates/coder.d7bdfc4.md new file mode 100644 index 00000000000..6fc669492f0 --- /dev/null +++ b/tests/runner-e2e/fixtures/hiring-templates/coder.d7bdfc4.md @@ -0,0 +1,64 @@ +# Coder Agent Template + +Use this template when hiring software engineers who implement code, debug issues, write tests, and coordinate with QA or engineering leadership. + +## Recommended Role Fields + +- `name`: `Coder`, `CodexCoder`, `ClaudeCoder`, or a model/tool-specific name +- `role`: `engineer` +- `title`: `Software Engineer` +- `icon`: `code` +- `capabilities`: `Implements coding tasks, writes and edits code, debugs issues, adds focused tests, and coordinates with QA and engineering leadership.` +- `adapterType`: `codex_local`, `claude_local`, `cursor`, or another coding adapter + +## `AGENTS.md` + +```md +You are agent {{agentName}} (Coder / Software Engineer) at {{companyName}}. + +When you wake up, follow the Paperclip skill. It contains the full heartbeat procedure. + +You are a software engineer. Your job is to implement coding tasks: + +- Write, edit, and debug code as assigned +- Follow existing code conventions and architecture +- Leave code better than you found it +- Comment your work clearly in task updates +- Ask for clarification when requirements are ambiguous +- Test your changes with the smallest verification that proves the work + +You report to {{managerTitle}}. Work only on tasks assigned to you or explicitly handed to you in comments. When done, mark the task done with a clear summary of what changed and how you verified it. + +Start actionable work in the same heartbeat; do not stop at a plan unless planning was requested. Leave durable progress with a clear next action. Use child issues for long or parallel delegated work instead of polling. Mark blocked work with owner and action. Respect budget, pause/cancel, approval gates, and company boundaries. + +Commit things in logical commits as you go when the work is good. If there are unrelated changes in the repo, work around them and do not revert them. Only stop and say you are blocked when there is an actual conflict you cannot resolve. + +Make sure you know the success condition for each task. If it was not described, pick a sensible one and state it in your task update. Before finishing, check whether the success condition was achieved. If it was not, keep iterating or escalate with a concrete blocker. + +Keep the work moving until it is done. If you need QA to review it, ask QA. If you need your manager to review it, ask them. If someone needs to unblock you, assign or hand back the ticket with a comment explaining exactly what you need. + +An implied addition to every prompt is: test it, make sure it works, and iterate until it does. If it is a shell script, run a safe version. If it is code, run the smallest relevant tests or checks. If browser verification is needed and you do not have browser capability, ask QA to verify. + +If you are asked to fix a deployed bug, fix the bug, identify the underlying reason it happened, add coverage or guardrails where practical, and ask QA to verify the fix when user-facing behavior changed. + +If the task is part of an existing PR and you are asked to address review feedback or failing checks after the PR has already been pushed, push the completed follow-up changes unless your company instructions say otherwise. + +If there is a blocker, explain the blocker and include your best guess for how to resolve it. Do not only say that it is blocked. + +When you run tests, do not default to the entire test suite. Run the minimal checks needed for confidence unless the task explicitly requires full release or PR verification. + +## Collaboration and handoffs + +- UX-facing changes → loop in `[UXDesigner](/{{issuePrefix}}/agents/uxdesigner)` for review of visual quality and flows. +- Security-sensitive changes (auth, crypto, secrets, permissions, adapter/tool access) → loop in `[SecurityEngineer](/{{issuePrefix}}/agents/securityengineer)` before merging. +- Browser validation / user-facing verification → hand to `[QA](/{{issuePrefix}}/agents/qa)` with a reproducible test plan. +- Skill or instruction quality changes → hand to the skill consultant or equivalent instruction owner. + +## Safety and permissions + +- Never commit secrets, credentials, or customer data. If you spot any in the diff, stop and escalate. +- Do not bypass pre-commit hooks, signing, or CI unless the task explicitly asks you to and the reason is documented in the commit message. +- Do not install new company-wide skills, grant broad permissions, or enable timer heartbeats as part of a code change — those are governance actions that belong on a separate ticket. + +You must always update your task with a comment before exiting a heartbeat. +``` diff --git a/tests/runner-e2e/hiring-template-cases.ts b/tests/runner-e2e/hiring-template-cases.ts index b11b369edfc..6291ce4a72a 100644 --- a/tests/runner-e2e/hiring-template-cases.ts +++ b/tests/runner-e2e/hiring-template-cases.ts @@ -38,7 +38,7 @@ export function hiringTemplateScenario(nonce: string, projectName: string) { marker, hireName, inputs: hiringTemplateInputs, initialPrompt: [ `Use the paperclip-create-agent skill to hire exactly one coding teammate named ${hireName}, reporting to you, with your native runner, model, and available AI connection.`, - "Before hiring, read that skill's SKILL.md, references/agent-instruction-templates.md, references/draft-review-checklist.md, and references/agents/coder.md. Use the AGENTS.md example from that coder reference as the teammate's managed instructions, filling all agentName, companyName, and managerTitle placeholders from this hire, company, and your actual title. Keep skills and reporting information in their configuration.", + "Before hiring, read that skill's SKILL.md, references/agent-instruction-templates.md, references/draft-review-checklist.md, and references/agents/coder.md. Use the AGENTS.md example from that coder reference as the teammate's managed instructions, filling all agentName, companyName, managerTitle, and issuePrefix placeholders from this hire, company, your actual title, and the company issue prefix. Keep skills and reporting information in their configuration.", `Have that coder prepare a JSON test fixture for a label normalizer in one assigned task in ${projectName}. Save the fixture as a Paperclip document with an object containing reference: ${JSON.stringify(marker)} and entries, an ordered array of objects with input and value. Inputs: ${JSON.stringify(hiringTemplateInputs)}.`, "For each value, trim its input, lowercase it, replace each sequence outside ASCII a-z and 0-9 with one hyphen, then trim leading and trailing hyphens. Link the task here and let the coder complete it.", ].join("\n\n"), diff --git a/tests/runner-e2e/hiring-template-flow.ts b/tests/runner-e2e/hiring-template-flow.ts index 233ca0b72b8..6976788f917 100644 --- a/tests/runner-e2e/hiring-template-flow.ts +++ b/tests/runner-e2e/hiring-template-flow.ts @@ -56,10 +56,10 @@ export async function readHiringTemplateSources(api: Pick, com executionContractHash: hiringTemplateHash(await readFile(new URL("../../server/src/onboarding-assets/default/AGENTS.md", import.meta.url), "utf8")) }; } -export function renderHiringCoderExample(reference: string, agentName: string, companyName: string, managerTitle: string) { +export function renderHiringCoderExample(reference: string, agentName: string, companyName: string, managerTitle: string, issuePrefix: string) { const example = reference.match(/```md\s*\n([\s\S]*?)\n```/)?.[1]; if (!example?.trim()) throw new Error("Production coder reference has no AGENTS.md example"); - return example.replaceAll("{{agentName}}", agentName).replaceAll("{{companyName}}", companyName).replaceAll("{{managerTitle}}", managerTitle); + return example.replaceAll("{{agentName}}", agentName).replaceAll("{{companyName}}", companyName).replaceAll("{{managerTitle}}", managerTitle).replaceAll("{{issuePrefix}}", issuePrefix); } export async function runHiringTemplateFlow(context: { @@ -71,6 +71,8 @@ export async function runHiringTemplateFlow(context: { const company = `/api/companies/${f.company.id}`; const account = f.aiConnection; if (!account) throw new Error("Hiring-template fixture requires a managed execution account"); + const issuePrefix = f.company.issuePrefix; + if (!issuePrefix) throw new Error("Hiring-template fixture requires the actual company issue prefix"); const evidence: HiringTemplateEvidence = { leadId: f.agent.id, chatIssueId: "", hireName: "", marker: "", projectId: "", inputs: [], expectedCeoFiles: {}, expectedSourceHashes: {}, servedSourceHashes: {}, assignedSkills: [], coderExample: "", @@ -102,7 +104,7 @@ export async function runHiringTemplateFlow(context: { }); scenario = hiringTemplateScenario(input.nonce, project.name); Object.assign(evidence, { hireName: scenario.hireName, marker: scenario.marker, inputs: scenario.inputs, projectId: project.id, - coderExample: renderHiringCoderExample(source.coderReference, scenario.hireName, f.company.name, String((await api.get<{ title: string }>(`/api/agents/${f.agent.id}`)).title)) }); + coderExample: renderHiringCoderExample(source.coderReference, scenario.hireName, f.company.name, String((await api.get<{ title: string }>(`/api/agents/${f.agent.id}`)).title), issuePrefix) }); await turn(scenario.initialPrompt, 2); evidence.chatIssueId = context.issue().id; const initial = await tasks(); diff --git a/tests/runner-e2e/hiring-template-scoring.ts b/tests/runner-e2e/hiring-template-scoring.ts index 21366460595..5cfe2f9320b 100644 --- a/tests/runner-e2e/hiring-template-scoring.ts +++ b/tests/runner-e2e/hiring-template-scoring.ts @@ -50,13 +50,35 @@ function matchingReadPath(value: unknown, file: string) { return typeof value === "string" && value.replace(/\\/g, "/").endsWith(`/paperclip-create-agent/${file}`); } function commandReads(command: unknown, file: string) { - if (typeof command !== "string") return false; - // Recognize bounded, direct read commands. Echoed paths, searches, writes, - // arbitrary scripts and unresolved shell variables are not read evidence. - const tokens = command.match(/"[^"]*"|'[^']*'|[^\s]+/g)?.map(s => s.replace(/^['"]|['"]$/g, "")) ?? []; - return tokens.some((token, i) => matchingReadPath(token, file) - && (/^(?:cat|head|tail|sed)$/.test(tokens[0] ?? "")) && i > 0 - && !tokens.some(t => /[|;&><]|\$/.test(t))); + if (typeof command !== "string" || /[|;&><`$\\\r\n]/.test(command)) return false; + // Admit only direct reads with explicit arguments. Unknown options, scripts, + // expansions and help/version output leave source coverage uncomparable. + const matches = [...command.matchAll(/"[^"\n]*"|'[^'\n]*'|[^\s"']+/g)]; + if (!matches.length || command.slice(0, matches[0]!.index).trim()) return false; + for (let i = 1; i < matches.length; i++) { + if (!/^\s+$/.test(command.slice(matches[i - 1]!.index! + matches[i - 1]![0].length, matches[i]!.index))) return false; + } + const last = matches.at(-1)!; + if (command.slice(last.index! + last[0].length).trim()) return false; + const tokens = matches.map(match => match[0].replace(/^['"]|['"]$/g, "")); + const program = tokens.shift(); + if (program === "head" || program === "tail") { + if (tokens[0] === "-n" || tokens[0] === "-c") { + tokens.shift(); + const count = tokens.shift(); + if (!count || !/^[1-9]\d*$/.test(count)) return false; + } else if (/^-[nc][1-9]\d*$/.test(tokens[0] ?? "")) tokens.shift(); + } else if (program === "sed") { + if (tokens.shift() !== "-n") return false; + if (tokens[0] === "-e") tokens.shift(); + const script = tokens.shift() ?? ""; + // Only printing from the beginning is supported, without editing or shell + // execution. Other valid sed programs still require different evidence. + if (!/^(?:p|1p|1,[1-9]\d*p)$/.test(script)) return false; + } else if (program !== "cat") return false; + if (tokens[0] === "--") tokens.shift(); + return tokens.length > 0 && tokens.every(token => token.length > 0 && !token.startsWith("-") && !/["'*?\[\]]/.test(token)) + && tokens.some(token => matchingReadPath(token, file)); } /** Only completed provider read operations count; prose and discovery listings do not. */ @@ -79,7 +101,7 @@ export function hiringTemplateReadReceipts(readRuns: HiringReadRun[], leadId: st const commandRead = item.type === "commandExecution" && item.exitCode === 0 && item.status === "completed" && typeof item.aggregatedOutput === "string" && item.aggregatedOutput.length > 0; const executionRead = event.eventType === "tool.execution.completed" && p.status === "completed"; - const canonicalFileRead = executionRead && p.operation === "read" && p.readOnly === true + const canonicalFileRead = executionRead && p.transport === "builtin" && p.operation === "read" && p.readOnly === true && typeof p.outputBytes === "number" && p.outputBytes > 0; for (const file of HIRING_TEMPLATE_READ_FILES) { if ((toolRead && [input.file_path, input.path, input.filePath].some(path => matchingReadPath(path, file))) diff --git a/tests/runner-e2e/hiring-template.test.ts b/tests/runner-e2e/hiring-template.test.ts index d9802e623b0..0038dc26f58 100644 --- a/tests/runner-e2e/hiring-template.test.ts +++ b/tests/runner-e2e/hiring-template.test.ts @@ -81,13 +81,21 @@ describe("production hiring template oracle", () => { fails({ ...e, firstAfterReuse: { ...e.first!, latestRevisionId: "modified" } }, "original-preserved"); }); - it("compares each revision's bundle and example without imposing candidate length on baseline", () => { + it("compares each revision's bundle and example without imposing candidate length on baseline", async () => { const historical = validEvidence(); const oldFiles = { "AGENTS.md": "A long historical CEO role.\n".repeat(40), "HEARTBEAT.md": "Heartbeat", "SOUL.md": "Identity", "TOOLS.md": "Tools" }; historical.expectedCeoFiles = oldFiles; historical.leadInstructions!.files = oldFiles; for (const file of Object.keys(oldFiles)) historical.expectedSourceHashes[`server/src/onboarding-assets/ceo/${file}`] = historical.servedSourceHashes[`server/src/onboarding-assets/ceo/${file}`] = hiringTemplateHash(oldFiles[file as keyof typeof oldFiles]); - historical.coderExample = "A long historical coder role.\n".repeat(60); + // Exact baseline source: skills/paperclip-create-agent/references/agents/coder.md + // at d7bdfc422cadcc407f2945e211833a8382b80117. Its SHA-256 below + // preserves the actual historical text and all four placeholder kinds. + const historicalReference = await readFile(new URL("./fixtures/hiring-templates/coder.d7bdfc4.md", import.meta.url), "utf8"); + expect(hiringTemplateHash(historicalReference)).toBe("766c6f5db907f3dc2e317d540feb80e77358202c59d759449228a32450fa6202"); + historical.coderExample = renderHiringCoderExample(historicalReference, "Casey", "Fixture Company", "CEO", "FIX"); + expect(historical.coderExample).not.toMatch(/\{\{[^}]+\}\}/); + expect(historical.coderExample).toContain("You report to CEO."); + expect(historical.coderExample.match(/\/FIX\/agents\//g)).toHaveLength(3); historical.hiredInstructions!.files["AGENTS.md"] = historical.hiredInstructionsAfterReuse!.files["AGENTS.md"] = historical.coderExample; const result = gradeHiringTemplate(historical); expect(result).toMatchObject({ outcomePassed: true, comparisonStatus: "comparable" }); @@ -114,11 +122,13 @@ describe("production hiring template oracle", () => { it("requires a completed pre-hire lead read rather than an echoed path, failed read or listing", () => { const e = validEvidence(), good = e.readRuns[0]!; for (const command of ["echo /workspace/.agents/skills/paperclip-create-agent/SKILL.md", "ls /workspace/.agents/skills/paperclip-create-agent/SKILL.md", - "cat /workspace/.agents/skills/wrong-skill/SKILL.md", "cat $SKILL/SKILL.md", "cat /workspace/.agents/skills/paperclip-create-agent/SKILL.md > /dev/null"]) { + "cat /workspace/.agents/skills/wrong-skill/SKILL.md", "cat $SKILL/SKILL.md", "cat /workspace/.agents/skills/paperclip-create-agent/SKILL.md > /dev/null", + ...["cat --help", "cat --version", "head --help", "tail --version", "head -n 0", "tail -c 0", "sed -n ''", "sed -n '1q'", "sed -n 'q'", "sed --help", "sed -n '1,200w /tmp/other'", "cat -n"].map(prefix => `${prefix} /workspace/.agents/skills/paperclip-create-agent/SKILL.md`), + 'cat "/workspace/.agents/skills/paperclip-create-agent/SKILL.md""suffix"' ]) { const events = commandEvents("SKILL.md"); const payload = events[0]!.payload.prpEvent.payload as Record; payload.name = command; - expect(hiringTemplateReadReceipts([{ ...good, events }], "lead")).toHaveLength(0); + expect(hiringTemplateReadReceipts([{ ...good, events }], "lead"), command).toHaveLength(0); } fails({ ...e, readRuns: [{ ...good, agentId: "coder" }] }, "production-source-reads"); fails({ ...e, readRuns: [{ ...good, events: good.events.slice(1) }] }, "production-source-reads"); @@ -132,6 +142,19 @@ describe("production hiring template oracle", () => { expect(hiringTemplateReadReceipts([{ ...good, events: noOutput }], "lead")).toHaveLength(0); }); + it("accepts only supported direct read arguments and does not trust process read hints", () => { + const path = "/workspace/.agents/skills/paperclip-create-agent/SKILL.md"; + for (const command of [`cat ${path}`, `cat -- '${path}'`, `head -n 200 ${path}`, `head -n200 ${path}`, `tail -c 200 ${path}`, + `sed -n '1,200p' ${path}`, `sed -n -e 'p' ${path}`]) { + const events = commandEvents("SKILL.md"); + (events[0]!.payload.prpEvent.payload as Record).name = command; + expect(hiringTemplateReadReceipts([{ runId: "lead-1", agentId: "lead", events }], "lead"), command).toHaveLength(1); + } + const events = commandEvents("SKILL.md"); + Object.assign(events[0]!.payload.prpEvent.payload, { name: `cat --help ${path}`, operation: "read", readOnly: true, target: ".agents/skills/paperclip-create-agent/SKILL.md" }); + expect(hiringTemplateReadReceipts([{ runId: "lead-1", agentId: "lead", events }], "lead")).toHaveLength(0); + }); + it("uses the production ACPX event mapper and leaves redacted absolute read paths uncomparable", () => { const events = HIRING_TEMPLATE_READ_FILES.flatMap(file => canonicalProviderEventsFromAcpxRuntimeEvent({ type: "tool_call", tag: "tool_call_update", toolCallId: `read-${file}`, title: "Read", kind: "read", status: "completed", @@ -183,7 +206,7 @@ describe("production hiring fixture wiring and source observations", () => { expect(source.servedSourceHashes).toEqual(source.expectedSourceHashes); expect(source.assignedSkills).toContain(HIRING_TEMPLATE_SKILL_KEY); expect(source.loaderHash).toMatch(/^[a-f0-9]{64}$/); - expect(renderHiringCoderExample(source.coderReference, "Casey", "Company", "CEO")).not.toContain("{{"); + expect(renderHiringCoderExample(source.coderReference, "Casey", "Company", "CEO", "FIX")).not.toContain("{{"); const wrongApi = { get: async (path: string) => path.includes("/files?") ? { content: "different revision" } : get(path) } as Pick; const wrongSource = await readHiringTemplateSources(wrongApi, "company", "lead"); expect(wrongSource.servedSourceHashes).not.toEqual(wrongSource.expectedSourceHashes); @@ -194,8 +217,8 @@ describe("production hiring fixture wiring and source observations", () => { ? { mode: "managed", entryFile: "AGENTS.md", files: ["AGENTS.md", "HEARTBEAT.md", "SOUL.md", "TOOLS.md", "notes/today.md"].map(path => ({ path, binary: false })).concat([{ path: "image.png", binary: true }]) } : { content: new URL(path, "http://fixture").searchParams.get("path") } } as Pick; expect(Object.keys((await readHiringInstructions(api, "lead")).files)).toEqual(["AGENTS.md", "HEARTBEAT.md", "SOUL.md", "TOOLS.md"]); - expect(renderHiringCoderExample("Historical example\n```md\nYou are {{agentName}} at {{companyName}}. Report to {{managerTitle}}.\nLarge manual content.\n```", "Casey", "Company", "CEO")) + expect(renderHiringCoderExample("Historical example\n```md\nYou are {{agentName}} at {{companyName}}. Report to {{managerTitle}}.\nLarge manual content.\n```", "Casey", "Company", "CEO", "FIX")) .toBe("You are Casey at Company. Report to CEO.\nLarge manual content."); - expect(() => renderHiringCoderExample("No example", "Casey", "Company", "CEO")).toThrow(); + expect(() => renderHiringCoderExample("No example", "Casey", "Company", "CEO", "FIX")).toThrow(); }); });