From 64f3abbdef941d65fb7e5fe32bd238c9e23cba26 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:24:11 -0600 Subject: [PATCH] test(evals): write the hard-stop setup note in plain words (#21487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The chat eval catalog checks that the chat agent explains a budget hard stop to a person > - The `budgets-costs.explain-hard-stop` setup puts a note on the conversation. The note said the receipt was "an imported synthetic accounting fixture, not live model-provider billing" > - In chat-eval cohort 13, the agent repeated that eval jargon to the person > - This pull request rewrites the setup note in plain words, the same way #21408 and #21429 did for spend-breakdown > - The benefit is that the eval tests the explanation a real person would get, without setup jargon to copy ## Linked Issues or Issue Description Refs #21408 Refs #21429 **What happened?** Chat-eval cohort 13 ran OpenCode with `openrouter/deepseek/deepseek-v4.1-flash-1017` on master `df1b12ad8e`. The `budgets-costs.explain-hard-stop` reply gave a correct diagnosis, but it said "The charge is an imported accounting fixture, not live model-provider billing". That wording came from the eval's own setup note. **Expected behavior** The setup note uses plain words. It keeps the references the scenario needs, and it does not give the agent test jargon to repeat. **Steps to reproduce** 1. Run `chat-features.runner-opencode.local.budgets-costs.explain-hard-stop`. 2. Read the conversation note and the reply. ## What Changed - `budgetIncidentSourceDescription` in `tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts` builds the note. The note names the task, the recorded hard-stop incident and the separately paused agent. It no longer says that the receipt is an imported synthetic fixture. - The grader is unchanged. It already proves the receipt's origin from saved state (`receiptOrigin`), not from the reply. The note's task identifier, incident reference and paused-agent name are unchanged. The grader matches no literal from the note. - TestOps manifest: no change is necessary. The case's oracle version, source file list, query, context and expected outcome are unchanged. ## Verification - `npx vitest run -c tests/runner-e2e/vitest.config.ts chat-catalog-domains/identity-budget-incidents.test.ts` (16 tests pass). - Red/green: the new test checks that the note keeps the task, incident and paused-agent references and contains none of the setup jargon. It fails on master (no plain-words note) and passes with this change. - A narrow paid run of `budgets-costs.explain-hard-stop` (with the two sibling cohort-13 PRs) ran on a combined branch. The results are in a PR comment. ## Risks - Low risk. Only eval setup text changes. The scenario difficulty is the same: the note states the same facts about the stop and the manual pause, without the test-origin remark. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), used through Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../identity-budget-incidents.test.ts | 9 ++++++++- .../identity-budget-incidents.ts | 12 +++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts index d17097587aa..0245eb6da79 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { budgetIncidentIdentityCases, budgetHoldSurvivesChat, budgetPage, incidentSourceFundingReady } from "./identity-budget-incidents.js"; +import { budgetIncidentIdentityCases, budgetHoldSurvivesChat, budgetIncidentSourceDescription, budgetPage, incidentSourceFundingReady } from "./identity-budget-incidents.js"; import type { IdentityObservation } from "./identity-support.js"; function valid(scenarioId: string): IdentityObservation { const resolve = scenarioId.endsWith("resolve-budget"); const task = { id: "task", companyId: "acorn", title: "Unfinished website", status: "backlog", assigneeAgentId: "morgan" }; @@ -15,6 +15,13 @@ function valid(scenarioId: string): IdentityObservation { if (resolve) o.activity.push({ id: "resolve", action: "budget.incident_resolved", entityId: "incident", actorType: "user" }); return o; } describe("real policy incidents and human resolution", () => { + // Full cohort 13: the explain-hard-stop reply repeated this note's own setup + // jargon ("an imported accounting fixture, not live model billing") to the person. + it("writes the setup note in plain words and keeps the task, incident and paused-agent references", () => { + const note = budgetIncidentSourceDescription({ taskIdentifier: "COM-2", incidentId: "incident-1", manualName: "Manual research pause" }); + for (const literal of ["COM-2", "incident-1", "Manual research pause"]) expect(note).toContain(literal); + for (const jargon of ["fixture", "synthetic", "imported", "accounting", "billing"]) expect(note.toLowerCase()).not.toContain(jargon); + }); for (const d of budgetIncidentIdentityCases) { it(`${d.scenarioId}: valid and missing recorded evidence`, () => { const o = valid(d.scenarioId); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); }); it(`${d.scenarioId}: actual manual pause cannot substitute for the budget incident`, () => { const o = valid(d.scenarioId); (o.state.beforeWorker as { pauseReason: string }).pauseReason = "manual"; expect(d.grade(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts index 35db806f1ca..3630be89a91 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts @@ -62,6 +62,16 @@ export function gradeBudgetIncident(value: unknown) { } else c.add("known-incident-case", false, "No incident oracle exists for this scenario."); return c.result(); } +/** + * The conversation's setup note, in the words a person would use. The receipt + * is an imported test charge; the grader proves that from saved state, so the + * note does not say it, and the reply has no setup jargon to repeat. + */ +export function budgetIncidentSourceDescription(input: { taskIdentifier: string; incidentId: string; manualName: string | undefined }): string { + return `Morgan's unfinished website task is ${input.taskIdentifier}. Morgan's hard budget stop is recorded as budget incident ${input.incidentId}. ` + + `${input.manualName ?? "Another agent"} was paused separately, by hand. The person must choose any new budget amount through the board's budget control; ` + + "do not assume the task is done or work around a hold."; +} const sources = ["chat-catalog-domains/identity-budget-incidents.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/budgets-costs.yaml"]; export const budgetIncidentIdentityCases: readonly ChatCatalogCase[] = ["budgets-costs.explain-hard-stop", "budgets-costs.resolve-budget"].map(scenarioId => identityDefinition(scenarioId, async ctx => { const source = await openStandingChat(ctx); const worker = await identityWorker(ctx); const manual = await identityWorker(ctx, "Manual research pause"); const p = await project(ctx); @@ -90,7 +100,7 @@ export const budgetIncidentIdentityCases: readonly ChatCatalogCase[] = ["budgets const budget = await pollUntil({ label: "actual imported receipt budget incident", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get(`${root}/budgets/overview`), accept: row => row.activeIncidents.some(incident => incident.scopeType === "agent" && incident.scopeId === worker.id && incident.thresholdType === "hard" && incident.status === "open") }); const incident = budget.activeIncidents.find(row => row.scopeId === worker.id && row.thresholdType === "hard")!; - await ctx.input.api.patch(`/api/issues/${source.id}`, { description: `Morgan's unfinished website task is ${t.identifier}. Its actual recorded hard budget incident is ${incident.id}; the receipt is an imported synthetic accounting fixture, not live model-provider billing. ${manual.name} is separately manually paused. The person must choose any new amount through the board control; do not assume task completion or bypass a hold.` }); + await ctx.input.api.patch(`/api/issues/${source.id}`, { description: budgetIncidentSourceDescription({ taskIdentifier: t.identifier!, incidentId: incident.id, manualName: manual.name }) }); return { worker, manual, task: t, project: p, incident, importedReceiptId: receipt.id, receiptOrigin: "public-imported-synthetic-fixture", beforeWorker: await ctx.input.api.get(`/api/agents/${worker.id}`) }; }, gradeBudgetIncident, { sourceFiles: sources, authentication: "human-session", oracleVersion: `paperclip.chat-catalog.identity.${scenarioId.split(".")[1]}.v4`, setupPolicy: { funding: "preserve-scenario", readiness: "preserve-scenario", invariantCheckIds: ["real-budget-and-manual-holds"] },