From a762b6bffd5bc944af8d7b10c25e4f1432158ea7 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:52:06 -0600 Subject: [PATCH] test(evals): keep the spend fixture's pointer to the tasks that record the charges (#21429) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - #21408 rewrote the spend-breakdown eval setup text in user words > - Its first wording ("Launch has recorded API charges on COM-3 (Morgan) and COM-4 (Riley)") dropped the old pointer that these tasks hold the charges > - In two paid runs the agent then gave correct totals but did not link the tasks that record them, which the case contract needs ("linked records support the totals") > - This pull request restores that pointer in plain words > - The benefit is that the setup gives the same information as before, without jargon ## Linked Issues or Issue Description Refs #21408. This commit was pushed to #21408 after it merged, so it is a follow-up. **What happened?** Paid runs 43451957487 and 43455387724: `budgets-costs.spend-breakdown` failed `actual-agent-breakdown-and-incomplete-total` because the reply did not link COM-3 and COM-4. **Expected behavior** The setup says where the charges are recorded, as the original text did. **Steps to reproduce** 1. Run `budgets-costs.spend-breakdown` with the #21408 wording. ## What Changed - The setup description now says "Launch's API charges this week are recorded on tasks COM-3 (Morgan) and COM-4 (Riley)." ## Verification - Paid run 43460401711 with this wording: `budgets-costs.spend-breakdown` passed. The reply links COM-3, COM-4, the project and the unpriced run, with no internal terms. ## Risks - Low. Eval setup text only. ## Model Used - Claude, claude-opus-5-5, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-domains/identity-accounting.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts index 4c5c15b0f94..d5b48d3bcb4 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts @@ -320,7 +320,7 @@ export const accountingIdentityCases: readonly ChatCatalogCase[] = [ const receipt = await ctx.input.api.post(`${root}/cost-events`, { agentId, issueId: work.id, projectId: work.projectId, provider: "synthetic-eval-import", biller: "synthetic-eval-import", model: "imported-fixture-api-usage", billingType: "metered_api", costStatus: "reported", costCents: cents, inputTokens: 0, outputTokens: 0, occurredAt, idempotencyKey: `identity-report:${ctx.input.nonce}:${receipts.length}` }); receipts.push(receipt.id); } - await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The requested reporting week is ${weekFrom} through ${weekTo} (UTC). Launch has recorded API charges on ${morganReceiptTask.identifier} (Morgan) and ${rileyReceiptTask.identifier} (Riley). Morgan also finished one work session on Launch (run ${state.originalRunId}) whose usage is saved but not yet priced. Leave out charges from other projects and earlier weeks. Plan subscription fees are separate from API charges.` }); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The requested reporting week is ${weekFrom} through ${weekTo} (UTC). Launch's API charges this week are recorded on tasks ${morganReceiptTask.identifier} (Morgan) and ${rileyReceiptTask.identifier} (Riley). Morgan also finished one work session on Launch (run ${state.originalRunId}) whose usage is saved but not yet priced. Leave out charges from other projects and earlier weeks. Plan subscription fees are separate from API charges.` }); return { ...state, riley, morganReceiptTask, rileyReceiptTask, weekFrom, weekTo, importedReceiptIds: receipts, receiptOrigin: "public-imported-synthetic-fixture" }; }, gradeAccounting, { authentication: "human-session", oracleVersion: "paperclip.chat-catalog.spend-breakdown.structural.v5", sourceFiles: [...sources, "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts"], expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] } }),