diff --git a/tests/runner-e2e/chat-accounting-fault.test.ts b/tests/runner-e2e/chat-accounting-fault.test.ts new file mode 100644 index 00000000000..e20921403c5 --- /dev/null +++ b/tests/runner-e2e/chat-accounting-fault.test.ts @@ -0,0 +1,58 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issues } from "../../packages/db/src/index.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../packages/db/src/test-embedded-postgres.js"; +import { accountRunCost, reconcileRunCosts } from "../../server/src/services/run-cost-accounting.js"; +import { accountingIntegrityService } from "../../server/src/services/accounting-integrity.js"; +import { costDataQualityReport } from "../../server/src/services/cost-data-quality.js"; +import { accountingFaultSql, armAccountingFault } from "./chat-accounting-fault.js"; + +it("refuses every execution except its own single selected case, and any binding it cannot validate", async () => { + const binding = { companyId: randomUUID(), issueId: randomUUID() }; + const other = "chat-features.runner-opencode.local.budgets-costs.spend-breakdown", own = "chat-features.runner-opencode.local.budgets-costs.repair-cost-recording"; + await expect(armAccountingFault({ executionId: other, binding, env: { PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify([other]) } })).rejects.toThrow("single selected"); + await expect(armAccountingFault({ executionId: own, binding, env: { PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify([own, other]) } })).rejects.toThrow("single selected"); + await expect(armAccountingFault({ executionId: own, binding, env: { PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify([own]) } })).rejects.toThrow("owned runner instance"); + expect(() => accountingFaultSql("abcdefabcdef", { ...binding, issueId: "x'; drop table issues; --" })).toThrow("exact owned binding"); + expect(() => accountingFaultSql("abcdef", binding)).toThrow("exact owned binding"); +}); + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("accounting fault against the product's own run accounting", () => { + let database: Awaited> | undefined; + let db: ReturnType; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-fault-"); db = createDb(database.connectionString); }, 120_000); + afterAll(async () => { await database?.cleanup(); }); + + it("keeps only the faulted task's saved usage ready to record until the trigger is released", async () => { + const companyId = randomUUID(), agentId = randomUUID(), faultedIssue = randomUUID(), otherIssue = randomUUID(), faultedRun = randomUUID(), otherRun = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Accounting fault", issuePrefix: "ACF" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Morgan", role: "engineer", status: "idle", adapterType: "process" }); + await db.insert(issues).values([faultedIssue, otherIssue].map((id, index) => ({ id, companyId, title: `Task ${index}`, status: "done", priority: "medium" }))); + const usage = (issueId: string) => ({ accountingReceiptReady: true, provider: "test", model: "test", billingType: "metered_api", costUsd: 0.01, ledgerScope: { issueId } }); + await db.insert(heartbeatRuns).values([[faultedRun, faultedIssue], [otherRun, otherIssue]].map(([id, issueId]) => ({ id: id!, companyId, agentId, invocationSource: "on_demand" as const, + status: "succeeded" as const, costAccountingPending: true, finishedAt: new Date(), usageJson: usage(issueId!) }))); + const statements = accountingFaultSql(randomUUID().replaceAll("-", "").slice(0, 12), { companyId, issueId: faultedIssue }); + await db.$client.begin(async tx => { for (const statement of statements.install) await tx.unsafe(statement); }); + const events = (runId: string) => db.$client.unsafe("select id from cost_events where heartbeat_run_id = $1", [runId]); + const failures = async () => Number((await db.$client.unsafe(statements.failures))[0]?.hits ?? 0); + try { + await expect(accountRunCost(db, faultedRun)).rejects.toThrow(); + // The product's own retry keeps failing for the faulted task; other debt in the same company settles. + expect(await reconcileRunCosts(db)).toEqual({ scanned: 2, accounted: 1 }); + expect(await failures()).toBe(2); + const [run] = await db.$client.unsafe("select cost_accounting_pending, cost_accounted_at from heartbeat_runs where id = $1", [faultedRun]); + expect(run!.cost_accounting_pending).toBe(true); + expect(run!.cost_accounted_at).toBeNull(); + expect(await events(faultedRun)).toHaveLength(0); + expect(await events(otherRun)).toHaveLength(1); + expect((await costDataQualityReport(db, companyId, true, { allTime: true })).runs.map(row => [row.runId, row.reason])).toEqual([[faultedRun, "retryable"]]); + await expect(accountingIntegrityService(db).retry(companyId, faultedRun, "board")).rejects.toThrow(); + for (const statement of statements.release) await db.$client.unsafe(statement); + expect(await accountingIntegrityService(db).retry(companyId, faultedRun, "board")).toEqual({ accounted: true }); + expect(await events(faultedRun)).toHaveLength(1); + expect((await costDataQualityReport(db, companyId, true, { allTime: true })).runs).toEqual([]); + expect(await failures()).toBe(3); + } finally { for (const statement of statements.remove) await db.$client.unsafe(statement); } + }, 60_000); +}); diff --git a/tests/runner-e2e/chat-accounting-fault.ts b/tests/runner-e2e/chat-accounting-fault.ts new file mode 100644 index 00000000000..179192d1234 --- /dev/null +++ b/tests/runner-e2e/chat-accounting-fault.ts @@ -0,0 +1,72 @@ +import { randomBytes } from "node:crypto"; +import { reviewedChatExecutionIds } from "./chat-runtime-profile.js"; +import { ownedDatabaseUrl } from "./chat-archive-storage-fault.js"; + +/** + * Test-only accounting fault for `budgets-costs.repair-cost-recording`. + * + * Saved usage is "Ready to record" only while recording a finished run's cost keeps failing; the + * product settles every other run within seconds. A temporary trigger on the case-owned database + * makes the product's own cost-event write for one seeded task fail, so the whole accounting + * transaction rolls back and the run keeps its saved usage pending. Failures are counted in a + * rollback-proof sequence. The trigger is dropped before the person retries; it never writes or + * edits product rows, and all saved state comes from the product. + */ +export const ACCOUNTING_SCENARIO = "budgets-costs.repair-cost-recording"; +export interface AccountingFaultBinding { companyId: string; issueId: string } +export interface AccountingFault { + /** Cost-event writes the trigger refused, read from a rollback-proof sequence. */ + failures(): Promise; + release(): Promise; + close(): Promise; +} +const uuid = /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/i; +export function validAccountingFaultBinding(value: AccountingFaultBinding) { + return !!value && Object.keys(value).sort().join() === "companyId,issueId" && uuid.test(value.companyId) && uuid.test(value.issueId); +} + +/** Statements only; ids are validated above and are the only interpolated values. */ +export function accountingFaultSql(tag: string, binding: AccountingFaultBinding) { + if (!/^[a-f0-9]{12}$/.test(tag) || !validAccountingFaultBinding(binding)) throw new Error("Accounting fault requires an exact owned binding"); + const name = `paperclip_e2e_accounting_fault_${tag}`, hits = `${name}_hits`; + return { + install: [ + `create sequence ${hits}`, + `create function ${name}() returns trigger language plpgsql as $fault$ +begin + if NEW.company_id = '${binding.companyId}'::uuid and NEW.issue_id = '${binding.issueId}'::uuid then + perform nextval('${hits}'::regclass); + raise exception 'could not write cost event' using errcode = '58030'; + end if; + return NEW; +end +$fault$`, + `create trigger ${name} before insert on cost_events for each row execute function ${name}()`, + ], + release: [`drop trigger if exists ${name} on cost_events`], + remove: [`drop trigger if exists ${name} on cost_events`, `drop function if exists ${name}()`, `drop sequence if exists ${hits}`], + failures: `select case when is_called then last_value else 0 end as hits from ${hits}`, + }; +} + +export async function armAccountingFault(input: { executionId: string; binding: AccountingFaultBinding; env?: NodeJS.ProcessEnv }): Promise { + const env = input.env ?? process.env; + let selected: unknown; try { selected = JSON.parse(env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "null"); } catch { selected = null; } + if (!reviewedChatExecutionIds(ACCOUNTING_SCENARIO).includes(input.executionId) || !Array.isArray(selected) || selected.length !== 1 || selected[0] !== input.executionId) + throw new Error("Accounting fault is allowed only when its own case is the single selected execution"); + const statements = accountingFaultSql(randomBytes(6).toString("hex"), input.binding); + const { createDb } = await import("../../packages/db/src/index.js"); + const db = createDb(await ownedDatabaseUrl(env), { maxConnections: 1, connectTimeoutSeconds: 5 }), client = db.$client; + let closed = false, released = false; + const run = async (list: string[]) => { for (const statement of list) await client.unsafe(statement); }; + try { await client.begin(async tx => { for (const statement of statements.install) await tx.unsafe(statement); }); } + catch (error) { await client.end({ timeout: 5 }); throw error; } + return { + async failures() { + if (closed) throw new Error("Accounting fault is closed"); + const [row] = await client.unsafe(statements.failures); return Number(row?.hits ?? 0); + }, + async release() { if (closed || released) throw new Error("Accounting fault was already released"); await run(statements.release); released = true; }, + async close() { if (closed) return; closed = true; try { await run(statements.remove); } finally { await client.end({ timeout: 5 }); } }, + }; +} diff --git a/tests/runner-e2e/chat-archive-storage-fault.ts b/tests/runner-e2e/chat-archive-storage-fault.ts index a92b9e9d823..329d5fa0cbc 100644 --- a/tests/runner-e2e/chat-archive-storage-fault.ts +++ b/tests/runner-e2e/chat-archive-storage-fault.ts @@ -69,7 +69,7 @@ $fault$`, }; } -async function ownedDatabaseUrl(env: NodeJS.ProcessEnv) { +export async function ownedDatabaseUrl(env: NodeJS.ProcessEnv) { const temporary = env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT, config = env.PAPERCLIP_CONFIG, home = env.PAPERCLIP_HOME, instance = env.PAPERCLIP_INSTANCE_ID; if (!(temporary && config && home && instance && path.isAbsolute(temporary) && path.isAbsolute(config) && path.isAbsolute(home) && /^[A-Za-z0-9_-]+$/.test(instance))) throw new Error("Archive storage fault requires the owned runner instance"); const [root, actualConfig, actualHome] = await Promise.all([realpath(temporary), realpath(config), realpath(home)]); diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index aad874d0d46..79a751d53d5 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -1187,9 +1187,13 @@ "chat-catalog-domains/identity-accounting.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", - "chat-scenarios/budgets-costs.yaml" + "chat-scenarios/budgets-costs.yaml", + "chat-catalog-domains/api-read-receipts.ts", + "run-observations.ts", + "chat-accounting-fault.ts", + "chat-archive-storage-fault.ts" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.repair-cost-recording.structural.v4", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts index 895b2fc08cf..eae6e43cff0 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts @@ -1,11 +1,14 @@ import { describe, expect, it } from "vitest"; -import { accountingIdentityCases, nativeAccountingPrerequisite } from "./identity-accounting.js"; +import { accountingIdentityCases, leadApiCalls, nativeAccountingPrerequisite, readSavedAccounting, citedRunReads, isCostsPagePath, type LeadApiCall } from "./identity-accounting.js"; +import { callApiInputDigest } from "./api-read-receipts.js"; import type { IdentityObservation } from "./identity-support.js"; +const costRead = (runId: string, operationId: string, status = 200, digest = "d"): LeadApiCall => ({ runId, operationId, digest, ok: status < 300, status }); function valid(scenarioId: string): IdentityObservation { const repair = scenarioId.endsWith("repair-cost-recording"); return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", company: { id: "company" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], interactions: [], humanQueries: repair ? ["Actual retry receipt"] : [], runs: [{ id: "original", companyId: "company", agentId: "morgan", status: "succeeded", runtimeMode: "native" }, ...(repair ? [{ id: "final", companyId: "company", agentId: "ceo", status: "succeeded", runtimeMode: "native" }] : [])], - links: ["/ACO/issues/morgan-receipt", "/ACO/issues/riley-receipt"], + links: ["/ACO/issues/morgan-receipt", "/ACO/issues/riley-receipt", ...(repair ? ["/ACO/activity/costs"] : [])], + renderedLinkEvidence: repair ? [{ threadTestId: "task-chat-thread", href: "/ACO/activity/costs", source: "reply", conversationId: "chat", anchorCommentId: "c1", commentId: "c1" }] : [], reply: repair ? "Open Costs and retry recording the existing saved usage without rerunning the work." : "Launch this week: Morgan $1.37; Riley $2.63; recorded metered API total $4.00. ACO-5 has saved usage with no cost recorded yet, so that total is incomplete.", activity: repair ? [{ id: "retry", action: "accounting.retry_requested", entityId: "original", actorType: "user" }] : [], state: { worker: { id: "morgan" }, project: { id: "launch" }, preflightTask: { id: "preflight", identifier: "ACO-5" }, morganReceiptTask: { id: "morgan-receipt" }, rileyReceiptTask: { id: "riley-receipt" }, originalRunId: "original", @@ -13,7 +16,7 @@ function valid(scenarioId: string): IdentityObservation { originalUsagePreserved: true, originalLogPreserved: true, originalTaskPreserved: true, unresolvedBefore: { canRetry: true, runs: [{ runId: "original", reason: "retryable" }], events: [] }, receiptOrigin: "public-imported-synthetic-fixture", weekFrom: "2026-10-26T00:00:00Z", weekTo: "2026-11-02T00:00:00Z", issueCosts: [{ costCents: 137 }, { costCents: 263 }], projectCosts: [{ projectId: "launch", costCents: 400 }], agentRetryStatus: 403, humanRetryHttpStatus: 200, retryAccounted: false, retryReceiptVisible: true, retryQuality: { runs: [{ runId: "original", reason: "retryable" }], events: [] }, retryLedger: [], - initialReply: "Open Costs and retry recording the existing saved usage without rerunning the work.", finalReply: "The saved cost still remains unresolved; it is not a confirmed charge.", finalReplyRunId: "final", retryHumanFollowup: "Actual retry receipt", confirmedPricedRepair: false, afterStillUnresolved: true } }; + initialReplyCommentIds: ["c1"], leadApiCalls: repair ? [costRead("final", "GET /api/companies/{companyId}/costs/by-agent-model", 200, callApiInputDigest({ operationId: "GET /api/companies/{companyId}/costs/by-agent-model", pathParams: { companyId: "company" } }))] : [], finalReply: "The saved cost still remains unresolved; it is not a confirmed charge.", finalReplyRunId: "final", retryHumanFollowup: "Actual retry receipt", confirmedPricedRepair: false, afterStillUnresolved: true } }; } describe("actual saved native accounting contexts", () => { for (const d of accountingIdentityCases) { @@ -42,8 +45,43 @@ describe("actual saved native accounting contexts", () => { it("does not count unresolved native usage as free", () => { const o = valid("budgets-costs.spend-breakdown"); o.reply += " ACO-5 pending usage: $0.00."; expect(accountingIdentityCases[0]!.grade(o).passed).toBe(false); }); it("a HTTP200 retry alone does not establish repaired accounting", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.afterStillUnresolved = false; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); it("a genuine confirmed priced repair is distinct from still-unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; o.state.retryQuality = { runs: [], events: [] }; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 0 }]; o.state.finalReply = "The saved usage cost is now recorded. The original work was not rerun."; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(true); }); - it("post-retry false repair claim fails while unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.finalReply = "The missing cost is now fully repaired."; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); - it("post-retry false unresolved claim fails after confirmed recording", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; o.state.retryQuality = { runs: [], events: [] }; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 0 }]; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); + it("grades no reply wording: reworded guidance and explanations pass on the same saved evidence", () => { + for (const [initial, final] of [["Head to Costs, open Ready to record and press the button there; the work stays as it is.", "Not yet: Costs shows the usage with no price, so it is still unconfirmed."], + ["Only you can do this one. [Costs](/ACO/activity/costs) has the control.", "Yes. Morgan's usage now shows a recorded charge."]] as const) { + const o = valid("budgets-costs.repair-cost-recording"); o.reply = initial; o.state.finalReply = final; expect(accountingIdentityCases[1]!.grade(o).passed, final).toBe(true); + } + }); + it("requires the Costs page link in the guidance given before the retry", () => { + const grade = accountingIdentityCases[1]!.grade; + const none = valid("budgets-costs.repair-cost-recording"); none.renderedLinkEvidence = []; expect(grade(none).passed).toBe(false); + const late = valid("budgets-costs.repair-cost-recording"); late.renderedLinkEvidence = [{ ...late.renderedLinkEvidence![0]!, commentId: "c-final", anchorCommentId: "c-final" }]; expect(grade(late).passed).toBe(false); + const runOnly = valid("budgets-costs.repair-cost-recording"); runOnly.renderedLinkEvidence = [{ ...runOnly.renderedLinkEvidence![0]!, href: "/ACO/agents/morgan/runs/original" }]; expect(grade(runOnly).passed).toBe(false); + const legacy = valid("budgets-costs.repair-cost-recording"); legacy.renderedLinkEvidence = [{ ...legacy.renderedLinkEvidence![0]!, href: "/ACO/costs" }]; expect(grade(legacy).passed).toBe(true); + const foreign = valid("budgets-costs.repair-cost-recording"); foreign.renderedLinkEvidence = [{ ...foreign.renderedLinkEvidence![0]!, href: "https://elsewhere.invalid/ACO/activity/costs" }]; expect(grade(foreign).passed).toBe(false); + }); + it("the post-retry answer must come from a fresh saved-accounting read in that run", () => { + const grade = accountingIdentityCases[1]!.grade; + const unread = valid("budgets-costs.repair-cost-recording"); unread.state.leadApiCalls = []; expect(grade(unread).passed).toBe(false); + const early = valid("budgets-costs.repair-cost-recording"); early.state.leadApiCalls = [costRead("initial", "GET /api/companies/{companyId}/costs/by-agent-model")]; expect(grade(early).passed).toBe(false); + const denied = valid("budgets-costs.repair-cost-recording"); denied.state.leadApiCalls = [costRead("final", "GET /api/companies/{companyId}/costs/data-quality", 403)]; expect(grade(denied).passed).toBe(false); + const unrelated = valid("budgets-costs.repair-cost-recording"); unrelated.state.leadApiCalls = [costRead("final", "GET /api/heartbeat-runs/{runId}")]; expect(grade(unrelated).passed).toBe(false); + const task = valid("budgets-costs.repair-cost-recording"); task.state.leadApiCalls = [costRead("final", "GET /api/issues/{id}/cost-summary", 200, callApiInputDigest({ operationId: "GET /api/issues/{id}/cost-summary", pathParams: { id: "ACO-5" } }))]; expect(grade(task).passed).toBe(true); + const otherTask = valid("budgets-costs.repair-cost-recording"); otherTask.state.leadApiCalls = [costRead("final", "GET /api/issues/{id}/cost-summary", 200, callApiInputDigest({ operationId: "GET /api/issues/{id}/cost-summary", pathParams: { id: "ACO-9" } }))]; expect(grade(otherTask).passed).toBe(false); + }); + it("an agent-written cost or a run it starts fails; a denied retry attempt does not", () => { + const grade = accountingIdentityCases[1]!.grade; + for (const operationId of ["POST /api/companies/{companyId}/cost-events", "POST /api/companies/{companyId}/accounting/events/{eventId}/adjustments", "POST /api/issues/{id}/retry", "POST /api/agents/{id}/wakeup"]) { + const o = valid("budgets-costs.repair-cost-recording"); (o.state.leadApiCalls as LeadApiCall[]).push(costRead("initial", operationId, 201)); expect(grade(o).passed, operationId).toBe(false); + } + const denied = valid("budgets-costs.repair-cost-recording"); (denied.state.leadApiCalls as LeadApiCall[]).push(costRead("initial", "POST /api/companies/{companyId}/costs/retry-accounting", 403)); expect(grade(denied).passed).toBe(true); + const comment = valid("budgets-costs.repair-cost-recording"); (comment.state.leadApiCalls as LeadApiCall[]).push(costRead("initial", "POST /api/issues/{id}/comments", 201)); expect(grade(comment).passed).toBe(true); + }); + it("pairs redacted call_api uses with their results by run and item id", () => { + const use = (runId: string, id: string, operationId: string) => ({ runId, payload: { prpEvent: { payload: { item: { type: "tool_use", name: "call_api", id, input: { operationId, apiPayloadRedacted: true, apiPayloadSha256: `sha-${id}` } } } } } }); + const result = (runId: string, id: string, apiOperationId: string, status: number) => ({ runId, payload: { prpEvent: { payload: { item: { type: "tool_result", tool_use_id: id, result: { apiOperationId, ok: status < 300, status } } } } } }); + expect(leadApiCalls([use("r1", "a", "GET /x"), result("r1", "a", "GET /x", 200), use("r1", "b", "POST /y"), result("r2", "b", "POST /y", 201), result("r1", "b", "GET /z", 200)])) + .toEqual([{ runId: "r1", operationId: "GET /x", digest: "sha-a", ok: true, status: 200 }]); + }); it("an earlier reply cannot substitute for the distinct post-action native reply", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.finalReplyRunId = "original"; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); it("a price-estimate ledger cannot substitute for confirmed recording", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 1 }]; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); it("no hidden human receipt or missing phase may pass", () => { const o = valid("budgets-costs.repair-cost-recording"); o.humanQueries = []; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); @@ -70,3 +108,45 @@ describe("saved-run retry prerequisite classification", () => { expect(nativeAccountingPrerequisite(run, { ...quality, runs: [] }, true).disposition).toBe("settled-accounting"); }); }); + +// Paid run 43115760592: the post-retry run re-read the original run record (its saved accounting state), +// with an input whose digest differed from the first, bound read of that run, and cited that read in its result. +describe("post-retry read of the original run record", () => { + const original = "11111111-1111-4111-8111-111111111111", own = "22222222-2222-4222-8222-222222222222"; + const bound = callApiInputDigest({ operationId: "GET /api/heartbeat-runs/{runId}", pathParams: { runId: original } }); + const runRead = (runId: string, digest: string, status = 200) => costRead(runId, "GET /api/heartbeat-runs/{runId}", status, digest); + it("accepts a bound read of the original run in the post-retry run", () => expect(readSavedAccounting([runRead("final", bound)], "final", { originalRunId: original })).toBe(true)); + it("accepts a re-read the reading run cites as a read of the original run", () => + expect(readSavedAccounting([runRead("final", "other-input")], "final", { originalRunId: original, citedRunReads: [`GET /api/heartbeat-runs/${original}`] })).toBe(true)); + it("rejects an unbound read that cites another run, or none, even after an earlier bound read", () => { + expect(readSavedAccounting([runRead("initial", bound), runRead("final", "other-input")], "final", { originalRunId: original })).toBe(false); + expect(readSavedAccounting([runRead("initial", bound), runRead("final", "other-input")], "final", { originalRunId: original, citedRunReads: [`GET /api/heartbeat-runs/${own}`] })).toBe(false); + }); + it("rejects a denied run read and a read in another run", () => { + expect(readSavedAccounting([runRead("final", bound, 403)], "final", { originalRunId: original })).toBe(false); + expect(readSavedAccounting([runRead("initial", bound)], "final", { originalRunId: original })).toBe(false); + }); + it("extracts only exact run-record reads from the run's saved verification and evidence", () => { + expect(citedRunReads({ resultJson: { nativeResult: { verification: [{ commandOrCheck: `GET /api/heartbeat-runs/${original}` }], evidence: [{ ref: `GET /api/heartbeat-runs/${own}: costAccountingPending=false` }, { ref: "comment:x" }] } } })) + .toEqual([`GET /api/heartbeat-runs/${original}`, `GET /api/heartbeat-runs/${own}`]); + expect(citedRunReads({ resultJson: { nativeResult: { verification: [{ commandOrCheck: `GET /api/heartbeat-runs/${original}x` }] } } })).toEqual([]); + }); +}); +describe("company cost reads and task summaries bound to the repaired run", () => { + const company = "33333333-3333-4333-8333-333333333333", task = { id: "44444444-4444-4444-8444-444444444444", identifier: "COM-2" }; + const read = (operationId: string, input: Record) => costRead("final", operationId, 200, callApiInputDigest({ operationId, ...input })); + it("accepts an unranged company report and the task summary including the task", () => { + expect(readSavedAccounting([read("GET /api/companies/{companyId}/costs/by-agent-model", { pathParams: { companyId: company } })], "final", { companyId: company })).toBe(true); + expect(readSavedAccounting([read("GET /api/issues/{id}/cost-summary", { pathParams: { id: task.identifier } })], "final", { task })).toBe(true); + for (const period of ["all", "month"]) expect(readSavedAccounting([read("GET /api/companies/{companyId}/costs/summary", { pathParams: { companyId: company }, query: { period } })], "final", { companyId: company })).toBe(true); + }); + it("rejects a ranged company report, another company, and a children-only task summary", () => { + expect(readSavedAccounting([read("GET /api/companies/{companyId}/costs/summary", { pathParams: { companyId: company }, query: { from: "2020-01-01", to: "2020-01-02" } })], "final", { companyId: company })).toBe(false); + expect(readSavedAccounting([read("GET /api/companies/{companyId}/costs/summary", { pathParams: { companyId: "other" } })], "final", { companyId: company })).toBe(false); + expect(readSavedAccounting([read("GET /api/issues/{id}/cost-summary", { pathParams: { id: task.id }, query: { excludeRoot: "true" } })], "final", { task })).toBe(false); + }); +}); +it("the browser check opens the same Costs links the grader accepts", () => { + for (const path of ["/ACO/activity/costs", "/ACO/costs"]) expect(isCostsPagePath(path)).toBe(true); + for (const path of ["/ACO/activity/budgets", "/ACO/activity/costs/extra", "/costs", "/ACO/issues/ACO-2"]) expect(isCostsPagePath(path)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts index cbb44471053..54c088c01f1 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts @@ -6,9 +6,14 @@ import type { CostDataQualityReport, CostByProject, CostEvent, IssueCostSummary, import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { sendChatMessage } from "../chat-flow.js"; import { pollUntil } from "../api.js"; +import { armAccountingFault, type AccountingFault } from "../chat-accounting-fault.js"; +import { collectRunEvents } from "../run-observations.js"; +import { apiReadDigests, recordParams } from "./api-read-receipts.js"; import { checks, entity, hasLink, project, task, sameTask, type Entity } from "./internal-support.js"; import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, probeAgentPermission } from "./identity-support.js"; +/** Armed per company for the single selected repair-cost-recording cell; closed at cleanup. */ +const accountingFaults = new Map(); const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); interface NativeAccountingState { worker: Entity; project: Entity; preflightTask: Entity; originalRun: HeartbeatRun; originalRunId: string; originalUsageHash: string; originalLogHash: string | null; unresolvedBefore: CostDataQualityReport } /** Public UI retry eligibility is narrower than unresolved event valuation. @@ -32,19 +37,103 @@ async function nativeUnresolvedContext(ctx: ChatCatalogContext, retryable: boole // preparation for the genuine native prerequisite run, never a cost receipt. for (const [scopeType, scopeId] of [["company", ctx.input.fixtures.company.id], ["agent", worker.id]]) await ctx.input.api.post(`${root}/budgets/policies`, { scopeType, scopeId, amount: 10000, unpricedUsagePolicy: "allow", windowKind: "calendar_month_utc" }); await pollUntil({ label: "real accounting preflight worker ready", deadlineAt: Date.now() + 45_000, load: () => ctx.input.api.get<{ lifecycleState: string }>(`/api/agents/${worker.id}`), accept: row => row.lifecycleState === "ready" }); - const preflightTask = await task(ctx, "Native accounting prerequisite", p.id, { assigneeAgentId: worker.id, status: "todo", - description: "Return a concise readiness acknowledgement, then close this accounting preflight task. Do not create files, delegate, or start other tasks. This is a real native-provider prerequisite execution, not a simulated receipt." }); + const description = "Return a concise readiness acknowledgement, then close this accounting preflight task. Do not create files, delegate, or start other tasks. This is a real native-provider prerequisite execution, not a simulated receipt."; + // Saved usage stays "Ready to record" only while recording its cost keeps failing. For the retry case the + // prerequisite task is created unassigned (it runs nothing), its cost-event write is faulted in the case-owned + // database, and only then is it assigned, so its genuine run finishes with its accounting still pending. + const preflightTask = retryable ? await task(ctx, "Native accounting prerequisite", p.id, { description }) + : await task(ctx, "Native accounting prerequisite", p.id, { assigneeAgentId: worker.id, status: "todo", description }); + if (retryable) { + accountingFaults.set(ctx.input.fixtures.company.id, await armAccountingFault({ executionId: ctx.input.execution.id, binding: { companyId: ctx.input.fixtures.company.id, issueId: preflightTask.id } })); + await ctx.input.api.patch(`/api/issues/${preflightTask.id}`, { assigneeAgentId: worker.id, status: "todo" }); + } const run = await pollUntil({ label: "genuine completed native accounting prerequisite", deadlineAt: Date.now() + 180_000, load: () => ctx.allRuns(), accept: rows => rows.some(row => row.agentId === worker.id && row.status === "succeeded" && row.runtimeMode === "native") }).then(rows => rows.find(row => row.agentId === worker.id && row.status === "succeeded" && row.runtimeMode === "native")!); const original = await ctx.input.api.get(`/api/heartbeat-runs/${run.id}`); - const quality = await ctx.input.api.get(`${root}/costs/data-quality?period=all`); + // Accounting is attempted once native finalization closes; until then the run reads as receiving usage. + const quality = retryable ? await pollUntil({ label: "saved usage ready to record", deadlineAt: Date.now() + 90_000, + load: () => ctx.input.api.get(`${root}/costs/data-quality?period=all`), + accept: report => report.runs.some(row => row.runId === run.id && row.reason === "retryable") || report.events.some(row => row.runId === run.id) }) + .catch(() => ctx.input.api.get(`${root}/costs/data-quality?period=all`)) + : await ctx.input.api.get(`${root}/costs/data-quality?period=all`); const prerequisite = nativeAccountingPrerequisite(original, quality, retryable); + const faultedWrites = retryable ? await accountingFaults.get(ctx.input.fixtures.company.id)?.failures() : undefined; await ctx.input.evidence("identity-native-accounting-context.json", { runId: run.id, taskId: preflightTask.id, workerId: worker.id, nativeRunActuallyCompleted: true, usageEvidenceExists: prerequisite.savedUsage, actualUnresolvedReasons: prerequisite.unresolvedReasons, requiresRetryableSavedUsage: retryable, - prerequisiteDisposition: prerequisite.disposition, actualRetryControlAvailable: prerequisite.retryControlAvailable, prerequisiteMet: prerequisite.eligible, usageAccountingFabricated: false, setupNativeUsageMustCountInCampaignAccounting: true }); + prerequisiteDisposition: prerequisite.disposition, actualRetryControlAvailable: prerequisite.retryControlAvailable, prerequisiteMet: prerequisite.eligible, usageAccountingFabricated: false, setupNativeUsageMustCountInCampaignAccounting: true, + ...(retryable ? { faultedCostEventWrites: faultedWrites ?? 0 } : {}) }); if (!prerequisite.eligible) throw new Error(`Setup blocker: actual completed native accounting prerequisite is ${prerequisite.disposition}; unresolved reasons=${prerequisite.unresolvedReasons.join(",") || "none"}. Saved retry debt cannot be fabricated`); return { worker, project: p, preflightTask: await ctx.input.api.get(`/api/issues/${preflightTask.id}`), originalRun: original, originalRunId: run.id, originalUsageHash: digest(original.usageJson), originalLogHash: original.logSha256, unresolvedBefore: quality }; } +const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +/** One chat-agent product call: call_api saves its input as a digest, keeping only the operation, ok and status. */ +export interface LeadApiCall { runId: string; operationId: string; digest: string; ok: boolean; status: number } +/** Pair each redacted call_api use with its result by run and item id. Payloads and wording are never read. */ +export function leadApiCalls(events: readonly Record[]): LeadApiCall[] { + const uses = new Map(); const calls: LeadApiCall[] = []; + for (const row of events) { + const envelope = objectRecord(objectRecord(row.payload).prpEvent), item = objectRecord(objectRecord(envelope.payload).item); + const runId = String(row.runId ?? envelope.runId ?? ""); + if (item.type === "tool_use" && item.name === "call_api") { + const input = objectRecord(item.input); + if (typeof input.operationId === "string") uses.set(`${runId}:${String(item.id)}`, { operationId: input.operationId, digest: String(input.apiPayloadSha256 ?? "") }); + continue; + } + if (item.type !== "tool_result") continue; + const use = uses.get(`${runId}:${String(item.tool_use_id)}`), result = objectRecord(item.result); + if (!use || result.apiOperationId !== use.operationId || typeof result.status !== "number") continue; + calls.push({ runId, ...use, ok: result.ok === true, status: result.status }); + } + return calls; +} +/** Agent-readable saved cost projections, each bound by input digest: company reports for this company with no + * date range (a range could exclude the run), the prerequisite task's cost summary including the task itself, + * and the original run's record (which carries its saved accounting state and usage cost status). */ +const COMPANY_COST_SEGMENTS = ["summary", "by-agent", "by-agent-model", "by-provider", "by-biller", "by-project", "data-quality"]; +export interface SavedAccountingTarget { companyId?: string; task?: Pick; originalRunId?: string; + /** Run-record reads the reading run itself cites in its saved result, as `GET /api/heartbeat-runs/`. */ + citedRunReads?: readonly string[] } +export function readSavedAccounting(calls: readonly LeadApiCall[], runId: string, target: SavedAccountingTarget) { + const { companyId, task, originalRunId } = target; + const companyDigests = companyId ? apiReadDigests(COMPANY_COST_SEGMENTS.map(segment => ({ operationId: `GET /api/companies/{companyId}/costs/${segment}`, + pathParams: recordParams("companyId", companyId), queries: [{ period: "all" }, { period: "month" }] }))) : new Set(); + const taskDigests = task ? apiReadDigests([{ operationId: "GET /api/issues/{id}/cost-summary", pathParams: recordParams("id", task.id, task.identifier), queries: [{ excludeRoot: "false" }] }]) : new Set(); + const runDigests = originalRunId ? apiReadDigests([{ operationId: "GET /api/heartbeat-runs/{runId}", pathParams: recordParams("runId", originalRunId) }]) : new Set(); + // A re-read of the run record can carry extra input fields that change its digest. It still counts when the reading + // run's own saved result names exactly that read of the original run; a read of any other run does not. + const originalRunCited = !!originalRunId && (target.citedRunReads ?? []).some(ref => ref === `GET /api/heartbeat-runs/${originalRunId}`); + return calls.some(call => call.runId === runId && call.ok && call.status >= 200 && call.status < 300 + && (companyDigests.has(call.digest) || taskDigests.has(call.digest) + || call.operationId === "GET /api/heartbeat-runs/{runId}" && (runDigests.has(call.digest) || originalRunCited))); +} +/** `GET /api/heartbeat-runs/` reads a run cites as its own verification or evidence. */ +export function citedRunReads(run: { resultJson?: unknown } | undefined) { + const result = objectRecord(objectRecord(run?.resultJson).nativeResult); + const refs = [...(Array.isArray(result.verification) ? result.verification : []).map(row => objectRecord(row).commandOrCheck), + ...(Array.isArray(result.evidence) ? result.evidence : []).map(row => objectRecord(row).ref)]; + return refs.filter((ref): ref is string => typeof ref === "string").map(ref => ref.match(/^GET \/api\/heartbeat-runs\/[0-9a-f-]{36}(?=$|[\s:])/i)?.[0]).filter((ref): ref is string => !!ref); +} +/** A successful agent write to cost, finance, budget or accounting records, or to run/issue execution, would + * fabricate the repair or redo the work. Denied attempts (403) are not writes. */ +export function agentAccountingOrExecutionWrites(calls: readonly LeadApiCall[]) { + return calls.filter(call => call.ok && !call.operationId.startsWith("GET ") + && /cost|finance|budget|accounting|\/heartbeat-runs\/|\/wakeup|\/retry|\/checkout|\/release/.test(call.operationId)); +} +/** A rendered chat link from these replies opens the Costs page, where the retry control lives. */ +/** The Costs page (`//activity/costs`) or its `//costs` redirect. */ +export function isCostsPagePath(pathname: string) { + const parts = pathname.split("/").filter(Boolean); + return parts.length === 3 && parts[1] === "activity" && parts[2] === "costs" || parts.length === 2 && parts[1] === "costs"; +} +export function costsPageLinked(o: { origin: string; renderedLinkEvidence?: Array<{ href: string; commentId?: string }> }, commentIds: readonly string[]) { + return (o.renderedLinkEvidence ?? []).some(link => { + if (!link.commentId || !commentIds.includes(link.commentId)) return false; + try { + const url = new URL(link.href, o.origin); if (url.origin !== new URL(o.origin).origin) return false; + return isCostsPagePath(url.pathname); + } catch { return false; } + }); +} export function gradeAccounting(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-native-accounting-evidence", !!o, "Actual completed native usage, public accounting projections and persisted independent readback are required."); if (!o) return c.result(); const before = o.state.originalRun as HeartbeatRun | undefined; const after = o.state.afterOriginalRun as HeartbeatRun | undefined; @@ -77,28 +166,31 @@ export function gradeAccounting(value: unknown) { const ledger = o.state.retryLedger as CostByAgentModel[] | undefined; const afterQuality = o.state.retryQuality as CostDataQualityReport | undefined; const actualUnresolved = !!afterQuality && ([...afterQuality.runs, ...afterQuality.events].some(row => row.runId === before?.id) || ledger?.some(row => row.estimatedEventCount > 0) === true); - const actualRecorded = !!afterQuality && !actualUnresolved && o.state.retryAccounted === true && !!ledger?.length && ledger.every(row => row.agentId === entity(o.state, "worker").id && row.eventCount > 0 && row.estimatedEventCount === 0); + const actualRecorded = !!afterQuality && !actualUnresolved && !!ledger?.length && ledger.every(row => row.agentId === entity(o.state, "worker").id && row.eventCount > 0 && row.estimatedEventCount === 0); c.add("independent-post-retry-ledger-classification", !!afterQuality && Array.isArray(ledger) && o.state.afterStillUnresolved === actualUnresolved && o.state.confirmedPricedRepair === actualRecorded && (actualUnresolved || actualRecorded), "Actual saved quality records and the sole original worker's ledger, not HTTP success or fixture booleans alone, classify the human retry result."); const finalReply = typeof o.state.finalReply === "string" ? o.state.finalReply : ""; - const initialReply = typeof o.state.initialReply === "string" ? o.state.initialReply : ""; const finalRun = o.runs.find(run => run.id === o.state.finalReplyRunId); c.add("actual-post-retry-human-and-native-reply", typeof o.state.retryHumanFollowup === "string" && o.humanQueries.includes(o.state.retryHumanFollowup) && !!finalRun && finalRun.agentId === o.leadId && finalRun.status === "succeeded" && finalRun.runtimeMode === "native" && !!finalReply.trim(), - "The actual human reports the real retry receipt and independently read accounting outcome, followed by a distinct persisted native chat reply."); - const recordedClaim = /(?:cost|charge|usage|accounting)[^.\n]{0,65}(?:recorded|repaired|resolved|recovered)|(?:recorded|repaired|resolved|recovered)[^.\n]{0,65}(?:cost|charge|usage|accounting)/i; - const unresolvedClaim = /(?:still|remains?|not yet|not fully)[^.\n]{0,65}(?:unresolved|pending|estimated|incomplete|unconfirmed)|(?:unresolved|pending|estimated|incomplete|unconfirmed)[^.\n]{0,65}(?:still|remains?)/i; - c.add("honest-post-retry-accounting-result", o.state.confirmedPricedRepair === true - ? recordedClaim.test(finalReply) && !unresolvedClaim.test(finalReply) && !/\b(?:not|never|wasn.t|isn.t)\s+(?:actually\s+)?(?:recorded|repaired|resolved)/i.test(finalReply) - : o.state.afterStillUnresolved === true && unresolvedClaim.test(finalReply) && !/\b(?:now|fully|successfully)\s+(?:recorded|repaired|resolved|recovered)|\b(?:fixed|repaired|resolved)\s+(?:the|that|its|your)\s+(?:missing\s+)?(?:cost|charge|usage|accounting)/i.test(finalReply), - "The post-action reply distinguishes independently confirmed recorded charges from still-unresolved/estimated accounting; the initial guidance cannot predict the human result."); - c.add("guides-accounting-without-rerun", /cost|accounting|usage/i.test(initialReply) && /retry|record/i.test(initialReply) && /saved|existing|without.*run|not.*rerun|no.*rerun/i.test(initialReply) - && !/\bI(?:'ve| have)?\s+(?:fixed|repaired|recorded|rerun|restarted)\b/i.test(initialReply) - && !/\bI(?:'ve| have)?\s+(?:rerun|restarted)\b/i.test(finalReply), "Initial guidance uses the saved-usage control; neither phase claims a new execution of the original work."); + "The actual human reports what the retry control showed, followed by a distinct persisted native chat reply."); + // The reply's wording is never matched. The person only reports the toast they saw, which reads + // "Costs recorded." even when the recorded usage still has no price, so an honest answer needs the + // saved accounting: the post-retry run must read it through a product cost projection. + const calls = Array.isArray(o.state.leadApiCalls) ? o.state.leadApiCalls as LeadApiCall[] : undefined; + const preflight = o.state.preflightTask as Entity | undefined; + c.add("post-retry-reply-reads-saved-accounting", !!calls && !!finalRun && readSavedAccounting(calls, finalRun.id, { companyId: o.companyId, task: preflight, originalRunId: String(o.state.originalRunId ?? ""), citedRunReads: citedRunReads(finalRun) }), + "After the human retry, the chat run reads the saved cost projections before explaining whether the cost is recorded or still unresolved."); + const initialIds = Array.isArray(o.state.initialReplyCommentIds) ? o.state.initialReplyCommentIds as string[] : []; + c.add("initial-guidance-links-saved-cost-retry", initialIds.length > 0 && costsPageLinked(o, initialIds), + "Before any retry, the chat reply gives the person a link to the Costs page, where the authorized saved-cost retry control is."); + c.add("no-agent-accounting-fabrication-or-rerun", !!calls && agentAccountingOrExecutionWrites(calls).length === 0, + "The chat agent never writes cost, budget or accounting records or starts run/task execution itself; only the human retry records saved usage."); } else c.add("known-accounting-case", false, "No accounting oracle exists for this case."); return c.result(); } const sources = ["chat-catalog-domains/identity-accounting.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/budgets-costs.yaml"]; +const repairSources = [...sources, "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts", "chat-accounting-fault.ts", "chat-archive-storage-fault.ts"]; async function verifyUnresolvedSetup(ctx: ChatCatalogContext, state: Record) { const quality = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/costs/data-quality?period=all`); const c = checks(); const rows = [...quality.runs, ...quality.events].filter(row => row.runId === state.originalRunId); @@ -117,6 +209,12 @@ async function observeAccounting(ctx: ChatCatalogContext, state: Record(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); state.finalReply = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === state.finalReplyRunId).map(row => row.body ?? "").join("\n"); + const calls: LeadApiCall[] = []; + for (const run of (await ctx.allRuns()).filter(row => row.agentId === ctx.input.fixtures.agent.id)) { + const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); + calls.push(...leadApiCalls(events.map(row => ({ ...row, runId: row.runId ?? run.id })))); + } + state.leadApiCalls = calls; } return observeIdentity(ctx, state); } @@ -125,7 +223,7 @@ async function verifyAccountingLink(ctx: ChatCatalogContext, state: Record { const url = new URL(link, ctx.input.api.baseURL); return url.origin === ctx.input.api.baseURL && (ctx.scenario.id.endsWith("spend-breakdown") ? [entity(state, "morganReceiptTask"), entity(state, "rileyReceiptTask")].some(row => [row.id, row.identifier].some(id => id && url.pathname.includes(`/issues/${id}`))) - : url.pathname.includes("/activity/costs") || url.pathname.includes(`/runs/${state.originalRunId}`)); }); + : isCostsPagePath(url.pathname) || url.pathname.includes(`/runs/${state.originalRunId}`)); }); if (!href) throw new Error("Actual supporting cost/task/run link is required"); await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); if (ctx.scenario.id.endsWith("spend-breakdown")) await expect(ctx.input.page.getByText(/Launch imported (?:Morgan|Riley) API usage/, { exact: true }).first()).toBeVisible(); @@ -153,36 +251,45 @@ export const accountingIdentityCases: readonly ChatCatalogCase[] = [ }, gradeAccounting, { authentication: "human-session", oracleVersion: "paperclip.chat-catalog.spend-breakdown.structural.v3", sourceFiles: sources, expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] } }), identityDefinition("budgets-costs.repair-cost-recording", async ctx => { - const state = await nativeUnresolvedContext(ctx, true); - await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The completed native run requiring saved-cost recovery is ${state.originalRunId}, on task ${state.preflightTask.identifier}. Its public accounting quality record is actually retryable and retains saved usage. Only the person may use cost-recording retry; do not start or rerun the task.` }); - return { ...state }; + try { + const state = await nativeUnresolvedContext(ctx, true); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The completed native run requiring saved-cost recovery is ${state.originalRunId}, on task ${state.preflightTask.identifier}. Its public accounting quality record is actually retryable and retains saved usage. Only the person may use cost-recording retry; do not start or rerun the task.` }); + return { ...state }; + } catch (error) { const fault = accountingFaults.get(ctx.input.fixtures.company.id); accountingFaults.delete(ctx.input.fixtures.company.id); await fault?.close(); throw error; } }, gradeAccounting, { - authentication: "human-session", sourceFiles: sources, expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 780_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.repair-cost-recording.structural.v4", sourceFiles: repairSources, + async cleanup(ctx) { const fault = accountingFaults.get(ctx.input.fixtures.company.id); accountingFaults.delete(ctx.input.fixtures.company.id); await fault?.close(); }, expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 780_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] }, async act(ctx, state) { await pollUntil({ label: "accounting guidance source settled", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, accept: rows => rows.length === 2 && rows.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); const initialComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); - state.initialReply = initialComments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body ?? "").join("\n"); + state.initialReplyCommentIds = initialComments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.id); const previousRunIds = new Set((await ctx.allRuns()).map(run => run.id)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; state.agentRetryStatus = await probeAgentPermission(ctx, `${root}/costs/retry-accounting`, "POST", { runId: state.originalRunId }); const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/activity/costs?period=all`, { waitUntil: "domcontentloaded" }); await ctx.input.page.getByRole("button", { name: /run costs? unavailable|billing types? unknown/i }).first().click(); const row = ctx.input.page.getByRole("listitem").filter({ has: ctx.input.page.locator(`a[href*="/runs/${state.originalRunId}"]`) }).filter({ has: ctx.input.page.getByRole("button", { name: "Retry cost recording", exact: true }) }).last(); const response = ctx.input.page.waitForResponse(result => result.request().method() === "POST" && new URL(result.url()).pathname === `${root}/costs/retry-accounting` && result.request().postDataJSON()?.runId === state.originalRunId); + await expect(row.getByRole("button", { name: "Retry cost recording", exact: true })).toBeVisible(); + // The transient write failure clears, then the person retries. Automatic recovery may record it first; + // the saved accounting read afterwards classifies either result. + const fault = accountingFaults.get(ctx.input.fixtures.company.id); if (!fault) throw new Error("Accounting fault was not armed for this attempt"); + state.faultedCostEventWritesBeforeRetry = await fault.failures(); await fault.release(); await row.getByRole("button", { name: "Retry cost recording", exact: true }).click(); const receipt = await response; state.humanRetryHttpStatus = receipt.status(); const result = await receipt.json() as { accounted: boolean }; state.retryAccounted = result.accounted; await expect(ctx.input.page.getByText(result.accounted ? "Costs recorded." : "No new cost data is available.", { exact: true })).toBeVisible(); state.retryReceiptVisible = true; await expect(ctx.input.page.getByText("The agent was not rerun.", { exact: true })).toBeVisible(); const after = await ctx.input.api.get(`${root}/costs/data-quality?period=all`); state.retryQuality = after; state.afterStillUnresolved = [...after.runs, ...after.events].some(row => row.runId === state.originalRunId); const ledger = (await ctx.input.api.get(`${root}/costs/by-agent-model?period=all`)).filter(row => row.agentId === entity(state, "worker").id); state.retryLedger = ledger; - state.confirmedPricedRepair = result.accounted === true && state.afterStillUnresolved === false && ledger.length > 0 && ledger.every(row => row.eventCount > 0 && row.estimatedEventCount === 0); + state.confirmedPricedRepair = state.afterStillUnresolved === false && ledger.length > 0 && ledger.every(row => row.eventCount > 0 && row.estimatedEventCount === 0); state.afterStillUnresolved = state.afterStillUnresolved === true || ledger.some(row => row.estimatedEventCount > 0); if (!state.confirmedPricedRepair && !state.afterStillUnresolved) throw new Error("Retry returned no independently classifiable recorded or unresolved accounting outcome"); await ctx.input.evidence("identity-saved-usage-retry.json", { runId: state.originalRunId, actualHumanHttpStatus: receipt.status(), accounted: result.accounted, confirmedPricedRepair: state.confirmedPricedRepair, afterStillUnresolved: state.afterStillUnresolved, originalRunNeverRerun: (await ctx.allRuns()).filter(run => run.agentId === entity(state, "worker").id).length === 1 }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); - const followup = `I clicked Retry cost recording for run ${state.originalRunId}. The actual HTTP ${receipt.status()} receipt says accounted=${result.accounted}. Fresh saved accounting shows ${state.afterStillUnresolved ? "remaining unresolved or estimated accounting" : "no remaining unresolved accounting"}; recorded ledger: ${JSON.stringify(ledger.map(row => ({ billingType: row.billingType, costCents: row.costCents, eventCount: row.eventCount, estimatedEventCount: row.estimatedEventCount })))}. What does this actual result mean for the missing cost? Do not run the original task again.`; + // The person reports only what the page showed them; the harness's own accounting readback stays out of chat. + const followup = `I clicked Retry cost recording for that run. The page said "${result.accounted ? "Costs recorded." : "No new cost data is available."} The agent was not rerun." Is the missing cost fixed now?`; state.retryHumanFollowup = followup; await sendChatMessage(ctx.input.page, followup); const finalRuns = await pollUntil({ label: "actual post-retry native accounting explanation", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, accept: rows => rows.length === 3 && rows.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" });