diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 1cbca97545e..fca10c34321 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -822,13 +822,12 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } it("does not reap active adapter executions started by another heartbeat service instance", async () => { - let releaseAdapter: (() => void) | null = null; + let releaseAdapter!: () => void; + const adapterRelease = new Promise(resolve => { releaseAdapter = resolve; }); const adapterStarted = new Promise((resolve) => { mockAdapterExecute.mockImplementationOnce(async () => { resolve(); - await new Promise((release) => { - releaseAdapter = release; - }); + await adapterRelease; return { exitCode: 0, signal: null, @@ -852,52 +851,55 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const executorHeartbeat = heartbeatService(db); const reaperHeartbeat = heartbeatService(db); - await executorHeartbeat.resumeQueuedRuns(); - await Promise.race([ - adapterStarted, - new Promise((_, reject) => { - setTimeout( - () => - reject( - new Error("Timed out waiting for adapter execution to start"), - ), - 3_000, - ); - }), - ]); + try { + await executorHeartbeat.resumeQueuedRuns(); + await Promise.race([ + adapterStarted, + new Promise((_, reject) => { + setTimeout( + () => + reject( + new Error("Timed out waiting for adapter execution to start"), + ), + 3_000, + ); + }), + ]); - await db - .update(heartbeatRuns) - .set({ - updatedAt: new Date("2026-03-19T00:00:00.000Z"), - }) - .where(eq(heartbeatRuns.id, runId)); + await db + .update(heartbeatRuns) + .set({ + updatedAt: new Date("2026-03-19T00:00:00.000Z"), + }) + .where(eq(heartbeatRuns.id, runId)); - const result = await reaperHeartbeat.reapOrphanedRuns({ - staleThresholdMs: 1, - }); - expect(result).toEqual({ reaped: 0, runIds: [] }); + const result = await reaperHeartbeat.reapOrphanedRuns({ + staleThresholdMs: 1, + }); + expect(result).toEqual({ reaped: 0, runIds: [] }); - const activeRun = await reaperHeartbeat.getRun(runId); - expect(activeRun?.status).toBe("running"); - expect(activeRun?.errorCode).toBeNull(); + const activeRun = await reaperHeartbeat.getRun(runId); + expect(activeRun?.status).toBe("running"); + expect(activeRun?.errorCode).toBeNull(); - const wakeup = await db - .select() - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, wakeupRequestId)) - .then((rows) => rows[0] ?? null); - expect(wakeup?.status).toBe("claimed"); + const wakeup = await db + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, wakeupRequestId)) + .then((rows) => rows[0] ?? null); + expect(wakeup?.status).toBe("claimed"); - if (!releaseAdapter) - throw new Error("Adapter release handle was not captured"); - releaseAdapter(); - const settledRun = await waitForRunToSettle( - executorHeartbeat, - runId, - 5_000, - ); - expect(settledRun?.status).toBe("succeeded"); + releaseAdapter(); + const settledRun = await waitForRunToSettle( + executorHeartbeat, + runId, + 5_000, + ); + expect(settledRun?.status).toBe("succeeded"); + } finally { + releaseAdapter(); + await executorHeartbeat.drainActiveRunExecutions(); + } }); async function seedStrandedIssueFixture(input: { @@ -3410,7 +3412,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); it("persists codex_local spawn identity before hot restart and never loses the live run for missing metadata", async () => { - let releaseAdapter: (() => void) | null = null; + let releaseAdapter!: () => void; + const adapterRelease = new Promise(resolve => { releaseAdapter = resolve; }); let spawnedPid: number | null = null; const adapterStarted = new Promise((resolve) => { mockAdapterExecute.mockImplementationOnce(async (rawInput?: unknown) => { @@ -3432,9 +3435,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { startedAt: new Date("2026-07-30T07:00:00.000Z").toISOString(), }); resolve(); - await new Promise((release) => { - releaseAdapter = release; - }); + await adapterRelease; return { exitCode: 0, signal: null, @@ -3459,87 +3460,90 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { includeIssue: false, }); const heartbeat = heartbeatService(db); - await heartbeat.resumeQueuedRuns(); - await Promise.race([ - adapterStarted, - new Promise((_, reject) => { - setTimeout( - () => - reject( - new Error("Timed out waiting for codex_local spawn identity"), - ), - 3_000, - ); - }), - ]); - - const running = await waitForValue(async () => - db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => { - const row = rows[0] ?? null; - return row?.status === "running" && row.processPid ? row : null; + try { + await heartbeat.resumeQueuedRuns(); + await Promise.race([ + adapterStarted, + new Promise((_, reject) => { + setTimeout( + () => + reject( + new Error("Timed out waiting for codex_local spawn identity"), + ), + 3_000, + ); }), - ); - const observedProcessStartedAt = await readProcessStartedAt(spawnedPid!); - expect(observedProcessStartedAt).not.toBeNull(); - expect(running).toMatchObject({ - id: runId, - status: "running", - processPid: spawnedPid, - processGroupId: null, - processStartedAt: new Date(observedProcessStartedAt!), - }); + ]); - await withTempPaperclipHome(async (home) => { - await writeHotRestartIntent({ - previousServerPid: process.pid, - previousServerVersion: "old-version", - requestedAt: new Date("2026-07-30T07:01:00.000Z"), - }); - await heartbeat.prepareHotRestartShutdown( - "SIGTERM", - new Date("2026-07-30T07:02:00.000Z"), - ); - - const adoption = await heartbeat.reconcileHotRestartAdoption( - new Date("2026-07-30T07:03:00.000Z"), - ); - expect(adoption).toMatchObject({ - mode: "reported", - adoptedRunIds: [runId], - finalizedWhileDownRunIds: [], - lostRunIds: [], - }); - const report = JSON.parse( - await fs.readFile(resolveHotRestartReportPath(home), "utf8"), - ) as { runs?: Array> }; - expect(report.runs).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - runId, - classification: "adopted", - reason: "process_pid_alive", + const running = await waitForValue(async () => + db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => { + const row = rows[0] ?? null; + return row?.status === "running" && row.processPid ? row : null; }), - ]), ); - expect(report.runs).not.toEqual( - expect.arrayContaining([ - expect.objectContaining({ - runId, - reason: "missing_process_metadata", - }), - ]), - ); - }); + const observedProcessStartedAt = await readProcessStartedAt(spawnedPid!); + expect(observedProcessStartedAt).not.toBeNull(); + expect(running).toMatchObject({ + id: runId, + status: "running", + processPid: spawnedPid, + processGroupId: null, + processStartedAt: new Date(observedProcessStartedAt!), + }); - if (!releaseAdapter) - throw new Error("Adapter release handle was not captured"); - releaseAdapter(); - const settled = await waitForRunToSettle(heartbeat, runId, 5_000); - expect(settled?.status).toBe("succeeded"); + await withTempPaperclipHome(async (home) => { + await writeHotRestartIntent({ + previousServerPid: process.pid, + previousServerVersion: "old-version", + requestedAt: new Date("2026-07-30T07:01:00.000Z"), + }); + await heartbeat.prepareHotRestartShutdown( + "SIGTERM", + new Date("2026-07-30T07:02:00.000Z"), + ); + + const adoption = await heartbeat.reconcileHotRestartAdoption( + new Date("2026-07-30T07:03:00.000Z"), + ); + expect(adoption).toMatchObject({ + mode: "reported", + adoptedRunIds: [runId], + finalizedWhileDownRunIds: [], + lostRunIds: [], + }); + const report = JSON.parse( + await fs.readFile(resolveHotRestartReportPath(home), "utf8"), + ) as { runs?: Array> }; + expect(report.runs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + runId, + classification: "adopted", + reason: "process_pid_alive", + }), + ]), + ); + expect(report.runs).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + runId, + reason: "missing_process_metadata", + }), + ]), + ); + }); + + releaseAdapter(); + const settled = await waitForRunToSettle(heartbeat, runId, 5_000); + expect(settled?.status).toBe("succeeded"); + } finally { + releaseAdapter(); + await heartbeat.drainActiveRunExecutions(); + } }); it("reports adopted hot-restart runs before startup reap can mark them process_lost", async () => { diff --git a/ui/src/components/BudgetPolicyCard.test.tsx b/ui/src/components/BudgetPolicyCard.test.tsx index 734a298dc61..b782842dfb9 100644 --- a/ui/src/components/BudgetPolicyCard.test.tsx +++ b/ui/src/components/BudgetPolicyCard.test.tsx @@ -29,11 +29,22 @@ describe("exact reservation editing", () => { const save = Array.from(container.querySelectorAll("button")).find(button => button.textContent === "Update reservation")!; expect(input.value).toBe("90071992547409.920000001"); expect(save.disabled).toBe(true); + for (const invalid of ["-1", "", "not an amount"]) { + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(input, invalid); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + expect(save.disabled).toBe(true); + expect(input.getAttribute("aria-invalid")).toBe("true"); + expect(container.querySelector('[role="alert"]')?.textContent).toBe("Enter a valid amount of zero or more."); + expect(input.getAttribute("aria-describedby")).toBe(container.querySelector('[role="alert"]')?.id); + } await act(async () => { Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(input, "0.000000001"); input.dispatchEvent(new Event("input", { bubbles: true })); }); await act(async () => save.click()); expect(onReservationChange).toHaveBeenCalledWith("0.0000001"); + expect(container.querySelector('[role="alert"]')).toBeNull(); }); }); diff --git a/ui/src/components/BudgetPolicyCard.tsx b/ui/src/components/BudgetPolicyCard.tsx index 267a25b776c..6f5afd2b400 100644 --- a/ui/src/components/BudgetPolicyCard.tsx +++ b/ui/src/components/BudgetPolicyCard.tsx @@ -1,5 +1,5 @@ import { centsToUsd, usdToCents } from "@paperclipai/shared"; -import { useEffect, useState } from "react"; +import { useEffect, useId, useState } from "react"; import type { BudgetPolicySummary } from "@paperclipai/shared"; import { AlertTriangle, PauseCircle, ShieldAlert, Wallet } from "lucide-react"; import { cn, formatCents } from "../lib/utils"; @@ -53,13 +53,16 @@ export function BudgetPolicyCard({ }, [summary.amount]); const [reservation, setReservation] = useState(centsToUsd(summary.reservationCents ?? 0)); + const reservationErrorId = useId(); useEffect(() => setReservation(centsToUsd(summary.reservationCents ?? 0)), [summary.reservationCents]); let reservationCents: string | null = null; try { if (!reservation.trim().startsWith("-")) reservationCents = usdToCents(reservation.trim()); } catch { /* Shown as invalid input below. */ } const reservationSection = onReservationChange ?
+ {reservationCents === null && }

An estimate held before each run starts. Zero disables the estimate. Actual provider charges may exceed it.

diff --git a/ui/src/pages/Costs.quota.test.tsx b/ui/src/pages/Costs.quota.test.tsx index de21cf6101f..6c76e3582a9 100644 --- a/ui/src/pages/Costs.quota.test.tsx +++ b/ui/src/pages/Costs.quota.test.tsx @@ -34,6 +34,18 @@ describe("Costs account quota polling", () => { }); afterEach(() => { act(() => root.unmount()); client.clear(); container.remove(); vi.unstubAllGlobals(); }); const flush = async () => { await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)); }); }; + it.each(["providers", "billers"] as const)("marks retained %s reports stale when their refresh fails", async (tab) => { + api.byBiller.mockResolvedValue([{ biller: "openai", costCents: 125, eventCount: 1, providerCount: 1, inputTokens: 1, outputTokens: 1, cachedInputTokens: 0 }]); + await act(async () => root.render()); + await flush(); + expect(container.textContent).not.toContain("Showing the last loaded data"); + const report = tab === "providers" ? api.byProvider : api.byBiller; + report.mockRejectedValue(new Error("Private report diagnostic")); + await act(async () => { await client.invalidateQueries(); }); await flush(); + expect(container.textContent).toContain("Showing the last loaded data"); + expect(container.textContent).toContain("OpenAI"); + expect(container.textContent).not.toContain("Private report diagnostic"); + }); it("keeps quota readings with a safe notice when the endpoint rejects", async () => { await act(async () => root.render()); await flush(); diff --git a/ui/src/pages/Costs.tsx b/ui/src/pages/Costs.tsx index 58176cfc65f..be222961a17 100644 --- a/ui/src/pages/Costs.tsx +++ b/ui/src/pages/Costs.tsx @@ -323,7 +323,7 @@ export function Costs({ return map; }, [spendData?.byAgentModel]); - const { data: providerData } = useQuery({ + const { data: providerData, error: providerError } = useQuery({ queryKey: queryKeys.usageByProvider(companyId, reportQueryFrom, reportQueryTo), queryFn: () => costsApi.byProvider(companyId, from || undefined, to || undefined), enabled: !!selectedCompanyId && customReady && (mainTab === "providers" || mainTab === "billers"), @@ -331,7 +331,7 @@ export function Costs({ staleTime: 10_000, }); - const { data: billerData } = useQuery({ + const { data: billerData, error: billerError } = useQuery({ queryKey: queryKeys.usageByBiller(companyId, reportQueryFrom, reportQueryTo), queryFn: () => costsApi.byBiller(companyId, from || undefined, to || undefined), enabled: !!selectedCompanyId && customReady && mainTab === "billers", @@ -339,7 +339,7 @@ export function Costs({ staleTime: 10_000, }); - const { data: weekData } = useQuery({ + const { data: weekData, error: weekError } = useQuery({ queryKey: queryKeys.usageByProvider(companyId, weekRange.from, weekRange.to), queryFn: () => costsApi.byProvider(companyId, weekRange.from, weekRange.to), enabled: !!selectedCompanyId && (mainTab === "providers" || mainTab === "billers"), @@ -347,7 +347,7 @@ export function Costs({ staleTime: 10_000, }); - const { data: weekBillerData } = useQuery({ + const { data: weekBillerData, error: weekBillerError } = useQuery({ queryKey: queryKeys.usageByBiller(companyId, weekRange.from, weekRange.to), queryFn: () => costsApi.byBiller(companyId, weekRange.from, weekRange.to), enabled: !!selectedCompanyId && mainTab === "billers", @@ -355,7 +355,7 @@ export function Costs({ staleTime: 10_000, }); - const { data: windowData } = useQuery({ + const { data: windowData, error: windowError } = useQuery({ queryKey: queryKeys.usageWindowSpend(companyId), queryFn: () => costsApi.windowSpend(companyId), enabled: !!selectedCompanyId && mainTab === "providers", @@ -677,7 +677,7 @@ export function Costs({
) : null} - {((spendData && spendError) || (financeData && financeError) || (budgetData && budgetError)) ? ( + {((spendData && spendError) || (financeData && financeError) || (budgetData && budgetError) || providerError || billerError || weekError || weekBillerError || windowError) ? (

Showing the last loaded data. Updates will resume automatically.

diff --git a/ui/storybook/fixtures/costsFinance.test.ts b/ui/storybook/fixtures/costsFinance.test.ts index 7f0422e28ea..3b8f672b612 100644 --- a/ui/storybook/fixtures/costsFinance.test.ts +++ b/ui/storybook/fixtures/costsFinance.test.ts @@ -44,6 +44,24 @@ describe("Costs story finance actions", () => { } }); + it("accepts equivalent invoice retries but rejects changed normalized content without altering charges", async () => { + const fixture = createCostsFinanceFixtures("preview"); + const invoice = { biller: "Example", externalId: "immutable-invoice", currency: "USD", lines: [ + { externalId: "fee-b", kind: "fee", amountCents: "50", occurredAt: charge.occurredAt }, + { externalId: "fee-a", kind: "fee", amountCents: "25", occurredAt: charge.occurredAt }, + ] }; + const saved = await (await fixture("accounting/invoices", request("accounting/invoices", invoice)))!.json(); + const equivalent = { ...invoice, lines: invoice.lines.slice().reverse().map(line => ({ ...line, amountCents: `${line.amountCents}.0000000` })) }; + expect(await (await fixture("accounting/invoices", request("accounting/invoices", equivalent)))!.json()).toEqual(saved); + for (const changed of [ + { ...invoice, currency: "EUR" }, + { ...invoice, lines: [{ ...invoice.lines[0], amountCents: "51" }, invoice.lines[1]] }, + { ...invoice, lines: [{ ...invoice.lines[0], occurredAt: "2000-01-01T00:00:00Z" }, invoice.lines[1]] }, + ]) expect((await fixture("accounting/invoices", request("accounting/invoices", changed)))!.status).toBe(409); + expect(await (await fixture("costs/finance-summary", request("costs/finance-summary")))!.json()).toMatchObject({ debitCents: 75, eventCount: 2 }); + expect(await (await fixture("accounting/invoices", request("accounting/invoices")))!.json()).toHaveLength(1); + }); + it("keeps timestamp bounds exact while date-only bounds include the full day", async () => { const fixture = createCostsFinanceFixtures("preview"); for (const [hour, id] of [["12", "at-boundary"], ["18", "after-boundary"]]) { diff --git a/ui/storybook/fixtures/costsFinance.ts b/ui/storybook/fixtures/costsFinance.ts index 8e89323edb6..a52c1aeaa9e 100644 --- a/ui/storybook/fixtures/costsFinance.ts +++ b/ui/storybook/fixtures/costsFinance.ts @@ -4,6 +4,7 @@ import { normalizeCents, createFinanceEventSchema, importBillingInvoiceSchema, i export function createCostsFinanceFixtures(companyId: string) { const events: FinanceEvent[] = []; const reports: BillingReconciliation[] = []; + const invoiceContents = new Map(); const createdAt = new Date(); let sequence = 0; const id = () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`; @@ -54,11 +55,18 @@ export function createCostsFinanceFixtures(companyId: string) { if (resource === "accounting/invoices") { if (request.method === "GET") return Response.json(reports.map(report => report.invoice)); const input = importBillingInvoiceSchema.parse(await request.json()); + input.lines.sort((a, b) => a.externalId.localeCompare(b.externalId)); + const canonical = (value: unknown): unknown => value instanceof Date ? value.toISOString() + : Array.isArray(value) ? value.map(canonical) + : value && typeof value === "object" ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, canonical(item)])) : value; + const contents = JSON.stringify(canonical(input)); let report = reports.find(row => row.invoice.biller === input.biller && row.invoice.externalId === input.externalId); + if (report && invoiceContents.get(report.invoice.id) !== contents) return Response.json({ error: "Invoice identifier already has different contents" }, { status: 409 }); if (!report) { const invoice = { id: id(), companyId, biller: input.biller, externalId: input.externalId, currency: input.currency, createdAt: createdAt.toISOString() }; report = { invoice, lines: input.lines.map(line => ({ id: id(), externalId: line.externalId, kind: line.kind, amountCents: line.amountCents, status: "unmatched", matchedEventId: null, recordedCents: null, differenceCents: null })) }; reports.push(report); + invoiceContents.set(invoice.id, contents); for (const line of input.lines) record({ idempotencyKey: `preview-invoice:${invoice.id}:${line.externalId}`, externalInvoiceId: invoice.externalId, biller: input.biller, currency: input.currency, amountCents: line.amountCents, occurredAt: line.occurredAt, direction: line.kind === "credit" ? "credit" : "debit", eventKind: line.kind === "credit" ? "credit_refund" : line.kind === "fee" ? "platform_fee" : "inference_charge", description: `Preview invoice ${input.externalId}` }); } return Response.json(report.invoice);