diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index b87e5115b9..8c645911b0 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -616,16 +616,20 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies - # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is - # stale for the merge tree. Resolve it inline instead of waiting on a - # policy-job artifact: that dependency put the policy job's queue and - # runtime (~60s) on every lane's critical path, and policy still - # validates resolution as a required check in parallel. + id: install_dependencies + env: + ACCOUNTING_GATE: ${{ matrix.group == 'general-server-without-chat' && matrix.shard_index == 0 }} + # Coverage is executable evidence: its lane must use the reviewed lock. + # The repository bot refreshes stale locks; never resolve its provider here. run: | - if ! pnpm install --frozen-lockfile; then - echo '::notice title=Lockfile::checked-in lockfile is stale for this merge tree; resolving it inline' - pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile + if [ "$ACCOUNTING_GATE" = "true" ]; then pnpm install --frozen-lockfile + else + if ! pnpm install --frozen-lockfile; then + echo '::notice title=Lockfile::checked-in lockfile is stale for this merge tree; resolving it inline' + pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile + pnpm install --frozen-lockfile + fi fi - name: Run grouped general test suites @@ -637,6 +641,22 @@ jobs: pnpm test:run:general -- --group '${{ matrix.group }}' fi + - name: Verify accounting coverage and crash recovery + if: ${{ !cancelled() && steps.install_dependencies.outcome == 'success' && matrix.group == 'general-server-without-chat' && matrix.shard_index == 0 }} + run: | + node --test scripts/test-accounting-mutations.test.mjs scripts/verify-accounting-test-results.test.mjs + pnpm --filter @paperclipai/paperclip-runner build:typescript + pnpm test:accounting + + - name: Upload accounting coverage + if: ${{ !cancelled() && steps.install_dependencies.outcome == 'success' && matrix.group == 'general-server-without-chat' && matrix.shard_index == 0 }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: accounting-coverage + path: coverage/accounting/ + if-no-files-found: error + retention-days: 14 + docker_context_integrity: name: Docker context integrity needs: gate diff --git a/doc/COST-ACCOUNTING.md b/doc/COST-ACCOUNTING.md new file mode 100644 index 0000000000..f824850b22 --- /dev/null +++ b/doc/COST-ACCOUNTING.md @@ -0,0 +1,272 @@ +# Cost accounting and budget enforcement + +This document describes the implementation after the September 2026 reliability audit. The execution ledger and the finance ledger serve different purposes: `cost_events` records USD usage attributed to agent work; `finance_events` records billing charges and credits in their original currencies. Provider invoices can be imported explicitly for comparison and reviewed corrections. Operators can also fetch OpenAI and Anthropic organization cost reports using a company-owned admin credential. Provider reports, invoices, and run estimates remain separate; Paperclip does not guess exchange rates. + +## UI scope + +The October 5 scope review preserves the existing Finance tab, headline totals, ledger summary, biller/kind breakdowns, recent events, API/CLI ingestion, and automatic Browser Use reporting. Recent financial events appear on the Finance tab only, following the October 6 Overview simplification. Cost accuracy, estimate labels, incomplete-spend warnings, user/project attribution, and stable background refresh remain in scope. + +The new charge-entry/provider-import dialog and Accounting health panel (including invoice review, corrections, inspection, and repair controls) are deferred from the board UI. They are not mounted or polled by Costs or Budgets. Their server services and authenticated operator API/CLI commands remain available. Existing Finance reports still load and refresh, including for organizations with no events; no existing Finance functionality is gated on adoption or hidden behind a feature flag. The new reservation and unknown-price budget options are available under **Advanced settings**, collapsed by default; pending/unpriced warnings remain visible. + +## From a run to a report + +1. Before provider dispatch, the heartbeat stores issue/project/billing-code attribution and marks accounting pending. Adapters normalize usage into input tokens (excluding cache reads, including cache writes), cached input tokens, output tokens, billing identity, and an optional USD price. Usage is per run unless the adapter explicitly declares session-cumulative counters. A missing price is distinct from a reported zero. Native runs require a fresh, complete per-run usage report for the terminal turn; attachment snapshots and partial reports cannot finalize a charge. The Codex runner carries report completeness separately from its accumulated counters so retained older fields cannot fill gaps in a newer partial report. +2. Eligible direct OpenAI API receipts receive a versioned token-price estimate when the provider supplies no dollars (see below). Native execution and supported CLI/ACPX adapters checkpoint usage before workspace and issue finalization. Native terminal events persist fresh same-turn usage before the runtime commits its result; replayed events rebuild the receipt after a stop between event and receipt persistence. Each checkpoint is versioned and fsynced to a private instance spool before its database write; database outages leave the file for replay. A failed checkpoint capture persists a sticky incomplete-accounting marker before its error reaches the adapter; heartbeat retries that write before failure finalization. Older complete attempts, spool replay, and recorder replacement cannot clear the marker or settle the reservation. Missing evidence requires recovery rather than treating an earlier zero-cost attempt as the whole run. The heartbeat stores the final adapter receipt on `heartbeat_runs`, including partial usage returned for failed or timed-out execution. `subscription_included` usage contributes tokens but no incremental monetary charge. An explicit cache-adjusted price takes precedence over the nominal price. +3. `accountRunCost` acquires the company accounting lock and then locks the run. In one PostgreSQL transaction it inserts idempotent cost receipts, updates monthly spend projections, evaluates budgets, increments lifetime runtime totals, settles any reservation, and acknowledges run accounting. The original reported amount is preserved; a reviewed correction appends an audit record and changes only the effective valuation. Every new run receipt has a company-scoped idempotency key. Duplicate delivery cannot increment totals again. Complete per-model receipts are split only when both their tokens and prices reconcile to the run total; otherwise the aggregate receipt is retained. +4. Startup and periodic recovery scan terminal pending runs, independently of heartbeat scheduling being enabled. Failed writes roll back, leave the marker pending, and retry. A bounded batch rotates failed attempts so one bad receipt cannot starve other pending runs. Dispatched runs that have not produced a final receipt remain pending, including when cancellation marks a run terminal before its provider stops. A late final receipt can then be recorded exactly once. Recovery does not invent a zero-valued receipt or price. Proven pre-provider failures are acknowledged without creating a charge. +5. Reports aggregate `cost_events`. Company and agent monthly counters and runtime lifetime counters are projections. New receipts use the attribution captured for the run. Deleted issue/project targets become unallocated; foreign-company references are rejected. Legacy project attribution is inferred only when run activity identifies one project. The unallocated bucket is included so project totals conserve company spend. + +The Overview always shows **By user** below **By agent**, including single-user companies. The synthetic `local-board` (Board) principal is excluded; any historical charges attached to it remain in **Unattributed** so spend is not lost. `GET /api/companies/:companyId/costs/by-user` uses the same authorization and date filters as the other cost reports. Attribution comes from the run's recorded `responsible_user_id`, not the current issue owner. Active users with no spend appear with zero totals; former members with spend remain visible. Charges with no valid company-scoped user attribution appear under **Unattributed**, preserving the company total. Runs are counted distinctly even when they have multiple receipts. Estimates and unpriced charges are labeled. + +Amounts remain denominated in **cents**, including fractional cents. Ledger amounts and spend projections use PostgreSQL `numeric(24,7)`, preserving nanodollar precision instead of rounding every run to a whole cent. Arithmetic uses integer nanodollars, with one half-away-from-zero rounding step at the storage boundary. Monetary inputs accept decimal strings; use strings when exact representation matters. Unsafe large numeric inputs are rejected. Existing JSON number fields remain for compatibility, alongside exact decimal fields such as `costCentsExact`, `spendCentsExact`, `amountCentsExact`, `netCentsExact`, and budget `observedAmountExact`. Display rounding never changes storage or admission decisions. Budget limits remain whole cents. + +## Ingestion and retries + +`POST /api/companies/:companyId/cost-events` accepts an optional `idempotencyKey` (1–200 characters). Retrying the same normalized receipt with the same key returns its existing event. Reusing the key with different content returns `409`. Keys are scoped to a company; callers must reuse the original `occurredAt`, not regenerate it on retry. Without a key, each POST intentionally creates a new event. Automatic run accounting always supplies keys. + +All linked agent, run, issue, project, and goal references must belong to the reporting company, and a linked run must belong to the reported agent. Non-finite/negative costs and negative/fractional token counts are rejected. The receipt, projections, budget incidents, approvals, and local activity-log rows commit together. Live activity publication and provider cancellation happen after commit; a notification or cancellation transport failure cannot undo a recorded charge. + +Finance ingestion uses the same company-scoped key/conflict semantics. Credits remain separate nonnegative events with `direction: credit`. The top-level finance summary is explicitly USD; `currencies` contains independent totals for every recorded currency. Biller/kind groups retain currency. No exchange rate is guessed and unlike currencies are never added into one monetary total. + +## Budgets and incomplete accounting + +Company, agent, and project policies all apply. Monthly windows use UTC calendar months; lifetime policies coexist with monthly policies. A zero or inactive limit does not enforce a stop. Raising a saved zero limit activates enforcement; editing a disabled positive limit preserves its disabled state unless activation is explicitly requested. Generic company/agent budget updates synchronize the policy in the same transaction as the legacy budget field. + +Active hard-stop policies block admission when recorded spend reaches the limit, when a terminal run still awaits accounting, or when usage lacks a reliable price. `unpricedUsagePolicy: block` is the default. An operator can explicitly choose `allow` to permit work despite missing prices; reports continue to identify incomplete pricing. This choice does not bypass pending accounting transactions. Subscription-included usage does not count as a monetary pricing gap. + +At a hard stop the budget service pauses the scope, creates one open incident/approval for the policy window, and persists a cancellation delivery version. Cancellation is retried after failures and may be delivered more than once. It rechecks the policy and limits cancellation to work that existed at the check, so a delayed delivery does not cancel newly admitted work after a budget grant. Provider shutdown is best effort. + +Raising a limit must exceed **current** observed spend. Resuming also requires all other hard-stop policies on that scope to permit work. Calendar rollover or disabling a policy releases budget-owned pauses only; manual pauses, terminated agents, and archived companies are preserved. Choosing to keep a scope paused dismisses the incident without generating a fresh approval on each admission check. A subsequent limit increase followed by another threshold crossing creates a new incident. + +A legacy budget pause with no policy remains blocked until a policy or explicit operator resume resolves it. Manual agent pauses keep the normal agent admission error. Accounting activity is excluded from task-progress evidence, so recording a receipt cannot suppress plan-only recovery. + +Policies optionally configure `reservationCents` (default zero), also editable as “Reserve per run (USD)” under **Advanced settings** on the Costs budget cards. Before dispatch, Paperclip locks the company, checks all relevant policies, and reserves the maximum configured estimate against company, agent, and project capacity. Concurrent dispatches cannot claim the same capacity. Zero disables the estimate; a zero-valued reservation still fences duplicate dispatch of the same run. Existing reservations retain their original amount after policy edits and carry across UTC month boundaries until settled. A native recovery owner may reuse its hold only after current budget and pause checks; its own in-progress accounting does not exempt it from other blockers. Daily heartbeat cost caps also compare exact stored cents before admitting work. + +A final accounting transaction settles the reservation against the actual charge. Positive pre-provider failure evidence releases it without a charge. Time passing, a missing process, cancellation intent, or a policy change alone does not prove provider work has stopped and never releases a held reservation. A run permanently missing its final receipt requires evidence recovery; the system has no “assume zero” button. + +Reservations are estimates, not provider-enforced spending caps. Delayed receipts and in-progress calls can exceed the estimate. Provider spending controls are needed for an external invoice ceiling. + +## Dates, completeness, and UI + +Cost and finance endpoints default to the current UTC month through now. Explicit `from`/`to` bounds remain inclusive; `period=all` requests all time. Reversed or malformed bounds are rejected. Date-filterable CLI cost/finance reports expose `--from`, `--to`, and `--all-time`. Operational accounting, budget, live quota and rolling-window commands reject these flags because their endpoints do not filter by arbitrary dates. The board sends explicit ranges, and month/year-to-date starts use UTC. Rolling spend excludes future events. The dashboard compares spend with the monthly cap only for Month to Date; historical ranges show the monthly limit without a utilization percentage. Provider report imports require explicit `from` and `to` dates in their payload, with an exclusive end date. To import yesterday, use its UTC midnight as `from` and today’s UTC midnight as `to`. + +Cost summaries expose `eventCount`, `estimatedEventCount`, `unpricedEventCount`, `pendingRunCount`, and `pricingComplete`. Estimated charges count toward budgets, while the UI distinguishes estimates from provider-reported dollars. Agent totals and expanded model rows show **Estimated** when every charge in that group is estimated, or **Partially estimated** for a mix. Groups with no estimates have no estimate badge. These labels follow the selected date range and use ledger-event counts, not run counts; they do not infer pricing status from the provider or model name. `pricingComplete` means no missing prices or pending receipts; it does not mean invoice reconciliation has occurred. The Costs page warns when totals are incomplete. Provider cards include cached tokens and calculate subscription shares using each token once. Reports normalize historical unqualified OpenAI events without a receipt fingerprint from inclusive input to exclusive input before grouping (provider-qualified `openai/...` models from Pi/OpenCode already use exclusive input and are preserved), so mixed historical/new totals count cache reads once across agents, models, providers, billers, projects, issues, and rolling windows. This read-time conversion preserves original ledger rows and charges; legacy Anthropic input already excludes cache reads. Provider spend is compared with the real company budget, not a fabricated proportional allocation. Run displays prefer cache-adjusted prices and preserve explicit zeroes. Session rotation uses an explicit layout marker on new raw usage snapshots; unmarked historical snapshots retain their original input threshold calculation so cache hits are not counted twice after upgrade. + +Provider pace warnings project month-to-date spend across the UTC month and compare it with the same company cap displayed by the bar. Run-history totals preserve separate cached input for legacy Claude/Gemini snapshots and provider-qualified OpenCode/Pi model snapshots, while retaining inclusive legacy Codex totals. Finance timelines use event time, then creation time, newest first, with bounded request limits. The headline event count covers USD only; non-USD events remain in their own currency summaries and in the timeline. + +Gemini token normalization follows the [CLI stream statistics](https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/output/stream-json-formatter.ts) and [API usage metadata](https://ai.google.dev/api/generate-content#UsageMetadata): cached prompt tokens are separated from input, and thinking tokens contribute to output. + +## Upgrade and operational limits + +The generated migrations widen monetary columns, add receipt identities and run recovery markers, and restrict incident uniqueness to open incidents. Existing whole-cent values retain their value. Column type changes and index creation acquire database locks; schedule upgrades according to database size. The new nonnegative check rejects invalid legacy ledger rows rather than silently rewriting them. + +Historical rounded charges, missing receipts, ambiguous token semantics, and absent provider prices cannot be reconstructed from these migrations. Old runs are not automatically re-billed. Imported invoices and reviewed corrections can resolve historical prices when there is evidence. Historical runtime totals are retained as an explicit baseline; the inspector identifies legacy totals that lack one instead of fabricating a reconstruction. When a receipt is available but has no price, operators can explicitly allow unpriced usage; doing so does not make the displayed total complete. Runs that permanently lose their final receipt remain pending and require recovery or explicit disabling of the hard-stop policy. Automated reconstruction of missing receipts from arbitrary provider logs is not implemented. + +Tests use disposable PostgreSQL databases and cover company isolation, concurrent duplicate delivery, sub-cent threshold crossings, complete rollback after a late write failure, pending-run recovery, mixed-model conservation, currency separation, monthly rollover, manual pause preservation, simultaneous policies, cancellation delivery failures, and recovery foreign-key lock compatibility. Adapter tests cover timeout receipts and cache semantics, alongside UI amount/share tests. Provider report tests exercise request/response contracts, credential isolation, pagination, revisions, transaction rollback and duplicate delivery with fixture HTTP responses. They do not establish live billing-account permissions. + +Managed Claude and AWS AgentCore report cumulative session usage. The runner saves a per-run baseline, subtracts it from fresh reports, and preserves it across retries and controller recovery. A new run attached to a warm session uses the last complete report as its baseline; reconnecting the same run never resets it. Partial, invalid or regressing reports remain pending. AgentCore reports awaiting interrupted-invocation metadata, and session-ceiling estimates whose token counts remain lower bounds, cannot complete a receipt or release its reservation. Legacy runner checkpoints without a baseline cannot establish historical per-run charges; a subsequent attachment with a complete baseline restores prospective accounting. AgentCore's returned prices remain labeled as estimates. + +ACPX usage may contain placeholder zero counters when its token breakdown is missing or partial. The runner's explicit completeness flag travels with the usage through the driver and durable replay. Per-turn reports are accumulated across retries and restored from the run log after a controller restart. A complete later turn cannot hide an incomplete earlier turn. A terminal event cannot settle such a receipt; the run stays pending until complete usage arrives. Explicitly complete zero usage remains valid and does not prevent retrying an initial turn that did no work. An empty zero-cost attempt before a paid turn does not block prospective Codex pricing; partial positive known spend remains preserved. + +## Reproducing reliability checks + +Run `pnpm test:accounting` for the accounting test typecheck, focused server tests, historical migration tests, V8 coverage, and mutation sentinels. The gate verifies the saved test reports contain passing results for all four crash cases and both historical migration cases; missing or skipped tests fail the gate. The thirteen accounting service modules each require at least 98% line coverage, 90% branch coverage, 96% statement coverage, and 96% function coverage. HTML and JSON reports are written to `coverage/accounting/`. These percentages do not include the full heartbeat service, adapters, browser UI, or child server processes. + +The CI lane that executes the coverage provider requires a frozen install from the checked-in lockfile. It neither resolves a stale manifest inline nor runs the accounting gate after an install failure. Dependency changes must receive the repository lockfile bot’s refresh before this gate can pass; other test lanes retain the repository’s existing install policy. + +The repository's lockfile bot owns dependency lock updates. The new coverage package requires its generated lock refresh before running frozen-install workflows. Live provider evals, optional E2E, Storybook visual checks and canary onboarding retain their frozen-only installs: a stale lock stops the job before tests or provider access; these jobs never resolve replacement dependencies as a fallback. Wait for the bot's reviewed lockfile refresh rather than retrying with an unfrozen install. + +The expanded accounting gate includes prospective Codex pricing and provider-report imports. All four deliberately introduced accounting mutations must be caught by their intended assertions. Separate regressions cover late pre-provider cancellation proof, reservation release without replacing the stop result, lost-channel receipt incompleteness, and exact reservation editing. The full test runner includes every adapter project configured in the root Vitest configuration; a roster check prevents silent omissions. + +The server suite includes four SIGKILL cases: before receipt insertion, after the ledger write but before runtime totals, at the last write before commit, and after commit but before acknowledging cancellation delivery. SQL barriers make these boundaries deterministic. After killing the server process group, the harness terminates the orphan database connection still waiting at the barrier, restarts the real server, and checks exactly one receipt, one set of totals, and one incident/approval. A second restart proves replay does not duplicate them. This tests application crash recovery, not a database power failure. Eight fixed random seeds exercise 640 mixed operations with concurrent receipt retries, policy changes, admission checks, recovery, and injected cancellation failures. + +The migration fixtures restore the predecessor column/index/journal shape, populate historical maximum integer amounts, rounded-zero receipts, finance credits/currencies, and closed incidents, then use the production migrator. An invalid negative legacy receipt must reject the upgrade without changing the schema or migration journal; an explicit repair allows retry. + +Run `pnpm exec playwright test --config tests/e2e/playwright.config.ts tests/e2e/cost-accounting.spec.ts` for the browser workflow. It boots a throwaway instance and the built UI, invokes a deterministic local provider through the real Claude adapter, checks cost display and the budget stop, rejects a manual wake while paused, raises the budget in the UI, and starts another run. It asserts exact cents and cache tokens through the API, then imports an invoice and applies a reviewed sub-cent correction through the operator API. It rejects a correction without a reason, replays the correction without duplicating it, and independently verifies the resulting totals. A separate workflow verifies that charges and credits submitted through the existing finance-events API still appear in Overview and Finance, with independent currency totals and idempotent invoice replay. The deferred accounting tools are absent. Screenshots are attached to the Playwright report. It uses no paid provider credentials or calls. + + +## Durable capture and operator recovery + +The spool lives at `/accounting-receipts`. Directories use mode 0700 and receipt files 0600. Files contain accounting fields only; transcript text, prompts, tools, and credentials are excluded. Publication awaits file and directory fsync on POSIX. On the first use in each process, every ancestor directory must also be readable and support fsync, including for an existing spool or credential-recovery path. This includes real target ancestors and directories containing each storage symlink. Reuse revalidates the complete ancestry and symlink targets; moving an unchanged leaf under a new parent invalidates the proof. Permission or flush failures block recording and credential exchange until corrected; restarting cannot waive an unfinished flush. Each record carries company/run identity, a controller source ID, sequence, timestamp, normalized usage, completeness, and price. Replay validates the schema and fingerprint and rotates failures behind fresh entries in bounded batches. Old controller snapshots are retained as evidence but cannot overwrite a replacement controller or an acknowledged run. Multiple provider attempts are accounted together; incomplete attempts and capture failures remain incomplete. + +Before replacing a stopped run's recorder, recovery saves every pending spool receipt for that company and run, independently of the bounded startup sweep. A save failure blocks replacement and retains the old source for retry. Native recovery then restores its latest journal snapshot; subsequent cumulative Codex run usage replaces that snapshot rather than adding it twice. OpenCode and ACPX per-turn reports are accumulated by turn, rebuilt from company/run-scoped durable events after a controller restart, and protected against replaying older reports. Missing earlier usage or terminal evidence keeps the aggregate incomplete; missing prices preserve known spend with an unpriced marker. Native turn prices are summed in integer nanodollars. A complete zero-cost, zero-token failed resume does not discard the successful Claude retry's model breakdown; other unattributed attempts retain the aggregate fallback. + +Settlement also drains every pending spool receipt for the run under the company accounting lock before checking completeness or acknowledging usage. A newer partial receipt prevents settlement of an older complete snapshot. Recovered journal writes and ledger settlement share a transaction; spool files are removed only after its commit, so a failed charge or projection write leaves the original evidence available for retry. A concurrent replay rename causes a rescan; repeated file movement defers settlement. + +Recovery indexes immutable spool-file identities once per batch, outside company locks. Each settlement skips indexed files belonging to other runs, revalidates its matching receipts, and checks newly published or renamed files. Standalone settlement builds the same index before taking its lock. This avoids repeatedly reading unrelated receipt contents while admissions and ledger writes wait. + +The spool survives process death on persistent local storage, including the tested boundary before the first database write. It does not survive destruction of that storage, and cannot recover provider activity that was never emitted as usage. Back up persistent instance storage together with PostgreSQL. No arbitrary provider-log scraping or automatic provider re-execution occurs during accounting recovery. + +Board-only routes under `/api/companies/:companyId/accounting` provide: + +| Route | Behavior | +| --- | --- | +| `GET /health` | Pending and unpriced counts, oldest pending time, last errors/retry counts, cancellation backlog, held capacity | +| `GET /inspect` | Independent ledger/projection and original run-receipt comparison; no data repair | +| `POST /repair` | Rebuild repairable totals using the reviewed `fingerprint` and a required `reason`; stale review returns 409 | +| `POST /retry` | Retry a company-owned `runId`; record the request and any failed attempt | +| `GET` / `POST /invoices` | List or idempotently import normalized invoice evidence and financial events | +| `POST /provider-costs/import` | Fetch completed daily provider reports for explicitly selected projects/workspaces | +| `GET /invoices/:invoiceId` | Compare exact amounts and expose unmatched, ambiguous, non-inference, or unsupported-currency lines | +| `GET` / `POST /events/:eventId/adjustments` | Read correction history or apply a reviewed correction | + +Agents cannot use operator endpoints. Board viewers may inspect but cannot mutate. Repairs and corrections record the operator and reason in local activity history. An inspection fingerprint binds the repair to the observed discrepancies; the service recomputes them under the company lock before writing. Missing original evidence and legacy runtime uncertainty are never silently repaired. New runtime totals reconcile against an explicit historical baseline plus acknowledged v2 run receipts, including backdated receipts. + +The CLI exposes `accounting health`, `accounting inspect`, `accounting repair`, `accounting retry`, `accounting invoices`, `accounting invoice:import`, `accounting invoice:review `, `accounting provider:import`, and `accounting event:correct `, using the ordinary authenticated company context. Mutations take `--payload-json`; inspection is the dry run for repair. + +## Invoice review and corrections + +Imports use the normalized format in [the invoice template](examples/accounting-invoice.json). Map provider exports to this format explicitly. Invoice identity is `(company, biller, externalId)`; same-content replay is harmless, different-content reuse returns 409. Line order does not affect identity. No provider credentials or remote API calls are needed. + +A match requires a unique company/biller-scoped `costEventId`, `runId` (optionally with model), or `providerRequestId`. All supplied identifiers must agree. Multiple candidates or multiple lines claiming one charge are ambiguous and cannot be applied as invoice-backed corrections. Non-USD invoices, fees, and credits remain visible evidence and are never silently converted into USD inference spend. + +Imports retain every invoice line as evidence. Inference lines matching an existing Finance debit for the same charge, biller, and currency do not create another debit, including when the invoice reports a different price. Ambiguous inference matches also remain evidence without adding a debit. Other lines create idempotent finance events in the same transaction, retaining their kind and currency; uniquely matched charges are linked so overlapping exports cannot count them twice. Fees and credits remain separate events. An import either commits invoice evidence and new timeline entries together or commits neither. Replaying an invoice creates no duplicate charge. Importing an invoice does not automatically change run valuations or revise an existing Finance entry; differences require operator review. Unrelated identifiers cannot establish that two records represent the same payment. + +A correction requires `idempotencyKey`, exact `expectedCents`, `correctedCents`, `reason`, and `pricing` provenance. An optional `invoiceLineId` must uniquely match and support the corrected amount. The expected value guards against stale review; the correction key prevents duplicate application after an ambiguous response. `reportedCostCents` and the original receipt fingerprint remain unchanged. `cost_adjustments` preserves every prior valuation, actor, reason, evidence, and pricing revision, including the original provenance in `previousPricing`. The effective `costCents`, projections, runtime totals for supported new receipts, and budget state update atomically. Pricing an unpriced receipt can release a budget-owned pause; other policies and manual pauses still apply. + +## Scale and fault qualification + +`pnpm benchmark:accounting` creates its own disposable PostgreSQL database **and receipt-spool home**. It never uses the caller's database or pending receipts. `PAPERCLIP_ACCOUNTING_BENCH_ROWS` selects 1,000–10,000,000 events (one million by default), spread across twelve UTC months. Results go to `coverage/accounting/scale.json`; copy that file before running coverage, which cleans the same output directory. There are no machine-dependent latency assertions in CI. + +The benchmark measures all-time reports, eight concurrent writers on one company, two active budgets, admission, eight concurrent ledger/budget/health report bundles competing with writers, durable checkpoints, 100,000 historical run rows, overview/writes/admission with 102 policies (100 unrelated to the writing agent), 1,000/10,000-file receipt backlogs, and recovery of 250 completed runs. An independent integrity check must find no discrepancies. Report bundles measure service/database work, not browser rendering, HTTP latency, provider quota calls, or financial-event queries. The fixture concentrates spend in one agent; it is not a production traffic model. + +The October 4 local comparison on macOS arm64, 10 CPUs, Node 25.6.1 measured the following at one million ledger events. PostgreSQL, API code, and storage shared one developer machine. These are observations, not service-level guarantees. + +| Measurement | Before | After | +| --- | ---: | ---: | +| Budgeted writes/second, eight writers | 20.1 | 35.1 | +| Budgeted write p95 | 396 ms | 227 ms | +| All-time summary p95 | 44 ms | 48 ms | +| All-time project grouping p95 | 71 ms | 75 ms | + +The expanded run measured report-bundle p95 of 678 ms with eight viewers and eight writers, write p95 of 569 ms under that read load, admission p95 of 103 ms, and durable checkpoint p95 of 11 ms. A 102-policy overview had p95 of 73 ms. Health over 100,000 historical runs had p95 of 19 ms. Scanning 10,000 pending receipt files added about 580 ms to recorder startup; receipts are retained until safely persisted. The API process's event-loop delay p95 was 12 ms. The 250-run recovery finished in three batches in 11.5 seconds, with zero integrity findings. + +At ten million events, budgeted throughput rose from 2.8 to 3.8 writes/s and p95 fell from 2.80 to 2.09 seconds. Recovery of 250 runs fell from 90.4 to 71.8 seconds, with no integrity discrepancies. However, eight concurrent report bundles plus eight writers reached 12.2-second report p95 and 12.4-second write p95. Warm all-time summary p95 was 649 ms; the first all-time summary took 59.2 seconds (44.5 seconds before the change). Seeding took 7.2 minutes. These larger runs shared developer hardware with test activity; report code was unchanged, so their variability is not evidence of a report-query regression. They do show that this implementation is **not qualified for heavy traffic on a ten-million-row company**. The aggregation and storage capacity limit remains; the recovery guard prevents it from also creating overlapping sweeps. + +Performance protections: + +- Cost writes aggregate the affected company, agent, and project policies in one ledger scan and one pending-run scan. Each policy retains its own scope and UTC/lifetime window. Exact ledger values remain authoritative, including unknown pricing and native recovery guards. +- A budget operation reuses its observation for thresholds, incident amounts, and reasons. The observation also carries its UTC window across a midnight boundary. It does not cache observations across ledger mutations. PostgreSQL filters policies to the affected scopes before returning rows. +- Overview policy and incident reads run in batches of four, so a large policy list does not enqueue one query per policy at once. +- Health, integrity inspection, and invoice comparison use read-only, repeatable-read snapshots. Their counts and details agree without taking the company accounting write lock. Repairs retain the write lock and revalidate the inspection fingerprint before making changes. +- One heartbeat service shares a running receipt replay/recovery/policy sweep across overlapping scheduler ticks. Success or failure releases the guard; a later tick can retry. This is a per-process scheduler protection, not a distributed lease. Ledger locks and receipt idempotency still protect multiple processes. + +The regression suite checks SQL scan counts, mixed-scope/window totals, pending/unpriced/native recovery guards, snapshot consistency while a writer commits, and coalesced recovery through both success and failure. It also retains the concurrency, crash recovery, exact-money, and mutation tests. Wall-clock benchmarks stay separate from CI correctness gates. + +Capacity remains finite. Budget decisions still sum authoritative ledger rows, so a very large company or lifetime project costs more per write. First writes after a UTC rollover rebuild monthly projections. Cold all-time reports can be much slower than warm reports. A large receipt backlog still requires directory scanning. Qualify the expected company write rate, storage latency, database pool size, and viewer concurrency on deployment hardware before increasing traffic; monitor receipt age and recovery duration as well as request latency. Sustained traffic near measured saturation needs further database aggregation work with equivalent repair and consistency guarantees. + +Monthly projections carry an explicit UTC month marker. The first write after upgrade or rollover initializes from the ledger; subsequent same-month writes use exact atomic increments. Backdated events do not increment the current month. Reports and budget decisions continue using the ledger. Composite project/date and partial unpriced indexes support targeted queries; pending recovery is indexed in its actual update-time order. + +Additional fault tests use independent Node writer processes, kill a recorder after fsync but before database persistence, and proxy PostgreSQL to discard the `COMMIT` acknowledgement after the server commits. Retrying all of these preserves exactly one charge and correct projections. Mutation tests remove deduplication, company filtering, projection updates, and the inclusive hard-stop comparison in Vite memory. Each must produce its specifically marked assertion failure after an unchanged baseline passes, with the other sentinels passing. Setup errors, timeouts, unrelated assertions, and skipped tests do not count as detected mutations. Workspace source is never rewritten by the mutation runner. + + +## Prospective Codex estimates + +Codex commonly reports tokens without dollars. For new receipts, `codex-pricing.ts` applies the immutable `openai-standard-2026-09-30` catalog only when both provider and biller are OpenAI, billing is metered API, usage is complete and per run, and the exact model ID is supported. The initial catalog covers `gpt-6-astra`, `gpt-6-sol`, `gpt-6.1-sol`, `gpt-6-luna`, and `gpt-5.6-sol`, from the [official pricing page](https://developers.openai.com/api/docs/pricing). Unknown models, unsupported tiers, session-cumulative receipts and mixed-model receipts remain unpriced. OpenAI-compatible custom endpoints do not establish OpenAI billing. Existing provider-reported amounts, including zero, take precedence. A Codex CLI, Cursor, or OpenCode terminal event without valid usage counters or a reported price is incomplete and unpriced; it cannot become a zero-dollar estimate. Its reservation remains held pending reliable accounting evidence. Explicitly reported zero counters remain distinct from absent counters. + +Calculation separates ordinary input, cached reads, cache writes, and output. Cache writes are a subset of input, not extra tokens. Integer arithmetic rounds once to nanodollars. An explicit pricing context can select the supported service/context tier; when request-level context is unavailable, the estimate records standard processing and short-request assumptions. A large cumulative run is not evidence that any individual request crossed a long-context threshold. Such estimates can differ from a bill because of tier/context assumptions, negotiated prices, or fees. They are useful estimates, not externally enforced budget ceilings. + +The recorder calculates and freezes the amount, rate version, base rates and assumptions **before** writing its durable spool. Finalization, a process restart, or a later catalog change cannot silently reprice that captured receipt. Native runs carry their managed AI connection's billing identity into this path. Estimated costs feed the same exact, idempotent accounting and budget transaction as provider-reported costs. The original receipt remains available after an invoice correction. Historical missing dollars are not backfilled automatically. + +## Getting financial events into Paperclip + +Financial events remain visible in **Costs → Finance → Recent financial events**. Ingestion is available through the API and CLI; the proposed entry/import dialog is deferred. + +1. **Existing finance-event ingestion:** `POST /api/companies/:companyId/finance-events` or `paperclipai finance event:create --payload-json ''` records charges, fees, credits, and refunds. Supply amounts in cents, the currency, occurrence time, and a description. Use a stable `idempotencyKey` to safely retry after an uncertain response. Existing callers without that optional key remain supported. +2. **Supported integrations:** Browser Use Cloud automatically records its provider charges as linked cost and finance events. Not every agent adapter writes financial events; ordinary inference receipts use the separate cost ledger. +3. **Operator invoice import:** `paperclipai accounting invoice:import --payload-json ''` accepts the normalized [invoice JSON format](examples/accounting-invoice.json). Stable biller/invoice/line IDs provide idempotency. Invoice import produces timeline events atomically and preserves evidence for separately reviewed run corrections. +4. **Operator provider-report import:** the API/CLI fetches API cost reports from OpenAI or Anthropic. Supply a company secret containing the provider organization admin key, the matching provider organization ID, explicit project/workspace IDs, and a UTC interval. Dates cover at most 31 completed days; the end date is exclusive. Use `default` only to intentionally include provider rows with no project/workspace ID. + +The provider importer uses fixed HTTPS endpoints, refuses redirects, verifies Anthropic organization identity, sends the selected OpenAI organization header, filters selected scopes and bounds time, pages and response size. It reads all pages before writing. Missing days, overlapping scoped results or invalid currencies/amounts reject the import. OpenAI amounts arrive in USD; Anthropic reports decimal cents. A complete bucket without a selected scope is an explicit zero. + +Each daily company/provider/account/scope snapshot has a revision. Reimporting unchanged data writes no event. Increases append debit deltas; decreases append credits, including corrections back to a previous total. A slow older fetch cannot overwrite a newer changed snapshot. The snapshot, finance event and activity record commit together. + +**Provider report totals are separate from invoice/manual-charge totals.** The finance summary exposes `providerReportedCentsExact`; report rows remain labeled in the timeline but are excluded from net charges and biller/kind charge groupings. This prevents importing a report and its invoice from adding the same expenditure twice. Neither is added to run-cost estimates. A manually entered charge and an invoice representing the same payment must still be kept distinct by the operator; Paperclip cannot infer that identity from unrelated IDs. + +API: `POST /api/companies/:companyId/accounting/provider-costs/import`. CLI: `paperclipai accounting provider:import --payload-json ''`, with an authenticated company owner/admin (or instance administrator/local operator) context. Ordinary members and agents cannot use this endpoint. + +Create a dedicated company secret through `POST /api/companies/:companyId/secrets` with the provider admin key as `value` and this explicit designation in its metadata: + +```json +{ + "providerMetadata": { + "providerBilling": { "provider": "openai", "accountId": "org_example" } + } +} +``` + +Use `anthropic` and its organization ID for Anthropic. The designation must match both import fields exactly; missing, malformed or mismatched designations are rejected before secret resolution or provider requests. The secret's top-level `provider` still identifies its storage backend, not its billing provider. Existing secret status, version and access-audit checks remain in force. A secret ID alone never authorizes sending an unrelated credential to a billing provider. + +Import payload: + +```json +{ + "provider": "openai", + "secretId": "00000000-0000-4000-8000-000000000001", + "accountId": "org_example", + "scopeIds": ["proj_example"], + "from": "2026-09-01", + "to": "2026-09-02" +} +``` + +Imports are initiated explicitly; this implementation does not schedule background billing synchronization. Repeated API/CLI invocations are safe for an external scheduler. It does not retrieve ChatGPT/Claude subscription invoices, apply exchange rates, or claim provider API reports cover every invoice item. [Anthropic's report](https://platform.claude.com/docs/en/manage-claude/usage-cost-api) excludes Priority Tier costs. The [OpenAI Costs API](https://platform.openai.com/docs/api-reference/usage/audio_transcriptions_object) requires organization billing access; an ordinary inference key may be insufficient. These accounts need to be configured before live imports work. + +## Subscription quotas and refresh behavior + +Company quota requests resolve the caller's accessible subscription AI connections and read each account with its own managed credential. They do not substitute the server host's login for a remote agent account. Private, revoked or cross-company connections are filtered through AI connection authorization. Unmanaged remote credentials require connecting the account in AI connections before its quota is available. + +Responses carry an opaque account identity and successful capture timestamp. The 30-second cache is scoped to company, user, connection, grant and credential revision; authorization and revision metadata are checked before reading cached results. Transient failures retain the last successful UI observation with its original timestamp. Rotation, revocation and explicit authentication rejection clear obsolete windows. Public messages never contain provider subprocess commands or raw diagnostics. + +Budget-policy saves use safe error messages and retain the operator's draft settings on failure. Detailed accounting diagnostics are available through the operator API/CLI, rather than a panel on every Costs tab. + +The shared Costs page retains loaded reports and expanded rows during background refresh failures, with a small stale-data notice. Initial failures remain visible. An initial budget-load failure shows one safe notice above the tabs, including when Budgets is hidden; successful polling restores incident controls automatically. Budget-load and stale-report notices remain independent when requests fail together. Company/date changes isolate cached reports. The run-count label is `runs: 0 api · 11 sub`: distinct recorded runs, not tasks or individual model requests. Input totals include cache reads, with the cached portion shown explicitly. + +Receipt identity and supplemental charges: `heartbeat:` idempotency keys are reserved for internally generated run receipts. Additional API-reported charges may link to the same run with their own keys. They contribute to monthly and acknowledged-run lifetime totals, while receipt-integrity checks compare only the original provider receipt. Recovery isolates each budget scope so a deleted target or failed scope does not prevent recovery elsewhere. + +### Review hardening + +Native restart recovery reuses a held reservation only when the coordinator's +current, unexpired lease owns the same run and project. Ordinary duplicate +provider dispatch remains rejected. A process adapter with no tokens, provider, +price, or explicit unknown-price marker does not create an unpriced charge; +provider adapters with missing prices still do. + +Budget policy updates may omit unchanged settings, including the amount for an +existing policy. Omitted settings are read under the accounting transaction lock. +Creating a policy still requires an amount. Partial edits preserve an explicitly +disabled policy; the legacy monthly-cap endpoints explicitly enable a positive +cap and disable a zero cap. Generic agent and company budget +edits deliver hard-stop cancellation after their transaction commits. + +Status-card update costs retain the ledger's fractional-cent storage precision. +The mutation gate retains partial results and per-run reports/logs under +`coverage/accounting/` when its baseline fails or a mutation survives. + +Failed native turns persist observed run-delta usage before semantic-result +finalization. A matching terminal failure, cancellation, or interruption closes +that usage snapshot; ledger acknowledgement still waits until the native coordinator +has a result or a terminal failure. A retryable failed heartbeat keeps its +reservation and can resume. An open native coordinator's pending accounting +blocks fresh admission without pausing or cancelling its own recovery. Actual +budget overruns, unpriced charges, and closed runs missing accounting still +enforce hard stops; manual pauses remain unchanged. Replacing its recorder restores the native run's +cumulative snapshot; recovery does not add the same tokens twice, and an empty +final result cannot erase earlier observed usage. Missing terminal evidence and +session-only totals remain pending. + +Budget admission completes before the final dispatch ownership check. A rejected +handoff releases a newly created hold using explicit evidence that the adapter +was never entered. Process adapter configuration and spawn failures likewise +return pre-provider proof; failures after a child exists cannot claim it. + +Scheduled retry gates acquire company, issue, then run locks, in the same order +as accounting and attribution writes. Delayed budget cancellation rechecks the +policy version and current blocking state when it writes durable stop intent, +under the company admission lock. External shutdown happens after that lock is +released so the provider can save its final receipt. Raising a budget before the +stop is claimed preserves newly admitted work, including previously queued rows; +a run with an already claimed stop cannot enter provider work after the grant. +For multiple provider attempts, known prices remain in the exact spend total +when another attempt is unpriced. The aggregate stays marked unpriced, so the +configured unknown-price policy still controls whether new work may start. diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 1efb6cbfc8..8998c7f3d9 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -141,7 +141,7 @@ Human auth tables (`users`, `sessions`, and provider-specific auth artifacts) ar - `issue_prefix` text not null - `issue_counter` int not null - `budget_monthly_cents` int not null default 0 -- `spent_monthly_cents` int not null default 0 +- `spent_monthly_cents` numeric(24,7) not null default 0 - `require_board_approval_for_new_agents` boolean not null default false - feedback sharing consent fields @@ -164,7 +164,7 @@ Invariant: every business record belongs to exactly one company. - `default_environment_id` uuid fk `environments.id` null - `context_mode` enum: `thin | fat` default `thin` - `budget_monthly_cents` int not null default 0 -- `spent_monthly_cents` int not null default 0 +- `spent_monthly_cents` numeric(24,7) not null default 0 - pause fields: `pause_reason`, `paused_at` - `permissions` jsonb not null default `{}` - `last_heartbeat_at` timestamptz null @@ -337,7 +337,9 @@ Invariants: - `cost_status` text not null default `reported`; `unpriced` when usage exists but no price was reported - `input_tokens` int not null default 0 - `output_tokens` int not null default 0 -- `cost_cents` int not null +- `cost_cents` numeric(24,7) not null +- `idempotency_key` text null; unique within a company when present +- `receipt_hash` text null; canonical immutable-receipt fingerprint - `occurred_at` timestamptz not null Invariant: each event must attach to agent and company; rollups are aggregation, never manually edited. @@ -1483,7 +1485,9 @@ for contracts, recovery behavior, Storybook, and acceptance workflows. - block new checkout/invocation for that agent - emit high-priority activity event -Board may override by raising budget or explicitly resuming agent. +Board may raise the budget or disable its hard stop. An explicit resume cannot bypass an active blocking policy. Budget-owned pauses release when the applicable UTC window expires; manual pauses remain unchanged. Active hard stops also block pending terminal-run accounting and unpriced usage unless the operator explicitly allows unpriced usage. + +Optional per-run budget reservations serialize available-capacity admission with ledger writes. Durable usage checkpoints, exact decimal reporting, operator integrity inspection and guarded repairs, and reviewed invoice corrections support accounting recovery. Reservations are estimates; provider charges can exceed them. See [Cost accounting and budget enforcement](COST-ACCOUNTING.md) for receipt recovery, retry guarantees, and provider-side limits. ## 13.3 Cost Event Ingestion @@ -1506,7 +1510,8 @@ Board may override by raising budget or explicitly resuming agent. Validation: - non-negative token counts -- `costCents >= 0` +- finite `costCents >= 0`, retaining fractional cents to seven decimal places +- optional company-scoped `idempotencyKey`: identical retries return the original receipt; changed content conflicts - company ownership checks for all linked entities ## 13.4 Rollups @@ -1514,6 +1519,8 @@ Validation: Read-time aggregate queries are acceptable for V1. Materialized rollups can be added later if query latency exceeds targets. +Cost and finance queries default to month-to-date in UTC; explicit bounds are inclusive and `period=all` requests all time. Project rollups retain an unallocated bucket and must conserve total spend. Cost summaries expose missing-price and pending-run counts. Finance summaries retain original currencies without adding unlike currencies together. + ## 14. UI Requirements (Board App) V1 UI routes: diff --git a/doc/SPEC.md b/doc/SPEC.md index 6731f25c50..df02a2053d 100644 --- a/doc/SPEC.md +++ b/doc/SPEC.md @@ -343,6 +343,8 @@ Full hierarchy: **Initiative** (company goal) → Projects → Milestones → Is ## 6. Cost Tracking [DRAFT] +The implemented receipt, recovery, precision, currency, and budget-admission contracts are described in [Cost accounting and budget enforcement](COST-ACCOUNTING.md) and [the V1 implementation spec](SPEC-implementation.md#13-cost-and-budget-system). + Token/LLM cost budgeting is a core part of Paperclip. External revenue and expense tracking is a future plugin. ### Cost Reporting @@ -366,6 +368,8 @@ Three tiers: 2. **Soft alerts** — configurable thresholds (e.g. warn at 80% of budget) 3. **Hard ceiling** — auto-pause the Agent when budget is hit. Board notified. Board can override/raise the limit. +The implementation also supports estimated per-run reservations, durable receipt recovery, exact decimal reporting, independent integrity inspection, and audited invoice corrections. Recorded spend and reservations do not replace a provider-enforced invoice cap. See [cost accounting](COST-ACCOUNTING.md). + Budgets can be set to **unlimited** (no ceiling). ### Open Questions diff --git a/doc/examples/accounting-invoice.json b/doc/examples/accounting-invoice.json new file mode 100644 index 0000000000..abc43fa906 --- /dev/null +++ b/doc/examples/accounting-invoice.json @@ -0,0 +1,15 @@ +{ + "biller": "anthropic", + "externalId": "example-invoice-2026-09", + "currency": "USD", + "lines": [ + { + "externalId": "example-line-1", + "kind": "inference", + "amountCents": "12.5000000", + "occurredAt": "2026-09-01T00:00:00Z", + "providerRequestId": "replace-with-the-provider-request-id", + "pricing": { "source": "provider_invoice", "version": "2026-09", "evidence": "example-invoice-2026-09" } + } + ] +} diff --git a/package.json b/package.json index ab0ae5c516..a8ad8c938e 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "typecheck:build-gaps": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk ensure-build-deps && pnpm --filter @paperclipai/server build && node scripts/run-typecheck-build-gaps.mjs", "test": "pnpm run test:run", "test:watch": "pnpm run preflight:workspace-links && vitest", + "test:accounting": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/server exec tsc -p tsconfig.accounting-tests.json && vitest run --config server/vitest.accounting.config.ts --reporter=default --reporter=json --outputFile.json=../coverage/accounting/server-tests.json && vitest run --project @paperclipai/db packages/db/src/cost-accounting-migration.test.ts --reporter=default --reporter=json --outputFile.json=coverage/accounting/migration-tests.json && node scripts/verify-accounting-test-results.mjs coverage/accounting/server-tests.json coverage/accounting/migration-tests.json && pnpm test:accounting:mutations", "test:run": "pnpm run preflight:workspace-links && node scripts/run-vitest-stable.mjs", "test:runner-acceptance": "vitest run --config tests/runner-acceptance/vitest.config.ts", "test:runner-acceptance:typecheck": "tsc -p tests/runner-acceptance/tsconfig.json", @@ -91,10 +92,13 @@ "connections:ingest-app-definitions": "node scripts/ingest-app-definitions.mjs", "test:lifecycle-baseline": "node tests/lifecycle-baseline/run.mjs", "test:lifecycle-baseline:support": "node --test tests/lifecycle-baseline/report.test.mjs", - "test:lifecycle-baseline:typecheck": "tsc -p tests/lifecycle-baseline/tsconfig.json" + "test:lifecycle-baseline:typecheck": "tsc -p tests/lifecycle-baseline/tsconfig.json", + "test:accounting:mutations": "node scripts/test-accounting-mutations.mjs", + "benchmark:accounting": "node cli/node_modules/tsx/dist/cli.mjs server/scripts/benchmark-accounting.ts" }, "devDependencies": { "@playwright/test": "^1.62.1", + "@vitest/coverage-v8": "^5.0.3", "agentmail": "^0.5.31", "cross-env": "^10.1.0", "esbuild": "^0.28.2", diff --git a/packages/db/src/cost-accounting-migration.test.ts b/packages/db/src/cost-accounting-migration.test.ts new file mode 100644 index 0000000000..f657225f19 --- /dev/null +++ b/packages/db/src/cost-accounting-migration.test.ts @@ -0,0 +1,131 @@ +import { randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import postgres from "postgres"; +import { describe, expect, it } from "vitest"; +import { applyPendingMigrations, inspectMigrations } from "./client.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const journal = JSON.parse(readFileSync(new URL("./migrations/meta/_journal.json", import.meta.url), "utf8")); +const hashes = journal.entries.filter((entry: { tag: string }) => entry.tag === "0312_easy_eternity").map((entry: { tag: string }) => `${entry.tag}.sql`).map((name: string) => + createHash("sha256").update(readFileSync(new URL(`./migrations/${name}`, import.meta.url))).digest("hex")); + +async function historicalDatabase() { + const database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-upgrade-"); + const sql = postgres(database.connectionString, { max: 1, onnotice: () => {} }); + // Restore the complete predecessor shape for the changed tables, including + // the old uniqueness predicate and migration journal. Upgrade uses the real + // production migrator, not hand-picked statements from the migration. + await sql.unsafe(` + DROP TABLE provider_billing_snapshots, cost_adjustments, billing_invoice_lines, billing_invoices, budget_reservations, run_usage_receipts, accounting_runtime_baselines; + DROP INDEX cost_events_company_project_occurred_idx; + DROP INDEX cost_events_unpriced_idx; + ALTER TABLE cost_events DROP COLUMN provider_request_id, DROP COLUMN reported_cost_cents, DROP COLUMN pricing_provenance; + ALTER TABLE heartbeat_runs DROP COLUMN accounting_projection_version, DROP COLUMN accounting_last_attempt_at, DROP COLUMN accounting_last_error, DROP COLUMN accounting_attempt_count; + ALTER TABLE budget_policies DROP COLUMN reservation_cents; + ALTER TABLE companies DROP COLUMN spend_month_utc; + ALTER TABLE agents DROP COLUMN spend_month_utc; + ALTER TABLE finance_events ALTER COLUMN amount_cents TYPE integer; + ALTER TABLE status_card_updates ALTER COLUMN cost_cents TYPE integer; + ALTER TABLE cost_events DROP CONSTRAINT cost_events_nonnegative_amounts; + ALTER TABLE cost_events DROP COLUMN idempotency_key, DROP COLUMN receipt_hash; + ALTER TABLE finance_events DROP COLUMN idempotency_key, DROP COLUMN receipt_hash; + ALTER TABLE heartbeat_runs DROP COLUMN cost_accounting_pending, DROP COLUMN cost_accounted_at; + ALTER TABLE budget_policies DROP COLUMN enforcement_version, DROP COLUMN enforcement_delivered_version, DROP COLUMN unpriced_usage_policy; + ALTER TABLE cost_events ALTER COLUMN cost_cents TYPE integer; + ALTER TABLE companies ALTER COLUMN spent_monthly_cents TYPE integer; + ALTER TABLE agents ALTER COLUMN spent_monthly_cents TYPE integer; + ALTER TABLE agent_runtime_state ALTER COLUMN total_cost_cents TYPE bigint; + ALTER TABLE budget_incidents ALTER COLUMN amount_observed TYPE integer; + DROP INDEX budget_incidents_policy_window_threshold_idx; + CREATE UNIQUE INDEX budget_incidents_policy_window_threshold_idx + ON budget_incidents (policy_id, window_start, threshold_type) WHERE status <> 'dismissed'; + `); + await sql`DELETE FROM drizzle.__drizzle_migrations WHERE hash IN ${sql(hashes)}`; + return { database, sql, close: async () => { await sql.end(); await database.cleanup(); } }; +} + +(support.supported ? describe : describe.skip)("cost accounting historical upgrade", () => { + it("preserves historical receipts, counters, references, finance currencies and incident history", async () => { + const f = await historicalDatabase(); + const { sql } = f; + const company = randomUUID(), agent = randomUUID(), run = randomUUID(), policy = randomUUID(); + try { + await sql`INSERT INTO companies (id,name,issue_prefix,spent_monthly_cents) VALUES (${company},'Historical','HIST',2147483647)`; + await sql`INSERT INTO agents (id,company_id,name,spent_monthly_cents) VALUES (${agent},${company},'Historical worker',2147483647)`; + await sql`INSERT INTO heartbeat_runs (id,company_id,agent_id,status,usage_json,finished_at) + VALUES (${run},${company},${agent},'succeeded','{"inputTokens":42,"costUsd":0.004}', '2025-12-31T23:59:59Z')`; + await sql`INSERT INTO agent_runtime_state (agent_id,company_id,adapter_type,total_cost_cents,total_input_tokens) + VALUES (${agent},${company},'codex_local',9007199254740993,42)`; + await sql`INSERT INTO cost_events (company_id,agent_id,heartbeat_run_id,provider,model,input_tokens,output_tokens,cost_cents,occurred_at) + VALUES (${company},${agent},${run},'historical','old-model',42,5,2147483647,'2025-12-31T23:59:59Z'), + (${company},${agent},${run},'historical','old-model',0,0,0,'2026-01-01T00:00:00Z')`; + await sql`INSERT INTO finance_events (company_id,event_kind,direction,biller,amount_cents,currency,occurred_at,external_invoice_id) + VALUES (${company},'adjustment','debit','historical',12345,'EUR',now(),'invoice-1'), + (${company},'adjustment','credit','historical',789,'USD',now(),'invoice-1')`; + await sql`INSERT INTO budget_policies (id,company_id,scope_type,scope_id,metric,window_kind,amount) + VALUES (${policy},${company},'agent',${agent},'billed_cents','calendar_month_utc',100)`; + await sql`INSERT INTO budget_incidents (company_id,policy_id,scope_type,scope_id,metric,window_kind,window_start,window_end,threshold_type,amount_limit,amount_observed,status) + VALUES (${company},${policy},'agent',${agent},'billed_cents','calendar_month_utc','2026-01-01Z','2026-02-01Z','hard',100,101,'resolved'), + (${company},${policy},'agent',${agent},'billed_cents','calendar_month_utc','2026-01-01Z','2026-02-01Z','hard',100,101,'dismissed')`; + const before = await sql`SELECT id,company_id,agent_id,heartbeat_run_id,provider,model,cost_cents::text,input_tokens,occurred_at FROM cost_events ORDER BY id`; + const financeBefore = await sql`SELECT id,event_kind,direction,amount_cents,currency,external_invoice_id FROM finance_events ORDER BY id`; + await applyPendingMigrations(f.database.connectionString); + await applyPendingMigrations(f.database.connectionString); + expect((await inspectMigrations(f.database.connectionString)).status).toBe("upToDate"); + const after = await sql`SELECT id,company_id,agent_id,heartbeat_run_id,provider,model,cost_cents::text,input_tokens,occurred_at FROM cost_events ORDER BY id`; + expect(after.map(row => ({ ...row, cost_cents: Number(row.cost_cents) }))).toEqual(before.map(row => ({ ...row, cost_cents: Number(row.cost_cents) }))); + expect((await sql`SELECT id,event_kind,direction,amount_cents,currency,external_invoice_id FROM finance_events ORDER BY id`).map(row => ({ ...row, amount_cents: Number(row.amount_cents) }))).toEqual(financeBefore); + expect(await sql`SELECT cost_accounting_pending,cost_accounted_at FROM heartbeat_runs WHERE id=${run}`).toEqual([{ cost_accounting_pending: false, cost_accounted_at: null }]); + expect(await sql`SELECT enforcement_version,enforcement_delivered_version,unpriced_usage_policy FROM budget_policies WHERE id=${policy}`) + .toEqual([{ enforcement_version: 0, enforcement_delivered_version: 0, unpriced_usage_policy: "block" }]); + for (const [table, column, expected] of [["companies","spent_monthly_cents","2147483647.0000000"],["agents","spent_monthly_cents","2147483647.0000000"],["agent_runtime_state","total_cost_cents","9007199254740993.0000000"]]) { + expect((await sql.unsafe(`SELECT ${column}::text AS value FROM ${table}`))[0].value).toBe(expected); + } + expect(await sql`SELECT DISTINCT idempotency_key,receipt_hash FROM cost_events`).toEqual([{ idempotency_key: null, receipt_hash: null }]); + expect((await sql`SELECT reservation_cents::text FROM budget_policies WHERE id=${policy}`)[0].reservation_cents).toBe("0.0000000"); + expect((await sql`SELECT spend_month_utc FROM companies WHERE id=${company}`)[0].spend_month_utc).toBeNull(); + await sql`INSERT INTO finance_events (company_id,event_kind,biller,amount_cents,currency,occurred_at) VALUES (${company},'adjustment','new',0.0000001,'USD',now())`; + expect((await sql`SELECT amount_cents::text FROM finance_events WHERE biller='new'`)[0].amount_cents).toBe("0.0000001"); + // Closed historical incidents must not prevent a fresh threshold crossing. + const insertIncident = () => sql`INSERT INTO budget_incidents (company_id,policy_id,scope_type,scope_id,metric,window_kind,window_start,window_end,threshold_type,amount_limit,amount_observed,status) + VALUES (${company},${policy},'agent',${agent},'billed_cents','calendar_month_utc','2026-01-01Z','2026-02-01Z','hard',150,150.4,'open')`; + await insertIncident(); + await expect(insertIncident()).rejects.toMatchObject({ code: "23505" }); + expect(await sql`SELECT status FROM budget_incidents ORDER BY status`).toEqual([{ status:"dismissed" },{ status:"open" },{ status:"resolved" }]); + await sql`INSERT INTO cost_events (company_id,agent_id,provider,model,cost_cents,occurred_at,idempotency_key) + VALUES (${company},${agent},'new','model',0.0000001,now(),'new-receipt')`; + expect((await sql`SELECT cost_cents::text FROM cost_events WHERE idempotency_key='new-receipt'`)[0].cost_cents).toBe("0.0000001"); + await expect(sql`INSERT INTO cost_events (company_id,agent_id,provider,model,cost_cents,occurred_at,idempotency_key) + VALUES (${company},${agent},'new','model',1,now(),'new-receipt')`).rejects.toMatchObject({ code: "23505" }); + // A development deployment can have applied this schema under an older + // migration number. Replaying the SQL must preserve both old and new rows. + const beforeReplay = await sql`SELECT id,cost_cents::text FROM cost_events ORDER BY id`; + const migration = readFileSync(new URL("./migrations/0312_easy_eternity.sql", import.meta.url), "utf8"); + await sql.begin(async (tx) => { + for (const statement of migration.split("--> statement-breakpoint")) await tx.unsafe(statement); + }); + expect(await sql`SELECT id,cost_cents::text FROM cost_events ORDER BY id`).toEqual(beforeReplay); + } finally { await f.close(); } + }, 60_000); + + it("rejects invalid historical amounts atomically and can retry after explicit repair", async () => { + const f = await historicalDatabase(); const { sql } = f; + const company = randomUUID(), agent = randomUUID(); + try { + await sql`INSERT INTO companies (id,name,issue_prefix) VALUES (${company},'Invalid historical','BAD')`; + await sql`INSERT INTO agents (id,company_id,name) VALUES (${agent},${company},'Worker')`; + await sql`INSERT INTO cost_events (company_id,agent_id,provider,model,cost_cents,occurred_at) VALUES (${company},${agent},'old','old',-1,now())`; + const journal = await sql`SELECT * FROM drizzle.__drizzle_migrations ORDER BY id`; + await expect(applyPendingMigrations(f.database.connectionString)).rejects.toThrow(); + expect(await sql`SELECT * FROM drizzle.__drizzle_migrations ORDER BY id`).toEqual(journal); + expect((await sql`SELECT data_type FROM information_schema.columns WHERE table_name='cost_events' AND column_name='cost_cents'`)[0].data_type).toBe("integer"); + expect(await sql`SELECT column_name FROM information_schema.columns WHERE table_name='cost_events' AND column_name='idempotency_key'`).toHaveLength(0); + expect((await sql`SELECT cost_cents FROM cost_events`)[0].cost_cents).toBe(-1); + await sql`UPDATE cost_events SET cost_cents=0`; + await applyPendingMigrations(f.database.connectionString); + expect((await inspectMigrations(f.database.connectionString)).status).toBe("upToDate"); + } finally { await f.close(); } + }, 60_000); +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts index fbf45dfea9..47abcf4602 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts @@ -42,8 +42,33 @@ import { type PrpEvent, type PrpStructuredRunResult, } from "./codex-app-server-driver.test-support.js"; +import { rehydrateRunnerdUsageNotification } from "../../live/runnerd-codex-transport.js"; describe("Codex app-server Codex driver", () => { + it("preserves incomplete runner receipts through the ACPX driver and later recovery", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport], { + driverIdentity: { kind: "acpx_runtime", displayName: "ACPX", version: "test" }, + }).openSession({ runId: "run-acpx-usage", normalizedSessionId: "session-acpx-usage", workingDirectory: WORKSPACE }); + const turn = await session.startTurn({ message: { role: "user", text: "Work." } }); + transport.push("turn/started", { threadId: "thread-1", turn: { id: turn.turnId, status: "inProgress" } }); + for (const complete of [true, false, true]) { + transport.push("thread/tokenUsage/updated", rehydrateRunnerdUsageNotification({ + provider: "acpx", runDeltaAvailable: complete, + cumulative: { inputTokens: 0, outputTokens: 0, providerCostUsd: 10 }, + runDelta: { inputTokens: 12, outputTokens: complete ? 4 : 0, providerCostUsd: 0 }, + }, "thread-1", turn.turnId)); + } + transport.push("turn/completed", { threadId: "thread-1", turn: { id: turn.turnId, status: "completed", items: [] } }); + const events = await collectUntilTerminal(session.events()); + expect(events.filter(event => event.payload.kind === "usage").map(event => event.payload.usage)).toMatchObject([ + { runDeltaComplete: true, runDelta: { inputTokens: 12, outputTokens: 4 } }, + { runDeltaComplete: false, runDelta: { inputTokens: 12, outputTokens: 0 } }, + { runDeltaComplete: true, runDelta: { inputTokens: 12, outputTokens: 4 } }, + ]); + expect(await session.usage()).toMatchObject({ runDeltaComplete: true }); + }); + it("admits a strictly bound semantic result from the durable runner", async () => { const transport = new FakeCodexTransport(); const session = await makeDriver([transport]).openSession({ diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 24f8f98cc8..55cdca9d5f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1996,6 +1996,23 @@ it("rehydrates durable cumulative usage for a cold thread read", () => { expect(rehydrateRunnerdThreadTokenUsage(null)).toBeNull(); }); +it.each([true, false, undefined, null, "true", 1])( + "preserves explicit runner usage completeness (%j)", + (runDeltaAvailable) => { + const runDelta = { inputTokens: 0, outputTokens: 0, providerCostUsd: 0 }; + const notification = rehydrateRunnerdUsageNotification( + { cumulative: { ...runDelta, providerCostUsd: 10 }, runDelta, runDeltaAvailable }, + "thread", + "turn", + ); + expect(notification.tokenUsage).toEqual({ + total: { ...runDelta, providerCostUsd: 10 }, + runDelta, + runDeltaComplete: runDeltaAvailable === true, + }); + }, +); + it("binds a durable semantic result to the active provider turn", () => { expect( rehydrateRunnerdResultNotification( diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index db0344becc..fe6168c252 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -33,6 +33,8 @@ import { type ExecuteNativeSessionOptions, } from "./native-session-runtime.js"; +import { rehydrateRunnerdUsageNotification } from "./live/runnerd-codex-transport.js"; + const identity = { runId: "run-recovery", sessionId: "session-recovery", @@ -6501,7 +6503,7 @@ describe("executeNativeSession recovery", () => { }); }); - it.each([false, true])("only replays the original ACPX envelope for a proven effect-free initial turn (prepared: %s)", async (prepared) => { + it.each([false, true].flatMap(prepared => [false, true].map(translated => ({ prepared, translated }))))("only replays the original ACPX envelope for a proven effect-free initial turn (prepared: $prepared, translated: $translated)", async ({ prepared, translated }) => { const executionInput = prepared ? preparedInput() : input; const checkpoint: PersistedNativeSession = { backendKind: "mock", @@ -6665,6 +6667,12 @@ describe("executeNativeSession recovery", () => { turnId: "turn-work", }, ]; + const receipt = effectFreeTurn[3]!.payload.usage as Record; + if (translated) { + effectFreeTurn[3]!.payload.usage = rehydrateRunnerdUsageNotification({ + provider: "acpx", cumulative: receipt.total, runDelta: receipt.runDelta, runDeltaAvailable: true, + }, "thread", "turn-work").tokenUsage; + } bySource.set("runner-recovery", effectFreeTurn); await expect( @@ -6709,6 +6717,28 @@ describe("executeNativeSession recovery", () => { expect(recoveryEnvelope.task).not.toHaveProperty("description"); } + const zeroUsage = effectFreeTurn[3]!.payload.usage as Record; + for (const unsafeUsage of [ + { ...zeroUsage, runDeltaComplete: false }, + { ...zeroUsage, runDeltaComplete: null }, + { ...zeroUsage, runDeltaComplete: "true" }, + { ...zeroUsage, unknownEvidence: 0 }, + { ...zeroUsage, total: { ...receipt.total as object, providerCostUsd: 1 } }, + { ...zeroUsage, runDelta: { ...receipt.runDelta as object, inputTokens: 1 } }, + { ...zeroUsage, runDelta: { ...receipt.runDelta as object, outputTokens: undefined } }, + { ...zeroUsage, total: { ...receipt.total as object, requests: 2 } }, + ]) { + startTurn.mockClear(); + const unsafeTurn = structuredClone(effectFreeTurn); + unsafeTurn[3]!.payload.usage = unsafeUsage; + bySource.set("runner-recovery", unsafeTurn); + await executeNativeSession({ input: executionInput, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); + const unsafeEnvelope = JSON.parse(startTurn.mock.calls[0]![0].message.text); + expect(unsafeEnvelope.task.prompt).toContain("semantic-result recovery for a prior completed provider turn"); + expect(unsafeEnvelope.task.prompt).not.toContain(executionInput.task.prompt); + } + startTurn.mockClear(); bySource.set("runner-recovery", [ ...effectFreeTurn.slice(0, 3), diff --git a/packages/shared/src/money.test.ts b/packages/shared/src/money.test.ts new file mode 100644 index 0000000000..3edebc8151 --- /dev/null +++ b/packages/shared/src/money.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { addCents, centsToUnits, centsToUsd, compareCents, formatUsdExact, MAX_MONEY_UNITS, normalizeCents, subtractCents, unitsToCents, usdToCents } from "./money.js"; + +describe("fixed-point money", () => { + it.each([ + ["0.00000005", "0.0000001"], ["-0.00000005", "-0.0000001"], + ["0.000000049", "0.0000000"], ["1e-7", "0.0000001"], + ["12345678901234567.1234567", "12345678901234567.1234567"], + ])("normalizes %s exactly", (input, expected) => expect(normalizeCents(input)).toBe(expected)); + it("does exact arithmetic beyond floating-point precision", () => { + expect(addCents("12345678901234567.1234567", "0.0000001")).toBe("12345678901234567.1234568"); + expect(subtractCents("9007199254740992.0000001", "9007199254740992")).toBe("0.0000001"); + expect(compareCents("9007199254740992.0000001", "9007199254740992")).toBe(1); + expect(addCents(0.1, 0.2)).toBe("0.3000000"); + }); + it("converts USD without intermediate multiplication or double rounding", () => { + expect(usdToCents("0.000000001")).toBe("0.0000001"); + expect(usdToCents("0.00000000049")).toBe("0.0000000"); + expect(usdToCents("0.0123456789")).toBe("1.2345679"); + expect(formatUsdExact("4.5")).toBe("$0.05"); + }); + it("round-trips the full database range", () => expect(centsToUnits(unitsToCents(MAX_MONEY_UNITS))).toBe(MAX_MONEY_UNITS)); + it.each(["0.0000001", "9007199254740992.0000001", "-0.0000001", unitsToCents(MAX_MONEY_UNITS)])("preserves an editable USD value for %s cents", cents => { + expect(usdToCents(centsToUsd(cents))).toBe(cents); + }); + it.each(["NaN", "Infinity", "", " 1", "1.2.3", "1e101", "100000000000000000", Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])("rejects invalid/out-of-range %s", input => { + expect(() => centsToUnits(input)).toThrow(); + }); +}); diff --git a/packages/shared/src/validators/company-skill.test.ts b/packages/shared/src/validators/company-skill.test.ts index ec28e5f933..bec89367a4 100644 --- a/packages/shared/src/validators/company-skill.test.ts +++ b/packages/shared/src/validators/company-skill.test.ts @@ -7,9 +7,27 @@ import { companySkillInstallCatalogSchema, companySkillInstallUpdateSchema, companySkillResetSchema, + companySkillTestRunCostSummarySchema, companySkillUpdateStatusSchema, } from "./company-skill.js"; +describe("skill test run cost summary", () => { + const tokens = { inputTokens: 10, cachedInputTokens: 5, outputTokens: 2 }; + + it.each([0, 0.0000001, 0.1234567, 25, 4_000_000_000.125])("preserves valid cents: %s", (costCents) => { + const response = { ...tokens, costCents }; + expect(companySkillTestRunCostSummarySchema.parse(response)).toEqual(response); + }); + + it.each([-0.0000001, -1, NaN, Infinity, -Infinity, "0.1234567", null])("rejects invalid cents: %s", (costCents) => { + expect(companySkillTestRunCostSummarySchema.safeParse({ ...tokens, costCents }).success).toBe(false); + }); + + it.each(["inputTokens", "cachedInputTokens", "outputTokens"])("still requires whole %s", (field) => { + expect(companySkillTestRunCostSummarySchema.safeParse({ ...tokens, costCents: 0.1234567, [field]: 0.5 }).success).toBe(false); + }); +}); + const catalogSkill = { id: "paperclipai:bundled:software-development:review", key: "paperclipai/bundled/software-development/review", diff --git a/packages/shared/src/validators/status-card.test.ts b/packages/shared/src/validators/status-card.test.ts index 26194a5d39..ea757373a8 100644 --- a/packages/shared/src/validators/status-card.test.ts +++ b/packages/shared/src/validators/status-card.test.ts @@ -1,5 +1,18 @@ import { describe, expect, it } from "vitest"; -import { statusCardRefreshPolicySchema } from "./status-card.js"; +import { statusCardRefreshPolicySchema, statusCardSchema, statusCardUpdateSchema } from "./status-card.js"; + +describe.each([ + ["status card daily spend", statusCardSchema.pick({ todayCostCents: true }), "todayCostCents"], + ["status card update spend", statusCardUpdateSchema.pick({ costCents: true }), "costCents"], +] as const)("%s", (_name, schema, field) => { + it.each([0, 0.0000001, 0.1234567, 25, 4_000_000_000.125])("preserves valid cents: %s", (amount) => { + expect(schema.parse({ [field]: amount })).toEqual({ [field]: amount }); + }); + + it.each([-0.0000001, -1, NaN, Infinity, -Infinity, "0.1234567", null])("rejects invalid cents: %s", (amount) => { + expect(schema.safeParse({ [field]: amount }).success).toBe(false); + }); +}); describe("statusCardRefreshPolicySchema", () => { it("accepts valid IANA timezones", () => { diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 293280c2d1..66a5caab0c 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -456,3 +456,40 @@ test("direct protocol concurrency override only lowers the configured ceiling", if (expected !== null) assert.equal(result.stdout, expected); } }); + +for (const workflow of ["release.yml", "e2e.yml", "storybook-visual.yml", "runner-live-evals.yml"]) { + test(`${workflow} keeps its frozen dependency install and fails closed on stale locks`, () => { + const text = readWorkflow(workflow); + const job = workflow === "release.yml" ? text.split(" smoke_canary_onboarding:\n")[1].split("\n # ----- Nightly lane")[0] : text; + const install = job.match(/(?:- name: Install (?:test )?dependencies\n\s+run: ([^\n]+)|- run: (pnpm install[^\n]*))/); + assert.ok(install, "source workflow must expose a dependency install step"); + const script = install[1] ?? install[2]; + for (const exitCode of [0, 17]) { + const run = spawnSync("bash", ["-euo", "pipefail", "-c", ` + pnpm() { printf '%s\n' "$*"; return "$INSTALL_EXIT"; } + ${script} + `], { encoding: "utf8", env: { ...process.env, INSTALL_EXIT: String(exitCode) } }); + assert.equal(run.status, exitCode, run.stderr); + assert.equal(run.stdout.trim(), "install --frozen-lockfile", "never resolve new packages or retry after a frozen install failure"); + } + }); +} + +test("the accounting CI lane installs only the reviewed lockfile and gates coverage on that install", () => { + const job = readWorkflow("pr-trusted.yml").split(" general_tests:\n")[1].split(" docker_context_integrity:\n")[0]; + const install = job.split(" - name: Install dependencies\n")[1].split(" - name: Run grouped general test suites\n")[0]; + const script = install.split(" run: |\n")[1]; + assert.ok(script, "accounting lane must have an inspectable install command"); + for (const exitCode of [0, 17]) { + const result = spawnSync("bash", ["-euo", "pipefail", "-c", ` + pnpm() { printf '%s\\n' "$*"; return "$INSTALL_EXIT"; } + ${script} + `], { encoding: "utf8", env: { ...process.env, ACCOUNTING_GATE: "true", INSTALL_EXIT: String(exitCode) } }); + assert.equal(result.status, exitCode, result.stderr); + assert.equal(result.stdout.trim(), "install --frozen-lockfile", "a stale lock must not resolve executable coverage code inline"); + } + assert.match(install, /id: install_dependencies/); + assert.match(install, /ACCOUNTING_GATE: \$\{\{ matrix\.group == 'general-server-without-chat' && matrix\.shard_index == 0 \}\}/); + const coverage = job.split(" - name: Verify accounting coverage and crash recovery\n")[1].split(" - name: Upload accounting coverage\n")[0]; + assert.match(coverage, /steps\.install_dependencies\.outcome == 'success'/, "failed installs must never load the coverage provider"); +}); diff --git a/scripts/test-accounting-mutations.mjs b/scripts/test-accounting-mutations.mjs new file mode 100644 index 0000000000..0dcd2bf30a --- /dev/null +++ b/scripts/test-accounting-mutations.mjs @@ -0,0 +1,57 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile, mkdir, copyFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +const root = fileURLToPath(new URL("..", import.meta.url)); +const directory = await mkdtemp(path.join(tmpdir(), "accounting-mutations-")); +const artifactDirectory = path.join(root, "coverage/accounting/mutations"); +const targets = { deduplication: "deduplicates a retried receipt", company: "isolates company reporting", projection: "conserves agent projections", threshold: "stops at the exact budget boundary" }; +const evidence = []; +const persist = () => writeFile(path.join(root, "coverage/accounting/mutations.json"), JSON.stringify({ generatedAt: new Date().toISOString(), evidence }, null, 2)); +await rm(artifactDirectory, { recursive: true, force: true }); +await mkdir(artifactDirectory, { recursive: true }); +try { + for (const name of ["baseline", ...Object.keys(targets)]) { + const report = path.join(directory, `${name}.json`); + const entry = { mutation: name, result: "error", assertions: [] }; + evidence.push(entry); + await persist(); + const result = await new Promise((resolve) => { + const child = spawn(process.execPath, ["node_modules/vitest/vitest.mjs", "run", "--config", "server/vitest.accounting-mutations.config.ts", "--reporter=json", `--outputFile=${report}`], { + cwd: root, env: { ...process.env, PAPERCLIP_ACCOUNTING_MUTATION: name === "baseline" ? "" : name }, stdio: ["ignore", "pipe", "pipe"], + }); + let log = ""; const capture = data => { log += data; }; + child.stdout.on("data", capture); child.stderr.on("data", capture); + child.once("error", error => resolve({ code: null, log: `${log}\n${error.message}` })); + child.once("close", code => resolve({ code, log })); + }); + await writeFile(path.join(artifactDirectory, `${name}.log`), result.log); + await copyFile(report, path.join(artifactDirectory, `${name}.json`)).catch(() => {}); + const json = JSON.parse(await readFile(report, "utf8").catch(() => { throw new Error(`${name} produced no test evidence: ${result.log}`); })); + const assertions = json.testResults.flatMap(file => file.assertionResults); + entry.assertions = assertions.map(({ title, status, failureMessages }) => ({ title, status, failureMessages })); + const completeRoster = assertions.length === 4 && Object.values(targets).every(title => assertions.filter(test => test.title === title).length === 1); + if (name === "baseline") { + entry.result = result.code === 0 && completeRoster && assertions.every(test => test.status === "passed") ? "passed" : "baseline_failed"; + if (entry.result !== "passed") throw new Error(`Mutation baseline failed or skipped: ${result.log}`); + } else { + const failures = assertions.filter(test => test.status === "failed"); + const sentinel = failures[0]; + // A marked AssertionError proves the intended comparison ran. Setup + // errors, timeouts, unrelated assertions, and extra failures are invalid. + const intendedFailure = failures.length === 1 && sentinel.title === targets[name] + && sentinel.failureMessages?.length === 1 + && sentinel.failureMessages[0].startsWith(`AssertionError: ACCOUNTING_ASSERTION ${name}:`); + entry.result = result.code === 1 && result.log.split(/\r?\n/).includes(`ACCOUNTING_MUTATION_APPLIED ${name}`) + && completeRoster && assertions.every(test => test.status === "passed" || test === sentinel) + && intendedFailure ? "killed" : "survived_or_invalid"; + if (entry.result !== "killed") throw new Error(`Mutation ${name} survived or failed without its sentinel: ${result.log}`); + } + await persist(); + process.stdout.write(`${name}: ${entry.result}\n`); + } +} finally { + // Failed baselines, surviving mutants, and missing reports are evidence too. + try { await persist(); } finally { await rm(directory, { recursive: true, force: true }); } +} diff --git a/scripts/test-accounting-mutations.test.mjs b/scripts/test-accounting-mutations.test.mjs new file mode 100644 index 0000000000..ddc81dcfeb --- /dev/null +++ b/scripts/test-accounting-mutations.test.mjs @@ -0,0 +1,49 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, copyFile, readFile, writeFile, rm, access } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +for (const scenario of ["success", "baseline", "survives", "missing", "database", "timeout", "wrong_assertion", "wrong_sentinel", "missing_marker", "extra_failure", "skipped"]) { + test(`mutation gate retains evidence: ${scenario}`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "mutation-evidence-test-")); + try { + await mkdir(path.join(root, "scripts")); + await mkdir(path.join(root, "node_modules/vitest"), { recursive: true }); + await copyFile(path.resolve("scripts/test-accounting-mutations.mjs"), path.join(root, "scripts/test-accounting-mutations.mjs")); + await writeFile(path.join(root, "node_modules/vitest/vitest.mjs"), ` +import { writeFileSync } from 'node:fs'; +const name = process.env.PAPERCLIP_ACCOUNTING_MUTATION; +const scenario = process.env.MUTATION_EVIDENCE_TEST_SCENARIO; +const titles = { deduplication: 'deduplicates a retried receipt', company: 'isolates company reporting', projection: 'conserves agent projections', threshold: 'stops at the exact budget boundary' }; +console.log(name && scenario !== 'missing_marker' ? 'ACCOUNTING_MUTATION_APPLIED ' + name : 'baseline diagnostics'); +if (name && scenario === 'missing') process.exit(1); +const fail = name ? scenario !== 'survives' : scenario === 'baseline'; +const failureMessage = scenario === 'database' ? 'Error: connection refused' + : scenario === 'timeout' ? 'Error: Test timed out in 10000ms' + : scenario === 'wrong_assertion' ? 'AssertionError: expected 1 to be 2' + : 'AssertionError: ACCOUNTING_ASSERTION ' + name + ': expected value to match'; +const assertionResults = Object.values(titles).map(title => { + const failed = fail && (!name || title === titles[scenario === 'wrong_sentinel' ? 'company' : name] || scenario === 'extra_failure'); + return { title, status: failed ? 'failed' : name && scenario === 'skipped' ? 'pending' : 'passed', failureMessages: failed ? [failureMessage] : [] }; +}); +writeFileSync(process.argv.find(arg => arg.startsWith('--outputFile=')).slice(13), JSON.stringify({ testResults: [{ assertionResults }] })); +process.exit(fail ? 1 : 0); +`); + if (scenario === "missing") { + await mkdir(path.join(root, "coverage/accounting/mutations"), { recursive: true }); + await writeFile(path.join(root, "coverage/accounting/mutations/deduplication.json"), '{"stale":true}'); + } + const result = spawnSync(process.execPath, ["scripts/test-accounting-mutations.mjs"], { cwd: root, + env: { ...process.env, MUTATION_EVIDENCE_TEST_SCENARIO: scenario }, encoding: "utf8" }); + assert.equal(result.status, scenario === "success" ? 0 : 1, result.stderr); + const report = JSON.parse(await readFile(path.join(root, "coverage/accounting/mutations.json"), "utf8")); + assert.equal(report.evidence.at(-1).result, { success: "killed", baseline: "baseline_failed", missing: "error" }[scenario] ?? "survived_or_invalid"); + const last = report.evidence.at(-1).mutation; + assert.ok((await readFile(path.join(root, `coverage/accounting/mutations/${last}.log`), "utf8")).length); + if (scenario === "missing") await assert.rejects(access(path.join(root, `coverage/accounting/mutations/${last}.json`))); + if (scenario !== "missing") assert.ok(JSON.parse(await readFile(path.join(root, `coverage/accounting/mutations/${last}.json`), "utf8")).testResults.length); + } finally { await rm(root, { recursive: true, force: true }); } + }); +} diff --git a/scripts/verify-accounting-test-results.mjs b/scripts/verify-accounting-test-results.mjs new file mode 100644 index 0000000000..f98a910cf8 --- /dev/null +++ b/scripts/verify-accounting-test-results.mjs @@ -0,0 +1,39 @@ +import { readFile } from "node:fs/promises"; + +const required = [ + { + file: "server/src/__tests__/cost-accounting-crash.test.ts", + suite: "accounting survives real server SIGKILL", + tests: ["before_receipt", "before_runtime_totals", "before_commit", "after_commit_before_delivery_ack"] + .map(phase => `recovers exactly once after ${phase}`), + }, + { + file: "packages/db/src/cost-accounting-migration.test.ts", + suite: "cost accounting historical upgrade", + tests: [ + "preserves historical receipts, counters, references, finance currencies and incident history", + "rejects invalid historical amounts atomically and can retry after explicit repair", + ], + }, +]; + +try { + const reports = process.argv.slice(2); + if (reports.length !== required.length) throw new Error("Expected server and migration JSON report paths"); + for (const [index, spec] of required.entries()) { + const report = JSON.parse(await readFile(reports[index], "utf8")); + if (report.success !== true || !Array.isArray(report.testResults)) throw new Error(`Unsuccessful or invalid report: ${reports[index]}`); + const files = report.testResults.filter(result => result.name?.replaceAll("\\", "/").endsWith(`/${spec.file}`)); + if (files.length !== 1 || !Array.isArray(files[0].assertionResults)) throw new Error(`Missing or duplicate suite: ${spec.file}`); + for (const title of spec.tests) { + const matches = files[0].assertionResults.filter(result => result.fullName === `${spec.suite} ${title}`); + if (matches.length !== 1 || matches[0].status !== "passed") { + throw new Error(`Required accounting test did not pass: ${spec.suite} ${title}`); + } + } + process.stdout.write(`Verified ${spec.tests.length} required tests: ${spec.file}\n`); + } +} catch (error) { + console.error(`Accounting gate: ${error.message}`); + process.exitCode = 1; +} diff --git a/scripts/verify-accounting-test-results.test.mjs b/scripts/verify-accounting-test-results.test.mjs new file mode 100644 index 0000000000..3aaf6eae2f --- /dev/null +++ b/scripts/verify-accounting-test-results.test.mjs @@ -0,0 +1,50 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +function reports() { + return [ + { success: true, testResults: [{ name: "/repo/server/src/__tests__/cost-accounting-crash.test.ts", assertionResults: + ["before_receipt", "before_runtime_totals", "before_commit", "after_commit_before_delivery_ack"].map(phase => ({ + fullName: `accounting survives real server SIGKILL recovers exactly once after ${phase}`, status: "passed", + })), + }] }, + { success: true, testResults: [{ name: "/repo/packages/db/src/cost-accounting-migration.test.ts", assertionResults: + ["preserves historical receipts, counters, references, finance currencies and incident history", + "rejects invalid historical amounts atomically and can retry after explicit repair"].map(title => ({ + fullName: `cost accounting historical upgrade ${title}`, status: "passed", + })), + }] }, + ]; +} + +for (const scenario of ["passed", "skipped_crash", "skipped_migration", "missing_case", "missing_suite", "duplicate_case", "duplicate_suite", "wrong_suite", "failed", "malformed", "missing_report"]) { + test(`required accounting evidence: ${scenario}`, async () => { + const directory = await mkdtemp(path.join(tmpdir(), "accounting-required-tests-")); + try { + const data = reports(); + const crash = data[0].testResults[0]; + const migration = data[1].testResults[0]; + if (scenario === "skipped_crash") crash.assertionResults[0].status = "pending"; + if (scenario === "skipped_migration") migration.assertionResults[1].status = "pending"; + if (scenario === "missing_case") crash.assertionResults.pop(); + if (scenario === "missing_suite") data[1].testResults = []; + if (scenario === "duplicate_case") migration.assertionResults.push(migration.assertionResults[0]); + if (scenario === "duplicate_suite") data[0].testResults.push(crash); + if (scenario === "wrong_suite") crash.assertionResults[0].fullName = "another test with the same title"; + if (scenario === "failed") data[0].success = false; + const files = data.map((_, index) => path.join(directory, `${index}.json`)); + for (const [index, report] of data.entries()) { + if (scenario === "missing_report" && index === 1) continue; + await writeFile(files[index], scenario === "malformed" ? "{" : JSON.stringify(report)); + } + const result = spawnSync(process.execPath, ["scripts/verify-accounting-test-results.mjs", ...files], { encoding: "utf8" }); + assert.equal(result.status, scenario === "passed" ? 0 : 1, result.stderr); + if (scenario === "passed") assert.match(result.stdout, /Verified 4 required tests:[\s\S]*Verified 2 required tests:/); + else assert.match(result.stderr, /Accounting gate:/); + } finally { await rm(directory, { recursive: true, force: true }); } + }); +} diff --git a/server/scripts/benchmark-accounting.ts b/server/scripts/benchmark-accounting.ts new file mode 100644 index 0000000000..05c26ffa3d --- /dev/null +++ b/server/scripts/benchmark-accounting.ts @@ -0,0 +1,169 @@ +import { randomUUID } from "node:crypto"; +import { reserveRunBudget } from "../src/services/budget-reservations.js"; +import { createRunUsageRecorder, indexPendingUsageReceipts, usageReceiptSpoolPath } from "../src/services/usage-receipts.js"; +import { budgetService } from "../src/services/budgets.js"; +import { sql } from "drizzle-orm"; +/** Reproducible synthetic PostgreSQL benchmark. Never reads DATABASE_URL or + * provider credentials; startEmbeddedPostgresTestDatabase owns all data. */ +import { performance, monitorEventLoopDelay } from "node:perf_hooks"; +import { mkdir, writeFile, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { agents, companies, projects, heartbeatRuns, budgetPolicies, createDb, startEmbeddedPostgresTestDatabase } from "@paperclipai/db"; +import { costService } from "../src/services/costs.js"; +import { withAccountingTransaction } from "../src/services/accounting-transaction.js"; +import { accountingIntegrityService } from "../src/services/accounting-integrity.js"; +import { reconcileRunCosts } from "../src/services/run-cost-accounting.js"; +const count = Number(process.env.PAPERCLIP_ACCOUNTING_BENCH_ROWS ?? 1_000_000); +if (!Number.isInteger(count) || count < 1000 || count > 10_000_000) throw new Error("Benchmark row count must be 1,000–10,000,000"); +// Recovery reads the instance receipt spool as well as the database. Always +// own both; a caller's real PAPERCLIP_HOME must never be scanned or replayed. +const previousHome = process.env.PAPERCLIP_HOME; +const benchmarkHome = await mkdtemp(path.join(os.tmpdir(), "accounting-benchmark-")); +process.env.PAPERCLIP_HOME = benchmarkHome; +const database = await startEmbeddedPostgresTestDatabase("accounting-scale-").catch(async error => { + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + await rm(benchmarkHome, { recursive: true, force: true }); + throw error; +}); +const db = createDb(database.connectionString); +const started = performance.now(); +const eventLoop = monitorEventLoopDelay({ resolution: 10 }); eventLoop.enable(); +function distribution(values: number[]) { + const sorted = [...values].sort((a,b) => a-b); + const pick = (p: number) => Number(sorted[Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1)].toFixed(2)); + return { samples: sorted.length, p50Ms: pick(0.5), p95Ms: pick(0.95), p99Ms: pick(0.99), maxMs: pick(1) }; +} +async function time(work: () => Promise) { const start = performance.now(); await work(); return performance.now() - start; } +try { + const [company] = await db.insert(companies).values({ name: "Scale fixture", issuePrefix: "SCALE" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning(); + const [project] = await db.insert(projects).values({ companyId: company.id, name: "Hot project" }).returning(); + const seedMs = await time(async () => { + await db.$client`insert into cost_events (company_id,agent_id,project_id,provider,biller,model,cost_cents,input_tokens,cached_input_tokens,output_tokens,occurred_at) + select ${company.id},${agent.id}, case when i % 100 = 0 then ${project.id}::uuid else null end, 'fixture','fixture','fixture', 0.1234567,7,11,3, + date_trunc('month',now() at time zone 'UTC') at time zone 'UTC' - (i % 12) * interval '1 month' + from generate_series(1,${count}) i`; + await db.$client`analyze cost_events`; + // Seed projections through an explicit reviewed repair, not benchmark-only SQL. + const integrity = accountingIntegrityService(db); const review = await integrity.inspect(company.id); + await integrity.repair(company.id, review.fingerprint, "Initialize synthetic benchmark projections", "benchmark"); + }); + const reports: number[] = [], projectReports: number[] = []; + for (let i = 0; i < 20; i++) { + reports.push(await time(() => costService(db).summary(company.id, { allTime: true }))); + projectReports.push(await time(() => costService(db).byProject(company.id, { allTime: true }))); + } + const writes: number[] = []; const concurrency = 8; const writeStarted = performance.now(); + for (let batch = 0; batch < 8; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, { + agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `bench:${batch}:${index}`, occurredAt: new Date(), + })).then(ms => writes.push(ms)))); + const writeMs = performance.now() - writeStarted; + await budgetService(db).upsertPolicy(company.id, { scopeType: "company", scopeId: company.id, amount: 2_000_000_000 }, "benchmark"); + await budgetService(db).upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 2_000_000_000 }, "benchmark"); + const budgetedWrites: number[] = []; + const budgetedStart = performance.now(); + for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, { + agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `budgeted:${batch}:${index}`, occurredAt: new Date(), + })).then(ms => budgetedWrites.push(ms)))); + const budgetedMs = performance.now() - budgetedStart; + const admissions: number[] = []; + for (let i = 0; i < 20; i++) { + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning(); + admissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null))); + } + const dashboard: number[] = [], writesWhilePolling: number[] = []; + // Eight viewers each issue the ledger queries used by the Costs overview, + // together with budget and accounting health reads, while eight writes run. + for (let batch = 0; batch < 4; batch++) await Promise.all([ + ...Array.from({ length: 8 }, () => time(() => Promise.all([ + costService(db).summary(company.id), costService(db).byAgent(company.id), + costService(db).byProject(company.id), costService(db).byAgentModel(company.id), + budgetService(db).overview(company.id), accountingIntegrityService(db).health(company.id), + ])).then(ms => dashboard.push(ms))), + ...Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, { + agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `polling:${batch}:${index}`, occurredAt: new Date(), + })).then(ms => writesWhilePolling.push(ms))), + ]); + const [checkpointRun] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning(); + const recorder = await createRunUsageRecorder(db, { companyId: company.id, runId: checkpointRun.id, adapterType: "process" }); + const checkpoints: number[] = []; + for (let i = 1; i <= 50; i++) checkpoints.push(await time(() => recorder.capture({ complete: false, usageBasis: "per_run", usage: { inputTokens: i, outputTokens: i }, costUsdExact: "0.000000001" }))); + // Include historical run-table size, independently of the pending index. + await db.$client`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending,finished_at) + select ${company.id},${agent.id},'succeeded',false, now() from generate_series(1,100000)`; + await db.$client`analyze heartbeat_runs`; + const health: number[] = []; + for (let i = 0; i < 20; i++) health.push(await time(() => accountingIntegrityService(db).health(company.id))); + const waits: number[] = []; + await Promise.all(Array.from({ length: concurrency }, () => { + const start = performance.now(); + return withAccountingTransaction(db, company.id, async tx => { waits.push(performance.now() - start); await tx.execute(sql`select pg_sleep(0.005)`); }); + })); + await db.$client`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending,usage_json,finished_at) + select ${company.id},${agent.id},'failed',true,'{"accountingReceiptReady":true,"costUsdExact":"0.000000001","inputTokens":1}'::jsonb, now() from generate_series(1,250)`; + const recovery: Array<{ milliseconds: number; scanned: number; accounted: number }> = []; + for (let i = 0; i < 3; i++) { + const before = performance.now(); const result = await reconcileRunCosts(db); recovery.push({ milliseconds: performance.now()-before, ...result }); + } + if (recovery.reduce((sum, r) => sum+r.accounted,0) !== 250) throw new Error("Benchmark recovery did not conserve receipt count"); + const spool = usageReceiptSpoolPath(); await mkdir(spool, { recursive: true }); + const spoolBacklogs: Array<{ files: number; indexMs: number; recorderStartMs: number }> = []; + const foreignRunId = randomUUID(); + let published = 0; + for (const files of [1000, 10000]) { + // Publish synthetic immutable files directly; timed checkpoints above use + // the real fsync path. These belong to another run and must be retained. + for (; published < files; published += 50) await Promise.all(Array.from({ length: 50 }, (_, offset) => writeFile(path.join(spool, `${published + offset}.json`), JSON.stringify({ + schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: company.id, runId: foreignRunId, + sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false }, + })))); + const indexMs = await time(() => indexPendingUsageReceipts()); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning(); + const recorderStartMs = await time(() => createRunUsageRecorder(db, { companyId: company.id, runId: run.id, adapterType: "process" })); + spoolBacklogs.push({ files, indexMs, recorderStartMs }); + } + await rm(spool, { recursive: true, force: true }); + // A large policy list must not make each write fetch every agent's policies. + const extraAgents = await db.insert(agents).values(Array.from({ length: 100 }, (_, i) => ({ companyId: company.id, name: `Idle ${i}`, role: "engineer", adapterType: "process" }))).returning(); + await db.insert(budgetPolicies).values(extraAgents.map(extra => ({ companyId: company.id, scopeType: "agent", scopeId: extra.id, windowKind: "calendar_month_utc", amount: 2_000_000_000 }))); + const largeOverview: number[] = []; + for (let i = 0; i < 10; i++) largeOverview.push(await time(async () => { + const overview = await budgetService(db).overview(company.id); + if (overview.policies.length !== 102) throw new Error("Benchmark policy fixture is incomplete"); + })); + const largePolicyWrites: number[] = [], largePolicyAdmissions: number[] = []; + const largePolicyStart = performance.now(); + for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, { + agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `large-policy:${batch}:${index}`, occurredAt: new Date(), + })).then(ms => largePolicyWrites.push(ms)))); + const largePolicyMs = performance.now() - largePolicyStart; + for (let i = 0; i < 20; i++) { + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning(); + largePolicyAdmissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null))); + } + const integrity = await accountingIntegrityService(db).inspect(company.id); + if (integrity.findings.length) throw new Error(`Benchmark violated accounting integrity: ${JSON.stringify(integrity.findings)}`); + const plan = await db.$client`explain (analyze,buffers,format json) select sum(cost_cents) from cost_events where company_id = ${company.id} and project_id = ${project.id} + and occurred_at >= date_trunc('month',now() at time zone 'UTC') at time zone 'UTC'`; + const report = { generatedAt: new Date().toISOString(), platform: `${os.platform()} ${os.arch()}`, cpus: os.cpus().length, node: process.version, + seedRows: count, seedMs, concurrency, allTimeSummary: distribution(reports), allTimeProjects: distribution(projectReports), + hotCompanyWrites: { activePolicies: 0, ...distribution(writes), writesPerSecond: writes.length / (writeMs / 1000) }, companyLockAcquisition: distribution(waits), + budgetedCompanyWrites: { activePolicies: 2, ...distribution(budgetedWrites), writesPerSecond: budgetedWrites.length / (budgetedMs / 1000) }, + admission: distribution(admissions), concurrentDashboard: distribution(dashboard), writesWhilePolling: distribution(writesWhilePolling), + durableCheckpoint: distribution(checkpoints), healthWith100kHistoricalRuns: distribution(health), + overviewWith102Policies: distribution(largeOverview), + writesWith102Policies: { activePolicies: 102, relevantPolicies: 2, ...distribution(largePolicyWrites), writesPerSecond: largePolicyWrites.length / (largePolicyMs / 1000) }, + admissionWith102Policies: distribution(largePolicyAdmissions), spoolBacklogs, + eventLoopDelay: { p95Ms: eventLoop.percentile(95) / 1e6, maxMs: eventLoop.max / 1e6 }, + recovery, projectBudgetQueryPlan: plan, integrityFindings: integrity.findings.length, totalMs: performance.now()-started }; + const output = path.resolve("coverage/accounting/scale.json"); await mkdir(path.dirname(output), { recursive: true }); await writeFile(output, JSON.stringify(report,null,2)); + process.stdout.write(JSON.stringify({ ...report, projectBudgetQueryPlan: "see coverage/accounting/scale.json" },null,2)+"\n"); +} finally { + eventLoop.disable(); + try { await db.$client.end(); await database.cleanup(); } + finally { + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + await rm(benchmarkHome, { recursive: true, force: true }); + } +} diff --git a/server/src/__tests__/browser-use-connection.test.ts b/server/src/__tests__/browser-use-connection.test.ts index b206fabe93..1c74d88945 100644 --- a/server/src/__tests__/browser-use-connection.test.ts +++ b/server/src/__tests__/browser-use-connection.test.ts @@ -1,3 +1,5 @@ +import { costService, createCostEventInTransaction } from "../services/costs.js"; +import { withAccountingTransaction } from "../services/accounting-transaction.js"; import { budgetService } from "../services/budgets.js"; import { buildPaperclipRuntimeMcpServers } from "../services/heartbeat.js"; import { resolveNativeRuntimeMcpSnapshot } from "../services/native-runtime/runtime-context.js"; @@ -480,6 +482,69 @@ const actor = { actorType: "user" as const, actorId: "browser-reviewer" }; f.access.refreshCatalog(f.connection.connectionId, actor), ).rejects.toThrow(); }); + it("rolls back the Browser Use charge and Finance event if a spend projection fails", async () => { + const f = await fixture(); + await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: 100, occurredAt: new Date() }); + await f.service.execute(f.binding, f.grant, randomUUID(), "browser_start", { task: "Read example.com" }); + f.complete(); + await db.execute(sql`create function fail_browser_projection() returns trigger language plpgsql as $$ begin raise exception 'injected projection failure'; end $$`); + await db.execute(sql.raw(`create trigger fail_browser_projection before update of spent_monthly_cents on companies for each row when (new.id = '${f.company.id}'::uuid) execute function fail_browser_projection()`)); + try { + await f.tick(); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1); + expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(0); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(100); + expect((await db.select().from(browserUseRuns).where(eq(browserUseRuns.companyId, f.company.id)))[0].accountedCents).toBe(0); + } finally { + await db.execute(sql`drop trigger fail_browser_projection on companies`); + await db.execute(sql`drop function fail_browser_projection()`); + } + await f.tick(); + await f.tick(); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("115.0000000"); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(2); + expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toMatchObject([{ amountCents: 15 }]); + expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].spentMonthlyCents).toBe(115); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(115); + }); + + it("settles Browser Use atomically behind another accounting writer without losing spend", async () => { + const f = await fixture(); + const charge = (costCents: number) => ({ agentId: f.agent.id, provider: "fixture", model: "fixture", costCents, occurredAt: new Date() }); + await costService(db).createEvent(f.company.id, charge(100)); + await f.service.execute(f.binding, f.grant, randomUUID(), "browser_start", { task: "Read example.com" }); + f.complete(); + let release!: () => void; + let locked!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { locked = resolve; }); + const writer = withAccountingTransaction(db, f.company.id, async (tx, publications) => { + await createCostEventInTransaction(tx, f.company.id, charge(5), publications); + locked(); + await released; + }); + await ready; + const browser = f.tick(); + try { + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() and wait_event_type = 'Lock' and pid <> pg_backend_pid()`); + expect(waiting.length).toBeGreaterThan(0); + }); + // Nothing from Browser Use may become visible before its projections. + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1); + expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(0); + } finally { + release(); + await Promise.all([writer, browser]); + } + await f.tick(); // Replaying the provider's cumulative total is a no-op. + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("120.0000000"); + expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].spentMonthlyCents).toBe(120); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(120); + expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toMatchObject([{ amountCents: 15 }]); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(3); + }); + it("observes delayed browser.ready, hides viewer credentials, accounts once and cleans up", async () => { const f = await fixture(); const invocation = randomUUID(); diff --git a/server/src/__tests__/codex-pricing.test.ts b/server/src/__tests__/codex-pricing.test.ts new file mode 100644 index 0000000000..6d47c81e11 --- /dev/null +++ b/server/src/__tests__/codex-pricing.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils"; +import { priceCodexReceipt } from "../services/codex-pricing.js"; + +const receipt: AdapterUsageCheckpoint = { + provider: "openai", + biller: "openai", + billingType: "metered_api", + model: "gpt-6-astra", + complete: true, + usageBasis: "per_run", + costUsd: null, + usage: { + inputTokens: 123536, + cachedInputTokens: 1567209, + outputTokens: 6625, + }, +}; +describe("prospective Codex pricing", () => { + it.each([ + ["gpt-6-sol", "12.200000000"], + ["gpt-6.1-sol", "12.100000000"], + ["gpt-6-luna", "0.610000000"], + ["gpt-5.6-sol", "24.400000000"], + ])("uses the distinct published cache rate for %s", (model, expected) => { + expect( + priceCodexReceipt({ + ...receipt, + model, + usage: { + inputTokens: 1_000_000, + cachedInputTokens: 1_000_000, + outputTokens: 1_000_000, + }, + }).costUsdExact, + ).toBe(expected); + }); + it("prices the staging token shape without counting cached input twice", () => { + const result = priceCodexReceipt(receipt); + expect(result.costUsdExact).toBe("3.133819000"); + expect(result.costStatus).toBe("estimated"); + expect(result.pricingProvenance).toMatchObject({ + source: "rate_card", + version: "openai-standard-2026-09-30", + }); + expect(result.pricingProvenance?.evidence).toContain( + "short per-request context assumed", + ); + expect(receipt.costUsdExact).toBeUndefined(); + }); + it.each([ + { complete: false }, + { complete: false, usage: { inputTokens: 0, outputTokens: 0 } }, + { costUsd: 0 }, + { costUsd: 4.21 }, + { costUsdExact: "0.004" }, + { cacheAdjustedCostUsd: 0.01 }, + { provider: "other" }, + { biller: "proxy" }, + { model: "gpt-6-astra-custom" }, + { model: "unknown" }, + { model: "constructor" }, + { model: "__proto__" }, + { usageBasis: undefined }, + { usageBasis: null }, + { billingType: "unknown" }, + { billingType: "subscription_included" }, + { usageBasis: "session_cumulative" }, + { pricingContext: { serviceTier: "unsupported" } }, + { usage: undefined }, + { usage: { inputTokens: -1, outputTokens: 0 } }, + { usage: { inputTokens: 1, outputTokens: 0, cacheWriteTokens: 2 } }, + ])( + "preserves reported costs and refuses ambiguous pricing: %j", + (override) => { + const input = { ...receipt, ...override } as AdapterUsageCheckpoint; + expect(priceCodexReceipt(input)).toBe(input); + }, + ); + it.each([ + ["standard", "short", "61.000000000"], + ["fast", "short", "122.000000000"], + ["flex", "short", "30.500000000"], + ["standard", "long", "97.000000000"], + ["batch", "long", "48.500000000"], + ] as const)("handles %s/%s rates", (serviceTier, contextTier, expected) => { + const result = priceCodexReceipt({ + ...receipt, + pricingContext: { serviceTier, contextTier }, + usage: { + inputTokens: 1_000_000, + cachedInputTokens: 1_000_000, + outputTokens: 1_000_000, + cacheWriteTokens: 0, + }, + }); + expect(result.costUsdExact).toBe(expected); + }); + it("charges cache writes once as a subset of input", () => { + expect( + priceCodexReceipt({ + ...receipt, + usage: { + inputTokens: 1_000_000, + cacheWriteTokens: 1_000_000, + outputTokens: 0, + }, + }).costUsdExact, + ).toBe("12.500000000"); + }); + it("retains nanodollar precision and does not reprice frozen receipts", () => { + const priced = priceCodexReceipt({ + ...receipt, + model: "gpt-6-luna", + usage: { inputTokens: 0, cachedInputTokens: 1, outputTokens: 0 }, + }); + expect(priced.costUsdExact).toBe("0.000000010"); + expect(priceCodexReceipt(priced)).toBe(priced); + }); +}); diff --git a/server/src/__tests__/cost-accounting-by-user.test.ts b/server/src/__tests__/cost-accounting-by-user.test.ts new file mode 100644 index 0000000000..2997bb30f7 --- /dev/null +++ b/server/src/__tests__/cost-accounting-by-user.test.ts @@ -0,0 +1,162 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { agents, authUsers, companies, companyMemberships, costEvents, createDb, heartbeatRuns, issues, type Db } from "@paperclipai/db"; +import { costService } from "../services/costs.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +describe("cost attribution by user (PostgreSQL)", () => { + let database: Awaited>, db: Db; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-by-user-"); db = createDb(database.connectionString); }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + + async function fixture() { + const [company] = await db.insert(companies).values({ name: "User costs", issuePrefix: `U${randomUUID().slice(0, 7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Shared worker", role: "engineer", adapterType: "process" }).returning(); + return { companyId: company.id, agentId: agent.id }; + } + async function member(companyId: string, name: string, status = "active") { + const userId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name, email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status }); + return userId; + } + async function run(f: Awaited>, responsibleUserId: string | null) { + return (await db.insert(heartbeatRuns).values({ ...f, responsibleUserId, status: "succeeded" }).returning())[0]; + } + async function charge(f: Awaited>, overrides: Partial = {}) { + await db.insert(costEvents).values({ ...f, provider: "openai", model: "fixture", costCents: 1, + billingType: "metered_api", occurredAt: new Date(), ...overrides }); + } + function trace() { + const queries: string[] = []; + const traced = drizzle(db.$client, { logger: { logQuery(query) { queries.push(query); } } }) as unknown as Db; + return { service: costService(traced), ledgerReads: () => queries.filter(query => query.startsWith("select") && query.includes('from "cost_events"')) }; + } + + it("reports spend for any number of users and only includes active humans with zero spend", async () => { + const f = await fixture(), foreign = await fixture(); + const { service, ledgerReads } = trace(); + expect(await service.byUser(f.companyId)).toEqual({ activeUserCount: 0, rows: [] }); + await member(f.companyId, "Alice"); + await member(f.companyId, "Invited", "pending"); + await member(f.companyId, "Suspended", "suspended"); + await member(f.companyId, "Former", "archived"); + await member(foreign.companyId, "Foreign"); + await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "agent", principalId: f.agentId }); + await charge(f); + expect(await service.byUser(f.companyId)).toMatchObject({ activeUserCount: 1, rows: [ + { userId: null, costCents: 1 }, { userName: "Alice", costCents: 0 }, + ] }); + expect(ledgerReads()).toHaveLength(2); + await member(f.companyId, "Bob"); + expect(await service.byUser(f.companyId)).toMatchObject({ activeUserCount: 2, rows: [ + { userId: null, costCents: 1 }, { userName: "Alice", costCents: 0 }, { userName: "Bob", costCents: 0 }, + ] }); + expect(ledgerReads()).toHaveLength(3); + }); + + it("excludes the Board principal by ID and preserves its historical charges as unattributed", async () => { + const f = await fixture(); + await db.insert(authUsers).values({ id: "local-board", name: "Board", email: "local@paperclip.local", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: "local-board" }); + const service = costService(db); + expect(await service.byUser(f.companyId)).toEqual({ activeUserCount: 0, rows: [] }); + // A real person is never filtered by their display name. + const human = await member(f.companyId, "Board"); + const humanRun = await run(f, human), boardRun = await run(f, "local-board"); + await charge(f, { heartbeatRunId: humanRun.id, costCents: 0.25 }); + await charge(f, { heartbeatRunId: boardRun.id, costCents: 1 }); + const report = await service.byUser(f.companyId); + expect(report).toMatchObject({ activeUserCount: 1, rows: [ + { userId: null, userName: null, costCentsExact: "1.0000000", runCount: 1 }, + { userId: human, userName: "Board", costCentsExact: "0.2500000", runCount: 1 }, + ] }); + expect(report.rows.some(row => row.userId === "local-board")).toBe(false); + expect((await service.summary(f.companyId)).spendCentsExact).toBe("1.2500000"); + }); + + it("preserves exact spend, legacy cache semantics, zero-spend members and distinct runs", async () => { + const f = await fixture(); + const alice = await member(f.companyId, "Alice"), bob = await member(f.companyId, "Bob"); + const former = await member(f.companyId, "Former", "archived"); + const a = await run(f, alice), b = await run(f, former); + const [otherAgent] = await db.insert(agents).values({ companyId: f.companyId, name: "Second worker", role: "engineer", adapterType: "process" }).returning(); + const c = await run({ ...f, agentId: otherAgent.id }, alice); + await charge(f, { heartbeatRunId: a.id, costCents: 0.1000001, inputTokens: 100, cachedInputTokens: 40, outputTokens: 5 }); + await charge(f, { heartbeatRunId: a.id, costCents: 0.2000002, receiptHash: "modern", costStatus: "estimated", inputTokens: 100, cachedInputTokens: 40, outputTokens: 10 }); + await charge(f, { heartbeatRunId: b.id, costCents: 0.4 }); + await charge(f, { heartbeatRunId: c.id, agentId: otherAgent.id, costCents: 0, costStatus: "unpriced", inputTokens: 9 }); + await charge(f, { heartbeatRunId: a.id, costCents: 0, costStatus: "unpriced", billingType: "subscription_included" }); + await charge(f, { costCents: 0.0000001 }); + const { service, ledgerReads } = trace(); + const report = await service.byUser(f.companyId, { allTime: true }); + expect(ledgerReads()).toHaveLength(1); + expect(report.rows.find(row => row.userId === alice)).toMatchObject({ + costCentsExact: "0.3000003", eventCount: 4, estimatedEventCount: 1, unpricedEventCount: 1, + inputTokens: 169, cachedInputTokens: 80, outputTokens: 15, runCount: 2, + }); + expect(report.rows.find(row => row.userId === bob)).toMatchObject({ costCentsExact: "0.0000000", eventCount: 0, runCount: 0 }); + expect(report.rows.find(row => row.userId === former)).toMatchObject({ userName: "Former", costCentsExact: "0.4000000", runCount: 1 }); + expect(report.rows.find(row => row.userId === null)).toMatchObject({ costCentsExact: "0.0000001", runCount: 0 }); + const sum = report.rows.reduce((total, row) => total + BigInt(row.costCentsExact.replace(".", "")), 0n); + expect(sum).toBe(7000004n); + expect((await service.summary(f.companyId, { allTime: true })).spendCentsExact).toBe("0.7000004"); + }); + + it("retains charges and active memberships when user profiles are missing", async () => { + const f = await fixture(); + const missing = [randomUUID(), randomUUID()].sort(); + const former = randomUUID(); + for (const userId of [...missing, former]) { + await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: userId, + status: userId === former ? "archived" : "active" }); + } + const ownerRun = await run(f, former); + await charge(f, { heartbeatRunId: ownerRun.id, costCents: 0.125 }); + const report = await costService(db).byUser(f.companyId); + expect(report).toMatchObject({ activeUserCount: 2, rows: [ + { userId: former, userName: "Former user", userImage: null, costCentsExact: "0.1250000", runCount: 1 }, + ...missing.map(userId => ({ userId, userName: "Unknown user", userImage: null, costCentsExact: "0.0000000", runCount: 0 })), + ] }); + expect((await costService(db).summary(f.companyId)).spendCentsExact).toBe("0.1250000"); + }); + + it("keeps every company charge but never attributes through a foreign run, agent or user", async () => { + const f = await fixture(), foreign = await fixture(); + const alice = await member(f.companyId, "Alice"); await member(f.companyId, "Bob"); + const outsider = await member(foreign.companyId, "Secret foreign name"); + const foreignRun = await run(foreign, alice); + const wrongUser = await run(f, outsider); + const unowned = await run(f, null); + const [otherAgent] = await db.insert(agents).values({ companyId: f.companyId, name: "Other", role: "engineer", adapterType: "process" }).returning(); + const wrongAgent = await run({ ...f, agentId: otherAgent.id }, alice); + for (const r of [foreignRun, wrongUser, unowned, wrongAgent]) await charge(f, { heartbeatRunId: r.id }); + await charge(foreign, { heartbeatRunId: foreignRun.id, costCents: 999 }); + const report = await costService(db).byUser(f.companyId, { allTime: true }); + expect(report.rows.find(row => row.userId === null)).toMatchObject({ costCentsExact: "4.0000000", eventCount: 4, runCount: 2 }); + expect(report.rows.find(row => row.userId === alice)?.costCents).toBe(0); + expect(JSON.stringify(report)).not.toContain(outsider); + expect(JSON.stringify(report)).not.toContain("Secret foreign name"); + }); + + it("filters receipt dates inclusively and never reassigns history when the issue owner changes", async () => { + const f = await fixture(); + const alice = await member(f.companyId, "Alice"), bob = await member(f.companyId, "Bob"); + const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Shared work", responsibleUserId: alice }).returning(); + const r = await run(f, alice); + const from = new Date("2001-01-01T00:00:00Z"), to = new Date("2001-01-02T00:00:00Z"); + for (const occurredAt of [new Date(from.getTime() - 1), from, to, new Date(to.getTime() + 1)]) { + await charge(f, { heartbeatRunId: r.id, issueId: issue.id, occurredAt }); + } + await db.update(issues).set({ responsibleUserId: bob }).where(eq(issues.id, issue.id)); + const service = costService(db); + expect((await service.byUser(f.companyId, { from, to })).rows.find(row => row.userId === alice)).toMatchObject({ costCents: 2, runCount: 1 }); + expect((await service.byUser(f.companyId, { allTime: true })).rows.find(row => row.userId === alice)?.costCents).toBe(4); + expect((await service.byUser(f.companyId)).rows.every(row => row.costCents === 0)).toBe(true); + await charge(f, { heartbeatRunId: r.id }); + expect((await service.byUser(f.companyId)).rows.find(row => row.userId === alice)?.costCents).toBe(1); + await expect(service.byUser(randomUUID())).rejects.toThrow("Company not found"); + }); +}); diff --git a/server/src/__tests__/cost-accounting-checkpoints.test.ts b/server/src/__tests__/cost-accounting-checkpoints.test.ts new file mode 100644 index 0000000000..01eb99e441 --- /dev/null +++ b/server/src/__tests__/cost-accounting-checkpoints.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint"; +import { parseClaudeStreamJson } from "../../../packages/adapters/claude-local/src/server/parse.js"; +import { parseCodexJsonl } from "../../../packages/adapters/codex-local/src/server/parse.js"; +import { parseCursorJsonl } from "../../../packages/adapters/cursor-local/src/server/parse.js"; +import { parseGeminiJsonl } from "../../../packages/adapters/gemini-local/src/server/parse.js"; +import { parseOpenCodeJsonl } from "../../../packages/adapters/opencode-local/src/server/parse.js"; +import { parsePiJsonl } from "../../../packages/adapters/pi-local/src/server/parse.js"; +const secret = "TRANSCRIPT_CONTENT_MUST_NOT_ENTER_ACCOUNTING_BUFFER"; +const fixtures = [ + { name: "claude", event: { type: "result", total_cost_usd: 0.0123, result: secret, usage: { input_tokens: 7, cache_read_input_tokens: 11, output_tokens: 3 } }, parse: parseClaudeStreamJson, input: 7, cached: 11, output: 3 }, + { name: "codex", event: { type: "turn.completed", text: secret, usage: { input_tokens: 18, cached_input_tokens: 11, output_tokens: 3 } }, parse: parseCodexJsonl, input: 7, cached: 11, output: 3 }, + { name: "cursor", event: { type: "result", result: secret, usage: { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 } }, parse: parseCursorJsonl, input: 7, cached: 11, output: 3 }, + { name: "gemini", event: { type: "result", message: secret, stats: { input_tokens: 18, cached: 11, output_tokens: 1, total_tokens: 21 } }, parse: parseGeminiJsonl, input: 7, cached: 11, output: 3 }, + { name: "opencode", event: { type: "step_finish", part: { text: secret, cost: 0.0123, tokens: { input: 5, cache: { read: 11, write: 2 }, output: 1, reasoning: 2 } } }, parse: parseOpenCodeJsonl, input: 7, cached: 11, output: 3 }, + { name: "pi", event: { type: "turn_end", message: { role: "assistant", content: secret, usage: { input: 5, cacheWrite: 2, cacheRead: 11, output: 3, cost: { total: 0.0123 } } } }, parse: parsePiJsonl, input: 7, cached: 11, output: 3 }, +]; +describe("durable accounting stream projection", () => { + it.each(fixtures)("captures split $name usage without retaining transcript content", async f => { + const onUsage = vi.fn(async (..._args: any[]) => {}), onLog = vi.fn(async (..._args: any[]) => {}); + const parse = vi.fn((source: string) => { + expect(source).not.toContain(secret); + const result = f.parse(source); return { usage: result.usage ?? undefined, complete: true }; + }); + const log = createUsageCheckpointLog(onLog, onUsage, parse); + const line = JSON.stringify(f.event) + "\n"; + for (let index = 0; index < line.length; index += 7) await log("stdout", line.slice(index,index+7)); + expect(onUsage).toHaveBeenCalledTimes(1); + expect(onUsage.mock.calls[0][0]).toMatchObject({ complete: true, attemptId: expect.any(String), usage: { inputTokens: f.input, cachedInputTokens: f.cached, outputTokens: f.output } }); + expect(onLog.mock.calls.map(call => call[1]).join("")).toBe(line); + }); + it("awaits durability before publication, suppresses identical snapshots and separates attempts", async () => { + const order: string[] = []; + const onUsage = vi.fn(async (..._args: any[]) => { order.push("durable"); }); + const parse = () => ({ usage: { inputTokens: 1, outputTokens: 2 }, complete: false }); + const log = createUsageCheckpointLog(async () => { order.push("published"); }, onUsage, parse); + const message = '{"type":"result","usage":{"input_tokens":1}}\n'; + await log("stdout", message); await log("stdout", message); + expect(order).toEqual(["durable", "published", "published"]); + const other = createUsageCheckpointLog(async () => {}, onUsage, parse); await other("stdout", message); + expect(onUsage.mock.calls[0][0].attemptId).not.toBe(onUsage.mock.calls[1][0].attemptId); + }); + it("propagates receipt failures and leaves ordinary logs and stderr alone when disabled", async () => { + const onLog = vi.fn(async (..._args: any[]) => {}); + const disabled = createUsageCheckpointLog(onLog, undefined, () => null); + await disabled("stdout", "ordinary output"); await disabled.flush(); + expect(onLog).toHaveBeenCalledWith("stdout", "ordinary output"); + onLog.mockClear(); + const onUsage = vi.fn(async () => { throw new Error("spool full"); }); + const parse = vi.fn(() => ({ usage: { inputTokens: 1, outputTokens: 0 }, complete: false })); + const log = createUsageCheckpointLog(onLog, onUsage, parse); + await log("stderr", '{"usage":1}\n'); await log("stdout", 'invalid json\n{"text":"plain"}\n'); + expect(onUsage).not.toHaveBeenCalled(); onLog.mockClear(); + await log("stdout", '{"usage":{"input_tokens":1}}\n'); + expect(onLog).toHaveBeenCalled(); + await expect(log.flush()).rejects.toThrow("spool full"); + const oversized = createUsageCheckpointLog(onLog, onUsage, parse); + await oversized("stdout", "x".repeat(8*1024*1024+1)); + await expect(oversized.flush()).rejects.toThrow("exceeds"); + }); +}); diff --git a/server/src/__tests__/cost-accounting-completion.test.ts b/server/src/__tests__/cost-accounting-completion.test.ts new file mode 100644 index 0000000000..7d8eab861d --- /dev/null +++ b/server/src/__tests__/cost-accounting-completion.test.ts @@ -0,0 +1,764 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; +import { eq } from "drizzle-orm"; +import { + agents, + companies, + companySecrets, + costEvents, + financeEvents, + heartbeatRuns, + providerBillingSnapshots, + createDb, +} from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { billingReconciliationService } from "../services/billing-reconciliation.js"; +import { financeService } from "../services/finance.js"; +import { + applyProviderDailyCosts, + importProviderDailyCosts, +} from "../services/provider-billing-import.js"; +import * as secretsModule from "../services/secrets.js"; +import { + createRunUsageRecorder, + persistUsageReceipt, +} from "../services/usage-receipts.js"; +import { accountRunCost } from "../services/run-cost-accounting.js"; +import { costService, getMonthlySpendTotal } from "../services/costs.js"; +import { budgetService } from "../services/budgets.js"; +import type { ImportProviderCosts } from "@paperclipai/shared"; + +describe("completed cost reporting paths (PostgreSQL)", () => { + let database: Awaited>; + let db: ReturnType; + let spool: string; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase( + "accounting-completion-", + ); + db = createDb(database.connectionString); + spool = await mkdtemp(join(tmpdir(), "completion-spool-")); + }, 90000); + afterAll(async () => { + await database?.cleanup(); + if (spool) await rm(spool, { recursive: true, force: true }); + }); + afterEach(() => vi.restoreAllMocks()); + const company = async () => + ( + await db + .insert(companies) + .values({ + name: "Completion", + issuePrefix: `C${randomUUID().slice(0, 7)}`, + }) + .returning() + )[0]; + it("reads monthly spend from the ledger with company, agent and UTC month boundaries", async () => { + const c = await company(); + const other = await company(); + const [first, second] = await db.insert(agents).values([ + { companyId: c.id, name: "First", spentMonthlyCents: 9999 }, + { companyId: c.id, name: "Second", spentMonthlyCents: 9999 }, + ]).returning(); + const [outsider] = await db.insert(agents).values({ companyId: other.id, name: "Other" }).returning(); + const now = new Date(); + const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)); + const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1)); + const base = { companyId: c.id, agentId: first.id, provider: "test", model: "test", costCents: 1.125, occurredAt: start }; + await db.insert(costEvents).values([ + base, + { ...base, agentId: second.id, costCents: 2.25, occurredAt: new Date(end.getTime() - 1) }, + { ...base, costCents: 500, occurredAt: new Date(start.getTime() - 1) }, + { ...base, costCents: 600, occurredAt: end }, + { ...base, companyId: other.id, agentId: outsider.id, costCents: 700 }, + ]); + expect(await getMonthlySpendTotal(db, { companyId: c.id })).toBe(3.375); + expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: first.id })).toBe(1.125); + expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: second.id })).toBe(2.25); + expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: outsider.id })).toBe(0); + expect(await getMonthlySpendTotal(db, { companyId: randomUUID() })).toBe(0); + }); + + it("scopes agent and model estimate counts to the company and selected period", async () => { + const c = await company(); + const other = await company(); + const [codie, bender] = await db.insert(agents).values([ + { companyId: c.id, name: "Codie", adapterType: "paperclip_runner" }, + { companyId: c.id, name: "Bender", adapterType: "claude_local" }, + ]).returning(); + const [outsider] = await db.insert(agents).values({ companyId: other.id, name: "Other", adapterType: "process" }).returning(); + const base = { companyId: c.id, agentId: codie.id, provider: "openai", biller: "openai", billingType: "metered_api", model: "gpt-6-astra", costCents: 100, occurredAt: new Date("2026-09-10T12:00:00Z") }; + await db.insert(costEvents).values([ + { ...base, costStatus: "estimated" }, + { ...base, costStatus: "estimated" }, + { ...base, model: "gpt-6-sol", costStatus: "reported" }, + { ...base, costStatus: "reported", occurredAt: new Date("2026-08-31T23:59:59Z") }, + { ...base, costStatus: "estimated", occurredAt: new Date("2026-10-01T00:00:00Z") }, + { ...base, agentId: bender.id, costStatus: "reported" }, + { ...base, companyId: other.id, agentId: outsider.id, costStatus: "estimated" }, + ]); + const range = { from: new Date("2026-09-01T00:00:00Z"), to: new Date("2026-09-30T23:59:59Z") }; + const service = costService(db); + const totals = await service.byAgent(c.id, range); + expect(totals).toHaveLength(2); + expect(totals.find(row => row.agentId === codie.id)).toMatchObject({ costCents: 300, eventCount: 3, estimatedEventCount: 2, apiRunCount: 0 }); + expect(totals.find(row => row.agentId === bender.id)).toMatchObject({ eventCount: 1, estimatedEventCount: 0 }); + const models = (await service.byAgentModel(c.id, range)).filter(row => row.agentId === codie.id); + expect(models).toHaveLength(2); + expect(models.find(row => row.model === "gpt-6-astra")).toMatchObject({ eventCount: 2, estimatedEventCount: 2 }); + expect(models.find(row => row.model === "gpt-6-sol")).toMatchObject({ eventCount: 1, estimatedEventCount: 0 }); + }); + it("imports inference, fees and credits atomically into the timeline, without duplicating on retry", async () => { + const c = await company(); + const service = billingReconciliationService(db); + const invoice = { + biller: "openai", + externalId: "invoice-1", + currency: "USD", + lines: [ + { + externalId: "usage", + kind: "inference" as const, + amountCents: "10.0000001", + occurredAt: "2026-09-01T00:00:00Z", + }, + { + externalId: "subscription", + kind: "fee" as const, + amountCents: "2000", + occurredAt: "2026-09-01T00:00:00Z", + }, + { + externalId: "refund", + kind: "credit" as const, + amountCents: "100", + occurredAt: "2026-09-01T00:00:00Z", + }, + ], + }; + const imports = await Promise.all([ + service.importInvoice(c.id, invoice, "board"), + service.importInvoice(c.id, invoice, "board"), + ]); + expect(imports[0].id).toBe(imports[1].id); + expect( + await financeService(db).list(c.id, { + from: new Date("2026-09-01"), + to: new Date("2026-09-30"), + }), + ).toHaveLength(3); + expect( + ( + await financeService(db).summary(c.id, { + from: new Date("2026-09-01"), + to: new Date("2026-09-30"), + }) + ).netCentsExact, + ).toBe("1910.0000001"); + expect( + await db.select().from(costEvents).where(eq(costEvents.companyId, c.id)), + ).toHaveLength(0); + await expect( + service.importInvoice( + c.id, + { ...invoice, lines: [{ ...invoice.lines[0], amountCents: "99" }] }, + "board", + ), + ).rejects.toThrow("different contents"); + expect( + await db + .select() + .from(financeEvents) + .where(eq(financeEvents.companyId, c.id)), + ).toHaveLength(3); + }); + it.each(["costEventId", "runId", "providerRequestId"] as const)("keeps overlapping invoice evidence out of Finance totals using %s", async (reference) => { + const c = await company(); + const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Browser worker" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: c.id, agentId: agent.id, invocationSource: "on_demand" }).returning(); + const charge = await costService(db).createEvent(c.id, { agentId: agent.id, heartbeatRunId: run.id, + provider: "browser-use-cloud", model: "test", providerRequestId: "request-1", costCents: "15.0000001", occurredAt: new Date() }); + const finance = financeService(db); + await finance.createEvent(c.id, { costEventId: charge.id, heartbeatRunId: run.id, + biller: "browser-use-cloud", eventKind: "inference_charge", amountCents: "15.0000001", occurredAt: new Date() }); + const service = billingReconciliationService(db); + const linked = { [reference]: reference === "costEventId" ? charge.id : reference === "runId" ? run.id : "request-1" }; + const invoice = { biller: "browser-use-cloud", externalId: "invoice-1", currency: "USD", lines: [ + { externalId: "usage", kind: "inference" as const, amountCents: "15.0000001", occurredAt: new Date().toISOString(), ...linked }, + ] }; + const imported = await service.importInvoice(c.id, invoice, "board"); + expect((await service.reconcile(c.id, imported.id)).lines[0]).toMatchObject({ status: "matched", matchedEventId: charge.id }); + await service.importInvoice(c.id, invoice, "board"); + // A second export, even with a revised price, is evidence for review and + // never a second debit for the same already-recorded purchase. + const revised = await service.importInvoice(c.id, { ...invoice, externalId: "invoice-2", lines: [ + { ...invoice.lines[0], amountCents: "16" }, + ] }, "board"); + expect((await service.reconcile(c.id, revised.id)).lines[0].status).toBe("difference"); + expect(await finance.summary(c.id)).toMatchObject({ netCentsExact: "15.0000001", eventCount: 1 }); + expect(await finance.list(c.id)).toHaveLength(1); + }); + + it("links a uniquely matched invoice debit and serializes overlapping exports", async () => { + const c = await company(); + const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning(); + const charge = await costService(db).createEvent(c.id, { agentId: agent.id, + provider: "test", model: "test", providerRequestId: "request", costCents: 15, occurredAt: new Date() }); + const service = billingReconciliationService(db); + await Promise.all(["first-export", "second-export"].map(externalId => service.importInvoice(c.id, { + biller: "test", externalId, currency: "USD", lines: [{ externalId: "usage", amountCents: "15", + providerRequestId: "request", model: "test", occurredAt: new Date().toISOString() }], + }, "board"))); + const rows = await financeService(db).list(c.id); + expect(rows).toHaveLength(1); + expect(rows[0].costEventId).toBe(charge.id); + expect((await financeService(db).summary(c.id)).netCentsExact).toBe("15.0000000"); + }); + + it.each(["fee", "credit", "other currency", "other biller", "prior credit", "provider report"] as const)("preserves distinct invoice entries: %s", async (scenario) => { + const c = await company(); + const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning(); + const charge = await costService(db).createEvent(c.id, { agentId: agent.id, + provider: "test", model: "test", costCents: 15, occurredAt: new Date() }); + await financeService(db).createEvent(c.id, { costEventId: charge.id, biller: "test", + eventKind: "inference_charge", direction: scenario === "prior credit" ? "credit" : "debit", + amountCents: 15, occurredAt: new Date(), + metadataJson: scenario === "provider report" ? { source: "provider_cost_report" } : undefined }); + const imported = await billingReconciliationService(db).importInvoice(c.id, { + biller: scenario === "other biller" ? "other" : "test", externalId: "invoice", + currency: scenario === "other currency" ? "EUR" : "USD", + lines: [{ externalId: "line", costEventId: charge.id, amountCents: "2", + kind: scenario === "fee" || scenario === "credit" ? scenario : "inference", occurredAt: new Date().toISOString() }], + }, "board"); + const rows = await financeService(db).list(c.id); + expect(rows).toHaveLength(2); + expect(rows.find(row => row.metadataJson?.invoiceId === imported.id)).toMatchObject({ + amountCentsExact: "2.0000000", currency: scenario === "other currency" ? "EUR" : "USD", + direction: scenario === "credit" ? "credit" : "debit", + }); + }); + + it("retains ambiguous run evidence without guessing an additional debit", async () => { + const c = await company(); + const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: c.id, agentId: agent.id, invocationSource: "on_demand" }).returning(); + for (const costCents of [5, 10]) await costService(db).createEvent(c.id, { + agentId: agent.id, heartbeatRunId: run.id, provider: "test", model: "test", costCents, occurredAt: new Date(), + }); + const service = billingReconciliationService(db); + const imported = await service.importInvoice(c.id, { biller: "test", externalId: "invoice", currency: "USD", + lines: [{ externalId: "usage", runId: run.id, amountCents: "15", occurredAt: new Date().toISOString() }], + }, "board"); + expect((await service.reconcile(c.id, imported.id)).lines[0].status).toBe("ambiguous"); + expect(await financeService(db).list(c.id)).toHaveLength(0); + expect((await costService(db).summary(c.id)).spendCentsExact).toBe("15.0000000"); + }); + + it("does not select a price when multiple invoice lines claim the same charge", async () => { + const c = await company(); + const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning(); + const charge = await costService(db).createEvent(c.id, { agentId: agent.id, + provider: "test", model: "test", costCents: 15, occurredAt: new Date() }); + const service = billingReconciliationService(db); + const imported = await service.importInvoice(c.id, { biller: "test", externalId: "invoice", currency: "USD", + lines: [15, 16].map(amount => ({ externalId: `line-${amount}`, costEventId: charge.id, + amountCents: String(amount), occurredAt: new Date().toISOString() })), + }, "board"); + expect((await service.reconcile(c.id, imported.id)).lines.map(line => line.status)).toEqual(["ambiguous", "ambiguous"]); + expect(await financeService(db).list(c.id)).toHaveLength(0); + }); + + it("records provider report corrections as deltas, including decreases and returns to an earlier total", async () => { + const c = await company(); + const input: ImportProviderCosts = { + provider: "openai", + accountId: "org_test", + secretId: randomUUID(), + scopeIds: ["project"], + from: "2026-09-01", + to: "2026-09-02", + }; + const update = (amountCents: string) => + applyProviderDailyCosts( + db, + c.id, + input, + [{ day: "2026-09-01", scopeId: "project", amountCents }], + "board", + ); + await update("100"); + await update("100"); + await update("120"); + await update("90"); + await update("100"); + const rows = await financeService(db).list(c.id, { + from: new Date("2026-09-01"), + to: new Date("2026-09-30"), + }); + expect(rows).toHaveLength(4); + expect(rows.filter((row) => row.direction === "credit")).toHaveLength(1); + const summary = await financeService(db).summary(c.id, { allTime: true }); + expect(summary).toMatchObject({ + netCentsExact: "0.0000000", + providerReportedCentsExact: "100.0000000", + }); + expect(await financeService(db).byBiller(c.id, { allTime: true })).toEqual( + [], + ); + expect(await financeService(db).byKind(c.id, { allTime: true })).toEqual( + [], + ); + const [snapshot] = await db + .select() + .from(providerBillingSnapshots) + .where(eq(providerBillingSnapshots.companyId, c.id)); + expect(snapshot.amountCents).toBe("100.0000000"); + expect(snapshot.revision).toBe(4); + await expect( + applyProviderDailyCosts( + db, + c.id, + input, + [{ day: "2026-09-01", scopeId: "project", amountCents: "80" }], + "board", + new Map([["2026-09-01:project", 0]]), + ), + ).rejects.toThrow("newer provider report"); + const other = await company(); + await applyProviderDailyCosts( + db, + other.id, + input, + [{ day: "2026-09-01", scopeId: "project", amountCents: "1" }], + "board", + ); + expect( + await db + .select() + .from(financeEvents) + .where(eq(financeEvents.companyId, c.id)), + ).toHaveLength(4); + }); + it.each([ + null, + {}, + { providerBilling: "openai" }, + { providerBilling: { provider: "anthropic", accountId: "org_test" } }, + { providerBilling: { provider: "openai", accountId: "org_other" } }, + { providerBilling: { provider: "openai" } }, + ])("never reads or transmits a secret without the matching billing designation: %j", async (providerMetadata) => { + const c = await company(); + const [secret] = await db.insert(companySecrets).values({ + companyId: c.id, key: "unrelated", name: "Unrelated credential", providerMetadata, + }).returning(); + const resolveSecretValue = vi.fn().mockResolvedValue("unrelated-private-key"); + vi.spyOn(secretsModule, "secretService").mockReturnValue({ resolveSecretValue } as unknown as ReturnType); + const fetcher = vi.fn().mockResolvedValue(Response.json({ data: [], has_more: false })); + await expect(importProviderDailyCosts(db, c.id, { + provider: "openai", accountId: "org_test", secretId: secret.id, + scopeIds: ["project"], from: "2026-09-01", to: "2026-09-02", + }, "board", fetcher)).rejects.toThrow("designated for this billing provider and account"); + expect(resolveSecretValue).not.toHaveBeenCalled(); + expect(fetcher).not.toHaveBeenCalled(); + expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, c.id))).toEqual([]); + }); + it("fetches with a company credential and keeps provider reports separate from invoice charges", async () => { + const c = await company(); + const [secret] = await db + .insert(companySecrets) + .values({ companyId: c.id, key: "billing", name: "Billing admin", providerMetadata: { providerBilling: { provider: "openai", accountId: "org_test" } } }) + .returning(); + const resolveSecretValue = vi.fn().mockResolvedValue("fixture-admin-key"); + vi.spyOn(secretsModule, "secretService").mockReturnValue({ + resolveSecretValue, + } as unknown as ReturnType); + const fetcher = vi.fn().mockImplementation(async () => + Response.json({ + data: [ + { + start_time: Date.parse("2026-09-01") / 1000, + end_time: Date.parse("2026-09-02") / 1000, + results: [ + { + project_id: "project", + amount: { value: 0.06, currency: "usd" }, + }, + ], + }, + ], + has_more: false, + }), + ); + const input: ImportProviderCosts = { + provider: "openai", + accountId: "org_test", + secretId: secret.id, + scopeIds: ["project"], + from: "2026-09-01", + to: "2026-09-02", + }; + expect( + await importProviderDailyCosts(db, c.id, input, "board", fetcher), + ).toEqual({ daysRead: 1, eventsCreated: 1 }); + expect(resolveSecretValue).toHaveBeenCalledWith( + c.id, + secret.id, + "latest", + expect.objectContaining({ + accessContext: expect.objectContaining({ + actorId: "board", + consumerId: "provider-billing-import", + }), + }), + ); + expect(fetcher.mock.calls[0][1]?.headers).toMatchObject({ + Authorization: "Bearer fixture-admin-key", + "OpenAI-Organization": "org_test", + }); + expect( + await importProviderDailyCosts(db, c.id, input, "board", fetcher), + ).toEqual({ daysRead: 1, eventsCreated: 0 }); + await billingReconciliationService(db).importInvoice( + c.id, + { + biller: "openai", + externalId: "separate-invoice", + currency: "USD", + lines: [ + { + externalId: "usage", + amountCents: "6", + kind: "inference", + occurredAt: "2026-09-01T00:00:00Z", + }, + ], + }, + "board", + ); + expect( + await financeService(db).summary(c.id, { allTime: true }), + ).toMatchObject({ + netCentsExact: "6.0000000", + providerReportedCentsExact: "6.0000000", + eventCount: 2, + }); + expect(await financeService(db).byBiller(c.id, { allTime: true })).toEqual([ + expect.objectContaining({ + biller: "openai", + netCentsExact: "6.0000000", + eventCount: 1, + }), + ]); + const other = await company(); + await expect( + importProviderDailyCosts(db, other.id, input, "board", fetcher), + ).rejects.toThrow("company billing credential"); + expect(resolveSecretValue).toHaveBeenCalledTimes(2); + }); + it("rejects a slow report when another importer changes the same provider snapshot", async () => { + const c = await company(); + const [secret] = await db + .insert(companySecrets) + .values({ companyId: c.id, key: "billing-race", name: "Billing admin", providerMetadata: { providerBilling: { provider: "openai", accountId: "org_test" } } }) + .returning(); + vi.spyOn(secretsModule, "secretService").mockReturnValue({ + resolveSecretValue: vi.fn().mockResolvedValue("fixture-admin-key"), + } as unknown as ReturnType); + const input: ImportProviderCosts = { + provider: "openai", + accountId: "org_test", + secretId: secret.id, + scopeIds: ["project"], + from: "2026-09-01", + to: "2026-09-02", + }; + const report = (value: number) => + Response.json({ + data: [ + { + start_time: Date.parse(input.from) / 1000, + end_time: Date.parse(input.to) / 1000, + results: [ + { project_id: "project", amount: { currency: "usd", value } }, + ], + }, + ], + has_more: false, + }); + let release!: (response: Response) => void; + let markStarted!: () => void; + const slowResponse = new Promise((resolve) => { + release = resolve; + }); + const started = new Promise((resolve) => { + markStarted = resolve; + }); + const slowImport = importProviderDailyCosts( + db, + c.id, + input, + "board", + async () => { + markStarted(); + return slowResponse; + }, + ); + await started; + await importProviderDailyCosts(db, c.id, input, "board", async () => + report(2), + ); + release(report(1)); + await expect(slowImport).rejects.toThrow("newer provider report"); + expect( + await financeService(db).summary(c.id, { allTime: true }), + ).toMatchObject({ + providerReportedCentsExact: "200.0000000", + eventCount: 1, + }); + }); + it("rolls back an entire provider import when a later snapshot is invalid", async () => { + const c = await company(); + const input: ImportProviderCosts = { + provider: "openai", + accountId: "org_test", + secretId: randomUUID(), + scopeIds: ["project"], + from: "2026-09-01", + to: "2026-09-03", + }; + await expect( + applyProviderDailyCosts( + db, + c.id, + input, + [ + { day: "2026-09-01", scopeId: "project", amountCents: "5" }, + { day: "2026-09-02", scopeId: "project", amountCents: "-1" }, + ], + "board", + ), + ).rejects.toThrow(); + expect( + await db + .select() + .from(providerBillingSnapshots) + .where(eq(providerBillingSnapshots.companyId, c.id)), + ).toEqual([]); + expect( + await db + .select() + .from(financeEvents) + .where(eq(financeEvents.companyId, c.id)), + ).toEqual([]); + }); + it("stores an explicit zero report and credits an amended report down to zero only once", async () => { + const c = await company(); + const input: ImportProviderCosts = { + provider: "anthropic", + accountId: "org_test", + secretId: randomUUID(), + scopeIds: ["default"], + from: "2026-09-01", + to: "2026-09-02", + }; + const update = (amountCents: string) => + applyProviderDailyCosts( + db, + c.id, + input, + [{ day: input.from, scopeId: "default", amountCents }], + "board", + ); + expect((await update("0")).eventsCreated).toBe(0); + expect((await update("0")).eventsCreated).toBe(0); + expect((await update("1.0000001")).eventsCreated).toBe(1); + expect((await update("0")).eventsCreated).toBe(1); + expect((await update("0")).eventsCreated).toBe(0); + expect( + await financeService(db).summary(c.id, { allTime: true }), + ).toMatchObject({ + providerReportedCentsExact: "0.0000000", + netCentsExact: "0.0000000", + eventCount: 2, + }); + }); + it("freezes estimated dollars before finalization and persists them exactly once into costs and budgets", async () => { + const c = await company(); + const budgets = budgetService(db); + await budgets.upsertPolicy(c.id, { scopeType: "company", scopeId: c.id, amount: 100 }, "board"); + const [agent] = await db + .insert(agents) + .values({ + companyId: c.id, + name: "Codie", + adapterType: "paperclip_runner", + }) + .returning(); + const [run] = await db + .insert(heartbeatRuns) + .values({ + companyId: c.id, + agentId: agent.id, + invocationSource: "on_demand", + status: "running", + }) + .returning(); + const recorder = await createRunUsageRecorder( + db, + { companyId: c.id, runId: run.id, adapterType: "paperclip_runner" }, + spool, + ); + const captured = await recorder.capture({ + provider: "openai", + biller: "openai", + billingType: "metered_api", + model: "gpt-6-astra", + usageBasis: "per_run", + complete: true, + costUsd: null, + usage: { + inputTokens: 123536, + cachedInputTokens: 1567209, + outputTokens: 6625, + }, + }); + expect(captured.costUsdExact).toBe("3.133819000"); + await db + .update(heartbeatRuns) + .set({ status: "failed", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, run.id)); + await accountRunCost(db, run.id); + await accountRunCost(db, run.id); + const [event] = await db + .select() + .from(costEvents) + .where(eq(costEvents.heartbeatRunId, run.id)); + expect(event.costStatus).toBe("estimated"); + expect(event.pricingProvenance).toMatchObject({ + source: "rate_card", + version: "openai-standard-2026-09-30", + }); + const summary = await costService(db).summary(c.id); + expect(summary.spendCentsExact).toBe("313.3819000"); + expect(summary.estimatedEventCount).toBe(1); + expect(await budgets.getInvocationBlock(c.id, agent.id)).not.toBeNull(); + }); + it("preserves cache-write evidence and pricing across multiple provider attempts", async () => { + const c = await company(); + const [agent] = await db + .insert(agents) + .values({ + companyId: c.id, + name: "Retry", + adapterType: "paperclip_runner", + }) + .returning(); + const [run] = await db + .insert(heartbeatRuns) + .values({ companyId: c.id, agentId: agent.id, status: "running" }) + .returning(); + const recorder = await createRunUsageRecorder( + db, + { companyId: c.id, runId: run.id, adapterType: "paperclip_runner" }, + spool, + ); + await recorder.capture({ + attemptId: randomUUID(), + complete: true, + provider: "openai", + biller: "openai", + billingType: "api", + model: "gpt-6-astra", + usageBasis: "per_run", + usage: { inputTokens: 10, cacheWriteTokens: 4, outputTokens: 2 }, + }); + const result = await recorder.capture({ + attemptId: randomUUID(), + complete: true, + provider: "openai", + biller: "openai", + billingType: "api", + costUsd: 2, + }); + expect(result).toMatchObject({ + complete: true, + costUsdExact: "2.000210000", + costStatus: "estimated", + pricingProvenance: { source: "rate_card", version: "per-attempt/v1" }, + usage: { + inputTokens: 10, + cacheWriteTokens: 4, + cachedInputTokens: 0, + outputTokens: 2, + }, + }); + await db + .update(heartbeatRuns) + .set({ status: "succeeded", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, run.id)); + await accountRunCost(db, run.id); + expect(await costService(db).summary(c.id)).toMatchObject({ + spendCentsExact: "200.0210000", + estimatedEventCount: 1, + }); + }); + it("recovers older durable receipts without pricing metadata while preserving explicit zero", async () => { + const c = await company(); + const sourceId = randomUUID(); + const [agent] = await db + .insert(agents) + .values({ + companyId: c.id, + name: "Legacy receipt", + adapterType: "process", + }) + .returning(); + const [run] = await db + .insert(heartbeatRuns) + .values({ + companyId: c.id, + agentId: agent.id, + status: "running", + usageJson: { accountingReceiptSourceId: sourceId }, + }) + .returning(); + const cases = [ + { costUsd: null }, + { costUsd: 0 }, + { costUsdExact: "0.000000001" }, + ]; + for (const [index, receipt] of cases.entries()) { + await persistUsageReceipt(db, { + schema: "paperclip/accounting-receipt/v1", + id: randomUUID(), + companyId: c.id, + runId: run.id, + adapterType: "process", + sourceId, + sequence: index + 1, + receivedAt: new Date().toISOString(), + receipt: { ...receipt, complete: true }, + }); + const [stored] = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, run.id)); + expect(stored.usageJson?.pricingProvenance).toEqual({ + source: index === 0 ? "unknown" : "provider_reported", + version: "accounting-receipt/v1", + }); + } + }); +}); diff --git a/server/src/__tests__/cost-accounting-concurrency.test.ts b/server/src/__tests__/cost-accounting-concurrency.test.ts new file mode 100644 index 0000000000..c4ccb2c577 --- /dev/null +++ b/server/src/__tests__/cost-accounting-concurrency.test.ts @@ -0,0 +1,113 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { and, eq } from "drizzle-orm"; +import { agents, agentRuntimeState, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, projects } from "@paperclipai/db"; +import { costService } from "../services/costs.js"; +import { budgetService, type BudgetServiceHooks } from "../services/budgets.js"; +import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +function random(seed: number) { + return () => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed / 2 ** 32; }; +} +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("seeded concurrent accounting invariants", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-interleavings-"); db = createDb(database.connectionString); }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + + it.each([1, 7, 42, 97, 31337, 65537, 104729, 0xdeadbeef])("conserves the ledger, totals, incidents and delivery state for seed %s", async (seed) => { + const rng = random(seed); + const [company] = await db.insert(companies).values({ name: `Interleaving ${seed}`, issuePrefix: `R${randomUUID().slice(0,7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "idle", adapterType: "process" }).returning(); + const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project" }).returning(); + const oracle = new Map(); + const operations: Array<() => Promise> = []; + let cancellationAttempts = 0; + let successfulDeliveries = 0; + const hooks: BudgetServiceHooks = { cancelWorkForScope: async (scope) => { + expect(scope.companyId).toBe(company.id); + expect(scope.createdBefore).toBeInstanceOf(Date); + cancellationAttempts++; + if (cancellationAttempts % 4 === 1) throw new Error(`Injected delivery interruption (seed ${seed})`); + successfulDeliveries++; + } }; + const budgets = budgetService(db, hooks); + const costs = costService(db, hooks); + const now = new Date(); + for (let index = 0; index < 12; index++) { + const ticks = Math.floor(rng() * 9_000_000) + 1; + const input = Math.floor(rng() * 300), cached = Math.floor(rng() * 500), output = Math.floor(rng() * 80); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + status: ["succeeded", "failed", "cancelled", "timed_out"][index % 4], finishedAt: now, costAccountingPending: true, + usageJson: { accountingReceiptReady: true, provider: "fixture", model: `model-${index % 3}`, billingType: "metered_api", inputTokens: input, cachedInputTokens: cached, outputTokens: output, costUsd: ticks / 1e9, ledgerScope: { projectId: project.id } }, + }).returning(); + oracle.set(`heartbeat:${run.id}:final`, { ticks: BigInt(ticks), input, cached, output, run: true }); + operations.push(() => accountRunCost(db, run.id, hooks), () => accountRunCost(db, run.id, hooks)); + } + for (let index = 0; index < 10; index++) { + const ticks = Math.floor(rng() * 3_000_000) + 1; + const key = `manual-${index}`; + const receipt = { idempotencyKey: key, agentId: agent.id, projectId: project.id, provider: "fixture", model: "manual", billingType: "metered_api", costCents: ticks / 1e7, occurredAt: now }; + oracle.set(key, { ticks: BigInt(ticks), input: 0, cached: 0, output: 0, run: false }); + operations.push(() => costs.createEvent(company.id, receipt), () => costs.createEvent(company.id, receipt)); + } + for (let index = 0; index < 12; index++) { + const amount = rng() < 0.5 ? 1 : 200; + const scopeType = index % 2 ? "agent" as const : "project" as const; + operations.push(() => budgets.upsertPolicy(company.id, { scopeType, scopeId: scopeType === "agent" ? agent.id : project.id, amount, notifyEnabled: false }, "board")); + operations.push(() => budgets.getInvocationBlock(company.id, agent.id, { projectId: project.id })); + operations.push(() => reconcileRunCosts(db, hooks)); + } + // Reproducible submission order and jitter; the database is free to choose + // the actual interleaving. The oracle is independent of execution order. + for (let i = operations.length - 1; i > 0; i--) { + const j = Math.floor(rng() * (i + 1)); [operations[i], operations[j]] = [operations[j], operations[i]]; + } + for (let offset = 0; offset < operations.length; offset += 8) { + await Promise.all(operations.slice(offset, offset + 8).map(async operation => { + await new Promise(resolve => setTimeout(resolve, Math.floor(rng() * 5))); + return operation(); + })); + const incidents = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, company.id), eq(budgetIncidents.status, "open"))); + expect(new Set(incidents.map(row => `${row.policyId}:${row.windowStart.toISOString()}:${row.thresholdType}`)).size).toBe(incidents.length); + } + await reconcileRunCosts(db, hooks); + const rows = await db.select().from(costEvents).where(eq(costEvents.companyId, company.id)); + expect(rows).toHaveLength(oracle.size); + for (const row of rows) { + const expected = oracle.get(row.idempotencyKey!); + expect(expected, `unexpected receipt ${row.idempotencyKey}; seed ${seed}`).toBeDefined(); + expect(BigInt(Math.round(row.costCents * 1e7))).toBe(expected!.ticks); + expect([row.inputTokens, row.cachedInputTokens, row.outputTokens]).toEqual([expected!.input, expected!.cached, expected!.output]); + } + const total = [...oracle.values()].reduce((sum, row) => sum + row.ticks, 0n); + expect(BigInt(Math.round((await costs.summary(company.id)).spendCents * 1e7))).toBe(total); + expect(BigInt(Math.round((await costs.byProject(company.id))[0].costCents * 1e7))).toBe(total); + const [companyProjection] = await db.select().from(companies).where(eq(companies.id, company.id)); + const [agentProjection] = await db.select().from(agents).where(eq(agents.id, agent.id)); + expect(BigInt(Math.round(companyProjection.spentMonthlyCents * 1e7))).toBe(total); + expect(BigInt(Math.round(agentProjection.spentMonthlyCents * 1e7))).toBe(total); + const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, agent.id)); + const runs = [...oracle.values()].filter(row => row.run); + expect(BigInt(Math.round(runtime.totalCostCents * 1e7))).toBe(runs.reduce((sum, row) => sum + row.ticks, 0n)); + expect([runtime.totalInputTokens,runtime.totalCachedInputTokens,runtime.totalOutputTokens]) + .toEqual(["input","cached","output"].map(key => runs.reduce((sum,row) => sum + Number(row[key as "input" | "cached" | "output"]),0))); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, company.id))).every(run => !run.costAccountingPending && run.costAccountedAt !== null)).toBe(true); + // Finish in a known hard-stop state, deliver failed effects, then grant more + // budget. This checks the final observable state, not only receipt counts. + await budgets.upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 1, notifyEnabled: false }, "board"); + expect(await budgets.getInvocationBlock(company.id, agent.id)).toMatchObject({ scopeType: "agent" }); + for (let pass = 0; pass < 4; pass++) await budgets.deliverPendingEnforcement(company.id); + expect(successfulDeliveries).toBeGreaterThan(0); + const policies = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, company.id)); + expect(policies.every(row => row.enforcementVersion === row.enforcementDeliveredVersion)).toBe(true); + const allIncidents = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, company.id)); + const approvalRows = await db.select().from(approvals).where(eq(approvals.companyId, company.id)); + expect(new Set(allIncidents.map(row => row.approvalId).filter(Boolean)).size).toBe(approvalRows.length); + await budgets.upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 200 }, "board"); + await budgets.upsertPolicy(company.id, { scopeType: "project", scopeId: project.id, amount: 200 }, "board"); + expect(await budgets.getInvocationBlock(company.id, agent.id, { projectId: project.id })).toBeNull(); + }, 60_000); +}); diff --git a/server/src/__tests__/cost-accounting-crash.test.ts b/server/src/__tests__/cost-accounting-crash.test.ts new file mode 100644 index 0000000000..d4e029a26d --- /dev/null +++ b/server/src/__tests__/cost-accounting-crash.test.ts @@ -0,0 +1,126 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { randomUUID, randomInt } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, companies, costEvents, heartbeatRuns, budgetPolicies, createDb } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const root = fileURLToPath(new URL("../../..", import.meta.url)); +async function until(read: () => Promise, accept: (value: T) => boolean, diagnostic: () => string = () => "", timeout = 40_000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + const value = await read(); + if (accept(value)) return value; + await new Promise(resolve => setTimeout(resolve, 30)); + } + throw new Error(`Accounting process checkpoint timed out. ${diagnostic()}`); +} + +(support.supported && process.platform !== "win32" ? describe : describe.skip)("accounting survives real server SIGKILL", () => { + let database: Awaited>; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-crash-"); }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + + it.each(["before_receipt", "before_runtime_totals", "before_commit", "after_commit_before_delivery_ack"] as const)("recovers exactly once after %s", async (phase) => { + const db = createDb(database.connectionString); + const observer = createDb(database.connectionString).$client; + const blocker = await createDb(database.connectionString).$client.reserve(); + const home = await mkdtemp(path.join(tmpdir(), "pc-accounting-kill-")); + const key = randomInt(1, 2_000_000_000); + const [company] = await db.insert(companies).values({ name: "Crash accounting", issuePrefix: `K${randomUUID().slice(0,7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "idle", adapterType: "process", runtimeConfig: { heartbeat: { enabled: false } } }).returning(); + const [policy] = await db.insert(budgetPolicies).values({ companyId: company.id, scopeType: "agent", scopeId: agent.id, metric: "billed_cents", windowKind: "calendar_month_utc", amount: 1, notifyEnabled: false }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "failed", finishedAt: new Date(), costAccountingPending: true, + usageJson: { accountingReceiptReady: true, provider: "fixture", model: "fixture", billingType: "metered_api", inputTokens: 7, cachedInputTokens: 11, outputTokens: 3, costUsd: 0.0125 }, + }).returning(); + const table = phase === "before_receipt" ? "cost_events" : phase === "before_runtime_totals" ? "agent_runtime_state" + : phase === "before_commit" ? "heartbeat_runs" : "budget_policies"; + const event = phase === "after_commit_before_delivery_ack" || phase === "before_commit" ? "UPDATE" : "INSERT"; + const condition = phase === "after_commit_before_delivery_ack" ? "AND NEW.enforcement_delivered_version > OLD.enforcement_delivered_version" + : phase === "before_commit" ? "AND OLD.cost_accounting_pending = false AND NEW.cost_accounted_at IS NOT NULL" : ""; + let child: ChildProcess | undefined; + let logs = ""; + async function kill() { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const stopped = new Promise((resolve) => child!.once("exit", () => resolve())); + process.kill(-child.pid!, "SIGKILL"); + await stopped; + } + async function start() { + const port = await new Promise((resolve, reject) => { + const listener = createServer(); listener.on("error", reject); + listener.listen(0, "127.0.0.1", () => { const address = listener.address() as { port: number }; listener.close(() => resolve(address.port)); }); + }); + logs = ""; + child = spawn(process.execPath, ["cli/node_modules/tsx/dist/cli.mjs", "server/src/index.ts"], { + cwd: root, detached: true, stdio: ["ignore", "pipe", "pipe"], + env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, + NODE_ENV: "test", DATABASE_URL: database.connectionString, PORT: String(port), + PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json"), PAPERCLIP_INSTANCE_ID: "accounting-crash", + PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", PAPERCLIP_BIND: "loopback", SERVE_UI: "false", + PAPERCLIP_OPEN_ON_LISTEN: "false", PAPERCLIP_TELEMETRY_DISABLED: "1", PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false", + HEARTBEAT_SCHEDULER_ENABLED: "false", PAPERCLIP_DECISION_SIGNING_SECRET: "accounting-crash-test-decision-secret", + PAPERCLIP_AGENT_JWT_SECRET: "accounting-crash-test-agent-secret", + }, + }); + const capture = (data: Buffer) => { logs = (logs + data.toString()).slice(-16000); }; + child.stdout!.on("data", capture); child.stderr!.on("data", capture); + return `http://127.0.0.1:${port}`; + } + try { + await blocker`SELECT pg_advisory_lock(${key})`; + await observer.unsafe(`CREATE FUNCTION accounting_crash_barrier() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.company_id = '${company.id}'::uuid ${condition} THEN PERFORM pg_advisory_lock(${key}); END IF; RETURN NEW; END $$; + CREATE TRIGGER accounting_crash_barrier BEFORE ${event} ON ${table} FOR EACH ROW EXECUTE FUNCTION accounting_crash_barrier()`); + await start(); + const waiting = await until(() => { + if (child!.exitCode !== null || child!.signalCode !== null) throw new Error(`Accounting server exited before the barrier: ${logs}`); + return observer`SELECT pid FROM pg_locks WHERE locktype='advisory' AND objid=${key} AND NOT granted`; + }, rows => rows.length > 0, () => logs); + const committed = phase === "after_commit_before_delivery_ack"; + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id))).toHaveLength(committed ? 1 : 0); + await kill(); + expect(child?.signalCode).toBe("SIGKILL"); + // A backend blocked inside our artificial barrier cannot observe its + // dead client until the query returns. Terminate that orphan connection + // before releasing the barrier, forcing PostgreSQL's disconnect rollback. + for (const row of waiting) await observer`SELECT pg_terminate_backend(${row.pid})`; + await until(() => observer`SELECT pid FROM pg_locks WHERE locktype='advisory' AND objid=${key} AND NOT granted`, rows => rows.length === 0); + await blocker`SELECT pg_advisory_unlock(${key})`; + await observer.unsafe(`DROP TRIGGER accounting_crash_barrier ON ${table}; DROP FUNCTION accounting_crash_barrier()`); + const baseUrl = await start(); + await until(() => observer`SELECT cost_accounting_pending,cost_accounted_at FROM heartbeat_runs WHERE id=${run.id}`, + rows => rows[0].cost_accounting_pending === false && rows[0].cost_accounted_at !== null, () => logs); + await until(() => observer`SELECT enforcement_version,enforcement_delivered_version FROM budget_policies WHERE id=${policy.id}`, + rows => rows[0].enforcement_version > 0 && rows[0].enforcement_version === rows[0].enforcement_delivered_version, () => logs); + const assertLedger = async () => { + expect((await observer`SELECT count(*)::int AS count,sum(cost_cents)::text AS cents FROM cost_events WHERE heartbeat_run_id=${run.id}`)[0]).toEqual({ count: 1, cents: "1.2500000" }); + expect((await observer`SELECT total_input_tokens,total_cached_input_tokens,total_output_tokens,total_cost_cents::text FROM agent_runtime_state WHERE agent_id=${agent.id}`)[0]) + .toEqual({ total_input_tokens: "7", total_cached_input_tokens: "11", total_output_tokens: "3", total_cost_cents: "1.2500000" }); + expect((await observer`SELECT spent_monthly_cents::text,status FROM agents WHERE id=${agent.id}`)[0]).toEqual({ spent_monthly_cents: "1.2500000", status: "paused" }); + expect((await observer`SELECT count(*)::int AS count FROM budget_incidents WHERE policy_id=${policy.id}`)[0].count).toBe(1); + expect((await observer`SELECT count(*)::int AS count FROM approvals WHERE company_id=${company.id}`)[0].count).toBe(1); + }; + await assertLedger(); + await until(async () => fetch(`${baseUrl}/api/health`).then(r => r.ok).catch(() => false), Boolean, () => logs); + await kill(); + const replayUrl = await start(); + await until(async () => fetch(`${replayUrl}/api/health`).then(r => r.ok).catch(() => false), Boolean, () => logs); + await until(async () => logs.includes(`Server startup recovery complete on 127.0.0.1:${new URL(replayUrl).port}`), Boolean, () => logs); + await assertLedger(); + } finally { + await kill(); + await blocker`SELECT pg_advisory_unlock_all()`; + blocker.release(); + await observer.unsafe(`DROP TRIGGER IF EXISTS accounting_crash_barrier ON ${table}; DROP FUNCTION IF EXISTS accounting_crash_barrier()`); + await observer.end(); + await rm(home, { recursive: true, force: true }); + } + }, 150_000); +}); diff --git a/server/src/__tests__/cost-accounting-edge-cases.test.ts b/server/src/__tests__/cost-accounting-edge-cases.test.ts new file mode 100644 index 0000000000..c6dc3e7f52 --- /dev/null +++ b/server/src/__tests__/cost-accounting-edge-cases.test.ts @@ -0,0 +1,241 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { activityLog, agents, companies, costEvents, createDb, goals, heartbeatRuns, issues, projects } from "@paperclipai/db"; +import { costService } from "../services/costs.js"; +import { financeService } from "../services/finance.js"; +import { budgetService } from "../services/budgets.js"; +import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js"; +import { receiptFingerprint } from "../services/receipt-fingerprint.js"; +import { subscribeCompanyLiveEvents } from "../services/live-events.js"; +import { logger } from "../middleware/logger.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("accounting validation and recovery edges (PostgreSQL)", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-edges-"); db = createDb(database.connectionString); }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Accounting edges", issuePrefix: `E${randomUUID().slice(0, 7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning(); + const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project" }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Work", projectId: project.id }).returning(); + const [goal] = await db.insert(goals).values({ companyId: company.id, title: "Goal" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id }).returning(); + const receipt = { agentId: agent.id, provider: "fixture", model: "fixture", billingType: "metered_api", costCents: 0.1234567, occurredAt: new Date() }; + const cost = await costService(db).createEvent(company.id, receipt); + const finance = { eventKind: "inference_charge", biller: "fixture", amountCents: 123, occurredAt: new Date() }; + return { company, agent, project, issue, goal, run, cost, receipt, finance, + links: { agentId: agent.id, projectId: project.id, issueId: issue.id, goalId: goal.id, heartbeatRunId: run.id, costEventId: cost.id } }; + } + + it.each(["agentId", "projectId", "issueId", "goalId", "heartbeatRunId", "costEventId"] as const)("rejects foreign and missing finance %s, accepts company-owned links", async field => { + const a = await fixture(), b = await fixture(); + const service = financeService(db); + await expect(service.createEvent(a.company.id, { ...a.finance, [field]: b.links[field] })).rejects.toMatchObject({ status: 422 }); + await expect(service.createEvent(a.company.id, { ...a.finance, [field]: randomUUID() })).rejects.toMatchObject({ status: 404 }); + expect(await service.list(a.company.id)).toHaveLength(0); + const event = await service.createEvent(a.company.id, { ...a.finance, ...a.links }, { actorType: "user", actorId: "board", agentId: a.agent.id }); + expect(event[field]).toBe(a.links[field]); + expect(await service.list(a.company.id)).toEqual([event]); + }); + + it("normalizes finance retries and rolls back changed keys and invalid amounts/currencies", async () => { + const f = await fixture(), service = financeService(db); + const receipt = { ...f.finance, idempotencyKey: "invoice", currency: "usd", metadataJson: { entries: [{ z: 2, a: 1 }, null] } }; + const event = await service.createEvent(f.company.id, receipt); + expect((await service.createEvent(f.company.id, { ...receipt, currency: "USD", metadataJson: { entries: [{ a: 1, z: 2 }, null] } })).id).toBe(event.id); + await expect(service.createEvent(f.company.id, { ...receipt, amountCents: 124 })).rejects.toMatchObject({ status: 409 }); + for (const invalid of [{ amountCents: -1 }, { amountCents: "1.2.3" }, { amountCents: Number.POSITIVE_INFINITY }, { currency: "US" }]) { + await expect(service.createEvent(f.company.id, { ...f.finance, ...invalid })).rejects.toMatchObject({ status: 422 }); + } + expect(await service.list(f.company.id)).toHaveLength(1); + expect((await db.select().from(activityLog).where(eq(activityLog.companyId, f.company.id))).filter(row => row.action === "finance_event.reported")).toHaveLength(1); + }); + + it("keeps finance credits, estimates, currencies, dates, and companies separate in every report", async () => { + const f = await fixture(), foreign = await fixture(), service = financeService(db); + const date = new Date("2026-08-15T12:00:00Z"); + await service.createEvent(f.company.id, { ...f.finance, occurredAt: date, amountCents: 150, estimated: true }); + await service.createEvent(f.company.id, { ...f.finance, occurredAt: new Date(date.getTime() + 1), eventKind: "credit_refund", direction: "credit", amountCents: 40 }); + await service.createEvent(f.company.id, { ...f.finance, occurredAt: date, amountCents: 900, currency: "EUR" }); + await service.createEvent(f.company.id, { ...f.finance, occurredAt: new Date("2026-08-16T00:00:00Z"), amountCents: 999 }); + await service.createEvent(foreign.company.id, { ...foreign.finance, occurredAt: date, amountCents: 999 }); + const range = { from: date, to: new Date(date.getTime() + 1) }; + expect(await service.summary(f.company.id, range)).toMatchObject({ debitCents: 150, creditCents: 40, estimatedDebitCents: 150, netCents: 110, eventCount: 2, + currencies: [expect.objectContaining({ currency: "EUR", netCents: 900, eventCount: 1 }), expect.objectContaining({ currency: "USD", netCents: 110, eventCount: 2 })] }); + expect(await service.byBiller(f.company.id, range)).toEqual([ + expect.objectContaining({ biller: "fixture", currency: "EUR", netCents: 900, eventCount: 1, kindCount: 1 }), + expect.objectContaining({ biller: "fixture", currency: "USD", netCents: 110, eventCount: 2, kindCount: 2 }), + ]); + expect(await service.byKind(f.company.id, range)).toEqual(expect.arrayContaining([ + expect.objectContaining({ eventKind: "inference_charge", currency: "EUR", netCents: 900 }), + expect.objectContaining({ eventKind: "inference_charge", currency: "USD", netCents: 150, estimatedDebitCents: 150 }), + expect.objectContaining({ eventKind: "credit_refund", currency: "USD", netCents: -40, billerCount: 1 }), + ])); + expect(await service.list(f.company.id, range, 1)).toEqual([expect.objectContaining({ direction: "credit", amountCents: 40 })]); + expect(await service.summary(randomUUID())).toMatchObject({ currencies: [], debitCents: 0, creditCents: 0, netCents: 0, eventCount: 0 }); + }); + + it("rejects missing companies, missing links, run/agent mismatches, and invalid cost inputs without side effects", async () => { + const f = await fixture(), costs = costService(db); + await expect(costs.createEvent(randomUUID(), f.receipt)).rejects.toMatchObject({ status: 404 }); + await expect(costs.summary(randomUUID())).rejects.toMatchObject({ status: 404 }); + for (const field of ["agentId", "issueId", "projectId", "goalId", "heartbeatRunId"]) { + await expect(costs.createEvent(f.company.id, { ...f.receipt, [field]: randomUUID() })).rejects.toMatchObject({ status: 404 }); + } + const [otherAgent] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning(); + await expect(costs.createEvent(f.company.id, { ...f.receipt, agentId: otherAgent.id, heartbeatRunId: f.run.id })).rejects.toThrow("does not belong to agent"); + for (const invalid of [{ costCents: -1 }, { costCents: NaN }, { inputTokens: 0.5 }, { cachedInputTokens: -1 }, { outputTokens: 2 ** 31 }]) { + await expect(costs.createEvent(f.company.id, { ...f.receipt, ...invalid })).rejects.toMatchObject({ status: 422 }); + } + expect((await costs.summary(f.company.id)).spendCents).toBe(f.receipt.costCents); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1); + }); + + it("aggregates billers with fractional cents, cache tokens, and subscription runs without cross-company leakage", async () => { + const f = await fixture(), foreign = await fixture(), costs = costService(db); + const date = new Date("2026-08-15T12:00:00Z"); + await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: date, biller: "gateway", provider: "a", model: "a", heartbeatRunId: f.run.id, inputTokens: 2, cachedInputTokens: 3, outputTokens: 5 }); + await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: date, biller: "gateway", provider: "b", model: "b", heartbeatRunId: f.run.id, billingType: "subscription_included", costCents: 0, inputTokens: 7, cachedInputTokens: 11, outputTokens: 13 }); + await costs.createEvent(foreign.company.id, { ...foreign.receipt, occurredAt: date, biller: "gateway", costCents: 999 }); + expect(await costs.byBiller(f.company.id, { from: date, to: date })).toEqual([{ + biller: "gateway", costCents: 0.1234567, costCentsExact: "0.1234567", inputTokens: 9, cachedInputTokens: 14, outputTokens: 18, + apiRunCount: 1, subscriptionRunCount: 1, subscriptionInputTokens: 7, subscriptionCachedInputTokens: 11, subscriptionOutputTokens: 13, providerCount: 2, modelCount: 2, + }]); + }); + + it("commits a receipt and its audit record even when a live subscriber throws", async () => { + const f = await fixture(); + const listener = vi.fn(() => { throw new Error("Injected disconnected live subscriber"); }); + const unsubscribe = subscribeCompanyLiveEvents(f.company.id, listener); + const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined); + try { + const receipt = { ...f.receipt, idempotencyKey: "committed-before-publication" }; + const event = await costService(db).createEvent(f.company.id, receipt, { actorType: "user", actorId: "board" }); + expect((await costService(db).createEvent(f.company.id, receipt)).id).toBe(event.id); + expect(listener).toHaveBeenCalledTimes(1); + expect(warning).toHaveBeenCalledOnce(); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(0.2469134); + expect(await db.select().from(activityLog).where(eq(activityLog.entityId, event.id))).toHaveLength(1); + } finally { unsubscribe(); warning.mockRestore(); } + }); + + it("reports lifetime history consistently across every cost grouping and finance view", async () => { + const f = await fixture(), costs = costService(db), finance = financeService(db); + const occurredAt = new Date("2000-01-01T00:00:00Z"); + await costs.createEvent(f.company.id, { ...f.receipt, occurredAt, costCents: 5, projectId: f.project.id, inputTokens: 2 }); + await finance.createEvent(f.company.id, { ...f.finance, occurredAt }); + const lifetime = { allTime: true }; + for (const read of [costs.byAgent, costs.byAgentModel, costs.byProvider, costs.byBiller, costs.byProject]) { + const rows = await read(f.company.id, lifetime); + expect(rows.reduce((sum, row) => sum + row.costCents, 0)).toBe(5.1234567); + expect(rows.reduce((sum, row) => sum + row.inputTokens, 0)).toBe(2); + } + expect((await costs.summary(f.company.id)).spendCents).toBe(0.1234567); + expect((await costs.summary(f.company.id, lifetime)).spendCents).toBe(5.1234567); + expect((await finance.summary(f.company.id)).eventCount).toBe(0); + expect((await finance.summary(f.company.id, lifetime)).debitCents).toBe(123); + expect(await finance.list(f.company.id, lifetime)).toHaveLength(1); + expect((await finance.byBiller(f.company.id, lifetime))[0].netCents).toBe(123); + expect((await finance.byKind(f.company.id, lifetime))[0].netCents).toBe(123); + await db.update(companies).set({ budgetMonthlyCents: 1 }).where(eq(companies.id, f.company.id)); + expect((await costs.summary(f.company.id)).utilizationPercent).toBe(12.35); + }); + + it.each(["company", "agent", "project"] as const)("rejects nonexistent and foreign %s budget scopes without creating policies", async scopeType => { + const f = await fixture(), foreign = await fixture(), budgets = budgetService(db); + await expect(budgets.upsertPolicy(f.company.id, { scopeType, scopeId: randomUUID(), amount: 10 }, "board")).rejects.toMatchObject({ status: 404 }); + await expect(budgets.upsertPolicy(f.company.id, { scopeType, scopeId: foreign[scopeType].id, amount: 10 }, "board")).rejects.toMatchObject({ status: 422 }); + expect(await budgets.listPolicies(f.company.id)).toHaveLength(0); + if (scopeType !== "company") { + await expect(budgets.getInvocationBlock(f.company.id, scopeType === "agent" ? foreign.agent.id : f.agent.id, + scopeType === "project" ? { projectId: foreign.project.id } : undefined)).rejects.toMatchObject({ status: 404 }); + } + }); + + it.each(["unpriced", "pending"] as const)("refuses an incident budget raise while accounting is %s, without partial mutations", async kind => { + const f = await fixture(), budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 10 }, "board"); + if (kind === "unpriced") { + await costService(db).createEvent(f.company.id, { ...f.receipt, costStatus: "unpriced", costCents: 0 }); + } else { + await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, finishedAt: new Date(), usageJson: { accountingReceiptReady: false } }).where(eq(heartbeatRuns.id, f.run.id)); + await budgets.getInvocationBlock(f.company.id, f.agent.id); + } + const [incident] = (await budgets.overview(f.company.id)).activeIncidents; + const resolution = { action: "raise_budget_and_resume" as const, amount: 100 }; + await expect(budgets.resolveIncident(f.company.id, randomUUID(), resolution, "board")).rejects.toMatchObject({ status: 404 }); + const foreign = await fixture(); + await expect(budgets.resolveIncident(foreign.company.id, incident.id, resolution, "board")).rejects.toMatchObject({ status: 404 }); + await expect(budgets.resolveIncident(f.company.id, incident.id, resolution, "board")).rejects.toThrow(kind === "unpriced" ? /unpriced usage/ : /finish accounting/); + expect((await budgets.listPolicies(f.company.id))[0].amount).toBe(10); + expect((await budgets.overview(f.company.id)).activeIncidents).toEqual([expect.objectContaining({ id: incident.id, status: "open" })]); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("paused"); + }); + + it.each(["company", "project"] as const)("preserves an operator's %s pause even without a policy", async scopeType => { + const f = await fixture(); + if (scopeType === "company") await db.update(companies).set({ status: "paused", pauseReason: "manual", pausedAt: new Date() }).where(eq(companies.id, f.company.id)); + else await db.update(projects).set({ pauseReason: "manual", pausedAt: new Date() }).where(eq(projects.id, f.project.id)); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id })) + .toMatchObject({ scopeType, reason: expect.stringContaining("paused and cannot start") }); + }); + + it("resolves a company incident and updates the legacy monthly budget together", async () => { + const f = await fixture(), budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2 }); + const [incident] = (await budgets.overview(f.company.id)).activeIncidents; + await budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 10 }, "board"); + expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0]).toMatchObject({ status: "active", pauseReason: null, budgetMonthlyCents: 10 }); + expect((await budgets.listPolicies(f.company.id))[0].amount).toBe(10); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + }); + + it.each([false, true])("records subscription tokens at zero incremental charge (split=%s)", async split => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status: "succeeded", costAccountingPending: true, + usageJson: { billingType: "subscription_included", costUsd: 1, inputTokens: 3, cachedInputTokens: 5, outputTokens: 7, + ...(split ? { usageByModel: [{ model: "a", costUsd: 1, usage: { inputTokens: 3, cachedInputTokens: 5, outputTokens: 7 } }] } : {}), + }, + }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(true); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([ + expect.objectContaining({ costCents: 0, costStatus: "reported", inputTokens: 3, cachedInputTokens: 5, outputTokens: 7, billingType: "subscription_included" }), + ]); + }); + + it("rotates a full poisoned recovery batch so the next valid receipt cannot starve", async () => { + const f = await fixture(), foreign = await fixture(); + const poison = await db.insert(heartbeatRuns).values(Array.from({ length: 100 }, () => ({ + companyId: f.company.id, agentId: f.agent.id, status: "failed", costAccountingPending: true, updatedAt: new Date(0), + usageJson: { accountingReceiptReady: true, costUsd: 0.01, ledgerScope: { projectId: foreign.project.id } }, + }))).returning(); + const [valid] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "succeeded", costAccountingPending: true, + updatedAt: new Date(1), usageJson: { accountingReceiptReady: true, costUsd: 0.02 }, + }).returning(); + const error = vi.spyOn(logger, "error").mockImplementation(() => undefined); + try { + expect(await reconcileRunCosts(db)).toEqual({ scanned: 100, accounted: 0 }); + expect(await reconcileRunCosts(db)).toEqual({ scanned: 100, accounted: 1 }); + expect(error).toHaveBeenCalledTimes(199); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, valid.id))).toEqual([expect.objectContaining({ costCents: 2 })]); + const rows = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.company.id)); + expect(rows.filter(row => row.costAccountingPending)).toHaveLength(100); + expect(rows.filter(row => poison.some(bad => bad.id === row.id)).every(row => row.costAccountedAt === null)).toBe(true); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(2.1234567); + expect(await accountRunCost(db, valid.id)).toBe(false); + expect(await accountRunCost(db, randomUUID())).toBe(false); + } finally { error.mockRestore(); } + }, 30_000); +}); + +it("fingerprints nested receipts independently of object order while preserving array order and values", () => { + const a = { items: [{ z: 2, a: 1 }, null], at: new Date("2026-01-01T00:00:00Z") }; + expect(receiptFingerprint(a)).toBe(receiptFingerprint({ at: a.at, items: [{ a: 1, z: 2 }, null] })); + expect(receiptFingerprint(a)).not.toBe(receiptFingerprint({ ...a, items: [null, { z: 2, a: 1 }] })); + expect(receiptFingerprint(a)).not.toBe(receiptFingerprint({ ...a, items: [{ z: 3, a: 1 }, null] })); +}); diff --git a/server/src/__tests__/cost-accounting-mutation-targets.test.ts b/server/src/__tests__/cost-accounting-mutation-targets.test.ts new file mode 100644 index 0000000000..ab1d24d06d --- /dev/null +++ b/server/src/__tests__/cost-accounting-mutation-targets.test.ts @@ -0,0 +1,45 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb } from "@paperclipai/db"; +import { costService } from "../services/costs.js"; +import { budgetService } from "../services/budgets.js"; +import { accountingIntegrityService } from "../services/accounting-integrity.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("accounting mutation sentinels", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-mutation-"); db = createDb(database.connectionString); }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Mutation", issuePrefix: `M${randomUUID().slice(0,7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning(); + return { company, agent, receipt: { agentId: agent.id, provider: "fixture", model: "fixture", costCents: "1.0000001", idempotencyKey: randomUUID(), occurredAt: new Date() } }; + } + it("deduplicates a retried receipt", async () => { + const f = await fixture(), service = costService(db); + const first = await service.createEvent(f.company.id, f.receipt); + const second = await service.createEvent(f.company.id, f.receipt); + expect(second.id, "ACCOUNTING_ASSERTION deduplication").toBe(first.id); + expect((await service.summary(f.company.id)).eventCount).toBe(1); + }); + it("isolates company reporting", async () => { + const f = await fixture(), other = await fixture(), service = costService(db); + await service.createEvent(f.company.id, f.receipt); + await service.createEvent(other.company.id, { ...other.receipt, costCents: 1000 }); + expect((await service.summary(f.company.id)).spendCentsExact, "ACCOUNTING_ASSERTION company").toBe("1.0000001"); + }); + it("conserves agent projections", async () => { + const f = await fixture(); + await costService(db).createEvent(f.company.id, f.receipt); + await costService(db).createEvent(f.company.id, { ...f.receipt, idempotencyKey: "second" }); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings, "ACCOUNTING_ASSERTION projection").toEqual([]); + }); + it("stops at the exact budget boundary", async () => { + const f = await fixture(); + const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1 }, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 1 }); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id), "ACCOUNTING_ASSERTION threshold").not.toBeNull(); + }); +}); diff --git a/server/src/__tests__/cost-accounting-operations-edges.test.ts b/server/src/__tests__/cost-accounting-operations-edges.test.ts new file mode 100644 index 0000000000..8fa73dd0af --- /dev/null +++ b/server/src/__tests__/cost-accounting-operations-edges.test.ts @@ -0,0 +1,551 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { accountingRuntimeBaselines, agents, agentRuntimeState, billingInvoiceLines, budgetPolicies, budgetReservations, companies, costEvents, createDb, heartbeatRuns, nativeRunFinalizations, issues, projects, runUsageReceipts } from "@paperclipai/db"; +import { accountingIntegrityService } from "../services/accounting-integrity.js"; +import { billingReconciliationService } from "../services/billing-reconciliation.js"; +import { budgetService, type BudgetEnforcementScope } from "../services/budgets.js"; +import { reserveRunBudget } from "../services/budget-reservations.js"; +import { costService } from "../services/costs.js"; +import { financeService } from "../services/finance.js"; +import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js"; +import { createRunUsageRecorder, persistUsageReceipt, replayUsageReceipts, spoolUsageReceipt, usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../services/usage-receipts.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("accounting operational edge cases", () => { + let database: Awaited>, db: ReturnType, directory: string; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-operational-edges-"); db = createDb(database.connectionString); directory = await fs.mkdtemp(path.join(os.tmpdir(), "accounting-edges-")); },30_000); + afterAll(async () => { vi.restoreAllMocks(); await database?.cleanup(); await fs.rm(directory, { recursive: true, force: true }); }); + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Edges", issuePrefix: `E${randomUUID().slice(0,7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning(); + const [project] = await db.insert(projects).values({ companyId: company.id, name: "Work" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running", usageJson: {} }).returning(); + return { company, agent, project, run }; + } + const usage = { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 }; + async function event(f: Awaited>, extras: Record = {}) { + return costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "model", costCents: "0.1234567", occurredAt: new Date(), ...extras }); + } + it("initializes an old month once, excludes backdated events, and repairs current-month drift", async () => { + const f = await fixture(); + await db.update(companies).set({ spendMonthUtc: "2000-01-01", spentMonthlyCents: 999 }).where(eq(companies.id,f.company.id)); + await event(f); await event(f, { occurredAt: new Date("2000-01-01") }); await event(f); + const [row] = await db.select().from(companies).where(eq(companies.id,f.company.id)); + expect(row.spentMonthlyCents).toBe(0.2469134); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + await db.update(agents).set({ spentMonthlyCents: 4 }).where(eq(agents.id,f.agent.id)); + const review = await accountingIntegrityService(db).inspect(f.company.id); + await accountingIntegrityService(db).repair(f.company.id,review.fingerprint,"Repair drift","board"); + await event(f); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + }); + it("keeps fractional finance credits and net totals exact beyond number precision", async () => { + const f = await fixture(), finance = financeService(db); + const base = { biller: "fixture", eventKind: "inference_charge", occurredAt: new Date() }; + await finance.createEvent(f.company.id,{ ...base, amountCents: "9007199254740992.0000001" }); + await finance.createEvent(f.company.id,{ ...base, direction: "credit", amountCents: "9007199254740992" }); + expect((await finance.summary(f.company.id)).netCentsExact).toBe("0.0000001"); + expect((await finance.byBiller(f.company.id))[0].netCentsExact).toBe("0.0000001"); + expect((await finance.byKind(f.company.id))[0].netCentsExact).toBe("0.0000001"); + }); + it("checks agent and project capacity, preserves reservations through policy edits, and releases proven bootstrap failures", async () => { + const f = await fixture(), budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id,{ scopeType:"agent",scopeId:f.agent.id,amount:20,reservationCents:"3" },"board"); + await budgets.upsertPolicy(f.company.id,{ scopeType:"project",scopeId:f.project.id,windowKind:"lifetime",amount:10,reservationCents:"6" },"board"); + await budgets.upsertPolicy(f.company.id,{ scopeType:"company",scopeId:f.company.id,amount:20,reservationCents:"1" },"board"); + const reserved = await reserveRunBudget(db,f.company.id,f.run.id,f.project.id); + expect(reserved.amountCents).toBe("6.0000000"); + await budgets.upsertPolicy(f.company.id,{ scopeType:"project",scopeId:f.project.id,windowKind:"lifetime",amount:10,reservationCents:"0" },"board"); + await budgets.upsertPolicy(f.company.id,{ scopeType:"agent",scopeId:f.agent.id,amount:20,reservationCents:"0" },"board"); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId,f.run.id)))[0].amountCents).toBe("6.0000000"); + await db.update(heartbeatRuns).set({ status:"failed",resultJson:{executionRecovery:{providerWorkStarted:false}} }).where(eq(heartbeatRuns.id,f.run.id)); + await accountRunCost(db,f.run.id); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId,f.run.id)))[0].state).toBe("released"); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId,f.company.id))).toEqual([]); + }); + it.each(["retryable_failure", "observed"])("keeps native %s receipts open through same-run recovery", async (phase) => { + const f = await fixture(); + const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recoverable native work" }).returning(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id)); + await reserveRunBudget(db, f.company.id, f.run.id, null); + await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase }); + const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" }; + const spool = path.join(directory, randomUUID()); + const recorder = await createRunUsageRecorder(db, input, spool); + await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(false); + await reconcileRunCosts(db); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull(); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held"); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id)); + const resumed = await createRunUsageRecorder(db, input, spool); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptReady).toBe(false); + const empty = await resumed.complete({ exitCode: 0, signal: null, timedOut: false }); + expect(empty).toMatchObject({ complete: false, usage: { inputTokens: 100, outputTokens: 10 } }); + // The provider's native delta is cumulative for this run across recovery. + await resumed.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 }); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.run.id)); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(true); + expect(await accountRunCost(db, f.run.id)).toBe(false); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ inputTokens: 150, outputTokens: 15, costCents: 15 }); + }); + + it.each([false, true])("recovers pending native receipts beyond the startup batch before replacement (save failure: %s)", async (failSave) => { + const f = await fixture(), other = await fixture(); + await db.update(heartbeatRuns).set({ runtimeMode: "native" }).where(eq(heartbeatRuns.id, f.run.id)); + await reserveRunBudget(db, f.company.id, f.run.id, null); + const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" }; + const spool = path.join(directory, randomUUID()); + const recorder = await createRunUsageRecorder(db, input, spool); + await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 }); + const before = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]; + const unavailable = vi.spyOn(db, "transaction").mockRejectedValue(new Error("Database unavailable")); + try { + await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 }); + await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 }); + } finally { unavailable.mockRestore(); } + const pending = (await fs.readdir(spool)).filter(name => name.endsWith(".json")); + expect(pending).toHaveLength(2); + for (const name of pending) await fs.rename(path.join(spool, name), path.join(spool, `zzz-${name}`)); + const unrelated: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), + companyId: other.company.id, runId: other.run.id, sourceId: randomUUID(), sequence: 1, + receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } }; + await Promise.all(Array.from({ length: 101 }, async (_, index) => { + await fs.writeFile(path.join(spool, `000-${index}.json`), JSON.stringify({ ...unrelated, id: randomUUID(), sequence: index + 1 })); + })); + expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 100, failed: 0 }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.inputTokens).toBe(100); + if (failSave) { + await db.execute(sql`create function reject_recovered_receipt() returns trigger language plpgsql as $$ begin raise exception 'receipt save failed'; end $$`); + await db.execute(sql`create trigger reject_recovered_receipt before insert on run_usage_receipts for each row execute function reject_recovered_receipt()`); + try { + await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId) + .toBe(before.usageJson?.accountingReceiptSourceId); + expect((await fs.readdir(spool)).filter(name => name.startsWith("zzz-"))).toHaveLength(2); + } finally { + await db.execute(sql`drop trigger reject_recovered_receipt on run_usage_receipts`); + await db.execute(sql`drop function reject_recovered_receipt()`); + } + } + const resumed = await createRunUsageRecorder(db, input, spool); + expect((await fs.readdir(spool)).filter(name => name.startsWith("zzz-"))).toHaveLength(0); + expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, f.run.id))).toHaveLength(3); + expect(await resumed.complete({ exitCode: 0, signal: null, timedOut: false })) + .toMatchObject({ complete: false, usage: { inputTokens: 200, outputTokens: 20 } }); + await resumed.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 220, outputTokens: 22 }, costUsd: 0.22 }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(true); + expect(await accountRunCost(db, f.run.id)).toBe(false); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]) + .toMatchObject({ inputTokens: 220, outputTokens: 22, costCents: 22 }); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled"); + expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 1, failed: 0 }); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(22); + }); + + it.each(["company", "agent", "project"] as const)("blocks fresh %s work without pausing native recovery", async (scopeType) => { + const f = await fixture(), cancelWorkForScope = vi.fn(async (_scope: BudgetEnforcementScope) => {}); + const budgets = budgetService(db, { cancelWorkForScope }); + const scopeId = f[scopeType].id; + await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100, reservationCents: "20" }, "board"); + const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover under budget" }).returning(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id)); + const original = await reserveRunBudget(db, f.company.id, f.run.id, f.project.id); + await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: "retryable_failure" }); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" }, path.join(directory, randomUUID())); + await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + // Also recover a budget-owned pause left by the previous implementation. + const table = { company: companies, agent: agents, project: projects }[scopeType]; + await db.update(table).set({ pauseReason: "budget", ...(scopeType === "project" ? {} : { status: "paused" }) }).where(eq(table.id, scopeId)); + await reconcileRunCosts(db, { cancelWorkForScope }); + expect((await db.select().from(table).where(eq(table.id, scopeId)))[0].pauseReason).toBeNull(); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle"); + expect(cancelWorkForScope.mock.calls.filter(([scope]) => scope.companyId === f.company.id)).toEqual([]); + expect((await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id }))?.reason).toContain("native run recovers"); + const [fresh] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running" }).returning(); + await expect(reserveRunBudget(db, f.company.id, fresh.id, f.project.id)).rejects.toThrow("native run recovers"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull(); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id)); + await db.update(nativeRunFinalizations).set({ leaseOwner: "recovery", leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id)); + expect(await reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "recovery")) + .toMatchObject({ id: original.id, reused: true, amountCents: "20.0000000", state: "held" }); + expect(await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id))).toHaveLength(1); + await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 }); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.run.id)); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(true); + expect(await accountRunCost(db, f.run.id)).toBe(false); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ inputTokens: 150, costCents: 15 }); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled"); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id })).toBeNull(); + }); + + it.each(["company", "agent", "project"] as const)("rechecks %s stops before native reservation reuse", async scopeType => { + const f = await fixture(), budgets = budgetService(db), scopeId = f[scopeType].id; + await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100, reservationCents: "20" }, "board"); + const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Retry after budget edit" }).returning(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id)); + const original = await reserveRunBudget(db, f.company.id, f.run.id, f.project.id); + await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: "retryable_failure", leaseOwner: "retry", leaseExpiresAt: new Date(Date.now() + 60_000) }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + await event(f, { costCents: 50, projectId: f.project.id }); + await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 40 }, "board"); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id)); + await expect(reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).rejects.toThrow(/budget/); + await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100 }, "board"); + expect(await reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).toMatchObject({ id: original.id, reused: true }); + const table = { company: companies, agent: agents, project: projects }[scopeType]; + await db.update(table).set({ pauseReason: "manual", ...(scopeType === "project" ? { pausedAt: new Date() } : { status: "paused" }) }).where(eq(table.id, scopeId)); + await expect(reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).rejects.toThrow(/paused/); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0]).toMatchObject({ id: original.id, state: "held", amountCents: "20.0000000" }); + }); + + it.each(["closed", "over_budget", "unpriced", "manual"])("preserves %s stops while native accounting is unfinished", async (stop) => { + const f = await fixture(), budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board"); + const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Pending native run" }).returning(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id)); + await reserveRunBudget(db, f.company.id, f.run.id, null); + await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: stop === "closed" ? "terminal_failure" : "retryable_failure" }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + if (stop === "over_budget") await event(f, { costCents: 100 }); + if (stop === "unpriced") await event(f, { costCents: 0, costStatus: "unpriced" }); + if (stop === "manual") await db.update(agents).set({ status: "paused", pauseReason: "manual" }).where(eq(agents.id, f.agent.id)); + await budgets.reconcilePolicies(); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]) + .toMatchObject({ status: "paused", pauseReason: stop === "manual" ? "manual" : "budget" }); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull(); + }); + + it.each(["complete", "partial", "receipt_failure", "ledger_failure", "cleanup_failure", "moved", "unstable_spool"])("settles the newest run receipt beyond a full replay batch (%s)", async (scenario) => { + const f = await fixture(), other = await fixture(), spool = usageReceiptSpoolPath(); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "process" }); + await reserveRunBudget(db, f.company.id, f.run.id, null); + await recorder.capture({ complete: true, usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 }); + const unavailable = vi.spyOn(db, "transaction").mockRejectedValue(new Error("Database unavailable")); + try { await recorder.capture({ complete: scenario !== "partial", usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 }); } + finally { unavailable.mockRestore(); } + const [pending] = (await fs.readdir(spool)).filter(name => name.endsWith(".json")); + const file = path.join(spool, `zzz-${pending}`); + await fs.rename(path.join(spool, pending), file); + const unrelated: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: other.company.id, + runId: other.run.id, sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } }; + for (let index = 0; index < 101; index++) await fs.writeFile(path.join(spool, `000-${index}.json`), JSON.stringify({ ...unrelated, id: randomUUID(), sequence: index + 1 })); + expect(await replayUsageReceipts(db)).toEqual({ replayed: 100, failed: 0 }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.inputTokens).toBe(100); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + if (scenario === "unstable_spool") { + const stat = fs.lstat.bind(fs); + const unstable = vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => { + if (target === file) throw Object.assign(new Error("Concurrent rename"), { code: "ENOENT" }); + return stat(target, ...args); + }); + try { await expect(accountRunCost(db, f.run.id)).rejects.toThrow("retry required"); } + finally { unstable.mockRestore(); } + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull(); + expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200); + } + if (scenario.endsWith("_failure") && scenario !== "cleanup_failure") { + const table = scenario === "receipt_failure" ? "run_usage_receipts" : "cost_events"; + await db.execute(sql`create function reject_settlement() returns trigger language plpgsql as $$ begin raise exception 'settlement save failed'; end $$`); + await db.execute(sql.raw(`create trigger reject_settlement before insert on ${table} for each row execute function reject_settlement()`)); + try { + await expect(accountRunCost(db, f.run.id)).rejects.toThrow(); + expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]) + .toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 100 } }); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]); + expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, f.run.id))).toHaveLength(1); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held"); + } finally { + await db.execute(sql.raw(`drop trigger reject_settlement on ${table}`)); + await db.execute(sql`drop function reject_settlement()`); + } + } + const remove = fs.rm.bind(fs), stat = fs.lstat.bind(fs); + let moved = false; + const rm = scenario === "cleanup_failure" ? vi.spyOn(fs, "rm").mockImplementation(async (target, options) => { + if (target === file) throw new Error("Temporary cleanup failure"); + return remove(target, options); + }) : null; + const lstat = scenario === "moved" ? vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => { + if (target === file && !moved) { moved = true; await fs.rename(file, `${file.slice(0, -5)}-moved.json`); } + return stat(target, ...args); + }) : null; + try { await reconcileRunCosts(db); } finally { rm?.mockRestore(); lstat?.mockRestore(); } + if (scenario === "partial") { + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]) + .toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 200, accountingReceiptReady: false } }); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]); + await recorder.capture({ complete: true, usage: { inputTokens: 300, outputTokens: 30 }, costUsd: 0.3 }); + expect(await accountRunCost(db, f.run.id)).toBe(true); + } + expect(await accountRunCost(db, f.run.id)).toBe(false); + const expected = scenario === "partial" ? 300 : 200; + const charges = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)); + expect(charges).toHaveLength(1); + expect(charges[0]).toMatchObject({ inputTokens: expected, outputTokens: expected / 10, costCents: expected / 10 }); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled"); + expect(await replayUsageReceipts(db)).toEqual({ replayed: scenario === "cleanup_failure" ? 2 : 1, failed: 0 }); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(expected / 10); + expect(await fs.readdir(spool)).toEqual([]); + }); + + it("leaves unrelated corrupt spool evidence alone and aborts handoff on unreadable or invalid matching receipts", async () => { + const f = await fixture(), spool = path.join(directory, randomUUID()); + const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" }; + await createRunUsageRecorder(db, input, spool); + const source = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId; + const matching = path.join(spool, "matching.json"); + await fs.writeFile(matching, JSON.stringify({ ...input, receipt: "invalid" })); + await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId).toBe(source); + const read = vi.spyOn(fs, "readFile").mockRejectedValueOnce(Object.assign(new Error("Unreadable receipt"), { code: "EACCES" })); + try { await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow("Unreadable receipt"); } finally { read.mockRestore(); } + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId).toBe(source); + await fs.rm(matching); + const unrelated = ["invalid", "null", "{}", JSON.stringify({ companyId: f.company.id }), + JSON.stringify({ companyId: f.company.id, runId: randomUUID() }), "x".repeat(1024 * 1024 + 1)]; + for (const [index, value] of unrelated.entries()) await fs.writeFile(path.join(spool, `${index}.json`), value); + await fs.mkdir(path.join(spool, "directory.json")); + const stat = vi.spyOn(fs, "lstat").mockRejectedValueOnce(Object.assign(new Error("Concurrent replay removed file"), { code: "ENOENT" })); + try { await createRunUsageRecorder(db, input, spool); } finally { stat.mockRestore(); } + await createRunUsageRecorder(db, input, spool); + expect(await fs.readdir(spool)).toHaveLength(unrelated.length + 1); + }); + + it("indexes a recovery backlog once outside company locks and still sees newly published receipts", async () => { + const f = await fixture(), other = await fixture(), spool = usageReceiptSpoolPath(); + const runs = [f.run]; + for (let i = 0; i < 7; i++) { + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running" }).returning(); + runs.push(run); + } + for (const run of runs) { + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }); + await recorder.capture({ complete: true, usage: { inputTokens: 50, outputTokens: 5 }, costUsd: 0.05 }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + } + const [first] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: other.company.id, + runId: other.run.id, sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } }; + const foreignFiles = new Set(); + for (let i = 0; i < 256; i++) foreignFiles.add(await spoolUsageReceipt({ ...envelope, id: randomUUID(), sequence: i + 1 })); + const reads = new Map(), read = fs.readFile.bind(fs); + let checkedLock = false, lockWasAvailable = false; + const spy = vi.spyOn(fs, "readFile").mockImplementation(async (file, options) => { + if (typeof file === "string" && foreignFiles.has(file)) { + reads.set(file, (reads.get(file) ?? 0) + 1); + if (!checkedLock) { + checkedLock = true; + await db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '100ms'`); + await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update"); + }); + lockWasAvailable = true; + // Published after the index's directory snapshot: the locked drain + // must discover it instead of trusting a stale batch inventory. + await spoolUsageReceipt({ ...envelope, id: randomUUID(), companyId: f.company.id, runId: f.run.id, + sourceId: String(first.usageJson?.accountingReceiptSourceId), sequence: 2, + receipt: { complete: true, usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 } }); + } + } + return read(file, options); + }); + try { await reconcileRunCosts(db); } finally { spy.mockRestore(); } + expect(lockWasAvailable).toBe(true); + expect(reads.size).toBe(256); + expect([...reads.values()]).toEqual(Array(256).fill(1)); + const charges = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)); + expect(charges).toHaveLength(8); + expect(charges.find(charge => charge.heartbeatRunId === f.run.id)).toMatchObject({ inputTokens: 200, costCents: 20 }); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(55); + const listing = vi.spyOn(fs, "readdir"); + try { + for (const run of runs) expect(await accountRunCost(db, run.id)).toBe(false); + // An acknowledged run with a stale pending flag still needs its locked + // cleanup, but neither form of duplicate finalization needs spool I/O. + await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(false); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountingPending).toBe(false); + expect(listing).not.toHaveBeenCalled(); + } finally { listing.mockRestore(); } + await Promise.all([...foreignFiles].map(file => fs.rm(file))); + }); + + it("reports over-limit advisory budgets without blocking, and tolerates legacy active zero limits", async () => { + const f = await fixture(), budgets = budgetService(db); + const policy = await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1, hardStopEnabled: false }, "board"); + await event(f, { costCents: "2.0000001" }); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ status: "hard_stop", observedAmountExact: "2.0000001", paused: false }); + // Older databases can contain an active zero policy; it must never become + // an accidental hard stop during an upgrade or a projection inspection. + await db.update(budgetPolicies).set({ amount: 0, isActive: true }).where(eq(budgetPolicies.id, policy.policyId)); + expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ status: "ok", amount: 0 }); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + }); + it("recovers late bootstrap proof without replacing the stop owner's terminal metadata", async () => { + const f = await fixture(); + await reserveRunBudget(db, f.company.id, f.run.id, null); + const stopped = { executionCancellation: { state: "acknowledged" } }; + await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: stopped, + usageJson: { accountingReceiptReady: false, accountingProviderWorkStarted: false }, + }).where(eq(heartbeatRuns.id, f.run.id)); + const integrity = accountingIntegrityService(db); + expect((await integrity.health(f.company.id)).items[0].state).toBe("retryable"); + expect((await integrity.inspect(f.company.id)).findings).toEqual([]); + await reconcileRunCosts(db); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(run).toMatchObject({ status: "cancelled", resultJson: stopped, costAccountingPending: false }); + expect(run.costAccountedAt).not.toBeNull(); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("released"); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]); + }); + it("rejects absent, foreign, finished and blocked dispatch without changing the ledger", async () => { + const f = await fixture(), other = await fixture(); + await expect(reserveRunBudget(db,f.company.id,randomUUID(),null)).rejects.toThrow("not found"); + await expect(reserveRunBudget(db,other.company.id,f.run.id,null)).rejects.toThrow("not found"); + await expect(reserveRunBudget(db,f.company.id,f.run.id,other.project.id)).rejects.toThrow("not found"); + await db.update(heartbeatRuns).set({status:"failed"}).where(eq(heartbeatRuns.id,f.run.id)); + await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("no longer"); + await db.update(heartbeatRuns).set({status:"running",costAccountedAt:new Date()}).where(eq(heartbeatRuns.id,f.run.id)); + await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("no longer"); + await db.update(heartbeatRuns).set({costAccountedAt:null}).where(eq(heartbeatRuns.id,f.run.id)); + await budgetService(db).upsertPolicy(f.company.id,{scopeType:"agent",scopeId:f.agent.id,amount:1},"board"); + await event(f,{costCents:1}); + await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("paused"); + await expect(budgetService(db).upsertPolicy(f.company.id,{scopeType:"agent",scopeId:f.agent.id,amount:1,reservationCents:-1},"board")).rejects.toThrow("negative"); + }); + it("distinguishes legacy totals, missing acknowledgements, missing receipts, and corrupted receipt amounts", async () => { + const f = await fixture(); const integrity = accountingIntegrityService(db); + await db.insert(agentRuntimeState).values({companyId:f.company.id,agentId:f.agent.id,adapterType:"process",totalCostCents:4}); + expect((await integrity.inspect(f.company.id)).findings[0].kind).toBe("legacy_runtime"); + await db.update(heartbeatRuns).set({status:"failed",costAccountingPending:true,usageJson:{...usage,costUsdExact:"0.01",accountingReceiptReady:true}}).where(eq(heartbeatRuns.id,f.run.id)); + expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_acknowledgement")).toBe(true); + expect(await integrity.retry(f.company.id,f.run.id,"board")).toEqual({accounted:true}); + await db.update(heartbeatRuns).set({usageJson:{...usage,costUsdExact:"0.02"}}).where(eq(heartbeatRuns.id,f.run.id)); + expect((await integrity.inspect(f.company.id)).findings.map(f=>f.kind)).toContain("receipt_mismatch"); + await db.update(heartbeatRuns).set({usageJson:{...usage,costUsdExact:"invalid"}}).where(eq(heartbeatRuns.id,f.run.id)); + expect((await integrity.inspect(f.company.id)).findings.find(f=>f.kind==="receipt_mismatch")?.expected.cents).toBe("invalid receipt"); + await db.delete(costEvents).where(eq(costEvents.heartbeatRunId,f.run.id)); + expect((await integrity.inspect(f.company.id)).findings.map(f=>f.kind)).toContain("missing_receipt"); + await expect(integrity.repair(f.company.id,"a".repeat(64)," ","board")).rejects.toThrow("reason"); + await expect(integrity.retry(f.company.id,randomUUID(),"board")).rejects.toThrow("not found"); + }); + it("records a failed operator retry and keeps it actionable", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({status:"failed",costAccountingPending:true,usageJson:{inputTokens:0.1,accountingReceiptReady:true}}).where(eq(heartbeatRuns.id,f.run.id)); + const integrity = accountingIntegrityService(db); + await expect(integrity.retry(f.company.id,f.run.id,"board")).rejects.toThrow("Invalid cost"); + const health = await integrity.health(f.company.id); + expect(health.items[0]).toMatchObject({state:"retryable",attempts:1,lastError:"Invalid cost receipt"}); + expect(health.items[0].lastAttemptAt).not.toBeNull(); + }); + it("matches provider request IDs and run/model pairs, leaves ambiguous and non-inference lines unresolved", async () => { + const f = await fixture(), service = billingReconciliationService(db); + await event(f,{heartbeatRunId:f.run.id,providerRequestId:"request"}); await event(f,{heartbeatRunId:f.run.id,model:"second"}); + const line = {amountCents:"1",occurredAt:new Date().toISOString()}; + const invoice = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"requests",currency:"USD",lines:[ + {...line,externalId:"provider",providerRequestId:"request"}, {...line,externalId:"model",runId:f.run.id,model:"second"}, + {...line,externalId:"ambiguous",runId:f.run.id}, {...line,externalId:"unknown",providerRequestId:"absent"}, + {...line,externalId:"unlinked"}, {...line,externalId:"fee",kind:"fee"}, {...line,externalId:"credit",kind:"credit"}, + ]},"board"); + expect((await service.list(f.company.id))[0].id).toBe(invoice.id); + const report = await service.reconcile(f.company.id,invoice.id); + expect(Object.fromEntries(report.lines.map(l=>[l.externalId,l.status]))).toEqual({provider:"difference",model:"difference",ambiguous:"ambiguous",unknown:"unmatched",unlinked:"unmatched",fee:"non_inference",credit:"non_inference"}); + await expect(service.importInvoice(f.company.id,{biller:"fixture",externalId:"missing-run",currency:"USD",lines:[{...line,externalId:"run",runId:randomUUID()}]},"board")).rejects.toThrow("run not found"); + }); + it("rejects contradictory invoice evidence and reused correction keys, and preserves legacy originals", async () => { + const f = await fixture(), service = billingReconciliationService(db); const charge = await event(f); + await db.update(costEvents).set({reportedCostCents:null}).where(eq(costEvents.id,charge.id)); + const input = {idempotencyKey:"correction",expectedCents:"0.1234567",correctedCents:"1",reason:"Reviewed",pricing:{source:"operator" as const}}; + await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:randomUUID()},"board")).rejects.toThrow("not found"); + const invoice = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"bad-evidence",currency:"USD",lines:[{externalId:"line",amountCents:"2",occurredAt:new Date().toISOString(),costEventId:charge.id}]},"board"); + const [line] = (await service.reconcile(f.company.id,invoice.id)).lines; + await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:line.id},"board")).rejects.toThrow("does not support"); + const noMatch = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"unverified",currency:"USD",lines:[{externalId:"line",amountCents:"1",occurredAt:new Date().toISOString()}]},"board"); + await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:(await service.reconcile(f.company.id,noMatch.id)).lines[0].id},"board")).rejects.toThrow("unverified"); + await service.adjust(f.company.id,charge.id,input,"board"); + await expect(service.adjust(f.company.id,charge.id,{...input,correctedCents:"2"},"board")).rejects.toThrow("different contents"); + expect((await db.select().from(costEvents).where(eq(costEvents.id,charge.id)))[0].reportedCostCents).toBe("0.1234567"); + }); + it("aggregates attempts exactly, rejects incomplete attempts, and never lets a late old attempt replace the current attempt", async () => { + const f = await fixture(), spool = path.join(directory,randomUUID()); + const recorder = await createRunUsageRecorder(db,{companyId:f.company.id,runId:f.run.id,adapterType:"process"},spool); + const first=randomUUID(), second=randomUUID(); + await recorder.capture({attemptId:first,usage,costUsdExact:"0.010000001",complete:true,billingType:"api"}); + await recorder.capture({attemptId:second,usage,costUsd:0.02,complete:false,billingType:"api"}); + await recorder.capture({attemptId:first,usage,costUsdExact:"0.010000001",complete:true,billingType:"api"}); + const result = await recorder.complete({exitCode:0,signal:null,timedOut:false,usage,costUsd:0.03,billingType:"api"}); + expect(result).toMatchObject({costUsdExact:"0.040000001",complete:true,usage:{inputTokens:14,cachedInputTokens:22,outputTokens:6}}); + await recorder.capture({attemptId:randomUUID(),usage,complete:false,billingType:"subscription"}); + const uncertain = await recorder.complete({exitCode:1,signal:null,timedOut:false,usage}); + expect(uncertain).toMatchObject({costUsd:0.040000001,costUsdExact:"0.040000001",costStatus:"unpriced",billingType:"unknown"}); + const partial = await recorder.complete({exitCode:1,signal:null,timedOut:false}); + expect(partial.complete).toBe(false); + expect(partial.usage?.inputTokens).toBe(21); + }); + it.each(["block", "allow"] as const)("keeps known attempted-run spend when unknown prices %s new work", async (unpricedUsagePolicy) => { + const f = await fixture(), budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, unpricedUsagePolicy }, "board"); + await reserveRunBudget(db, f.company.id, f.run.id, null); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "process" }, path.join(directory, randomUUID())); + await recorder.capture({ attemptId: randomUUID(), complete: true, billingType: "metered_api", usage, costUsdExact: "0.010000001" }); + const aggregate = await recorder.capture({ attemptId: randomUUID(), complete: true, billingType: "metered_api", usage, costUsd: null, costStatus: "unpriced" }); + expect(aggregate).toMatchObject({ costUsdExact: "0.010000001", costStatus: "unpriced", complete: true }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(true); + expect(await accountRunCost(db, f.run.id)).toBe(false); + expect(await costService(db).summary(f.company.id)).toMatchObject({ spendCentsExact: "1.0000001", unpricedEventCount: 1, pricingComplete: false, pendingRunCount: 0 }); + const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)); + expect(runtime.totalCostCents).toBe(1.0000001); + const [reservation] = await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)); + expect(reservation.state).toBe("settled"); + expect((await budgets.getInvocationBlock(f.company.id, f.agent.id)) !== null).toBe(unpricedUsagePolicy === "block"); + }); + + it("refuses invalid checkpoints and does not certify a run after capture failure", async () => { + const f = await fixture(); + const recorder = await createRunUsageRecorder(db,{companyId:f.company.id,runId:f.run.id,adapterType:"process"},path.join(directory,randomUUID())); + await expect(recorder.capture({complete:true,costUsdExact:"NaN"})).rejects.toThrow(); + const result = await recorder.complete({exitCode:0,signal:null,timedOut:false,usage,costUsd:0.01}); + expect(result.complete).toBe(false); + await expect(createRunUsageRecorder(db,{companyId:f.company.id,runId:randomUUID(),adapterType:"process"},path.join(directory,randomUUID()))).rejects.toThrow("cannot accept"); + }); + it("preserves disk errors, rejects missing runs, and handles invalid spool entries without deleting evidence", async () => { + const f=await fixture(), spool=path.join(directory,randomUUID()); + const envelope:UsageReceiptEnvelope={schema:"paperclip/accounting-receipt/v1",id:randomUUID(),companyId:f.company.id,runId:randomUUID(),sourceId:randomUUID(),sequence:1,receivedAt:new Date().toISOString(),adapterType:"process",receipt:{complete:true,costUsd:1}}; + await expect(persistUsageReceipt(db,envelope)).rejects.toThrow("not found"); + const rename=vi.spyOn(fs,"rename").mockRejectedValueOnce(new Error("Disk rename unavailable")); + try { await expect(spoolUsageReceipt(envelope,spool)).rejects.toThrow("Disk rename"); } finally { rename.mockRestore(); } + expect(await fs.readdir(spool)).toEqual([]); + await fs.writeFile(path.join(spool,"large.json"),"x".repeat(1024*1024+1)); + expect((await replayUsageReceipts(db,spool)).failed).toBe(1); + const read=vi.spyOn(fs,"readdir").mockRejectedValueOnce(Object.assign(new Error("denied"),{code:"EACCES"})); + try { await expect(replayUsageReceipts(db,spool)).rejects.toThrow("denied"); } finally { read.mockRestore(); } + }); + it("uses the isolated instance spool and retains evidence when retry rotation itself fails", async () => { + const f = await fixture(); + const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: f.company.id, runId: f.run.id, + sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: true, costUsd: 0 } }; + const file = await spoolUsageReceipt(envelope); + expect(path.dirname(file)).toBe(usageReceiptSpoolPath()); + expect((await replayUsageReceipts(db)).replayed).toBe(1); + const spool = path.join(directory, randomUUID()); await fs.mkdir(spool); + const bad = path.join(spool, "bad.json"); await fs.writeFile(bad, "invalid"); + const rename = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("read-only directory")); + try { expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 0, failed: 1 }); } finally { rename.mockRestore(); } + expect(await fs.readFile(bad, "utf8")).toBe("invalid"); + }); + +}); diff --git a/server/src/__tests__/cost-accounting-operators.test.ts b/server/src/__tests__/cost-accounting-operators.test.ts new file mode 100644 index 0000000000..03c3ceb98e --- /dev/null +++ b/server/src/__tests__/cost-accounting-operators.test.ts @@ -0,0 +1,502 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { agents, agentRuntimeState, budgetReservations, companies, costEvents, costAdjustments, createDb, heartbeatRuns, runUsageReceipts, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { accountingIntegrityService } from "../services/accounting-integrity.js"; +import { billingReconciliationService } from "../services/billing-reconciliation.js"; +import { budgetService } from "../services/budgets.js"; +import { reserveRunBudget } from "../services/budget-reservations.js"; +import { costService } from "../services/costs.js"; +import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js"; +import { createRunUsageRecorder, persistUsageReceipt, replayUsageReceipts, spoolUsageReceipt, type UsageReceiptEnvelope } from "../services/usage-receipts.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint"; +import { runChildProcess } from "@paperclipai/adapter-utils/server-utils"; +import { createClaudeStreamParser, parseClaudeStreamJson } from "../../../packages/adapters/claude-local/src/server/parse.js"; + +import { parseCodexJsonl } from "../../../packages/adapters/codex-local/src/server/parse.js"; +import { parseOpenCodeJsonl } from "../../../packages/adapters/opencode-local/src/server/parse.js"; +import { parseCursorJsonl } from "../../../packages/adapters/cursor-local/src/server/parse.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("accounting operations and durable receipts (PostgreSQL)", () => { + let database: Awaited>; + let db: ReturnType; + let directory: string; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-operators-"); db = createDb(database.connectionString); directory = await fs.mkdtemp(path.join(os.tmpdir(), "accounting-spool-test-")); }, 30_000); + afterAll(async () => { await database?.cleanup(); await fs.rm(directory, { recursive: true, force: true }); }); + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Accounting", issuePrefix: `A${randomUUID().slice(0,7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning(); + return { company, agent }; + } + async function runFor(f: Awaited>, status = "running") { + return (await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, invocationSource: "on_demand", status }).returning())[0]; + } + async function receipt(f: Awaited>, costCents: number | string = "1.1234567") { + return costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test-model", costCents, occurredAt: new Date(), idempotencyKey: randomUUID() }); + } + + it("keeps the last nanocent exact in storage, reports, projections and admission", async () => { + const f = await fixture(); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 2_000_000_000 }, "board"); + await receipt(f, "1999999999.9999999"); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1999999999.9999999"); + await receipt(f, "0.0000001"); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull(); + const huge = await fixture(); + await receipt(huge, "99999999999999999.0000001"); + expect((await accountingIntegrityService(db).inspect(huge.company.id)).findings).toEqual([]); + }); + + it("admits only affordable concurrent reservations and settles atomically with the receipt", async () => { + const f = await fixture(); const runs = await Promise.all(Array.from({ length: 4 }, () => runFor(f))); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 10, reservationCents: "6" }, "board"); + const results = await Promise.allSettled(runs.map(run => reserveRunBudget(db, f.company.id, run.id, null))); + expect(results.filter(r => r.status === "fulfilled")).toHaveLength(1); + const index = results.findIndex(r => r.status === "fulfilled"); const run = runs[index]; + expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("6.0000000"); + await expect(reserveRunBudget(db, f.company.id, run.id, null)).rejects.toThrow("already reserved"); + // Reservations never disappear merely because wall-clock time passed. + await db.update(budgetReservations).set({ createdAt: new Date("2000-01-01") }).where(eq(budgetReservations.runId, run.id)); + const denied = runs[(index + 1) % runs.length]; + await expect(reserveRunBudget(db, f.company.id, denied.id, null)).rejects.toThrow("reserved"); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), usageJson: { costUsdExact: "0.04", billingType: "metered_api", accountingReceiptReady: true } }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(true); + expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000"); + await reserveRunBudget(db, f.company.id, denied.id, null); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + }); + + it("reuses a held native reservation only for its current recovery lease", async () => { + const f = await fixture(); const run = await runFor(f); + const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover" }).returning(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, run.id)); + await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: run.id, issueId: issue.id, + phase: "running", leaseOwner: "successor", leaseExpiresAt: new Date(Date.now() + 60_000) }); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 10, reservationCents: "6" }, "board"); + const original = await reserveRunBudget(db, f.company.id, run.id, null); + await expect(reserveRunBudget(db, f.company.id, run.id, null)).rejects.toThrow("already reserved"); + await expect(reserveRunBudget(db, f.company.id, run.id, null, {}, "stale-owner")).rejects.toThrow("already reserved"); + expect(await reserveRunBudget(db, f.company.id, run.id, null, {}, "successor")).toEqual({ ...original, reused: true }); + expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("6.0000000"); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, run.id)); + await expect(reserveRunBudget(db, f.company.id, run.id, null, {}, "successor")).rejects.toThrow("already reserved"); + }); + + it.each([false, true])("settles absent usage without hiding explicit unknown pricing (%s)", async (unpriced) => { + const f = await fixture(); const run = await runFor(f); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board"); + await reserveRunBudget(db, f.company.id, run.id, null); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory); + await recorder.complete({ exitCode: 0, signal: null, timedOut: false, ...(unpriced ? { provider: "moonshot", costStatus: "unpriced" as const, costUsd: null } : {}) }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(true); + expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000"); + const summary = await costService(db).summary(f.company.id); + expect(summary.unpricedEventCount).toBe(unpriced ? 1 : 0); + const block = await budgetService(db).getInvocationBlock(f.company.id, f.agent.id); + if (unpriced) expect(block).not.toBeNull(); else expect(block).toBeNull(); + }); + + it.each([ + ...["turn.failed", "error", "turn.completed"].map(type => ({ + label: `Codex ${type}`, adapterType: "codex_local", parse: () => parseCodexJsonl(JSON.stringify({ type })), + })), + ...["result", "step_finish"].map(type => ({ + label: `Cursor ${type}`, adapterType: "cursor", parse: () => parseCursorJsonl(JSON.stringify({ type })), + })), + { label: "OpenCode step_finish", adapterType: "opencode_local", parse: () => parseOpenCodeJsonl(JSON.stringify({ type: "step_finish" })) }, + ])("keeps $label without usage unpriced and its reservation held", async ({ adapterType, parse }) => { + const f = await fixture(); const run = await runFor(f); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board"); + await reserveRunBudget(db, f.company.id, run.id, null); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType }, directory); + const parsed = parse(); + const accounting = { provider: "openai", biller: "openai", billingType: "api" as const, model: "gpt-6-astra", usageBasis: "per_run" as const, + usage: parsed.usageReported ? parsed.usage : undefined, costUsd: null, costStatus: "unpriced" as const }; + await recorder.capture({ ...accounting, complete: parsed.usageComplete }); + await recorder.complete({ ...accounting, exitCode: 1, signal: null, timedOut: false, usageComplete: parsed.usageComplete }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(stored.usageJson).toMatchObject({ costUsd: null, costUsdExact: null, costStatus: "unpriced", accountingReceiptReady: false }); + expect(await accountRunCost(db, run.id)).toBe(false); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]); + expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("10.0000000"); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull(); + }); + + it("recovers a flushed zero-usage OpenCode receipt after stopping before the adapter result is saved", async () => { + const f = await fixture(); const run = await runFor(f); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board"); + await reserveRunBudget(db, f.company.id, run.id, null); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "opencode_local" }, directory); + const log = createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => ({ + provider: "openai", biller: "openai", billingType: "unknown", model: "openai/test", usageBasis: "per_run", + usage: parseOpenCodeJsonl(records).usage, costUsd: null, complete: false, + })); + await log("stdout", JSON.stringify({ type: "step_finish", part: { tokens: { input: 0, output: 0 } } })); + await log.flush({ complete: true }); + // No recorder.complete(adapterResult): the process stopped during cleanup. + await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(true); + const events = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ inputTokens: 0, outputTokens: 0, costCents: 0, costStatus: "unpriced" }); + expect((await accountingIntegrityService(db).health(f.company.id))).toMatchObject({ heldReservationCents: "0.0000000", pendingRunCount: 0 }); + expect(await accountRunCost(db, run.id)).toBe(false); + }); + + it("keeps an interrupted partial receipt pending and settles its reservation when evidence becomes complete", async () => { + const f = await fixture(); const run = await runFor(f); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board"); + await reserveRunBudget(db, f.company.id, run.id, null); + const usage = { accountingReceiptReady: false, costUsdExact: "0.01", inputTokens: 5, provider: "test", billingType: "metered_api" }; + await db.update(heartbeatRuns).set({ status: "interrupted", usageJson: usage }).where(eq(heartbeatRuns.id, run.id)); + const integrity = accountingIntegrityService(db); + expect((await integrity.health(f.company.id)).pendingRunCount).toBe(1); + expect((await integrity.health(f.company.id)).items[0]).toMatchObject({ runId: run.id, state: "waiting_for_receipt" }); + expect((await costService(db).summary(f.company.id)).pendingRunCount).toBe(1); + expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_acknowledgement")).toBe(true); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull(); + expect(await accountRunCost(db, run.id)).toBe(false); + expect((await integrity.health(f.company.id)).heldReservationCents).toBe("10.0000000"); + await db.update(heartbeatRuns).set({ usageJson: { ...usage, accountingReceiptReady: true } }).where(eq(heartbeatRuns.id, run.id)); + await reconcileRunCosts(db); + expect((await costService(db).summary(f.company.id))).toMatchObject({ pendingRunCount: 0, spendCents: 1 }); + expect((await integrity.health(f.company.id)).heldReservationCents).toBe("0.0000000"); + expect((await integrity.inspect(f.company.id)).findings).toEqual([]); + expect(await accountRunCost(db, run.id)).toBe(false); + }); + + it("independently detects and repairs corrupted projections with a stale-review guard and audit", async () => { + const f = await fixture(); const run = await runFor(f, "succeeded"); + await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { accountingReceiptReady: true, costUsdExact: "0.012345678", inputTokens: 7, cachedInputTokens: 3, outputTokens: 11 } }).where(eq(heartbeatRuns.id, run.id)); + await accountRunCost(db, run.id); + const service = accountingIntegrityService(db); + expect((await service.inspect(f.company.id)).findings).toEqual([]); + await db.update(companies).set({ spentMonthlyCents: 999 }).where(eq(companies.id, f.company.id)); + await db.update(agents).set({ spentMonthlyCents: 888 }).where(eq(agents.id, f.agent.id)); + await db.update(agentRuntimeState).set({ totalCostCents: 777, totalInputTokens: 600 }).where(eq(agentRuntimeState.agentId, f.agent.id)); + const review = await service.inspect(f.company.id); + expect(review.findings.map(f => f.kind)).toEqual(["agent_projection", "company_projection", "runtime_projection"]); + await db.update(companies).set({ spentMonthlyCents: 998 }).where(eq(companies.id, f.company.id)); + await expect(service.repair(f.company.id, review.fingerprint, "Fix drift", "board")).rejects.toThrow("changed since inspection"); + const fresh = await service.inspect(f.company.id); + expect((await service.repair(f.company.id, fresh.fingerprint, "Fix drift", "board")).findings).toEqual([]); + const audit = await db.execute(sql`select details from activity_log where company_id = ${f.company.id} and action = 'accounting.projections_repaired'`); + expect(audit).toHaveLength(1); + expect(await accountRunCost(db, run.id)).toBe(false); + expect((await service.inspect(f.company.id)).findings).toEqual([]); + }); + + it.each(["before", "after", "concurrent"])("accounts separate run charges %s the provider receipt without false integrity findings", async (timing) => { + const f = await fixture(); const run = await runFor(f, "succeeded"); + await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { costUsdExact: "0.01", inputTokens: 7, cachedInputTokens: 3, outputTokens: 11 } }).where(eq(heartbeatRuns.id, run.id)); + const extra = { agentId: f.agent.id, heartbeatRunId: run.id, provider: "test", model: "tool", costCents: "0.1234567", inputTokens: 2, cachedInputTokens: 4, outputTokens: 6, occurredAt: new Date(), idempotencyKey: "extra-charge" }; + const costs = costService(db); + if (timing === "before") await costs.createEvent(f.company.id, extra); + if (timing === "concurrent") await Promise.all([costs.createEvent(f.company.id, extra), accountRunCost(db, run.id)]); + else await accountRunCost(db, run.id); + const event = await costs.createEvent(f.company.id, extra); + // Replaying either source cannot count it twice. + await costs.createEvent(f.company.id, extra); + expect(await accountRunCost(db, run.id)).toBe(false); + const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)); + expect(runtime).toMatchObject({ totalCostCents: 1.1234567, totalInputTokens: 9, totalCachedInputTokens: 7, totalOutputTokens: 17 }); + const integrity = accountingIntegrityService(db); + expect((await integrity.inspect(f.company.id)).findings).toEqual([]); + await billingReconciliationService(db).adjust(f.company.id, event.id, { idempotencyKey: "extra-correction", expectedCents: "0.1234567", correctedCents: "0.2234567", reason: "Provider correction", pricing: { source: "provider_invoice", evidence: "external charge" } }, "board"); + expect((await costs.summary(f.company.id)).spendCentsExact).toBe("1.2234567"); + expect((await integrity.inspect(f.company.id)).findings).toEqual([]); + await db.update(agentRuntimeState).set({ totalCostCents: 999 }).where(eq(agentRuntimeState.agentId, f.agent.id)); + const review = await integrity.inspect(f.company.id); + expect(review.findings.map(f => f.kind)).toEqual(["runtime_projection"]); + expect((await integrity.repair(f.company.id, review.fingerprint, "Fix drift", "board")).findings).toEqual([]); + // An extra charge cannot conceal a missing original provider receipt. + await db.delete(costEvents).where(and(eq(costEvents.heartbeatRunId, run.id), eq(costEvents.idempotencyKey, `heartbeat:${run.id}:final`))); + expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_receipt")).toBe(true); + }); + + it("rejects public charges that impersonate the provider receipt namespace", async () => { + const f = await fixture(); + await expect(costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test", costCents: 1, occurredAt: new Date(), idempotencyKey: "heartbeat:fake:final" })).rejects.toThrow("reserved"); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]); + }); + + it.each(["zero", "paid", "tokens"] as const)("preserves model attribution after a retry with %s earlier usage", async kind => { + const f = await fixture(); const run = await runFor(f); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory); + await recorder.capture({ attemptId: randomUUID(), provider: "anthropic", billingType: "metered_api", + usage: { inputTokens: kind === "tokens" ? 1 : 0, outputTokens: 0, cacheWriteTokens: undefined }, + costUsd: kind === "paid" ? 0.001 : 0, complete: true }); + const success = { attemptId: randomUUID(), provider: "anthropic", biller: "anthropic", billingType: "metered_api" as const, model: "mixed", + usage: { inputTokens: 30, outputTokens: 4, cachedInputTokens: 100 }, costUsd: 0.007, complete: true, + usageByModel: [ + { model: "large", costUsd: 0.005, usage: { inputTokens: 20, outputTokens: 3, cachedInputTokens: 100 } }, + { model: "small", costUsd: 0.002, usage: { inputTokens: 10, outputTokens: 1 } }, + ] }; + await recorder.capture(success); + const final = await recorder.complete({ ...success, exitCode: 0, signal: null, timedOut: false, usageComplete: true }); + expect(final.usageByModel).toEqual(kind === "zero" ? success.usageByModel : undefined); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(true); + const rows = await costService(db).byProvider(f.company.id); + expect(rows.map(row => row.model).sort()).toEqual(kind === "zero" ? ["large", "small"] : ["mixed"]); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe(kind === "paid" ? "0.8000000" : "0.7000000"); + }); + + it("imports invoices idempotently and appends reviewed corrections without losing the provider receipt", async () => { + const f = await fixture(); + const originalPricing = { source: "rate_card" as const, version: "2026-09-01", inputCentsPerMillion: "12.3456789" }; + const event = await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test-model", + costCents: "1.1234567", occurredAt: new Date(), pricingProvenance: originalPricing }); + const service = billingReconciliationService(db); + const invoice = { biller: "test", externalId: "invoice-1", currency: "usd", lines: [{ externalId: "request-1", amountCents: "2.0000001", occurredAt: new Date().toISOString(), costEventId: event.id }] }; + const imported = await service.importInvoice(f.company.id, invoice, "board"); + expect((await service.importInvoice(f.company.id, invoice, "board")).id).toBe(imported.id); + await expect(service.importInvoice(f.company.id, { ...invoice, currency: "EUR" }, "board")).rejects.toThrow("different contents"); + const review = await service.reconcile(f.company.id, imported.id); + expect(review.lines[0]).toMatchObject({ status: "difference", recordedCents: "1.1234567", differenceCents: "0.8765434" }); + const input = { idempotencyKey: "fix-1", expectedCents: "1.1234567", correctedCents: "2.0000001", invoiceLineId: review.lines[0].id, reason: "Provider invoice", pricing: { source: "provider_invoice" as const, evidence: "invoice-1" } }; + const corrected = await service.adjust(f.company.id, event.id, input, "board"); + expect(corrected.previousPricing).toEqual(originalPricing); + expect(corrected.pricing).toEqual(input.pricing); + expect((await service.adjust(f.company.id, event.id, input, "board")).id).toBe(corrected.id); + expect((await service.reconcile(f.company.id, imported.id)).lines[0].status).toBe("matched"); + const [row] = await db.select().from(costEvents).where(eq(costEvents.id, event.id)); + expect(row.reportedCostCents).toBe("1.1234567"); expect(row.receiptHash).toBe(event.receiptHash); + expect((await service.adjustments(f.company.id, event.id))).toHaveLength(1); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + await expect(service.adjust(f.company.id, event.id, { ...input, idempotencyKey: "stale" }, "board")).rejects.toThrow("changed since review"); + const second = await service.adjust(f.company.id, event.id, { ...input, invoiceLineId: undefined, idempotencyKey: "fix-2", + expectedCents: "2.0000001", correctedCents: "3", pricing: { source: "operator" }, reason: "Reviewed updated evidence" }, "board"); + expect(second.previousPricing).toEqual(input.pricing); + expect((await service.adjustments(f.company.id, event.id))).toHaveLength(2); + }); + + it("refuses cross-company references, guesses, duplicate invoice matches, and currency conversion", async () => { + const f = await fixture(), foreign = await fixture(); const event = await receipt(f); + const service = billingReconciliationService(db); + const base = { biller: "test", externalId: "invoice", currency: "USD", lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString(), costEventId: event.id }] }; + await expect(service.importInvoice(foreign.company.id, base, "board")).rejects.toThrow("not found"); + await expect(service.adjust(foreign.company.id, event.id, { idempotencyKey: "bad", expectedCents: 0, correctedCents: 1, reason: "test", pricing: { source: "operator" } }, "board")).rejects.toThrow("not found"); + const duplicate = await service.importInvoice(f.company.id, { ...base, lines: [...base.lines, { ...base.lines[0], externalId: "second" }] }, "board"); + expect((await service.reconcile(f.company.id, duplicate.id)).lines.map(l => l.status)).toEqual(["ambiguous", "ambiguous"]); + const eur = await service.importInvoice(f.company.id, { ...base, externalId: "eur", currency: "EUR" }, "board"); + expect((await service.reconcile(f.company.id, eur.id)).lines[0].status).toBe("unsupported_currency"); + await expect(service.reconcile(foreign.company.id, eur.id)).rejects.toThrow("not found"); + expect(await db.select().from(costAdjustments).where(eq(costAdjustments.companyId, f.company.id))).toEqual([]); + }); + + it.each(["provider_invoice", "rate_card"] as const)("prices unknown usage from %s and preserves its estimate status", async (source) => { + const f = await fixture(); const run = await runFor(f, "failed"); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 10 }, "board"); + await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { accountingReceiptReady: true, inputTokens: 9, provider: "test" } }).where(eq(heartbeatRuns.id, run.id)); + await accountRunCost(db, run.id); + expect((await accountingIntegrityService(db).health(f.company.id)).unpricedEventCount).toBe(1); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull(); + const [event] = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id)); + await billingReconciliationService(db).adjust(f.company.id, event.id, { idempotencyKey: "pricing", expectedCents: 0, correctedCents: "1.0000001", reason: "Verified provider invoice", pricing: { source, evidence: "billing-export" } }, "board"); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect((await accountingIntegrityService(db).health(f.company.id)).unpricedEventCount).toBe(0); + expect((await costService(db).summary(f.company.id)).estimatedEventCount).toBe(source === "rate_card" ? 1 : 0); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + }); + + it("spools a receipt during database failure, replays once, fences old sources, and preserves partial uncertainty", async () => { + const f = await fixture(); const run = await runFor(f); const spool = path.join(directory, randomUUID()); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, spool); + const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Database unavailable")); + await recorder.capture({ costUsdExact: "0.010000001", usage: { inputTokens: 7, outputTokens: 2 }, complete: false }); + transaction.mockRestore(); + expect((await fs.readdir(spool)).filter(n => n.endsWith(".json"))).toHaveLength(1); + expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 1, failed: 0 }); + expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 0, failed: 0 }); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(false); + expect((await accountingIntegrityService(db).health(f.company.id)).items[0].state).toBe("waiting_for_receipt"); + await recorder.complete({ exitCode: 0, signal: null, timedOut: false, costUsdExact: "0.010000001", usage: { inputTokens: 7, outputTokens: 2 } }); + expect(await accountRunCost(db, run.id)).toBe(true); + const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, run.id)); + expect(receipts).toHaveLength(2); + await persistUsageReceipt(db, receipts[0].receiptJson as UsageReceiptEnvelope); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.0000001"); + const tampered = { ...(receipts[0].receiptJson as UsageReceiptEnvelope), receipt: { complete: true, costUsd: 123 } }; + await expect(persistUsageReceipt(db, tampered)).rejects.toThrow("Conflicting usage"); + }); + + it("settles a complete streamed Claude receipt when the final stdout tail clips its result", async () => { + const f = await fixture(); const run = await runFor(f); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, reservationCents: 10 }, "board"); + await reserveRunBudget(db, f.company.id, run.id, null); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory); + const consume = createClaudeStreamParser(); + const log = createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => { + const parsed = consume(records); + return { provider: "anthropic", biller: "anthropic", model: "claude-fable-5-1", billingType: "metered_api", + usage: parsed.usage ?? undefined, costUsd: parsed.costUsd, complete: parsed.resultJson !== null }; + }); + const proc = await runChildProcess(run.id, process.execPath, ["-e", `console.log(JSON.stringify({ + type: "result", subtype: "success", result: "x".repeat(5 * 1024 * 1024), + total_cost_usd: 0.012345678, usage: { input_tokens: 7, cache_read_input_tokens: 3, output_tokens: 2 } + }))`], { cwd: directory, env: {}, timeoutSec: 10, graceSec: 1, onLog: log }); + await log.flush(); + const tail = parseClaudeStreamJson(proc.stdout); + expect(tail.resultJson).toBeNull(); + const final = await recorder.complete({ exitCode: proc.exitCode, signal: proc.signal, timedOut: proc.timedOut, + provider: "anthropic", billingType: "metered_api", usage: tail.usage ?? undefined, costUsd: tail.costUsd, + usageComplete: tail.resultJson !== null }); + expect(final).toMatchObject({ complete: true, costUsd: 0.012345678, usage: { inputTokens: 7, cachedInputTokens: 3, outputTokens: 2 } }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(true); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.2345678"); + expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000"); + }); + + it("holds failed retry accounting when flush throws before final completion", async () => { + const f = await fixture(); const run = await runFor(f); const spool = path.join(directory, randomUUID()); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, reservationCents: 10 }, "board"); + await reserveRunBudget(db, f.company.id, run.id, null); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool); + const attempt = () => { + const consume = createClaudeStreamParser(); + return createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => { + const parsed = consume(records); + return { provider: "anthropic", billingType: "metered_api", usage: parsed.usage ?? undefined, + costUsd: parsed.costUsd, complete: parsed.resultJson !== null }; + }); + }; + const rejectedResume = attempt(); + await rejectedResume("stdout", JSON.stringify({ type: "result", subtype: "error_during_execution", total_cost_usd: 0, + usage: { input_tokens: 0, output_tokens: 0 } }) + "\n"); + await rejectedResume.flush(); + const [prior] = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, run.id)); + expect((prior.receiptJson as UsageReceiptEnvelope).receipt.complete).toBe(true); + const paidRetry = attempt(); + const write = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("Receipt disk full")); + try { + await paidRetry("stdout", JSON.stringify({ type: "result", subtype: "success", total_cost_usd: 0.25, + usage: { input_tokens: 10, output_tokens: 2 } }) + "\n"); + await expect(paidRetry.flush()).rejects.toThrow("Receipt disk full"); + } finally { write.mockRestore(); } + // This is the adapter's throw path: complete() is never called. + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + const read = async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]; + expect(await read()).toMatchObject({ costAccountingPending: true, costAccountedAt: null, + accountingLastError: "usage_capture_failed", usageJson: { accountingReceiptReady: false, accountingCaptureFailed: true } }); + expect(await accountRunCost(db, run.id)).toBe(false); + // Old and later complete envelopes must not clear lost-capture evidence. + await spoolUsageReceipt(prior.receiptJson as UsageReceiptEnvelope, spool); + await spoolUsageReceipt({ ...prior.receiptJson as UsageReceiptEnvelope, id: randomUUID(), sequence: 100 }, spool); + expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 2, failed: 0 }); + expect((await read()).usageJson?.accountingReceiptReady).toBe(false); + const replacement = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool); + expect((await replacement.complete({ exitCode: 0, signal: null, timedOut: false, costUsd: 0.25 })).complete).toBe(false); + // Even a finalizer overwriting readiness or claiming pre-provider failure cannot erase the fence. + await db.update(heartbeatRuns).set({ usageJson: { ...(await read()).usageJson, accountingReceiptReady: true }, + resultJson: { executionRecovery: { providerWorkStarted: false } } }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(false); + expect((await read()).costAccountedAt).toBeNull(); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id))).toEqual([]); + expect(await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id))).toEqual([]); + expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("10.0000000"); + }); + + it.each(["capture", "complete", "invalid"] as const)("persists the failure fence before %s rejects", async kind => { + const f = await fixture(); const run = await runFor(f); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory); + await recorder.persistFailure(); // Healthy failure finalization has no fence to save. + await recorder.capture({ costUsd: 0, complete: true }); + const rename = vi.spyOn(fs, "rename"); + if (kind !== "invalid") rename.mockRejectedValueOnce(new Error("Receipt disk full")); + try { + const result = kind === "complete" + ? recorder.complete({ exitCode: 0, signal: null, timedOut: false, costUsd: 1 }) + : recorder.capture({ costUsd: kind === "invalid" ? -1 : 1, complete: true }); + await expect(result).rejects.toThrow(); + const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(stored.usageJson).toMatchObject({ accountingCaptureFailed: true, accountingReceiptReady: false }); + } finally { rename.mockRestore(); } + }); + + it("retries a failed durable fence before allowing failure finalization", async () => { + const f = await fixture(); const run = await runFor(f); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory); + await recorder.capture({ costUsd: 0, complete: true }); + const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Database unavailable")); + try { + await expect(recorder.capture({ costUsd: -1, complete: true })).rejects.toThrow(); + expect(transaction).toHaveBeenCalledTimes(1); + transaction.mockRejectedValueOnce(new Error("Database still unavailable")); + await expect(recorder.persistFailure()).rejects.toThrow("Database still unavailable"); + await recorder.persistFailure(); + } finally { transaction.mockRestore(); } + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(false); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toMatchObject({ + costAccountedAt: null, accountingLastError: "usage_capture_failed", + usageJson: { accountingCaptureFailed: true, accountingReceiptReady: false }, + }); + }); + + it("does not let a replaced recorder's failed capture poison its successor", async () => { + const f = await fixture(); const run = await runFor(f); + const input = { companyId: f.company.id, runId: run.id, adapterType: "process" }; + const old = await createRunUsageRecorder(db, input, directory); + const current = await createRunUsageRecorder(db, input, directory); + await current.capture({ costUsd: 0.25, complete: true }); + await expect(old.capture({ costUsd: -1, complete: true })).rejects.toThrow(); + await old.persistFailure(); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(true); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("25.0000000"); + }); + + it.each(["more_tokens", "more_cost", "new_attempt"])("keeps additional incomplete provider work pending (%s)", async (kind) => { + const f = await fixture(); const run = await runFor(f); + const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory); + await recorder.capture({ provider: "anthropic", usage: { inputTokens: 7, outputTokens: 2 }, costUsd: 0.01, complete: true }); + if (kind === "new_attempt") await recorder.capture({ attemptId: randomUUID(), provider: "anthropic", usage: { inputTokens: 1, outputTokens: 0 }, complete: false }); + const final = await recorder.complete({ exitCode: 1, signal: null, timedOut: false, usageComplete: false, + usage: { inputTokens: kind === "more_tokens" ? 8 : 7, outputTokens: 2 }, costUsd: kind === "more_cost" ? 0.02 : 0.01 }); + expect(final.complete).toBe(false); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + expect(await accountRunCost(db, run.id)).toBe(false); + }); + + it.skipIf(process.platform === "win32")("rejects recorder setup before dispatch when its new directory parent cannot be flushed", async () => { + const f = await fixture(); const run = await runFor(f); + const spool = path.join(directory, randomUUID(), "receipts"); + const realParent = await fs.realpath(directory); + const open = fs.open.bind(fs); + const probe = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + if (args[0] === realParent) throw Object.assign(new Error("unflushed parent"), { code: "EACCES" }); + return open(...args); + }); + try { + await expect(createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool)).rejects.toMatchObject({ code: "EACCES" }); + expect(probe.mock.calls.some(([file]) => String(file).endsWith(".probe"))).toBe(false); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].usageJson?.accountingReceiptSourceId).toBeUndefined(); + } finally { probe.mockRestore(); } + await expect(createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool)).resolves.toBeDefined(); + }); + + it("retains invalid spool evidence and bounds recovery work", async () => { + const spool = path.join(directory, randomUUID()); await fs.mkdir(spool); + await fs.writeFile(path.join(spool, "bad.json"), "not json"); + expect(await replayUsageReceipts(db, spool, 1)).toEqual({ replayed: 0, failed: 1 }); + expect((await fs.readdir(spool))[0]).toMatch(/^retry-/); + expect(await replayUsageReceipts(db, path.join(spool, "missing"))).toEqual({ replayed: 0, failed: 0 }); + const f = await fixture(); const run = await runFor(f); + const unknown: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: f.company.id, runId: run.id, + sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: true, costUsd: 10 } }; + await spoolUsageReceipt(unknown, spool); + expect((await replayUsageReceipts(db, spool, 1)).replayed).toBe(1); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].costAccountingPending).toBe(false); + }); +}); diff --git a/server/src/__tests__/cost-accounting-performance.test.ts b/server/src/__tests__/cost-accounting-performance.test.ts new file mode 100644 index 0000000000..f4d8428da2 --- /dev/null +++ b/server/src/__tests__/cost-accounting-performance.test.ts @@ -0,0 +1,266 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, issues, nativeRunFinalizations, projects, type Db } from "@paperclipai/db"; +import { billingReconciliationService } from "../services/billing-reconciliation.js"; +import { accountingIntegrityService } from "../services/accounting-integrity.js"; +import { budgetService, budgetServiceInTransaction } from "../services/budgets.js"; +import { withAccountingTransaction } from "../services/accounting-transaction.js"; +import { costService } from "../services/costs.js"; +import { createCostAccountingReconciler } from "../services/run-cost-accounting.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +function deferred() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +describe("accounting performance invariants (PostgreSQL)", () => { + let database: Awaited>, db: Db; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-performance-"); db = createDb(database.connectionString); }, 30_000); + afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); }); + afterAll(async () => { await database?.cleanup(); }); + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Performance", issuePrefix: `P${randomUUID().slice(0, 7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process" }).returning(); + const [project] = await db.insert(projects).values({ companyId: company.id, name: "Work" }).returning(); + return { company, agent, project }; + } + function trace() { + const queries: string[] = []; + const traced = drizzle(db.$client, { logger: { logQuery(query) { queries.push(query); } } }) as unknown as Db; + return { traced, ledgerReads: () => queries.filter(query => query.startsWith("select") && query.includes('from "cost_events"')) }; + } + + it("reads health while a writer holds the company lock and keeps counts/items in one snapshot", async () => { + const f = await fixture(); + const writer = await db.$client.reserve(); + await writer`begin`; + try { + await writer`select id from companies where id = ${f.company.id} for no key update`; + const original = db.transaction.bind(db); + const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => { + // A lock attempt fails promptly. There is no elapsed-time pass/fail assertion. + await tx.execute(sql`set local lock_timeout = '200ms'`); + const execute = tx.execute.bind(tx); + let counted = false; + vi.spyOn(tx, "execute").mockImplementation(query => { + const result = execute(query); + const executeRaw = result.execute.bind(result); + vi.spyOn(result, "execute").mockImplementation(async () => { + const rows = await executeRaw(); + if (!counted) { + counted = true; + // Commit a pending row between health's counts and item list. + await writer`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending) + values (${f.company.id},${f.agent.id},'failed',true)`; + await writer`commit`; + } + return rows; + }); + return result; + }); + return callback(tx); + }, config)); + expect(await accountingIntegrityService(db).health(f.company.id)).toMatchObject({ pendingRunCount: 0, items: [] }); + expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" }); + transaction.mockRestore(); + expect(await accountingIntegrityService(db).health(f.company.id)).toMatchObject({ pendingRunCount: 1, items: [expect.objectContaining({ state: "retryable" })] }); + } finally { await writer`rollback`; writer.release(); } + // Remove the fixture's pending debt before the global recovery tests. + await db.execute(sql`delete from heartbeat_runs where company_id = ${f.company.id}`); + await expect(accountingIntegrityService(db).health(randomUUID())).rejects.toThrow("Company not found"); + }); + + it("keeps summary incompleteness consistent when an unpriced run settles between reads", async () => { + const f = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, + status: "failed", costAccountingPending: true, finishedAt: new Date() }).returning(); + const writer = await db.$client.reserve(); + await writer`begin`; + try { + await writer`select id from companies where id = ${f.company.id} for no key update`; + const original = db.transaction.bind(db); + const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => { + await tx.execute(sql`set local lock_timeout = '200ms'`); + const select = tx.select.bind(tx); + vi.spyOn(tx, "select").mockImplementation((...args: Parameters) => { + const query = select(...args); + const from = query.from.bind(query); + vi.spyOn(query, "from").mockImplementation(table => { + const result = from(table); + if (table === costEvents) { + const execute = result.execute.bind(result); + vi.spyOn(result, "execute").mockImplementation(async () => { + const rows = await execute(); + await writer`insert into cost_events (company_id,agent_id,provider,model,cost_cents,cost_status,billing_type,occurred_at) + values (${f.company.id},${f.agent.id},'fixture','fixture',0,'unpriced','metered_api',now())`; + await writer`update heartbeat_runs set cost_accounting_pending=false where id=${run.id}`; + await writer`commit`; + return rows; + }); + } + return result; + }); + return query; + }); + return callback(tx); + }, config)); + expect(await costService(db).summary(f.company.id)).toMatchObject({ pendingRunCount: 1, unpricedEventCount: 0, pricingComplete: false }); + transaction.mockRestore(); + expect(await costService(db).summary(f.company.id)).toMatchObject({ pendingRunCount: 0, unpricedEventCount: 1, pricingComplete: false }); + } finally { await writer`rollback`; writer.release(); } + }); + + it("shares receipt replay across overlapping ticks and releases the guard after replay failure", async () => { + const reconcile = createCostAccountingReconciler(db); + const gate = deferred(); + const read = vi.spyOn(fs, "readdir").mockImplementationOnce(async () => { await gate.promise; throw new Error("Spool unavailable"); }); + const first = reconcile(), second = reconcile(); + expect(second).toBe(first); + expect(read).toHaveBeenCalledTimes(1); + const settled = Promise.allSettled([first, second]); + gate.resolve(); + expect(await settled).toEqual([expect.objectContaining({ status: "rejected" }), expect.objectContaining({ status: "rejected" })]); + read.mockRestore(); + expect(await reconcile()).toMatchObject({ scanned: 0, accounted: 0 }); + }); + + it("keeps the guard through budget delivery and retries after reconciliation failure", async () => { + const f = await fixture(); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board"); + await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: 1, occurredAt: new Date() }); + const entered = deferred(), gate = deferred(); + const cancelWorkForScope = vi.fn(async () => { entered.resolve(); await gate.promise; }); + const reconcile = createCostAccountingReconciler(db, { cancelWorkForScope }); + const first = reconcile(); + try { + await entered.promise; + expect(reconcile()).toBe(first); + expect(cancelWorkForScope).toHaveBeenCalledTimes(1); + } finally { gate.resolve(); } + await first; + const select = vi.spyOn(db, "select").mockImplementationOnce(() => { throw new Error("Database unavailable"); }); + await expect(reconcile()).rejects.toThrow("Database unavailable"); + select.mockRestore(); + const next = reconcile(); expect(next).not.toBe(first); + await next; + }); + + it("uses one ledger observation per overview policy and per admission policy", async () => { + const f = await fixture(); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100 }, "board"); + const { traced, ledgerReads } = trace(); + expect((await budgetService(traced).overview(f.company.id)).policies[0]).toMatchObject({ observedAmountExact: "0.0000000", status: "ok" }); + expect(ledgerReads()).toHaveLength(1); + await budgetService(traced).getInvocationBlock(f.company.id, f.agent.id); + expect(ledgerReads()).toHaveLength(2); + }); + + it("shares a scan across mixed policy scopes/windows without trusting monthly projections", async () => { + const f = await fixture(), foreign = await fixture(); + const [other] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning(); + const values = [ + { scopeType: "company", scopeId: f.company.id, windowKind: "calendar_month_utc", amount: 4 }, + { scopeType: "agent", scopeId: f.agent.id, windowKind: "lifetime", amount: 10 }, + { scopeType: "project", scopeId: f.project.id, windowKind: "lifetime", amount: 9 }, + { scopeType: "agent", scopeId: other.id, windowKind: "calendar_month_utc", amount: 1 }, + ]; + await db.insert(budgetPolicies).values(values.map(value => ({ ...value, companyId: f.company.id }))); + const base = { provider: "fixture", model: "fixture", costCents: 3, occurredAt: new Date() }; + const [event] = await db.insert(costEvents).values([ + { ...base, companyId: f.company.id, agentId: f.agent.id, projectId: f.project.id, costCents: 7, occurredAt: new Date("2000-01-01") }, + { ...base, companyId: f.company.id, agentId: f.agent.id, projectId: f.project.id }, + { ...base, companyId: f.company.id, agentId: other.id }, + { ...base, companyId: foreign.company.id, agentId: foreign.agent.id, costCents: 999 }, + ]).returning(); + await db.update(companies).set({ spentMonthlyCents: 0 }).where(eq(companies.id, f.company.id)); + const { traced, ledgerReads } = trace(); + await withAccountingTransaction(traced, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event)); + expect(ledgerReads()).toHaveLength(1); + const incidents = await db.execute<{ scope_type: string; amount_observed: string }>(sql`select scope_type,amount_observed::text from budget_incidents where company_id = ${f.company.id} order by scope_type`); + expect(incidents).toEqual([ + { scope_type: "agent", amount_observed: "10.0000000" }, + { scope_type: "company", amount_observed: "6.0000000" }, + { scope_type: "project", amount_observed: "10.0000000" }, + ]); + }); + + it("keeps unpriced and pending-run guards scoped when batching native recovery observations", async () => { + const f = await fixture(); + const [other] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning(); + await db.insert(budgetPolicies).values([ + { companyId: f.company.id, scopeType: "company", scopeId: f.company.id, windowKind: "calendar_month_utc", amount: 100 }, + { companyId: f.company.id, scopeType: "agent", scopeId: f.agent.id, windowKind: "lifetime", amount: 100 }, + { companyId: f.company.id, scopeType: "project", scopeId: f.project.id, windowKind: "lifetime", amount: 100 }, + ]); + const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover" }).returning(); + const [native] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, + status: "failed", runtimeMode: "native", nativeIssueId: issue.id, costAccountingPending: true, finishedAt: new Date() }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: native.id, issueId: issue.id, phase: "retryable_failure" }); + const base = { companyId: f.company.id, provider: "fixture", model: "fixture", costCents: 0, occurredAt: new Date() }; + const [event] = await db.insert(costEvents).values([ + { ...base, agentId: f.agent.id, projectId: f.project.id }, + { ...base, agentId: other.id, projectId: f.project.id, occurredAt: new Date("2000-01-01"), costStatus: "unpriced" }, + ]).returning(); + await withAccountingTransaction(db, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event)); + const overview = await budgetService(db).overview(f.company.id); + expect(overview.activeIncidents.map(row => row.scopeType)).toEqual(["project"]); + expect(overview.policies).toEqual(expect.arrayContaining([ + expect.objectContaining({ scopeType: "company", status: "ok", pendingRunCount: 1, unpricedEventCount: 0 }), + expect.objectContaining({ scopeType: "agent", status: "ok", pendingRunCount: 1, unpricedEventCount: 0 }), + expect.objectContaining({ scopeType: "project", status: "hard_stop", pendingRunCount: 0, unpricedEventCount: 1 }), + ])); + await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "failed", + costAccountingPending: true, finishedAt: new Date(), usageJson: { ledgerScope: { projectId: f.project.id } } }); + await withAccountingTransaction(db, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event)); + expect((await budgetService(db).overview(f.company.id)).activeIncidents.map(row => row.scopeType).sort()).toEqual(["agent", "company", "project"]); + }); + + + it("keeps incident amounts and overview windows tied to the observed UTC month across midnight", async () => { + const f = await fixture(); + await db.insert(budgetPolicies).values({ companyId: f.company.id, scopeType: "company", scopeId: f.company.id, + windowKind: "calendar_month_utc", amount: 1 }); + const [event] = await db.insert(costEvents).values({ companyId: f.company.id, agentId: f.agent.id, + provider: "fixture", model: "fixture", costCents: 1, occurredAt: new Date("2026-01-31T12:00:00Z") }).returning(); + const traced = drizzle(db.$client, { logger: { logQuery(query) { + if (query.startsWith("select") && query.includes('from "cost_events"')) vi.setSystemTime(new Date("2026-02-01T00:00:00Z")); + } } }) as unknown as Db; + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-01-31T23:59:59.999Z")); + await withAccountingTransaction(traced, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event)); + const incident = (await budgetService(db).overview(f.company.id)).activeIncidents[0]; + expect(incident).toMatchObject({ amountObserved: 1, windowStart: new Date("2026-01-01T00:00:00Z"), windowEnd: new Date("2026-02-01T00:00:00Z") }); + vi.setSystemTime(new Date("2026-01-31T23:59:59.999Z")); + expect((await budgetService(traced).overview(f.company.id)).policies[0]).toMatchObject({ + observedAmountExact: "1.0000000", windowStart: new Date("2026-01-01T00:00:00Z"), windowEnd: new Date("2026-02-01T00:00:00Z"), + }); + // The next operation uses February and releases the prior month's pause. + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + }); + + + it.each(["inspection", "invoice comparison"])("does not take the company write lock during %s", async kind => { + const f = await fixture(); + const invoice = await billingReconciliationService(db).importInvoice(f.company.id, { biller: "fixture", externalId: "invoice", currency: "USD", + lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString() }] }, "board"); + const writer = await db.$client.reserve(); + await writer`begin`; + try { + await writer`select id from companies where id = ${f.company.id} for no key update`; + const original = db.transaction.bind(db); + const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => { + await tx.execute(sql`set local lock_timeout = '200ms'`); + return callback(tx); + }, config)); + if (kind === "inspection") expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + else expect((await billingReconciliationService(db).reconcile(f.company.id, invoice.id)).lines).toHaveLength(1); + expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" }); + } finally { await writer`rollback`; writer.release(); } + }); + +}); diff --git a/server/src/__tests__/cost-accounting-processes.test.ts b/server/src/__tests__/cost-accounting-processes.test.ts new file mode 100644 index 0000000000..ed30d35515 --- /dev/null +++ b/server/src/__tests__/cost-accounting-processes.test.ts @@ -0,0 +1,110 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createConnection, createServer, type Socket } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, companies, costEvents, createDb, heartbeatRuns } from "@paperclipai/db"; +import { costService } from "../services/costs.js"; +import { accountingIntegrityService } from "../services/accounting-integrity.js"; +import { replayUsageReceipts } from "../services/usage-receipts.js"; +import { accountRunCost } from "../services/run-cost-accounting.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +const support = await getEmbeddedPostgresTestSupport(); +const root = fileURLToPath(new URL("../../..", import.meta.url)); +(support.supported && process.platform !== "win32" ? describe : describe.skip)("accounting across process and connection failures", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-processes-"); db = createDb(database.connectionString); }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Process", issuePrefix: `P${randomUUID().slice(0,7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning(); + return { company, agent }; + } + function worker(mode: string, input: unknown) { + return spawn(process.execPath, ["cli/node_modules/tsx/dist/cli.mjs", "server/src/__tests__/fixtures/accounting-writer.ts", mode, JSON.stringify(input)], { + cwd: root, detached: true, stdio: ["ignore", "pipe", "pipe"], + env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, PAPERCLIP_ACCOUNTING_TEST_DATABASE: database.connectionString, PAPERCLIP_TELEMETRY_DISABLED: "1" }, + }); + } + it("deduplicates the same receipt from independent server processes", async () => { + const f = await fixture(); + const receipt = { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: "0.1234567", idempotencyKey: "shared-key", occurredAt: new Date().toISOString() }; + const children = Array.from({ length: 4 }, () => worker("write", { companyId: f.company.id, receipt })); + try { + await Promise.all(children.map(child => new Promise((resolve, reject) => { + let log = ""; child.stdout.on("data", d => log += d); child.stderr.on("data", d => log += d); + child.once("error", reject); child.once("exit", code => code === 0 && log.includes("WRITES_COMMITTED") ? resolve() : reject(new Error(log))); + }))); + expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))).toHaveLength(1); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + } finally { for (const child of children) if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid!, "SIGKILL"); } + }, 45_000); + + it("recovers a receipt after SIGKILL before its first database persistence", async () => { + const f = await fixture(); const directory = await mkdtemp(path.join(tmpdir(), "accounting-early-kill-")); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running", costAccountingPending: true, usageJson: { accountingReceiptReady: false } }).returning(); + const child = worker("receipt", { companyId: f.company.id, runId: run.id, directory }); + try { + await new Promise((resolve, reject) => { + let log = ""; child.stdout.on("data", data => { log += data; if (log.includes("RECEIPT_DURABLE")) resolve(); }); + child.stderr.on("data", data => log += data); child.once("error", reject); child.once("exit", () => reject(new Error(`Exited before checkpoint: ${log}`))); + }); + const stopped = new Promise(resolve => child.once("exit", resolve)); process.kill(-child.pid!, "SIGKILL"); await stopped; + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + expect(await replayUsageReceipts(db, directory)).toEqual({ replayed: 1, failed: 0 }); + expect(await accountRunCost(db, run.id)).toBe(true); + expect(await accountRunCost(db, run.id)).toBe(false); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.2345678"); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + } finally { + if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid!, "SIGKILL"); + await rm(directory, { recursive: true, force: true }); + } + }, 45_000); + + it("retries safely when PostgreSQL commits but its acknowledgement is lost", async () => { + const f = await fixture(); const target = new URL(database.connectionString); const sockets = new Set(); let dropped = false; + // Parse PostgreSQL server frames, suppress precisely CommandComplete(COMMIT), + // then break TCP. The database has committed while the writer sees failure. + const proxy = createServer(client => { + const upstream = createConnection({ host: target.hostname, port: Number(target.port) }); sockets.add(client); sockets.add(upstream); + client.on("error", () => {}); upstream.on("error", () => client.destroy()); client.pipe(upstream); + let pending = Buffer.alloc(0); + upstream.on("data", chunk => { + pending = Buffer.concat([pending, chunk]); + while (pending.length >= 5) { + const size = pending.readInt32BE(1) + 1; + if (size < 5 || size > 16 * 1024 * 1024) { client.destroy(); upstream.destroy(); return; } + if (pending.length < size) return; + const message = pending.subarray(0, size); pending = pending.subarray(size); + if (!dropped && message[0] === 67 && message.subarray(5).toString() === "COMMIT\0") { + dropped = true; client.destroy(); upstream.destroy(); return; + } + client.write(message); + } + }); + client.on("close", () => { sockets.delete(client); upstream.destroy(); }); + upstream.on("close", () => { sockets.delete(upstream); client.destroy(); }); + }); + await new Promise(resolve => proxy.listen(0, "127.0.0.1", resolve)); + const forwarded = new URL(database.connectionString); forwarded.hostname = "127.0.0.1"; forwarded.port = String((proxy.address() as { port: number }).port); forwarded.searchParams.set("sslmode", "disable"); + const writer = createDb(forwarded.toString()); + const receipt = { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: "0.9999999", idempotencyKey: "ambiguous-commit", occurredAt: new Date() }; + try { + await expect(costService(writer).createEvent(f.company.id, receipt)).rejects.toThrow(); + expect(dropped).toBe(true); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1); + await costService(writer).createEvent(f.company.id, receipt); + expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("0.9999999"); + expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + } finally { + await writer.$client.end({ timeout: 1 }); for (const socket of sockets) socket.destroy(); await new Promise(resolve => proxy.close(() => resolve())); + } + }, 30_000); +}); diff --git a/server/src/__tests__/cost-accounting-reliability.test.ts b/server/src/__tests__/cost-accounting-reliability.test.ts new file mode 100644 index 0000000000..1f3cb3b8cf --- /dev/null +++ b/server/src/__tests__/cost-accounting-reliability.test.ts @@ -0,0 +1,741 @@ +import { upsertBudgetPolicySchema } from "@paperclipai/shared"; +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { activityLog, agentRuntimeState, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, companies, completionContracts, costEvents, createDb, financeEvents, goals, heartbeatRuns, issues, nativeRunResults, projects } from "@paperclipai/db"; +import { reserveRunBudget } from "../services/budget-reservations.js"; +import { createRunDispatch } from "../modules/run-dispatch/index.js"; +import { costService, createCostEventInTransaction } from "../services/costs.js"; +import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "../services/budgets.js"; +import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js"; +import { financeService } from "../services/finance.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { hoistModuleGraph } from "./helpers/hoist-module-graph.js"; +import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const databaseDescribe = support.supported ? describe : describe.skip; + +databaseDescribe("cost accounting reliability (PostgreSQL)", () => { + const services = hoistModuleGraph(() => {}, async () => { + const [agentModule, companyModule, approvalModule] = await Promise.all([ + import("../services/agents.js"), import("../services/companies.js"), import("../services/approvals.js"), + ]); + return { agentService: agentModule.agentService, companyService: companyModule.companyService, approvalService: approvalModule.approvalService }; + }); + let db: ReturnType; + let database: Awaited>; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-cost-reliability-"); + db = createDb(database.connectionString); + }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Accounting", issuePrefix: `T${randomUUID().slice(0, 7)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "active", adapterType: "codex_local" }).returning(); + const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project", status: "in_progress" }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: project.id, title: "Work" }).returning(); + const [goal] = await db.insert(goals).values({ companyId: company.id, title: "Goal" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, invocationSource: "on_demand" }).returning(); + const receipt = { agentId: agent.id, provider: "test", model: "test", billingType: "metered_api", costCents: 100, occurredAt: new Date() }; + return { company, agent, project, issue, goal, run, receipt }; + } + + it.each([false, true])("deletes linked accounting records before company runs (concurrent writer: %s)", async (concurrentWriter) => { + const f = await fixture(); + const other = await fixture(); + await costService(db).createEvent(other.company.id, other.receipt); + const charge = await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id, issueId: f.issue.id }); + await financeService(db).createEvent(f.company.id, { + eventKind: "platform_fee", direction: "debit", biller: "paperclip", amountCents: 5, currency: "USD", occurredAt: new Date(), + heartbeatRunId: f.run.id, costEventId: charge.id, + }); + const [policy] = await db.insert(budgetPolicies).values({ companyId: f.company.id, scopeType: "company", scopeId: f.company.id, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 1000 }).returning(); + await db.insert(budgetIncidents).values({ companyId: f.company.id, policyId: policy.id, scopeType: "company", scopeId: f.company.id, + metric: policy.metric, windowKind: policy.windowKind, windowStart: new Date(), windowEnd: new Date(), + thresholdType: "warning", amountLimit: 1000, amountObserved: 100 }); + let removing: Promise<{ value?: unknown; error?: unknown }> | undefined; + try { + if (concurrentWriter) { + await db.transaction(async tx => { + await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update"); + removing = services.value.companyService(db).remove(f.company.id).then(value => ({ value }), error => ({ error })); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() + and wait_event_type = 'Lock' and query like '%companies%for no key update%'`); + expect(waiting.length).toBeGreaterThan(0); + }, { timeout: 3000 }); + await createCostEventInTransaction(tx as unknown as ReturnType, f.company.id, { ...f.receipt, heartbeatRunId: f.run.id }, []); + }); + expect(await removing!).toMatchObject({ value: { id: f.company.id } }); + } else expect(await services.value.companyService(db).remove(f.company.id)).toMatchObject({ id: f.company.id }); + expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(0); + for (const table of [heartbeatRuns, costEvents, financeEvents, budgetPolicies, budgetIncidents]) { + expect(await db.select().from(table).where(eq(table.companyId, f.company.id))).toHaveLength(0); + } + expect((await costService(db).summary(other.company.id)).spendCents).toBe(100); + expect(await services.value.companyService(db).remove(f.company.id)).toBeNull(); + } finally { await removing; } + }); + + it("serializes agent deletion before child locks and preserves a concurrent settled charge", async () => { + const f = await fixture(); + let removing: Promise<{ value?: unknown; error?: unknown }> | undefined; + try { + await db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '1s'`); + await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update"); + await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update"); + removing = services.value.agentService(db).remove(f.agent.id).then(value => ({ value }), error => ({ error })); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select query from pg_stat_activity where datname = current_database() + and wait_event_type = 'Lock' and (query like '%companies%for no key update%' or query like 'delete from "heartbeat_runs"%')`); + expect(waiting.length).toBeGreaterThan(0); + }, { timeout: 3000 }); + await createCostEventInTransaction(tx as unknown as ReturnType, f.company.id, + { ...f.receipt, heartbeatRunId: f.run.id }, []); + }); + // Existing charge FKs prohibit hard deletion. Preserve the settled charge + // and the atomic deletion failure, rather than deadlocking its writer. + expect(await removing!).toMatchObject({ error: { cause: { code: "23503" } } }); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(100); + expect(await db.select().from(agents).where(eq(agents.id, f.agent.id))).toHaveLength(1); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id))).toHaveLength(1); + } finally { await removing; } + }); + + it("preserves an in-flight native result when company deletion reaches its existing foreign-key restriction", async () => { + const f = await fixture(); + await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id }); + const contractHash = randomUUID(); + const [contract] = await db.insert(completionContracts).values({ companyId: f.company.id, issueId: f.issue.id, + revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", + risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: {}, canonicalSha256: contractHash, createdByActorType: "system", createdByActorId: "test" }).returning(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id, + completionContractId: contract.id, completionContractSha256: contractHash }).where(eq(heartbeatRuns.id, f.run.id)); + let removing: Promise<{ value?: unknown; error?: unknown }> | undefined; + try { + await db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '1s'`); + await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update"); + removing = services.value.companyService(db).remove(f.company.id).then(value => ({ value }), error => ({ error })); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() + and wait_event_type = 'Lock' and query like 'delete from "heartbeat_runs"%'`); + expect(waiting.length).toBeGreaterThan(0); + }, { timeout: 3000 }); + const store = new NativeRunCoordinatorStore(tx as unknown as ReturnType, { + companyId: f.company.id, issueId: f.issue.id, runId: f.run.id, agentId: f.agent.id, + normalizedSessionId: randomUUID(), runnerSourceInstanceId: randomUUID(), + completionContractId: contract.id, completionContractSha256: contractHash, + completionContractRevision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision, + completionContractCriterionIds: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.criteria.map(c => c.criterionId), + }); + expect(await store.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL })) + .toMatchObject({ disposition: "committed" }); + }); + // Native artifacts already prevent hard deletion via restrictive FKs. + // Preserve that atomic failure; the accounting lock must not introduce a + // deadlock that aborts the native result instead (Postgres 40P01). + expect(await removing!).toMatchObject({ error: { cause: { code: "23503" } } }); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.companyId, f.company.id))).toHaveLength(1); + expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(1); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(100); + } finally { await removing; } + }); + + it("counts cache reads once across mixed historical and receipt-backed reports without changing charges", async () => { + const f = await fixture(); + const costs = costService(db); + const scope = { agentId: f.agent.id, issueId: f.issue.id, projectId: f.project.id, heartbeatRunId: f.run.id, + provider: "openai", biller: "openai", model: "gpt-test", occurredAt: new Date() }; + // These two layouts both describe 100 input (80 cached) and 10 output. + for (const billingType of ["metered_api", "subscription_included"]) { + await db.insert(costEvents).values({ companyId: f.company.id, ...scope, billingType, + inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 7 }); + await costs.createEvent(f.company.id, { ...scope, billingType, + inputTokens: 20, cachedInputTokens: 80, outputTokens: 10, costCents: 11 }); + } + const [provider, models] = await Promise.all([costs.byProvider(f.company.id), costs.byAgentModel(f.company.id)]); + for (const rows of [provider, models]) { + expect(rows).toHaveLength(2); + for (const row of rows) { + expect(row).toMatchObject({ inputTokens: 40, cachedInputTokens: 160, outputTokens: 20, costCents: 18 }); + expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(220); + } + } + const [agent] = await costs.byAgent(f.company.id); + const [biller] = await costs.byBiller(f.company.id); + const [project] = await costs.byProject(f.company.id); + const issue = await costs.issueTreeSummary(f.company.id, f.issue.id); + const windows = await costs.windowSpend(f.company.id); + expect(windows).toHaveLength(3); + for (const row of [agent, biller, project, issue, ...windows]) { + expect(row).toMatchObject({ inputTokens: 80, cachedInputTokens: 320, outputTokens: 40, costCents: 36 }); + expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(440); + } + for (const row of [agent, biller, provider.find(row => row.billingType === "subscription_included")!]) { + expect(row).toMatchObject({ subscriptionInputTokens: 40, subscriptionCachedInputTokens: 160, subscriptionOutputTokens: 20 }); + } + // Reading reports never rewrites historical evidence or retroactively bills. + const stored = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)); + expect(stored.filter(row => row.receiptHash === null).map(row => row.inputTokens)).toEqual([100, 100]); + expect((await costs.summary(f.company.id)).spendCents).toBe(36); + // Legacy Anthropic input was already exclusive; retain its cache semantics. + await db.insert(costEvents).values({ companyId: f.company.id, ...scope, provider: "anthropic", billingType: "metered_api", + inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 0 }); + expect((await costs.byProvider(f.company.id)).find(row => row.provider === "anthropic")).toMatchObject({ inputTokens: 100, cachedInputTokens: 80, outputTokens: 10 }); + }); + + it.each([ + ["openai/gpt-test", 100], ["openai/vendor/model", 100], + ["gpt-test", 20], ["different/model", 20], ["openai/", 20], + ] as const)("preserves historical input semantics for %s", async (model, inputTokens) => { + const f = await fixture(), costs = costService(db); + await db.insert(costEvents).values({ companyId: f.company.id, agentId: f.agent.id, + issueId: f.issue.id, projectId: f.project.id, heartbeatRunId: f.run.id, + provider: "openai", biller: "openai", model, occurredAt: new Date(), + inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 7 }); + const rows = [ + ...await costs.byAgent(f.company.id), ...await costs.byProvider(f.company.id), + ...await costs.byAgentModel(f.company.id), ...await costs.byBiller(f.company.id), + ...await costs.byProject(f.company.id), ...await costs.windowSpend(f.company.id), + await costs.issueTreeSummary(f.company.id, f.issue.id), + ...(await costs.byUser(f.company.id)).rows, + ]; + for (const row of rows) expect(row).toMatchObject({ inputTokens, cachedInputTokens: 80, outputTokens: 10, costCents: 7 }); + expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))[0].inputTokens).toBe(100); + }); + + it.each((["company", "agent", "project"] as const).flatMap(scopeType => [0, 500].map(previousAmount => ({ scopeType, previousAmount }))))("edits $scopeType limits without confusing saved zero and disabled positive amounts ($previousAmount)", async ({ scopeType, previousAmount }) => { + const f = await fixture(), cancel = vi.fn(async () => {}); + const budgets = budgetService(db, { cancelWorkForScope: cancel }); + const scopeId = scopeType === "company" ? f.company.id : scopeType === "agent" ? f.agent.id : f.project.id; + await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: previousAmount, isActive: false }, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, projectId: f.project.id, costCents: 100 }); + expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ amount: 0, isActive: false }); + const updated = await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 50 }, "board"); + expect(updated).toMatchObject({ isActive: previousAmount === 0, paused: previousAmount === 0 }); + expect(cancel).toHaveBeenCalledTimes(previousAmount === 0 ? 1 : 0); + expect((await budgets.overview(f.company.id)).activeIncidents).toHaveLength(previousAmount === 0 ? 1 : 0); + }); + + it.each(["issueId", "projectId", "goalId", "heartbeatRunId"] as const)("rejects a foreign-company %s without writing anything", async (field) => { + const a = await fixture(); const b = await fixture(); + const foreignIds = { issueId: b.issue.id, projectId: b.project.id, goalId: b.goal.id, heartbeatRunId: b.run.id }; + await expect(costService(db).createEvent(a.company.id, { ...a.receipt, [field]: foreignIds[field] })).rejects.toThrow(/company/); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, a.company.id))).toHaveLength(0); + }); + + it("allows admission inside a caller transaction holding a company foreign-key lock", async () => { + const f = await fixture(); + // The recovery service dispatches through another connection while its + // transaction still owns FK KEY SHARE locks on the company. Accounting + // must serialize without requesting the conflicting FOR UPDATE mode. + await db.transaction(async (tx) => { + await tx.insert(issues).values({ companyId: f.company.id, title: "Concurrent recovery" }); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + }); + }, 5000); + + it.each([false, true])("takes the attributed issue lock before the run during native recovery (attribution changes: %s)", async (changeAttribution) => { + const f = await fixture(); + const [nextIssue] = await db.insert(issues).values({ companyId: f.company.id, title: "New attribution" }).returning(); + const usage = { provider: "openai", billingType: "metered_api", costUsd: 0.5, inputTokens: 10, outputTokens: 2, + accountingReceiptReady: true, ledgerScope: { issueId: f.issue.id } }; + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id, status: "failed", finishedAt: new Date(), + costAccountingPending: true, usageJson: usage }).where(eq(heartbeatRuns.id, f.run.id)); + let accounting: Promise<{ value?: boolean; error?: unknown }> | undefined; + try { + await db.transaction(async tx => { + // recordRetryableFailure holds the issue while updating its run. + await tx.select().from(issues).where(eq(issues.id, f.issue.id)).for("update"); + accounting = accountRunCost(db, f.run.id).then(value => ({ value }), error => ({ error })); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() + and wait_event_type = 'Lock' and query like '%issues%for key share%'`); + expect(waiting.length).toBeGreaterThan(0); + }, { timeout: 3000 }); + await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update", { noWait: true }); + await tx.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "failed" }, + ...(changeAttribution ? { usageJson: { ...usage, ledgerScope: { issueId: nextIssue.id } } } : {}), + }).where(eq(heartbeatRuns.id, f.run.id)); + }); + if (changeAttribution) { + expect((await accounting!).error).toMatchObject({ status: 409, message: "Run cost attribution changed. Retry accounting." }); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0); + expect(await accountRunCost(db, f.run.id)).toBe(true); + } else expect(await accounting!).toEqual({ value: true }); + const charges = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)); + expect(charges).toHaveLength(1); + expect(charges[0]).toMatchObject({ issueId: changeAttribution ? nextIssue.id : f.issue.id, costCents: 50 }); + } finally { await accounting; } + }); + + it("takes the company lock before retry issue/run locks during an attributed cost write", async () => { + const f = await fixture(); + await db.update(issues).set({ status: "in_progress", assigneeAgentId: f.agent.id }).where(eq(issues.id, f.issue.id)); + await db.update(heartbeatRuns).set({ status: "scheduled_retry", scheduledRetryAt: new Date(0), scheduledRetryReason: "workspace_busy", + contextSnapshot: { issueId: f.issue.id, projectId: f.project.id }, + }).where(eq(heartbeatRuns.id, f.run.id)); + let promotion: ReturnType["promoteScheduledRetry"]> | undefined; + await db.transaction(async (tx) => { + await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update"); + promotion = createRunDispatch(db).promoteScheduledRetry({ companyId: f.company.id, runId: f.run.id }); + // Observe the actual blocked company-lock query before checking the issue; + // no timing guess or timeout is used as proof of lock ordering. + await vi.waitFor(async () => { + const result = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() + and pid <> pg_backend_pid() and wait_event_type = 'Lock' and query like '%companies%for no key update%'`); + expect(result.length).toBeGreaterThan(0); + }, { timeout: 3000 }); + await tx.execute(sql`select id from issues where id = ${f.issue.id} for key share nowait`); + await createCostEventInTransaction(tx as unknown as ReturnType, f.company.id, + { ...f.receipt, issueId: f.issue.id, projectId: f.project.id }, []); + }); + expect((await promotion)?.outcome).toBe("promoted"); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(100); + }, 10000); + + it("locks company before activating an approved hire while a receipt writer is active", async () => { + const f = await fixture(); + await db.update(agents).set({ status: "pending_approval" }).where(eq(agents.id, f.agent.id)); + const [approval] = await db.insert(approvals).values({ companyId: f.company.id, type: "hire_agent", status: "pending", + payload: { agentId: f.agent.id, budgetMonthlyCents: 1000 } }).returning(); + let approving: Promise | undefined; + await db.transaction(async tx => { + await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update"); + approving = services.value.approvalService(db).approve(approval.id, "board"); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() + and pid <> pg_backend_pid() and wait_event_type = 'Lock' and query like '%companies%for no key update%'`); + expect(waiting.length).toBeGreaterThan(0); + }, { timeout: 3000 }); + await tx.execute(sql`select id from agents where id = ${f.agent.id} for no key update nowait`); + await createCostEventInTransaction(tx as unknown as ReturnType, f.company.id, f.receipt, []); + }); + await expect(approving).resolves.toMatchObject({ applied: true }); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle"); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(100); + }, 10000); + + it("deduplicates concurrent retries and rejects changed receipt content", async () => { + const f = await fixture(); + const receipt = { ...f.receipt, idempotencyKey: "provider-receipt-1" }; + const events = await Promise.all(Array.from({ length: 8 }, () => costService(db).createEvent(f.company.id, receipt))); + expect(new Set(events.map((event) => event.id)).size).toBe(1); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(100); + await expect(costService(db).createEvent(f.company.id, { ...receipt, costCents: 200 })).rejects.toThrow(/different|conflict/i); + // Keys are scoped to the reporting company. + const other = await fixture(); + await expect(costService(db).createEvent(other.company.id, { ...other.receipt, idempotencyKey: receipt.idempotencyKey })).resolves.toBeTruthy(); + }); + + it("retains sub-cent spend and enforces the sum, not rounded individual receipts", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1 }, "board"); + for (let i = 0; i < 3; i++) await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 0.4 }); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(1.2); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" }); + }); + + it("conserves project totals when a run touches multiple projects and includes unallocated spend", async () => { + const f = await fixture(); + const [otherProject] = await db.insert(projects).values({ companyId: f.company.id, name: "Other" }).returning(); + const [otherIssue] = await db.insert(issues).values({ companyId: f.company.id, projectId: otherProject.id, title: "Other" }).returning(); + await db.insert(activityLog).values([f.issue, otherIssue].map((issue) => ({ companyId: f.company.id, actorType: "agent", actorId: f.agent.id, action: "issue.updated", entityType: "issue", entityId: issue.id, runId: f.run.id }))); + await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id, projectId: f.project.id }); + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 30 }); + const rows = await costService(db).byProject(f.company.id); + expect(rows.reduce((sum, row) => sum + row.costCents, 0)).toBe(130); + expect(rows.find((row) => row.projectId === f.project.id)?.costCents).toBe(100); + expect(rows.find((row) => row.projectId === null)?.costCents).toBe(30); + }); + + it("does not enforce inactive policies, and checks both monthly and lifetime limits", async () => { + const f = await fixture(); const budgets = budgetService(db); + await costService(db).createEvent(f.company.id, f.receipt); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1, isActive: false }, "board"); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1000, windowKind: "calendar_month_utc" }, "board"); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50, windowKind: "lifetime" }, "board"); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 2000, windowKind: "calendar_month_utc" }, "board"); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" }); + }); + + it("creates one approval under concurrent evaluation and opens a new incident after a raised budget is reached", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board"); + const event = await costService(db).createEvent(f.company.id, f.receipt); + await Promise.all(Array.from({ length: 8 }, () => budgets.evaluateCostEvent(event))); + expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(1); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 150 }, "board"); + await costService(db).createEvent(f.company.id, f.receipt); + expect(await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard"), eq(budgetIncidents.status, "open")))).toHaveLength(1); + expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(2); + }); + + it("commits accounting despite cancellation delivery failure and retries delivery safely", async () => { + const f = await fixture(); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board"); + const cancel = vi.fn().mockRejectedValueOnce(new Error("injected cancellation transport failure")).mockResolvedValue(undefined); + const costs = costService(db, { cancelWorkForScope: cancel }); + const receipt = { ...f.receipt, idempotencyKey: "retry-after-cancel" }; + await expect(costs.createEvent(f.company.id, receipt)).resolves.toBeTruthy(); + await costs.createEvent(f.company.id, receipt); + expect(cancel).toHaveBeenCalledTimes(2); + expect((await costs.summary(f.company.id)).spendCents).toBe(100); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" }); + }); + + it("recovers live scopes despite deleted agent and project policies and an isolated transaction failure", async () => { + const f = await fixture(); const other = await fixture(); const budgets = budgetService(db); + const [removedAgent] = await db.insert(agents).values({ companyId: f.company.id, name: "Removed", adapterType: "process" }).returning(); + const [removedProject] = await db.insert(projects).values({ companyId: f.company.id, name: "Removed" }).returning(); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: removedAgent.id, amount: 100 }, "board"); + await budgets.upsertPolicy(f.company.id, { scopeType: "project", scopeId: removedProject.id, amount: 100 }, "board"); + await db.delete(agents).where(eq(agents.id, removedAgent.id)); + await db.delete(projects).where(eq(projects.id, removedProject.id)); + for (const live of [f, other]) { + await budgets.upsertPolicy(live.company.id, { scopeType: "agent", scopeId: live.agent.id, amount: 100 }, "board"); + await db.update(agents).set({ status: "paused", pauseReason: "budget" }).where(eq(agents.id, live.agent.id)); + } + // Restrict the scanned scope list so the injected first transaction fails + // on the stale scope, ahead of both live policies in this real database. + const query = vi.spyOn(db, "selectDistinct").mockReturnValueOnce({ from: () => ({ orderBy: async () => [ + { companyId: f.company.id, scopeType: "project", scopeId: removedProject.id }, + { companyId: f.company.id, scopeType: "agent", scopeId: removedAgent.id }, + ...[f, other].map(live => ({ companyId: live.company.id, scopeType: "agent", scopeId: live.agent.id })), + ] }) } as never); + const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Injected scope recovery failure")); + try { await budgets.reconcilePolicies(); } finally { query.mockRestore(); transaction.mockRestore(); } + for (const live of [f, other]) { + const [agent] = await db.select().from(agents).where(eq(agents.id, live.agent.id)); + expect(agent).toMatchObject({ status: "idle", pauseReason: null }); + } + // A subsequent sweep uses actual policy discovery and skips both deleted scopes. + await expect(budgets.reconcilePolicies()).resolves.toBeUndefined(); + }); + + it("recovers terminal run accounting exactly once across simultaneous live and recovery workers", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status: "timed_out", finishedAt: new Date(), costAccountingPending: true, + usageJson: { inputTokens: 20, cachedInputTokens: 100, outputTokens: 10, costUsd: 0.004, provider: "test", model: "test", billingType: "metered_api", ledgerScope: { issueId: f.issue.id, projectId: f.project.id } }, + }).where(eq(heartbeatRuns.id, f.run.id)); + await Promise.all([accountRunCost(db, f.run.id), accountRunCost(db, f.run.id), reconcileRunCosts(db)]); + const [totals] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)); + expect(totals).toMatchObject({ totalInputTokens: 20, totalCachedInputTokens: 100, totalOutputTokens: 10, totalCostCents: 0.4 }); + const events = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ costCents: 0.4, projectId: f.project.id }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(run.costAccountingPending).toBe(false); + expect(run.costAccountedAt).toBeInstanceOf(Date); + // Late terminal metadata may re-arm the pending flag. The durable + // acknowledgement, not the delivery flag, owns exactly-once totals. + await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id)); + await accountRunCost(db, f.run.id); + expect((await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)))[0].totalCostCents).toBe(0.4); + }); + + it("rolls back the ledger and counters on accounting failure, retaining a retryable run", async () => { + const f = await fixture(); const other = await fixture(); + await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, usageJson: { + inputTokens: 10, costUsd: 1, provider: "test", model: "test", ledgerScope: { projectId: other.project.id }, + } }).where(eq(heartbeatRuns.id, f.run.id)); + await expect(accountRunCost(db, f.run.id)).rejects.toThrow(/company/); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0); + expect(await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id))).toHaveLength(0); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountingPending).toBe(true); + }); + + it("rolls back a receipt, counters, and budget incidents if a later runtime write fails", async () => { + const f = await fixture(); + await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50 }, "board"); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), costAccountingPending: true, usageJson: { + inputTokens: 10, costUsd: 1, provider: "test", billingType: "metered_api", model: "test", + } }).where(eq(heartbeatRuns.id, f.run.id)); + await db.execute(sql`create function reject_runtime_accounting() returns trigger language plpgsql as $$ begin raise exception 'injected runtime write failure'; end $$`); + await db.execute(sql`create trigger reject_runtime_accounting before insert on agent_runtime_state for each row execute function reject_runtime_accounting()`); + try { + await expect(accountRunCost(db, f.run.id)).rejects.toThrow(); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0); + expect(await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, f.company.id))).toHaveLength(0); + expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(0); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ spentMonthlyCents: 0, status: "active" }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: true, costAccountedAt: null }); + } finally { + await db.execute(sql`drop trigger reject_runtime_accounting on agent_runtime_state`); + await db.execute(sql`drop function reject_runtime_accounting()`); + } + await accountRunCost(db, f.run.id); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(100); + }); + + it("releases an expired monthly budget pause while preserving manual pauses", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50 }, "board"); + const event = await costService(db).createEvent(f.company.id, f.receipt); + const now = new Date(); + await db.update(costEvents).set({ occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)) }).where(eq(costEvents.id, event.id)); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ status: "idle", pauseReason: null }); + await db.update(agents).set({ status: "paused", pauseReason: "manual" }).where(eq(agents.id, f.agent.id)); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 200 }, "board"); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ status: "paused", pauseReason: "manual" }); + }); + + it("uses live observed spend for incident resolution and synchronizes company budgets", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 50 }, "board"); + await costService(db).createEvent(f.company.id, f.receipt); + const [incident] = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard"))); + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 25 }); + await expect(budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 125 }, "board")).rejects.toThrow("New budget must exceed current observed spend"); + await budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 175 }, "board"); + expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0]).toMatchObject({ budgetMonthlyCents: 175, status: "active", pauseReason: null }); + }); + + it("exposes missing prices and blocks strict budgets until explicitly allowed", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, inputTokens: 500, costCents: 0, costStatus: "unpriced" }); + expect(await costService(db).summary(f.company.id)).toMatchObject({ spendCents: 0, unpricedEventCount: 1, pricingComplete: false }); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ reason: "Agent cannot start work because recorded usage has no reliable price." }); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, unpricedUsagePolicy: "allow" }, "board"); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ unpricedEventCount: 1, unpricedUsagePolicy: "allow" }); + }); + + it("keeps subscription-included usage outside monetary pricing gaps", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, inputTokens: 500, costCents: 0, costStatus: "unpriced", billingType: "subscription_included" }); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: true }); + }); + + it("defaults reports to the UTC month and excludes future events from rolling spend", async () => { + const f = await fixture(); const costs = costService(db); const now = new Date(); + await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 1, 1)) }); + await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: new Date(now.getTime() + 86_400_000) }); + expect((await costs.summary(f.company.id)).spendCents).toBe(0); + expect((await costs.summary(f.company.id, { allTime: true })).spendCents).toBe(200); + expect(await costs.windowSpend(f.company.id)).toEqual([]); + }); + + it("keeps finance currencies separate and deduplicates invoice-line retries", async () => { + const f = await fixture(); const finance = financeService(db); + const usd = { biller: "vendor", eventKind: "platform_fee", amountCents: 100, currency: "usd", occurredAt: new Date(), idempotencyKey: "invoice:line:1", metadataJson: { a: 1, b: 2 } }; + await Promise.all(Array.from({ length: 6 }, () => finance.createEvent(f.company.id, usd))); + await finance.createEvent(f.company.id, { ...usd, metadataJson: { b: 2, a: 1 } }); + await finance.createEvent(f.company.id, { ...usd, currency: "EUR", idempotencyKey: "invoice:line:2" }); + const summary = await finance.summary(f.company.id); + expect(summary).toMatchObject({ currency: "USD", debitCents: 100, netCents: 100 }); + expect(summary.currencies).toEqual(expect.arrayContaining([expect.objectContaining({ currency: "USD", netCents: 100 }), expect.objectContaining({ currency: "EUR", netCents: 100 })])); + expect(await finance.byBiller(f.company.id)).toHaveLength(2); + expect(await finance.byKind(f.company.id)).toHaveLength(2); + expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(2); + await expect(finance.createEvent(f.company.id, { ...usd, amountCents: 200 })).rejects.toThrow(/different/); + }); + + it("synchronizes budgets through generic agent and company updates", async () => { + const f = await fixture(); + const { agentService, companyService } = services.value; + await agentService(db).update(f.agent.id, { budgetMonthlyCents: 50 }); + await companyService(db).update(f.company.id, { budgetMonthlyCents: 75 }); + await costService(db).createEvent(f.company.id, f.receipt); + const policies = await budgetService(db).listPolicies(f.company.id); + expect(policies).toEqual(expect.arrayContaining([expect.objectContaining({ scopeType: "agent", amount: 50 }), expect.objectContaining({ scopeType: "company", amount: 75 })])); + expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "company" }); + }); + + it("delivers cancellation after generic budget updates commit", async () => { + const f = await fixture(); + await costService(db).createEvent(f.company.id, f.receipt); + const cancelWorkForScope = vi.fn(async (scope) => { + const policies = await budgetService(db).listPolicies(f.company.id); + expect(policies.some(policy => policy.scopeType === scope.scopeType && policy.amount < 100)).toBe(true); + }); + await services.value.agentService(db, { cancelWorkForScope }).update(f.agent.id, { budgetMonthlyCents: 50 }); + await services.value.companyService(db, { cancelWorkForScope }).update(f.company.id, { budgetMonthlyCents: 75 }); + expect(cancelWorkForScope).toHaveBeenCalledWith(expect.objectContaining({ scopeType: "agent", scopeId: f.agent.id })); + expect(cancelWorkForScope).toHaveBeenCalledWith(expect.objectContaining({ scopeType: "company", scopeId: f.company.id })); + const rows = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, f.company.id)); + expect(rows.every(row => row.enforcementVersion === row.enforcementDeliveredVersion)).toBe(true); + }); + + it("merges partial policy edits with current settings under the accounting lock", async () => { + const f = await fixture(); const budgets = budgetService(db); + const identity = { scopeType: "agent" as const, scopeId: f.agent.id }; + await budgets.upsertPolicy(f.company.id, { ...identity, amount: 100, hardStopEnabled: true }, "board"); + await budgets.upsertPolicy(f.company.id, { ...identity, amount: 200, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false }, "other-operator"); + const saved = await budgets.upsertPolicy(f.company.id, upsertBudgetPolicySchema.parse({ ...identity, reservationCents: "10" }), "board"); + expect(saved).toMatchObject({ amount: 200, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false, reservationCents: "10.0000000" }); + await budgets.upsertPolicy(f.company.id, { ...identity, isActive: false }, "other-operator"); + const disabled = await budgets.upsertPolicy(f.company.id, upsertBudgetPolicySchema.parse({ ...identity, amount: 300 }), "board"); + expect(await db.select({ amount: budgetPolicies.amount, isActive: budgetPolicies.isActive }).from(budgetPolicies).where(eq(budgetPolicies.id, disabled.policyId))).toEqual([{ amount: 300, isActive: false }]); + expect(disabled).toMatchObject({ amount: 0, isActive: false, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false, reservationCents: "10.0000000" }); + await expect(budgets.upsertPolicy(f.company.id, { scopeType: "project", scopeId: f.project.id, reservationCents: "1" }, "board")).rejects.toThrow("Amount is required"); + }); + + it("conserves both spend and tokens across complete per-model receipts", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), costAccountingPending: true, usageJson: { + provider: "anthropic", billingType: "metered_api", model: "mixed", costUsd: 0.007, + inputTokens: 30, outputTokens: 4, cachedInputTokens: 100, + usageByModel: [ + { model: "large", costUsd: 0.005, usage: { inputTokens: 20, outputTokens: 3, cachedInputTokens: 100 } }, + { model: "small", costUsd: 0.002, usage: { inputTokens: 10, outputTokens: 1 } }, + ], + } }).where(eq(heartbeatRuns.id, f.run.id)); + await accountRunCost(db, f.run.id); + const rows = await costService(db).byProvider(f.company.id); + expect(rows).toEqual(expect.arrayContaining([expect.objectContaining({ model: "large", costCents: 0.5, inputTokens: 20 }), expect.objectContaining({ model: "small", costCents: 0.2, inputTokens: 10 })])); + expect((await costService(db).summary(f.company.id)).spendCents).toBe(0.7); + const [totals] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)); + expect(totals).toMatchObject({ totalInputTokens: 30, totalCachedInputTokens: 100, totalOutputTokens: 4, totalCostCents: 0.7 }); + }); + it("blocks admission while a terminal run awaits accounting, even under an allow-unpriced policy", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1000, unpricedUsagePolicy: "allow" }, "board"); + await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ reason: expect.stringContaining("await accounting") }); + expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: false, pendingRunCount: 1 }); + await accountRunCost(db, f.run.id); + expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: false, pendingRunCount: 0, unpricedEventCount: 1 }); + expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + }); + + it("waits for a late provider receipt after cancellation instead of acknowledging a zero charge", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status: "cancelled", costAccountingPending: true, usageJson: { accountingReceiptReady: false } }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await accountRunCost(db, f.run.id)).toBe(false); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: true, costAccountedAt: null }); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0); + await db.update(heartbeatRuns).set({ usageJson: { accountingReceiptReady: true, costUsd: 0.25, inputTokens: 10 } }).where(eq(heartbeatRuns.id, f.run.id)); + await Promise.all([accountRunCost(db, f.run.id), accountRunCost(db, f.run.id)]); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ costCents: 25, inputTokens: 10 }); + expect((await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)))[0].totalCostCents).toBe(25); + }); + + it("acknowledges proven pre-provider failures without inventing a charge", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, + resultJson: { executionRecovery: { providerWorkStarted: false } }, + }).where(eq(heartbeatRuns.id, f.run.id)); + await accountRunCost(db, f.run.id); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: false, costAccountedAt: expect.any(Date) }); + }); + + it("records spend after attribution targets are deleted", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, usageJson: { + costUsd: 0.01, ledgerScope: { projectId: f.project.id, issueId: f.issue.id }, + } }).where(eq(heartbeatRuns.id, f.run.id)); + await db.delete(issues).where(eq(issues.id, f.issue.id)); + await db.delete(projects).where(eq(projects.id, f.project.id)); + await accountRunCost(db, f.run.id); + expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))[0]).toMatchObject({ costCents: 1, issueId: null, projectId: null }); + }); + + it("preserves dismissed incidents and warning incidents during admission checks", async () => { + const f = await fixture(); const budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 80 }); + for (let i = 0; i < 3; i++) expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); + expect((await budgets.overview(f.company.id)).activeIncidents).toHaveLength(1); + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 20 }); + const [incident] = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard"))); + await budgets.resolveIncident(f.company.id, incident.id, { action: "keep_paused" }, "board"); + for (let i = 0; i < 3; i++) expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeTruthy(); + expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(1); + }); + + it("does not revive terminated agents or archived companies during budget reconciliation", async () => { + const f = await fixture(); const budgets = budgetService(db); + await db.update(agents).set({ status: "terminated", pauseReason: "budget" }).where(eq(agents.id, f.agent.id)); + await db.update(companies).set({ status: "archived", pauseReason: "budget" }).where(eq(companies.id, f.company.id)); + for (const [scopeType, scopeId] of [["agent", f.agent.id], ["company", f.company.id]] as const) { + await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 1000 }, "board"); + } + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2000 }); + expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("terminated"); + expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].status).toBe("archived"); + }); + + it.each(["agent", "company", "project"] as const)("limits delayed %s budget cancellation to work that preceded the policy check", async (scopeType) => { + const f = await fixture(); + const { heartbeatService } = await import("../services/heartbeat.js"); + const heartbeat = heartbeatService(db); + const scopeId = scopeType === "agent" ? f.agent.id : scopeType === "company" ? f.company.id : f.project.id; + const cutoff = new Date(); + await db.update(heartbeatRuns).set({ status: "queued", createdAt: new Date(cutoff.getTime() - 1000), contextSnapshot: { projectId: f.project.id } }).where(eq(heartbeatRuns.id, f.run.id)); + const [newRun] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, invocationSource: "on_demand", status: "scheduled_retry", + scheduledRetryAt: new Date(cutoff.getTime() + 60000), createdAt: new Date(cutoff.getTime() + 1000), contextSnapshot: { projectId: f.project.id }, + }).returning(); + await heartbeat.cancelBudgetScopeWork({ companyId: f.company.id, scopeType, scopeId, createdBefore: cutoff }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].status).toBe("cancelled"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, newRun.id)))[0].status).toBe("scheduled_retry"); + }); + + it.each(["agent", "company", "project"] as const)("rechecks delayed %s enforcement after a grant admits an old queued run", async (scopeType) => { + const f = await fixture(); + const { heartbeatService } = await import("../services/heartbeat.js"); + const heartbeat = heartbeatService(db); + const budgets = budgetService(db); + const scopeId = scopeType === "agent" ? f.agent.id : scopeType === "company" ? f.company.id : f.project.id; + const policyInput = { scopeType, scopeId, amount: 1 }; + await db.update(heartbeatRuns).set({ status: "queued", contextSnapshot: { projectId: f.project.id } }).where(eq(heartbeatRuns.id, f.run.id)); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.company.id, agentId: f.agent.id, source: "automation", payload: { projectId: f.project.id } }).returning(); + await budgets.upsertPolicy(f.company.id, policyInput, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, projectId: f.project.id, costCents: 2 }); + let release!: () => void; + const paused = new Promise(resolve => { release = resolve; }); + let snapshotReady!: (scope: BudgetEnforcementScope) => void; + const snapshot = new Promise(resolve => { snapshotReady = resolve; }); + const delivery = budgetService(db, { cancelWorkForScope: async (scope) => { + snapshotReady(scope); await paused; await heartbeat.cancelBudgetScopeWork(scope); + } }).deliverPendingEnforcement(f.company.id); + try { + expect((await snapshot).enforcement).toMatchObject({ policyId: expect.any(String), version: expect.any(Number) }); + await budgets.upsertPolicy(f.company.id, { ...policyInput, amount: 100 }, "board"); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id)); + await reserveRunBudget(db, f.company.id, f.run.id, f.project.id); + } finally { release(); } + await delivery; + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ status: "running", resultJson: null }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued"); + }); + + it("keeps a durably claimed budget stop fenced after a later grant", async () => { + const f = await fixture(), budgets = budgetService(db); + await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board"); + await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2 }); + const [policy] = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, f.company.id)); + const scope: BudgetEnforcementScope = { companyId: f.company.id, scopeType: "company", scopeId: f.company.id, + enforcement: { policyId: policy.id, version: policy.enforcementVersion } }; + expect(await withCurrentBudgetEnforcement(db, scope, async (tx) => { + await tx.update(heartbeatRuns).set({ status: "running", resultJson: { cancellation: { reason: "Budget stop" } } }).where(eq(heartbeatRuns.id, f.run.id)); + return true; + })).toBe(true); + await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100 }, "board"); + await expect(reserveRunBudget(db, f.company.id, f.run.id, null)).rejects.toThrow("cancellation already requested"); + const effect = vi.fn(async () => true); + for (const invalid of [ + scope, + { ...scope, enforcement: { ...scope.enforcement!, policyId: randomUUID() } }, + { ...scope, scopeId: randomUUID() }, + { ...scope, scopeType: "agent" as const }, + { ...scope, enforcement: { ...scope.enforcement!, version: policy.enforcementVersion + 1 } }, + ]) expect(await withCurrentBudgetEnforcement(db, invalid, effect)).toBeNull(); + expect(effect).not.toHaveBeenCalled(); + }); + +}); diff --git a/server/src/__tests__/cost-accounting-routes.test.ts b/server/src/__tests__/cost-accounting-routes.test.ts new file mode 100644 index 0000000000..34eb3bce46 --- /dev/null +++ b/server/src/__tests__/cost-accounting-routes.test.ts @@ -0,0 +1,91 @@ +import express from "express"; +import request from "supertest"; +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb } from "@paperclipai/db"; +import { costRoutes } from "../routes/costs.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("accounting operator authorization", () => { + let database: Awaited>; + let db: ReturnType; + let companyId: string, foreignId: string, agentId: string; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("accounting-authz-"); db = createDb(database.connectionString); + const rows = await db.insert(companies).values([{ name: "A", issuePrefix: "AUTH_A" }, { name: "B", issuePrefix: "AUTH_B" }]).returning(); + companyId = rows[0].id; foreignId = rows[1].id; + agentId = (await db.insert(agents).values({ companyId, name: "Worker", role: "engineer", adapterType: "process" }).returning())[0].id; + },30_000); + afterAll(async () => { await database?.cleanup(); }); + function app(actor: Record) { + const app = express(); app.use(express.json()); app.use((req,_res,next) => { req.actor = actor as typeof req.actor; next(); }); + app.use("/api", costRoutes(db)); app.use(errorHandler); return app; + } + const operations = [ + ["get", "health", undefined], ["get", "inspect", undefined], ["get", "invoices", undefined], + ["get", `invoices/${randomUUID()}`, undefined], ["get", `events/${randomUUID()}/adjustments`, undefined], + ["post", "repair", { fingerprint: "a".repeat(64), reason: "test" }], + ["post", "provider-costs/import", {}], ["post", "retry", { runId: randomUUID() }], ["post", "invoices", {}], ["post", `events/${randomUUID()}/adjustments`, {}], + ] as const; + it.each(operations)("denies agents for %s %s before reading or validating operator payloads", async (method, route, body) => { + const response = await request(app({ type: "agent", companyId, agentId }))[method](`/api/companies/${companyId}/accounting/${route}`).send(body); + expect(response.status).toBe(403); + }); + it.each(operations)("denies nonmembers for %s %s", async (method, route, body) => { + const response = await request(app({ type: "board", source: "session", userId: "operator", companyIds: [foreignId] }))[method](`/api/companies/${companyId}/accounting/${route}`).send(body); + expect(response.status).toBe(403); + }); + it("allows operator inspection and rejects viewer mutations", async () => { + const board = { type: "board", source: "session", userId: "operator", companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole: "viewer" }] }; + const response = await request(app(board)).get(`/api/companies/${companyId}/accounting/inspect`); + expect(response.status).toBe(200); expect(response.body.findings).toEqual([]); + expect((await request(app(board)).post(`/api/companies/${companyId}/accounting/repair`).send({ fingerprint: response.body.fingerprint, reason: "test" })).status).toBe(403); + }); + it.each(["member", "viewer", "unknown", undefined])("denies provider credential use by %s before payload validation", async (membershipRole) => { + const actor = { type: "board", source: "session", userId: "member", companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole }] }; + const response = await request(app(actor)).post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({}); + expect(response.status).toBe(403); + }); + it.each([ + { source: "session", memberships: "owner" }, + { source: "session", memberships: "admin" }, + { source: "session", isInstanceAdmin: true }, + { source: "local_implicit" }, + ])("allows billing operators through to payload validation: %j", async (input) => { + const actor = { ...input, type: "board", userId: "operator", companyIds: [companyId], memberships: input.memberships ? [{ companyId, status: "active", membershipRole: input.memberships }] : [] }; + const response = await request(app(actor)).post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({}); + expect(response.status).toBe(400); + }); + it("does not accept an inactive admin or an admin role from another company", async () => { + for (const membership of [ + { companyId, status: "suspended", membershipRole: "admin" }, + { companyId: foreignId, status: "active", membershipRole: "admin" }, + ]) { + const response = await request(app({ type: "board", source: "session", userId: "operator", companyIds: [companyId], memberships: [membership] })) + .post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({}); + expect(response.status).toBe(403); + } + }); + it("authorizes the user cost report and validates its date range", async () => { + const path = `/api/companies/${companyId}/costs/by-user`; + const board = app({ type: "board", source: "local_implicit", userId: "operator" }); + const response = await request(board).get(`${path}?period=all`); + expect(response.status).toBe(200); + expect(response.body).toEqual({ activeUserCount: 0, rows: [] }); + for (const query of ["from=invalid", "from=2026-02-01&to=2026-01-01", "period=all&from=2026-01-01"]) { + expect((await request(board).get(`${path}?${query}`)).status).toBe(400); + } + for (const actor of [ + { type: "board", source: "session", userId: "outsider", companyIds: [foreignId] }, + { type: "agent", companyId: foreignId, agentId }, + ]) expect((await request(app(actor)).get(path)).status).toBe(403); + expect((await request(app({ type: "none" })).get(path)).status).toBe(401); + }); + + it("validates correction and repair payloads before mutating", async () => { + const board = app({ type: "board", source: "local_implicit", userId: "operator" }); + expect((await request(board).post(`/api/companies/${companyId}/accounting/repair`).send({ fingerprint: "old", reason: "" })).status).toBe(400); + expect((await request(board).post(`/api/companies/${companyId}/accounting/events/${randomUUID()}/adjustments`).send({ idempotencyKey: "x", expectedCents: "0", correctedCents: "-0.000000001", reason: "Invalid", pricing: { source: "operator" } })).status).toBe(400); + }); +}); diff --git a/server/src/__tests__/email-channels.integration.test.ts b/server/src/__tests__/email-channels.integration.test.ts index adcdda0184..7c76f22fb1 100644 --- a/server/src/__tests__/email-channels.integration.test.ts +++ b/server/src/__tests__/email-channels.integration.test.ts @@ -944,6 +944,25 @@ describe("AgentMail durable email pipeline", () => { ); }); + it.each([ + ["1999999999.9999999", true], + ["2000000000.0000000", false], + ["2000000000.0000001", false], + ])("compares exact monthly email spend %s at the budget boundary", async (spent, allowed) => { + const f = await fixture(); + await f.receive(f.message()); + const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.companyId, f.companyId)); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, status: "running", runtimeMode: "native", nativeIssueId: conversation.issueId, contextSnapshot: { issueId: conversation.issueId } }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, conversation.issueId)); + await db.update(agents).set({ budgetMonthlyCents: 2000000000, spentMonthlyCents: sql`${spent}::numeric` }).where(eq(agents.id, f.agentId)); + const request = emailSendSchema.parse({ endpointId: f.endpointId, conversationId: conversation.id, replyToMessageId: [...f.messages.keys()][0], text: "Exact budget reply", idempotencyKey: randomUUID() }); + const queued = f.service.queueSend(f.companyId, request, { agentId: f.agentId, runId }); + if (allowed) expect((await queued).outcome).toBe("queued"); + else await expect(queued).rejects.toThrow("Agent is not available to send email"); + expect(f.sends).toHaveLength(0); + }); + it("enforces one inbox owner across companies and reconnects the same identity", async () => { const f = await fixture(); const other = await fixture(); diff --git a/server/src/__tests__/fixtures/accounting-writer.ts b/server/src/__tests__/fixtures/accounting-writer.ts new file mode 100644 index 0000000000..dc4fc8a80b --- /dev/null +++ b/server/src/__tests__/fixtures/accounting-writer.ts @@ -0,0 +1,20 @@ +// Dedicated real-process fault fixture. Only invoked with a throwaway test DB. +import { createDb } from "@paperclipai/db"; +import { costService } from "../../services/costs.js"; +import { createRunUsageRecorder } from "../../services/usage-receipts.js"; +const [mode, encoded] = process.argv.slice(2); +const input = JSON.parse(encoded); +if (!process.env.PAPERCLIP_ACCOUNTING_TEST_DATABASE) throw new Error("Explicit test database required"); +const db = createDb(process.env.PAPERCLIP_ACCOUNTING_TEST_DATABASE); +try { + if (mode === "receipt") { + const recorder = await createRunUsageRecorder(db, { companyId: input.companyId, runId: input.runId, adapterType: "process" }, input.directory); + // The receipt is fsynced before persistUsageReceipt enters a transaction. + // Freeze at this boundary, so the parent can SIGKILL us before any DB row. + db.transaction = async () => { process.stdout.write("RECEIPT_DURABLE\n"); return new Promise(() => {}); }; + await recorder.capture({ costUsdExact: "0.012345678", usage: { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 }, complete: true, billingType: "metered_api" }); + } else { + await Promise.all(Array.from({ length: 4 }, () => costService(db).createEvent(input.companyId, { ...input.receipt, occurredAt: new Date(input.receipt.occurredAt) }))); + process.stdout.write("WRITES_COMMITTED\n"); + } +} finally { await db.$client.end(); } diff --git a/server/src/__tests__/heartbeat-cost-accounting.test.ts b/server/src/__tests__/heartbeat-cost-accounting.test.ts index 32ce3a1328..69c1157619 100644 --- a/server/src/__tests__/heartbeat-cost-accounting.test.ts +++ b/server/src/__tests__/heartbeat-cost-accounting.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + normalizeAdapterRunUsage, resolveCacheAdjustedCostUsd, resolveLedgerCostStatus, } from "../services/heartbeat.js"; @@ -35,6 +36,13 @@ describe("heartbeat cost accounting", () => { })).toBe("unpriced"); }); + it("distinguishes missing receipts, reported zeroes, and included subscription usage", () => { + const noTokens = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }; + expect(resolveLedgerCostStatus({ ...noTokens, costUsd: null })).toBe("unpriced"); + expect(resolveLedgerCostStatus({ ...noTokens, costUsd: 0 })).toBe("reported"); + expect(resolveLedgerCostStatus({ ...noTokens, costUsd: null, billingType: "subscription_included" })).toBe("reported"); + }); + it("marks reported CLI cost as priced", () => { expect(resolveLedgerCostStatus({ costUsd: 1.25, @@ -86,3 +94,17 @@ describe("heartbeat cost accounting", () => { })).toBe(1.5); }); }); + + +describe("adapter usage basis", () => { + const prior = { inputTokens: 100, cachedInputTokens: 500, outputTokens: 20 }; + it.each([undefined, null, "per_run"] as const)("keeps equal consecutive run usage when the basis is %s", (basis) => { + expect(normalizeAdapterRunUsage(prior, prior, basis)).toEqual(prior); + }); + it("subtracts session totals only when explicitly declared", () => { + expect(normalizeAdapterRunUsage({ inputTokens: 110, cachedInputTokens: 700, outputTokens: 25 }, prior, "session_cumulative")) + .toEqual({ inputTokens: 10, cachedInputTokens: 200, outputTokens: 5 }); + expect(normalizeAdapterRunUsage({ inputTokens: 5, cachedInputTokens: 10, outputTokens: 2 }, prior, "session_cumulative")) + .toEqual({ inputTokens: 5, cachedInputTokens: 10, outputTokens: 2 }); + }); +}); diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 4711ad72c2..52c4f2b849 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -7,6 +7,7 @@ import { agentRuntimeState, companies, costEvents, + budgetReservations, createDb, documentRevisions, documents, @@ -827,6 +828,9 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { afterContinuationDispatchCheck = async ({ runId: guardedRunId, issueId: guardedIssueId }) => { expect(guardedRunId).toBe(runId); expect(guardedIssueId).toBe(issueId); + // Budget admission must finish before this last ownership check. Putting + // an awaited reservation inside dispatch lets work escape the row lock. + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]?.state).toBe("held"); ordering.push("validated"); parkPromise = Promise.resolve( db @@ -870,6 +874,8 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { expect(issue?.status).toBe("backlog"); expect(ordering).toEqual(["validated", "parked"]); expect(countExecuteCallsForRun(runId)).toBe(0); + await heartbeat.drainActiveRunExecutions(); + expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]?.state).toBe("released"); }); it("rate-limits skipped generic timer wakes by advancing the timer baseline", async () => { @@ -1311,6 +1317,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { }); }); + it.each(["1999999999.9999999", "2000000000.0000000"])("compares exact daily spend before admitting work (%s)", async costCents => { + const { companyId, agentId } = await seedCompanyAndAgent({ heartbeatConfig: { maxDailyCostCents: 2_000_000_000 } }); + await db.insert(costEvents).values({ companyId, agentId, provider: "test", biller: "test", billingType: "metered_api", model: "test", costCents: costCents as unknown as number, occurredAt: new Date() }); + const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual" }); + if (costCents === "1999999999.9999999") expect(run).not.toBeNull(); + else { expect(run).toBeNull(); expect(mockAdapterExecute).not.toHaveBeenCalled(); } + }); + it("treats zero daily cost cap as a hard stop", async () => { const { agentId } = await seedCompanyAndAgent({ heartbeatConfig: { diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index ac83f76c55..bc2b2282e1 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -796,6 +796,36 @@ describe("openapi routes", () => { }); }); + it("documents exact money, reviewed correction inputs, and board-only accounting access", () => { + const { spec } = loadSpecRoutes(); + const base = "/api/companies/{companyId}/accounting/"; + for (const [suffix, methods] of [ + ["health", ["get"]], ["inspect", ["get"]], ["repair", ["post"]], ["retry", ["post"]], + ["invoices", ["get", "post"]], ["invoices/{invoiceId}", ["get"]], ["events/{eventId}/adjustments", ["get", "post"]], + ] as const) { + for (const method of methods) { + const operation = spec.paths[`${base}${suffix}`][method]; + expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" }); + expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]); + expect(operation.responses["403"]).toBeDefined(); + } + } + const adjustment = spec.paths[`${base}events/{eventId}/adjustments`].post; + expect(adjustment.responses["201"]).toBeDefined(); + expect(adjustment.responses["409"]).toBeDefined(); + expect(adjustment.responses["422"]).toBeDefined(); + expect(adjustment.requestBody.content["application/json"].schema.required).toEqual(expect.arrayContaining([ + "idempotencyKey", "expectedCents", "correctedCents", "reason", "pricing", + ])); + for (const [endpoint, field] of [["cost-events", "costCents"], ["finance-events", "amountCents"]]) { + const operation = spec.paths[`/api/companies/{companyId}/${endpoint}`].post; + expect(operation.requestBody.content["application/json"].schema.properties[field]).toMatchObject({ oneOf: [{ type: "string" }, { type: "number" }] }); + expect(operation.responses["409"]).toBeDefined(); + } + const parameters = spec.paths["/api/companies/{companyId}/costs/summary"].get.parameters; + expect(parameters).toEqual(expect.arrayContaining([expect.objectContaining({ name: "period", in: "query", schema: expect.objectContaining({ enum: ["month", "all"] }) })])); + }); + it("documents board-only repository discovery and selection", () => { const { spec } = loadSpecRoutes(); const discovery = spec.paths["/api/companies/{companyId}/project-repositories"].get; diff --git a/server/src/__tests__/provider-billing-import.test.ts b/server/src/__tests__/provider-billing-import.test.ts new file mode 100644 index 0000000000..aa590c6696 --- /dev/null +++ b/server/src/__tests__/provider-billing-import.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, it, vi } from "vitest"; +import { fetchProviderDailyCosts } from "../services/provider-billing-import.js"; +import type { ImportProviderCosts } from "@paperclipai/shared"; + +const input: ImportProviderCosts = { + provider: "openai", + secretId: "00000000-0000-4000-8000-000000000001", + accountId: "org_test", + scopeIds: ["proj_test"], + from: "2026-09-01", + to: "2026-09-02", +}; +const start = Date.parse(input.from) / 1000; +function page(value = 0.06) { + return { + data: [ + { + start_time: start, + end_time: start + 86400, + results: [ + { project_id: "proj_test", amount: { currency: "usd", value } }, + ], + }, + ], + has_more: false, + next_page: null, + }; +} +const response = (body: unknown) => + new Response(JSON.stringify(body), { status: 200 }); +describe("provider cost report ingestion", () => { + it("uses fixed endpoints, admin identity and scoped daily amounts in the right currency units", async () => { + const fetcher = vi.fn(async () => response(page())); + expect( + await fetchProviderDailyCosts(input, "private-admin", fetcher), + ).toEqual([ + { day: "2026-09-01", scopeId: "proj_test", amountCents: "6.0000000" }, + ]); + const [url, options] = fetcher.mock.calls[0] as unknown as [ + URL, + RequestInit, + ]; + expect(url.origin).toBe("https://api.openai.com"); + expect(url.searchParams.get("group_by[]")).toBe("project_id"); + expect(options.redirect).toBe("error"); + expect(options.headers).toMatchObject({ + "OpenAI-Organization": "org_test", + }); + }); + it("verifies Anthropic organization identity and treats its amount as cents", async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce(response({ id: "org_test" })) + .mockResolvedValueOnce( + response({ + data: [ + { + starting_at: "2026-09-01T00:00:00Z", + ending_at: "2026-09-02T00:00:00Z", + results: [ + { + workspace_id: "proj_test", + amount: "123.78912", + currency: "USD", + }, + ], + }, + ], + has_more: false, + }), + ); + expect( + ( + await fetchProviderDailyCosts( + { ...input, provider: "anthropic" }, + "private-admin", + fetcher, + ) + )[0].amountCents, + ).toBe("123.7891200"); + await expect( + fetchProviderDailyCosts( + { ...input, provider: "anthropic" }, + "private-admin", + vi.fn(async () => response({ id: "other" })), + ), + ).rejects.toThrow("different provider organization"); + }); + it("excludes unrelated projects and supports an explicit empty daily bucket", async () => { + const body = page(); + body.data[0].results[0].project_id = "other-company-project"; + expect( + ( + await fetchProviderDailyCosts( + input, + "private", + vi.fn(async () => response(body)), + ) + )[0].amountCents, + ).toBe("0.0000000"); + }); + it.each([ + "missing-day", + "missing-group", + "overlap", + "currency", + "negative", + "period", + "cursor", + "shape", + ])( + "rejects %s rather than importing partial or invented totals", + async (kind) => { + const body: any = page(); + if (kind === "missing-day") body.data = []; + if (kind === "missing-group") delete body.data[0].results[0].project_id; + if (kind === "overlap") body.data.push(body.data[0]); + if (kind === "currency") body.data[0].results[0].amount.currency = "EUR"; + if (kind === "negative") body.data[0].results[0].amount.value = -1; + if (kind === "period") body.data[0].end_time++; + if (kind === "cursor") body.has_more = true; + if (kind === "shape") body.data[0].results[0].amount = {}; + await expect( + fetchProviderDailyCosts( + input, + "private", + vi.fn(async () => response(body)), + ), + ).rejects.toThrow(); + }, + ); + it("does not expose upstream errors or credentials", async () => { + await expect( + fetchProviderDailyCosts( + input, + "private-secret", + vi.fn( + async () => + new Response("private-secret raw diagnostics", { status: 403 }), + ), + ), + ).rejects.toThrow("Check the admin credential"); + }); + it("bounds response size and does not follow redirects", async () => { + await expect( + fetchProviderDailyCosts( + input, + "private", + vi.fn(async () => new Response("x".repeat(2_000_001))), + ), + ).rejects.toThrow("too large"); + }); + it("collects all pages before returning and rejects a repeated cursor", async () => { + const first = { ...page(), has_more: true, next_page: "next" }; + const second = page(0.01); + second.data[0].start_time += 86400; + second.data[0].end_time += 86400; + const fetcher = vi + .fn() + .mockResolvedValueOnce(response(first)) + .mockResolvedValueOnce(response(second)); + expect( + await fetchProviderDailyCosts( + { ...input, to: "2026-09-03" }, + "private", + fetcher, + ), + ).toHaveLength(2); + const empty = { data: [], has_more: true, next_page: "same" }; + await expect( + fetchProviderDailyCosts( + input, + "private", + vi.fn(async () => response(empty)), + ), + ).rejects.toThrow("cursor"); + }); + it("preserves a valid report if closing its already-consumed response stream fails", async () => { + let sent = false; + const reader = { + read: async () => + sent + ? { done: true } + : ((sent = true), + { done: false, value: Buffer.from(JSON.stringify(page())) }), + cancel: async () => { + throw new Error("closed transport"); + }, + }; + const fetcher = vi + .fn() + .mockResolvedValue({ + ok: true, + body: { getReader: () => reader }, + } as unknown as Response); + expect( + (await fetchProviderDailyCosts(input, "private", fetcher))[0].amountCents, + ).toBe("6.0000000"); + }); + it("includes unallocated costs only when the default scope was explicitly selected", async () => { + const body = page(); + (body.data[0].results[0] as { project_id: string | null }).project_id = + null; + expect( + await fetchProviderDailyCosts( + { ...input, scopeIds: ["default"] }, + "private", + vi.fn(async () => response(body)), + ), + ).toEqual([ + { day: "2026-09-01", scopeId: "default", amountCents: "6.0000000" }, + ]); + }); + it("rejects a provider amount with the other provider's unit/shape and missing period boundaries", async () => { + const wrong: any = page(); + wrong.data[0].results[0].amount = "6"; + wrong.data[0].results[0].currency = "USD"; + await expect( + fetchProviderDailyCosts( + input, + "private", + vi.fn(async () => response(wrong)), + ), + ).rejects.toThrow("invalid billing amount"); + const missing: any = page(); + delete missing.data[0].start_time; + delete missing.data[0].end_time; + await expect( + fetchProviderDailyCosts( + input, + "private", + vi.fn(async () => response(missing)), + ), + ).rejects.toThrow("unexpected billing period"); + }); +}); diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index 285060b754..1cb575ed0d 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -1078,7 +1078,7 @@ describeEmbeddedPostgres("status card routes", () => { model: "gpt-5.4", inputTokens: 5200, outputTokens: 980, - costCents: 2, + costCents: 0.1234567, occurredAt: new Date(), }); const summaryWrite = await request(writerApp).put(`/api/status-cards/${created.body.id}/summary`).send({ @@ -1111,7 +1111,7 @@ describeEmbeddedPostgres("status card routes", () => { summaryBody: "**Decide:** unblock launch approval.\n\n**Recent work:** launch review is waiting.", watchedIssueCount: 1, todayTokens: 6180, - todayCostCents: 2, + todayCostCents: 0.1234567, }; const detail = await request(boardApp).get(`/api/status-cards/${created.body.id}`); expect(detail.status).toBe(200); @@ -1143,7 +1143,7 @@ describeEmbeddedPostgres("status card routes", () => { model: "gpt-5.4", inputTokens: 1300, outputTokens: 410, - costCents: 1, + costCents: 3_000_000_000.25, occurredAt: new Date(), }); const incrementalWrite = await request(createApp(db, agentActor(company.id, summarizer.id, updateRun.id))) @@ -1155,7 +1155,7 @@ describeEmbeddedPostgres("status card routes", () => { model: "gpt-5.4", }); expect(incrementalWrite.status).toBe(200); - expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "incremental"))).toMatchObject({ inputTokens: 1300, outputTokens: 410 }); + expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "incremental"))).toMatchObject({ inputTokens: 1300, outputTokens: 410, costCents: 3_000_000_000.25 }); const revisions = await request(boardApp).get(`/api/status-cards/${created.body.id}/summary-revisions`); expect(revisions.status).toBe(200); expect(revisions.body.map((row: { revisionNumber: number }) => row.revisionNumber)).toEqual([2, 1]); diff --git a/server/tsconfig.accounting-tests.json b/server/tsconfig.accounting-tests.json new file mode 100644 index 0000000000..02b4f94e44 --- /dev/null +++ b/server/tsconfig.accounting-tests.json @@ -0,0 +1,19 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "..", + "noEmit": true + }, + "include": [ + "src/__tests__/cost-accounting-*.test.ts", + "src/__tests__/codex-pricing.test.ts", + "src/__tests__/provider-billing-import.test.ts", + "src/__tests__/company-quota-windows.test.ts", + "src/__tests__/fixtures/accounting-writer.ts", + "src/**/*.d.ts", + "vitest.accounting-mutations.config.ts", + "vitest.accounting.config.ts", + "../tests/e2e/cost-accounting.spec.ts" + ], + "exclude": [] +} diff --git a/server/vitest.accounting-mutations.config.ts b/server/vitest.accounting-mutations.config.ts new file mode 100644 index 0000000000..f92b15dba5 --- /dev/null +++ b/server/vitest.accounting-mutations.config.ts @@ -0,0 +1,27 @@ +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; +import base from "./vitest.config.js"; +export const mutations = { + deduplication: { file: "/services/costs.ts", from: "idempotencyKey: parsed.data.idempotencyKey ?? null,", to: "idempotencyKey: null,", test: "deduplicates a retried receipt" }, + company: { file: "/services/costs.ts", from: "const conditions: ReturnType[] = [eq(costEvents.companyId, companyId)];", to: "const conditions: ReturnType[] = [sql`true`];", test: "isolates company reporting" }, + projection: { file: "/services/costs.ts", from: "+ ${delta}::numeric", to: "+ 0::numeric", test: "conserves agent projections" }, + threshold: { file: "/services/budgets.ts", from: "compareCents(observed.totalExact, policy.amount) >= 0", to: "compareCents(observed.totalExact, policy.amount) > 0", test: "stops at the exact budget boundary" }, +}; +const name = process.env.PAPERCLIP_ACCOUNTING_MUTATION; +if (name && !(name in mutations)) throw new Error(`Unknown accounting mutation: ${name}`); +const mutation = name ? mutations[name as keyof typeof mutations] : null; +export default defineConfig({ + ...base, root: fileURLToPath(new URL(".", import.meta.url)), + plugins: [...(base.plugins ?? []), { + name: "accounting-mutation-test-only", enforce: "pre", + transform(code, id) { + if (!mutation || !id.split("?")[0].endsWith(mutation.file)) return; + if (!code.includes(mutation.from)) throw new Error(`Mutation anchor missing: ${name}`); + // Replace precisely one site in Vite's in-memory module. Workspace source + // stays untouched, including while another test process is running. + console.error(`ACCOUNTING_MUTATION_APPLIED ${name}`); + return code.replace(mutation.from, mutation.to); + }, + }], + test: { ...base.test, include: ["src/__tests__/cost-accounting-mutation-targets.test.ts"], coverage: { enabled: false } }, +}); diff --git a/server/vitest.accounting.config.ts b/server/vitest.accounting.config.ts new file mode 100644 index 0000000000..36ade4c483 --- /dev/null +++ b/server/vitest.accounting.config.ts @@ -0,0 +1,43 @@ +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; +import base from "./vitest.config.js"; + +export default defineConfig({ + ...base, + root: fileURLToPath(new URL(".", import.meta.url)), + test: { + ...base.test, + include: [ + "src/__tests__/cost-accounting-*.test.ts", + "src/__tests__/budgets-service.test.ts", + "src/__tests__/costs-service.test.ts", + "src/__tests__/codex-pricing.test.ts", + "src/__tests__/provider-billing-import.test.ts", + ], + coverage: { + enabled: true, + provider: "v8", + reportOnFailure: true, + reportsDirectory: "../coverage/accounting", + reporter: ["text", "json", "json-summary", "html"], + // Every module must retain its own floor; a well-covered helper cannot + // hide a regression in ingestion, recovery, finance, or enforcement. + thresholds: { perFile: true, lines: 98, branches: 90, functions: 96, statements: 96 }, + include: [ + "src/services/accounting-transaction.ts", + "src/services/accounting-integrity.ts", + "src/services/billing-reconciliation.ts", + "src/services/budget-reservations.ts", + "src/services/usage-receipts.ts", + "src/services/receipt-fingerprint.ts", + "src/services/cost-date-range.ts", + "src/services/run-cost-accounting.ts", + "src/services/costs.ts", + "src/services/finance.ts", + "src/services/budgets.ts", + "src/services/codex-pricing.ts", + "src/services/provider-billing-import.ts", + ], + }, + }, +}); diff --git a/tests/e2e/cloud-auth.spec.ts b/tests/e2e/cloud-auth.spec.ts index e18084feef..d7bacd91f9 100644 --- a/tests/e2e/cloud-auth.spec.ts +++ b/tests/e2e/cloud-auth.spec.ts @@ -1,6 +1,12 @@ import { randomUUID } from "node:crypto"; import { expect, test } from "@playwright/test"; +test.afterEach(async ({ page }) => { + // Health polling may still be reading route.fetch() responses when the + // assertions finish. Drain those handlers before Playwright disposes them. + await page.unrouteAll({ behavior: "wait" }); +}); + // The tenant UI and task database are real. Cloud is an external dependency: // simulate its entry endpoint and independent session states at the HTTP edge. for (const cloudOrigin of ["https://my.paperclip.app", "https://my-staging.paperclip.app"]) {