Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-10-21 16:27:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: make accounting durable and synchronize cost reporting

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip committed 2026-10-06 21:32:48 -07:00
1 parent a6306ba606
commit 2fe2dd06bd
99 files changed
+59636 -2587
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/__tests__/approvals-service.test.ts
+15 -3
View File
@@ -1,5 +1,11 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { approvalService } from "../services/approvals.ts";
import { companies } from "@paperclipai/db";
vi.mock("../services/budgets.js", () => ({
budgetService: () => ({ deliverPendingEnforcement: vi.fn(async () => {}) }),
budgetServiceInTransaction: () => ({ upsertPolicy: vi.fn(async () => {}) }),
}));
const mockAgentService = vi.hoisted(() => ({
activatePendingApproval: vi.fn(),
@@ -39,8 +45,11 @@ function createApproval(status: string): ApprovalRecord {
function createDbStub(selectResults: ApprovalRecord[][], updateResults: ApprovalRecord[]) {
const pendingSelectResults = [...selectResults];
const selectWhere = vi.fn(async () => pendingSelectResults.shift() ?? []);
const from = vi.fn(() => ({ where: selectWhere }));
let inTransaction = false;
const selectWhere = vi.fn(async () => (inTransaction ? pendingSelectResults.shift() : pendingSelectResults[0]) ?? []);
const from = vi.fn((table) => table === companies
? { where: () => ({ for: async () => [{ id: "company-1" }] }) }
: { where: selectWhere });
const select = vi.fn(() => ({ from }));
const returning = vi.fn(async () => updateResults);
@@ -49,7 +58,10 @@ function createDbStub(selectResults: ApprovalRecord[][], updateResults: Approval
const update = vi.fn(() => ({ set }));
return {
db: { select, update, transaction: vi.fn(async (callback: (db: unknown) => Promise<unknown>) => callback({ select, update })) },
db: { select, update, transaction: vi.fn(async (callback: (db: unknown) => Promise<unknown>) => {
inTransaction = true;
try { return await callback({ select, update }); } finally { inTransaction = false; }
}) },
selectWhere,
returning,
};
server/src/__tests__/budgets-service.test.ts
+22 -309
View File
@@ -1,4 +1,5 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import {
agents,
@@ -22,313 +23,8 @@ vi.mock("../services/activity-log.js", () => ({
logActivity: mockLogActivity,
}));
type SelectResult = unknown[];
function createDbStub(selectResults: SelectResult[]) {
const pendingSelects = [...selectResults];
const selectWhere = vi.fn(() => {
const query = Promise.resolve(pendingSelects.shift() ?? []);
return Object.assign(query, { for: () => query });
});
const selectThen = vi.fn((resolve: (value: unknown[]) => unknown) => Promise.resolve(resolve(pendingSelects.shift() ?? [])));
const selectOrderBy = vi.fn(async () => pendingSelects.shift() ?? []);
const selectFrom = vi.fn(() => ({
where: selectWhere,
then: selectThen,
orderBy: selectOrderBy,
}));
const select = vi.fn(() => ({
from: selectFrom,
}));
const insertValues = vi.fn();
const insertReturning = vi.fn(async () => pendingInserts.shift() ?? []);
const insert = vi.fn(() => ({
values: insertValues.mockImplementation(() => ({
returning: insertReturning,
})),
}));
const updateSet = vi.fn();
const updateWhere = vi.fn(async () => pendingUpdates.shift() ?? []);
const update = vi.fn(() => ({
set: updateSet.mockImplementation(() => ({
where: updateWhere,
})),
}));
const pendingInserts: unknown[][] = [];
const pendingUpdates: unknown[][] = [];
return {
db: {
select,
insert,
update,
transaction: async (callback: (db: unknown) => Promise<unknown>) => callback({ select, insert, update }),
},
queueInsert: (rows: unknown[]) => {
pendingInserts.push(rows);
},
queueUpdate: (rows: unknown[] = []) => {
pendingUpdates.push(rows);
},
selectWhere,
insertValues,
updateSet,
};
}
describe("budgetService", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("creates a hard-stop incident and pauses an agent when spend exceeds a budget", async () => {
const policy = {
id: "policy-1",
companyId: "company-1",
scopeType: "agent",
scopeId: "agent-1",
metric: "billed_cents",
windowKind: "calendar_month_utc",
amount: 100,
warnPercent: 80,
hardStopEnabled: true,
notifyEnabled: false,
isActive: true,
};
const dbStub = createDbStub([
[policy],
[{ total: 150 }],
[],
[{
companyId: "company-1",
name: "Budget Agent",
status: "running",
pauseReason: null,
}],
[{ id: "agent-1", companyId: "company-1", status: "running" }],
]);
dbStub.queueInsert([{
id: "approval-1",
companyId: "company-1",
status: "pending",
}]);
dbStub.queueInsert([{
id: "incident-1",
companyId: "company-1",
policyId: "policy-1",
approvalId: "approval-1",
}]);
dbStub.queueUpdate([]);
const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);
const service = budgetService(dbStub.db as any, { cancelWorkForScope });
await service.evaluateCostEvent({
companyId: "company-1",
agentId: "agent-1",
projectId: null,
} as any);
expect(dbStub.insertValues).toHaveBeenCalledWith(
expect.objectContaining({
companyId: "company-1",
type: "budget_override_required",
status: "pending",
}),
);
expect(dbStub.insertValues).toHaveBeenCalledWith(
expect.objectContaining({
companyId: "company-1",
policyId: "policy-1",
thresholdType: "hard",
amountLimit: 100,
amountObserved: 150,
approvalId: "approval-1",
}),
);
expect(dbStub.updateSet).toHaveBeenCalledWith(
expect.objectContaining({
status: "paused",
pauseReason: "budget",
pausedAt: expect.any(Date),
}),
);
expect(mockLogActivity).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
action: "budget.hard_threshold_crossed",
entityId: "incident-1",
}),
);
expect(cancelWorkForScope).toHaveBeenCalledWith({
companyId: "company-1",
scopeType: "agent",
scopeId: "agent-1",
});
});
it("blocks new work when an agent hard-stop remains exceeded even if the agent is not paused yet", async () => {
const agentPolicy = {
id: "policy-agent-1",
companyId: "company-1",
scopeType: "agent",
scopeId: "agent-1",
metric: "billed_cents",
windowKind: "calendar_month_utc",
amount: 100,
warnPercent: 80,
hardStopEnabled: true,
notifyEnabled: true,
isActive: true,
};
const dbStub = createDbStub([
[{
status: "running",
pauseReason: null,
companyId: "company-1",
name: "Budget Agent",
}],
[{
status: "active",
name: "Paperclip",
}],
[],
[agentPolicy],
[{ total: 120 }],
]);
const service = budgetService(dbStub.db as any);
const block = await service.getInvocationBlock("company-1", "agent-1");
expect(block).toEqual({
scopeType: "agent",
scopeId: "agent-1",
scopeName: "Budget Agent",
reason: "Agent cannot start because its budget hard-stop is still exceeded.",
});
});
it("surfaces a budget-owned company pause distinctly from a manual pause", async () => {
const dbStub = createDbStub([
[{
status: "idle",
pauseReason: null,
companyId: "company-1",
name: "Budget Agent",
}],
[{
status: "paused",
pauseReason: "budget",
name: "Paperclip",
}],
]);
const service = budgetService(dbStub.db as any);
const block = await service.getInvocationBlock("company-1", "agent-1");
expect(block).toEqual({
scopeType: "company",
scopeId: "company-1",
scopeName: "Paperclip",
reason: "Company is paused because its budget hard-stop was reached.",
});
});
it("uses live observed spend when raising a budget incident", async () => {
const dbStub = createDbStub([
[{
id: "incident-1",
companyId: "company-1",
policyId: "policy-1",
amountObserved: 120,
approvalId: "approval-1",
}],
[{
id: "policy-1",
companyId: "company-1",
scopeType: "company",
scopeId: "company-1",
metric: "billed_cents",
windowKind: "calendar_month_utc",
}],
[{ total: 150 }],
]);
const service = budgetService(dbStub.db as any);
await expect(
service.resolveIncident(
"company-1",
"incident-1",
{ action: "raise_budget_and_resume", amount: 140 },
"board-user",
),
).rejects.toThrow("New budget must exceed current observed spend");
});
it("syncs company monthly budget when raising and resuming a company incident", async () => {
const now = new Date();
const dbStub = createDbStub([
[{
id: "incident-1",
companyId: "company-1",
policyId: "policy-1",
scopeType: "company",
scopeId: "company-1",
metric: "billed_cents",
windowKind: "calendar_month_utc",
windowStart: now,
windowEnd: now,
thresholdType: "hard",
amountLimit: 100,
amountObserved: 120,
status: "open",
approvalId: "approval-1",
resolvedAt: null,
createdAt: now,
updatedAt: now,
}],
[{
id: "policy-1",
companyId: "company-1",
scopeType: "company",
scopeId: "company-1",
metric: "billed_cents",
windowKind: "calendar_month_utc",
amount: 100,
}],
[{ total: 120 }],
[{ id: "approval-1", status: "approved" }],
[{
companyId: "company-1",
name: "Paperclip",
status: "paused",
pauseReason: "budget",
pausedAt: now,
}],
]);
const service = budgetService(dbStub.db as any);
await service.resolveIncident(
"company-1",
"incident-1",
{ action: "raise_budget_and_resume", amount: 175 },
"board-user",
);
expect(dbStub.updateSet).toHaveBeenCalledWith(
expect.objectContaining({
budgetMonthlyCents: 175,
updatedAt: expect.any(Date),
}),
);
});
});
// Budget behavior is exercised against PostgreSQL below and in
// cost-accounting-reliability.test.ts, including transaction and retry paths.
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
@@ -493,7 +189,7 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => {
expect(agentAfterHardStop).toMatchObject({ status: "paused", pauseReason: "budget" });
expect(agentAfterHardStop?.pausedAt).toBeInstanceOf(Date);
expect(cancelWorkForScope).toHaveBeenCalledTimes(2);
expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "agent", scopeId: agentId });
expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "agent", scopeId: agentId, createdBefore: expect.any(Date), enforcement: { policyId: expect.any(String), version: expect.any(Number) } });
const block = await service.getInvocationBlock(companyId, agentId);
expect(block).toEqual({
@@ -538,6 +234,23 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => {
}
});
it("distinguishes a company budget pause from a manual company pause", async () => {
const { companyId, agentId } = await createBudgetFixture();
const service = budgetService(db);
await db.update(companies).set({ status: "paused", pauseReason: "budget" }).where(eq(companies.id, companyId));
expect(await service.getInvocationBlock(companyId, agentId)).toMatchObject({
scopeType: "company",
scopeId: companyId,
reason: "Budget pause requires a policy or an explicit operator resume.",
});
await db.update(companies).set({ pauseReason: null }).where(eq(companies.id, companyId));
expect(await service.getInvocationBlock(companyId, agentId)).toMatchObject({
scopeType: "company",
scopeId: companyId,
reason: "Company is paused and cannot start new work.",
});
});
it("hard-stops project work until a valid budget raise resumes it and overview reconciles ledger spend", async () => {
const { companyId, agentId, projectId } = await createBudgetFixture();
const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);
@@ -578,7 +291,7 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => {
.from(projects);
expect(projectAfterHardStop?.pauseReason).toBe("budget");
expect(projectAfterHardStop?.pausedAt).toBeInstanceOf(Date);
expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "project", scopeId: projectId });
expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "project", scopeId: projectId, createdBefore: expect.any(Date), enforcement: { policyId: expect.any(String), version: expect.any(Number) } });
const overviewWhileBlocked = await service.overview(companyId);
expect(overviewWhileBlocked.pausedProjectCount).toBe(1);
server/src/__tests__/costs-service.test.ts
+29 -18
View File
@@ -221,6 +221,21 @@ describe("cost routes", () => {
});
});
it("requires an explicit all-time request and rejects conflicting ranges", () => {
const { parseCostDateRange } = loadCostParsers();
expect(parseCostDateRange({})).toBeUndefined();
expect(parseCostDateRange({ period: "all" })).toEqual({ allTime: true });
for (const input of [
{ from: "2026-09-02", to: "2026-09-01" },
{ from: ["2026-09-01"] },
{ from: "2026-02-30" },
{ from: "" },
{ from: "09/01/2026" },
{ period: "all", from: "2026-09-01" },
{ period: "unknown" },
]) expect(() => parseCostDateRange(input)).toThrow();
});
it("returns 400 for an invalid 'from' date string", async () => {
const { parseCostDateRange } = loadCostParsers();
expect(() => parseCostDateRange({ from: "not-a-date" })).toThrow(/invalid 'from' date/i);
@@ -270,7 +285,7 @@ describe("cost routes", () => {
it("returns 400 for invalid finance event list limits", async () => {
const { parseCostLimit } = loadCostParsers();
expect(() => parseCostLimit({ limit: "0" })).toThrow(/invalid 'limit'/i);
for (const limit of ["0", "25garbage", "1.5", ["25"], "501"]) expect(() => parseCostLimit({ limit })).toThrow(/invalid 'limit'/i);
});
it("accepts valid finance event list limits", async () => {
@@ -352,7 +367,8 @@ describe("cost routes", () => {
});
it("allows authorized board users to update an agent budget and budget policy", async () => {
mockAgentService.update.mockResolvedValueOnce({
mockAgentService.getById.mockResolvedValueOnce({ id: "agent-1", companyId: "company-1", name: "Budget Agent", budgetMonthlyCents: 100, spentMonthlyCents: 0 });
mockAgentService.getById.mockResolvedValueOnce({
id: "agent-1",
companyId: "company-1",
name: "Budget Agent",
@@ -373,30 +389,22 @@ describe("cost routes", () => {
.send({ budgetMonthlyCents: 2500 });
expect(res.status).toBe(200);
expect(mockAgentService.update).toHaveBeenCalledWith("agent-1", { budgetMonthlyCents: 2500 });
expect(res.body.budgetMonthlyCents).toBe(2500);
expect(mockAgentService.update).not.toHaveBeenCalled();
expect(mockBudgetService.upsertPolicy).toHaveBeenCalledTimes(1);
expect(mockBudgetService.upsertPolicy).toHaveBeenCalledWith(
"company-1",
{
scopeType: "agent",
scopeId: "agent-1",
amount: 2500,
isActive: true,
windowKind: "calendar_month_utc",
},
"board-user",
);
expect(mockLogActivity).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
companyId: "company-1",
actorType: "user",
actorId: "board-user",
agentId: null,
action: "agent.budget_updated",
entityType: "agent",
entityId: "agent-1",
details: { budgetMonthlyCents: 2500 },
}),
);
// Policy mutation owns the atomic entity update and activity log.
expect(mockLogActivity).not.toHaveBeenCalled();
});
});
@@ -543,7 +551,9 @@ describeEmbeddedPostgres("cost and finance aggregate overflow handling", () => {
const [byAgentModelRow] = await costs.byAgentModel(companyId, range);
expect(byAgentRow?.costCents).toBe(4_000_000_000);
expect(byAgentRow?.inputTokens).toBe(4_000_000_000);
// Historical OpenAI input includes these ten cached tokens.
expect(byAgentRow?.inputTokens).toBe(3_999_999_990);
expect(byAgentRow!.inputTokens + byAgentRow!.cachedInputTokens).toBe(4_000_000_000);
expect(byProjectRow?.costCents).toBe(4_000_000_000);
expect(byAgentModelRow?.costCents).toBe(4_000_000_000);
});
@@ -692,7 +702,8 @@ describeEmbeddedPostgres("cost and finance aggregate overflow handling", () => {
issueCount: 3,
includeDescendants: true,
costCents: 600,
inputTokens: 60,
costCentsExact: "600.0000000",
inputTokens: 54,
cachedInputTokens: 6,
outputTokens: 12,
runCount: 0,
server/src/__tests__/heartbeat-list.test.ts
+2 -1
View File
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import { costEvents, agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
@@ -26,6 +26,7 @@ describeEmbeddedPostgres("heartbeat list", () => {
}, 20_000);
afterEach(async () => {
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(companies);
server/src/__tests__/heartbeat-process-recovery.test.ts
+40
View File
@@ -33,7 +33,9 @@ import {
agentWakeupRequests,
approvals,
authUsers,
budgetIncidents,
budgetPolicies,
budgetReservations,
chatActions,
chatConversations,
chatDeliveries,
@@ -601,6 +603,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
await db.delete(issueTreeHolds);
await db.delete(agentTaskSessions);
await db.delete(issueApprovals);
await db.delete(budgetIncidents);
await db.delete(approvals);
await db.update(issues).set({ lastStatusDecisionId: null });
await db.delete(statusDecisionEffects);
@@ -7798,6 +7801,36 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
},
);
it.each([{}, { command: "/nonexistent-paperclip-bootstrap-command" }])("releases holds for real process bootstrap failures: %j", async (config) => {
const { execute } = await import("../adapters/process/execute.js");
mockAdapterExecute.mockImplementationOnce(execute);
const { runId, agentId } = await seedRunFixture({ runtimeMode: "legacy", agentStatus: "idle", runStatus: "queued", includeIssue: false });
await db.update(agents).set({ adapterType: "process", adapterConfig: config }).where(eq(agents.id, agentId));
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await heartbeat.drainActiveRunExecutions();
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "failed", costAccountingPending: false, costAccountedAt: expect.any(Date) });
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]?.state).toBe("released");
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, runId))).toEqual([]);
});
it("releases the dispatch reservation after a proven pre-provider adapter failure", async () => {
mockAdapterExecute.mockResolvedValueOnce({
exitCode: 1, signal: null, timedOut: false,
errorMessage: "Provider startup failed", provider: "test", model: "test-model",
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
});
const { runId } = await seedRunFixture({ runtimeMode: "legacy", agentStatus: "idle", runStatus: "queued", includeIssue: false });
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await heartbeat.drainActiveRunExecutions();
const run = await heartbeat.getRun(runId);
expect(run).toMatchObject({ status: "failed", costAccountingPending: false });
expect(run?.costAccountedAt).not.toBeNull();
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]).toMatchObject({ state: "released" });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, runId))).toEqual([]);
});
it.each(["single Stop", "agent pause"] as const)(
"fences adapter registration while an earlier no-owner %s waits to commit",
async (operation) => {
@@ -7922,6 +7955,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
releaseAdapter();
await heartbeat.drainActiveRunExecutions();
const settledRun = await heartbeat.getRun(runId);
expect(settledRun?.costAccountingPending).toBe(false);
expect(settledRun?.costAccountedAt).not.toBeNull();
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]).toMatchObject({ state: "released" });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, runId))).toEqual([]);
expect({
status: settledRun?.status,
errorCode: settledRun?.errorCode,
@@ -11313,6 +11350,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
await waitForRunToSettle(heartbeat, sourceRun.id, 5_000);
}
expect(sourceRun?.id).not.toBe(runId);
// Durable accounting is bookkeeping, not evidence that the agent acted
// on its plan. A receipt must not suppress plan-only recovery.
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, String(sourceRunId)))).toHaveLength(1);
expect(sourceRun?.livenessState).toBe("plan_only");
if (withLegacyReview) {
expect(await db.select().from(issues).where(eq(issues.id, legacyReviewId))).toEqual(legacyReviewBefore);
server/src/__tests__/plugin-managed-agents.test.ts
+3
View File
@@ -11,6 +11,7 @@ import {
agentInstructionHeads,
agents,
approvals,
budgetPolicies,
companies,
createDb,
pluginEntities,
@@ -107,6 +108,7 @@ describeEmbeddedPostgres("plugin-managed agents", () => {
await db.delete(pluginManagedResources);
await db.delete(pluginCompanySettings);
await db.delete(approvals);
await db.delete(budgetPolicies);
await db.delete(principalPermissionGrants);
await db.delete(agents);
await db.delete(plugins);
@@ -499,6 +501,7 @@ describeEmbeddedPostgres("plugin-managed agents", () => {
const relinked = await services.agents.managedReconcile({ companyId, agentKey: "wiki-maintainer" });
expect(relinked.status).toBe("relinked");
expect(relinked.agentId).toBe(agentId);
expect(relinked.agent?.urlKey).toBe("renamed-wiki-agent");
const [binding] = await db.select().from(pluginEntities);
expect(binding?.data).toMatchObject({ agentId });
server/src/__tests__/routines-routes.test.ts
+4 -4
View File
@@ -166,10 +166,10 @@ function registerModuleMocks() {
}
async function createApp(actor: Record<string, unknown>) {
const [{ errorHandler }, { routineRoutes }] = await Promise.all([
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
vi.importActual<typeof import("../routes/routines.js")>("../routes/routines.js"),
]);
// Load the route after its mocks are installed. Concurrent importActual
// calls through the middleware barrel can cache unmocked dependencies.
const { errorHandler } = await import("../middleware/error-handler.js");
const { routineRoutes } = await import("../routes/routines.js");
const app = express();
app.use(express.json({ verify: (req, _res, buf) => { (req as any).rawBody = buf; } }));
app.use((req, _res, next) => {
server/src/__tests__/server-startup-feedback-export.test.ts
+25
View File
@@ -57,6 +57,7 @@ const {
reason: null,
}));
const heartbeatServiceMock = {
reconcileCostAccounting: vi.fn(async () => ({ scanned: 0, accounted: 0 })),
resolveSchedulingSuppression: resolveHeartbeatSchedulingSuppressionMock,
recoverNativeRunsAfterRestart: vi.fn(async () => ({
restartKind: "hard",
@@ -559,11 +560,13 @@ describe("startServer feedback export wiring", () => {
}) as typeof setInterval);
try {
await startServer();
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1);
expect(intervalCallback).not.toBeNull();
intervalCallback?.();
await new Promise<void>((resolve) => setImmediate(resolve));
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2);
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(2);
expect(retiredDetector).not.toHaveBeenCalled();
} finally {
delete (runtime as Partial<typeof runtime>).reconcileProductivityReviews;
@@ -660,6 +663,7 @@ describe("startServer feedback export wiring", () => {
// reaped at startup and on the interval.
expect(heartbeatServiceFactoryMock).toHaveBeenCalledTimes(1);
expect(heartbeatServiceMock.sweepPendingCleanupLeases).toHaveBeenCalled();
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
expect(intervalCallback).not.toBeNull();
intervalCallback?.();
await Promise.resolve();
@@ -667,12 +671,33 @@ describe("startServer feedback export wiring", () => {
expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1);
expect(routineServiceMock.tickScheduledTriggers).not.toHaveBeenCalled();
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(2);
expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled();
} finally {
setIntervalSpy.mockRestore();
}
});
it.each([false, true])("retries failed accounting recovery with heartbeat scheduling enabled: %s", async (enabled) => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
heartbeatServiceMock.reconcileCostAccounting.mockRejectedValueOnce(new Error("temporary ledger outage"));
let tick: (() => void) | undefined;
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => {
tick = callback;
return 1 as unknown as ReturnType<typeof setInterval>;
}) as typeof setInterval);
try {
expect((await startServer()).server).toBe(fakeServer);
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
expect(tick).toBeDefined();
tick?.();
await new Promise<void>((resolve) => setImmediate(resolve));
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(2);
} finally {
timer.mockRestore();
}
});
it("does not replay hot-restart adoption when the orphan reaper retries", async () => {
loadConfigMock.mockReturnValue(buildTestConfig({
heartbeatSchedulerEnabled: true,
server/src/index.ts
+9 -4
View File
@@ -879,6 +879,7 @@ async function startServerWithDatabaseTeardown(
const heartbeat = config.heartbeatSchedulerEnabled
? heartbeatService(db as any, { pluginWorkerManager })
: null;
const accountingHeartbeat = heartbeat ?? heartbeatService(db as any, { pluginWorkerManager });
const decisionServiceOptions = {
wakeOriginAgent: createDecisionWakeOriginAgent(heartbeat?.wakeup ?? null),
};
@@ -1230,8 +1231,7 @@ async function startServerWithDatabaseTeardown(
// so a just-failed lease does not draw a retry on every tick. The startup
// sweep passes zero, so a restart retries a stranded orphan at once.
const ENVIRONMENT_LEASE_CLEANUP_SWEEP_BACKOFF_MS = 5 * 60 * 1000;
const environmentLeaseCleanupHeartbeat =
heartbeat ?? heartbeatService(db as any, { pluginWorkerManager });
const environmentLeaseCleanupHeartbeat = accountingHeartbeat;
const chatCompletionDeliveries = chatCompletionDeliveryService(db as any, environmentLeaseCleanupHeartbeat);
// Activity publication happens after the status transaction commits. This is
// a best-effort fast path; the durable outbox and sweeps remain authoritative.
@@ -1449,6 +1449,7 @@ async function startServerWithDatabaseTeardown(
},
"worktree run-execution cutoff state",
);
await accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed; pending receipts will retry"));
const heartbeatSchedulingSuppression = await heartbeat.resolveSchedulingSuppression();
// Reap orphaned runs before timer ticks start so wakeups cannot coalesce
@@ -1464,8 +1465,7 @@ async function startServerWithDatabaseTeardown(
// workers before cleanup or retry admission, including unmanaged installs.
await app.locals.bundledPluginsStartup;
try {
const nativeRecovery =
await heartbeat.recoverNativeRunsAfterRestart();
const nativeRecovery = await heartbeat.recoverNativeRunsAfterRestart();
if (nativeRecovery.dispositions.length > 0) {
logger.info(
{
@@ -1763,6 +1763,9 @@ async function startServerWithDatabaseTeardown(
}));
trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending().catch((err) => logger.error({ err }, "chat completion delivery failed")));
trackHeartbeatSchedulerWork(accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed")));
trackHeartbeatSchedulerWork(connectionDeliveries.sweepPending().catch((err) => logger.error({ err }, "connection continuation delivery failed")));
trackHeartbeatSchedulerWork(app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "tool review recovery failed")));
trackHeartbeatSchedulerWork(app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "gateway token cleanup failed")));
@@ -1839,8 +1842,10 @@ async function startServerWithDatabaseTeardown(
// is still required. A failed acquire can leak a paid provider sandbox, so
// this path retries the teardown at startup and on the interval, exactly as
// the enabled path does.
await accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed; pending receipts will retry"));
await runEnvironmentLeaseCleanupSweep(0);
startHeartbeatSchedulerInterval(() => {
trackHeartbeatSchedulerWork(accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed")));
scheduleExternalObjectRefreshSweep(new Date());
scheduleEnvironmentLeaseCleanupSweep();
scheduleGitHubConnectionEventPoll();
server/src/modules/run-dispatch/adapters/postgres.ts
+13 -8
View File
@@ -15,9 +15,11 @@ import {
issues,
} from "@paperclipai/db";
import { ISSUE_DISPOSITION_REPAIR_RETRY_REASON } from "@paperclipai/shared";
import { withAccountingTransaction } from "../../../services/accounting-transaction.js";
import type { ActivityPublication } from "../../../services/activity-log.js";
import { parseObject } from "../../../adapters/utils.js";
import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokability.js";
import { budgetService } from "../../../services/budgets.js";
import { budgetService, budgetServiceInTransaction } from "../../../services/budgets.js";
import { isCompletedOnboardingHandoffWake } from "../../../services/chat-completion-delivery.js";
import { isHeartbeatWakeOnDemandEnabled } from "../../../services/heartbeat-policy.js";
import { collectDispositionRepairSourceState } from "../../../services/recovery/disposition-repair.js";
@@ -190,7 +192,7 @@ export function createPostgresRunDispatchAdapter(
async function withIssueThenRunLocks<T>(
input: { runId: string; companyId: string },
onMissing: () => T,
operation: (tx: Db, run: HeartbeatRun) => Promise<T>,
operation: (tx: Db, run: HeartbeatRun, publications: ActivityPublication[]) => Promise<T>,
): Promise<T> {
// Queue editing and claiming already use issue -> wake -> run. Read the
// immutable issue reference without a lock first, then acquire issue ->
@@ -206,8 +208,9 @@ export function createPostgresRunDispatchAdapter(
const hintedIssueId = readNonEmptyString(parseObject(hint.contextSnapshot).issueId);
const result: { kind: "missing" | "retry" } | { kind: "value"; value: T } =
await db.transaction(async (tx) => {
const typedTx = tx as unknown as Db;
await withAccountingTransaction(db, input.companyId, async (typedTx, publications) => {
// Budget gates mutate policy state. Acquire company before issue/run,
// matching receipt writers (including their attribution FK locks).
if (hintedIssueId) {
await typedTx
.select({ id: issues.id })
@@ -231,7 +234,7 @@ export function createPostgresRunDispatchAdapter(
const lockedIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
if (lockedIssueId !== hintedIssueId) return { kind: "retry" as const };
return { kind: "value" as const, value: await operation(typedTx, run) };
return { kind: "value" as const, value: await operation(typedTx, run, publications) };
});
if (result.kind === "missing") return onMissing();
@@ -247,12 +250,13 @@ export function createPostgresRunDispatchAdapter(
input: LoadGateFactsInput,
now: Date,
tx?: unknown,
publications?: ActivityPublication[],
): Promise<LoadGateFactsResult> {
// Semantic adapter operations pass their transaction here so the fact
// read and the state transition share one unit of work. This helper is
// deliberately not exposed through the module's public API.
const dbOrTx = (tx as Db | undefined) ?? db;
const budgetsForRead = tx ? budgetService(dbOrTx) : budgets;
const budgetsForRead = tx ? budgetServiceInTransaction(dbOrTx, publications) : budgets;
const treeControlForRead = tx ? issueTreeControlService(dbOrTx) : treeControlSvc;
const issuesSvcForRead = tx ? issueService(dbOrTx) : issuesSvc;
@@ -747,7 +751,7 @@ export function createPostgresRunDispatchAdapter(
): Promise<PromoteScheduledRetryOutcome> {
const now = input.now;
const promoteLockedRun = async (tx: Db, run: HeartbeatRun) => {
const promoteLockedRun = async (tx: Db, run: HeartbeatRun, publications: ActivityPublication[]) => {
if (
run.status !== "scheduled_retry" ||
!run.scheduledRetryAt ||
@@ -768,6 +772,7 @@ export function createPostgresRunDispatchAdapter(
},
now,
tx,
publications,
);
if (!factsResult.agentFound) {
@@ -1010,7 +1015,7 @@ export function createPostgresRunDispatchAdapter(
input: DispatchResolvedInteractionInput<T>,
): Promise<DispatchResolvedInteractionOutcome<T>> {
const dispatchLockedRun = async (tx: Db, run: HeartbeatRun) => {
if (run.status !== input.expectedStatus) {
if (run.status !== input.expectedStatus || run.resultJson?.cancellation || run.resultJson?.startupCancellation) {
return { dispatched: false as const, cancellation: { outcome: "lost_race" as const } };
}
const { issueId, decision: initialDecision } = await decideCurrentRunStaleness(
server/src/routes/agents.ts
+1 -1
View File
@@ -546,7 +546,7 @@ export function agentRoutes(
const KNOWN_INSTRUCTIONS_BUNDLE_KEY_SET: ReadonlySet<string> = new Set(KNOWN_INSTRUCTIONS_BUNDLE_KEYS);
const router = Router();
const svc = agentService(db);
const svc = agentService(db, { cancelWorkForScope: scope => heartbeat.cancelBudgetScopeWork(scope) });
const access = accessService(db);
const approvalsSvc = approvalService(db);
const budgets = budgetService(db);
server/src/routes/companies.ts
+4 -1
View File
@@ -55,6 +55,7 @@ import {
accessService,
agentService,
budgetService,
heartbeatService,
buildExportFidelityReport,
collectExportFidelityCounts,
companyArtifactsService,
@@ -278,7 +279,9 @@ export function companyRoutes(db: Db, storage?: StorageService, options?: Compan
const router = Router();
const importTransferSpoolRoot =
options?.importTransferSpoolRoot ?? resolveDefaultImportTransferSpoolRoot();
const svc = companyService(db);
const svc = companyService(db, {
cancelWorkForScope: (scope) => heartbeatService(db).cancelBudgetScopeWork(scope),
});
const agents = agentService(db);
const portability = companyPortabilityService(db, storage);
const access = accessService(db);
server/src/routes/costs.ts
+82 -75
View File
@@ -1,4 +1,10 @@
import { importProviderDailyCosts } from "../services/provider-billing-import.js";
import { importProviderCostsSchema } from "@paperclipai/shared";
import { accountingIntegrityService } from "../services/accounting-integrity.js";
import { billingReconciliationService } from "../services/billing-reconciliation.js";
import { adjustCostSchema, importBillingInvoiceSchema, repairAccountingSchema, retryAccountingSchema } from "@paperclipai/shared";
import { Router } from "express";
import { z } from "zod";
import type { Db } from "@paperclipai/db";
import {
createCostEventSchema,
@@ -18,28 +24,37 @@ import {
issueService,
heartbeatService,
accessService,
logActivity,
} from "../services/index.js";
import { assertBoard, assertCompanyAccess, getAccessibleResource, getActorInfo } from "./authz.js";
import { fetchCompanyQuotaWindows } from "../services/quota-windows.js";
import { badRequest } from "../errors.js";
import { badRequest, forbidden } from "../errors.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
const reportDateSchema = z.union([z.iso.datetime({ offset: true }), z.iso.date()]);
export function parseCostDateRange(query: Record<string, unknown>) {
if (query.period !== undefined && query.period !== "all" && query.period !== "month") throw badRequest("invalid 'period'");
if (query.period === "all") {
if (query.from !== undefined || query.to !== undefined) throw badRequest("all-time period cannot have date bounds");
return { allTime: true };
}
if (query.from !== undefined && !reportDateSchema.safeParse(query.from).success) throw badRequest("invalid 'from' date");
if (query.to !== undefined && !reportDateSchema.safeParse(query.to).success) throw badRequest("invalid 'to' date");
const fromRaw = query.from as string | undefined;
const toRaw = query.to as string | undefined;
const from = fromRaw ? new Date(fromRaw) : undefined;
const to = toRaw ? new Date(toRaw) : undefined;
if (from && isNaN(from.getTime())) throw badRequest("invalid 'from' date");
if (to && isNaN(to.getTime())) throw badRequest("invalid 'to' date");
if (from && to && from > to) throw badRequest("from must not be after to");
return (from || to) ? { from, to } : undefined;
}
export function parseCostLimit(query: Record<string, unknown>) {
const raw = Array.isArray(query.limit) ? query.limit[0] : query.limit;
const raw = query.limit;
if (raw == null || raw === "") return 100;
const limit = typeof raw === "number" ? raw : Number.parseInt(String(raw), 10);
if (!Number.isFinite(limit) || limit <= 0 || limit > 500) {
const limit = typeof raw === "number" || typeof raw === "string" ? Number(raw) : Number.NaN;
if (!Number.isInteger(limit) || limit <= 0 || limit > 500) {
throw badRequest("invalid 'limit' value");
}
return limit;
@@ -120,22 +135,11 @@ export function costRoutes(
return;
}
const actor = getActorInfo(req);
const event = await costs.createEvent(companyId, {
...req.body,
occurredAt: new Date(req.body.occurredAt),
});
const actor = getActorInfo(req);
await logActivity(db, {
companyId,
actorType: actor.actorType,
actorId: actor.actorId,
agentId: actor.agentId,
action: "cost.reported",
entityType: "cost_event",
entityId: event.id,
details: { costCents: event.costCents, model: event.model },
});
}, { actorType: actor.actorType, actorId: actor.actorId, agentId: actor.agentId });
res.status(201).json(event);
});
@@ -145,27 +149,11 @@ export function costRoutes(
assertCompanyAccess(req, companyId);
assertBoard(req);
const actor = getActorInfo(req);
const event = await finance.createEvent(companyId, {
...req.body,
occurredAt: new Date(req.body.occurredAt),
});
const actor = getActorInfo(req);
await logActivity(db, {
companyId,
actorType: actor.actorType,
actorId: actor.actorId,
agentId: actor.agentId,
action: "finance_event.reported",
entityType: "finance_event",
entityId: event.id,
details: {
amountCents: event.amountCents,
biller: event.biller,
eventKind: event.eventKind,
direction: event.direction,
},
});
}, { actorType: actor.actorType, actorId: actor.actorId, agentId: actor.agentId });
res.status(201).json(event);
});
@@ -189,6 +177,14 @@ export function costRoutes(
res.json(summary);
});
router.get("/companies/:companyId/costs/by-user", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
if (!(await assertCompanyCostReadAllowed(req, res, companyId))) return;
const range = parseCostDateRange(req.query);
res.json(await costs.byUser(companyId, range));
});
router.get("/companies/:companyId/costs/by-agent", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
@@ -331,34 +327,19 @@ export function costRoutes(
assertBoard(req);
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
const company = await companies.update(companyId, { budgetMonthlyCents: req.body.budgetMonthlyCents });
if (!company) {
res.status(404).json({ error: "Company not found" });
return;
}
await logActivity(db, {
companyId,
actorType: "user",
actorId: req.actor.userId ?? "board",
action: "company.budget_updated",
entityType: "company",
entityId: companyId,
details: { budgetMonthlyCents: req.body.budgetMonthlyCents },
});
await budgets.upsertPolicy(
companyId,
{
scopeType: "company",
scopeId: companyId,
amount: req.body.budgetMonthlyCents,
isActive: req.body.budgetMonthlyCents > 0,
windowKind: "calendar_month_utc",
},
req.actor.userId ?? "board",
);
res.json(company);
res.json(await companies.getById(companyId));
});
router.patch("/agents/:agentId/budgets", validate(updateBudgetSchema), async (req, res) => {
@@ -368,37 +349,63 @@ export function costRoutes(
assertBoard(req);
const updated = await agents.update(agentId, { budgetMonthlyCents: req.body.budgetMonthlyCents });
if (!updated) {
res.status(404).json({ error: "Agent not found" });
return;
}
const actor = getActorInfo(req);
await logActivity(db, {
companyId: updated.companyId,
actorType: actor.actorType,
actorId: actor.actorId,
agentId: actor.agentId,
action: "agent.budget_updated",
entityType: "agent",
entityId: updated.id,
details: { budgetMonthlyCents: updated.budgetMonthlyCents },
});
await budgets.upsertPolicy(
updated.companyId,
agent.companyId,
{
scopeType: "agent",
scopeId: updated.id,
amount: updated.budgetMonthlyCents,
scopeId: agent.id,
amount: req.body.budgetMonthlyCents,
isActive: req.body.budgetMonthlyCents > 0,
windowKind: "calendar_month_utc",
},
req.actor.type === "board" ? req.actor.userId ?? "board" : null,
);
res.json(updated);
res.json(await agents.getById(agentId));
});
const integrity = accountingIntegrityService(db, budgetHooks);
const billing = billingReconciliationService(db, budgetHooks);
// Operational accounting evidence and repairs are board-only. These payloads
// can contain company-wide billing identifiers even for an agent's own run.
router.use("/companies/:companyId/accounting", (req, _res, next) => {
assertCompanyAccess(req, req.params.companyId as string); assertBoard(req); next();
});
router.post("/companies/:companyId/accounting/provider-costs/import", (req, _res, next) => {
// Unlike ledger inspection, this operation discloses a stored credential
// to a provider. Ordinary company membership does not authorize that use.
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
assertBoard(req);
const membership = req.actor.memberships?.find((item) => item.companyId === companyId);
if (req.actor.source !== "local_implicit" && !req.actor.isInstanceAdmin
&& !(membership?.status === "active" && ["owner", "admin"].includes(String(membership.membershipRole)))) {
throw forbidden("Company admin access required to import provider billing.");
}
next();
}, validate(importProviderCostsSchema), async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
assertBoard(req);
const result = await importProviderDailyCosts(db, companyId, req.body, getActorInfo(req).actorId);
res.json(result);
});
router.get("/companies/:companyId/accounting/health", async (req, res) => res.json(await integrity.health(req.params.companyId as string)));
router.get("/companies/:companyId/accounting/inspect", async (req, res) => res.json(await integrity.inspect(req.params.companyId as string)));
router.post("/companies/:companyId/accounting/repair", validate(repairAccountingSchema), async (req, res) =>
res.json(await integrity.repair(req.params.companyId as string, req.body.fingerprint, req.body.reason, getActorInfo(req).actorId)));
router.post("/companies/:companyId/accounting/retry", validate(retryAccountingSchema), async (req, res) =>
res.json(await integrity.retry(req.params.companyId as string, req.body.runId, getActorInfo(req).actorId)));
router.get("/companies/:companyId/accounting/invoices", async (req, res) => res.json(await billing.list(req.params.companyId as string)));
router.post("/companies/:companyId/accounting/invoices", validate(importBillingInvoiceSchema), async (req, res) =>
res.status(201).json(await billing.importInvoice(req.params.companyId as string, req.body, getActorInfo(req).actorId)));
router.get("/companies/:companyId/accounting/invoices/:invoiceId", async (req, res) =>
res.json(await billing.reconcile(req.params.companyId as string, req.params.invoiceId as string)));
router.get("/companies/:companyId/accounting/events/:eventId/adjustments", async (req, res) =>
res.json(await billing.adjustments(req.params.companyId as string, req.params.eventId as string)));
router.post("/companies/:companyId/accounting/events/:eventId/adjustments", validate(adjustCostSchema), async (req, res) =>
res.status(201).json(await billing.adjust(req.params.companyId as string, req.params.eventId as string, req.body, getActorInfo(req).actorId)));
return router;
}
server/src/routes/openapi.ts
+56 -3
View File
@@ -127,6 +127,11 @@ import {
updateBudgetSchema,
upsertBudgetPolicySchema,
resolveBudgetIncidentSchema,
repairAccountingSchema,
retryAccountingSchema,
importBillingInvoiceSchema,
importProviderCostsSchema,
adjustCostSchema,
// Sidebar
upsertSidebarOrderPreferenceSchema,
// Announcements
@@ -1327,6 +1332,7 @@ const PUBLIC_OPERATIONS = new Set([
]);
const BOARD_ONLY_PREFIXES = [
"/api/companies/{companyId}/accounting/",
"/api/announcements/",
"/api/auth/",
"/api/admin/",
@@ -5324,9 +5330,16 @@ registry.registerPath({
// ─── Costs ───────────────────────────────────────────────────────────────────
const costReportQuerySchema = z.object({
period: z.enum(["month", "all"]).optional().describe("Defaults to the current UTC month. All-time cannot be combined with date bounds."),
from: z.union([z.iso.datetime({ offset: true }), z.iso.date()]).optional().describe("Inclusive start bound."),
to: z.union([z.iso.datetime({ offset: true }), z.iso.date()]).optional().describe("Inclusive end bound; must not precede from."),
});
const costSummaryPaths = [
"summary",
"by-agent",
"by-user",
"by-agent-model",
"by-provider",
"by-biller",
@@ -5345,7 +5358,12 @@ for (const segment of costSummaryPaths) {
path: `/api/companies/{companyId}/costs/${segment}`,
tags: ["costs"],
summary: `Cost report: ${segment}`,
request: { params: z.object({ companyId: z.string() }) },
request: {
params: z.object({ companyId: z.string() }),
...(!["window-spend", "quota-windows"].includes(segment) ? {
query: segment === "finance-events" ? costReportQuerySchema.extend({ limit: z.coerce.number().int().min(1).max(500).optional() }) : costReportQuerySchema,
} : {}),
},
responses: { 200: r.ok(), 401: r.unauthorized },
});
}
@@ -5355,11 +5373,12 @@ registry.registerPath({
path: "/api/companies/{companyId}/cost-events",
tags: ["costs"],
summary: "Record a cost event",
description: "Amounts are USD cents, including fractional cents. Decimal strings preserve exact nanodollar precision. Reuse the idempotency key and original receipt on retries; conflicting reuse returns 409.",
request: {
params: z.object({ companyId: z.string() }),
body: jsonBody(createCostEventSchema),
},
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized },
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 409: r.conflict },
});
registry.registerPath({
@@ -5367,13 +5386,47 @@ registry.registerPath({
path: "/api/companies/{companyId}/finance-events",
tags: ["costs"],
summary: "Record a finance event",
description: "Amounts accept exact decimal cents in the recorded currency. Credits use a nonnegative amount with direction credit. No currency conversion is inferred. Receipt-key conflicts return 409.",
request: {
params: z.object({ companyId: z.string() }),
body: jsonBody(createFinanceEventSchema),
},
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized },
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 409: r.conflict },
});
for (const [suffix, summary] of [
["health", "Inspect pending receipts, unpriced charges, cancellation backlog, and held reservations"],
["inspect", "Independently compare ledger evidence and stored totals"],
["invoices", "List imported provider invoices"],
["invoices/{invoiceId}", "Review exact invoice matches and unresolved differences"],
["events/{eventId}/adjustments", "Read append-only correction history and prior pricing evidence"],
] as const) {
registry.registerPath({
method: "get", path: `/api/companies/{companyId}/accounting/${suffix}`, tags: ["costs"], summary,
description: "Company-scoped board access is required. Board viewers may inspect; agents cannot access operator accounting evidence.",
request: { params: z.object({ companyId: z.string(),
...(suffix.includes("invoiceId") ? { invoiceId: z.string() } : {}),
...(suffix.includes("eventId") ? { eventId: z.string() } : {}),
}) },
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
});
}
for (const operation of [
{ suffix: "repair", schema: repairAccountingSchema, status: 200, summary: "Repair reviewed ledger projections", description: "Requires an audit reason and the current inspection fingerprint. A stale inspection returns 409. Missing evidence is never invented." },
{ suffix: "retry", schema: retryAccountingSchema, status: 200, summary: "Retry accounting for a company-owned run", description: "Retries accounting only. Provider work is never re-executed; incomplete evidence remains pending." },
{ suffix: "provider-costs/import", schema: importProviderCostsSchema, status: 200, summary: "Import a scoped provider cost report", description: "Reads up to 31 completed UTC days from OpenAI or Anthropic with a company admin credential. Requires explicit provider account and project/workspace identifiers. All pages validate before writing. Reimports append only the difference; run costs are not repriced. Report totals remain separate from invoices." },
{ suffix: "invoices", schema: importBillingInvoiceSchema, status: 201, summary: "Import normalized invoice evidence", description: "Idempotent by company, biller, and external invoice ID. Conflicting reuse returns 409. Import atomically creates finance timeline events and does not reprice run costs." },
{ suffix: "events/{eventId}/adjustments", schema: adjustCostSchema, status: 201, summary: "Apply a reviewed cost correction", description: "Requires the expected exact cents, an idempotency key, reason, and pricing evidence. Original amounts and pricing revisions remain in history. Stale reviews return 409; ambiguous or contradictory invoice evidence returns 422." },
] as const) {
registry.registerPath({
method: "post", path: `/api/companies/{companyId}/accounting/${operation.suffix}`, tags: ["costs"],
summary: operation.summary, description: `${operation.description} Company-scoped board write access is required; viewers and agents cannot mutate accounting.`,
request: { params: z.object({ companyId: z.string(), ...(operation.suffix.includes("eventId") ? { eventId: z.string() } : {}) }), body: jsonBody(operation.schema) },
responses: { [operation.status]: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict, 422: r.unprocessable },
});
}
registry.registerPath({
method: "post",
path: "/api/companies/{companyId}/budgets/policies",
server/src/services/accounting-integrity.ts
+121
View File
@@ -0,0 +1,121 @@
import { and, eq, sql } from "drizzle-orm";
import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db";
import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared";
import { conflict, notFound } from "../errors.js";
import { receiptFingerprint } from "./receipt-fingerprint.js";
import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js";
import { logActivity } from "./activity-log.js";
import { accountRunCost } from "./run-cost-accounting.js";
import type { BudgetServiceHooks } from "./budgets.js";
/** Independent aggregation: deliberately does not reuse writer projections. */
async function inspect(tx: Db, companyId: string): Promise<AccountingInspection> {
const findings: AccountingFinding[] = [];
const monthly = await tx.execute<{ kind: "company_projection" | "agent_projection"; id: string; actual: string; expected: string }>(sql`
with month_events as (select agent_id, cost_cents from cost_events where company_id = ${companyId}
and occurred_at >= date_trunc('month', now() at time zone 'UTC') at time zone 'UTC'
and occurred_at < (date_trunc('month', now() at time zone 'UTC') + interval '1 month') at time zone 'UTC')
select 'company_projection' as kind, id, spent_monthly_cents::text as actual,
(select coalesce(sum(cost_cents),0)::text from month_events) as expected from companies where id = ${companyId}
union all select 'agent_projection', a.id, a.spent_monthly_cents::text,
coalesce((select sum(e.cost_cents) from month_events e where e.agent_id = a.id),0)::text from agents a where a.company_id = ${companyId}`);
for (const row of monthly) if (normalizeCents(row.actual) !== normalizeCents(row.expected)) findings.push({ kind: row.kind, entityId: row.id, repairable: true,
actual: { cents: normalizeCents(row.actual) }, expected: { cents: normalizeCents(row.expected) } });
const runtime = await tx.execute<{ id: string; baseline: string | null; actual: Record<string, string>; expected: Record<string, string> }>(sql`
with totals as (select r.agent_id, sum(e.cost_cents) as cents, sum(e.input_tokens) as input, sum(e.cached_input_tokens) as cached, sum(e.output_tokens) as output
from heartbeat_runs r join cost_events e on e.heartbeat_run_id = r.id and e.company_id = r.company_id
where r.company_id = ${companyId} and r.accounting_projection_version = 'v2' and r.cost_accounted_at is not null group by r.agent_id)
select a.agent_id as id, b.agent_id as baseline,
jsonb_build_object('cents', a.total_cost_cents::text, 'input', a.total_input_tokens::text, 'cached', a.total_cached_input_tokens::text, 'output', a.total_output_tokens::text) as actual,
jsonb_build_object('cents', (coalesce(b.cost_cents,0) + coalesce(t.cents,0))::text, 'input', (coalesce(b.input_tokens,0) + coalesce(t.input,0))::text,
'cached', (coalesce(b.cached_input_tokens,0) + coalesce(t.cached,0))::text, 'output', (coalesce(b.output_tokens,0) + coalesce(t.output,0))::text) as expected
from agent_runtime_state a left join accounting_runtime_baselines b on b.agent_id = a.agent_id and b.company_id = a.company_id
left join totals t on t.agent_id = a.agent_id where a.company_id = ${companyId} order by a.agent_id`);
for (const row of runtime) {
if (!row.baseline) { findings.push({ kind: "legacy_runtime", entityId: row.id, repairable: false, actual: row.actual, expected: {} }); continue; }
row.actual.cents = normalizeCents(row.actual.cents); row.expected.cents = normalizeCents(row.expected.cents);
if (Object.keys(row.expected).some(key => row.actual[key] !== row.expected[key])) findings.push({ kind: "runtime_projection", entityId: row.id, repairable: true, actual: row.actual, expected: row.expected });
}
const incomplete = await tx.execute<{ id: string; acknowledged: boolean; events: number }>(sql`
select r.id, r.cost_accounted_at is not null as acknowledged, count(e.id)::int as events from heartbeat_runs r
left join cost_events e on e.heartbeat_run_id = r.id and e.company_id = r.company_id and e.idempotency_key like ('heartbeat:' || r.id::text || ':%')
where r.company_id = ${companyId} and r.status in ('succeeded','failed','timed_out','cancelled','interrupted')
and (r.cost_accounting_pending or r.accounting_projection_version = 'v2')
and r.result_json->'executionRecovery'->>'providerWorkStarted' is distinct from 'false'
and r.usage_json->>'accountingProviderWorkStarted' is distinct from 'false'
group by r.id having r.cost_accounted_at is null or count(e.id) = 0 order by r.id`);
for (const row of incomplete) findings.push({ kind: row.acknowledged ? "missing_receipt" : "missing_acknowledgement", entityId: row.id, repairable: false,
actual: { acknowledged: String(row.acknowledged), events: String(row.events) }, expected: { acknowledged: "true", events: ">=1" } });
// Verify new run receipts independently of the projections. Corrections
// change effective valuation, while reported_cost_cents preserves the
// provider amount this acknowledgement originally consumed.
const receipts = await tx.execute<{ id: string; usage: Record<string, unknown>; actual: Record<string, string> }>(sql`
select r.id, r.usage_json as usage, jsonb_build_object('cents', sum(coalesce(e.reported_cost_cents,e.cost_cents))::text,
'input', sum(e.input_tokens)::text, 'cached', sum(e.cached_input_tokens)::text, 'output', sum(e.output_tokens)::text) as actual
from heartbeat_runs r join cost_events e on e.heartbeat_run_id = r.id and e.company_id = r.company_id and e.idempotency_key like ('heartbeat:' || r.id::text || ':%')
where r.company_id = ${companyId} and r.accounting_projection_version = 'v2' and r.cost_accounted_at is not null group by r.id order by r.id`);
for (const row of receipts) {
const usage = row.usage ?? {};
const cost = usage.cacheAdjustedCostUsd ?? usage.costUsdExact ?? usage.costUsd ?? 0;
let cents: string;
try { cents = usage.billingType === "subscription_included" ? "0.0000000" : usdToCents(String(cost)); }
catch { cents = "invalid receipt"; }
const expected = { cents, input: String(usage.inputTokens ?? 0), cached: String(usage.cachedInputTokens ?? 0), output: String(usage.outputTokens ?? 0) };
row.actual.cents = normalizeCents(row.actual.cents);
if (Object.entries(expected).some(([key,value]) => row.actual[key] !== value)) findings.push({ kind: "receipt_mismatch", entityId: row.id, repairable: false, actual: row.actual, expected });
}
findings.sort((a,b) => a.kind.localeCompare(b.kind) || a.entityId.localeCompare(b.entityId));
return { companyId, findings, fingerprint: receiptFingerprint({ companyId, findings }), checkedAt: new Date().toISOString() };
}
export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {}) {
return {
inspect: (companyId: string) => withAccountingReadSnapshot(db, companyId, tx => inspect(tx, companyId)),
repair: (companyId: string, fingerprint: string, reason: string, actorId: string) => withAccountingTransaction(db, companyId, async (tx, publications) => {
if (!reason.trim()) throw conflict("A repair reason is required");
const before = await inspect(tx, companyId);
if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing");
for (const finding of before.findings.filter(f => f.repairable)) {
if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId));
if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId)));
if (finding.kind === "runtime_projection") await tx.update(agentRuntimeState).set({ totalCostCents: sql`${finding.expected.cents}::numeric`,
totalInputTokens: sql`${finding.expected.input}::bigint`, totalCachedInputTokens: sql`${finding.expected.cached}::bigint`, totalOutputTokens: sql`${finding.expected.output}::bigint`, updatedAt: new Date(),
}).where(and(eq(agentRuntimeState.agentId, finding.entityId), eq(agentRuntimeState.companyId, companyId)));
}
await logActivity(tx, { companyId, actorType: "user", actorId, action: "accounting.projections_repaired", entityType: "company", entityId: companyId,
details: { fingerprint, reason, repairs: before.findings.filter(f => f.repairable) } }, publications);
return inspect(tx, companyId);
}),
retry: async (companyId: string, runId: string, actorId: string) => {
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
if (!run) throw notFound("Accounting run not found");
await logActivity(db, { companyId, actorType: "user", actorId, action: "accounting.retry_requested", entityType: "heartbeat_run", entityId: runId });
try { return { accounted: await accountRunCost(db, runId, hooks) }; }
catch (error) {
await db.update(heartbeatRuns).set({ accountingLastAttemptAt: new Date(), accountingAttemptCount: sql`${heartbeatRuns.accountingAttemptCount} + 1`,
accountingLastError: error instanceof Error ? error.message.slice(0,1000) : "Accounting retry failed",
}).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId)));
throw error;
}
},
health: (companyId: string): Promise<AccountingHealth> => withAccountingReadSnapshot(db, companyId, async tx => {
const [counts] = await tx.execute<{ pending: number; unpriced: number; oldest: string | null; cancellations: number; reserved: string }>(sql`
select (select count(*)::int from heartbeat_runs where company_id = ${companyId} and cost_accounting_pending and status in ('succeeded','failed','timed_out','cancelled','interrupted')) as pending,
(select count(*)::int from cost_events where company_id = ${companyId} and cost_status = 'unpriced' and billing_type <> 'subscription_included') as unpriced,
(select min(coalesce(finished_at,created_at))::text from heartbeat_runs where company_id = ${companyId} and cost_accounting_pending and status in ('succeeded','failed','timed_out','cancelled','interrupted')) as oldest,
(select count(*)::int from budget_policies where company_id = ${companyId} and enforcement_version > enforcement_delivered_version) as cancellations,
(select coalesce(sum(amount_cents),0)::text from budget_reservations where company_id = ${companyId} and state = 'held') as reserved`);
const pending = await tx.execute<{ runId: string; agentId: string; state: "waiting_for_receipt" | "retryable"; lastError: string | null; attempts: number; since: string; lastAttemptAt: string | null }>(sql`
select id as "runId", agent_id as "agentId", case when usage_json->>'accountingReceiptReady' = 'false'
and usage_json->>'accountingProviderWorkStarted' is distinct from 'false'
and result_json->'executionRecovery'->>'providerWorkStarted' is distinct from 'false' then 'waiting_for_receipt' else 'retryable' end as state,
accounting_last_error as "lastError", accounting_attempt_count as attempts, coalesce(finished_at,created_at)::text as since, accounting_last_attempt_at::text as "lastAttemptAt"
from heartbeat_runs where company_id = ${companyId} and cost_accounting_pending and status in ('succeeded','failed','timed_out','cancelled','interrupted') order by coalesce(finished_at,created_at),id limit 100`);
const unpriced = await tx.execute<{ runId: string | null; costEventId: string; agentId: string; since: string }>(sql`
select heartbeat_run_id as "runId", id as "costEventId", agent_id as "agentId", occurred_at::text as since from cost_events
where company_id = ${companyId} and cost_status = 'unpriced' and billing_type <> 'subscription_included' order by occurred_at,id limit 100`);
return { companyId, pendingRunCount: counts.pending, unpricedEventCount: counts.unpriced, oldestPendingAt: counts.oldest, pendingCancellationCount: counts.cancellations,
heldReservationCents: normalizeCents(counts.reserved), items: [...pending, ...unpriced.map(row => ({ ...row, state: "unpriced" as const, lastError: null, attempts: 0, lastAttemptAt: null }))] };
}),
};
}
server/src/services/accounting-transaction.ts
+40
View File
@@ -0,0 +1,40 @@
import { eq } from "drizzle-orm";
import { companies, type Db } from "@paperclipai/db";
import { publishActivity, type ActivityPublication } from "./activity-log.js";
import { notFound } from "../errors.js";
import { logger } from "../middleware/logger.js";
/** Serialize ledger and budget mutations in one company. Always acquire this
* before locking a run or a policy, so concurrent writers use one lock order.
* NO KEY UPDATE permits FK KEY SHARE locks held by recovery callers. */
export async function withAccountingTransaction<T>(db: Db, companyId: string, work: (tx: Db, publications: ActivityPublication[]) => Promise<T>): Promise<T> {
const publications: ActivityPublication[] = [];
const result = await db.transaction(async (transaction) => {
const tx = transaction as unknown as Db;
const [company] = await tx.select({ id: companies.id }).from(companies)
.where(eq(companies.id, companyId)).for("no key update");
if (!company) throw notFound("Company not found");
return work(tx, publications);
});
publishAccountingActivities(companyId, publications);
return result;
}
/** Consistent operator reports must not wait for or delay accounting writers.
* Reads see one committed snapshot; repairs still revalidate under the lock. */
export function withAccountingReadSnapshot<T>(db: Db, companyId: string, work: (tx: Db) => Promise<T>): Promise<T> {
return db.transaction(async transaction => {
const tx = transaction as unknown as Db;
const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId));
if (!company) throw notFound("Company not found");
return work(tx);
}, { isolationLevel: "repeatable read", accessMode: "read only" });
}
/** Database history is durable even when the live subscriber channel fails. */
export function publishAccountingActivities(companyId: string, publications: ActivityPublication[]) {
for (const publication of publications) {
try { publishActivity(publication); }
catch (error) { logger.warn({ err: error, companyId }, "Accounting committed; live activity publication failed"); }
}
}
server/src/services/agents.ts
+17 -3
View File
@@ -1,3 +1,6 @@
import { budgetServiceInTransaction, deliverBudgetEnforcement, type BudgetServiceHooks } from "./budgets.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import type { ActivityPublication } from "./activity-log.js";
import { agentAppearanceSchema, randomAgentAppearance, resolveAgentAppearance, agentAvatarUrl } from "@paperclipai/shared";
import { createHash, randomBytes } from "node:crypto";
import { and, desc, eq, gte, inArray, lt, ne, or, sql } from "drizzle-orm";
@@ -342,7 +345,7 @@ export function deduplicateAgentName(
return `${candidateName} ${Date.now()}`;
}
export function agentService(db: Db) {
export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
const secretsSvc = secretService(db);
function currentUtcMonthWindow(now = new Date()) {
@@ -791,7 +794,7 @@ export function agentService(db: Db) {
const beforeConfig = shouldRecordRevision ? buildConfigSnapshot(existing) : null;
type AgentUpdateResult = Awaited<ReturnType<typeof getById>>;
const applyUpdate = async (txDb: Db): Promise<AgentUpdateResult> => {
const applyUpdate = async (txDb: Db, publications: ActivityPublication[] = []): Promise<AgentUpdateResult> => {
const current = data.status !== undefined
? await txDb.select().from(agents).where(eq(agents.id, id)).for("update").then(rows => rows[0] ?? null)
: existing;
@@ -843,6 +846,11 @@ export function agentService(db: Db) {
);
}
if (normalizedPatch.budgetMonthlyCents !== undefined) {
await budgetServiceInTransaction(txDb, publications).upsertPolicy(existing.companyId, {
scopeType: "agent", scopeId: id, amount: normalizedPatch.budgetMonthlyCents, isActive: normalizedPatch.budgetMonthlyCents > 0, windowKind: "calendar_month_utc",
}, options?.recordRevision?.createdByUserId ?? null);
}
const normalizedUpdated = await agentService(txDb).getById(updated.id);
if (!normalizedUpdated) {
throw notFound("Agent not found");
@@ -869,6 +877,12 @@ export function agentService(db: Db) {
return normalizedUpdated;
};
if (normalizedPatch.budgetMonthlyCents !== undefined) {
const result = await withAccountingTransaction(db, existing.companyId, applyUpdate);
await deliverBudgetEnforcement(db, budgetHooks, existing.companyId);
return result;
}
const transaction = (db as unknown as {
transaction?: (callback: (tx: unknown) => Promise<AgentUpdateResult>) => Promise<AgentUpdateResult>;
}).transaction;
@@ -1075,7 +1089,7 @@ export function agentService(db: Db) {
});
}
return db.transaction(async (tx) => {
return withAccountingTransaction(db, existing.companyId, async (tx) => {
await tx
.select({ id: agents.id })
.from(agents)
server/src/services/approvals.ts
+8 -4
View File
@@ -5,7 +5,8 @@ import { approvalComments, approvals } from "@paperclipai/db";
import { notFound, unprocessable } from "../errors.js";
import { redactCurrentUserText } from "../log-redaction.js";
import { agentService } from "./agents.js";
import { budgetService } from "./budgets.js";
import { budgetService, budgetServiceInTransaction } from "./budgets.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { notifyHireApproved } from "./hire-hook.js";
import { instanceSettingsService } from "./instance-settings.js";
@@ -142,10 +143,12 @@ export function approvalService(db: Db) {
approve: async (id: string, decidedByUserId: string, decisionNote?: string | null) => {
const now = new Date();
const result = await db.transaction(async tx => {
const txDb = tx as unknown as Db;
const existing = await getExistingApproval(id);
// Receipt writers lock company before agent. Hire approval must use that
// order too, including activation of an existing pending agent.
const result = await withAccountingTransaction(db, existing.companyId, async (txDb, publications) => {
const agentsSvc = agentService(txDb);
const budgets = budgetService(txDb);
const budgets = budgetServiceInTransaction(txDb, publications);
const { approval: updated, applied } = await resolveApproval(
id,
"approved",
@@ -210,6 +213,7 @@ export function approvalService(db: Db) {
return { approval: updated, applied, hireApprovedAgentId };
});
if (result.hireApprovedAgentId) {
await budgetService(db).deliverPendingEnforcement(result.approval.companyId);
void notifyHireApproved(db, {
companyId: result.approval.companyId,
agentId: result.hireApprovedAgentId,
server/src/services/billing-reconciliation.ts
+167
View File
@@ -0,0 +1,167 @@
import { createFinanceEventInTransaction } from "./finance.js";
import { and, desc, eq, sql } from "drizzle-orm";
import { agentRuntimeState, costEvents, costAdjustments, billingInvoices, billingInvoiceLines, financeEvents, heartbeatRuns, type Db } from "@paperclipai/db";
import { adjustCostSchema, importBillingInvoiceSchema, normalizeCents, subtractCents, type AdjustCost, type ImportBillingInvoice } from "@paperclipai/shared";
import { conflict, notFound, unprocessable } from "../errors.js";
import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js";
import { receiptFingerprint } from "./receipt-fingerprint.js";
import { logActivity } from "./activity-log.js";
import { updateMonthlySpendProjections } from "./costs.js";
import { budgetServiceInTransaction, deliverBudgetEnforcement, type BudgetServiceHooks } from "./budgets.js";
async function invoiceCostCandidates(tx: Db, companyId: string, biller: string, line: {
costEventId?: string | null; runId?: string | null; providerRequestId?: string | null; model?: string | null;
}) {
if (!(line.costEventId || line.runId || line.providerRequestId)) return [];
return tx.select({ id: costEvents.id, cents: sql<string>`${costEvents.costCents}::text`, status: costEvents.costStatus })
.from(costEvents).where(and(eq(costEvents.companyId, companyId), eq(costEvents.biller, biller),
line.costEventId ? eq(costEvents.id, line.costEventId) : undefined,
line.runId ? eq(costEvents.heartbeatRunId, line.runId) : undefined,
line.providerRequestId ? eq(costEvents.providerRequestId, line.providerRequestId) : undefined,
line.model ? eq(costEvents.model, line.model) : undefined)).limit(2);
}
export function billingReconciliationService(db: Db, hooks: BudgetServiceHooks = {}) {
return {
importInvoice: (companyId: string, raw: ImportBillingInvoice, actorId: string) => withAccountingTransaction(db, companyId, async (tx, publications) => {
const input = importBillingInvoiceSchema.parse(raw);
// Line order in an export is not part of invoice identity.
input.lines.sort((a,b) => a.externalId.localeCompare(b.externalId));
const receiptHash = receiptFingerprint(input);
const [existing] = await tx.select().from(billingInvoices).where(and(eq(billingInvoices.companyId, companyId), eq(billingInvoices.biller, input.biller), eq(billingInvoices.externalId, input.externalId)));
if (existing) {
if (existing.receiptHash !== receiptHash) throw conflict("Invoice identifier already has different contents");
return existing;
}
// Validate explicit references, even on non-USD invoices that we cannot
// compare with the USD cost ledger. Never infer cross-company matches.
for (const line of input.lines) {
if (line.costEventId) {
const [event] = await tx.select({ id: costEvents.id }).from(costEvents).where(and(eq(costEvents.companyId, companyId), eq(costEvents.id, line.costEventId)));
if (!event) throw notFound("Invoice cost event not found");
}
if (line.runId) {
const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, line.runId)));
if (!run) throw notFound("Invoice run not found");
}
}
const [invoice] = await tx.insert(billingInvoices).values({ companyId, biller: input.biller, externalId: input.externalId, currency: input.currency, receiptHash, importedBy: actorId }).returning();
await tx.insert(billingInvoiceLines).values(input.lines.map(line => ({ ...line, companyId, invoiceId: invoice.id, occurredAt: new Date(line.occurredAt), pricing: line.pricing ?? {} })));
const matches = [];
const matchCounts = new Map<string, number>();
for (const line of input.lines) {
const candidates = line.kind === "inference" ? await invoiceCostCandidates(tx, companyId, input.biller, line) : [];
matches.push({ line, candidates });
if (candidates.length === 1) matchCounts.set(candidates[0].id, (matchCounts.get(candidates[0].id) ?? 0) + 1);
}
for (const { line, candidates } of matches) {
// Ambiguous evidence cannot authorize another debit. Keep every line
// for reconciliation, including differences from an existing charge.
if (candidates.length > 1 || candidates.length === 1 && matchCounts.get(candidates[0].id)! > 1) continue;
const costEventId = line.costEventId ?? candidates[0]?.id;
if (line.kind === "inference" && costEventId) {
const [recorded] = await tx.select({ id: financeEvents.id }).from(financeEvents).where(and(
eq(financeEvents.companyId, companyId), eq(financeEvents.costEventId, costEventId),
eq(financeEvents.biller, input.biller), eq(financeEvents.currency, input.currency),
eq(financeEvents.eventKind, "inference_charge"), eq(financeEvents.direction, "debit"),
sql`${financeEvents.metadataJson}->>'source' is distinct from 'provider_cost_report'`,
)).limit(1);
if (recorded) continue;
}
await createFinanceEventInTransaction(tx, publications, companyId, {
idempotencyKey: `invoice:${invoice.id}:${receiptFingerprint(line.externalId)}`,
biller: input.biller, externalInvoiceId: input.externalId,
eventKind: line.kind === "inference" ? "inference_charge" : line.kind === "credit" ? "credit_refund" : "platform_fee",
direction: line.kind === "credit" ? "credit" : "debit",
amountCents: line.amountCents, currency: input.currency, occurredAt: new Date(line.occurredAt),
description: `Invoice ${input.externalId} · ${line.externalId}`.slice(0, 500),
costEventId, heartbeatRunId: line.runId,
metadataJson: { source: "provider_invoice", invoiceId: invoice.id, externalLineId: line.externalId },
}, { actorType: "user", actorId });
}
await logActivity(tx, { companyId, actorType: "user", actorId, action: "accounting.invoice_imported", entityType: "billing_invoice", entityId: invoice.id,
details: { biller: invoice.biller, externalId: invoice.externalId, currency: invoice.currency, lineCount: input.lines.length, receiptHash } }, publications);
return invoice;
}),
list: (companyId: string) => db.select().from(billingInvoices).where(eq(billingInvoices.companyId, companyId)).orderBy(desc(billingInvoices.createdAt)).limit(100),
reconcile: (companyId: string, invoiceId: string) => withAccountingReadSnapshot(db, companyId, async tx => {
const [invoice] = await tx.select().from(billingInvoices).where(and(eq(billingInvoices.companyId, companyId), eq(billingInvoices.id, invoiceId)));
if (!invoice) throw notFound("Invoice not found");
const lines = await tx.select().from(billingInvoiceLines).where(and(eq(billingInvoiceLines.companyId, companyId), eq(billingInvoiceLines.invoiceId, invoiceId))).orderBy(billingInvoiceLines.externalId);
const rows = [];
for (const line of lines) {
const candidates = invoice.currency !== "USD" || line.kind !== "inference" ? []
: await invoiceCostCandidates(tx, companyId, invoice.biller, line);
const match = candidates.length === 1 ? candidates[0] : null;
const differenceCents = match ? subtractCents(line.amountCents, match.cents) : null;
const status = invoice.currency !== "USD" ? "unsupported_currency" : line.kind !== "inference" ? "non_inference"
: candidates.length > 1 ? "ambiguous" : !match ? "unmatched" : differenceCents === "0.0000000" && match.status !== "unpriced" ? "matched" : "difference";
rows.push({ ...line, amountCents: normalizeCents(line.amountCents), status, matchedEventId: match?.id ?? null,
recordedCents: match ? normalizeCents(match.cents) : null, differenceCents });
}
// Multiple invoice lines cannot independently revalue the same charge.
const counts = new Map<string, number>();
for (const row of rows) if (row.matchedEventId) counts.set(row.matchedEventId, (counts.get(row.matchedEventId) ?? 0) + 1);
for (const row of rows) if (row.matchedEventId && counts.get(row.matchedEventId)! > 1) row.status = "ambiguous";
return { invoice, lines: rows };
}),
adjustments: (companyId: string, eventId: string) => db.select().from(costAdjustments).where(and(eq(costAdjustments.companyId, companyId), eq(costAdjustments.costEventId, eventId))).orderBy(desc(costAdjustments.createdAt)),
adjust: async (companyId: string, eventId: string, raw: AdjustCost, actorId: string) => {
const input = adjustCostSchema.parse(raw);
const result = await withAccountingTransaction(db, companyId, async (tx, publications) => {
const hash = receiptFingerprint({ eventId, ...input });
const [existing] = await tx.select().from(costAdjustments).where(and(eq(costAdjustments.companyId, companyId), eq(costAdjustments.idempotencyKey, input.idempotencyKey)));
if (existing) {
if (existing.receiptHash !== hash) throw conflict("Correction key already has different contents");
return existing;
}
const [event] = await tx.select({ event: costEvents, exact: sql<string>`${costEvents.costCents}::text` }).from(costEvents)
.where(and(eq(costEvents.companyId, companyId), eq(costEvents.id, eventId))).for("update");
if (!event) throw notFound("Cost event not found");
if (normalizeCents(event.exact) !== input.expectedCents) throw conflict("Charge changed since review; review it again");
if (input.invoiceLineId) {
const reportLine = await tx.select({ line: billingInvoiceLines, invoice: billingInvoices }).from(billingInvoiceLines)
.innerJoin(billingInvoices, eq(billingInvoices.id, billingInvoiceLines.invoiceId)).where(and(eq(billingInvoiceLines.id, input.invoiceLineId), eq(billingInvoiceLines.companyId, companyId), eq(billingInvoices.companyId, companyId)));
const row = reportLine[0];
if (!row) throw notFound("Invoice line not found");
if (row.invoice.currency !== "USD" || row.line.kind !== "inference" || row.invoice.biller !== event.event.biller
|| normalizeCents(row.line.amountCents) !== input.correctedCents
|| row.line.costEventId && row.line.costEventId !== eventId
|| row.line.runId && row.line.runId !== event.event.heartbeatRunId
|| row.line.providerRequestId && row.line.providerRequestId !== event.event.providerRequestId
|| row.line.model && row.line.model !== event.event.model) throw unprocessable("Invoice line does not support this correction");
const review = await billingReconciliationService(tx).reconcile(companyId, row.invoice.id);
const reviewedLine = review.lines.find(line => line.id === input.invoiceLineId);
if (!reviewedLine || !["matched", "difference"].includes(reviewedLine.status) || reviewedLine.matchedEventId !== eventId)
throw unprocessable("Invoice match is ambiguous or unverified; review an explicit correction without attaching this line");
const [used] = await tx.select().from(costAdjustments).where(and(eq(costAdjustments.companyId, companyId), eq(costAdjustments.invoiceLineId, input.invoiceLineId)));
if (used) throw conflict("Invoice line has already been applied");
}
const [adjustment] = await tx.insert(costAdjustments).values({ companyId, costEventId: eventId, invoiceLineId: input.invoiceLineId,
idempotencyKey: input.idempotencyKey, receiptHash: hash, previousCents: input.expectedCents, correctedCents: input.correctedCents,
previousStatus: event.event.costStatus, previousPricing: event.event.pricingProvenance ?? {},
reason: input.reason, actorId, pricing: input.pricing }).returning();
// Original provider value and fingerprint remain immutable. cost_cents
// is the effective valuation, backed by this append-only correction.
const [updated] = await tx.update(costEvents).set({ costCents: sql`${input.correctedCents}::numeric`,
reportedCostCents: event.event.reportedCostCents ?? normalizeCents(event.exact), costStatus: input.pricing.source === "rate_card" ? "estimated" : "reported", pricingProvenance: input.pricing,
}).where(eq(costEvents.id, eventId)).returning();
await updateMonthlySpendProjections(tx, companyId, updated.agentId, subtractCents(input.correctedCents, input.expectedCents), updated.occurredAt);
if (updated.heartbeatRunId) {
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, updated.heartbeatRunId), eq(heartbeatRuns.companyId, companyId)));
if (run?.costAccountedAt && run.accountingProjectionVersion === "v2") await tx.update(agentRuntimeState)
.set({ totalCostCents: sql`${agentRuntimeState.totalCostCents} + ${subtractCents(input.correctedCents, input.expectedCents)}::numeric`, updatedAt: new Date() })
.where(and(eq(agentRuntimeState.agentId, updated.agentId), eq(agentRuntimeState.companyId, companyId)));
}
const budgets = budgetServiceInTransaction(tx, publications);
await budgets.evaluateCostEvent(updated);
await budgets.getInvocationBlock(companyId, updated.agentId, { projectId: updated.projectId });
await logActivity(tx, { companyId, actorType: "user", actorId, action: "accounting.charge_corrected", entityType: "cost_event", entityId: eventId,
details: { adjustmentId: adjustment.id, previousCents: input.expectedCents, correctedCents: input.correctedCents, reason: input.reason, pricing: input.pricing } }, publications);
return adjustment;
});
await deliverBudgetEnforcement(db, hooks, companyId);
return result;
},
};
}
server/src/services/browser-use.ts
+23 -44
View File
@@ -16,9 +16,7 @@ import {
issues,
heartbeatRuns,
agents,
companies,
costEvents,
financeEvents,
} from "@paperclipai/db";
import {
BROWSER_USE_IDLE_MS,
@@ -41,7 +39,9 @@ import { secretService } from "./secrets.js";
import { budgetService, type BudgetServiceHooks } from "./budgets.js";
import { issueTreeControlService } from "./issue-tree-control.js";
import { accessService } from "./access.js";
import { getMonthlySpendTotal } from "./costs.js";
import { createCostEventInTransaction } from "./costs.js";
import { createFinanceEventInTransaction } from "./finance.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { logActivity } from "./activity-log.js";
import { browserUseViewports } from "./browser-use-viewport.js";
import { forbidden, notFound, conflict } from "../errors.js";
@@ -911,36 +911,32 @@ export function browserUseService(
const cents = Math.max(0, Math.round(Number(summary.totalCostUsd) * 100));
if (!Number.isSafeInteger(cents))
throw new BrowserUseError(502, "Browser Use returned invalid usage.");
const [issue] = await db
.select({ projectId: issues.projectId })
.from(issues)
.where(and(eq(issues.id, s.issueId), eq(issues.companyId, s.companyId)));
const event = await db.transaction(async (tx) => {
const event = await withAccountingTransaction(db, s.companyId, async (tx, publications) => {
const [issue] = await tx
.select({ projectId: issues.projectId })
.from(issues)
.where(and(eq(issues.id, s.issueId), eq(issues.companyId, s.companyId)));
const [current] = await tx
.select()
.from(runs)
.where(eq(runs.id, run.id))
.for("update");
if (cents <= current.accountedCents) return null;
const [e] = await tx
.insert(costEvents)
.values({
companyId: s.companyId,
agentId: s.agentId,
issueId: s.issueId,
projectId: issue?.projectId,
heartbeatRunId: run.heartbeatRunId,
provider: "browser-use-cloud",
biller: "browser-use-cloud",
billingType: "metered_api",
model: summary.model,
costCents: cents - current.accountedCents,
occurredAt: new Date(),
billingCode: `browser-use-cloud:${run.id}`,
})
.returning();
await tx.insert(financeEvents).values({
companyId: e.companyId,
const e = await createCostEventInTransaction(tx, s.companyId, {
agentId: s.agentId,
issueId: s.issueId,
projectId: issue?.projectId,
heartbeatRunId: run.heartbeatRunId,
provider: "browser-use-cloud",
biller: "browser-use-cloud",
billingType: "metered_api",
model: summary.model,
costCents: cents - current.accountedCents,
occurredAt: new Date(),
billingCode: `browser-use-cloud:${run.id}`,
idempotencyKey: `browser-use-cloud:${run.id}:${cents}`,
}, publications);
await createFinanceEventInTransaction(tx, publications, s.companyId, {
agentId: e.agentId,
issueId: e.issueId,
projectId: e.projectId,
@@ -963,23 +959,6 @@ export function browserUseService(
.where(eq(runs.id, run.id));
return e;
});
await db
.update(agents)
.set({
spentMonthlyCents: await getMonthlySpendTotal(db, {
companyId: s.companyId,
agentId: s.agentId,
}),
})
.where(and(eq(agents.companyId, s.companyId), eq(agents.id, s.agentId)));
await db
.update(companies)
.set({
spentMonthlyCents: await getMonthlySpendTotal(db, {
companyId: s.companyId,
}),
})
.where(eq(companies.id, s.companyId));
// Re-evaluate on retries too: accounting may have committed immediately
// before a crash interrupted budget enforcement.
const lastEvent =
server/src/services/budget-reservations.ts
+68
View File
@@ -0,0 +1,68 @@
import { and, eq, sql } from "drizzle-orm";
import { agents, budgetPolicies, budgetReservations, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { centsToUnits, unitsToCents } from "@paperclipai/shared";
import { conflict, notFound } from "../errors.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { budgetServiceInTransaction, budgetPoliciesForRun, computeObservedSpend } from "./budgets.js";
/** Reserve before dispatch, under the same company lock as charges and policy
* changes. Estimates constrain admission; they cannot cap a provider's bill.
* A reservation survives timeouts and restarts until accounting proves closure. */
export async function reserveRunBudget(db: Db, companyId: string, runId: string, projectId: string | null, ledgerScope: Record<string, unknown> = {}, recoveryLeaseOwner?: string) {
return withAccountingTransaction(db, companyId, async (tx, publications) => {
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))).for("update");
if (!run) throw notFound("Run not found");
if (run.resultJson?.cancellation || run.resultJson?.startupCancellation) throw conflict("Run cancellation already requested");
if (run.costAccountedAt || !["queued", "running"].includes(run.status)) throw conflict("Run can no longer start provider work");
const [existing] = await tx.select().from(budgetReservations).where(and(eq(budgetReservations.companyId, companyId), eq(budgetReservations.runId, runId)));
if (existing) {
// The native coordinator fences recovery ownership. Reuse the original
// hold only for that live owner, never for an ordinary duplicate dispatch.
const [owner] = recoveryLeaseOwner && run.runtimeMode === "native" && existing.state === "held"
&& existing.projectId === projectId ? await tx.select({ runId: nativeRunFinalizations.runId })
.from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.companyId, companyId),
eq(nativeRunFinalizations.runId, runId), eq(nativeRunFinalizations.leaseOwner, recoveryLeaseOwner),
sql`${nativeRunFinalizations.leaseExpiresAt} > now()`)).for("update") : [];
if (owner) {
// Recovery has already made this run queued/running, so its own pending
// accounting is excluded. Current overruns, other unfinished runs and
// scope pauses still apply before any more paid work starts.
const block = await budgetServiceInTransaction(tx, publications).getInvocationBlock(companyId, run.agentId, { projectId });
if (block) throw conflict(block.reason);
const [agent] = await tx.select({ status: agents.status }).from(agents)
.where(and(eq(agents.companyId, companyId), eq(agents.id, run.agentId)));
if (agent?.status === "paused") throw conflict("Agent is paused and cannot start new work");
return { ...existing, reused: true };
}
throw conflict("Provider dispatch has already reserved this run");
}
const block = await budgetServiceInTransaction(tx, publications).getInvocationBlock(companyId, run.agentId, { projectId });
if (block) throw conflict(block.reason);
const policies = await tx.select().from(budgetPolicies).where(and(
eq(budgetPolicies.companyId, companyId), eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true),
eq(budgetPolicies.metric, "billed_cents"), sql`${budgetPolicies.amount} > 0`,
budgetPoliciesForRun(companyId, run.agentId, projectId),
));
const amount = policies.reduce((max, policy) => { const next = centsToUnits(policy.reservationCents); return next > max ? next : max; }, 0n);
for (const policy of policies) {
const [held] = await tx.select({ amount: sql<string>`coalesce(sum(${budgetReservations.amountCents}), 0)::text` }).from(budgetReservations).where(and(
eq(budgetReservations.companyId, companyId), eq(budgetReservations.state, "held"),
policy.scopeType === "agent" ? eq(budgetReservations.agentId, run.agentId) : undefined,
policy.scopeType === "project" ? eq(budgetReservations.projectId, projectId!) : undefined,
));
const observed = await computeObservedSpend(tx, policy);
// Even zero-estimate runs cannot enter a scope whose available capacity
// is completely reserved. Outstanding reservations carry across months.
const committed = centsToUnits(observed.totalExact) + centsToUnits(held.amount);
const limit = centsToUnits(policy.amount);
if (committed >= limit || committed + amount > limit) throw conflict("Available budget is reserved by unfinished runs");
}
// A zero-valued row also fences duplicate dispatch when estimates are off.
const [reservation] = await tx.insert(budgetReservations).values({ companyId, runId, agentId: run.agentId, projectId,
amountCents: unitsToCents(amount), providerStartedAt: new Date() }).returning();
await tx.update(heartbeatRuns).set({ costAccountingPending: true,
usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ accountingReceiptReady: false, ledgerScope: { ...(run.usageJson?.ledgerScope as object ?? {}), ...ledgerScope, projectId } })}::jsonb`,
}).where(eq(heartbeatRuns.id, runId));
return { ...reservation, reused: false };
});
}
server/src/services/budgets.ts
+336 -246
View File
@@ -1,4 +1,5 @@
import { and, desc, eq, gte, inArray, lt, ne, sql } from "drizzle-orm";
import { compareCents, normalizeCents } from "@paperclipai/shared";
import { and, desc, eq, gte, inArray, lt, ne, sql, or, isNull, type SQL } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { recordAgentStatusEvent } from "./resource-lifecycle-events.js";
import {
@@ -8,6 +9,8 @@ import {
budgetPolicies,
companies,
costEvents,
heartbeatRuns,
nativeRunFinalizations,
projects,
} from "@paperclipai/db";
import type {
@@ -23,8 +26,10 @@ import type {
BudgetThresholdType,
BudgetWindowKind,
} from "@paperclipai/shared";
import { notFound, unprocessable } from "../errors.js";
import { logActivity } from "./activity-log.js";
import { HttpError, notFound, unprocessable } from "../errors.js";
import { logger } from "../middleware/logger.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js";
type ScopeRecord = {
companyId: string;
@@ -40,6 +45,10 @@ export type BudgetEnforcementScope = {
companyId: string;
scopeType: BudgetScopeType;
scopeId: string;
/** Snapshot taken under the company lock, before a later budget grant can admit new work. */
createdBefore?: Date;
/** Revalidated at the durable cancellation write, under the admission lock. */
enforcement?: { policyId: string; version: number };
};
export type BudgetServiceHooks = {
@@ -65,13 +74,13 @@ function resolveWindow(windowKind: BudgetWindowKind, now = new Date()) {
}
function budgetStatusFromObserved(
observedAmount: number,
observedAmount: string,
amount: number,
warnPercent: number,
): BudgetPolicySummary["status"] {
if (amount <= 0) return "ok";
if (observedAmount >= amount) return "hard_stop";
if (observedAmount >= Math.ceil((amount * warnPercent) / 100)) return "warning";
if (compareCents(observedAmount, amount) >= 0) return "hard_stop";
if (compareCents(observedAmount, Math.ceil((amount * warnPercent) / 100)) >= 0) return "warning";
return "ok";
}
@@ -141,29 +150,84 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
};
}
async function computeObservedAmount(
db: Db,
policy: Pick<PolicyRow, "companyId" | "scopeType" | "scopeId" | "windowKind" | "metric">,
) {
if (policy.metric !== "billed_cents") return 0;
const conditions = [eq(costEvents.companyId, policy.companyId)];
if (policy.scopeType === "agent") conditions.push(eq(costEvents.agentId, policy.scopeId));
if (policy.scopeType === "project") conditions.push(eq(costEvents.projectId, policy.scopeId));
const { start, end } = resolveWindow(policy.windowKind as BudgetWindowKind);
if (policy.windowKind === "calendar_month_utc") {
conditions.push(gte(costEvents.occurredAt, start));
conditions.push(lt(costEvents.occurredAt, end));
// Keep a large dashboard from filling the database pool with queued reads.
async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
const result: R[] = [];
for (let start = 0; start < rows.length; start += 4) {
result.push(...await Promise.all(rows.slice(start, start + 4).map(read)));
}
return result;
}
const [row] = await db
.select({
total: sql<number>`coalesce(sum(${costEvents.costCents}), 0)::double precision`,
})
.from(costEvents)
.where(and(...conditions));
/** Filter in PostgreSQL so one run never loads unrelated agents' policies. */
export function budgetPoliciesForRun(companyId: string, agentId: string, projectId: string | null) {
return or(
and(eq(budgetPolicies.scopeType, "company"), eq(budgetPolicies.scopeId, companyId)),
and(eq(budgetPolicies.scopeType, "agent"), eq(budgetPolicies.scopeId, agentId)),
projectId ? and(eq(budgetPolicies.scopeType, "project"), eq(budgetPolicies.scopeId, projectId)) : undefined,
);
}
return Number(row?.total ?? 0);
type SpendPolicy = Pick<PolicyRow, "companyId" | "scopeType" | "scopeId" | "windowKind" | "metric">;
type ObservedSpend = { windowStart: Date; windowEnd: Date; total: number; totalExact: string; unpricedEventCount: number; pendingRunCount: number; recoveringRunCount: number };
/** One ledger scan and one pending-run scan for the affected policies. The
* company lock still protects decisions; nothing is cached across mutations.
* Each aggregate keeps its own scope/window, including mixed lifetime and
* monthly policies. An outer predicate excludes rows no policy needs. */
async function computeObservedSpends(db: Db, policies: SpendPolicy[]): Promise<ObservedSpend[]> {
const now = new Date();
const windows = policies.map(policy => resolveWindow(policy.windowKind as BudgetWindowKind, now));
const billable = policies.map((policy, index) => ({ policy, index })).filter(({ policy }) => policy.metric === "billed_cents");
const ledgerColumns: Record<string, SQL> = {};
const pendingColumns: Record<string, SQL> = {};
const ledgerPredicates: SQL[] = [], pendingPredicates: SQL[] = [];
for (const { policy, index } of billable) {
const { start, end } = windows[index];
const ledger = and(
eq(costEvents.companyId, policy.companyId),
policy.scopeType === "agent" ? eq(costEvents.agentId, policy.scopeId) : undefined,
policy.scopeType === "project" ? eq(costEvents.projectId, policy.scopeId) : undefined,
policy.windowKind === "calendar_month_utc" ? and(gte(costEvents.occurredAt, start), lt(costEvents.occurredAt, end)) : undefined,
)!;
const pending = and(
eq(heartbeatRuns.companyId, policy.companyId), eq(heartbeatRuns.costAccountingPending, true),
inArray(heartbeatRuns.status, ["succeeded", "failed", "timed_out", "cancelled", "interrupted"]),
policy.scopeType === "agent" ? eq(heartbeatRuns.agentId, policy.scopeId) : undefined,
policy.scopeType === "project" ? sql`${heartbeatRuns.usageJson}->'ledgerScope'->>'projectId' = ${policy.scopeId}` : undefined,
policy.windowKind === "calendar_month_utc" ? sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) >= ${start.toISOString()}::timestamptz
and coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) < ${end.toISOString()}::timestamptz` : undefined,
)!;
ledgerPredicates.push(ledger); pendingPredicates.push(pending);
ledgerColumns[`total${index}`] = sql`coalesce(sum(${costEvents.costCents}) filter (where ${ledger}), 0)::text`;
ledgerColumns[`unpriced${index}`] = sql`count(*) filter (where ${ledger} and ${costEvents.costStatus} = 'unpriced' and ${costEvents.billingType} <> 'subscription_included')::int`;
pendingColumns[`pending${index}`] = sql`count(*) filter (where ${pending})::int`;
pendingColumns[`recovering${index}`] = sql`count(*) filter (where ${pending} and ${heartbeatRuns.runtimeMode} = 'native'
and exists (select 1 from ${nativeRunFinalizations} where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
and ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId}
and ${nativeRunFinalizations.resultId} is null and ${nativeRunFinalizations.phase} <> 'terminal_failure'))::int`;
}
const [ledger = {}] = billable.length ? await db.select(ledgerColumns).from(costEvents).where(or(...ledgerPredicates)) : [];
const [pending = {}] = billable.length ? await db.select(pendingColumns).from(heartbeatRuns).where(or(...pendingPredicates)) : [];
return policies.map((_, index) => ({
windowStart: windows[index].start, windowEnd: windows[index].end,
total: Number(ledger[`total${index}`] ?? 0), totalExact: normalizeCents(String(ledger[`total${index}`] ?? 0)),
unpricedEventCount: Number(ledger[`unpriced${index}`] ?? 0), pendingRunCount: Number(pending[`pending${index}`] ?? 0),
recoveringRunCount: Number(pending[`recovering${index}`] ?? 0),
}));
}
export async function computeObservedSpend(db: Db, policy: SpendPolicy) {
return (await computeObservedSpends(db, [policy]))[0];
}
function observedBlocks(policy: PolicyRow, observed: Awaited<ReturnType<typeof computeObservedSpend>>) {
if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false;
return compareCents(observed.totalExact, policy.amount) >= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || (policy.unpricedUsagePolicy !== "allow" && observed.unpricedEventCount > 0);
}
async function policyBlocks(db: Db, policy: PolicyRow) {
if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false;
return observedBlocks(policy, await computeObservedSpend(db, policy));
}
function buildApprovalPayload(input: {
@@ -171,6 +235,7 @@ function buildApprovalPayload(input: {
scopeName: string;
thresholdType: BudgetThresholdType;
amountObserved: number;
amountObservedExact: string;
windowStart: Date;
windowEnd: Date;
}) {
@@ -183,11 +248,12 @@ function buildApprovalPayload(input: {
thresholdType: input.thresholdType,
budgetAmount: input.policy.amount,
observedAmount: input.amountObserved,
observedAmountExact: input.amountObservedExact,
warnPercent: input.policy.warnPercent,
windowStart: input.windowStart.toISOString(),
windowEnd: input.windowEnd.toISOString(),
policyId: input.policy.id,
guidance: "Raise the budget and resume the scope, or keep the scope paused.",
guidance: "Resolve pending or unpriced accounting, raise the budget, or keep the scope paused.",
};
}
@@ -211,7 +277,9 @@ async function markApprovalStatus(
.where(eq(approvals.id, approvalId));
}
export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
/** Internal: caller must hold the company accounting transaction lock. */
export function budgetServiceInTransaction(db: Db, publications: ActivityPublication[] = []) {
const recordActivity = (input: LogActivityInput) => logActivity(db, input, publications);
async function pauseScopeForBudget(policy: PolicyRow) {
const now = new Date();
if (policy.scopeType === "agent") {
@@ -235,7 +303,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
pausedAt: now,
updatedAt: now,
})
.where(eq(projects.id, policy.scopeId));
.where(and(eq(projects.id, policy.scopeId), or(isNull(projects.pausedAt), eq(projects.pauseReason, "budget"))));
return;
}
@@ -247,19 +315,24 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
pausedAt: now,
updatedAt: now,
})
.where(eq(companies.id, policy.scopeId));
.where(and(eq(companies.id, policy.scopeId), or(eq(companies.status, "active"), and(eq(companies.status, "paused"), eq(companies.pauseReason, "budget")))));
}
async function pauseAndCancelScopeForBudget(policy: PolicyRow) {
await pauseScopeForBudget(policy);
await hooks.cancelWorkForScope?.({
companyId: policy.companyId,
scopeType: policy.scopeType as BudgetScopeType,
scopeId: policy.scopeId,
});
await db.update(budgetPolicies).set({
enforcementVersion: sql`${budgetPolicies.enforcementVersion} + 1`,
}).where(eq(budgetPolicies.id, policy.id));
}
async function resumeScopeFromBudget(policy: PolicyRow) {
const policies = await db.select().from(budgetPolicies).where(and(
eq(budgetPolicies.companyId, policy.companyId), eq(budgetPolicies.scopeType, policy.scopeType),
eq(budgetPolicies.scopeId, policy.scopeId), eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true),
));
for (const candidate of policies) {
if (await policyBlocks(db, candidate)) return;
}
const now = new Date();
if (policy.scopeType === "agent") {
await db.transaction(async tx => {
@@ -294,7 +367,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
pausedAt: null,
updatedAt: now,
})
.where(and(eq(companies.id, policy.scopeId), eq(companies.pauseReason, "budget")));
.where(and(eq(companies.id, policy.scopeId), eq(companies.status, "paused"), eq(companies.pauseReason, "budget")));
}
async function getPolicyRow(policyId: string) {
@@ -317,8 +390,9 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
const observedAmount = await computeObservedAmount(db, policy);
const { start, end } = resolveWindow(policy.windowKind as BudgetWindowKind);
const observed = await computeObservedSpend(db, policy);
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
const { windowStart: start, windowEnd: end } = observed;
const amount = policy.isActive ? policy.amount : 0;
const utilizationPercent =
amount > 0 ? Number(((observedAmount / amount) * 100).toFixed(2)) : 0;
@@ -332,14 +406,19 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
windowKind: policy.windowKind as BudgetWindowKind,
amount,
observedAmount,
observedAmountExact,
reservationCents: policy.reservationCents,
unpricedEventCount,
pendingRunCount,
unpricedUsagePolicy: policy.unpricedUsagePolicy as "block" | "allow",
remainingAmount: amount > 0 ? Math.max(0, amount - observedAmount) : 0,
utilizationPercent,
warnPercent: policy.warnPercent,
hardStopEnabled: policy.hardStopEnabled,
notifyEnabled: policy.notifyEnabled,
isActive: policy.isActive,
status: policy.isActive
? budgetStatusFromObserved(observedAmount, amount, policy.warnPercent)
status: observedBlocks(policy, observed) ? "hard_stop" : policy.isActive
? budgetStatusFromObserved(observedAmountExact, amount, policy.warnPercent)
: "ok",
paused: scope.paused,
pauseReason: scope.pauseReason,
@@ -351,9 +430,9 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
async function createIncidentIfNeeded(
policy: PolicyRow,
thresholdType: BudgetThresholdType,
amountObserved: number,
observed: Awaited<ReturnType<typeof computeObservedSpend>>,
) {
const { start, end } = resolveWindow(policy.windowKind as BudgetWindowKind);
const { windowStart: start, windowEnd: end } = observed;
const existing = await db
.select()
.from(budgetIncidents)
@@ -362,18 +441,21 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
eq(budgetIncidents.policyId, policy.id),
eq(budgetIncidents.windowStart, start),
eq(budgetIncidents.thresholdType, thresholdType),
ne(budgetIncidents.status, "dismissed"),
or(eq(budgetIncidents.status, "open"), and(eq(budgetIncidents.status, "dismissed"), eq(budgetIncidents.amountLimit, policy.amount))),
),
)
.orderBy(desc(budgetIncidents.createdAt))
.then((rows) => rows[0] ?? null);
if (existing) return { incident: existing, created: false };
const { total: amountObserved, totalExact: amountObservedExact } = observed;
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
const payload = buildApprovalPayload({
policy,
scopeName: normalizeScopeName(policy.scopeType as BudgetScopeType, scope.name),
thresholdType,
amountObserved,
amountObservedExact,
windowStart: start,
windowEnd: end,
});
@@ -406,7 +488,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
windowEnd: end,
thresholdType,
amountLimit: policy.amount,
amountObserved,
amountObserved: sql`${amountObservedExact}::numeric`,
status: "open",
approvalId: approval?.id ?? null,
})
@@ -436,11 +518,14 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
policyId: string,
approvalStatus: "approved" | "rejected" | null,
decidedByUserId: string | null,
thresholdType?: BudgetThresholdType,
) {
const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"),
thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined);
const openRows = await db
.select()
.from(budgetIncidents)
.where(and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open")));
.where(condition);
await db
.update(budgetIncidents)
@@ -449,7 +534,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
resolvedAt: new Date(),
updatedAt: new Date(),
})
.where(and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open")));
.where(condition);
if (!approvalStatus || !decidedByUserId) return;
for (const row of openRows) {
@@ -467,40 +552,87 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
: [];
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
return Promise.all(
rows.map(async (row) => {
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
return {
id: row.id,
companyId: row.companyId,
policyId: row.policyId,
scopeType: row.scopeType as BudgetScopeType,
scopeId: row.scopeId,
scopeName: normalizeScopeName(row.scopeType as BudgetScopeType, scope.name),
metric: row.metric as BudgetMetric,
windowKind: row.windowKind as BudgetWindowKind,
windowStart: row.windowStart,
windowEnd: row.windowEnd,
thresholdType: row.thresholdType as BudgetThresholdType,
amountLimit: row.amountLimit,
amountObserved: row.amountObserved,
status: row.status as BudgetIncident["status"],
approvalId: row.approvalId ?? null,
approvalStatus: row.approvalId ? approvalStatusById.get(row.approvalId) ?? null : null,
resolvedAt: row.resolvedAt ?? null,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}),
);
return mapBudgetReads(rows, async (row) => {
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
return {
id: row.id,
companyId: row.companyId,
policyId: row.policyId,
scopeType: row.scopeType as BudgetScopeType,
scopeId: row.scopeId,
scopeName: normalizeScopeName(row.scopeType as BudgetScopeType, scope.name),
metric: row.metric as BudgetMetric,
windowKind: row.windowKind as BudgetWindowKind,
windowStart: row.windowStart,
windowEnd: row.windowEnd,
thresholdType: row.thresholdType as BudgetThresholdType,
amountLimit: row.amountLimit,
amountObserved: row.amountObserved,
status: row.status as BudgetIncident["status"],
approvalId: row.approvalId ?? null,
approvalStatus: row.approvalId ? approvalStatusById.get(row.approvalId) ?? null : null,
resolvedAt: row.resolvedAt ?? null,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
});
}
async function reconcileScope(companyId: string, scopeType: BudgetScopeType, scopeId: string) {
const scope = await resolveScopeRecord(db, scopeType, scopeId);
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
const policies = await db.select().from(budgetPolicies).where(and(
eq(budgetPolicies.companyId, companyId), eq(budgetPolicies.scopeType, scopeType), eq(budgetPolicies.scopeId, scopeId),
));
let recoveringPolicy: PolicyRow | undefined;
for (const policy of policies) {
const observed = await computeObservedSpend(db, policy);
if (observedBlocks(policy, observed)) {
await createIncidentIfNeeded(policy, "hard", observed);
await pauseAndCancelScopeForBudget(policy);
const title = scopeType === "company" ? "Company" : scopeType === "agent" ? "Agent" : "Project";
if (observed.pendingRunCount > 0) {
return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work while completed runs await accounting.` };
}
if (policy.unpricedUsagePolicy !== "allow" && observed.unpricedEventCount > 0) {
return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work because recorded usage has no reliable price.` };
}
return { scopeType, scopeId, scopeName: scope.name, reason: scopeType === "project"
? "Project cannot start work because its budget hard-stop is still exceeded."
: `${title} is paused because its budget hard-stop was reached.` };
}
// An open native coordinator must finish the original run's accounting.
// Deny fresh work without pausing/cancelling that recovery or its hold.
if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
// A closed UTC window, inactive policy, or budget raise releases only
// budget-owned pauses. Manual pauses remain operator decisions.
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) {
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft");
}
}
// Agent status has its own admission gate and error code. Preserve a
// manual pause without misclassifying it as a budget failure.
if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") {
return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` };
}
if (scope.paused && scope.pauseReason === "budget" && !policies.length) {
return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." };
}
if (scope.pauseReason === "budget" && policies[0]) await resumeScopeFromBudget(policies[0]);
if (recoveringPolicy) return { scopeType, scopeId, scopeName: scope.name,
reason: "New work must wait while a native run recovers its unfinished accounting." };
return null;
}
return {
reconcileScope,
listPolicies: async (companyId: string): Promise<BudgetPolicy[]> => {
const rows = await listPolicyRows(companyId);
return rows.map((row) => ({
...row,
scopeType: row.scopeType as BudgetScopeType,
unpricedUsagePolicy: row.unpricedUsagePolicy as "block" | "allow",
metric: row.metric as BudgetMetric,
windowKind: row.windowKind as BudgetWindowKind,
}));
@@ -516,10 +648,9 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
throw unprocessable("Budget scope does not belong to company");
}
if (input.reservationCents !== undefined && compareCents(input.reservationCents, 0) < 0) throw unprocessable("Reservation cannot be negative");
const metric = input.metric ?? "billed_cents";
const windowKind = input.windowKind ?? (input.scopeType === "project" ? "lifetime" : "calendar_month_utc");
const amount = Math.max(0, Math.floor(input.amount));
const nextIsActive = amount > 0 && (input.isActive ?? true);
const existing = await db
.select()
.from(budgetPolicies)
@@ -534,15 +665,22 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
)
.then((rows) => rows[0] ?? null);
if (!existing && input.amount === undefined) throw unprocessable("Amount is required for a new budget policy");
const amount = Math.max(0, Math.floor(input.amount ?? existing!.amount));
// A saved zero disables enforcement. Raising it is an explicit limit
// setting; editing a disabled positive policy must preserve that choice.
const nextIsActive = amount > 0 && (input.isActive ?? (existing?.amount === 0 ? true : existing?.isActive) ?? true);
const now = new Date();
const row = existing
? await db
.update(budgetPolicies)
.set({
amount,
reservationCents: normalizeCents(input.reservationCents ?? existing.reservationCents ?? 0),
warnPercent: input.warnPercent ?? existing.warnPercent,
hardStopEnabled: input.hardStopEnabled ?? existing.hardStopEnabled,
notifyEnabled: input.notifyEnabled ?? existing.notifyEnabled,
unpricedUsagePolicy: input.unpricedUsagePolicy ?? existing.unpricedUsagePolicy,
isActive: nextIsActive,
updatedByUserId: actorUserId,
updatedAt: now,
@@ -559,9 +697,11 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
metric,
windowKind,
amount,
reservationCents: normalizeCents(input.reservationCents ?? 0),
warnPercent: input.warnPercent ?? 80,
hardStopEnabled: input.hardStopEnabled ?? true,
notifyEnabled: input.notifyEnabled ?? true,
unpricedUsagePolicy: input.unpricedUsagePolicy ?? "block",
isActive: nextIsActive,
createdByUserId: actorUserId,
updatedByUserId: actorUserId,
@@ -589,28 +729,33 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
.where(eq(agents.id, input.scopeId));
}
if (amount > 0) {
const observedAmount = await computeObservedAmount(db, row);
if (observedAmount < amount) {
if (row.isActive && amount > 0) {
const observed = await computeObservedSpend(db, row);
if (!observedBlocks(row, observed)) {
await resumeScopeFromBudget(row);
await resolveOpenIncidentsForPolicy(row.id, actorUserId ? "approved" : null, actorUserId);
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard");
if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) {
await createIncidentIfNeeded(row, "soft", observed);
} else {
await resolveOpenIncidentsForPolicy(row.id, null, null, "soft");
}
} else {
const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100);
if (row.notifyEnabled && observedAmount >= softThreshold) {
await createIncidentIfNeeded(row, "soft", observedAmount);
if (row.notifyEnabled && compareCents(observed.totalExact, softThreshold) >= 0 && (!row.hardStopEnabled || compareCents(observed.totalExact, row.amount) < 0)) {
await createIncidentIfNeeded(row, "soft", observed);
}
if (row.hardStopEnabled && observedAmount >= row.amount) {
if (observedBlocks(row, observed)) {
await resolveOpenSoftIncidents(row.id);
await createIncidentIfNeeded(row, "hard", observedAmount);
await createIncidentIfNeeded(row, "hard", observed);
await pauseAndCancelScopeForBudget(row);
}
}
} else {
await resumeScopeFromBudget(row);
await resolveOpenIncidentsForPolicy(row.id, actorUserId ? "approved" : null, actorUserId);
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service");
}
await logActivity(db, {
await recordActivity({
companyId,
actorType: "user",
actorId: actorUserId ?? "board",
@@ -630,7 +775,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
overview: async (companyId: string): Promise<BudgetOverview> => {
const rows = await listPolicyRows(companyId);
const policies = await Promise.all(rows.map((row) => buildPolicySummary(row)));
const policies = await mapBudgetReads(rows, buildPolicySummary);
const activeIncidentRows = await db
.select()
.from(budgetIncidents)
@@ -641,8 +786,8 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
companyId,
policies,
activeIncidents,
pausedAgentCount: policies.filter((policy) => policy.scopeType === "agent" && policy.paused).length,
pausedProjectCount: policies.filter((policy) => policy.scopeType === "project" && policy.paused).length,
pausedAgentCount: new Set(policies.filter((policy) => policy.scopeType === "agent" && policy.paused).map((policy) => policy.scopeId)).size,
pausedProjectCount: new Set(policies.filter((policy) => policy.scopeType === "project" && policy.paused).map((policy) => policy.scopeId)).size,
pendingApprovalCount: activeIncidents.filter((incident) => incident.approvalStatus === "pending").length,
};
},
@@ -655,26 +800,21 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
and(
eq(budgetPolicies.companyId, event.companyId),
eq(budgetPolicies.isActive, true),
inArray(budgetPolicies.scopeType, ["company", "agent", "project"]),
budgetPoliciesForRun(event.companyId, event.agentId, event.projectId),
),
);
const relevantPolicies = candidatePolicies.filter((policy) => {
if (policy.scopeType === "company") return policy.scopeId === event.companyId;
if (policy.scopeType === "agent") return policy.scopeId === event.agentId;
if (policy.scopeType === "project") return Boolean(event.projectId) && policy.scopeId === event.projectId;
return false;
});
for (const policy of relevantPolicies) {
if (policy.metric !== "billed_cents" || policy.amount <= 0) continue;
const observedAmount = await computeObservedAmount(db, policy);
const policies = candidatePolicies.filter(policy => policy.metric === "billed_cents" && policy.amount > 0);
const observations = await computeObservedSpends(db, policies);
for (const [index, policy] of policies.entries()) {
const observed = observations[index];
const observedAmount = observed.total;
const softThreshold = Math.ceil((policy.amount * policy.warnPercent) / 100);
if (policy.notifyEnabled && observedAmount >= softThreshold) {
const softIncident = await createIncidentIfNeeded(policy, "soft", observedAmount);
if (policy.notifyEnabled && compareCents(observed.totalExact, softThreshold) >= 0 && (!policy.hardStopEnabled || compareCents(observed.totalExact, policy.amount) < 0)) {
const softIncident = await createIncidentIfNeeded(policy, "soft", observed);
if (softIncident?.created) {
await logActivity(db, {
await recordActivity({
companyId: policy.companyId,
actorType: "system",
actorId: "budget_service",
@@ -691,12 +831,12 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
}
}
if (policy.hardStopEnabled && observedAmount >= policy.amount) {
if (observedBlocks(policy, observed)) {
await resolveOpenSoftIncidents(policy.id);
const hardIncident = await createIncidentIfNeeded(policy, "hard", observedAmount);
const hardIncident = await createIncidentIfNeeded(policy, "hard", observed);
await pauseAndCancelScopeForBudget(policy);
if (hardIncident?.created) {
await logActivity(db, {
await recordActivity({
companyId: policy.companyId,
actorType: "system",
actorId: "budget_service",
@@ -721,147 +861,18 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
agentId: string,
context?: { issueId?: string | null; projectId?: string | null },
) => {
const agent = await db
.select({
status: agents.status,
pauseReason: agents.pauseReason,
companyId: agents.companyId,
name: agents.name,
})
.from(agents)
.where(eq(agents.id, agentId))
.then((rows) => rows[0] ?? null);
if (!agent || agent.companyId !== companyId) throw notFound("Agent not found");
const company = await db
.select({
status: companies.status,
pauseReason: companies.pauseReason,
name: companies.name,
})
.from(companies)
.where(eq(companies.id, companyId))
.then((rows) => rows[0] ?? null);
if (!company) throw notFound("Company not found");
if (company.status === "paused") {
return {
scopeType: "company" as const,
scopeId: companyId,
scopeName: company.name,
reason:
company.pauseReason === "budget"
? "Company is paused because its budget hard-stop was reached."
: "Company is paused and cannot start new work.",
};
const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [
{ scopeType: "company", scopeId: companyId }, { scopeType: "agent", scopeId: agentId },
...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []),
];
for (const { scopeType, scopeId } of scopes) {
if ((await resolveScopeRecord(db, scopeType, scopeId)).companyId !== companyId) throw notFound("Budget scope not found");
}
const companyPolicy = await db
.select()
.from(budgetPolicies)
.where(
and(
eq(budgetPolicies.companyId, companyId),
eq(budgetPolicies.scopeType, "company"),
eq(budgetPolicies.scopeId, companyId),
eq(budgetPolicies.isActive, true),
eq(budgetPolicies.metric, "billed_cents"),
),
)
.then((rows) => rows[0] ?? null);
if (companyPolicy && companyPolicy.hardStopEnabled && companyPolicy.amount > 0) {
const observed = await computeObservedAmount(db, companyPolicy);
if (observed >= companyPolicy.amount) {
return {
scopeType: "company" as const,
scopeId: companyId,
scopeName: company.name,
reason: "Company cannot start new work because its budget hard-stop is exceeded.",
};
}
for (const { scopeType, scopeId } of scopes) {
const block = await reconcileScope(companyId, scopeType, scopeId);
if (block) return block;
}
if (agent.status === "paused" && agent.pauseReason === "budget") {
return {
scopeType: "agent" as const,
scopeId: agentId,
scopeName: agent.name,
reason: "Agent is paused because its budget hard-stop was reached.",
};
}
const agentPolicy = await db
.select()
.from(budgetPolicies)
.where(
and(
eq(budgetPolicies.companyId, companyId),
eq(budgetPolicies.scopeType, "agent"),
eq(budgetPolicies.scopeId, agentId),
eq(budgetPolicies.isActive, true),
eq(budgetPolicies.metric, "billed_cents"),
),
)
.then((rows) => rows[0] ?? null);
if (agentPolicy && agentPolicy.hardStopEnabled && agentPolicy.amount > 0) {
const observed = await computeObservedAmount(db, agentPolicy);
if (observed >= agentPolicy.amount) {
return {
scopeType: "agent" as const,
scopeId: agentId,
scopeName: agent.name,
reason: "Agent cannot start because its budget hard-stop is still exceeded.",
};
}
}
const candidateProjectId = context?.projectId ?? null;
if (!candidateProjectId) return null;
const project = await db
.select({
id: projects.id,
name: projects.name,
companyId: projects.companyId,
pauseReason: projects.pauseReason,
pausedAt: projects.pausedAt,
})
.from(projects)
.where(eq(projects.id, candidateProjectId))
.then((rows) => rows[0] ?? null);
if (!project || project.companyId !== companyId) return null;
const projectPolicy = await db
.select()
.from(budgetPolicies)
.where(
and(
eq(budgetPolicies.companyId, companyId),
eq(budgetPolicies.scopeType, "project"),
eq(budgetPolicies.scopeId, project.id),
eq(budgetPolicies.isActive, true),
eq(budgetPolicies.metric, "billed_cents"),
),
)
.then((rows) => rows[0] ?? null);
if (projectPolicy && projectPolicy.hardStopEnabled && projectPolicy.amount > 0) {
const observed = await computeObservedAmount(db, projectPolicy);
if (observed >= projectPolicy.amount) {
return {
scopeType: "project" as const,
scopeId: project.id,
scopeName: project.name,
reason: "Project cannot start work because its budget hard-stop is still exceeded.",
};
}
}
if (!project.pausedAt || project.pauseReason !== "budget") return null;
return {
scopeType: "project" as const,
scopeId: project.id,
scopeName: project.name,
reason: "Project is paused because its budget hard-stop was reached.",
};
return null;
},
resolveIncident: async (
@@ -881,8 +892,12 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
const policy = await getPolicyRow(incident.policyId);
if (input.action === "raise_budget_and_resume") {
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
const currentObserved = await computeObservedAmount(db, policy);
if (nextAmount <= currentObserved) {
const observed = await computeObservedSpend(db, policy);
if (policy.unpricedUsagePolicy !== "allow" && observed.unpricedEventCount > 0) {
throw unprocessable("Resolve unpriced usage or explicitly allow it in the budget policy before resuming");
}
if (observed.pendingRunCount > 0) throw unprocessable("Completed runs must finish accounting before resuming");
if (compareCents(observed.totalExact, nextAmount) >= 0) {
throw unprocessable("New budget must exceed current observed spend");
}
@@ -934,7 +949,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
await markApprovalStatus(db, incident.approvalId ?? null, "rejected", input.decisionNote, actorUserId);
}
await logActivity(db, {
await recordActivity({
companyId: incident.companyId,
actorType: "user",
actorId: actorUserId,
@@ -959,3 +974,78 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
},
};
}
/** Fence the durable stop intent against policy edits and admission. External
* process shutdown must happen after this transaction, without holding locks
* that the executor needs to persist its final usage receipt. */
export async function withCurrentBudgetEnforcement<T>(db: Db, scope: BudgetEnforcementScope, work: (tx: Db) => Promise<T>): Promise<T | null> {
return withAccountingTransaction(db, scope.companyId, async (tx) => {
if (scope.enforcement) {
const [policy] = await tx.select().from(budgetPolicies).where(and(
eq(budgetPolicies.id, scope.enforcement.policyId), eq(budgetPolicies.companyId, scope.companyId),
));
if (!policy || policy.scopeType !== scope.scopeType || policy.scopeId !== scope.scopeId
|| policy.enforcementVersion !== scope.enforcement.version || !(await policyBlocks(tx, policy))) return null;
}
return work(tx);
});
}
/** Cancellation is an at-least-once external effect. A failed delivery never
* rolls back committed spend, and its version remains pending for recovery. */
export async function deliverBudgetEnforcement(db: Db, hooks: BudgetServiceHooks, companyId?: string) {
if (!hooks.cancelWorkForScope) return;
const pending = await db.select().from(budgetPolicies).where(and(
sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`,
companyId ? eq(budgetPolicies.companyId, companyId) : undefined,
));
for (const policy of pending) {
try {
const cancellation = await withAccountingTransaction(db, policy.companyId, async (tx) => {
const [current] = await tx.select().from(budgetPolicies).where(eq(budgetPolicies.id, policy.id));
if (!current || current.enforcementVersion !== policy.enforcementVersion || !(await policyBlocks(tx, current))) return null;
return { companyId: policy.companyId, scopeType: policy.scopeType as BudgetScopeType, scopeId: policy.scopeId, createdBefore: new Date(), enforcement: { policyId: policy.id, version: policy.enforcementVersion } };
});
if (cancellation) await hooks.cancelWorkForScope(cancellation);
await db.update(budgetPolicies).set({ enforcementDeliveredVersion: policy.enforcementVersion })
.where(and(eq(budgetPolicies.id, policy.id), eq(budgetPolicies.enforcementVersion, policy.enforcementVersion)));
} catch (error) {
logger.warn({ err: error, policyId: policy.id }, "Budget cancellation delivery pending; accounting is committed");
}
}
}
export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
const reads = budgetServiceInTransaction(db);
async function mutate<T>(companyId: string, work: (service: ReturnType<typeof budgetServiceInTransaction>) => Promise<T>) {
const result = await withAccountingTransaction(db, companyId, (tx, publications) => work(budgetServiceInTransaction(tx, publications)));
await deliverBudgetEnforcement(db, hooks, companyId);
return result;
}
return {
...reads,
deliverPendingEnforcement: (companyId?: string) => deliverBudgetEnforcement(db, hooks, companyId),
reconcilePolicies: async () => {
const scopes = await db.selectDistinct({ companyId: budgetPolicies.companyId, scopeType: budgetPolicies.scopeType, scopeId: budgetPolicies.scopeId })
.from(budgetPolicies).orderBy(budgetPolicies.companyId, budgetPolicies.scopeType, budgetPolicies.scopeId);
// Each scope commits independently: deleted targets and a failed scope
// must not roll back recovery or cancellation delivery for other work.
for (const scope of scopes) {
try {
await mutate(scope.companyId, (service) => service.reconcileScope(scope.companyId, scope.scopeType as BudgetScopeType, scope.scopeId));
} catch (error) {
if (error instanceof HttpError && error.status === 404) continue;
logger.warn({ err: error, ...scope }, "Budget scope recovery pending; continuing remaining scopes");
}
}
},
upsertPolicy: (companyId: string, input: BudgetPolicyUpsertInput, actorUserId: string | null) =>
mutate(companyId, (service) => service.upsertPolicy(companyId, input, actorUserId)),
evaluateCostEvent: (event: typeof costEvents.$inferSelect) =>
mutate(event.companyId, (service) => service.evaluateCostEvent(event)),
getInvocationBlock: (companyId: string, agentId: string, context?: { issueId?: string | null; projectId?: string | null }) =>
mutate(companyId, (service) => service.getInvocationBlock(companyId, agentId, context)),
resolveIncident: (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) =>
mutate(companyId, (service) => service.resolveIncident(companyId, incidentId, input, actorUserId)),
};
}
server/src/services/codex-pricing.ts
+98
View File
@@ -0,0 +1,98 @@
import { centsToUsd, unitsToCents, usdToUnits } from "@paperclipai/shared";
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
// Immutable, reviewed rate card. Existing receipts retain their selected rates.
// https://developers.openai.com/api/docs/pricing (retrieved 2026-09-30)
const VERSION = "openai-standard-2026-09-30";
const RATES: Record<string, readonly [string, string, string, string]> = {
"gpt-6-astra": ["10", "1", "12.5", "50"],
"gpt-6-sol": ["2", "0.2", "2.5", "10"],
"gpt-6.1-sol": ["2", "0.1", "2.5", "10"],
"gpt-6-luna": ["0.1", "0.01", "0.125", "0.5"],
"gpt-5.6-sol": ["4", "0.4", "5", "20"],
};
/** Price new, direct-API receipts only. A token-price estimate is not an invoice.
* Run totals cannot establish per-request context length or cache-write splits;
* preserve those assumptions instead of inferring a long request from a long run. */
export function priceCodexReceipt(
receipt: AdapterUsageCheckpoint,
): AdapterUsageCheckpoint {
if (
!receipt.complete ||
receipt.costUsd != null ||
receipt.costUsdExact != null ||
receipt.cacheAdjustedCostUsd != null ||
receipt.provider !== "openai" ||
receipt.biller !== "openai" ||
!["api", "metered_api"].includes(receipt.billingType ?? "") ||
receipt.usageBasis !== "per_run" ||
!receipt.usage ||
receipt.usageByModel?.length
)
return receipt;
const model = receipt.model ?? "";
const rates = Object.hasOwn(RATES, model) ? RATES[model] : undefined;
if (!rates) return receipt;
const usage = receipt.usage;
const counts = [
usage.inputTokens - (usage.cacheWriteTokens ?? 0),
usage.cachedInputTokens ?? 0,
usage.cacheWriteTokens ?? 0,
usage.outputTokens,
];
if (counts.some((n) => !Number.isSafeInteger(n) || n < 0)) return receipt;
const context = receipt.pricingContext;
if (
context?.serviceTier &&
!["standard", "default", "priority", "fast", "flex", "batch"].includes(
context.serviceTier,
)
)
return receipt;
const tierNumerator =
context?.serviceTier === "priority" || context?.serviceTier === "fast"
? 2n
: 1n;
const tierDenominator =
context?.serviceTier === "flex" || context?.serviceTier === "batch"
? 2n
: 1n;
const units = counts.reduce((total, count, index) => {
const longNumerator =
context?.contextTier === "long" ? (index === 3 ? 3n : 4n) : 2n;
return (
total +
BigInt(count) * usdToUnits(rates[index]) * tierNumerator * longNumerator
);
}, 0n);
const divisor = 1_000_000n * tierDenominator * 2n;
const costCents = unitsToCents((units + divisor / 2n) / divisor);
const assumptions = [
!context?.serviceTier
? "standard processing assumed"
: `${context.serviceTier} processing`,
!context?.contextTier
? "short per-request context assumed"
: `${context.contextTier} per-request context`,
usage.cacheWriteTokens === undefined
? "cache writes not separately reported"
: "cache writes reported",
];
return {
...receipt,
costUsdExact: centsToUsd(costCents),
costStatus: "estimated",
pricingProvenance: {
source: "rate_card",
version: VERSION,
evidence: `https://developers.openai.com/api/docs/pricing; ${assumptions.join("; ")}`,
inputCentsPerMillion: unitsToCents(usdToUnits(rates[0])),
cachedInputCentsPerMillion: unitsToCents(usdToUnits(rates[1])),
cacheWriteCentsPerMillion: unitsToCents(usdToUnits(rates[2])),
outputCentsPerMillion: unitsToCents(usdToUnits(rates[3])),
serviceTier: context?.serviceTier ?? "standard",
contextTier: context?.contextTier ?? "short",
},
};
}
server/src/services/companies.ts
+31 -4
View File
@@ -1,3 +1,5 @@
import { publishAccountingActivities } from "./accounting-transaction.js";
import { budgetServiceInTransaction, deliverBudgetEnforcement, type BudgetServiceHooks } from "./budgets.js";
import { and, count, eq, gte, inArray, isNull, lt, notInArray, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
@@ -18,6 +20,8 @@ import {
heartbeatRunEvents,
costEvents,
financeEvents,
budgetPolicies,
budgetIncidents,
issueReadStates,
approvalComments,
approvals,
@@ -47,7 +51,7 @@ import {
} from "./issue-prefix.js";
import { environmentService } from "./environments.js";
import { heartbeatService } from "./heartbeat.js";
import { logActivity } from "./activity-log.js";
import { logActivity, type ActivityPublication } from "./activity-log.js";
import { builtInAgentService } from "./built-in-agents.js";
@@ -66,7 +70,7 @@ const SYSTEM_COMPANY_ACTOR: CompanyActivityActor = {
runId: null,
};
export function companyService(db: Db) {
export function companyService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
const environmentsSvc = environmentService(db);
const heartbeat = heartbeatService(db);
const builtInAgents = builtInAgentService(db);
@@ -332,12 +336,16 @@ export function companyService(db: Db) {
data: Partial<typeof companies.$inferInsert> & { logoAssetId?: string | null },
actor: CompanyActivityActor = SYSTEM_COMPANY_ACTOR,
) => {
const budgetPublications: ActivityPublication[] = [];
const result = await db.transaction(async (tx) => {
const existing = await getCompanyQuery(tx)
.where(eq(companies.id, id))
.then((rows) => rows[0] ?? null);
if (!existing) return null;
if (data.budgetMonthlyCents !== undefined) {
await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, id)).for("no key update");
}
const { logoAssetId, ...companyPatch } = data;
const willReactivate = existing.status !== "active" && companyPatch.status === "active";
const willArchive = existing.status !== "archived" && companyPatch.status === "archived";
@@ -431,6 +439,13 @@ export function companyService(db: Db) {
await tx.delete(assets).where(eq(assets.id, existing.logoAssetId));
}
if (data.budgetMonthlyCents !== undefined) {
await budgetServiceInTransaction(tx as unknown as Db, budgetPublications).upsertPolicy(id, {
scopeType: "company", scopeId: id, amount: data.budgetMonthlyCents, isActive: data.budgetMonthlyCents > 0, windowKind: "calendar_month_utc",
}, actor.actorType === "user" ? actor.actorId : null);
const [budgetUpdated] = await tx.select().from(companies).where(eq(companies.id, id));
Object.assign(updated, budgetUpdated);
}
const [hydrated] = await hydrateCompanySpend([{
...updated,
logoAssetId: logoAssetId === undefined ? existing.logoAssetId : logoAssetId,
@@ -448,6 +463,8 @@ export function companyService(db: Db) {
};
});
if (!result) return null;
publishAccountingActivities(id, budgetPublications);
if (data.budgetMonthlyCents !== undefined) await deliverBudgetEnforcement(db, budgetHooks, id);
// Post-commit, fire-and-forget, and BEFORE any finalization that
// could throw: a Cloud-pinned primary company that crossed the
// archived boundary (either direction) rings the harness so the
@@ -537,6 +554,18 @@ export function companyService(db: Db) {
remove: (id: string) =>
db.transaction(async (tx) => {
// Exclude accounting writers before taking child locks. KEY SHARE must
// remain compatible: native writers can already hold a child row while
// saving a company-scoped result. FOR UPDATE would deadlock that save.
const [existing] = await tx.select({ id: companies.id }).from(companies)
.where(eq(companies.id, id)).for("no key update");
if (!existing) return null;
// Finance can reference costs and both can reference runs. Incidents
// reference policies and approvals; delete these dependents first.
await tx.delete(financeEvents).where(eq(financeEvents.companyId, id));
await tx.delete(costEvents).where(eq(costEvents.companyId, id));
await tx.delete(budgetIncidents).where(eq(budgetIncidents.companyId, id));
await tx.delete(budgetPolicies).where(eq(budgetPolicies.companyId, id));
// Delete from child tables in dependency order
const companyRunIds = await tx
.select({ id: heartbeatRuns.id })
@@ -557,8 +586,6 @@ export function companyService(db: Db) {
await tx.delete(agentApiKeys).where(eq(agentApiKeys.companyId, id));
await tx.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, id));
await tx.delete(issueComments).where(eq(issueComments.companyId, id));
await tx.delete(costEvents).where(eq(costEvents.companyId, id));
await tx.delete(financeEvents).where(eq(financeEvents.companyId, id));
await tx.delete(approvalComments).where(eq(approvalComments.companyId, id));
await tx.delete(approvals).where(eq(approvals.companyId, id));
await tx.delete(companySecrets).where(eq(companySecrets.companyId, id));
server/src/services/company-skills.ts
+1 -1
View File
@@ -6623,7 +6623,7 @@ export function companySkillService(db: Db) {
const rows = await db
.select({
issueId: costEvents.issueId,
costCents: sql<number>`coalesce(sum(${costEvents.costCents}), 0)::int`,
costCents: sql<number>`coalesce(sum(${costEvents.costCents}), 0)::double precision`,
inputTokens: sql<number>`coalesce(sum(${costEvents.inputTokens}), 0)::int`,
cachedInputTokens: sql<number>`coalesce(sum(${costEvents.cachedInputTokens}), 0)::int`,
outputTokens: sql<number>`coalesce(sum(${costEvents.outputTokens}), 0)::int`,
server/src/services/cost-date-range.ts
+12
View File
@@ -0,0 +1,12 @@
export interface CostDateRange {
from?: Date;
to?: Date;
allTime?: boolean;
}
/** Default reports match the UTC month used by monetary budgets. Explicit
* bounds preserve the existing inclusive API; all-time must be requested. */
export function resolveCostDateRange(range?: CostDateRange, now = new Date()): CostDateRange {
if (range?.allTime || range?.from || range?.to) return range;
return { from: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)), to: now };
}
server/src/services/costs.ts
+247 -79
View File
@@ -1,21 +1,32 @@
import { agentAvatarUrl, resolveAgentAppearance } from "@paperclipai/shared";
import { and, desc, eq, gte, isNotNull, isNull, lt, lte, sql } from "drizzle-orm";
import { receiptFingerprint } from "./receipt-fingerprint.js";
import { agentAvatarUrl, resolveAgentAppearance, createCostEventSchema, normalizeCents, type MoneyInput, type CostByUserReport } from "@paperclipai/shared";
import { and, desc, eq, gte, inArray, isNotNull, isNull, lt, lte, ne, sql } from "drizzle-orm";
import { alias } from "drizzle-orm/pg-core";
import type { Db } from "@paperclipai/db";
import { activityLog, agents, companies, costEvents, heartbeatRuns, issues, projects } from "@paperclipai/db";
import { notFound, unprocessable } from "../errors.js";
import { budgetService, type BudgetServiceHooks } from "./budgets.js";
import { activityLog, agents, agentRuntimeState, authUsers, companies, companyMemberships, costEvents, heartbeatRuns, issues, projects, goals } from "@paperclipai/db";
import { notFound, unprocessable, conflict } from "../errors.js";
import { budgetService, budgetServiceInTransaction, type BudgetServiceHooks } from "./budgets.js";
import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js";
import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js";
import { visibleIssueCondition } from "./issue-visibility.js";
export interface CostDateRange {
from?: Date;
to?: Date;
}
import { resolveCostDateRange, type CostDateRange } from "./cost-date-range.js";
export type { CostDateRange } from "./cost-date-range.js";
const METERED_BILLING_TYPE = "metered_api";
const SUBSCRIPTION_BILLING_TYPES = ["subscription_included", "subscription_overage"] as const;
function sumAsNumber(column: typeof costEvents.costCents | typeof costEvents.inputTokens | typeof costEvents.cachedInputTokens | typeof costEvents.outputTokens) {
// Pre-receipt Codex events stored cache reads inside input; Pi/OpenCode used
// provider-qualified model IDs and already excluded cache reads. Normalize each
// historical row before aggregation so mixed old/new groups share the current
// exclusive-input contract. Preserve the original ledger and monetary amounts.
const ordinaryInputTokens = sql<number>`case
when ${costEvents.receiptHash} is null and ${costEvents.provider} = 'openai'
and ${costEvents.model} not like 'openai/_%'
then greatest(0, ${costEvents.inputTokens} - ${costEvents.cachedInputTokens})
else ${costEvents.inputTokens} end`;
function sumAsNumber(column: typeof costEvents.costCents | typeof costEvents.inputTokens | typeof costEvents.cachedInputTokens | typeof costEvents.outputTokens | ReturnType<typeof sql>) {
return sql<number>`coalesce(sum(${column}), 0)::double precision`;
}
@@ -28,6 +39,26 @@ function currentUtcMonthWindow(now = new Date()) {
};
}
/** Increment an initialized month under the company lock. On upgrade or UTC
* rollover, initialize from the ledger in the same transaction exactly once.
* Reports, budgets and the integrity checker continue reading the ledger. */
export async function updateMonthlySpendProjections(db: Db, companyId: string, agentId: string, amountCents: MoneyInput, occurredAt: Date) {
const { start, end } = currentUtcMonthWindow();
const month = start.toISOString().slice(0, 10);
const delta = occurredAt >= start && occurredAt < end ? normalizeCents(amountCents) : "0.0000000";
for (const table of [companies, agents] as const) {
const id = table === companies ? companyId : agentId;
const companyCondition = table === companies ? eq(companies.id, companyId) : eq(agents.companyId, companyId);
await db.update(table).set({
spentMonthlyCents: sql`case when ${table.spendMonthUtc} = ${month} then ${table.spentMonthlyCents} + ${delta}::numeric else (
select coalesce(sum(${costEvents.costCents}),0) from ${costEvents} where ${costEvents.companyId} = ${companyId}
and ${costEvents.occurredAt} >= ${start.toISOString()}::timestamptz and ${costEvents.occurredAt} < ${end.toISOString()}::timestamptz
${table === agents ? sql`and ${costEvents.agentId} = ${agentId}` : sql``}) end`,
spendMonthUtc: month, updatedAt: new Date(),
}).where(and(eq(table.id, id), companyCondition));
}
}
export async function getMonthlySpendTotal(
db: Db,
scope: { companyId: string; agentId?: string | null },
@@ -50,60 +81,88 @@ export async function getMonthlySpendTotal(
return Number(row?.total ?? 0);
}
export async function createCostEventInTransaction(db: Db, companyId: string, data: Omit<typeof costEvents.$inferInsert, "companyId" | "receiptHash" | "costCents"> & { costCents: MoneyInput }, publications: ActivityPublication[] = [], actor?: Pick<LogActivityInput, "actorType" | "actorId" | "agentId">) {
const parsed = createCostEventSchema.safeParse({ ...data, occurredAt: data.occurredAt.toISOString() });
if (!parsed.success) throw unprocessable("Invalid cost receipt", parsed.error.flatten());
const values = {
...parsed.data,
issueId: parsed.data.issueId ?? null,
projectId: parsed.data.projectId ?? null,
goalId: parsed.data.goalId ?? null,
heartbeatRunId: parsed.data.heartbeatRunId ?? null,
billingCode: parsed.data.billingCode ?? null,
idempotencyKey: parsed.data.idempotencyKey ?? null,
costCents: normalizeCents(parsed.data.costCents),
occurredAt: new Date(parsed.data.occurredAt),
};
const receiptHash = receiptFingerprint(values);
// Older receipts hashed the numeric API value. Retain replay compatibility
// only when the decimal value survives that representation exactly.
let legacyHash: string | null = null;
try {
if (normalizeCents(Number(values.costCents)) === values.costCents) legacyHash = receiptFingerprint({ ...values, costCents: Number(values.costCents) });
} catch { /* The legacy number may round beyond the decimal storage limit. */ }
if (values.idempotencyKey) {
const [existing] = await db.select().from(costEvents).where(and(
eq(costEvents.companyId, companyId), eq(costEvents.idempotencyKey, values.idempotencyKey),
));
if (existing) {
if (existing.receiptHash !== receiptHash && existing.receiptHash !== legacyHash) throw conflict("Idempotency key already used for a different cost receipt");
const [exact] = await db.select({ costCents: sql<string>`${costEvents.costCents}::text` }).from(costEvents).where(eq(costEvents.id, existing.id));
return { ...existing, costCentsExact: normalizeCents(exact.costCents) };
}
}
const links = [
[agents, values.agentId, "Agent"], [issues, values.issueId, "Issue"],
[projects, values.projectId, "Project"], [goals, values.goalId, "Goal"],
[heartbeatRuns, values.heartbeatRunId, "Heartbeat run"],
] as const;
for (const [table, id, label] of links) {
if (!id) continue;
const [row] = await db.select({ companyId: table.companyId }).from(table).where(eq(table.id, id));
if (!row) throw notFound(`${label} not found`);
if (row.companyId !== companyId) throw unprocessable(`${label} does not belong to company`);
}
if (values.heartbeatRunId) {
const [run] = await db.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, values.heartbeatRunId));
if (run.agentId !== values.agentId) throw unprocessable("Heartbeat run does not belong to agent");
}
const [event] = await db.insert(costEvents).values({ ...values, costCents: sql`${values.costCents}::numeric`, reportedCostCents: values.costCents, id: data.id, companyId, receiptHash }).returning();
await updateMonthlySpendProjections(db, companyId, event.agentId, values.costCents, event.occurredAt);
// Separately reported charges linked to an already-accounted run contribute
// to lifetime totals too. Before acknowledgement, accountRunCost includes
// all of the run's events atomically when it initializes the projection.
if (event.heartbeatRunId) {
const [run] = await db.select({ acknowledged: heartbeatRuns.costAccountedAt, version: heartbeatRuns.accountingProjectionVersion })
.from(heartbeatRuns).where(and(eq(heartbeatRuns.id, event.heartbeatRunId), eq(heartbeatRuns.companyId, companyId)));
if (run.acknowledged && run.version === "v2") await db.update(agentRuntimeState).set({
totalCostCents: sql`${agentRuntimeState.totalCostCents} + ${values.costCents}::numeric`,
totalInputTokens: sql`${agentRuntimeState.totalInputTokens} + ${event.inputTokens}`,
totalCachedInputTokens: sql`${agentRuntimeState.totalCachedInputTokens} + ${event.cachedInputTokens}`,
totalOutputTokens: sql`${agentRuntimeState.totalOutputTokens} + ${event.outputTokens}`,
updatedAt: new Date(),
}).where(and(eq(agentRuntimeState.agentId, event.agentId), eq(agentRuntimeState.companyId, companyId)));
}
await budgetServiceInTransaction(db, publications).evaluateCostEvent(event);
await logActivity(db, {
companyId, actorType: actor?.actorType ?? "system", actorId: actor?.actorId ?? "cost_accounting", agentId: actor?.agentId ?? event.agentId,
runId: event.heartbeatRunId, action: "cost.reported", entityType: "cost_event", entityId: event.id,
details: { costCents: event.costCents, costCentsExact: values.costCents, model: event.model, costStatus: event.costStatus },
}, publications);
return { ...event, costCentsExact: values.costCents };
}
export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
const budgets = budgetService(db, budgetHooks);
return {
createEvent: async (companyId: string, data: Omit<typeof costEvents.$inferInsert, "companyId">) => {
const agent = await db
.select()
.from(agents)
.where(eq(agents.id, data.agentId))
.then((rows) => rows[0] ?? null);
if (!agent) throw notFound("Agent not found");
if (agent.companyId !== companyId) {
throw unprocessable("Agent does not belong to company");
}
const event = await db
.insert(costEvents)
.values({
...data,
companyId,
biller: data.biller ?? data.provider,
billingType: data.billingType ?? "unknown",
cachedInputTokens: data.cachedInputTokens ?? 0,
})
.returning()
.then((rows) => rows[0]);
const [agentMonthSpend, companyMonthSpend] = await Promise.all([
getMonthlySpendTotal(db, { companyId, agentId: event.agentId }),
getMonthlySpendTotal(db, { companyId }),
]);
await db
.update(agents)
.set({
spentMonthlyCents: agentMonthSpend,
updatedAt: new Date(),
})
.where(eq(agents.id, event.agentId));
await db
.update(companies)
.set({
spentMonthlyCents: companyMonthSpend,
updatedAt: new Date(),
})
.where(eq(companies.id, companyId));
await budgets.evaluateCostEvent(event);
createEvent: async (companyId: string, data: Omit<typeof costEvents.$inferInsert, "companyId" | "receiptHash" | "costCents"> & { costCents: MoneyInput }, actor?: Pick<LogActivityInput, "actorType" | "actorId" | "agentId">) => {
if (data.idempotencyKey?.startsWith("heartbeat:")) throw unprocessable("The heartbeat idempotency namespace is reserved for run receipts");
const event = await withAccountingTransaction(db, companyId, (tx, publications) => createCostEventInTransaction(tx, companyId, data, publications, actor));
await budgets.deliverPendingEnforcement(companyId);
return event;
},
summary: async (companyId: string, range?: CostDateRange) => {
summary: async (companyId: string, range?: CostDateRange) => withAccountingReadSnapshot(db, companyId, async (db) => {
const company = await db
.select()
.from(companies)
@@ -112,17 +171,28 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
if (!company) throw notFound("Company not found");
range = resolveCostDateRange(range);
const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];
if (range?.from) conditions.push(gte(costEvents.occurredAt, range.from));
if (range?.to) conditions.push(lte(costEvents.occurredAt, range.to));
const [{ total }] = await db
const [{ total, totalExact, eventCount, unpricedEventCount, estimatedEventCount }] = await db
.select({
eventCount: sql<number>`count(*)::int`,
estimatedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'estimated')::int`,
unpricedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'unpriced' and ${costEvents.billingType} <> 'subscription_included')::int`,
total: sumAsNumber(costEvents.costCents),
totalExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
})
.from(costEvents)
.where(and(...conditions));
const pendingConditions = [eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.costAccountingPending, true),
inArray(heartbeatRuns.status, ["succeeded", "failed", "timed_out", "cancelled", "interrupted"])];
if (range?.from) pendingConditions.push(sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) >= ${range.from.toISOString()}::timestamptz`);
if (range?.to) pendingConditions.push(sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) <= ${range.to.toISOString()}::timestamptz`);
const [pending] = await db.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns).where(and(...pendingConditions));
const pendingRunCount = pending?.count ?? 0;
const spendCents = Number(total);
const utilization =
company.budgetMonthlyCents > 0
@@ -131,11 +201,17 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
return {
companyId,
eventCount,
unpricedEventCount,
estimatedEventCount,
pendingRunCount,
pricingComplete: unpricedEventCount === 0 && pendingRunCount === 0,
spendCents,
spendCentsExact: normalizeCents(totalExact),
budgetCents: company.budgetMonthlyCents,
utilizationPercent: Number(utilization.toFixed(2)),
};
},
}),
issueTreeSummary: async (
companyId: string,
@@ -238,7 +314,8 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
.select({
issueCount: sql<number>`count(distinct ${issues.id})::int`,
costCents: sumAsNumber(costEvents.costCents),
inputTokens: sumAsNumber(costEvents.inputTokens),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
})
@@ -270,6 +347,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
issueCount: Number(costRow?.issueCount ?? 0),
includeDescendants: true,
costCents: Number(costRow?.costCents ?? 0),
costCentsExact: normalizeCents(costRow?.costCentsExact ?? 0),
inputTokens: Number(costRow?.inputTokens ?? 0),
cachedInputTokens: Number(costRow?.cachedInputTokens ?? 0),
outputTokens: Number(costRow?.outputTokens ?? 0),
@@ -278,7 +356,79 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
};
},
byUser: (companyId: string, range?: CostDateRange): Promise<CostByUserReport> => withAccountingReadSnapshot(db, companyId, async tx => {
// Keep membership, identity and spend in one non-blocking read snapshot.
const members = await tx.select({
userId: companyMemberships.principalId,
status: companyMemberships.status,
userName: authUsers.name,
userImage: authUsers.image,
}).from(companyMemberships)
.leftJoin(authUsers, eq(authUsers.id, companyMemberships.principalId))
.where(and(
eq(companyMemberships.companyId, companyId),
eq(companyMemberships.principalType, "user"),
ne(companyMemberships.principalId, "local-board"),
));
const activeMembers = members.filter(member => member.status === "active");
const activeUserCount = activeMembers.length;
const bounds = resolveCostDateRange(range);
const conditions = [eq(costEvents.companyId, companyId)];
if (bounds.from) conditions.push(gte(costEvents.occurredAt, bounds.from));
if (bounds.to) conditions.push(lte(costEvents.occurredAt, bounds.to));
const totals = await tx.select({
userId: companyMemberships.principalId,
eventCount: sql<number>`count(*)::int`,
estimatedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'estimated')::int`,
unpricedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'unpriced' and ${costEvents.billingType} <> 'subscription_included')::int`,
costCents: sumAsNumber(costEvents.costCents),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
runCount: sql<number>`count(distinct ${heartbeatRuns.id})::int`,
}).from(costEvents)
.leftJoin(heartbeatRuns, and(
eq(heartbeatRuns.id, costEvents.heartbeatRunId),
eq(heartbeatRuns.companyId, costEvents.companyId),
eq(heartbeatRuns.agentId, costEvents.agentId),
))
// Never resolve names through a foreign-company run or membership. Retain
// former members' historical spend; missing attribution remains in the total.
// The synthetic Board principal is not a person. Any historical charges
// attached to it remain in Unattributed so the report still reconciles.
.leftJoin(companyMemberships, and(
eq(companyMemberships.companyId, companyId),
eq(companyMemberships.principalType, "user"),
ne(companyMemberships.principalId, "local-board"),
eq(companyMemberships.principalId, heartbeatRuns.responsibleUserId),
))
.where(and(...conditions))
.groupBy(companyMemberships.principalId)
.orderBy(desc(sql`sum(${costEvents.costCents})`), companyMemberships.principalId);
const directory = new Map(members.map(member => [member.userId, member]));
const rows: CostByUserReport["rows"] = totals.map(row => ({
...row,
userName: row.userId ? directory.get(row.userId)?.userName ?? "Former user" : null,
userImage: row.userId ? directory.get(row.userId)?.userImage ?? null : null,
costCentsExact: normalizeCents(row.costCentsExact),
}));
const represented = new Set(rows.map(row => row.userId));
for (const member of activeMembers.sort((a, b) => (a.userName ?? a.userId).localeCompare(b.userName ?? b.userId))) {
if (represented.has(member.userId)) continue;
rows.push({
userId: member.userId, userName: member.userName ?? "Unknown user", userImage: member.userImage,
eventCount: 0, estimatedEventCount: 0, unpricedEventCount: 0,
costCents: 0, costCentsExact: "0.0000000",
inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, runCount: 0,
});
}
return { activeUserCount, rows };
}),
byAgent: async (companyId: string, range?: CostDateRange) => {
range = resolveCostDateRange(range);
const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];
if (range?.from) conditions.push(gte(costEvents.occurredAt, range.from));
if (range?.to) conditions.push(lte(costEvents.occurredAt, range.to));
@@ -289,8 +439,11 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
agentName: agents.name,
agentAppearance: agents.appearance,
agentStatus: agents.status,
eventCount: sql<number>`count(*)::int`,
estimatedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'estimated')::int`,
costCents: sumAsNumber(costEvents.costCents),
inputTokens: sumAsNumber(costEvents.inputTokens),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
apiRunCount:
@@ -300,7 +453,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
subscriptionCachedInputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.cachedInputTokens} else 0 end), 0)::double precision`,
subscriptionInputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.inputTokens} else 0 end), 0)::double precision`,
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${ordinaryInputTokens} else 0 end), 0)::double precision`,
subscriptionOutputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.outputTokens} else 0 end), 0)::double precision`,
})
@@ -316,6 +469,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
},
byProvider: async (companyId: string, range?: CostDateRange) => {
range = resolveCostDateRange(range);
const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];
if (range?.from) conditions.push(gte(costEvents.occurredAt, range.from));
if (range?.to) conditions.push(lte(costEvents.occurredAt, range.to));
@@ -327,7 +481,8 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
billingType: costEvents.billingType,
model: costEvents.model,
costCents: sumAsNumber(costEvents.costCents),
inputTokens: sumAsNumber(costEvents.inputTokens),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
apiRunCount:
@@ -337,7 +492,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
subscriptionCachedInputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.cachedInputTokens} else 0 end), 0)::double precision`,
subscriptionInputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.inputTokens} else 0 end), 0)::double precision`,
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${ordinaryInputTokens} else 0 end), 0)::double precision`,
subscriptionOutputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.outputTokens} else 0 end), 0)::double precision`,
})
@@ -348,6 +503,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
},
byBiller: async (companyId: string, range?: CostDateRange) => {
range = resolveCostDateRange(range);
const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];
if (range?.from) conditions.push(gte(costEvents.occurredAt, range.from));
if (range?.to) conditions.push(lte(costEvents.occurredAt, range.to));
@@ -356,7 +512,8 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
.select({
biller: costEvents.biller,
costCents: sumAsNumber(costEvents.costCents),
inputTokens: sumAsNumber(costEvents.inputTokens),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
apiRunCount:
@@ -366,7 +523,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
subscriptionCachedInputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.cachedInputTokens} else 0 end), 0)::double precision`,
subscriptionInputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.inputTokens} else 0 end), 0)::double precision`,
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${ordinaryInputTokens} else 0 end), 0)::double precision`,
subscriptionOutputTokens:
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.outputTokens} else 0 end), 0)::double precision`,
providerCount: sql<number>`count(distinct ${costEvents.provider})::int`,
@@ -390,15 +547,17 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
{ label: "7d", hours: 168 },
] as const;
const now = new Date();
const results = await Promise.all(
windows.map(async ({ label, hours }) => {
const since = new Date(Date.now() - hours * 60 * 60 * 1000);
const since = new Date(now.getTime() - hours * 60 * 60 * 1000);
const rows = await db
.select({
provider: costEvents.provider,
biller: sql<string>`case when count(distinct ${costEvents.biller}) = 1 then min(${costEvents.biller}) else 'mixed' end`,
costCents: sumAsNumber(costEvents.costCents),
inputTokens: sumAsNumber(costEvents.inputTokens),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
})
@@ -407,6 +566,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
and(
eq(costEvents.companyId, companyId),
gte(costEvents.occurredAt, since),
lte(costEvents.occurredAt, now),
),
)
.groupBy(costEvents.provider)
@@ -418,6 +578,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
window: label as string,
windowHours: hours,
costCents: row.costCents,
costCentsExact: normalizeCents(row.costCentsExact),
inputTokens: row.inputTokens,
cachedInputTokens: row.cachedInputTokens,
outputTokens: row.outputTokens,
@@ -429,6 +590,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
},
byAgentModel: async (companyId: string, range?: CostDateRange) => {
range = resolveCostDateRange(range);
const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];
if (range?.from) conditions.push(gte(costEvents.occurredAt, range.from));
if (range?.to) conditions.push(lte(costEvents.occurredAt, range.to));
@@ -446,8 +608,11 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
biller: costEvents.biller,
billingType: costEvents.billingType,
model: costEvents.model,
eventCount: sql<number>`count(*)::int`,
estimatedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'estimated')::int`,
costCents: sumAsNumber(costEvents.costCents),
inputTokens: sumAsNumber(costEvents.inputTokens),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
})
@@ -473,9 +638,9 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
byProject: async (companyId: string, range?: CostDateRange) => {
const issueIdAsText = sql<string>`${issues.id}::text`;
const runProjectLinks = db
.selectDistinctOn([activityLog.runId, issues.projectId], {
.select({
runId: activityLog.runId,
projectId: issues.projectId,
projectId: sql<string>`min(${issues.projectId}::text)::uuid`.as("linked_project_id"),
})
.from(activityLog)
.innerJoin(
@@ -493,10 +658,12 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
isNotNull(issues.projectId),
),
)
.orderBy(activityLog.runId, issues.projectId, desc(activityLog.createdAt))
.groupBy(activityLog.runId)
.having(sql`count(distinct ${issues.projectId}) = 1`)
.as("run_project_links");
const effectiveProjectId = sql<string | null>`coalesce(${costEvents.projectId}, ${runProjectLinks.projectId})`;
const effectiveProjectId = sql<string | null>`coalesce(${costEvents.projectId}, case when ${costEvents.receiptHash} is null then ${runProjectLinks.projectId} end)`;
range = resolveCostDateRange(range);
const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];
if (range?.from) conditions.push(gte(costEvents.occurredAt, range.from));
if (range?.to) conditions.push(lte(costEvents.occurredAt, range.to));
@@ -505,18 +672,19 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
return db
.select({
projectId: effectiveProjectId,
projectId: projects.id,
projectName: projects.name,
costCents: costCentsExpr,
inputTokens: sumAsNumber(costEvents.inputTokens),
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
inputTokens: sumAsNumber(ordinaryInputTokens),
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
outputTokens: sumAsNumber(costEvents.outputTokens),
})
.from(costEvents)
.leftJoin(runProjectLinks, eq(costEvents.heartbeatRunId, runProjectLinks.runId))
.innerJoin(projects, sql`${projects.id} = ${effectiveProjectId}`)
.where(and(...conditions, sql`${effectiveProjectId} is not null`))
.groupBy(effectiveProjectId, projects.name)
.leftJoin(projects, and(eq(projects.companyId, companyId), sql`${projects.id} = ${effectiveProjectId}`))
.where(and(...conditions))
.groupBy(projects.id, projects.name)
.orderBy(desc(costCentsExpr));
},
};
server/src/services/dashboard.ts
+2 -1
View File
@@ -1,4 +1,4 @@
import { and, eq, gte, sql } from "drizzle-orm";
import { and, eq, gte, lte, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { agents, approvals, companies, costEvents, heartbeatRuns, issues } from "@paperclipai/db";
import { notFound } from "../errors.js";
@@ -96,6 +96,7 @@ export function dashboardService(db: Db) {
and(
eq(costEvents.companyId, companyId),
gte(costEvents.occurredAt, monthStart),
lte(costEvents.occurredAt, now),
),
));
server/src/services/email-channels.ts
+4 -3
View File
@@ -1,7 +1,7 @@
import { HttpError } from "../errors.js";
import { createHash, randomUUID } from "node:crypto";
import WebSocket from "ws";
import { and, asc, eq, inArray, isNull, ne, sql } from "drizzle-orm";
import { and, asc, eq, getTableColumns, inArray, isNull, ne, sql } from "drizzle-orm";
import {
type Db,
agents,
@@ -27,6 +27,7 @@ import {
companyMemberships,
instanceUserRoles,
} from "@paperclipai/db";
import { compareCents } from "@paperclipai/shared";
import type {
AgentPermissions,
EmailEndpointSetupInput,
@@ -522,7 +523,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
);
await authorizeRead(endpoint.companyId, issueId, actor);
const [agent] = await db
.select()
.select({ ...getTableColumns(agents), spentMonthlyCentsExact: sql<string>`${agents.spentMonthlyCents}::text` })
.from(agents)
.where(
and(
@@ -534,7 +535,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
!agent ||
["paused", "terminated", "pending_approval"].includes(agent.status) ||
(agent.budgetMonthlyCents > 0 &&
agent.spentMonthlyCents >= agent.budgetMonthlyCents)
compareCents(agent.spentMonthlyCentsExact, agent.budgetMonthlyCents) >= 0)
)
throw forbidden("Agent is not available to send email");
if (!actor.runId)
server/src/services/finance.ts
+84 -35
View File
@@ -1,12 +1,14 @@
import { and, desc, eq, gte, lte, sql } from "drizzle-orm";
import { and, desc, eq, getTableColumns, gte, lte, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { agents, costEvents, financeEvents, goals, heartbeatRuns, issues, projects } from "@paperclipai/db";
import { notFound, unprocessable } from "../errors.js";
import { notFound, unprocessable, conflict } from "../errors.js";
export interface FinanceDateRange {
from?: Date;
to?: Date;
}
import { resolveCostDateRange, type CostDateRange as FinanceDateRange } from "./cost-date-range.js";
import { createFinanceEventSchema, normalizeCents, subtractCents, type MoneyInput } from "@paperclipai/shared";
import { receiptFingerprint } from "./receipt-fingerprint.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js";
export type { CostDateRange as FinanceDateRange } from "./cost-date-range.js";
async function assertBelongsToCompany(
db: Db,
@@ -28,19 +30,29 @@ async function assertBelongsToCompany(
}
function rangeConditions(companyId: string, range?: FinanceDateRange) {
range = resolveCostDateRange(range);
const conditions: ReturnType<typeof eq>[] = [eq(financeEvents.companyId, companyId)];
if (range?.from) conditions.push(gte(financeEvents.occurredAt, range.from));
if (range?.to) conditions.push(lte(financeEvents.occurredAt, range.to));
return conditions;
}
export function financeService(db: Db) {
const debitExpr = sql<number>`coalesce(sum(case when ${financeEvents.direction} = 'debit' then ${financeEvents.amountCents} else 0 end), 0)::double precision`;
const creditExpr = sql<number>`coalesce(sum(case when ${financeEvents.direction} = 'credit' then ${financeEvents.amountCents} else 0 end), 0)::double precision`;
const estimatedDebitExpr = sql<number>`coalesce(sum(case when ${financeEvents.direction} = 'debit' and ${financeEvents.estimated} = true then ${financeEvents.amountCents} else 0 end), 0)::double precision`;
return {
createEvent: async (companyId: string, data: Omit<typeof financeEvents.$inferInsert, "companyId">) => {
export async function createFinanceEventInTransaction(tx: Db, publications: ActivityPublication[], companyId: string, data: Omit<typeof financeEvents.$inferInsert, "companyId" | "receiptHash" | "amountCents"> & { amountCents: MoneyInput }, actor?: Pick<LogActivityInput, "actorType" | "actorId" | "agentId">) {
const db = tx;
const parsed = createFinanceEventSchema.safeParse({ ...data, occurredAt: data.occurredAt.toISOString() });
if (!parsed.success) throw unprocessable("Invalid finance receipt", parsed.error.flatten());
const values = { ...parsed.data, amountCents: normalizeCents(parsed.data.amountCents), occurredAt: new Date(parsed.data.occurredAt) };
const receiptHash = receiptFingerprint(values);
let legacyHash: string | null = null;
try { if (normalizeCents(Number(values.amountCents)) === values.amountCents) legacyHash = receiptFingerprint({ ...values, amountCents: Number(values.amountCents) }); } catch { /* Decimal exceeds the legacy number range. */ }
if (values.idempotencyKey) {
const [existing] = await db.select().from(financeEvents).where(and(eq(financeEvents.companyId, companyId), eq(financeEvents.idempotencyKey, values.idempotencyKey)));
if (existing) {
if (existing.receiptHash !== receiptHash && existing.receiptHash !== legacyHash) throw conflict("Idempotency key already used for a different finance receipt");
const [exact] = await db.select({ amount: sql<string>`${financeEvents.amountCents}::text` }).from(financeEvents).where(eq(financeEvents.id, existing.id));
return { ...existing, amountCentsExact: normalizeCents(exact.amount) };
}
}
if (data.agentId) await assertBelongsToCompany(db, agents, data.agentId, companyId, "Agent");
if (data.issueId) await assertBelongsToCompany(db, issues, data.issueId, companyId, "Issue");
if (data.projectId) await assertBelongsToCompany(db, projects, data.projectId, companyId, "Project");
@@ -51,35 +63,60 @@ export function financeService(db: Db) {
const event = await db
.insert(financeEvents)
.values({
...data,
companyId,
currency: data.currency ?? "USD",
direction: data.direction ?? "debit",
estimated: data.estimated ?? false,
...values, amountCents: sql`${values.amountCents}::numeric`, companyId, receiptHash, id: data.id,
})
.returning()
.then((rows) => rows[0]);
return event;
},
await logActivity(db, {
companyId, actorType: actor?.actorType ?? "system", actorId: actor?.actorId ?? "finance_accounting", agentId: actor?.agentId,
action: "finance_event.reported", entityType: "finance_event", entityId: event.id,
details: { amountCents: event.amountCents, amountCentsExact: values.amountCents, currency: event.currency, biller: event.biller, eventKind: event.eventKind, direction: event.direction },
}, publications);
return { ...event, amountCentsExact: values.amountCents };
}
export function financeService(db: Db) {
const debitExpr = sql<string>`coalesce(sum(case when ${financeEvents.direction} = 'debit' and ${financeEvents.metadataJson}->>'source' is distinct from 'provider_cost_report' then ${financeEvents.amountCents} else 0 end), 0)`;
const creditExpr = sql<string>`coalesce(sum(case when ${financeEvents.direction} = 'credit' and ${financeEvents.metadataJson}->>'source' is distinct from 'provider_cost_report' then ${financeEvents.amountCents} else 0 end), 0)`;
const estimatedDebitExpr = sql<string>`coalesce(sum(case when ${financeEvents.direction} = 'debit' and ${financeEvents.estimated} = true and ${financeEvents.metadataJson}->>'source' is distinct from 'provider_cost_report' then ${financeEvents.amountCents} else 0 end), 0)`;
return {
createEvent: async (companyId: string, data: Omit<typeof financeEvents.$inferInsert, "companyId" | "receiptHash" | "amountCents"> & { amountCents: MoneyInput }, actor?: Pick<LogActivityInput, "actorType" | "actorId" | "agentId">) => withAccountingTransaction(db, companyId, (tx, publications) => createFinanceEventInTransaction(tx, publications, companyId, data, actor)),
summary: async (companyId: string, range?: FinanceDateRange) => {
const conditions = rangeConditions(companyId, range);
const [row] = await db
const rows = await db
.select({
debitCents: debitExpr,
creditCents: creditExpr,
estimatedDebitCents: estimatedDebitExpr,
currency: financeEvents.currency,
providerReportedCentsExact: sql<string>`coalesce(sum(case when ${financeEvents.metadataJson}->>'source' = 'provider_cost_report' then case when ${financeEvents.direction} = 'credit' then -${financeEvents.amountCents} else ${financeEvents.amountCents} end else 0 end), 0)::text`,
debitCents: sql<number>`${debitExpr}::double precision`,
debitCentsExact: sql<string>`${debitExpr}::text`,
creditCents: sql<number>`${creditExpr}::double precision`,
creditCentsExact: sql<string>`${creditExpr}::text`,
estimatedDebitCents: sql<number>`${estimatedDebitExpr}::double precision`,
estimatedDebitCentsExact: sql<string>`${estimatedDebitExpr}::text`,
eventCount: sql<number>`count(*)::int`,
})
.from(financeEvents)
.where(and(...conditions));
.where(and(...conditions))
.groupBy(financeEvents.currency)
.orderBy(financeEvents.currency);
const row = rows.find((row) => row.currency === "USD");
return {
companyId,
currency: "USD" as const,
providerReportedCentsExact: normalizeCents(row?.providerReportedCentsExact ?? 0),
providerReportedCents: Number(row?.providerReportedCentsExact ?? 0),
currencies: rows.map((row) => ({ ...row, netCents: Number(subtractCents(row.debitCentsExact, row.creditCentsExact)), netCentsExact: subtractCents(row.debitCentsExact, row.creditCentsExact) })),
debitCents: Number(row?.debitCents ?? 0),
creditCents: Number(row?.creditCents ?? 0),
netCents: Number(row?.debitCents ?? 0) - Number(row?.creditCents ?? 0),
netCents: Number(subtractCents(row?.debitCentsExact ?? 0, row?.creditCentsExact ?? 0)),
netCentsExact: subtractCents(row?.debitCentsExact ?? 0, row?.creditCentsExact ?? 0),
debitCentsExact: normalizeCents(row?.debitCentsExact ?? 0),
creditCentsExact: normalizeCents(row?.creditCentsExact ?? 0),
estimatedDebitCentsExact: normalizeCents(row?.estimatedDebitCentsExact ?? 0),
estimatedDebitCents: Number(row?.estimatedDebitCents ?? 0),
eventCount: Number(row?.eventCount ?? 0),
};
@@ -87,44 +124,56 @@ export function financeService(db: Db) {
byBiller: async (companyId: string, range?: FinanceDateRange) => {
const conditions = rangeConditions(companyId, range);
conditions.push(sql`${financeEvents.metadataJson}->>'source' is distinct from 'provider_cost_report'`);
return db
.select({
currency: financeEvents.currency,
biller: financeEvents.biller,
debitCents: debitExpr,
creditCents: creditExpr,
estimatedDebitCents: estimatedDebitExpr,
debitCents: sql<number>`${debitExpr}::double precision`,
debitCentsExact: sql<string>`${debitExpr}::text`,
creditCents: sql<number>`${creditExpr}::double precision`,
creditCentsExact: sql<string>`${creditExpr}::text`,
estimatedDebitCents: sql<number>`${estimatedDebitExpr}::double precision`,
estimatedDebitCentsExact: sql<string>`${estimatedDebitExpr}::text`,
eventCount: sql<number>`count(*)::int`,
kindCount: sql<number>`count(distinct ${financeEvents.eventKind})::int`,
netCents: sql<number>`(${debitExpr} - ${creditExpr})::double precision`,
netCentsExact: sql<string>`(${debitExpr} - ${creditExpr})::text`,
})
.from(financeEvents)
.where(and(...conditions))
.groupBy(financeEvents.biller)
.groupBy(financeEvents.biller, financeEvents.currency)
.orderBy(desc(sql`(${debitExpr} - ${creditExpr})::double precision`), financeEvents.biller);
},
byKind: async (companyId: string, range?: FinanceDateRange) => {
const conditions = rangeConditions(companyId, range);
conditions.push(sql`${financeEvents.metadataJson}->>'source' is distinct from 'provider_cost_report'`);
return db
.select({
currency: financeEvents.currency,
eventKind: financeEvents.eventKind,
debitCents: debitExpr,
creditCents: creditExpr,
estimatedDebitCents: estimatedDebitExpr,
debitCents: sql<number>`${debitExpr}::double precision`,
debitCentsExact: sql<string>`${debitExpr}::text`,
creditCents: sql<number>`${creditExpr}::double precision`,
creditCentsExact: sql<string>`${creditExpr}::text`,
estimatedDebitCents: sql<number>`${estimatedDebitExpr}::double precision`,
estimatedDebitCentsExact: sql<string>`${estimatedDebitExpr}::text`,
eventCount: sql<number>`count(*)::int`,
billerCount: sql<number>`count(distinct ${financeEvents.biller})::int`,
netCents: sql<number>`(${debitExpr} - ${creditExpr})::double precision`,
netCentsExact: sql<string>`(${debitExpr} - ${creditExpr})::text`,
})
.from(financeEvents)
.where(and(...conditions))
.groupBy(financeEvents.eventKind)
.groupBy(financeEvents.eventKind, financeEvents.currency)
.orderBy(desc(sql`(${debitExpr} - ${creditExpr})::double precision`), financeEvents.eventKind);
},
list: async (companyId: string, range?: FinanceDateRange, limit: number = 100) => {
const conditions = rangeConditions(companyId, range);
return db
.select()
.select({ ...getTableColumns(financeEvents), amountCentsExact: sql<string>`${financeEvents.amountCents}::text` })
.from(financeEvents)
.where(and(...conditions))
.orderBy(desc(financeEvents.occurredAt), desc(financeEvents.createdAt))
server/src/services/heartbeat.ts
+263 -229
View File
@@ -22,8 +22,11 @@ import {
type NativeWorkspaceFinalizationOwnership,
} from "./native-runtime/native-workspace-finalization-ownership.js";
import { hasStopOnlyCleanup, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js";
import { reserveRunBudget } from "./budget-reservations.js";
import { accountRunCost, createCostAccountingReconciler } from "./run-cost-accounting.js";
import { createRunUsageRecorder } from "./usage-receipts.js";
import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { compareCents, hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { externalConversationStateSql, nonIdleSlackIssueCondition } from "./slack-conversation-state.js";
import { settleSlackConversation } from "./slack-conversation-lifecycle.js";
import { publicChatTaskUrl } from "./chat-task-url.js";
@@ -90,7 +93,7 @@ import {
} from "@paperclipai/adapter-utils/execution-target";
import { agentService } from "./agents.js";
import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "./agent-instruction-working-copies.js";
import { normalizeLegacyRunnerProvider } from "@paperclipai/adapter-utils";
import { normalizeLegacyRunnerProvider, resolveManagedOpenAiBilling } from "@paperclipai/adapter-utils";
import fs from "node:fs/promises";
import path from "node:path";
import { execFile as execFileCallback } from "node:child_process";
@@ -318,7 +321,6 @@ import {
appendWithByteCap,
MAX_EXCERPT_BYTES,
} from "../adapters/utils.js";
import { costService } from "./costs.js";
import {
authorizeChatConversationForBoundRun,
isExternalChatWaitAuthorizationContention,
@@ -350,7 +352,7 @@ import {
import { reportRunFailure } from "./run-failure-report.js";
import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";
import { companySkillService } from "./company-skills.js";
import { budgetService, type BudgetEnforcementScope } from "./budgets.js";
import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js";
import { secretService, type MissingRuntimeBinding } from "./secrets.js";
import {
resolveDefaultAgentWorkspaceDir,
@@ -691,6 +693,7 @@ const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 50;
const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [
"environment.lease_acquired",
"environment.lease_released",
"cost.reported",
];
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
const EXTERNAL_ATTACHMENT_OMISSIONS_KEY = "externalAttachmentOmissions";
@@ -5344,21 +5347,23 @@ function resolveLedgerBiller(result: AdapterExecutionResult): string {
);
}
function normalizeBilledCostCents(
export function normalizeBilledCostCents(
costUsd: number | null | undefined,
billingType: BillingType,
): number {
if (billingType === "subscription_included") return 0;
if (typeof costUsd !== "number" || !Number.isFinite(costUsd)) return 0;
return Math.max(0, Math.round(costUsd * 100));
return Math.max(0, Number((costUsd * 100).toFixed(7)));
}
export function resolveLedgerCostStatus(input: {
costUsd: number | null | undefined;
billingType?: BillingType;
inputTokens: number;
cachedInputTokens: number;
outputTokens: number;
}): CostStatus {
if (input.billingType === "subscription_included") return "reported";
// A paused turn can have neither a token receipt nor a cost receipt. Zero
// normalized counters do not establish that its billed cost was zero.
return typeof input.costUsd === "number" &&
@@ -5564,12 +5569,13 @@ function readRawUsageTotals(usageJson: unknown): UsageTotals | null {
};
}
function deriveNormalizedUsageDelta(
export function normalizeAdapterRunUsage(
current: UsageTotals | null,
previous: UsageTotals | null,
usageBasis?: "per_run" | "session_cumulative" | null,
): UsageTotals | null {
if (!current) return null;
if (!previous) return { ...current };
if (!previous || usageBasis !== "session_cumulative") return { ...current };
const inputTokens =
current.inputTokens >= previous.inputTokens
@@ -12172,7 +12178,7 @@ export function heartbeatService(
// Adapters that declare per-run usage (e.g. the ACPX lane reports each
// turn's tokens, not session totals) must not be session-delta'd, or
// consecutive runs would be undercounted.
if (!sessionId || !rawUsage || usageBasis === "per_run") {
if (!sessionId || !rawUsage || usageBasis !== "session_cumulative") {
return {
normalizedUsage: rawUsage,
previousRawUsage: null as UsageTotals | null,
@@ -12185,7 +12191,7 @@ export function heartbeatService(
});
const previousRawUsage = readRawUsageTotals(previousRun?.usageJson);
return {
normalizedUsage: deriveNormalizedUsageDelta(rawUsage, previousRawUsage),
normalizedUsage: normalizeAdapterRunUsage(rawUsage, previousRawUsage, usageBasis),
previousRawUsage,
derivedFromSessionTotals: previousRawUsage !== null,
};
@@ -12949,6 +12955,10 @@ export function heartbeatService(
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null);
// Preserve the receipt-source fence when finalization enriches usage. A
// late spool replay must still be able to complete an unfinished receipt.
if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };
// Cancelling a queued run that never acquired provider execution is
// positive bootstrap evidence. It must not hold unrelated queued messages.
if (previousStatus && (status === "cancelled" || status === "interrupted")) {
@@ -13043,6 +13053,10 @@ export function heartbeatService(
patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };
}
// Preserve the receipt-source fence when finalization enriches usage. A
// late spool replay must still be able to complete an unfinished receipt.
if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };
// Cancelling a queued run that never acquired provider execution is
// positive bootstrap evidence. It must not hold unrelated queued messages.
if (
@@ -16428,9 +16442,8 @@ export function heartbeatService(
.then((rows) => rows[0] ?? null);
}
// No code path raises the `ai_connection_busy` error any more. This
// function stays because a stored run row can still carry that error code
// from an earlier release.
// Credential rotation can briefly contend with a fresh runtime read. Keep
// the task on its automatic pre-provider retry path while the lock clears.
async function finalizeAiConnectionBusyDeferral(
run: typeof heartbeatRuns.$inferSelect,
error: HttpError,
@@ -16952,7 +16965,7 @@ export function heartbeatService(
if (checkCostCap && policy.maxDailyCostCents !== null) {
const [row] = await client
.select({
total: sql<number>`coalesce(sum(${costEvents.costCents})::bigint, 0)`,
total: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
})
.from(costEvents)
.where(
@@ -16964,7 +16977,7 @@ export function heartbeatService(
),
);
const observed = Number(row?.total ?? 0);
if (observed >= policy.maxDailyCostCents) {
if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) {
return {
reason: "heartbeat.daily_cost_limit",
observed,
@@ -20030,32 +20043,7 @@ export function heartbeatService(
normalizedUsage?: UsageTotals | null,
) {
await ensureRuntimeState(agent);
const usage = normalizedUsage ?? normalizeUsageTotals(result.usage);
const inputTokens = usage?.inputTokens ?? 0;
const outputTokens = usage?.outputTokens ?? 0;
const cachedInputTokens = usage?.cachedInputTokens ?? 0;
const billingType = normalizeLedgerBillingType(result.billingType);
const billedCostUsd = resolveCacheAdjustedCostUsd(result);
const additionalCostCents = normalizeBilledCostCents(
billedCostUsd,
billingType,
);
const hasTokenUsage =
inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0;
const costStatus = resolveLedgerCostStatus({
costUsd: billedCostUsd,
inputTokens,
cachedInputTokens,
outputTokens,
});
const provider = result.provider ?? "unknown";
const biller = resolveLedgerBiller(result);
const ledgerScope = await resolveLedgerScopeForRun(
db,
agent.companyId,
run,
);
await accountRunCost(db, run.id, budgetHooks);
await db
.update(agentRuntimeState)
.set({
@@ -20064,34 +20052,10 @@ export function heartbeatService(
lastRunId: run.id,
lastRunStatus: run.status,
lastError: run.error ?? null,
totalInputTokens: sql`${agentRuntimeState.totalInputTokens} + ${inputTokens}`,
totalOutputTokens: sql`${agentRuntimeState.totalOutputTokens} + ${outputTokens}`,
totalCachedInputTokens: sql`${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`,
totalCostCents: sql`${agentRuntimeState.totalCostCents} + ${additionalCostCents}`,
updatedAt: new Date(),
})
.where(eq(agentRuntimeState.agentId, agent.id));
if (additionalCostCents > 0 || hasTokenUsage) {
const costs = costService(db, budgetHooks);
await costs.createEvent(agent.companyId, {
heartbeatRunId: run.id,
agentId: agent.id,
issueId: ledgerScope.issueId,
projectId: ledgerScope.projectId,
billingCode: ledgerScope.billingCode,
provider,
biller,
billingType,
costStatus,
model: result.model ?? "unknown",
inputTokens,
cachedInputTokens,
outputTokens,
costCents: additionalCostCents,
occurredAt: new Date(),
});
}
}
// A 403 from claimQueuedRun comes from the run's own persisted identity
@@ -21126,6 +21090,13 @@ export function heartbeatService(
issueContext.executionWorkspacePreference,
}
: null;
const storedLedgerScope = parseObject(parseObject(run.usageJson).ledgerScope);
const runLedgerScope = Object.keys(storedLedgerScope).length > 0
? storedLedgerScope
: await resolveLedgerScopeForRun(db, agent.companyId, run);
await db.update(heartbeatRuns).set({
usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ ledgerScope: runLedgerScope })}::jsonb`,
}).where(eq(heartbeatRuns.id, run.id));
const continuationSummary = issueRef && !isConversation(issueContext)
? await getIssueContinuationSummaryDocument(db, issueRef.id)
: null;
@@ -22907,7 +22878,7 @@ export function heartbeatService(
},
);
}
const dispatchResolvedInteractionContinuationWithAtomicGate = async <T>(
const dispatchResolvedInteractionContinuationAfterAdmission = async <T>(
dispatch: (markDispatchStarted: () => void) => Promise<T>,
): Promise<
{ dispatched: true; resultPromise: Promise<T> } | { dispatched: false }
@@ -22973,6 +22944,30 @@ export function heartbeatService(
}
return { dispatched: false };
};
const dispatchResolvedInteractionContinuationWithAtomicGate = async <T>(
dispatch: (markDispatchStarted: () => void) => Promise<T>,
) => {
// Admission can wait for accounting locks. Finish that wait before the
// final ownership gate, whose callback must enter the adapter directly.
const reservation = await reserveRunBudget(db, run.companyId, run.id,
readNonEmptyString(runLedgerScope.projectId), runLedgerScope, runOptions.nativeLeaseOwner);
let entered = false;
try {
return await dispatchResolvedInteractionContinuationAfterAdmission((markDispatchStarted) => {
entered = true;
return dispatch(markDispatchStarted);
});
} finally {
if (!entered && !reservation.reused) {
await db.update(heartbeatRuns).set({ costAccountingPending: true,
usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`,
}).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt)));
await accountRunCost(db, run.id, budgetHooks).catch((err) => {
logger.warn({ err, runId: run.id }, "Undispatched reservation release remains pending");
});
}
}
};
if (!executionTarget || executionTarget.kind === "local") {
try {
runScratch = await prepareHeartbeatRunScratch({
@@ -24800,6 +24795,7 @@ export function heartbeatService(
adapterFinalizeOutcome = status;
};
const usageRecorder = await createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType });
let adapterResult: AdapterExecutionResult;
const runGoalControlRequestId = readNonEmptyString(
context.goalControlRequestId,
@@ -24882,8 +24878,8 @@ export function heartbeatService(
);
const guardedDispatch =
await dispatchResolvedInteractionContinuationWithAtomicGate(
(markDispatchStarted) =>
executePaperclipNativeSession({
(markDispatchStarted) => {
return executePaperclipNativeSession({
db,
execution: nativeExecution,
getFreshSessionHandoff: getNativeFreshSessionHandoff,
@@ -24940,11 +24936,14 @@ export function heartbeatService(
hasChanges: () => instructionCopies.hasChanges({ companyId: agent.companyId, runId: run.id, target: executionTarget }),
collectStopped: collectStoppedInstructions,
} : undefined,
onUsage: async receipt => { await usageRecorder.capture(receipt); },
preparationSpans: nativeRunnerPreparationSpans,
// Bootstrap with executable/home discovery while keeping
// configured provider values and the server-selected
// workspace boundary authoritative.
managedGitHub: !useHostGitHub && githubSelection.configured,
billingIdentity: managedAiRuntime ? { provider: managedAiRuntime.attribution.provider, biller: managedAiRuntime.attribution.provider === "openai" ? resolveManagedOpenAiBilling(managedAiRuntime.config.managedAiRouting)?.biller ?? managedAiRuntime.attribution.provider : managedAiRuntime.attribution.provider, billingType: managedAiRuntime.attribution.method === "subscription" ? "subscription_included" : "metered_api" } : undefined,
managedAiCredentialIdentity: managedAiRuntime?.identity,
managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record<string, unknown>).CODEX_HOME) : undefined,
runnerEnvironment: {
@@ -25017,7 +25016,8 @@ export function heartbeatService(
markDispatchStarted();
await persistRunProcessMetadata(run.id, meta);
},
}),
});
},
);
if (!guardedDispatch.dispatched) return;
nativeDispatchStarted = true;
@@ -25133,6 +25133,7 @@ export function heartbeatService(
onLog,
onMeta: onAdapterMeta,
onEvent: onAdapterEvent,
onUsage: async receipt => { await usageRecorder.capture(receipt); },
onExecutionPhase: executionControl.phases.enter,
startupTraceContext: getStartupTraceContext(),
onRuntimeProgress: async (progress) => {
@@ -25197,6 +25198,17 @@ export function heartbeatService(
if (tail) await appendIdentityRedactedLog(stream, tail);
}
if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave };
if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) {
const captured = await usageRecorder.complete(adapterResult);
adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
} else {
// Stop may already own the terminal result. Preserve its metadata
// while durably recording the proof needed to release admission.
await db.update(heartbeatRuns).set({ costAccountingPending: true,
usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`,
}).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt)));
}
adapterResult = applyWorkspaceRestoreFailure(adapterResult);
// A returned result can include a failed restore. Keep the workspace
// barrier closed until required files have been restored.
@@ -25637,65 +25649,67 @@ export function heartbeatService(
? "timed_out"
: "failed";
const cacheAdjustedCostUsd = resolveCacheAdjustedCostUsd(adapterResult);
const usageJson =
normalizedUsage ||
adapterResult.costUsd != null ||
cacheAdjustedCostUsd != null
? ({
...(normalizedUsage ?? {}),
...(rawUsage
? {
rawInputTokens: rawUsage.inputTokens,
rawInputIncludesCached: false,
rawCachedInputTokens: rawUsage.cachedInputTokens,
rawOutputTokens: rawUsage.outputTokens,
}
: {}),
...(sessionUsageResolution.derivedFromSessionTotals
? { usageSource: "session_delta" }
: adapterResult.usageBasis === "per_run"
? { usageSource: "per_run" }
: {}),
...((nextSessionState.displayId ??
nextSessionState.legacySessionId)
? {
persistedSessionId:
nextSessionState.displayId ??
nextSessionState.legacySessionId,
}
: {}),
sessionReused:
runtimeForAdapter.sessionId != null ||
runtimeForAdapter.sessionDisplayId != null,
taskSessionReused: taskSessionForRun != null,
freshSession:
runtimeForAdapter.sessionId == null &&
runtimeForAdapter.sessionDisplayId == null,
sessionRotated: sessionCompaction.rotate,
sessionRotationReason: sessionCompaction.reason,
configFreshness: configFreshnessResultMetadata,
provider:
readNonEmptyString(adapterResult.provider) ?? "unknown",
biller: resolveLedgerBiller(adapterResult),
model: readNonEmptyString(adapterResult.model) ?? "unknown",
...(adapterResult.costUsd != null
? { costUsd: adapterResult.costUsd }
: {}),
...(cacheAdjustedCostUsd != null
? { cacheAdjustedCostUsd }
: {}),
costStatus: resolveLedgerCostStatus({
costUsd: cacheAdjustedCostUsd,
inputTokens: normalizedUsage?.inputTokens ?? 0,
cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0,
outputTokens: normalizedUsage?.outputTokens ?? 0,
}),
billingType: normalizeLedgerBillingType(
adapterResult.billingType,
),
} as Record<string, unknown>)
: null;
const cacheAdjustedCostUsd = adapterResult.costUsdExact != null && adapterResult.cacheAdjustedCostUsd == null
? null : resolveCacheAdjustedCostUsd(adapterResult);
const usageJson: Record<string, unknown> = {
accountingReceiptReady: adapterResult.usageComplete !== false,
costUsdExact: adapterResult.costUsdExact ?? null,
providerRequestId: adapterResult.providerRequestId ?? null,
...(normalizedUsage ?? {}),
...(adapterResult.usageByModel ? { usageByModel: adapterResult.usageByModel } : {}),
...(rawUsage
? {
rawInputTokens: rawUsage.inputTokens,
rawInputIncludesCached: false,
rawCachedInputTokens: rawUsage.cachedInputTokens,
rawOutputTokens: rawUsage.outputTokens,
}
: {}),
...(sessionUsageResolution.derivedFromSessionTotals
? { usageSource: "session_delta" }
: adapterResult.usageBasis === "per_run"
? { usageSource: "per_run" }
: {}),
...((nextSessionState.displayId ??
nextSessionState.legacySessionId)
? {
persistedSessionId:
nextSessionState.displayId ??
nextSessionState.legacySessionId,
}
: {}),
sessionReused:
runtimeForAdapter.sessionId != null ||
runtimeForAdapter.sessionDisplayId != null,
taskSessionReused: taskSessionForRun != null,
freshSession:
runtimeForAdapter.sessionId == null &&
runtimeForAdapter.sessionDisplayId == null,
sessionRotated: sessionCompaction.rotate,
sessionRotationReason: sessionCompaction.reason,
configFreshness: configFreshnessResultMetadata,
provider:
readNonEmptyString(adapterResult.provider) ?? "unknown",
biller: resolveLedgerBiller(adapterResult),
model: readNonEmptyString(adapterResult.model) ?? "unknown",
...(adapterResult.costUsd != null
? { costUsd: adapterResult.costUsd }
: {}),
...(cacheAdjustedCostUsd != null
? { cacheAdjustedCostUsd }
: {}),
pricingProvenance: adapterResult.pricingProvenance,
costStatus: adapterResult.costStatus ?? resolveLedgerCostStatus({
costUsd: cacheAdjustedCostUsd ?? (adapterResult.costUsdExact != null ? Number(adapterResult.costUsdExact) : null),
billingType: normalizeLedgerBillingType(adapterResult.billingType),
inputTokens: normalizedUsage?.inputTokens ?? 0,
cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0,
outputTokens: normalizedUsage?.outputTokens ?? 0,
}),
billingType: normalizeLedgerBillingType(
adapterResult.billingType,
),
};
const persistedResultJson = cancellationResultJson(latestRun ?? run, outcome, mergeHeartbeatRunResultJson(
mergeRunStopMetadataForAgent(agent, outcome, {
@@ -25719,13 +25733,17 @@ export function heartbeatService(
adapterResult.summary ?? null,
), runErrorCode, runErrorMessage);
const ledgerScope = runLedgerScope;
const finalRunPatch: Partial<typeof heartbeatRuns.$inferInsert> = {
// Accounting must acknowledge even a proven pre-provider failure:
// that transaction releases the reservation without creating a charge.
costAccountingPending: true,
finishedAt: new Date(),
error: runErrorMessage,
errorCode: runErrorCode,
exitCode: adapterResult.exitCode,
signal: adapterResult.signal,
usageJson,
usageJson: { ...usageJson, ledgerScope },
resultJson: persistedResultJson,
sessionIdAfter:
nextSessionState.displayId ?? nextSessionState.legacySessionId,
@@ -25763,6 +25781,8 @@ export function heartbeatService(
.set({
...finalRunPatch,
resultJson: cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage),
usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },
finishedAt:
persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,
updatedAt: new Date(),
@@ -25777,6 +25797,9 @@ export function heartbeatService(
.then((rows) => rows[0] ?? null);
}
if (!persistedRun) {
await accountRunCost(db, run.id, budgetHooks).catch((err) => {
logger.error({ err, runId: run.id }, "Late run accounting queued for recovery");
});
logger.info(
{
runId: run.id,
@@ -25789,6 +25812,10 @@ export function heartbeatService(
}
}
if (persistedRun) {
// Accounting recovery is independent of workspace or issue finalization.
await accountRunCost(db, persistedRun.id, budgetHooks).catch((err) => {
logger.error({ err, runId: run.id }, "Run accounting queued for recovery");
});
persistedRun =
(await classifyAndPersistRunLiveness(
persistedRun,
@@ -29512,7 +29539,7 @@ export function heartbeatService(
return { scanned: candidates.length, dispatched, recovered, deferred };
}
async function listProjectScopedRunIds(companyId: string, projectId: string) {
async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) {
const runIssueId = sql<
string | null
>`${heartbeatRuns.contextSnapshot} ->> 'issueId'`;
@@ -29533,6 +29560,7 @@ export function heartbeatService(
.where(
and(
eq(heartbeatRuns.companyId, companyId),
createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined,
inArray(heartbeatRuns.status, [
...CANCELLABLE_HEARTBEAT_RUN_STATUSES,
]),
@@ -29546,6 +29574,7 @@ export function heartbeatService(
async function listProjectScopedWakeupIds(
companyId: string,
projectId: string,
database: Db = db,
) {
const wakeIssueId = sql<
string | null
@@ -29554,7 +29583,7 @@ export function heartbeatService(
string | null
>`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;
const rows = await db
const rows = await database
.selectDistinctOn([agentWakeupRequests.id], {
id: agentWakeupRequests.id,
})
@@ -29584,63 +29613,72 @@ export function heartbeatService(
async function cancelPendingWakeupsForBudgetScope(
scope: BudgetEnforcementScope,
) {
const now = new Date();
let wakeupIds: string[] = [];
return withCurrentBudgetEnforcement(db, scope, async (tx) => {
const now = new Date();
let wakeupIds: string[] = [];
if (scope.scopeType === "company") {
wakeupIds = await db
.select({ id: agentWakeupRequests.id })
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, scope.companyId),
inArray(agentWakeupRequests.status, [
"queued",
"deferred_issue_execution",
]),
sql`${agentWakeupRequests.runId} is null`,
),
)
.then((rows) => rows.map((row) => row.id));
} else if (scope.scopeType === "agent") {
wakeupIds = await db
.select({ id: agentWakeupRequests.id })
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, scope.companyId),
eq(agentWakeupRequests.agentId, scope.scopeId),
inArray(agentWakeupRequests.status, [
"queued",
"deferred_issue_execution",
]),
sql`${agentWakeupRequests.runId} is null`,
),
)
.then((rows) => rows.map((row) => row.id));
} else {
wakeupIds = await listProjectScopedWakeupIds(
scope.companyId,
scope.scopeId,
);
}
if (scope.scopeType === "company") {
wakeupIds = await tx
.select({ id: agentWakeupRequests.id })
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, scope.companyId),
inArray(agentWakeupRequests.status, [
"queued",
"deferred_issue_execution",
]),
sql`${agentWakeupRequests.runId} is null`,
),
)
.then((rows) => rows.map((row) => row.id));
} else if (scope.scopeType === "agent") {
wakeupIds = await tx
.select({ id: agentWakeupRequests.id })
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, scope.companyId),
eq(agentWakeupRequests.agentId, scope.scopeId),
inArray(agentWakeupRequests.status, [
"queued",
"deferred_issue_execution",
]),
sql`${agentWakeupRequests.runId} is null`,
),
)
.then((rows) => rows.map((row) => row.id));
} else {
wakeupIds = await listProjectScopedWakeupIds(
scope.companyId,
scope.scopeId,
tx,
);
}
if (wakeupIds.length === 0) return 0;
if (wakeupIds.length === 0) return 0;
await db
.update(agentWakeupRequests)
.set({
status: "cancelled",
finishedAt: now,
error: "Cancelled due to budget pause",
updatedAt: now,
})
.where(inArray(agentWakeupRequests.id, wakeupIds));
await tx
.update(agentWakeupRequests)
.set({
status: "cancelled",
finishedAt: now,
error: "Cancelled due to budget pause",
updatedAt: now,
})
.where(and(
inArray(agentWakeupRequests.id, wakeupIds),
inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]),
isNull(agentWakeupRequests.runId),
scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined,
));
return wakeupIds.length;
return wakeupIds.length;
});
}
type CancelRunOptions = {
budgetEnforcement?: BudgetEnforcementScope;
/** Optional board request identity, atomically reserved for native Stop. */
cancellationRequestId?: string;
cancellationRequestedByUserId?: string | null;
@@ -29725,25 +29763,40 @@ export function heartbeatService(
// Established legacy processes must still be stopped if the database is
// unavailable. Only native or not-yet-dispatched preparation needs this
// additional durable fence before its existing cancellation path.
if (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control))) {
const [fenced] = await db.update(heartbeatRuns).set({
// Record handoff intent before native cancellation can finalize and release
// the run. Only its audited stop acknowledgement suppresses recovery.
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||
${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true
? { reassignmentStopRequested: true } : {})}::jsonb ||
${JSON.stringify({ cancellation })}::jsonb ||
jsonb_build_object('startupCancellation', jsonb_build_object(
'requestedAt', ${new Date().toISOString()}::text,
'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy'
and ${heartbeatRuns.runtimeModeResolvedAt} is null
and ${heartbeatRuns.executionStage} = 'preparing'
and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false)
))`,
}).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status,
pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],
))).returning();
if (!fenced) return getRun(runId);
if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) {
const fence = async (tx: Db) => {
const [fenced] = await tx.update(heartbeatRuns).set({
// Record handoff intent before native cancellation can finalize and release
// the run. Only its audited stop acknowledgement suppresses recovery.
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||
${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true
? { reassignmentStopRequested: true } : {})}::jsonb ||
${JSON.stringify({ cancellation })}::jsonb ||
jsonb_build_object('startupCancellation', jsonb_build_object(
'requestedAt', ${new Date().toISOString()}::text,
'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy'
and ${heartbeatRuns.runtimeModeResolvedAt} is null
and ${heartbeatRuns.executionStage} = 'preparing'
and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false)
))`,
}).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status,
pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],
))).returning();
return fenced ?? null;
};
let fenced: typeof run | null;
try {
fenced = options.budgetEnforcement
? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence)
: await fence(db);
} catch (error) {
stopOwnership?.release();
throw error;
}
if (!fenced) {
stopOwnership?.release();
return getRun(runId);
}
run = fenced;
}
const resultJson = { ...(agent
@@ -30118,35 +30171,15 @@ export function heartbeatService(
}
async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) {
if (scope.scopeType === "agent") {
await cancelActiveForAgentInternal(
scope.scopeId,
"Cancelled due to budget pause",
);
await cancelPendingWakeupsForBudgetScope(scope);
return;
}
const runIds =
scope.scopeType === "company"
? await db
.select({ id: heartbeatRuns.id })
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, scope.companyId),
inArray(heartbeatRuns.status, [
...CANCELLABLE_HEARTBEAT_RUN_STATUSES,
]),
),
)
.then((rows) => rows.map((row) => row.id))
: await listProjectScopedRunIds(scope.companyId, scope.scopeId);
for (const runId of runIds) {
await cancelRunInternal(runId, "Cancelled due to budget pause");
}
const runIds = scope.scopeType === "project"
? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore)
: await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, scope.companyId),
scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined,
scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined,
inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),
)).then((rows) => rows.map((row) => row.id));
for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope });
await cancelPendingWakeupsForBudgetScope(scope);
}
@@ -30515,6 +30548,7 @@ export function heartbeatService(
scanSilentActiveRuns,
reconcileTaskWatchdogs,
reconcileCostAccounting: createCostAccountingReconciler(db, budgetHooks),
buildRunOutputSilence,
server/src/services/native-runtime/native-session-executor.test.ts
+326 -3
View File
@@ -1,5 +1,7 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { readFileSync } from "node:fs";
import { PgDialect } from "drizzle-orm/pg-core";
import type { SQL } from "drizzle-orm";
import {
access,
cp,
@@ -195,6 +197,18 @@ const state = vi.hoisted(() => ({
release: null as null | (() => void),
}));
const accountingEvents = vi.hoisted(() => ({ committed: undefined as undefined | ((event: PrpEvent) => Promise<void>), duplicate: undefined as undefined | ((event: PrpEvent) => Promise<void>) }));
vi.mock("./paperclip-control-plane-port.js", async importOriginal => {
const original = await importOriginal<typeof import("./paperclip-control-plane-port.js")>();
return { ...original, PaperclipControlPlanePort: class extends original.PaperclipControlPlanePort {
constructor(...args: ConstructorParameters<typeof original.PaperclipControlPlanePort>) {
super(...args);
accountingEvents.committed = args[2]?.onCommittedEvent;
accountingEvents.duplicate = args[2]?.onDuplicateEvent;
}
} };
});
const grokCopyBack = vi.hoisted(() => vi.fn(async (_input: { readSandboxAuth: () => Promise<Buffer>; hostHomeDir: string }) => undefined));
vi.mock("@paperclipai/adapter-grok-local/server", async importOriginal => ({
...await importOriginal<typeof import("@paperclipai/adapter-grok-local/server")>(),
@@ -309,6 +323,7 @@ import {
nativeUsageCostUsd,
nativeUsageBiller,
normalizeNativeUsage,
resolveNativeBilling,
parseRemoteRunnerProcessIdentity,
REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
verifyRemoteRunnerReattachment,
@@ -984,6 +999,18 @@ describe("native incomplete-bootstrap evidence", () => {
});
describe("native provider usage normalization", () => {
it.each([
{}, { inputTokens: 1 }, { outputTokens: 1 },
{ inputTokens: 0, outputTokens: -1 }, { inputTokens: 0.5, outputTokens: 1 },
{ inputTokens: 10, outputTokens: 1, cacheReadTokens: 20 },
{ inputTokens: 10, outputTokens: 1, cacheReadTokens: -1 },
])("does not invent complete native counters from %j", runDelta => {
expect(normalizeNativeUsage({ runDelta, total: { inputTokens: 100, outputTokens: 10 } }, { inputIncludesCacheReads: true })).toBeUndefined();
});
it("does not use a cumulative price or protocol-default zero for an incomplete delta", () => {
expect(nativeUsageCostUsd({ runDelta: { inputTokens: 10, outputTokens: 1 }, providerCostUsd: 100 })).toBeUndefined();
expect(nativeUsageCostUsd({ runDelta: { providerCostUsd: 0 } })).toBeUndefined();
});
it.each([["cursor", "cursor"], ["copilot", "github"], ["pi", "openrouter"]] as const)("keeps %s cost unknown and attributes its actual biller", (agent, biller) => {
const provider = { kind: "acpx", agent, model: "exact-model" } as NativeExecutionInput["provider"];
expect(nativeUsageBiller(provider)).toBe(biller);
@@ -991,6 +1018,35 @@ describe("native provider usage normalization", () => {
expect(nativeUsageCostUsd(usage, provider)).toBeUndefined();
expect(normalizeNativeUsage(usage)).toMatchObject({ inputTokens: 100, outputTokens: 12 });
});
it("retains cache writes and avoids counting Codex cache hits twice", () => {
expect(normalizeNativeUsage({ runDelta: { inputTokens: 120, cacheReadTokens: 100, cacheWriteTokens: 5, outputTokens: 10 } }, { inputIncludesCacheReads: true }))
.toEqual({ inputTokens: 20, cachedInputTokens: 100, cacheWriteTokens: 5, outputTokens: 10 });
});
it("does not treat a protocol-default price as free usage", () => {
expect(nativeUsageCostUsd({ runDelta: { inputTokens: 20, outputTokens: 10, providerCostUsd: 0 } })).toBeUndefined();
});
it("reports the actual runtime billing identity", () => {
expect(resolveNativeBilling({ kind: "opencode", model: "openrouter/deepseek/test", permissionMode: "allow" }))
.toEqual({ provider: "deepseek", biller: "openrouter", billingType: "unknown" });
expect(resolveNativeBilling({ kind: "acpx", agent: "claude", model: "claude-test", permissionMode: "approve-all",
profile: { driverKind: "acpx_runtime", protocolVersion: 1, acpxVersion: "0.13.1", agent: "claude", agentProfileVersion: 1,
agentServerPackage: "@zed-industries/claude-agent-acp", agentServerVersion: "1", agentRuntimePackage: null,
agentRuntimeVersion: null, commandDigest: "fixture" },
}, { ANTHROPIC_API_KEY: "test" }))
.toEqual({ provider: "anthropic", biller: "anthropic", billingType: "metered_api" });
});
it("prefers the adjusted turn price to cumulative session prices", () => {
expect(nativeUsageCostUsd({ providerCostUsd: 10, runDelta: { inputTokens: 10, providerCostUsd: 2, cacheAdjustedCostUsd: 1 } })).toBe(1);
});
it.each(["OPENAI_BASE_URL", "OPENAI_API_BASE", "OPENAI_API_BASE_URL"])("does not price native or managed proxy traffic as direct OpenAI using %s", key => {
const provider = { kind: "codex", model: "gpt-6-astra" } as Parameters<typeof resolveNativeBilling>[0];
expect(resolveNativeBilling(provider, { OPENAI_API_KEY: "fixture", [key]: "https://proxy.example/v1" }).biller).toBe("unknown");
expect(resolveNativeBilling(provider, { [key]: "https://proxy.example/v1" }, { provider: "openai", biller: "openai", billingType: "metered_api" }).biller).toBe("unknown");
expect(resolveNativeBilling(provider, {}, { provider: "openai", biller: "openai", billingType: "metered_api" })).toMatchObject({ biller: "openai", billingType: "metered_api" });
});
it("reads remote runner run-delta tokens and provider cost", () => {
const usage = {
total: {
@@ -4852,6 +4908,7 @@ function leaseDb(
updates: Array<{ table: unknown; values: Record<string, unknown> }> = [],
runnerProfileJson: Record<string, unknown> = {},
runStatus = "running",
priorAccountingEvents: PrpEvent[] = [],
): Db {
const coordinator: LeaseCoordinator = {
runId: boundExecution.binding.runId,
@@ -4910,13 +4967,15 @@ function leaseDb(
checkoutRunId: null,
},
]
: [];
: table === heartbeatRunEvents
? priorAccountingEvents.map((event, seq) => ({ seq, eventType: event.eventType, sourceSeq: event.sourceSeq, payload: { prpEvent: event } }))
: [];
const query = {
then: Promise.resolve(rows).then.bind(Promise.resolve(rows)),
where: () => query,
orderBy: () => query,
for: () => query,
limit: () => Promise.resolve(rows),
limit: () => query,
};
return query;
},
@@ -4924,7 +4983,9 @@ function leaseDb(
const insert = (table: unknown) => ({
values: (values: Record<string, unknown>) => {
updates.push({ table, values });
return { returning: async () => [values] };
const query = { returning: async () => [values], onConflictDoUpdate: () => query, onConflictDoNothing: () => query,
then: Promise.resolve([values]).then.bind(Promise.resolve([values])) };
return query;
},
});
const tx = {
@@ -5177,6 +5238,268 @@ describe("native startup restart detachment", () => {
});
});
describe("native terminal-turn accounting", () => {
it.each(["claude", "codex"].flatMap(agent => [false, true].flatMap(restart =>
[false, true].flatMap(partialFirst => [false, true].map(zeroFirst => ({ agent, restart, partialFirst, zeroFirst }))),
)))("accumulates ACPX turns for $agent (restart: $restart, partial: $partialFirst, zero first: $zeroFirst)", async ({ agent, restart, partialFirst, zeroFirst }) => {
const { rehydrateRunnerdUsageNotification } = await import("@paperclipai/paperclip-runner/live");
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const event = (eventType: string, turnId: string, sourceSeq: number, payload: object = {}) => ({
eventType, turnId, sourceSeq, sourceInstanceId: "provider", emittedAt: new Date().toISOString(), payload,
}) as unknown as PrpEvent;
const usage = (turnId: string, inputTokens: number, outputTokens: number, complete: boolean) => ({
kind: "usage", usage: rehydrateRunnerdUsageNotification({ provider: "acpx", runDeltaAvailable: complete,
cumulative: { inputTokens: 0, outputTokens: 0, providerCostUsd: 10 },
runDelta: { inputTokens, outputTokens, cacheReadTokens: 0, cacheWriteTokens: 0, providerCostUsd: 0 },
}, "session", turnId).tokenUsage,
});
const first = [event("turn.started", "first", 1), event("item.completed", "first", 2, usage("first", zeroFirst ? 0 : 12, zeroFirst ? 0 : 4, !partialFirst)), event("turn.failed", "first", 3)];
const second = [event("turn.started", "second", 4), event("item.completed", "second", 5, usage("second", 20, 6, true)), event("turn.completed", "second", 6)];
state.execute.mockReset().mockImplementationOnce(async () => {
for (const entry of [...(restart ? [] : first), ...second]) await accountingEvents.committed!(entry);
for (const entry of [...first, ...second]) await accountingEvents.duplicate!(entry);
return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" }, turnId: "second",
normalizedSessionId: "session", providerSessionId: null, driverKind: "acpx_runtime", driverVersion: "1",
nativeEventCount: 6, highestContiguousSourceSeq: 6, usage: null };
});
const result = await executePaperclipNativeSession({ db: leaseDb(execution, {}, {}, [], {}, "running", restart ? first : []),
execution: { ...execution, provider: { kind: "acpx", agent, model: agent === "codex" ? "gpt-6-astra" : "claude-sonnet-4-6" } } as NativeExecutionInput,
runnerInstanceId: "runner", runnerEnvironment: { OPENAI_API_KEY: "fixture" }, onUsage });
expect(result).toMatchObject({ usageComplete: !partialFirst,
usage: { inputTokens: partialFirst || zeroFirst ? 20 : 32, outputTokens: partialFirst || zeroFirst ? 6 : 10 } });
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: !partialFirst }));
if (partialFirst) expect(result.costStatus).toBe("unpriced");
if (zeroFirst && !partialFirst) {
expect(result.costUsdExact).toBeUndefined();
expect(result.costUsd).toBeNull();
}
if (agent === "codex" && !partialFirst) {
const { priceCodexReceipt } = await import("../codex-pricing.js");
const priced = priceCodexReceipt(onUsage.mock.calls.at(-1)![0]);
expect(priced.costStatus).toBe("estimated");
expect(Number(priced.costUsdExact)).toBeCloseTo(zeroFirst ? 0.0005 : 0.00082, 9);
}
});
it.each(["claude", "codex"].flatMap(agent =>
["missing", "partial", "complete_zero", "later_partial", "recovered"].map(scenario => ({ agent, scenario })),
))("honors runner completeness for ACPX $agent ($scenario)", async ({ agent, scenario }) => {
const { rehydrateRunnerdUsageNotification } = await import("@paperclipai/paperclip-runner/live");
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const complete = ["complete_zero", "recovered"].includes(scenario);
const zero = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, providerCostUsd: 0 };
const full = { ...zero, inputTokens: 12, outputTokens: 4 };
const reports = scenario === "complete_zero" ? [{ runDelta: zero, runDeltaAvailable: true }]
: scenario === "later_partial" ? [{ runDelta: full, runDeltaAvailable: true }, { runDelta: zero, runDeltaAvailable: false }]
: scenario === "recovered" ? [{ runDelta: zero, runDeltaAvailable: false }, { runDelta: full, runDeltaAvailable: true }]
: [{ runDelta: scenario === "partial" ? { ...zero, inputTokens: 12 } : zero, runDeltaAvailable: false }];
state.execute.mockReset().mockImplementationOnce(async () => {
for (const observe of [accountingEvents.committed!, accountingEvents.duplicate!]) {
for (const report of reports) {
const notification = rehydrateRunnerdUsageNotification({ provider: "acpx", cumulative: { ...full, providerCostUsd: 10 }, ...report }, "session", "turn");
await observe({ eventType: "item.completed", turnId: "turn", payload: { kind: "usage", usage: notification.tokenUsage } } as unknown as PrpEvent);
}
await observe({ eventType: "turn.completed", turnId: "turn", payload: {} } as unknown as PrpEvent);
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete }));
if (!complete) expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ costUsd: null, costStatus: "unpriced" }));
}
return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" }, turnId: "turn",
normalizedSessionId: "session", providerSessionId: null, driverKind: "acpx_runtime", driverVersion: "1", nativeEventCount: 2, highestContiguousSourceSeq: 2,
usage: { runDelta: full, runDeltaComplete: true } };
});
const result = await executePaperclipNativeSession({ db: leaseDb(), execution: { ...execution,
provider: { kind: "acpx", agent, model: agent === "codex" ? "gpt-6-astra" : "claude-sonnet-4-6" } } as NativeExecutionInput,
runnerInstanceId: "runner", onUsage });
expect(result).toMatchObject({ usageComplete: complete });
if (!complete) expect(result).toMatchObject({ costUsd: null, costStatus: "unpriced" });
else expect(result.usage).toMatchObject({ inputTokens: scenario === "complete_zero" ? 0 : 12, outputTokens: scenario === "complete_zero" ? 0 : 4 });
});
it.each([0, 0.25])("keeps interrupted AgentCore accounting pending despite cumulative cost %s", async providerCostUsd => {
const { rehydrateRunnerdUsageNotification } = await import("@paperclipai/paperclip-runner/live");
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const total = { inputTokens: 10, outputTokens: 2, cacheReadTokens: 0, cacheWriteTokens: 0, providerCostUsd };
const notification = rehydrateRunnerdUsageNotification({ provider: "aws_agentcore", cumulative: total, runDelta: null, runDeltaAvailable: false }, "session", "turn");
state.execute.mockReset().mockImplementationOnce(async () => {
for (const observe of [accountingEvents.committed!, accountingEvents.duplicate!]) {
await observe({ eventType: "item.completed", turnId: "turn", payload: { kind: "usage", usage: notification.tokenUsage } } as unknown as PrpEvent);
await observe({ eventType: "turn.interrupted", turnId: "turn", payload: {} } as unknown as PrpEvent);
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: false, costStatus: "unpriced" }));
}
return { result: { summary: "interrupted" }, terminal: { runTerminalState: "cancelled" }, turnId: "turn",
normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 2, highestContiguousSourceSeq: 2, usage: { total } };
});
const result = await executePaperclipNativeSession({ db: leaseDb(), execution: { ...execution, provider: { kind: "aws_agentcore", model: "managed-model", agentCoreProfile: { profileId: "agentcore-test" } } } as unknown as NativeExecutionInput, runnerInstanceId: "runner", onUsage });
expect(result).toMatchObject({ usageComplete: false, costStatus: "unpriced", billingType: "metered_api" });
});
it.each(["claude_managed", "aws_agentcore"] as const)("settles fresh runner receipts for %s and preserves estimated pricing", async kind => {
const { rehydrateRunnerdUsageNotification } = await import("@paperclipai/paperclip-runner/live");
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const total = { inputTokens: 121, outputTokens: 18, cacheReadTokens: 33, cacheWriteTokens: 15, providerCostUsd: 1.25 };
const delta = { inputTokens: 21, outputTokens: 8, cacheReadTokens: 13, cacheWriteTokens: 5, providerCostUsd: 0.25 };
const notification = rehydrateRunnerdUsageNotification({ provider: kind, cumulative: total, runDelta: delta, runDeltaAvailable: true }, "session", "turn");
state.execute.mockReset().mockImplementationOnce(async () => {
for (const observe of [accountingEvents.committed!, accountingEvents.duplicate!]) {
await observe({ eventType: "item.completed", turnId: "turn", payload: { kind: "usage", usage: notification.tokenUsage } } as unknown as PrpEvent);
await observe({ eventType: "turn.completed", turnId: "turn", payload: {} } as unknown as PrpEvent);
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true, costUsd: 0.25,
costStatus: kind === "aws_agentcore" ? "estimated" : undefined,
usage: { inputTokens: 26, outputTokens: 8, cachedInputTokens: 13, cacheWriteTokens: 5 } }));
}
return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" },
turnId: "turn", normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 2, highestContiguousSourceSeq: 2, usage: { total } };
});
const result = await executePaperclipNativeSession({ db: leaseDb(), execution: { ...execution, provider: { kind, model: "managed-model", managedProfile: { profileId: "managed-test" }, agentCoreProfile: { profileId: "agentcore-test" } } } as unknown as NativeExecutionInput, runnerInstanceId: "runner", onUsage });
expect(result).toMatchObject({ usageComplete: true, costUsd: 0.25, billingType: "metered_api", costStatus: kind === "aws_agentcore" ? "estimated" : undefined });
});
it.each([
{ provider: "codex", restart: false }, { provider: "codex", restart: true },
{ provider: "opencode", restart: false }, { provider: "opencode", restart: true },
] as const)("preserves earlier turns for $provider (restart: $restart)", async ({ provider, restart }) => {
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const event = (eventType: string, turnId: string, sourceSeq: number, payload: object = {}) => ({
eventType, turnId, sourceSeq, sourceInstanceId: "provider", emittedAt: new Date().toISOString(), payload,
}) as unknown as PrpEvent;
const first = [event("turn.started", "first", 1), event("item.completed", "first", 2, { kind: "usage", usage: {
runDelta: { inputTokens: 10, cacheReadTokens: 3, cacheWriteTokens: 1, outputTokens: 2, providerCostUsd: 0.25 },
} }), event("turn.failed", "first", 3)];
const second = [event("turn.started", "second", 4), event("item.completed", "second", 5, { kind: "usage", usage: {
runDelta: { inputTokens: 20, cacheReadTokens: 6, cacheWriteTokens: 2, outputTokens: 4, providerCostUsd: 0.5 },
} }), event("turn.completed", "second", 6)];
state.execute.mockReset().mockImplementationOnce(async () => {
for (const entry of [...(restart ? [] : first), ...second]) await accountingEvents.committed!(entry);
for (const entry of provider === "opencode" ? [...first, ...second] : second) await accountingEvents.duplicate!(entry);
return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" }, turnId: "second",
normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 6,
highestContiguousSourceSeq: 6, usage: null };
});
const result = await executePaperclipNativeSession({
db: leaseDb(execution, {}, {}, [], {}, "running", restart ? first : []),
execution: { ...execution, provider: { kind: provider, model: provider === "opencode" ? "openai/test" : "gpt-6-astra" } } as NativeExecutionInput,
runnerInstanceId: "runner", onUsage,
});
const usage = provider === "opencode"
? { inputTokens: 33, cachedInputTokens: 9, cacheWriteTokens: 3, outputTokens: 6 }
: { inputTokens: 14, cachedInputTokens: 6, cacheWriteTokens: 2, outputTokens: 4 };
expect(result).toMatchObject({ usageComplete: true, costUsd: provider === "opencode" ? 0.75 : 0.5, usage });
if (provider === "opencode") expect(result.costUsdExact).toBe("0.750000000");
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true, costUsd: result.costUsd, usage }));
});
it.each(["missing_usage", "partial_usage", "missing_terminal", "unknown_price", "late_update"] as const)("keeps OpenCode retry accounting conservative (%s)", async scenario => {
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
let seq = 0;
const emit = (eventType: string, turnId: string, payload: object = {}) => accountingEvents.committed!({
eventType, turnId, sourceSeq: ++seq, sourceInstanceId: "provider", emittedAt: new Date().toISOString(), payload,
} as unknown as PrpEvent);
const usage = (inputTokens: number, outputTokens: number | undefined, providerCostUsd: number | undefined) => ({
kind: "usage", usage: { runDelta: { inputTokens, outputTokens, providerCostUsd } },
});
state.execute.mockReset().mockImplementationOnce(async () => {
await emit("turn.started", "first");
if (scenario !== "missing_usage") await emit("item.completed", "first",
usage(10, scenario === "partial_usage" ? undefined : 2, scenario === "unknown_price" ? undefined : 0.25));
if (scenario !== "missing_terminal") await emit("turn.failed", "first");
await emit("turn.started", "second");
expect(onUsage.mock.calls.at(-1)![0].complete).toBe(false);
await emit("item.completed", "second", usage(20, 4, 0.5));
await emit("turn.completed", "second");
if (scenario === "late_update") await emit("item.completed", "first", usage(12, 4, 0.35));
return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" }, turnId: "second",
normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: seq,
highestContiguousSourceSeq: seq, usage: null };
});
const result = await executePaperclipNativeSession({ db: leaseDb(),
execution: { ...execution, provider: { kind: "opencode", model: "openai/test" } } as NativeExecutionInput,
runnerInstanceId: "runner", onUsage });
const complete = ["unknown_price", "late_update"].includes(scenario);
const cost = ["missing_usage", "unknown_price"].includes(scenario) ? 0.5 : scenario === "late_update" ? 0.85 : 0.75;
expect(result).toMatchObject({ usageComplete: complete, costUsd: cost });
expect(result.costStatus).toBe(scenario === "late_update" ? undefined : "unpriced");
expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ complete, costUsd: cost, costStatus: result.costStatus });
if (scenario === "late_update") expect(result.usage).toMatchObject({ inputTokens: 32, outputTokens: 8 });
});
it.each(["committed", "duplicate"] as const)("saves completed usage before returning to result commitment (%s)", async mode => {
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const failure = new Error("stopped after result commit, before executor returns");
state.execute.mockReset().mockImplementationOnce(async () => {
const observe = accountingEvents[mode]!;
await observe({ eventType: "item.completed", turnId: "turn", emittedAt: new Date().toISOString(), payload: { kind: "usage", usage: { runDelta: { inputTokens: 120, cacheReadTokens: 100, outputTokens: 10, providerCostUsd: 0.25 } } } } as unknown as PrpEvent);
await observe({ eventType: "turn.completed", turnId: "turn", emittedAt: new Date().toISOString(), payload: { status: "completed" } } as unknown as PrpEvent);
// The runtime cannot call completeRun until appendEvent has returned.
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true, costUsd: 0.25, usage: { inputTokens: 20, cachedInputTokens: 100, outputTokens: 10 } }));
throw failure;
});
await expect(executePaperclipNativeSession({ db: leaseDb(), execution: { ...execution, provider: { kind: "codex", model: "gpt-6-astra" } } as NativeExecutionInput, runnerInstanceId: "runner", onUsage })).rejects.toBe(failure);
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true }));
});
it("clears provisional unpriced status when direct database persistence receives complete usage", async () => {
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
state.execute.mockReset().mockImplementationOnce(async () => {
await accountingEvents.committed!({ eventType: "item.completed", turnId: "turn", emittedAt: new Date().toISOString(), payload: { kind: "usage", usage: { runDelta: { inputTokens: 10, outputTokens: 2, providerCostUsd: 0.25 } } } } as unknown as PrpEvent);
return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" },
turnId: "turn", normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
});
await executePaperclipNativeSession({ db: leaseDb(execution, {}, {}, updates), execution, runnerInstanceId: "runner" });
const dialect = new PgDialect();
const writes = updates.filter(update => update.values.usageJson).map(update =>
JSON.parse(dialect.sqlToQuery(update.values.usageJson as SQL).params[0] as string) as Record<string, unknown>);
expect(writes[0]).toMatchObject({ accountingReceiptReady: false, costStatus: "unpriced" });
// Apply the same JSON merge as the database: omitted keys retain the old value.
expect(Object.assign({}, ...writes)).toMatchObject({ accountingReceiptReady: true, costStatus: null, costUsd: 0.25 });
});
it.each(["stale_warm_zero", "partial", "fresh", "fresh_zero", "new_turn", "later_partial", "merged_partial"])("requires fresh complete usage at native completion (%s)", async scenario => {
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const usage = { inputTokens: 120, cacheReadTokens: 100, outputTokens: 10, providerCostUsd: 0 };
const runDelta = scenario === "partial" ? { inputTokens: 120 } : scenario === "fresh_zero" ? { inputTokens: 0, outputTokens: 0 } : usage;
state.execute.mockReset().mockImplementationOnce(async () => {
if (scenario !== "stale_warm_zero") await accountingEvents.committed!({ eventType: "item.completed", turnId: "turn", emittedAt: new Date().toISOString(), payload: { kind: "usage", usage: { runDelta, runDeltaComplete: scenario !== "merged_partial" } } } as unknown as PrpEvent);
if (scenario === "new_turn") await accountingEvents.committed!({ eventType: "turn.started", turnId: "next-turn", emittedAt: new Date().toISOString(), payload: {} } as unknown as PrpEvent);
if (scenario === "later_partial") await accountingEvents.committed!({ eventType: "item.completed", turnId: "turn", emittedAt: new Date().toISOString(), payload: { kind: "usage", usage: { runDelta: { inputTokens: 130 } } } } as unknown as PrpEvent);
await accountingEvents.committed!({ eventType: "turn.completed", turnId: scenario === "new_turn" ? "next-turn" : "turn", emittedAt: new Date().toISOString(), payload: {} } as unknown as PrpEvent);
if (!["fresh", "fresh_zero"].includes(scenario)) expect(onUsage.mock.calls.every(([receipt]) => !receipt.complete)).toBe(true);
return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" },
turnId: scenario === "new_turn" ? "next-turn" : "turn", normalizedSessionId: "session",
providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1,
highestContiguousSourceSeq: 1,
usage: { runDelta: scenario === "stale_warm_zero" ? { inputTokens: 0, outputTokens: 0, providerCostUsd: 0 } : runDelta },
};
});
const result = await executePaperclipNativeSession({ db: leaseDb(), execution: { ...execution, provider: { kind: "codex", model: "gpt-6-astra" } } as NativeExecutionInput, runnerInstanceId: "runner", runnerEnvironment: { OPENAI_API_KEY: "fixture" }, onUsage });
const complete = ["fresh", "fresh_zero"].includes(scenario);
expect(result.usageComplete).toBe(complete);
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete }));
if (!complete) {
expect(result.costStatus).toBe("unpriced");
expect(result.costUsd).toBeNull();
expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: null, costStatus: "unpriced" });
}
});
beforeEach(() => vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })) } as unknown as ReturnType<typeof issueServiceModule.issueService>));
afterEach(() => vi.restoreAllMocks());
it.each(["turn.failed", "turn.cancelled", "turn.interrupted", "missing_terminal", "session_total", "empty_usage", "different_turn"])("preserves run usage for %s without certifying incomplete evidence", async (scenario) => {
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const failure = new Error("provider_transport_failed: no semantic result");
state.execute.mockReset().mockImplementationOnce(async () => {
const usage = { inputTokens: 120, cacheReadTokens: 100, outputTokens: 10, providerCostUsd: 0.25 };
await accountingEvents.committed!({ eventType: "item.completed", turnId: "turn", emittedAt: new Date().toISOString(), payload: { kind: "usage", usage: scenario === "session_total" ? { total: usage } : { runDelta: scenario === "empty_usage" ? { requests: 1 } : usage } } } as unknown as PrpEvent);
if (scenario !== "missing_terminal") await accountingEvents.committed!({ eventType: scenario.startsWith("turn.") ? scenario : "turn.failed", turnId: scenario === "different_turn" ? "another-turn" : "turn", emittedAt: new Date().toISOString(), payload: { status: "failed" } } as unknown as PrpEvent);
throw failure;
});
await expect(executePaperclipNativeSession({ db: leaseDb(), execution: { ...execution, provider: { kind: "codex", model: "gpt-6-astra" } } as NativeExecutionInput, runnerInstanceId: "runner", onUsage })).rejects.toBe(failure);
if (scenario === "session_total") expect(onUsage).not.toHaveBeenCalled();
else if (scenario === "empty_usage") expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: false, usage: undefined, costUsd: null, costStatus: "unpriced" }));
else {
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: scenario.startsWith("turn."), costUsd: 0.25, usageBasis: "per_run", usage: { inputTokens: 20, cachedInputTokens: 100, outputTokens: 10 } }));
expect(onUsage.mock.calls[0]?.[0]).toMatchObject({ complete: false });
}
});
});
describe("native resumed preparation timing", () => {
it("keeps answered-question ingress at the run root rather than charging it to preparation", async () => {
const answeredAtMs = Date.now();
server/src/services/native-runtime/native-session-executor.ts
+226 -38
View File
@@ -11,6 +11,7 @@ import { readLocalAiCredentialFile } from "../local-ai-credential-file.js";
import { prepareGrokRunnerCredentials } from "./grok-runner-credentials.js";
import { copyBackGrokAuth } from "@paperclipai/adapter-grok-local/server";
import { inferOpenAiCompatibleBiller, type AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import {
isSupportedRemoteCodexVersion,
@@ -65,7 +66,7 @@ import type {
AdapterExecutionResult,
AdapterRuntimeEvent,
} from "../../adapters/index.js";
import type { NativeFinalizationResult } from "@paperclipai/shared";
import { usdToUnits, type NativeFinalizationResult } from "@paperclipai/shared";
import type {
HarnessRuntimeRequestResolution,
NativeExecutionInput,
@@ -7320,6 +7321,7 @@ export async function executePaperclipNativeSession(input: {
collectStopped: () => Promise<void>;
};
/** Persist task-level continuity before a durable goal can outlive this run. */
onUsage?: (receipt: AdapterUsageCheckpoint) => Promise<void>;
onGoalCheckpoint?: (snapshot: PersistedNativeSession) => Promise<void>;
sessionGoalControl?: NativeSessionGoalControl | null;
resumeSessionGoalHeartbeat?: boolean;
@@ -7331,6 +7333,7 @@ export async function executePaperclipNativeSession(input: {
/** Private grant materialization; never a user-configured host path. */
managedAiCredentialHome?: string;
managedAiCredentialIdentity?: string;
billingIdentity?: Pick<AdapterExecutionResult, "provider" | "biller" | "billingType">;
runnerExecutionTarget?: AdapterExecutionTarget | null;
/** Resolved per-run authorization; not an independent instance setting. */
runnerIngressAuthorized?: boolean;
@@ -7912,6 +7915,76 @@ async function executePaperclipNativeSessionWithinScope(
binding: { ...input.execution.binding, normalizedSessionId: nativeSessionKey(input.execution), runnerSourceInstanceId: effectiveRunnerInstanceId },
resolve: resolveNativeRuntimeRequest,
});
let observedAccountingUsage: Record<string, unknown> | null = null;
let observedAccountingTurn: string | undefined;
let nativeAccountingComplete = false;
// OpenCode and ACPX report their current turn; Codex reports a cumulative run delta.
// Rebuild per-turn state from the durable event log when a controller resumes.
const turnAccounting = ["opencode", "acpx"].includes(input.execution.provider.kind) ? createNativeTurnAccounting(input.execution.provider) : null;
if (turnAccounting) {
const history = await input.db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, input.execution.binding.companyId), eq(heartbeatRunEvents.runId, input.execution.binding.runId),
or(sql`${heartbeatRunEvents.payload}->'prpEvent'->'payload'->>'kind' = 'usage'`,
inArray(heartbeatRunEvents.eventType, ["turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"])),
)).orderBy(heartbeatRunEvents.seq);
for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent);
}
const persistAccountingUsage = async (usage: Record<string, unknown> | null, complete: boolean) => {
const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" });
const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider) : undefined;
complete = complete && accountingUsage !== undefined;
const costStatus = complete && usage?.accountingCostIncomplete !== true
? input.execution.provider.kind === "aws_agentcore" ? "estimated" as const : undefined
: "unpriced" as const;
const billing = resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity);
const costUsdExact = typeof usage?.accountingCostUsdExact === "string" ? usage.accountingCostUsdExact : undefined;
await input.onUsage?.({ usage: accountingUsage, ...billing, complete, usageBasis: "per_run", costUsdExact,
model: input.execution.provider.model ?? "unknown", costUsd: accountingCost ?? null, costStatus });
if (!input.onUsage) await input.db.update(heartbeatRuns).set({
costAccountingPending: true,
usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({
...accountingUsage, ...billing, accountingReceiptReady: complete, model: input.execution.provider.model ?? "unknown",
costUsd: accountingCost ?? null, costUsdExact: costUsdExact ?? null, costStatus: costStatus ?? null, usageSource: "per_run",
})}::jsonb`,
}).where(and(eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), isNull(heartbeatRuns.costAccountedAt)));
};
// appendEvent awaits this receipt before the runtime may commit completeRun.
// Replayed events must rebuild it too after a stop between event and receipt.
const observeAccountingEvent = async (event: PrpEvent) => {
if (turnAccounting) {
if (!turnAccounting.observe(event)) return;
const snapshot = turnAccounting.snapshot();
observedAccountingUsage = snapshot.usage;
observedAccountingTurn = snapshot.turnId;
await persistAccountingUsage(snapshot.usage, snapshot.complete);
return;
}
const payload = record(event.payload);
const usage = record(payload.usage);
const delta = record(usage.runDelta);
if (event.eventType === "turn.started") {
observedAccountingTurn = undefined;
if (observedAccountingUsage) await persistAccountingUsage(observedAccountingUsage, false);
}
// A warm attachment seeds a zero snapshot before any provider report.
// Only a fresh run delta for this turn can close its accounting. A later
// malformed report invalidates completeness instead of retaining a
// smaller earlier total or falling back to cumulative session usage.
if (payload.kind === "usage") {
observedAccountingTurn = undefined;
if (Object.hasOwn(usage, "runDelta")) {
observedAccountingUsage = { runDelta: delta, runDeltaComplete: usage.runDeltaComplete };
if (normalizeNativeUsage(observedAccountingUsage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" })) {
observedAccountingTurn = event.turnId;
}
}
if (observedAccountingUsage) await persistAccountingUsage(observedAccountingUsage, false);
}
if (["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(event.eventType)
&& observedAccountingUsage && observedAccountingTurn !== undefined && event.turnId === observedAccountingTurn) {
await persistAccountingUsage(observedAccountingUsage, true);
}
};
let completedConversationReply: PrpEvent | null = null;
const controlPlane = new PaperclipControlPlanePort(
input.db,
@@ -7929,6 +8002,7 @@ async function executePaperclipNativeSessionWithinScope(
{
privateKeyPem: input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY,
onCommittedEvent: async (event) => {
await observeAccountingEvent(event);
await toolTrace.observe(event);
if (event.eventType === "item.completed" &&
record(event.payload).kind === "agentMessage" &&
@@ -8136,6 +8210,7 @@ async function executePaperclipNativeSessionWithinScope(
);
},
onDuplicateEvent: async (event) => {
await observeAccountingEvent(event);
// A crash can happen after the event commit but before its callback
// finishes. Recover only idempotent durable projections here; activity,
// publication, logging, trace, and metric effects remain committed-only.
@@ -8687,6 +8762,19 @@ async function executePaperclipNativeSessionWithinScope(
},
{ parentName: "task.run" },
);
// Persist provider accounting before any workspace/issue finalization. A
// detached controller or failed finalizer must not lose a completed turn.
// session.usage() may be an attachment baseline, a partial report, or a
// cumulative session total. Use the observed run delta, and require its
// turn to match the terminal result before certifying it as complete.
if (turnAccounting) {
const snapshot = turnAccounting.finish(native.turnId);
observedAccountingUsage = snapshot.usage;
nativeAccountingComplete = snapshot.complete && snapshot.turnId === native.turnId;
} else nativeAccountingComplete = observedAccountingUsage !== null
&& observedAccountingTurn !== undefined && observedAccountingTurn === native.turnId;
native = { ...native, usage: observedAccountingUsage };
await persistAccountingUsage(native.usage, nativeAccountingComplete);
try {
await completeManagedNativeCredentialTurn(managedCredentialSession);
} catch {
@@ -9325,10 +9413,16 @@ async function executePaperclipNativeSessionWithinScope(
summary: native.result.summary,
sessionId: native.normalizedSessionId,
sessionDisplayId: native.providerSessionId ?? native.normalizedSessionId,
provider: nativeUsageBiller(input.execution.provider),
...resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity),
model: input.execution.provider.model,
usage: normalizeNativeUsage(native.usage),
costUsd: nativeUsageCostUsd(native.usage, input.execution.provider),
usage: normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }),
costUsd: normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" })
? nativeUsageCostUsd(native.usage, input.execution.provider) ?? null : null,
costUsdExact: typeof native.usage?.accountingCostUsdExact === "string" ? native.usage.accountingCostUsdExact : undefined,
costStatus: nativeAccountingComplete && native.usage?.accountingCostIncomplete !== true
? input.execution.provider.kind === "aws_agentcore" ? "estimated" : undefined
: "unpriced",
usageComplete: nativeAccountingComplete,
usageBasis: "per_run",
nativeFinalization: finalization,
};
@@ -9359,6 +9453,9 @@ function numericUsageField(
function nativeUsageMeasurement(usage: Record<string, unknown>) {
const nestedUsage = record(usage.usage);
// An explicit partial delta must not fall through to a full session total.
if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta);
if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta);
const candidates = [
record(usage.runDelta),
record(nestedUsage.runDelta),
@@ -9393,6 +9490,74 @@ export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): s
return "openai";
}
function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) {
const turns = new Map<string, { usage: Record<string, unknown> | null; terminal: boolean }>();
const sequences = new Map<string, number>();
let currentTurn: string | undefined;
const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"];
const snapshot = () => {
if (turns.size === 0) return { usage: null, turnId: undefined, complete: false };
let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
let cost = 0n, costKnown = false, costIncomplete = false, complete = true;
for (const [turnId, turn] of turns) {
const normalized = normalizeNativeUsage(turn.usage);
complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined;
inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0);
outputTokens += normalized?.outputTokens ?? 0;
cacheReadTokens += normalized?.cachedInputTokens ?? 0;
cacheWriteTokens += normalized?.cacheWriteTokens ?? 0;
const price = nativeUsageCostUsd(turn.usage, provider);
if (price === undefined) costIncomplete = true;
else { cost += usdToUnits(price); costKnown = true; }
}
// A free empty attempt contributes no price evidence for later paid work.
// Keep an incomplete zero subtotal absent so complete aggregate tokens can
// still receive an estimate. Preserve positive known spend and true zero totals.
const retainCost = costKnown && (!costIncomplete || cost > 0n);
const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) };
const current = currentTurn ? turns.get(currentTurn) : undefined;
return { usage: { runDelta, accountingCostIncomplete: costIncomplete,
...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) },
turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete };
};
return {
observe(event: PrpEvent) {
if (!event) return false;
const payload = record(event.payload);
const terminal = terminalTypes.includes(event.eventType);
if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false;
// Historical replay can revisit an earlier report after restoration.
// Re-persist its aggregate, but never rewind or count that report twice.
if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) {
if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true;
sequences.set(event.sourceInstanceId, event.sourceSeq);
}
const key = event.turnId ?? "unattributed";
const turn = turns.get(key) ?? { usage: null, terminal: false };
if (event.eventType === "turn.started") {
currentTurn = event.turnId;
turn.usage = null;
turn.terminal = false;
} else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId;
if (payload.kind === "usage") {
const usage = record(payload.usage);
turn.usage = Object.hasOwn(usage, "runDelta")
? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null;
}
if (terminal) turn.terminal = true;
turns.set(key, turn);
return true;
},
snapshot,
finish(turnId: string | null) {
const turn = turnId ? turns.get(turnId) : undefined;
if (turn) turn.terminal = true;
return snapshot();
},
};
}
export function nativeUsageCostUsd(
usage: Record<string, unknown> | null,
provider?: NativeExecutionInput["provider"],
@@ -9403,50 +9568,73 @@ export function nativeUsageCostUsd(
if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined;
if (!usage) return undefined;
const measurement = nativeUsageMeasurement(usage);
const direct =
numericUsageField(usage, [
"providerCostUsd",
"cacheAdjustedCostUsd",
"costUsd",
]) ??
numericUsageField(measurement, [
"providerCostUsd",
"cacheAdjustedCostUsd",
"costUsd",
]);
if (direct !== undefined) return direct;
const cost = record(usage.cost);
const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta");
const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])
?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]));
// PRP currently defaults an absent providerCostUsd to zero. That zero is
// not proof of a free request when the runtime observed token consumption.
if (direct !== undefined && direct > 0) return direct;
const normalized = normalizeNativeUsage(usage);
if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0;
const cost = record(hasRunDelta ? measurement.cost : usage.cost);
const currency =
typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
if (currency !== "USD") return undefined;
return numericUsageField(cost, ["amount", "total"]);
}
export function normalizeNativeUsage(usage: Record<string, unknown> | null) {
if (!usage) return undefined;
/** Billing identity follows the selected runtime and resolved credentials.
* Unknown authentication remains unknown; a provider name does not prove billing mode. */
export function resolveNativeBilling(provider: NativeExecutionInput["provider"], env: NodeJS.ProcessEnv = {}, identity?: Pick<AdapterExecutionResult, "provider" | "biller" | "billingType">): Pick<AdapterExecutionResult, "provider" | "biller" | "billingType"> {
if (identity) return identity.provider === "openai" && identity.billingType === "metered_api"
? { ...identity, biller: inferOpenAiCompatibleBiller(env, identity.biller ?? "unknown") }
: identity;
if (provider.kind === "claude_managed") return { provider: "anthropic", biller: "anthropic", billingType: "metered_api" };
if (provider.kind === "aws_agentcore") return { provider: "aws_agentcore", biller: "aws_agentcore", billingType: "metered_api" };
if (provider.kind === "acpx" && ["cursor", "copilot"].includes(provider.agent)) {
const biller = nativeUsageBiller(provider);
return { provider: biller, biller, billingType: "unknown" };
}
if (provider.kind === "acpx" && provider.agent === "claude") {
if (env.CLAUDE_CODE_USE_BEDROCK === "1") return { provider: "anthropic", biller: "aws_bedrock", billingType: "metered_api" };
return { provider: "anthropic", biller: "anthropic", billingType: env.ANTHROPIC_API_KEY ? "metered_api" : env.CLAUDE_CODE_OAUTH_TOKEN ? "subscription_included" : "unknown" };
}
if (provider.kind === "opencode" || (provider.kind === "acpx" && provider.agent === "pi")) {
const parts = (provider.model ?? "").split("/");
const biller = parts.length > 1 ? parts[0] : provider.kind === "acpx" ? nativeUsageBiller(provider) : "unknown";
const modelProvider = biller === "openrouter" && parts.length > 2 ? parts[1] : biller;
return { provider: modelProvider, biller, billingType: "unknown" };
}
if (provider.kind !== "codex" && !(provider.kind === "acpx" && provider.agent === "codex"))
return { provider: "unknown", biller: "unknown", billingType: "unknown" };
return { provider: "openai", biller: env.OPENAI_API_KEY ? inferOpenAiCompatibleBiller(env, "openai") : "unknown", billingType: env.OPENAI_API_KEY ? "metered_api" : "unknown" };
}
export function normalizeNativeUsage(usage: Record<string, unknown> | null, options: { inputIncludesCacheReads?: boolean } = {}) {
if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined;
const measurement = nativeUsageMeasurement(usage);
const cache = record(measurement.cache);
const cachedInputTokens =
numericUsageField(measurement, [
"cachedInputTokens",
"cacheReadInputTokens",
"cacheReadTokens",
"cachedReadTokens",
]) ?? numericUsageField(cache, ["read"]);
const count = (source: Record<string, unknown>, keys: string[]) => {
for (const key of keys) {
if (source[key] === undefined) continue;
const value = source[key];
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN;
}
return undefined;
};
const input = count(measurement, ["inputTokens", "input", "promptTokens"]);
const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]);
const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]);
const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]);
if (input === undefined || outputTokens === undefined
|| ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined;
const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0)
+ (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0);
if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined;
return {
inputTokens:
numericUsageField(measurement, [
"inputTokens",
"input",
"promptTokens",
]) ?? 0,
outputTokens:
numericUsageField(measurement, [
"outputTokens",
"output",
"completionTokens",
]) ?? 0,
inputTokens, outputTokens,
...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }),
};
}
server/src/services/plugin-managed-agents.ts
+3 -1
View File
@@ -337,7 +337,7 @@ export function pluginManagedAgentService(
// and its canonical history. Transfer only its proven stable-key marker to
// the replacement installation, atomically with the new scoped bindings.
await upsertBinding(companyId, declaration, agentId, {}, adapterType, tx);
const [relinked] = await tx.update(agents).set({
await tx.update(agents).set({
metadata: managedMetadata(options.pluginId, options.pluginKey, declaration, agent.metadata),
updatedAt: new Date(),
}).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))).returning();
@@ -346,6 +346,8 @@ export function pluginManagedAgentService(
action: "plugin.managed_agent.relinked", entityType: "agent", entityId: agentId,
details: { sourcePluginKey: options.pluginKey, managedResourceKey: declaration.agentKey, previousPluginId },
});
const relinked = await agentService(tx as unknown as Db).getById(agentId);
if (!relinked) throw notFound("Managed agent not found after relink");
return relinked as Agent;
});
}
server/src/services/provider-billing-import.ts
+376
View File
@@ -0,0 +1,376 @@
import { z } from "zod";
import { and, eq, gte, inArray, lt } from "drizzle-orm";
import {
companySecrets,
providerBillingSnapshots,
type Db,
} from "@paperclipai/db";
import {
centsToUnits,
usdToUnits,
unitsToCents,
importProviderCostsSchema,
type ImportProviderCosts,
} from "@paperclipai/shared";
import { secretService } from "./secrets.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { createFinanceEventInTransaction } from "./finance.js";
import { forbidden, unprocessable } from "../errors.js";
const pageSchema = z.object({
data: z
.array(
z.object({
start_time: z.number().int().optional(),
end_time: z.number().int().optional(),
starting_at: z.iso.datetime({ offset: true }).optional(),
ending_at: z.iso.datetime({ offset: true }).optional(),
results: z
.array(
z.object({
project_id: z.string().nullable().optional(),
workspace_id: z.string().nullable().optional(),
amount: z.union([
z.string(),
z.object({ value: z.number().finite(), currency: z.string() }),
]),
currency: z.string().optional(),
}),
)
.max(5000),
}),
)
.max(31),
has_more: z.boolean(),
next_page: z.string().max(4000).nullable().optional(),
});
export interface DailyProviderCost {
day: string;
scopeId: string;
amountCents: string;
}
async function boundedJson(response: Response) {
if (!response.ok || !response.body)
throw unprocessable(
"Provider billing is unavailable. Check the admin credential and selected account.",
);
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
for (;;) {
const next = await reader.read();
if (next.done) break;
size += next.value.byteLength;
if (size > 2_000_000)
throw unprocessable(
"Provider billing response is too large; use a shorter period.",
);
chunks.push(next.value);
}
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
} finally {
await reader.cancel().catch(() => undefined);
}
}
/** Complete every page before writing. Group only by project/workspace so a
* provider's line-item changes cannot manufacture duplicate charges. */
export async function fetchProviderDailyCosts(
input: ImportProviderCosts,
credential: string,
fetcher = fetch,
): Promise<DailyProviderCost[]> {
const parsed = importProviderCostsSchema.parse(input);
const from = Date.parse(parsed.from),
to = Date.parse(parsed.to);
const base = new URL(
parsed.provider === "openai"
? "https://api.openai.com/v1/organization/costs"
: "https://api.anthropic.com/v1/organizations/cost_report",
);
base.searchParams.set(
parsed.provider === "openai" ? "start_time" : "starting_at",
parsed.provider === "openai"
? String(from / 1000)
: new Date(from).toISOString(),
);
base.searchParams.set(
parsed.provider === "openai" ? "end_time" : "ending_at",
parsed.provider === "openai"
? String(to / 1000)
: new Date(to).toISOString(),
);
base.searchParams.set("bucket_width", "1d");
base.searchParams.set("limit", "31");
base.searchParams.append(
"group_by[]",
parsed.provider === "openai" ? "project_id" : "workspace_id",
);
const headers: Record<string, string> =
parsed.provider === "openai"
? {
Authorization: `Bearer ${credential}`,
"OpenAI-Organization": parsed.accountId,
}
: { "x-api-key": credential, "anthropic-version": "2023-06-01" };
headers["User-Agent"] = "Paperclip/1.0 (https://paperclip.ing)";
const totals = new Map<string, bigint>();
const seen = new Set<string>();
const days = new Set<number>();
// Missing requested scopes in an otherwise complete daily report mean zero.
for (let time = from; time < to; time += 86400000)
for (const scope of parsed.scopeIds)
totals.set(`${new Date(time).toISOString().slice(0, 10)}:${scope}`, 0n);
const cursors = new Set<string>();
const signal = AbortSignal.timeout(30_000);
if (parsed.provider === "anthropic") {
const identity = z.object({ id: z.string() }).parse(
await boundedJson(
await fetcher("https://api.anthropic.com/v1/organizations/me", {
headers,
signal,
redirect: "error",
}),
),
);
if (identity.id !== parsed.accountId)
throw unprocessable(
"The admin credential belongs to a different provider organization.",
);
}
for (let page = 0; page < 100; page++) {
const response = await fetcher(base, {
headers,
signal,
redirect: "error",
});
const body = pageSchema.parse(await boundedJson(response));
for (const bucket of body.data) {
const start =
parsed.provider === "openai"
? (bucket.start_time ?? NaN) * 1000
: Date.parse(bucket.starting_at ?? "");
const end =
parsed.provider === "openai"
? (bucket.end_time ?? NaN) * 1000
: Date.parse(bucket.ending_at ?? "");
if (
!Number.isFinite(start) ||
start < from ||
end > to ||
end - start !== 86400000 ||
start % 86400000 !== 0
)
throw unprocessable("Provider returned an unexpected billing period.");
days.add(start);
for (const result of bucket.results) {
const scopeField = parsed.provider === "openai" ? "project_id" : "workspace_id";
if (!Object.hasOwn(result, scopeField)) {
throw unprocessable("Provider report is missing the requested project/workspace grouping.");
}
const scope =
(parsed.provider === "openai"
? result.project_id
: result.workspace_id) ?? "default";
if (!parsed.scopeIds.includes(scope)) continue;
const identity = `${new Date(start).toISOString().slice(0, 10)}:${scope}`;
if (seen.has(identity))
throw unprocessable("Provider returned overlapping billing buckets.");
seen.add(identity);
const currency =
typeof result.amount === "object"
? result.amount.currency
: result.currency;
if (currency?.toUpperCase() !== "USD")
throw unprocessable("Only USD provider cost reports are supported.");
const units =
parsed.provider === "openai" && typeof result.amount === "object"
? usdToUnits(result.amount.value)
: parsed.provider === "anthropic" &&
typeof result.amount === "string"
? centsToUnits(result.amount)
: -1n;
if (units < 0n)
throw unprocessable("Provider returned an invalid billing amount.");
totals.set(identity, units);
}
}
if (!body.has_more) {
if (days.size !== (to - from) / 86400000)
throw unprocessable(
"Provider report is incomplete; no financial events were changed.",
);
return [...totals].map(([key, value]) => ({
day: key.slice(0, 10),
scopeId: key.slice(11),
amountCents: unitsToCents(value),
}));
}
if (!body.next_page || cursors.has(body.next_page))
throw unprocessable("Provider returned an invalid billing cursor.");
cursors.add(body.next_page);
base.searchParams.set("page", body.next_page);
}
throw unprocessable("Provider billing report exceeded the page limit.");
}
export async function importProviderDailyCosts(
db: Db,
companyId: string,
raw: ImportProviderCosts,
actorId: string,
fetcher = fetch,
) {
const input = importProviderCostsSchema.parse(raw);
const [secret] = await db
.select()
.from(companySecrets)
.where(
and(
eq(companySecrets.id, input.secretId),
eq(companySecrets.companyId, companyId),
),
);
if (!secret || secret.scope !== "company")
throw forbidden("Select a company billing credential.");
// Metadata is an explicit operator designation, not the secret-storage
// provider (which may be local encryption, AWS, etc.). Never send a generic
// company secret to an external API merely because the caller knows its ID.
const designation = secret.providerMetadata?.providerBilling;
if (!designation || typeof designation !== "object" || Array.isArray(designation)
|| !("provider" in designation) || designation.provider !== input.provider
|| !("accountId" in designation) || designation.accountId !== input.accountId) {
throw forbidden("Select a company secret designated for this billing provider and account.");
}
const credential = await secretService(db).resolveSecretValue(
companyId,
secret.id,
"latest",
{
accessContext: {
consumerType: "system",
consumerId: "provider-billing-import",
actorType: "user",
actorId,
},
},
);
// Compare database revisions, not application clocks: concurrent importers
// can run on different hosts or start in the same millisecond.
const snapshots = await db
.select()
.from(providerBillingSnapshots)
.where(
and(
eq(providerBillingSnapshots.companyId, companyId),
eq(providerBillingSnapshots.provider, input.provider),
eq(providerBillingSnapshots.accountId, input.accountId),
inArray(providerBillingSnapshots.scopeId, input.scopeIds),
gte(providerBillingSnapshots.day, input.from),
lt(providerBillingSnapshots.day, input.to),
),
);
const expectedRevisions = new Map(
snapshots.map((row) => [`${row.day}:${row.scopeId}`, row.revision]),
);
const daily = await fetchProviderDailyCosts(input, credential, fetcher);
return applyProviderDailyCosts(
db,
companyId,
input,
daily,
actorId,
expectedRevisions,
);
}
export async function applyProviderDailyCosts(
db: Db,
companyId: string,
input: ImportProviderCosts,
daily: DailyProviderCost[],
actorId: string,
expectedRevisions?: ReadonlyMap<string, number>,
) {
return withAccountingTransaction(db, companyId, async (tx, publications) => {
let eventsCreated = 0;
for (const day of daily) {
const where = and(
eq(providerBillingSnapshots.companyId, companyId),
eq(providerBillingSnapshots.provider, input.provider),
eq(providerBillingSnapshots.accountId, input.accountId),
eq(providerBillingSnapshots.scopeId, day.scopeId),
eq(providerBillingSnapshots.day, day.day),
);
const [previous] = await tx
.select()
.from(providerBillingSnapshots)
.where(where);
const delta =
centsToUnits(day.amountCents) -
centsToUnits(previous?.amountCents ?? "0");
if (previous && delta === 0n) continue;
if (
expectedRevisions &&
(previous?.revision ?? 0) !==
(expectedRevisions.get(`${day.day}:${day.scopeId}`) ?? 0)
)
throw unprocessable(
"A newer provider report was imported. Fetch the report again.",
);
const revision = (previous?.revision ?? 0) + 1;
const [snapshot] = previous
? await tx
.update(providerBillingSnapshots)
.set({
amountCents: day.amountCents,
revision,
updatedAt: new Date(),
})
.where(where)
.returning()
: await tx
.insert(providerBillingSnapshots)
.values({
companyId,
provider: input.provider,
accountId: input.accountId,
...day,
revision,
})
.returning();
if (!delta) continue;
await createFinanceEventInTransaction(
tx,
publications,
companyId,
{
idempotencyKey: `provider-report:${snapshot.id}:${revision}`,
biller: input.provider,
eventKind: "inference_charge",
direction: delta < 0n ? "credit" : "debit",
currency: "USD",
amountCents: unitsToCents(delta < 0n ? -delta : delta),
occurredAt: new Date(`${day.day}T00:00:00.000Z`),
description:
`${input.provider} cost report · ${day.scopeId} · ${day.day}${revision > 1 ? " (correction)" : ""}`.slice(
0,
500,
),
metadataJson: {
source: "provider_cost_report",
accountId: input.accountId,
scopeId: day.scopeId,
revision,
totalCents: day.amountCents,
},
},
{ actorType: "user", actorId },
);
eventsCreated++;
}
return { daysRead: daily.length, eventsCreated };
});
}
server/src/services/receipt-fingerprint.ts
+12
View File
@@ -0,0 +1,12 @@
import { createHash } from "node:crypto";
function canonical(value: unknown): unknown {
if (value instanceof Date) return value.toISOString();
if (Array.isArray(value)) return value.map(canonical);
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, canonical(item)]));
return value;
}
export function receiptFingerprint(value: unknown): string {
return createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex");
}
server/src/services/recovery/service.ts
+6
View File
@@ -3835,6 +3835,12 @@ export function recoveryService(
current.companyId,
current.id,
);
// Budget admission may wait behind another recovery worker. Read the
// execution paths after the persisted counter so a newly reserved
// successor cannot be mistaken for permission to schedule the next one.
const currentState = await collectDispositionRepairSourceState(db, { issue: current });
if (currentState.hasActiveExecutionPath || currentState.hasDurableWaitingPath) return "skipped";
if (!episode && currentState.fingerprint !== state.fingerprint) return "skipped";
const runAttempt =
previousAttempt?.fingerprint === state.fingerprint
? previousAttempt.attempt
server/src/services/run-cost-accounting.ts
+196
View File
@@ -0,0 +1,196 @@
import { usdToUnits, unitsToCents } from "@paperclipai/shared";
import { and, asc, eq, inArray, sql } from "drizzle-orm";
import { accountingRuntimeBaselines, budgetReservations, agentRuntimeState, agents, costEvents, heartbeatRuns, nativeRunFinalizations, issues, projects, type Db } from "@paperclipai/db";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { conflict } from "../errors.js";
import { createCostEventInTransaction } from "./costs.js";
import { budgetService, deliverBudgetEnforcement, type BudgetServiceHooks } from "./budgets.js";
import { logger } from "../middleware/logger.js";
import { promises as fs } from "node:fs";
import { replayUsageReceipts, indexPendingUsageReceipts, recoverPendingRunUsageReceipts, type UsageReceiptIndex } from "./usage-receipts.js";
const terminalStatuses = ["succeeded", "failed", "timed_out", "cancelled", "interrupted"];
const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
const text = (value: unknown) => typeof value === "string" && value.length > 0 ? value : null;
const amount = (value: unknown) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
/** The run's persisted usage is a durable receipt. Ledger, runtime totals, and
* acknowledgement commit together, including after a restart or failed run. */
export async function accountRunCost(db: Db, runId: string, hooks: BudgetServiceHooks = {}, receiptIndex?: UsageReceiptIndex) {
const [identity] = await db.select({ companyId: heartbeatRuns.companyId, status: heartbeatRuns.status,
pending: heartbeatRuns.costAccountingPending, accountedAt: heartbeatRuns.costAccountedAt,
issueId: sql<string | null>`${heartbeatRuns.usageJson}->'ledgerScope'->>'issueId'`,
}).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
if (!identity) return false;
// Finalization can call twice. Avoid disk work for settled or active runs,
// but retain the locked recheck, stale-marker cleanup, and stop delivery.
const index = identity.pending && !identity.accountedAt && terminalStatuses.includes(identity.status)
? receiptIndex ?? await indexPendingUsageReceipts() : undefined;
let recoveredFiles: string[] = [];
const accounted = await withAccountingTransaction(db, identity.companyId, async (tx, publications) => {
// Native recovery owns the issue before updating its run. Acquire the
// charge's issue FK lock first so inserting it cannot reverse that order.
const issueId = text(identity.issueId);
if (issueId) await tx.select({ id: issues.id }).from(issues).where(and(
eq(issues.companyId, identity.companyId), eq(issues.id, issueId),
)).for("key share");
let [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update");
if (!run?.costAccountingPending || !terminalStatuses.includes(run.status)) return false;
if (run.costAccountedAt) {
await tx.update(heartbeatRuns).set({ costAccountingPending: false }).where(eq(heartbeatRuns.id, run.id));
return false;
}
// The bounded global replay can leave a newer receipt behind an older
// complete snapshot. Drain this run under the accounting lock before
// deciding completeness, and retain disk evidence until the outer commit.
recoveredFiles = await recoverPendingRunUsageReceipts(tx, { companyId: identity.companyId, runId }, undefined, { retainFiles: true, index });
[run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
if (text(object(object(run.usageJson).ledgerScope).issueId) !== issueId) {
throw conflict("Run cost attribution changed. Retry accounting.");
}
const preProviderFailure = object(object(run.resultJson).executionRecovery).providerWorkStarted === false
|| object(run.usageJson).accountingProviderWorkStarted === false;
// A failed native turn is not necessarily the end of this heartbeat: the
// coordinator can resume the same run. Only its closed provider boundary
// authorizes acknowledgement; finalization may still run after a result.
if (run.runtimeMode === "native" && !preProviderFailure) {
const [coordinator] = await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));
if (coordinator && !coordinator.resultId && coordinator.phase !== "terminal_failure") return false;
}
// A stop can mark the run terminal before the provider's shutdown receipt
// arrives. Keep the debt pending instead of acknowledging an invented zero
// and permanently discarding that late receipt.
if (!preProviderFailure && object(run.usageJson).accountingReceiptReady === false) return false;
// Acknowledgement is part of the same transaction. Budget checks inside
// this transaction must not mistake this receipt for missing accounting.
await tx.update(heartbeatRuns).set({ costAccountingPending: false, costAccountedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
await tx.update(budgetReservations).set({ state: preProviderFailure ? "released" : "settled", settledAt: new Date() })
.where(and(eq(budgetReservations.companyId, run.companyId), eq(budgetReservations.runId, run.id), eq(budgetReservations.state, "held")));
if (preProviderFailure) return true;
const usage = object(run.usageJson);
const scope = object(usage.ledgerScope);
// Deleted attribution targets become unallocated; a deletion must never
// prevent recording provider spend. Foreign-company references still fail.
for (const [table, key] of [[issues, "issueId"], [projects, "projectId"]] as const) {
const id = text(scope[key]);
if (id) {
const [linked] = await tx.select({ id: table.id }).from(table).where(eq(table.id, id));
if (!linked) scope[key] = null;
}
}
const costUsd = amount(usage.cacheAdjustedCostUsd) ?? text(usage.costUsdExact) ?? amount(usage.costUsd);
const billingType = text(usage.billingType) ?? "unknown";
const costUnits = billingType === "subscription_included" ? 0n : usdToUnits(costUsd ?? 0);
const costCents = unitsToCents(costUnits);
const [agent] = await tx.select({ adapterType: agents.adapterType }).from(agents).where(eq(agents.id, run.agentId));
const hasBillableEvidence = agent?.adapterType !== "process" || costUsd !== null || usage.costStatus === "unpriced"
|| ["inputTokens", "cachedInputTokens", "outputTokens"].some(key => (amount(usage[key]) ?? 0) > 0)
|| (text(usage.provider) !== null && usage.provider !== "unknown");
const receipt = {
idempotencyKey: `heartbeat:${run.id}:final`,
agentId: run.agentId, heartbeatRunId: run.id,
issueId: text(scope.issueId), projectId: text(scope.projectId), billingCode: text(scope.billingCode),
provider: text(usage.provider) ?? "unknown", biller: text(usage.biller) ?? text(usage.provider) ?? "unknown",
providerRequestId: text(usage.providerRequestId),
pricingProvenance: object(usage.pricingProvenance),
model: text(usage.model) ?? "unknown", billingType,
costStatus: (costUsd === null || usage.costStatus === "unpriced") && billingType !== "subscription_included" && hasBillableEvidence ? "unpriced" : usage.costStatus === "estimated" ? "estimated" : "reported",
inputTokens: amount(usage.inputTokens) ?? 0, cachedInputTokens: amount(usage.cachedInputTokens) ?? 0, outputTokens: amount(usage.outputTokens) ?? 0,
costCents, occurredAt: run.finishedAt ?? run.createdAt,
};
const parts = Array.isArray(usage.usageByModel) ? usage.usageByModel.map(object) : [];
const tokensMatch = ["inputTokens", "cachedInputTokens", "outputTokens"].every((key) =>
parts.reduce((sum, part) => sum + (amount(object(part.usage)[key]) ?? 0), 0) === (amount(usage[key]) ?? 0));
const costMatches = costUsd !== null && parts.reduce((sum, part) => sum + usdToUnits(amount(part.costUsd) ?? 0), 0n) === usdToUnits(costUsd);
const validParts = parts.length > 0 && tokensMatch && costMatches
&& parts.every((part) => text(part.model) && amount(part.costUsd) !== null)
&& new Set(parts.map((part) => part.model)).size === parts.length;
if (validParts) {
let allocated = 0n;
for (const [index, part] of parts.entries()) {
const partUsage = object(part.usage);
const remainder = costUnits - allocated;
const reported = usdToUnits(amount(part.costUsd) ?? 0);
const partUnits = billingType === "subscription_included" ? 0n : index === parts.length - 1 ? remainder : reported < remainder ? reported : remainder;
allocated += partUnits;
const partCents = unitsToCents(partUnits);
await createCostEventInTransaction(tx, run.companyId, {
...receipt, idempotencyKey: `heartbeat:${run.id}:model:${index}`, model: text(part.model)!,
inputTokens: amount(partUsage.inputTokens) ?? 0, cachedInputTokens: amount(partUsage.cachedInputTokens) ?? 0,
outputTokens: amount(partUsage.outputTokens) ?? 0, costCents: partCents,
}, publications);
}
} else {
await createCostEventInTransaction(tx, run.companyId, receipt, publications);
}
const [event] = await tx.select({
inputTokens: sql<string>`coalesce(sum(${costEvents.inputTokens}),0)::text`,
cachedInputTokens: sql<string>`coalesce(sum(${costEvents.cachedInputTokens}),0)::text`,
outputTokens: sql<string>`coalesce(sum(${costEvents.outputTokens}),0)::text`,
costCents: sql<string>`coalesce(sum(${costEvents.costCents}),0)::text`,
}).from(costEvents).where(and(eq(costEvents.companyId, run.companyId), eq(costEvents.heartbeatRunId, run.id)));
await tx.insert(agentRuntimeState).values({ agentId: run.agentId, companyId: run.companyId, adapterType: agent.adapterType }).onConflictDoNothing();
await tx.execute(sql`insert into accounting_runtime_baselines (agent_id,company_id,cost_cents,input_tokens,cached_input_tokens,output_tokens)
select agent_id,company_id,total_cost_cents,total_input_tokens,total_cached_input_tokens,total_output_tokens
from agent_runtime_state where agent_id = ${run.agentId} on conflict do nothing`);
await tx.update(agentRuntimeState).set({
totalInputTokens: sql`${agentRuntimeState.totalInputTokens} + ${event.inputTokens}`,
totalCachedInputTokens: sql`${agentRuntimeState.totalCachedInputTokens} + ${event.cachedInputTokens}`,
totalOutputTokens: sql`${agentRuntimeState.totalOutputTokens} + ${event.outputTokens}`,
totalCostCents: sql`${agentRuntimeState.totalCostCents} + ${event.costCents}::numeric`,
updatedAt: new Date(),
}).where(eq(agentRuntimeState.agentId, run.agentId));
await tx.update(heartbeatRuns).set({ costAccountingPending: false, costAccountedAt: new Date(), accountingProjectionVersion: "v2", accountingLastError: null, accountingLastAttemptAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
return true;
});
for (const file of recoveredFiles) {
try { await fs.rm(file, { force: true }); }
catch (error) { logger.warn({ err: error, runId }, "Accounting committed; recovered receipt cleanup will retry"); }
}
await deliverBudgetEnforcement(db, hooks, identity.companyId);
return accounted;
}
export async function reconcileRunCosts(db: Db, hooks: BudgetServiceHooks = {}) {
const pending = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.costAccountingPending, true), inArray(heartbeatRuns.status, terminalStatuses),
sql`(${heartbeatRuns.resultJson}->'executionRecovery'->>'providerWorkStarted' = 'false'
or ${heartbeatRuns.usageJson}->>'accountingProviderWorkStarted' = 'false'
or not exists (select 1 from ${nativeRunFinalizations} where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
and ${heartbeatRuns.runtimeMode} = 'native' and ${nativeRunFinalizations.resultId} is null
and ${nativeRunFinalizations.phase} <> 'terminal_failure'))`,
sql`(${heartbeatRuns.usageJson}->>'accountingReceiptReady' is distinct from 'false' or ${heartbeatRuns.resultJson}->'executionRecovery'->>'providerWorkStarted' = 'false' or ${heartbeatRuns.usageJson}->>'accountingProviderWorkStarted' = 'false')`,
)).orderBy(asc(heartbeatRuns.updatedAt), asc(heartbeatRuns.id)).limit(100);
let accounted = 0;
const receiptIndex = pending.length > 0 ? await indexPendingUsageReceipts() : undefined;
for (const run of pending) {
try { if (await accountRunCost(db, run.id, hooks, receiptIndex)) accounted++; }
catch (error) {
// Move failed attempts behind other pending work so a poisoned receipt
// cannot monopolize the bounded batch on every recovery tick.
await db.update(heartbeatRuns).set({ updatedAt: new Date(), accountingLastAttemptAt: new Date(), accountingAttemptCount: sql`${heartbeatRuns.accountingAttemptCount} + 1`, accountingLastError: error instanceof Error ? error.message.slice(0, 1000) : "Accounting failed" }).where(eq(heartbeatRuns.id, run.id));
logger.error({ err: error, runId: run.id }, "Run accounting remains pending for retry");
}
}
await budgetService(db, hooks).reconcilePolicies();
return { scanned: pending.length, accounted };
}
/** Share one full sweep across overlapping scheduler ticks. Do not enqueue
* another sweep: the next tick retries after either success or failure. */
export function createCostAccountingReconciler(db: Db, hooks: BudgetServiceHooks = {}) {
let inFlight: Promise<Awaited<ReturnType<typeof reconcileRunCosts>>> | undefined;
return () => {
if (!inFlight) {
inFlight = (async () => {
try {
await replayUsageReceipts(db);
return await reconcileRunCosts(db, hooks);
} finally {
inFlight = undefined;
}
})();
}
return inFlight;
};
}
server/src/services/runtime-skill-cache.ts
+14 -2
View File
@@ -165,6 +165,18 @@ async function setTreeMode(directory: string, readonly: boolean): Promise<void>
if (readonly) await fs.chmod(directory, 0o555);
}
// Darwin requires a directory being renamed to be writable by its owner.
// Change only the directory itself, while the publication lock is held; the
// contents remain read-only. Restore the mode even when publication fails.
async function renameCacheDirectory(source: string, destination: string) {
const stat = await fs.lstat(source);
const directory = stat.isDirectory() && !stat.isSymbolicLink();
if (directory) await fs.chmod(source, 0o700);
let moved = false;
try { await fs.rename(source, destination); moved = true; }
finally { if (directory) await fs.chmod(moved ? destination : source, stat.mode & 0o777); }
}
async function removeTree(directory: string): Promise<void> {
await setTreeMode(directory, false);
await fs.rm(directory, { recursive: true, force: true });
@@ -202,9 +214,9 @@ export async function resolveRuntimeSkillCache(
if (!await matches(spec, staging)) throw new Error("Runtime skill cache validation failed");
// Lifecycle mutations can update the DB while this builder owns the filesystem lock.
if (!await stillInstalled()) throw new Error("Skill was renamed or removed during preparation");
await fs.rename(spec.entry, path.join(spec.root, `.invalid-${spec.fingerprint}-${randomUUID()}`))
await renameCacheDirectory(spec.entry, path.join(spec.root, `.invalid-${spec.fingerprint}-${randomUUID()}`))
.catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; });
await fs.rename(staging, spec.entry);
await renameCacheDirectory(staging, spec.entry);
return path.join(spec.entry, "files");
} finally { await removeTree(staging); }
});
server/src/services/status-cards.ts
+3 -3
View File
@@ -179,7 +179,7 @@ export function statusCardService(
: Promise.resolve(null),
db.select({
tokens: sql<number>`coalesce(sum(coalesce(${statusCardUpdates.inputTokens}, 0) + coalesce(${statusCardUpdates.outputTokens}, 0)), 0)::int`,
costCents: sql<number>`coalesce(sum(${statusCardUpdates.costCents}), 0)::int`,
costCents: sql<number>`coalesce(sum(${statusCardUpdates.costCents}), 0)::double precision`,
})
.from(statusCardUpdates)
.where(and(eq(statusCardUpdates.cardId, card.id), gte(statusCardUpdates.startedAt, dayStart)))
@@ -877,7 +877,7 @@ export function statusCardService(
? await tx.select({
inputTokens: sql<number>`coalesce(sum(${costEvents.inputTokens}), 0)::int`,
outputTokens: sql<number>`coalesce(sum(${costEvents.outputTokens}), 0)::int`,
costCents: sql<number>`coalesce(sum(${costEvents.costCents}), 0)::int`,
costCents: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
}).from(costEvents).where(eq(costEvents.heartbeatRunId, actor.runId))
: [];
const existingUpdate = await tx.select().from(statusCardUpdates)
@@ -892,7 +892,7 @@ export function statusCardService(
changeSummary: input.changeSummary,
inputTokens: Number(usage[0]?.inputTokens ?? 0),
outputTokens: Number(usage[0]?.outputTokens ?? 0),
costCents: Number(usage[0]?.costCents ?? 0),
costCents: sql`${usage[0]?.costCents ?? "0"}::numeric`,
};
if (existingUpdate) {
await tx.update(statusCardUpdates).set(updateValues).where(eq(statusCardUpdates.id, existingUpdate.id));
server/src/services/usage-receipts.ts
+289
View File
@@ -0,0 +1,289 @@
import { priceCodexReceipt } from "./codex-pricing.js";
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import path from "node:path";
import { z } from "zod";
import { and, eq, isNull, sql } from "drizzle-orm";
import { heartbeatRuns, runUsageReceipts, type Db } from "@paperclipai/db";
import type { AdapterExecutionResult, AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import { pricingProvenanceSchema, usdToUnits } from "@paperclipai/shared";
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
import { conflict, notFound } from "../errors.js";
import { logger } from "../middleware/logger.js";
import { receiptFingerprint } from "./receipt-fingerprint.js";
import { withAccountingTransaction } from "./accounting-transaction.js";
import { ensureDurableDirectory, syncDirectory } from "../lib/durable-directory.js";
const token = z.number().int().nonnegative().max(2_147_483_647);
const price = z.number().finite().nonnegative().nullable().optional();
const usage = z.object({ inputTokens: token, outputTokens: token, cachedInputTokens: token.optional(), cacheWriteTokens: token.optional() });
const checkpointSchema = z.object({
attemptId: z.string().uuid().optional(), usage: usage.optional(),
usageByModel: z.array(z.object({ model: z.string().min(1).max(250), usage, costUsd: z.number().finite().nonnegative() })).max(500).optional(),
usageBasis: z.enum(["per_run", "session_cumulative"]).nullable().optional(),
provider: z.string().max(250).nullable().optional(), biller: z.string().max(250).nullable().optional(),
model: z.string().max(250).nullable().optional(), billingType: z.string().max(100).nullable().optional(),
costUsd: price, cacheAdjustedCostUsd: price,
costStatus: z.enum(["reported", "estimated", "unpriced"]).optional(),
pricingContext: z.object({ serviceTier: z.string().max(50).optional(), contextTier: z.enum(["short", "long"]).optional() }).optional(),
pricingProvenance: pricingProvenanceSchema.optional(),
costUsdExact: z.string().max(160).refine(value => { try { return !value.startsWith("-") && usdToUnits(value) >= 0n; } catch { return false; } }, "Invalid exact USD amount").nullable().optional(),
providerRequestId: z.string().max(250).nullable().optional(), complete: z.boolean(),
});
const envelopeSchema = z.object({
schema: z.literal("paperclip/accounting-receipt/v1"), id: z.string().uuid(), companyId: z.string().uuid(),
runId: z.string().uuid(), sourceId: z.string().uuid(), sequence: z.number().int().positive(), receivedAt: z.iso.datetime(),
adapterType: z.string().max(100), receipt: checkpointSchema,
});
export type UsageReceiptEnvelope = z.infer<typeof envelopeSchema>;
export const usageReceiptSpoolPath = () => path.join(resolvePaperclipInstanceRoot(), "accounting-receipts");
export async function spoolUsageReceipt(envelope: UsageReceiptEnvelope, directory = usageReceiptSpoolPath()) {
const parsed = envelopeSchema.parse(envelope);
await ensureDurableDirectory(directory);
const target = path.join(directory, `${parsed.id}.json`);
const temporary = `${target}.${randomUUID()}.tmp`;
const handle = await fs.open(temporary, "wx", 0o600);
try {
try { await handle.writeFile(JSON.stringify(parsed)); await handle.sync(); }
finally { await handle.close(); }
await fs.rename(temporary, target); await syncDirectory(directory);
} catch (error) { await fs.rm(temporary, { force: true }); throw error; }
return target;
}
export async function persistUsageReceipt(db: Db, raw: UsageReceiptEnvelope) {
const envelope = envelopeSchema.parse(raw);
const hash = receiptFingerprint(envelope);
return withAccountingTransaction(db, envelope.companyId, async tx => {
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, envelope.runId), eq(heartbeatRuns.companyId, envelope.companyId))).for("update");
if (!run) throw notFound("Accounting run not found");
const [existing] = await tx.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.companyId, envelope.companyId), eq(runUsageReceipts.runId, envelope.runId), eq(runUsageReceipts.sourceId, envelope.sourceId), eq(runUsageReceipts.sequence, envelope.sequence)));
if (existing && existing.receiptHash !== hash) throw conflict("Conflicting usage checkpoint replay");
if (!existing) await tx.insert(runUsageReceipts).values({
id: envelope.id, companyId: envelope.companyId, runId: envelope.runId, sourceId: envelope.sourceId,
sequence: envelope.sequence, receiptHash: hash, receiptJson: envelope, complete: envelope.receipt.complete, receivedAt: new Date(envelope.receivedAt),
});
const previous = run.usageJson ?? {};
// A restarted/replaced controller owns a fresh source. Old spool files
// remain auditable but cannot overwrite its snapshots or acknowledged work.
if (run.costAccountedAt || previous.accountingReceiptSourceId !== envelope.sourceId || Number(previous.accountingReceiptSequence ?? 0) >= envelope.sequence) return;
const receipt = envelope.receipt;
const billingType = receipt.billingType === "api" ? "metered_api" : receipt.billingType === "subscription" ? "subscription_included" : receipt.billingType ?? "unknown";
await tx.update(heartbeatRuns).set({
costAccountingPending: true,
accountingLastError: null,
usageJson: { ...previous, ...receipt.usage,
provider: receipt.provider ?? "unknown", biller: receipt.biller ?? receipt.provider ?? "unknown", model: receipt.model ?? "unknown", billingType,
costUsd: receipt.costUsd ?? null, costUsdExact: receipt.costUsdExact ?? null, cacheAdjustedCostUsd: receipt.cacheAdjustedCostUsd ?? null,
usageByModel: receipt.usageByModel ?? null, usageSource: receipt.usageBasis ?? "per_run", providerRequestId: receipt.providerRequestId ?? null,
accountingReceiptReady: receipt.complete && receipt.usageBasis !== "session_cumulative", accountingReceiptId: envelope.id,
accountingReceiptSequence: envelope.sequence, accountingReceiptReceivedAt: envelope.receivedAt,
costStatus: receipt.costStatus ?? null,
pricingProvenance: receipt.pricingProvenance ?? { source: receipt.costUsd != null || receipt.costUsdExact != null ? "provider_reported" : "unknown", version: "accounting-receipt/v1" },
},
}).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt)));
});
}
export async function replayUsageReceipts(db: Db, directory = usageReceiptSpoolPath(), limit = 100) {
let files: string[];
try { files = (await fs.readdir(directory)).filter(name => name.endsWith(".json")).sort(); }
catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return { replayed: 0, failed: 0 }; throw error; }
let replayed = 0, failed = 0;
for (const name of files.slice(0, limit)) {
const file = path.join(directory, name);
try {
const info = await fs.lstat(file);
if (!info.isFile() || info.size > 1024 * 1024) throw new Error("Invalid accounting spool entry");
await persistUsageReceipt(db, envelopeSchema.parse(JSON.parse(await fs.readFile(file, "utf8"))));
await fs.rm(file, { force: true }); replayed++;
} catch (error) {
failed++;
// Rotate failures behind fresh entries without discarding the evidence.
await fs.rename(file, path.join(directory, `retry-${Date.now()}-${randomUUID()}.json`)).catch(() => undefined);
logger.error({ err: error, receiptFile: name }, "Accounting spool receipt remains pending");
}
}
return { replayed, failed };
}
type ReceiptIdentity = { companyId: unknown; runId: unknown } | null;
export type UsageReceiptIndex = Map<string, ReceiptIdentity>;
function receiptIdentity(raw: unknown): ReceiptIdentity {
return raw && typeof raw === "object" && "companyId" in raw && "runId" in raw
? { companyId: raw.companyId, runId: raw.runId } : null;
}
/** Build once per recovery batch, before acquiring any accounting lock.
* Spool publications are immutable, uniquely named files. Cache identities,
* never receipt contents: matching evidence is revalidated during settlement. */
export async function indexPendingUsageReceipts(directory = usageReceiptSpoolPath()): Promise<UsageReceiptIndex> {
const index: UsageReceiptIndex = new Map();
let names: string[];
try { names = await fs.readdir(directory); }
// Indexing is only an optimization. The locked drain reports missing or
// unreadable storage through the existing per-run recovery error path.
catch { return index; }
for (const name of names.filter(name => name.endsWith(".json"))) {
const file = path.join(directory, name);
try {
const info = await fs.lstat(file);
if (!info.isFile() || info.size > 1024 * 1024) { index.set(file, null); continue; }
index.set(file, receiptIdentity(JSON.parse(await fs.readFile(file, "utf8"))));
} catch (error) {
if (error instanceof SyntaxError) index.set(file, null);
// Unreadable or concurrently moved entries remain unknown and are
// checked again under the lock, preserving per-run failure reporting.
}
}
return index;
}
export async function recoverPendingRunUsageReceipts(db: Db, input: { companyId: string; runId: string }, directory = usageReceiptSpoolPath(), options: { retainFiles?: boolean; index?: UsageReceiptIndex } = {}) {
// The bounded sweep may not reach this run. Its durable evidence must
// reach the journal before source replacement or ledger acknowledgement.
// A failed save aborts the operation so recovery can retry.
const recovered = new Set<string>();
// A concurrent failed replay can rename a file after releasing its company
// lock. Rescan disappearing paths rather than certify an incomplete scan.
for (let scan = 0; scan < 3; scan++) {
let names: string[];
try { names = await fs.readdir(directory); }
catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return [...recovered]; throw error; }
let moved = false;
for (const name of names.filter(name => name.endsWith(".json"))) {
const file = path.join(directory, name);
if (recovered.has(file)) continue;
const indexed = options.index?.get(file);
if (indexed !== undefined && (indexed?.companyId !== input.companyId || indexed?.runId !== input.runId)) continue;
let raw: unknown;
try {
const info = await fs.lstat(file);
if (!info.isFile() || info.size > 1024 * 1024) continue;
raw = JSON.parse(await fs.readFile(file, "utf8"));
} catch (error) {
// The background replayer can move/remove a file while we scan. Invalid
// JSON remains on disk for its normal recovery/error reporting path.
if ((error as NodeJS.ErrnoException).code === "ENOENT") { moved = true; continue; }
if (error instanceof SyntaxError) continue;
throw error;
}
const identity = receiptIdentity(raw);
options.index?.set(file, identity);
if (identity?.companyId !== input.companyId || identity?.runId !== input.runId) continue;
await persistUsageReceipt(db, envelopeSchema.parse(raw));
// Settlement uses the caller's transaction: deleting before its commit
// would lose the only durable evidence if the later ledger write fails.
recovered.add(file);
if (!options.retainFiles) await fs.rm(file, { force: true });
}
if (!moved) return [...recovered];
}
throw new Error("Accounting spool changed during recovery; retry required");
}
export async function createRunUsageRecorder(db: Db, input: { companyId: string; runId: string; adapterType: string }, directory = usageReceiptSpoolPath()) {
const sourceId = randomUUID();
// Verify durable storage before starting paid work, including a directory
// sync. The database binding fences checkpoints from replaced controllers.
await ensureDurableDirectory(directory);
const probe = await fs.open(path.join(directory, `${sourceId}.probe`), "wx", 0o600);
try { await probe.sync(); } finally { await probe.close(); await fs.rm(path.join(directory, `${sourceId}.probe`)); }
await syncDirectory(directory);
await recoverPendingRunUsageReceipts(db, input, directory);
const bound = await db.update(heartbeatRuns).set({ usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ accountingReceiptSourceId: sourceId, accountingReceiptSequence: 0, accountingReceiptReady: false })}::jsonb` })
.where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id, runtimeMode: heartbeatRuns.runtimeMode, usageJson: heartbeatRuns.usageJson });
if (bound.length !== 1) throw conflict("Run cannot accept a new usage recorder");
let sequence = 0;
let currentAttempt = "default";
const attempts = new Map<string, AdapterUsageCheckpoint>();
// Native run deltas retain their baseline across same-run recovery. Restore
// the previous snapshot as the same attempt, never add it a second time.
// An empty final result must not erase the last durable observation.
const priorReceiptId = bound[0].usageJson?.accountingReceiptId;
if (bound[0].runtimeMode === "native" && typeof priorReceiptId === "string") {
const [previous] = await db.select().from(runUsageReceipts).where(and(
eq(runUsageReceipts.id, priorReceiptId), eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.companyId, input.companyId),
));
if (previous) attempts.set("default", envelopeSchema.parse(previous.receiptJson).receipt as AdapterUsageCheckpoint);
}
let chain = Promise.resolve();
let captureFailed = false;
async function capture(raw: AdapterUsageCheckpoint) {
const parsed = checkpointSchema.parse(raw) as AdapterUsageCheckpoint;
const attempt = parsed.attemptId ?? currentAttempt;
if (!attempts.has(attempt)) currentAttempt = attempt;
const previous = attempts.get(attempt);
const priced = priceCodexReceipt({ ...parsed, usage: parsed.usage ?? previous?.usage,
complete: parsed.complete && !(previous?.usage && !parsed.usage) });
priced.pricingProvenance ??= {
source: priced.costUsd != null || priced.costUsdExact != null || priced.cacheAdjustedCostUsd != null ? "provider_reported" : "unknown",
version: "accounting-receipt/v1",
};
attempts.set(attempt, priced);
const parts = [...attempts.values()];
const latest = attempts.get(currentAttempt)!;
const receipt: AdapterUsageCheckpoint = { ...latest, complete: latest.complete && !captureFailed, usage: {
inputTokens: parts.reduce((sum, item) => sum + (item.usage?.inputTokens ?? 0), 0),
cachedInputTokens: parts.reduce((sum, item) => sum + (item.usage?.cachedInputTokens ?? 0), 0),
...(parts.some(item => item.usage?.cacheWriteTokens !== undefined) ? { cacheWriteTokens: parts.reduce((sum, item) => sum + (item.usage?.cacheWriteTokens ?? 0), 0) } : {}),
outputTokens: parts.reduce((sum, item) => sum + (item.usage?.outputTokens ?? 0), 0),
} };
if (parts.length > 1) {
let total = 0n, priced = true;
for (const part of parts) {
const subscription = ["subscription", "subscription_included"].includes(part.billingType ?? "");
const cost = subscription ? 0 : part.cacheAdjustedCostUsd ?? part.costUsdExact ?? part.costUsd;
if (cost == null || (!subscription && part.costStatus === "unpriced")) priced = false;
if (cost != null) total += usdToUnits(cost);
}
if (new Set(parts.map(part => part.billingType ?? "unknown")).size > 1) receipt.billingType = "unknown";
const decimal = `${total / 1_000_000_000n}.${String(total % 1_000_000_000n).padStart(9, "0")}`;
receipt.costUsdExact = decimal;
receipt.costUsd = Number(decimal);
receipt.cacheAdjustedCostUsd = null;
// A rejected resume can report a complete, explicit zero before the
// successful retry. It contributes neither spend nor tokens and must
// not discard the successful attempt's known model attribution.
const attributed = parts.filter(part => {
const cost = part.cacheAdjustedCostUsd ?? part.costUsdExact ?? part.costUsd;
return !part.complete || part.costStatus === "unpriced" || cost == null || usdToUnits(cost) !== 0n
|| Object.values(part.usage ?? {}).some(value => (value ?? 0) !== 0);
});
receipt.usageByModel = attributed.length === 1 ? attributed[0].usageByModel : undefined;
receipt.costStatus = priced && parts.some(part => part.costStatus === "estimated") ? "estimated" : priced ? "reported" : "unpriced";
receipt.pricingProvenance = { source: receipt.costStatus === "estimated" ? "rate_card" : "unknown", version: "per-attempt/v1", evidence: "Sum of attempted-run receipts; component prices are retained in the immutable receipt journal." };
// Every attempted provider must supply a final receipt before the total
// can be considered complete, even if a later attempt succeeded.
receipt.complete = !captureFailed && parts.every(part => part.complete);
}
const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), ...input, sourceId, sequence: ++sequence, receivedAt: new Date().toISOString(), receipt };
const file = await spoolUsageReceipt(envelope, directory);
try { await persistUsageReceipt(db, envelope); await fs.rm(file, { force: true }); }
catch (error) { logger.warn({ err: error, runId: input.runId, receiptId: envelope.id }, "Usage receipt is durable on disk; database replay pending"); }
return receipt;
}
return {
capture: (receipt: AdapterUsageCheckpoint) => {
const result = chain.then(() => capture(receipt)); chain = result.then(() => undefined, () => { captureFailed = true; }); return result;
},
complete: async (result: AdapterExecutionResult) => {
await chain;
const final = checkpointSchema.parse({ ...result, complete: result.usageComplete !== false }) as AdapterUsageCheckpoint;
const previous = attempts.get(currentAttempt);
// A bounded stdout tail can lose a terminal record already saved from the
// full stream. Keep that stronger evidence, but never hide additional
// usage/charges or certify a snapshot inherited from a replaced recorder.
if (sequence > 0 && previous?.complete && result.usageComplete === false) {
const moreTokens = (["inputTokens", "cachedInputTokens", "cacheWriteTokens", "outputTokens"] as const)
.some(key => (final.usage?.[key] ?? 0) > (previous.usage?.[key] ?? 0));
const finalPrice = final.cacheAdjustedCostUsd ?? final.costUsdExact ?? final.costUsd;
const previousPrice = previous.cacheAdjustedCostUsd ?? previous.costUsdExact ?? previous.costUsd;
const moreCost = finalPrice != null && (previousPrice == null || usdToUnits(finalPrice) > usdToUnits(previousPrice));
if (!moreTokens && !moreCost) return capture(previous);
}
return capture(final);
},
};
}
server/vitest.config.ts
+6 -1
View File
@@ -5,6 +5,12 @@ import { defineConfig } from "vitest/config";
export default defineConfig({
resolve: {
alias: [
{
find: /^@paperclipai\/paperclip-runner\/live$/,
replacement: fileURLToPath(
new URL("../packages/paperclip-runner/src/live/index.ts", import.meta.url),
),
},
{
find: /^@paperclipai\/paperclip-runner$/,
replacement: fileURLToPath(
@@ -37,7 +43,6 @@ export default defineConfig({
isolate: true,
maxConcurrency: 1,
maxWorkers: 1,
minWorkers: 1,
pool: "forks",
// Server suites share process state and one embedded Postgres instance,
// so tests inside a file must run one at a time. Do not set