mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-24 10:13:11 +02:00
Keep fresh budget admission blocked without cancelling the native recovery needed to close accounting. Drain every matching disk receipt under the accounting lock and retain files until ledger commit. Add PostgreSQL rollback, backlog, movement, and budget-scope regressions. Co-Authored-By: Paperclip <noreply@paperclip.ing>
1034 lines
41 KiB
TypeScript
1034 lines
41 KiB
TypeScript
import { compareCents, normalizeCents } from "@paperclipai/shared";
|
|
import { and, desc, eq, gte, inArray, lt, ne, sql, or, isNull } from "drizzle-orm";
|
|
import type { Db } from "@paperclipai/db";
|
|
import {
|
|
agents,
|
|
approvals,
|
|
budgetIncidents,
|
|
budgetPolicies,
|
|
companies,
|
|
costEvents,
|
|
heartbeatRuns,
|
|
nativeRunFinalizations,
|
|
projects,
|
|
} from "@paperclipai/db";
|
|
import type {
|
|
BudgetIncident,
|
|
BudgetIncidentResolutionInput,
|
|
BudgetMetric,
|
|
BudgetOverview,
|
|
PauseReason,
|
|
BudgetPolicy,
|
|
BudgetPolicySummary,
|
|
BudgetPolicyUpsertInput,
|
|
BudgetScopeType,
|
|
BudgetThresholdType,
|
|
BudgetWindowKind,
|
|
} from "@paperclipai/shared";
|
|
import { HttpError, notFound, unprocessable } from "../errors.js";
|
|
import { logger } from "../middleware/logger.js";
|
|
import { withAccountingTransaction } from "./accounting-transaction.js";
|
|
import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js";
|
|
|
|
type ScopeRecord = {
|
|
companyId: string;
|
|
name: string;
|
|
paused: boolean;
|
|
pauseReason: PauseReason | null;
|
|
};
|
|
|
|
type PolicyRow = typeof budgetPolicies.$inferSelect;
|
|
type IncidentRow = typeof budgetIncidents.$inferSelect;
|
|
|
|
export type BudgetEnforcementScope = {
|
|
companyId: string;
|
|
scopeType: BudgetScopeType;
|
|
scopeId: string;
|
|
/** Snapshot taken under the company lock, before a later budget grant can admit new work. */
|
|
createdBefore?: Date;
|
|
/** Revalidated at the durable cancellation write, under the admission lock. */
|
|
enforcement?: { policyId: string; version: number };
|
|
};
|
|
|
|
export type BudgetServiceHooks = {
|
|
cancelWorkForScope?: (scope: BudgetEnforcementScope) => Promise<void>;
|
|
};
|
|
|
|
function currentUtcMonthWindow(now = new Date()) {
|
|
const year = now.getUTCFullYear();
|
|
const month = now.getUTCMonth();
|
|
const start = new Date(Date.UTC(year, month, 1, 0, 0, 0, 0));
|
|
const end = new Date(Date.UTC(year, month + 1, 1, 0, 0, 0, 0));
|
|
return { start, end };
|
|
}
|
|
|
|
function resolveWindow(windowKind: BudgetWindowKind, now = new Date()) {
|
|
if (windowKind === "lifetime") {
|
|
return {
|
|
start: new Date(Date.UTC(1970, 0, 1, 0, 0, 0, 0)),
|
|
end: new Date(Date.UTC(9999, 0, 1, 0, 0, 0, 0)),
|
|
};
|
|
}
|
|
return currentUtcMonthWindow(now);
|
|
}
|
|
|
|
function budgetStatusFromObserved(
|
|
observedAmount: string,
|
|
amount: number,
|
|
warnPercent: number,
|
|
): BudgetPolicySummary["status"] {
|
|
if (amount <= 0) return "ok";
|
|
if (compareCents(observedAmount, amount) >= 0) return "hard_stop";
|
|
if (compareCents(observedAmount, Math.ceil((amount * warnPercent) / 100)) >= 0) return "warning";
|
|
return "ok";
|
|
}
|
|
|
|
function normalizeScopeName(scopeType: BudgetScopeType, name: string) {
|
|
if (scopeType === "company") return name;
|
|
return name.trim().length > 0 ? name : scopeType;
|
|
}
|
|
|
|
async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
|
|
if (scopeType === "company") {
|
|
const row = await db
|
|
.select({
|
|
companyId: companies.id,
|
|
name: companies.name,
|
|
status: companies.status,
|
|
pauseReason: companies.pauseReason,
|
|
pausedAt: companies.pausedAt,
|
|
})
|
|
.from(companies)
|
|
.where(eq(companies.id, scopeId))
|
|
.then((rows) => rows[0] ?? null);
|
|
if (!row) throw notFound("Company not found");
|
|
return {
|
|
companyId: row.companyId,
|
|
name: row.name,
|
|
paused: row.status === "paused" || Boolean(row.pausedAt),
|
|
pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null,
|
|
};
|
|
}
|
|
|
|
if (scopeType === "agent") {
|
|
const row = await db
|
|
.select({
|
|
companyId: agents.companyId,
|
|
name: agents.name,
|
|
status: agents.status,
|
|
pauseReason: agents.pauseReason,
|
|
})
|
|
.from(agents)
|
|
.where(eq(agents.id, scopeId))
|
|
.then((rows) => rows[0] ?? null);
|
|
if (!row) throw notFound("Agent not found");
|
|
return {
|
|
companyId: row.companyId,
|
|
name: row.name,
|
|
paused: row.status === "paused",
|
|
pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null,
|
|
};
|
|
}
|
|
|
|
const row = await db
|
|
.select({
|
|
companyId: projects.companyId,
|
|
name: projects.name,
|
|
pauseReason: projects.pauseReason,
|
|
pausedAt: projects.pausedAt,
|
|
})
|
|
.from(projects)
|
|
.where(eq(projects.id, scopeId))
|
|
.then((rows) => rows[0] ?? null);
|
|
if (!row) throw notFound("Project not found");
|
|
return {
|
|
companyId: row.companyId,
|
|
name: row.name,
|
|
paused: Boolean(row.pausedAt),
|
|
pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null,
|
|
};
|
|
}
|
|
|
|
export async function computeObservedSpend(
|
|
db: Db,
|
|
policy: Pick<PolicyRow, "companyId" | "scopeType" | "scopeId" | "windowKind" | "metric">,
|
|
) {
|
|
if (policy.metric !== "billed_cents") return { total: 0, totalExact: "0.0000000", unpricedEventCount: 0, pendingRunCount: 0, recoveringRunCount: 0 };
|
|
|
|
const conditions = [eq(costEvents.companyId, policy.companyId)];
|
|
if (policy.scopeType === "agent") conditions.push(eq(costEvents.agentId, policy.scopeId));
|
|
if (policy.scopeType === "project") conditions.push(eq(costEvents.projectId, policy.scopeId));
|
|
const { start, end } = resolveWindow(policy.windowKind as BudgetWindowKind);
|
|
if (policy.windowKind === "calendar_month_utc") {
|
|
conditions.push(gte(costEvents.occurredAt, start));
|
|
conditions.push(lt(costEvents.occurredAt, end));
|
|
}
|
|
|
|
const [row] = await db
|
|
.select({
|
|
unpricedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'unpriced' and ${costEvents.billingType} <> 'subscription_included')::int`,
|
|
total: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
|
})
|
|
.from(costEvents)
|
|
.where(and(...conditions));
|
|
|
|
const pendingConditions = [eq(heartbeatRuns.companyId, policy.companyId), eq(heartbeatRuns.costAccountingPending, true),
|
|
inArray(heartbeatRuns.status, ["succeeded", "failed", "timed_out", "cancelled", "interrupted"])];
|
|
if (policy.scopeType === "agent") pendingConditions.push(eq(heartbeatRuns.agentId, policy.scopeId));
|
|
if (policy.scopeType === "project") pendingConditions.push(sql`${heartbeatRuns.usageJson}->'ledgerScope'->>'projectId' = ${policy.scopeId}`);
|
|
if (policy.windowKind === "calendar_month_utc") {
|
|
pendingConditions.push(sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) >= ${start.toISOString()}::timestamptz`, sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) < ${end.toISOString()}::timestamptz`);
|
|
}
|
|
const [pending] = await db.select({
|
|
count: sql<number>`count(*)::int`,
|
|
recovering: sql<number>`count(*) filter (where ${heartbeatRuns.runtimeMode} = 'native'
|
|
and exists (select 1 from ${nativeRunFinalizations}
|
|
where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
|
|
and ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId}
|
|
and ${nativeRunFinalizations.resultId} is null
|
|
and ${nativeRunFinalizations.phase} <> 'terminal_failure'))::int`,
|
|
}).from(heartbeatRuns).where(and(...pendingConditions));
|
|
return { total: Number(row?.total ?? 0), totalExact: normalizeCents(row?.total ?? 0), unpricedEventCount: Number(row?.unpricedEventCount ?? 0),
|
|
pendingRunCount: pending?.count ?? 0, recoveringRunCount: pending?.recovering ?? 0 };
|
|
}
|
|
|
|
async function computeObservedAmount(db: Db, policy: PolicyRow) {
|
|
return (await computeObservedSpend(db, policy)).total;
|
|
}
|
|
async function reached(db: Db, policy: PolicyRow, threshold: number) {
|
|
return compareCents((await computeObservedSpend(db, policy)).totalExact, threshold) >= 0;
|
|
}
|
|
function observedBlocks(policy: PolicyRow, observed: Awaited<ReturnType<typeof computeObservedSpend>>) {
|
|
if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false;
|
|
return compareCents(observed.totalExact, policy.amount) >= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || (policy.unpricedUsagePolicy !== "allow" && observed.unpricedEventCount > 0);
|
|
}
|
|
async function policyBlocks(db: Db, policy: PolicyRow) {
|
|
if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false;
|
|
return observedBlocks(policy, await computeObservedSpend(db, policy));
|
|
}
|
|
|
|
function buildApprovalPayload(input: {
|
|
policy: PolicyRow;
|
|
scopeName: string;
|
|
thresholdType: BudgetThresholdType;
|
|
amountObserved: number;
|
|
amountObservedExact: string;
|
|
windowStart: Date;
|
|
windowEnd: Date;
|
|
}) {
|
|
return {
|
|
scopeType: input.policy.scopeType,
|
|
scopeId: input.policy.scopeId,
|
|
scopeName: input.scopeName,
|
|
metric: input.policy.metric,
|
|
windowKind: input.policy.windowKind,
|
|
thresholdType: input.thresholdType,
|
|
budgetAmount: input.policy.amount,
|
|
observedAmount: input.amountObserved,
|
|
observedAmountExact: input.amountObservedExact,
|
|
warnPercent: input.policy.warnPercent,
|
|
windowStart: input.windowStart.toISOString(),
|
|
windowEnd: input.windowEnd.toISOString(),
|
|
policyId: input.policy.id,
|
|
guidance: "Resolve pending or unpriced accounting, raise the budget, or keep the scope paused.",
|
|
};
|
|
}
|
|
|
|
async function markApprovalStatus(
|
|
db: Db,
|
|
approvalId: string | null,
|
|
status: "approved" | "rejected",
|
|
decisionNote: string | null | undefined,
|
|
decidedByUserId: string,
|
|
) {
|
|
if (!approvalId) return;
|
|
await db
|
|
.update(approvals)
|
|
.set({
|
|
status,
|
|
decisionNote: decisionNote ?? null,
|
|
decidedByUserId,
|
|
decidedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(eq(approvals.id, approvalId));
|
|
}
|
|
|
|
/** Internal: caller must hold the company accounting transaction lock. */
|
|
export function budgetServiceInTransaction(db: Db, publications: ActivityPublication[] = []) {
|
|
const recordActivity = (input: LogActivityInput) => logActivity(db, input, publications);
|
|
async function pauseScopeForBudget(policy: PolicyRow) {
|
|
const now = new Date();
|
|
if (policy.scopeType === "agent") {
|
|
await db
|
|
.update(agents)
|
|
.set({
|
|
status: "paused",
|
|
pauseReason: "budget",
|
|
pausedAt: now,
|
|
updatedAt: now,
|
|
})
|
|
.where(and(eq(agents.id, policy.scopeId), inArray(agents.status, ["active", "idle", "running", "error"])));
|
|
return;
|
|
}
|
|
|
|
if (policy.scopeType === "project") {
|
|
await db
|
|
.update(projects)
|
|
.set({
|
|
pauseReason: "budget",
|
|
pausedAt: now,
|
|
updatedAt: now,
|
|
})
|
|
.where(and(eq(projects.id, policy.scopeId), or(isNull(projects.pausedAt), eq(projects.pauseReason, "budget"))));
|
|
return;
|
|
}
|
|
|
|
await db
|
|
.update(companies)
|
|
.set({
|
|
status: "paused",
|
|
pauseReason: "budget",
|
|
pausedAt: now,
|
|
updatedAt: now,
|
|
})
|
|
.where(and(eq(companies.id, policy.scopeId), or(eq(companies.status, "active"), and(eq(companies.status, "paused"), eq(companies.pauseReason, "budget")))));
|
|
}
|
|
|
|
async function pauseAndCancelScopeForBudget(policy: PolicyRow) {
|
|
await pauseScopeForBudget(policy);
|
|
await db.update(budgetPolicies).set({
|
|
enforcementVersion: sql`${budgetPolicies.enforcementVersion} + 1`,
|
|
}).where(eq(budgetPolicies.id, policy.id));
|
|
}
|
|
|
|
async function resumeScopeFromBudget(policy: PolicyRow) {
|
|
const policies = await db.select().from(budgetPolicies).where(and(
|
|
eq(budgetPolicies.companyId, policy.companyId), eq(budgetPolicies.scopeType, policy.scopeType),
|
|
eq(budgetPolicies.scopeId, policy.scopeId), eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true),
|
|
));
|
|
for (const candidate of policies) {
|
|
if (await policyBlocks(db, candidate)) return;
|
|
}
|
|
const now = new Date();
|
|
if (policy.scopeType === "agent") {
|
|
await db
|
|
.update(agents)
|
|
.set({
|
|
status: "idle",
|
|
pauseReason: null,
|
|
pausedAt: null,
|
|
updatedAt: now,
|
|
})
|
|
.where(and(eq(agents.id, policy.scopeId), eq(agents.status, "paused"), eq(agents.pauseReason, "budget")));
|
|
return;
|
|
}
|
|
|
|
if (policy.scopeType === "project") {
|
|
await db
|
|
.update(projects)
|
|
.set({
|
|
pauseReason: null,
|
|
pausedAt: null,
|
|
updatedAt: now,
|
|
})
|
|
.where(and(eq(projects.id, policy.scopeId), eq(projects.pauseReason, "budget")));
|
|
return;
|
|
}
|
|
|
|
await db
|
|
.update(companies)
|
|
.set({
|
|
status: "active",
|
|
pauseReason: null,
|
|
pausedAt: null,
|
|
updatedAt: now,
|
|
})
|
|
.where(and(eq(companies.id, policy.scopeId), eq(companies.status, "paused"), eq(companies.pauseReason, "budget")));
|
|
}
|
|
|
|
async function getPolicyRow(policyId: string) {
|
|
const policy = await db
|
|
.select()
|
|
.from(budgetPolicies)
|
|
.where(eq(budgetPolicies.id, policyId))
|
|
.then((rows) => rows[0] ?? null);
|
|
if (!policy) throw notFound("Budget policy not found");
|
|
return policy;
|
|
}
|
|
|
|
async function listPolicyRows(companyId: string) {
|
|
return db
|
|
.select()
|
|
.from(budgetPolicies)
|
|
.where(eq(budgetPolicies.companyId, companyId))
|
|
.orderBy(desc(budgetPolicies.updatedAt));
|
|
}
|
|
|
|
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
|
|
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
|
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = await computeObservedSpend(db, policy);
|
|
const { start, end } = resolveWindow(policy.windowKind as BudgetWindowKind);
|
|
const amount = policy.isActive ? policy.amount : 0;
|
|
const utilizationPercent =
|
|
amount > 0 ? Number(((observedAmount / amount) * 100).toFixed(2)) : 0;
|
|
return {
|
|
policyId: policy.id,
|
|
companyId: policy.companyId,
|
|
scopeType: policy.scopeType as BudgetScopeType,
|
|
scopeId: policy.scopeId,
|
|
scopeName: normalizeScopeName(policy.scopeType as BudgetScopeType, scope.name),
|
|
metric: policy.metric as BudgetMetric,
|
|
windowKind: policy.windowKind as BudgetWindowKind,
|
|
amount,
|
|
observedAmount,
|
|
observedAmountExact,
|
|
reservationCents: policy.reservationCents,
|
|
unpricedEventCount,
|
|
pendingRunCount,
|
|
unpricedUsagePolicy: policy.unpricedUsagePolicy as "block" | "allow",
|
|
remainingAmount: amount > 0 ? Math.max(0, amount - observedAmount) : 0,
|
|
utilizationPercent,
|
|
warnPercent: policy.warnPercent,
|
|
hardStopEnabled: policy.hardStopEnabled,
|
|
notifyEnabled: policy.notifyEnabled,
|
|
isActive: policy.isActive,
|
|
status: await policyBlocks(db, policy) ? "hard_stop" : policy.isActive
|
|
? budgetStatusFromObserved(observedAmountExact, amount, policy.warnPercent)
|
|
: "ok",
|
|
paused: scope.paused,
|
|
pauseReason: scope.pauseReason,
|
|
windowStart: start,
|
|
windowEnd: end,
|
|
};
|
|
}
|
|
|
|
async function createIncidentIfNeeded(
|
|
policy: PolicyRow,
|
|
thresholdType: BudgetThresholdType,
|
|
amountObserved: number,
|
|
) {
|
|
const { start, end } = resolveWindow(policy.windowKind as BudgetWindowKind);
|
|
const existing = await db
|
|
.select()
|
|
.from(budgetIncidents)
|
|
.where(
|
|
and(
|
|
eq(budgetIncidents.policyId, policy.id),
|
|
eq(budgetIncidents.windowStart, start),
|
|
eq(budgetIncidents.thresholdType, thresholdType),
|
|
or(eq(budgetIncidents.status, "open"), and(eq(budgetIncidents.status, "dismissed"), eq(budgetIncidents.amountLimit, policy.amount))),
|
|
),
|
|
)
|
|
.orderBy(desc(budgetIncidents.createdAt))
|
|
.then((rows) => rows[0] ?? null);
|
|
if (existing) return { incident: existing, created: false };
|
|
|
|
const amountObservedExact = (await computeObservedSpend(db, policy)).totalExact;
|
|
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
|
const payload = buildApprovalPayload({
|
|
policy,
|
|
scopeName: normalizeScopeName(policy.scopeType as BudgetScopeType, scope.name),
|
|
thresholdType,
|
|
amountObserved,
|
|
amountObservedExact,
|
|
windowStart: start,
|
|
windowEnd: end,
|
|
});
|
|
|
|
const approval = thresholdType === "hard"
|
|
? await db
|
|
.insert(approvals)
|
|
.values({
|
|
companyId: policy.companyId,
|
|
type: "budget_override_required",
|
|
requestedByUserId: null,
|
|
requestedByAgentId: null,
|
|
status: "pending",
|
|
payload,
|
|
})
|
|
.returning()
|
|
.then((rows) => rows[0] ?? null)
|
|
: null;
|
|
|
|
const incident = await db
|
|
.insert(budgetIncidents)
|
|
.values({
|
|
companyId: policy.companyId,
|
|
policyId: policy.id,
|
|
scopeType: policy.scopeType,
|
|
scopeId: policy.scopeId,
|
|
metric: policy.metric,
|
|
windowKind: policy.windowKind,
|
|
windowStart: start,
|
|
windowEnd: end,
|
|
thresholdType,
|
|
amountLimit: policy.amount,
|
|
amountObserved: sql`${amountObservedExact}::numeric`,
|
|
status: "open",
|
|
approvalId: approval?.id ?? null,
|
|
})
|
|
.returning()
|
|
.then((rows) => rows[0] ?? null);
|
|
return incident ? { incident, created: true } : null;
|
|
}
|
|
|
|
async function resolveOpenSoftIncidents(policyId: string) {
|
|
await db
|
|
.update(budgetIncidents)
|
|
.set({
|
|
status: "resolved",
|
|
resolvedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(
|
|
and(
|
|
eq(budgetIncidents.policyId, policyId),
|
|
eq(budgetIncidents.thresholdType, "soft"),
|
|
eq(budgetIncidents.status, "open"),
|
|
),
|
|
);
|
|
}
|
|
|
|
async function resolveOpenIncidentsForPolicy(
|
|
policyId: string,
|
|
approvalStatus: "approved" | "rejected" | null,
|
|
decidedByUserId: string | null,
|
|
thresholdType?: BudgetThresholdType,
|
|
) {
|
|
const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"),
|
|
thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined);
|
|
const openRows = await db
|
|
.select()
|
|
.from(budgetIncidents)
|
|
.where(condition);
|
|
|
|
await db
|
|
.update(budgetIncidents)
|
|
.set({
|
|
status: "resolved",
|
|
resolvedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(condition);
|
|
|
|
if (!approvalStatus || !decidedByUserId) return;
|
|
for (const row of openRows) {
|
|
await markApprovalStatus(db, row.approvalId ?? null, approvalStatus, "Resolved via budget update", decidedByUserId);
|
|
}
|
|
}
|
|
|
|
async function hydrateIncidentRows(rows: IncidentRow[]): Promise<BudgetIncident[]> {
|
|
const approvalIds = rows.map((row) => row.approvalId).filter((value): value is string => Boolean(value));
|
|
const approvalRows = approvalIds.length > 0
|
|
? await db
|
|
.select({ id: approvals.id, status: approvals.status })
|
|
.from(approvals)
|
|
.where(inArray(approvals.id, approvalIds))
|
|
: [];
|
|
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
|
|
|
|
return Promise.all(
|
|
rows.map(async (row) => {
|
|
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
|
|
return {
|
|
id: row.id,
|
|
companyId: row.companyId,
|
|
policyId: row.policyId,
|
|
scopeType: row.scopeType as BudgetScopeType,
|
|
scopeId: row.scopeId,
|
|
scopeName: normalizeScopeName(row.scopeType as BudgetScopeType, scope.name),
|
|
metric: row.metric as BudgetMetric,
|
|
windowKind: row.windowKind as BudgetWindowKind,
|
|
windowStart: row.windowStart,
|
|
windowEnd: row.windowEnd,
|
|
thresholdType: row.thresholdType as BudgetThresholdType,
|
|
amountLimit: row.amountLimit,
|
|
amountObserved: row.amountObserved,
|
|
status: row.status as BudgetIncident["status"],
|
|
approvalId: row.approvalId ?? null,
|
|
approvalStatus: row.approvalId ? approvalStatusById.get(row.approvalId) ?? null : null,
|
|
resolvedAt: row.resolvedAt ?? null,
|
|
createdAt: row.createdAt,
|
|
updatedAt: row.updatedAt,
|
|
};
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function reconcileScope(companyId: string, scopeType: BudgetScopeType, scopeId: string) {
|
|
const scope = await resolveScopeRecord(db, scopeType, scopeId);
|
|
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
|
|
const policies = await db.select().from(budgetPolicies).where(and(
|
|
eq(budgetPolicies.companyId, companyId), eq(budgetPolicies.scopeType, scopeType), eq(budgetPolicies.scopeId, scopeId),
|
|
));
|
|
let recoveringPolicy: PolicyRow | undefined;
|
|
for (const policy of policies) {
|
|
const observed = await computeObservedSpend(db, policy);
|
|
if (observedBlocks(policy, observed)) {
|
|
await createIncidentIfNeeded(policy, "hard", observed.total);
|
|
await pauseAndCancelScopeForBudget(policy);
|
|
const title = scopeType === "company" ? "Company" : scopeType === "agent" ? "Agent" : "Project";
|
|
if ((await computeObservedSpend(db, policy)).pendingRunCount > 0) {
|
|
return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work while completed runs await accounting.` };
|
|
}
|
|
if (policy.unpricedUsagePolicy !== "allow" && (await computeObservedSpend(db, policy)).unpricedEventCount > 0) {
|
|
return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work because recorded usage has no reliable price.` };
|
|
}
|
|
return { scopeType, scopeId, scopeName: scope.name, reason: scopeType === "project"
|
|
? "Project cannot start work because its budget hard-stop is still exceeded."
|
|
: `${title} is paused because its budget hard-stop was reached.` };
|
|
}
|
|
// An open native coordinator must finish the original run's accounting.
|
|
// Deny fresh work without pausing/cancelling that recovery or its hold.
|
|
if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
|
|
// A closed UTC window, inactive policy, or budget raise releases only
|
|
// budget-owned pauses. Manual pauses remain operator decisions.
|
|
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
|
|
if (!policy.isActive || !policy.notifyEnabled || !(await reached(db, policy, Math.ceil(policy.amount * policy.warnPercent / 100)))) {
|
|
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft");
|
|
}
|
|
}
|
|
// Agent status has its own admission gate and error code. Preserve a
|
|
// manual pause without misclassifying it as a budget failure.
|
|
if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") {
|
|
return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` };
|
|
}
|
|
if (scope.paused && scope.pauseReason === "budget" && !policies.length) {
|
|
return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." };
|
|
}
|
|
if (scope.pauseReason === "budget" && policies[0]) await resumeScopeFromBudget(policies[0]);
|
|
if (recoveringPolicy) return { scopeType, scopeId, scopeName: scope.name,
|
|
reason: "New work must wait while a native run recovers its unfinished accounting." };
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
reconcileScope,
|
|
listPolicies: async (companyId: string): Promise<BudgetPolicy[]> => {
|
|
const rows = await listPolicyRows(companyId);
|
|
return rows.map((row) => ({
|
|
...row,
|
|
scopeType: row.scopeType as BudgetScopeType,
|
|
unpricedUsagePolicy: row.unpricedUsagePolicy as "block" | "allow",
|
|
metric: row.metric as BudgetMetric,
|
|
windowKind: row.windowKind as BudgetWindowKind,
|
|
}));
|
|
},
|
|
|
|
upsertPolicy: async (
|
|
companyId: string,
|
|
input: BudgetPolicyUpsertInput,
|
|
actorUserId: string | null,
|
|
): Promise<BudgetPolicySummary> => {
|
|
const scope = await resolveScopeRecord(db, input.scopeType, input.scopeId);
|
|
if (scope.companyId !== companyId) {
|
|
throw unprocessable("Budget scope does not belong to company");
|
|
}
|
|
|
|
if (input.reservationCents !== undefined && compareCents(input.reservationCents, 0) < 0) throw unprocessable("Reservation cannot be negative");
|
|
const metric = input.metric ?? "billed_cents";
|
|
const windowKind = input.windowKind ?? (input.scopeType === "project" ? "lifetime" : "calendar_month_utc");
|
|
const existing = await db
|
|
.select()
|
|
.from(budgetPolicies)
|
|
.where(
|
|
and(
|
|
eq(budgetPolicies.companyId, companyId),
|
|
eq(budgetPolicies.scopeType, input.scopeType),
|
|
eq(budgetPolicies.scopeId, input.scopeId),
|
|
eq(budgetPolicies.metric, metric),
|
|
eq(budgetPolicies.windowKind, windowKind),
|
|
),
|
|
)
|
|
.then((rows) => rows[0] ?? null);
|
|
|
|
if (!existing && input.amount === undefined) throw unprocessable("Amount is required for a new budget policy");
|
|
const amount = Math.max(0, Math.floor(input.amount ?? existing!.amount));
|
|
const nextIsActive = amount > 0 && (input.isActive ?? existing?.isActive ?? true);
|
|
const now = new Date();
|
|
const row = existing
|
|
? await db
|
|
.update(budgetPolicies)
|
|
.set({
|
|
amount,
|
|
reservationCents: normalizeCents(input.reservationCents ?? existing.reservationCents ?? 0),
|
|
warnPercent: input.warnPercent ?? existing.warnPercent,
|
|
hardStopEnabled: input.hardStopEnabled ?? existing.hardStopEnabled,
|
|
notifyEnabled: input.notifyEnabled ?? existing.notifyEnabled,
|
|
unpricedUsagePolicy: input.unpricedUsagePolicy ?? existing.unpricedUsagePolicy,
|
|
isActive: nextIsActive,
|
|
updatedByUserId: actorUserId,
|
|
updatedAt: now,
|
|
})
|
|
.where(eq(budgetPolicies.id, existing.id))
|
|
.returning()
|
|
.then((rows) => rows[0])
|
|
: await db
|
|
.insert(budgetPolicies)
|
|
.values({
|
|
companyId,
|
|
scopeType: input.scopeType,
|
|
scopeId: input.scopeId,
|
|
metric,
|
|
windowKind,
|
|
amount,
|
|
reservationCents: normalizeCents(input.reservationCents ?? 0),
|
|
warnPercent: input.warnPercent ?? 80,
|
|
hardStopEnabled: input.hardStopEnabled ?? true,
|
|
notifyEnabled: input.notifyEnabled ?? true,
|
|
unpricedUsagePolicy: input.unpricedUsagePolicy ?? "block",
|
|
isActive: nextIsActive,
|
|
createdByUserId: actorUserId,
|
|
updatedByUserId: actorUserId,
|
|
})
|
|
.returning()
|
|
.then((rows) => rows[0]);
|
|
|
|
if (input.scopeType === "company" && windowKind === "calendar_month_utc") {
|
|
await db
|
|
.update(companies)
|
|
.set({
|
|
budgetMonthlyCents: amount,
|
|
updatedAt: now,
|
|
})
|
|
.where(eq(companies.id, input.scopeId));
|
|
}
|
|
|
|
if (input.scopeType === "agent" && windowKind === "calendar_month_utc") {
|
|
await db
|
|
.update(agents)
|
|
.set({
|
|
budgetMonthlyCents: amount,
|
|
updatedAt: now,
|
|
})
|
|
.where(eq(agents.id, input.scopeId));
|
|
}
|
|
|
|
if (row.isActive && amount > 0) {
|
|
const observedAmount = await computeObservedAmount(db, row);
|
|
if (!(await policyBlocks(db, row))) {
|
|
await resumeScopeFromBudget(row);
|
|
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard");
|
|
if (row.notifyEnabled && (await reached(db, row, Math.ceil(row.amount * row.warnPercent / 100)))) {
|
|
await createIncidentIfNeeded(row, "soft", observedAmount);
|
|
} else {
|
|
await resolveOpenIncidentsForPolicy(row.id, null, null, "soft");
|
|
}
|
|
} else {
|
|
const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100);
|
|
if (row.notifyEnabled && (await reached(db, row, softThreshold)) && (!row.hardStopEnabled || !(await reached(db, row, row.amount)))) {
|
|
await createIncidentIfNeeded(row, "soft", observedAmount);
|
|
}
|
|
if (await policyBlocks(db, row)) {
|
|
await resolveOpenSoftIncidents(row.id);
|
|
await createIncidentIfNeeded(row, "hard", observedAmount);
|
|
await pauseAndCancelScopeForBudget(row);
|
|
}
|
|
}
|
|
} else {
|
|
await resumeScopeFromBudget(row);
|
|
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service");
|
|
}
|
|
|
|
await recordActivity({
|
|
companyId,
|
|
actorType: "user",
|
|
actorId: actorUserId ?? "board",
|
|
action: "budget.policy_upserted",
|
|
entityType: "budget_policy",
|
|
entityId: row.id,
|
|
details: {
|
|
scopeType: row.scopeType,
|
|
scopeId: row.scopeId,
|
|
amount: row.amount,
|
|
windowKind: row.windowKind,
|
|
},
|
|
});
|
|
|
|
return buildPolicySummary(row);
|
|
},
|
|
|
|
overview: async (companyId: string): Promise<BudgetOverview> => {
|
|
const rows = await listPolicyRows(companyId);
|
|
const policies = await Promise.all(rows.map((row) => buildPolicySummary(row)));
|
|
const activeIncidentRows = await db
|
|
.select()
|
|
.from(budgetIncidents)
|
|
.where(and(eq(budgetIncidents.companyId, companyId), eq(budgetIncidents.status, "open")))
|
|
.orderBy(desc(budgetIncidents.createdAt));
|
|
const activeIncidents = await hydrateIncidentRows(activeIncidentRows);
|
|
return {
|
|
companyId,
|
|
policies,
|
|
activeIncidents,
|
|
pausedAgentCount: new Set(policies.filter((policy) => policy.scopeType === "agent" && policy.paused).map((policy) => policy.scopeId)).size,
|
|
pausedProjectCount: new Set(policies.filter((policy) => policy.scopeType === "project" && policy.paused).map((policy) => policy.scopeId)).size,
|
|
pendingApprovalCount: activeIncidents.filter((incident) => incident.approvalStatus === "pending").length,
|
|
};
|
|
},
|
|
|
|
evaluateCostEvent: async (event: typeof costEvents.$inferSelect) => {
|
|
const candidatePolicies = await db
|
|
.select()
|
|
.from(budgetPolicies)
|
|
.where(
|
|
and(
|
|
eq(budgetPolicies.companyId, event.companyId),
|
|
eq(budgetPolicies.isActive, true),
|
|
inArray(budgetPolicies.scopeType, ["company", "agent", "project"]),
|
|
),
|
|
);
|
|
|
|
const relevantPolicies = candidatePolicies.filter((policy) => {
|
|
if (policy.scopeType === "company") return policy.scopeId === event.companyId;
|
|
if (policy.scopeType === "agent") return policy.scopeId === event.agentId;
|
|
if (policy.scopeType === "project") return Boolean(event.projectId) && policy.scopeId === event.projectId;
|
|
return false;
|
|
});
|
|
|
|
for (const policy of relevantPolicies) {
|
|
if (policy.metric !== "billed_cents" || policy.amount <= 0) continue;
|
|
const observed = await computeObservedSpend(db, policy);
|
|
const observedAmount = observed.total;
|
|
const softThreshold = Math.ceil((policy.amount * policy.warnPercent) / 100);
|
|
|
|
if (policy.notifyEnabled && compareCents(observed.totalExact, softThreshold) >= 0 && (!policy.hardStopEnabled || compareCents(observed.totalExact, policy.amount) < 0)) {
|
|
const softIncident = await createIncidentIfNeeded(policy, "soft", observedAmount);
|
|
if (softIncident?.created) {
|
|
await recordActivity({
|
|
companyId: policy.companyId,
|
|
actorType: "system",
|
|
actorId: "budget_service",
|
|
action: "budget.soft_threshold_crossed",
|
|
entityType: "budget_incident",
|
|
entityId: softIncident.incident.id,
|
|
details: {
|
|
scopeType: policy.scopeType,
|
|
scopeId: policy.scopeId,
|
|
amountObserved: observedAmount,
|
|
amountLimit: policy.amount,
|
|
},
|
|
});
|
|
}
|
|
}
|
|
|
|
if (observedBlocks(policy, observed)) {
|
|
await resolveOpenSoftIncidents(policy.id);
|
|
const hardIncident = await createIncidentIfNeeded(policy, "hard", observedAmount);
|
|
await pauseAndCancelScopeForBudget(policy);
|
|
if (hardIncident?.created) {
|
|
await recordActivity({
|
|
companyId: policy.companyId,
|
|
actorType: "system",
|
|
actorId: "budget_service",
|
|
action: "budget.hard_threshold_crossed",
|
|
entityType: "budget_incident",
|
|
entityId: hardIncident.incident.id,
|
|
details: {
|
|
scopeType: policy.scopeType,
|
|
scopeId: policy.scopeId,
|
|
amountObserved: observedAmount,
|
|
amountLimit: policy.amount,
|
|
approvalId: hardIncident.incident.approvalId ?? null,
|
|
},
|
|
});
|
|
}
|
|
}
|
|
}
|
|
},
|
|
|
|
getInvocationBlock: async (
|
|
companyId: string,
|
|
agentId: string,
|
|
context?: { issueId?: string | null; projectId?: string | null },
|
|
) => {
|
|
const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [
|
|
{ scopeType: "company", scopeId: companyId }, { scopeType: "agent", scopeId: agentId },
|
|
...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []),
|
|
];
|
|
for (const { scopeType, scopeId } of scopes) {
|
|
if ((await resolveScopeRecord(db, scopeType, scopeId)).companyId !== companyId) throw notFound("Budget scope not found");
|
|
}
|
|
for (const { scopeType, scopeId } of scopes) {
|
|
const block = await reconcileScope(companyId, scopeType, scopeId);
|
|
if (block) return block;
|
|
}
|
|
return null;
|
|
},
|
|
|
|
resolveIncident: async (
|
|
companyId: string,
|
|
incidentId: string,
|
|
input: BudgetIncidentResolutionInput,
|
|
actorUserId: string,
|
|
): Promise<BudgetIncident> => {
|
|
const incident = await db
|
|
.select()
|
|
.from(budgetIncidents)
|
|
.where(eq(budgetIncidents.id, incidentId))
|
|
.then((rows) => rows[0] ?? null);
|
|
if (!incident) throw notFound("Budget incident not found");
|
|
if (incident.companyId !== companyId) throw notFound("Budget incident not found");
|
|
|
|
const policy = await getPolicyRow(incident.policyId);
|
|
if (input.action === "raise_budget_and_resume") {
|
|
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
|
|
const currentObserved = await computeObservedAmount(db, policy);
|
|
if (policy.unpricedUsagePolicy !== "allow" && (await computeObservedSpend(db, policy)).unpricedEventCount > 0) {
|
|
throw unprocessable("Resolve unpriced usage or explicitly allow it in the budget policy before resuming");
|
|
}
|
|
if ((await computeObservedSpend(db, policy)).pendingRunCount > 0) throw unprocessable("Completed runs must finish accounting before resuming");
|
|
if (await reached(db, policy, nextAmount)) {
|
|
throw unprocessable("New budget must exceed current observed spend");
|
|
}
|
|
|
|
const now = new Date();
|
|
await db
|
|
.update(budgetPolicies)
|
|
.set({
|
|
amount: nextAmount,
|
|
isActive: true,
|
|
updatedByUserId: actorUserId,
|
|
updatedAt: now,
|
|
})
|
|
.where(eq(budgetPolicies.id, policy.id));
|
|
|
|
if (policy.scopeType === "company" && policy.windowKind === "calendar_month_utc") {
|
|
await db
|
|
.update(companies)
|
|
.set({ budgetMonthlyCents: nextAmount, updatedAt: now })
|
|
.where(eq(companies.id, policy.scopeId));
|
|
}
|
|
|
|
if (policy.scopeType === "agent" && policy.windowKind === "calendar_month_utc") {
|
|
await db
|
|
.update(agents)
|
|
.set({ budgetMonthlyCents: nextAmount, updatedAt: now })
|
|
.where(eq(agents.id, policy.scopeId));
|
|
}
|
|
|
|
await resumeScopeFromBudget(policy);
|
|
await db
|
|
.update(budgetIncidents)
|
|
.set({
|
|
status: "resolved",
|
|
resolvedAt: now,
|
|
updatedAt: now,
|
|
})
|
|
.where(and(eq(budgetIncidents.policyId, policy.id), eq(budgetIncidents.status, "open")));
|
|
|
|
await markApprovalStatus(db, incident.approvalId ?? null, "approved", input.decisionNote, actorUserId);
|
|
} else {
|
|
await db
|
|
.update(budgetIncidents)
|
|
.set({
|
|
status: "dismissed",
|
|
resolvedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(eq(budgetIncidents.id, incident.id));
|
|
await markApprovalStatus(db, incident.approvalId ?? null, "rejected", input.decisionNote, actorUserId);
|
|
}
|
|
|
|
await recordActivity({
|
|
companyId: incident.companyId,
|
|
actorType: "user",
|
|
actorId: actorUserId,
|
|
action: "budget.incident_resolved",
|
|
entityType: "budget_incident",
|
|
entityId: incident.id,
|
|
details: {
|
|
action: input.action,
|
|
amount: input.amount ?? null,
|
|
scopeType: incident.scopeType,
|
|
scopeId: incident.scopeId,
|
|
},
|
|
});
|
|
|
|
const [updated] = await hydrateIncidentRows([{
|
|
...incident,
|
|
status: input.action === "raise_budget_and_resume" ? "resolved" : "dismissed",
|
|
resolvedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
}]);
|
|
return updated!;
|
|
},
|
|
};
|
|
}
|
|
|
|
/** Fence the durable stop intent against policy edits and admission. External
|
|
* process shutdown must happen after this transaction, without holding locks
|
|
* that the executor needs to persist its final usage receipt. */
|
|
export async function withCurrentBudgetEnforcement<T>(db: Db, scope: BudgetEnforcementScope, work: (tx: Db) => Promise<T>): Promise<T | null> {
|
|
return withAccountingTransaction(db, scope.companyId, async (tx) => {
|
|
if (scope.enforcement) {
|
|
const [policy] = await tx.select().from(budgetPolicies).where(and(
|
|
eq(budgetPolicies.id, scope.enforcement.policyId), eq(budgetPolicies.companyId, scope.companyId),
|
|
));
|
|
if (!policy || policy.scopeType !== scope.scopeType || policy.scopeId !== scope.scopeId
|
|
|| policy.enforcementVersion !== scope.enforcement.version || !(await policyBlocks(tx, policy))) return null;
|
|
}
|
|
return work(tx);
|
|
});
|
|
}
|
|
|
|
/** Cancellation is an at-least-once external effect. A failed delivery never
|
|
* rolls back committed spend, and its version remains pending for recovery. */
|
|
export async function deliverBudgetEnforcement(db: Db, hooks: BudgetServiceHooks, companyId?: string) {
|
|
if (!hooks.cancelWorkForScope) return;
|
|
const pending = await db.select().from(budgetPolicies).where(and(
|
|
sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`,
|
|
companyId ? eq(budgetPolicies.companyId, companyId) : undefined,
|
|
));
|
|
for (const policy of pending) {
|
|
try {
|
|
const cancellation = await withAccountingTransaction(db, policy.companyId, async (tx) => {
|
|
const [current] = await tx.select().from(budgetPolicies).where(eq(budgetPolicies.id, policy.id));
|
|
if (!current || current.enforcementVersion !== policy.enforcementVersion || !(await policyBlocks(tx, current))) return null;
|
|
return { companyId: policy.companyId, scopeType: policy.scopeType as BudgetScopeType, scopeId: policy.scopeId, createdBefore: new Date(), enforcement: { policyId: policy.id, version: policy.enforcementVersion } };
|
|
});
|
|
if (cancellation) await hooks.cancelWorkForScope(cancellation);
|
|
await db.update(budgetPolicies).set({ enforcementDeliveredVersion: policy.enforcementVersion })
|
|
.where(and(eq(budgetPolicies.id, policy.id), eq(budgetPolicies.enforcementVersion, policy.enforcementVersion)));
|
|
} catch (error) {
|
|
logger.warn({ err: error, policyId: policy.id }, "Budget cancellation delivery pending; accounting is committed");
|
|
}
|
|
}
|
|
}
|
|
|
|
export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
|
|
const reads = budgetServiceInTransaction(db);
|
|
async function mutate<T>(companyId: string, work: (service: ReturnType<typeof budgetServiceInTransaction>) => Promise<T>) {
|
|
const result = await withAccountingTransaction(db, companyId, (tx, publications) => work(budgetServiceInTransaction(tx, publications)));
|
|
await deliverBudgetEnforcement(db, hooks, companyId);
|
|
return result;
|
|
}
|
|
return {
|
|
...reads,
|
|
deliverPendingEnforcement: (companyId?: string) => deliverBudgetEnforcement(db, hooks, companyId),
|
|
reconcilePolicies: async () => {
|
|
const scopes = await db.selectDistinct({ companyId: budgetPolicies.companyId, scopeType: budgetPolicies.scopeType, scopeId: budgetPolicies.scopeId })
|
|
.from(budgetPolicies).orderBy(budgetPolicies.companyId, budgetPolicies.scopeType, budgetPolicies.scopeId);
|
|
// Each scope commits independently: deleted targets and a failed scope
|
|
// must not roll back recovery or cancellation delivery for other work.
|
|
for (const scope of scopes) {
|
|
try {
|
|
await mutate(scope.companyId, (service) => service.reconcileScope(scope.companyId, scope.scopeType as BudgetScopeType, scope.scopeId));
|
|
} catch (error) {
|
|
if (error instanceof HttpError && error.status === 404) continue;
|
|
logger.warn({ err: error, ...scope }, "Budget scope recovery pending; continuing remaining scopes");
|
|
}
|
|
}
|
|
},
|
|
upsertPolicy: (companyId: string, input: BudgetPolicyUpsertInput, actorUserId: string | null) =>
|
|
mutate(companyId, (service) => service.upsertPolicy(companyId, input, actorUserId)),
|
|
evaluateCostEvent: (event: typeof costEvents.$inferSelect) =>
|
|
mutate(event.companyId, (service) => service.evaluateCostEvent(event)),
|
|
getInvocationBlock: (companyId: string, agentId: string, context?: { issueId?: string | null; projectId?: string | null }) =>
|
|
mutate(companyId, (service) => service.getInvocationBlock(companyId, agentId, context)),
|
|
resolveIncident: (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) =>
|
|
mutate(companyId, (service) => service.resolveIncident(companyId, incidentId, input, actorUserId)),
|
|
};
|
|
}
|