diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index acb0353706d..053a49216d9 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -1769,7 +1769,10 @@ hard stop refuses new work even while evaluation lags. waits on another run's cleanup or accounting completeness. Only these refuse it: - spend at the limit (ledger plus observed running spend), or unpriced usage - under a blocking policy (the hard stop); + a provider reported, under a blocking policy (the hard stop). A charge that + accounting settled without its receipt (lost, closure evidence that never + arrived, or a failed capture) is recorded and shown in Costs, but it is never + a company gate; - an explicit pause of the company, project or agent; - an inactive company. @@ -1986,12 +1989,12 @@ its display label after fifteen minutes. Elapsed time never discards a late receipt. It settles the debt only for a native run whose recovery has ended (terminal failure or an accepted result) and whose closure evidence has not arrived after ten minutes (the remote runner reconnect grace plus a margin): the known usage is recorded as unpriced with -provenance saying closure evidence never arrived, an activity entry names the -run, and the blocking unpriced-usage policy keeps counting that charge unless -a control-plane restart caused it. A restart-derived receipt (the run replaced -its receipt source, recovered from a restart, or lost a provider turn on -restore) never hard-stops or pauses a company under the blocking policy, -whatever its token count; its activity entry and provenance remain the notice. A lost +provenance saying closure evidence never arrived, and an activity entry names +the run. Like any charge accounting settled without its receipt, it never +hard-stops or pauses a company under the blocking policy, whatever its token +count; its activity entry and provenance remain the notice. A verified local +process stop closes the receipt source for a failed, cancelled, timed-out or +interrupted run alike, so a restart-cancelled run settles at once. A lost receipt that already carries a provider-reported cost settles at that cost as an estimate marked as a lower bound, not as unpriced. A failed native run that never started provider work (no usage receipt, no replaced receipt source, and a diff --git a/server/src/__tests__/closed-native-receipt-accounting.test.ts b/server/src/__tests__/closed-native-receipt-accounting.test.ts index fb1e56abd27..c03b8aa17b5 100644 --- a/server/src/__tests__/closed-native-receipt-accounting.test.ts +++ b/server/src/__tests__/closed-native-receipt-accounting.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { - agents, budgetIncidents, budgetPolicies, budgetReservations, companies, costEvents, createDb, environmentLeases, environments, + agentWakeupRequests, agents, budgetIncidents, budgetPolicies, budgetReservations, companies, costEvents, createDb, environmentLeases, environments, heartbeatRunEvents, heartbeatRuns, issues, activityLog, completionContracts, nativeRunFinalizations, nativeRunResults, type Db, } from "@paperclipai/db"; @@ -595,14 +595,27 @@ const support = await getEmbeddedPostgresTestSupport(); expect(await f.block()).toBeNull(); }); - it("does not count an empty fence map as restart evidence for a timed-out closure", async () => { + // G1 wave s9802 (runs bf9ea6b4, c2e222a2, aa2c9f03, f64f9f46): a restart + // cancelled these runs, and recovery verified each local process stopped. + // The stop proof counted only for failed runs, so each cancelled run held its + // task's Retry for the full 10-minute closure bound. + it.each(["cancelled", "timed_out", "interrupted"] as const)("settles a %s run once its local stop is verified", async status => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status, errorCode: "issue_terminalized" }).where(eq(heartbeatRuns.id, f.run.id)); + expect(await reconcileRunCosts(db)).toMatchObject({ accounted: 1 }); + const run = await f.readRun(); + expect(run.costAccountingPending).toBe(false); + expect(run.usageJson).not.toHaveProperty("accountingClosureTimedOut"); + }); + + it("records a timed-out closure with an empty fence map without blocking", async () => { const f = await acceptedResultFixture({ stopEvidence: false, finishedMinutesAgo: 11, restart: false }); await db.update(budgetPolicies).set({ unpricedUsagePolicy: "block" }).where(eq(budgetPolicies.id, f.policy.id)); await db.update(heartbeatRuns).set({ usageJson: sql`${heartbeatRuns.usageJson} || '{"accountingReceiptSourceFences":{}}'::jsonb` }) .where(eq(heartbeatRuns.id, f.run.id)); expect(await reconcileRunCosts(db)).toMatchObject({ accounted: 1 }); await flushBudgetEvaluationWork(db); - expect((await f.block())?.reason).toMatch(/no reliable price|budget hard-stop/); + expect(await f.block()).toBeNull(); }); it("keeps waiting for the bound when the provider process was never verified stopped", async () => { @@ -655,21 +668,27 @@ const support = await getEmbeddedPostgresTestSupport(); expect(next!).toBeGreaterThan(Date.now()); }); - it("keeps a settled capture failure under the blocking unpriced-usage policy", async () => { + // Accounting is never a company gate (plan decision 1): a charge accounting + // settled without its receipt is recorded and shown in Costs, but it never + // trips the blocking unpriced-usage policy. + it("records a settled capture failure without tripping the blocking unpriced-usage policy", async () => { const f = await fixture({ unpricedUsagePolicy: "block" }); await captureFailed(f, 11, { inputTokens: 1200, outputTokens: 80, model: "unknown-model" }); expect(await reconcileRunCosts(db)).toMatchObject({ accounted: 1 }); await flushBudgetEvaluationWork(db); - expect((await f.block())?.reason).toMatch(/no reliable price|budget hard-stop/); + expect(await f.charges()).toMatchObject([{ costStatus: "unpriced" }]); + expect(await f.block()).toBeNull(); + const [company] = await db.select().from(companies).where(eq(companies.id, f.companyId)); + expect(company).toMatchObject({ status: "active", pauseReason: null }); }); - it("keeps a capture failure after a receipt-source handoff under the blocking policy", async () => { + it("records a capture failure after a receipt-source handoff without blocking", async () => { const f = await fixture({ unpricedUsagePolicy: "block" }); await captureFailed(f, 11, { inputTokens: 1200, outputTokens: 80, model: "unknown-model", accountingReceiptSourceFences: { [f.sourceId]: 2 } }); expect(await reconcileRunCosts(db)).toMatchObject({ accounted: 1 }); await flushBudgetEvaluationWork(db); - expect((await f.block())?.reason).toMatch(/no reliable price|budget hard-stop/); + expect(await f.block()).toBeNull(); }); it("settles a capture failure at no less than its observed live price", async () => { @@ -716,12 +735,12 @@ const support = await getEmbeddedPostgresTestSupport(); expect((await readCostAccountingWork(db, Date.now())).nextRunAt).toBeNull(); }); - it("stops blocking once a complete late receipt accounts for a timed-out closure", async () => { + it("prices a timed-out closure from a complete late receipt, never blocking", async () => { const f = await acceptedResultFixture({ stopEvidence: false, finishedMinutesAgo: 11, restart: false }); await db.update(budgetPolicies).set({ unpricedUsagePolicy: "block" }).where(eq(budgetPolicies.id, f.policy.id)); expect(await reconcileRunCosts(db)).toMatchObject({ accounted: 1 }); await flushBudgetEvaluationWork(db); - expect((await f.block())?.reason).toMatch(/no reliable price|budget hard-stop/); + expect(await f.block()).toBeNull(); await persistUsageReceipt(db, { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: f.companyId, runId: f.run.id, adapterType: "paperclip_runner", sourceId: f.sourceId, sequence: 6, receivedAt: new Date().toISOString(), @@ -795,7 +814,7 @@ const support = await getEmbeddedPostgresTestSupport(); expect(await f.block()).toBeNull(); }); - it("keeps the blocking policy for a timed-out closure whose only recovery was an ordinary same-run retry", async () => { + it("records a timed-out closure whose only recovery was an ordinary same-run retry without blocking", async () => { const f = await acceptedResultFixture({ stopEvidence: false, finishedMinutesAgo: 11, restart: false }); await db.update(budgetPolicies).set({ unpricedUsagePolicy: "block" }).where(eq(budgetPolicies.id, f.policy.id)); await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.run.id, agentId: f.agentId, seq: 10, @@ -803,15 +822,31 @@ const support = await getEmbeddedPostgresTestSupport(); payload: { restartKind: "hot", trigger: "same_run_retry", runnerDisposition: "reattach_remote_runner" } }); expect(await reconcileRunCosts(db)).toMatchObject({ accounted: 1 }); await flushBudgetEvaluationWork(db); - expect((await f.block())?.reason).toMatch(/no reliable price|budget hard-stop/); + expect(await f.block()).toBeNull(); }); - it("keeps the blocking policy for a timed-out closure with no restart evidence, since the run may still spend", async () => { + // G1 wave s9802: the timed-out closure's unpriced charge hard-stopped the + // company and cancelled an accepted board Retry. Its exposure is recorded; + // it never holds the company. + it("records a timed-out closure with no restart evidence without holding the company", async () => { const f = await acceptedResultFixture({ stopEvidence: false, finishedMinutesAgo: 11, restart: false }); await db.update(budgetPolicies).set({ unpricedUsagePolicy: "block" }).where(eq(budgetPolicies.id, f.policy.id)); + // The board's accepted Retry, saved while the run's accounting was open. + const [retry] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + source: "on_demand", reason: "task_retry", status: "deferred_issue_execution", requestedByActorType: "user", + updatedAt: new Date(), payload: { issueId: f.issue.id, taskRetry: { sourceRunId: f.run.id }, + executionWait: { reason: "execution_settling", message: "Retry is waiting for the previous execution to stop." } } }).returning(); expect(await reconcileRunCosts(db)).toMatchObject({ accounted: 1 }); await flushBudgetEvaluationWork(db); - expect((await f.block())?.reason).toMatch(/no reliable price|budget hard-stop/); + expect((await f.readRun()).usageJson).toMatchObject({ accountingClosureTimedOut: true }); + expect(await f.block()).toBeNull(); + // The cleared hold re-dispatches it: still saved, never cancelled, and due now. + const [promoted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.id)); + expect(promoted.status).toBe("deferred_issue_execution"); + expect(promoted.updatedAt.getTime()).toBeLessThanOrEqual(Date.now() - 29_000); + expect(await budgetService(db).getInvocationBlock(f.companyId, f.agentId, { issueId: null, projectId: null })).toBeNull(); + const [company] = await db.select().from(companies).where(eq(companies.id, f.companyId)); + expect(company).toMatchObject({ status: "active", pauseReason: null }); }); it("keeps the debt pending while native recovery can still produce a result", async () => { diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index 7196458a43a..226bf36dfb6 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -124,6 +124,36 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => expect((await taskInterruptionPresentation(db, parent))?.retryWait?.reason).not.toBe("dependency_blocked"); }); + // G1 wave s9802 t02: an accepted Retry held by the unpriced-usage refusal + // read as a budget wait. It is an accounting matter; name it so. + it("labels a Retry held by unpriced usage as accounting, not budget", async () => { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Unpriced", issuePrefix: `U${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "paperclip_runner" }); + const [task] = await db.insert(issues).values({ companyId, title: "Task", identifier: "CHA-2", + status: "blocked", assigneeAgentId: agentId }).returning(); + const [todo] = await db.insert(issues).values({ companyId, title: "Not started", identifier: "CHA-3", + status: "todo", assigneeAgentId: agentId }).returning(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: task.id, scopeKind: "issue", + status: "failed", runtimeMode: "native", errorCode: "native_session_cleanup_quarantined", finishedAt: new Date(Date.now() - 60_000) }); + const refusal = "Company cannot start work because recorded usage has no reliable price."; + await db.insert(agentWakeupRequests).values([{ companyId, agentId, source: "on_demand", reason: "task_retry", + status: "deferred_issue_execution", requestedByActorType: "user", requestedAt: new Date(Date.now() - 30_000), + payload: { issueId: task.id, taskRetry: { sourceRunId: runId }, executionWait: { reason: "budget.blocked", message: refusal } } }, + { companyId, agentId, source: "assignment", reason: "budget.blocked", status: "skipped", error: refusal, + requestedAt: new Date(), payload: { issueId: todo.id } }]); + const held = await taskInterruptionPresentation(db, task); + expect(held?.retryWait).toMatchObject({ reason: "accounting.unpriced_usage", message: expect.stringContaining("has no price") }); + expect(held?.retryWait?.message).not.toContain("budget to allow"); + const waiting = (await taskInterruptionPresentations(db, companyId, [todo])).get(todo.id); + expect(waiting?.retryWait).toMatchObject({ reason: "accounting.unpriced_usage", message: expect.stringContaining("has no price") }); + // A budget hard stop keeps its budget label. + await db.update(agentWakeupRequests).set({ payload: { issueId: task.id, taskRetry: { sourceRunId: runId }, + executionWait: { reason: "budget.blocked", message: "Company is paused because its budget hard-stop was reached." } } }) + .where(eq(agentWakeupRequests.reason, "task_retry")); + expect((await taskInterruptionPresentation(db, task))?.retryWait?.reason).toBe("budget.blocked"); + }); + it("shows a queued-message Interrupt as handing off, not as a failed attempt", async () => { const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), userId = "board-user"; await db.insert(companies).values({ id: companyId, name: "Interrupted task", issuePrefix: `Q${companyId.slice(0, 7)}` }); diff --git a/server/src/services/budget-exposure.ts b/server/src/services/budget-exposure.ts index dcb5ace3101..38e87d0438f 100644 --- a/server/src/services/budget-exposure.ts +++ b/server/src/services/budget-exposure.ts @@ -127,44 +127,30 @@ async function readObservedSpends(db: Db, policies: SpendPolicy[], now: Date): P and ${budgetReservations.runId} = ${heartbeatRuns.id} and ${budgetReservations.state} = 'held')`; ledgerPredicates.push(ledger); heldPredicates.push(held); runPredicates.push(run); ledgerColumns[`total${index}`] = sql`coalesce(sum(${costEvents.costCents}) filter (where ${ledger}), 0)::text`; - // "Usage with no reliable price" means observed usage that could not be - // priced. These unpriced charges are not that, so they never trip the - // blocking policy (they stay visible in Costs): - // - a receipt lost with its stopped provider (restart-derived, any usage); + // "Usage with no reliable price" means usage from a complete receipt that + // could not be priced. These unpriced charges are not that, so they never + // trip the blocking policy (they stay visible in Costs): + // - a final charge accounting settled without its receipt: lost with a + // stopped provider, closure evidence that never arrived, or a failed + // capture (its receipt is never certified complete, even if a later + // report adds usage). Accounting is never a company gate (plan decision 1); its + // exposure is recorded, and the activity entry is the board's notice; // - a charge that carries a known partial cost; // - metered API or subscription billing with no observed tokens. // Unknown billing with no tokens (for example Cursor Cloud) still counts. - // The lost-receipt exemption covers only that run's settled final charge; - // later growth from a complete receipt is ordinary usage. - // A final charge settled only because closure evidence never arrived is - // genuinely unknown spend (its provider may still have run), so none of - // these exemptions apply to it, unless a control-plane restart caused it. - // A restart-derived receipt never trips the blocking policy, whatever its - // token count: the run replaced its receipt source, recovered from the - // restart, or lost a provider turn on restore. Its activity entry and - // provenance remain the board's notice. + // The receipt-unavailable exemption covers only that run's settled final + // charge; later growth from a complete receipt is ordinary usage. const finalCharge = sql`${costEvents.idempotencyKey} = 'heartbeat:' || ${costEvents.heartbeatRunId}::text || ':final'`; - // Every usage recorder binds an empty fence map; only a replaced receipt - // source (a non-empty map) is restart evidence. - const restartDerived = (alias: string) => sql.raw(`(coalesce(${alias}.usage_json->'accountingReceiptSourceFences', '{}'::jsonb) <> '{}'::jsonb - or coalesce(${alias}.usage_json->'pricingProvenance'->>'evidence', '') like '%lost on restore%' - or exists (select 1 from heartbeat_run_events restart_event where restart_event.run_id = ${alias}.id - and restart_event.company_id = ${alias}.company_id and restart_event.event_type = 'native.recovery.transition' - and restart_event.payload->>'trigger' is distinct from 'same_run_retry'))`); - const closureTimedOutCharge = sql`(${finalCharge} - and exists (select 1 from ${heartbeatRuns} timed_out_run where timed_out_run.id = ${costEvents.heartbeatRunId} - and timed_out_run.company_id = ${costEvents.companyId} - and timed_out_run.usage_json->>'accountingClosureTimedOut' = 'true' - -- A failed capture is unknown spend whatever its source history. - and (timed_out_run.usage_json->>'accountingCaptureFailed' = 'true' or not ${restartDerived("timed_out_run")})))`; - const exemptFromUnpricedBlock = sql`(not ${closureTimedOutCharge} and ((${finalCharge} + const exemptFromUnpricedBlock = sql`((${finalCharge} and exists (select 1 from ${heartbeatRuns} lost_receipt_run where lost_receipt_run.id = ${costEvents.heartbeatRunId} and lost_receipt_run.company_id = ${costEvents.companyId} and (lost_receipt_run.usage_json->>'accountingReceiptUnavailable' = 'true' + or lost_receipt_run.usage_json->>'accountingClosureTimedOut' = 'true' + or lost_receipt_run.usage_json->>'accountingCaptureFailed' = 'true' or coalesce(lost_receipt_run.usage_json->'pricingProvenance'->>'evidence', '') like '%lost on restore%'))) or ${costEvents.costCents} > 0 or (${costEvents.billingType} in ('metered_api', 'api', 'subscription') and ${costEvents.inputTokens} = 0 and ${costEvents.cachedInputTokens} = 0 - and ${costEvents.outputTokens} = 0)))`; + and ${costEvents.outputTokens} = 0))`; ledgerColumns[`unpriced${index}`] = sql`count(*) filter (where ${ledger} and ${costEvents.costStatus} = 'unpriced' and ${costEvents.billingType} <> 'subscription_included' and not ${exemptFromUnpricedBlock})::int`; runColumns[`live${index}`] = sql`coalesce(sum(${liveCost}) filter (where ${run} and ${inWindow}), 0)::text`; diff --git a/server/src/services/run-cost-accounting.ts b/server/src/services/run-cost-accounting.ts index 305ba884b36..81c7f524a33 100644 --- a/server/src/services/run-cost-accounting.ts +++ b/server/src/services/run-cost-accounting.ts @@ -71,9 +71,11 @@ const remoteExecutionTerminated = sql`exists (select 1 from ${environmentLeases} /** A native run's receipt source is closed only with positive stop evidence, * after recovery ended in terminal failure without a result: * - Local: the run no longer records a process identity and its latest - * server-authored process event is the verified local stop. Only a failed - * run closes this way; a cancelled local run keeps its accounting drain, - * since Stop recovery can still reattach a recorder. + * server-authored process event is the verified local stop. This holds for + * any terminal status (failed, cancelled, timed out, interrupted): a stopped + * process reports no more usage. Settlement still imports spooled receipts, + * and a receipt a reattached recorder delivers later posts through + * late-receipt reconciliation. * - Remote: every allocation has a termination receipt for its sandbox, so * neither the original nor a reattached runner can report usage. That proof * does not clear the recorded remote process identity. @@ -86,9 +88,11 @@ const settledNativeCoordinator = sql`exists (select 1 from ${nativeRunFinalizati and ((${nativeRunFinalizations.resultId} is null and ${nativeRunFinalizations.phase} = 'terminal_failure') or ${nativeRunFinalizations.resultId} is not null))`; +// A verified local stop closes the receipt source for any terminal status: a +// restart cancels, times out or interrupts a run as often as it fails one. const closedNativeReceiptSource = sql`${heartbeatRuns.runtimeMode} = 'native' and ${settledNativeCoordinator} - and ((${heartbeatRuns.status} = 'failed' + and ((${heartbeatRuns.status} in ('failed', 'cancelled', 'timed_out', 'interrupted') and ${heartbeatRuns.processPid} is null and ${heartbeatRuns.processGroupId} is null and (select process_event.event_type from ${heartbeatRunEvents} process_event where process_event.company_id = ${heartbeatRuns.companyId} and process_event.run_id = ${heartbeatRuns.id} @@ -120,8 +124,10 @@ const nativeProviderWorkNeverStarted = sql`${heartbeatRuns.runtimeMode} = 'nativ /** Waiting for closure evidence is bounded. Past the remote runner reconnect * grace plus a margin, a terminal run whose recovery has settled no longer - * holds new company work: its debt settles as unpriced, flagged so the - * blocking policy still counts it and the board sees which run it was. */ + * holds new company work: its debt settles as unpriced and is flagged so the + * board sees which run it was. The flag also exempts the charge from the + * blocking unpriced-usage policy (budget-exposure.ts): accounting is never a + * company gate. */ /** A run's accounting settled: an admission hold may have cleared. Held work * (queued runs, saved messages and Retries, and tasks whose start or * continuation was refused) waits passively, so wake its owners. */ @@ -144,8 +150,9 @@ function closureEvidenceTimedOut(at: Date = new Date()) { * certify what it missed. It is bounded like missing closure evidence: past * the same bound, once native recovery will not run the provider again, its * known usage settles as unpriced (or as a lower-bound estimate) and is - * flagged so the blocking policy still counts it. Without the bound such a - * run held company admission until its UTC budget window rolled over. */ + * flagged, which exempts it from the blocking unpriced-usage policy + * (budget-exposure.ts). Without the bound such a run held company admission + * until its UTC budget window rolled over. */ function captureFailureBoundPassed(at: Date = new Date()) { return sql`${heartbeatRuns.usageJson}->>'accountingCaptureFailed' = 'true' and ${heartbeatRuns.status} in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') @@ -396,8 +403,10 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService // A complete receipt arriving later may price this charge (see usage-receipts). ...(receiptUnavailable ? { usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ accountingReceiptUnavailable: true, - // Unlike a verified stop, the run may still have spent: the blocking - // unpriced-usage policy keeps counting this charge. + // Unlike a verified stop, the run may still have spent. The flag names + // the run in Costs; like any charge accounting settled without its + // receipt, it is exempt from the blocking unpriced-usage policy + // (budget-exposure.ts), because accounting is never a company gate. ...(closureTimedOut ? { accountingClosureTimedOut: true } : {}), // Late reconciliation measures a later receipt from the accepted money. ...(lostReceiptEstimate ? { costUsdExact: lostReceiptEstimate.costUsdExact, costStatus: "estimated", diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index ad8b01139cf..f4b9d443c51 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -95,6 +95,14 @@ function boundedJsonString(value: SQL, limit: number) { return sql`case when jsonb_typeof(${value}) = 'string' then left(${value} #>> '{}', ${limit}) else null end`; } +/** Projected reason for a budget.blocked refusal caused by unpriced usage. */ +export const UNPRICED_USAGE_WAIT = "accounting.unpriced_usage"; +/** Admission's unpriced-usage refusal text (budgets.ts). Only this fixed + * phrase is matched; the stored message itself is never displayed. */ +const unpricedUsageWait = (reason: SQL, message: SQL) => + sql`case when ${reason} = 'budget.blocked' and ${message} like '%recorded usage has no reliable price%' + then ${UNPRICED_USAGE_WAIT} else ${reason} end`; + // Persisted wait messages can contain diagnostics. Ordinary task views expose // only these stable categories, never arbitrary payloads or source identifiers. const retryWaitMessages: Record = { @@ -109,6 +117,9 @@ const retryWaitMessages: Record = { execution_active: "Retry is waiting for the current run to finish. Your message is saved.", decision_pending: "Retry is waiting for a pending approval or question to be resolved. Your message is saved.", "budget.blocked": "Retry is waiting for the budget to allow new work. Check Costs for unrecorded or unpriced usage.", + // Admission records an unpriced-usage refusal as budget.blocked; the + // projection shows it as the accounting matter it is. + [UNPRICED_USAGE_WAIT]: "Retry is waiting because recorded usage has no price. Check Costs, or allow unpriced usage in the budget policy. Your message is saved.", finalizing_saved_result: "Paperclip is finishing the saved result before Retry can continue.", // Another click cannot release a copyback owner that stopped without reporting. // Retry starts a new recovery generation; the agent does not repeat its work. @@ -136,6 +147,7 @@ function dependencyRetryWaitMessage(dependencies: RetryDependencies): string { // assigned-task liveness sweep admits it once the hold clears; no Retry needed. const admissionWaitMessages: Record = { "budget.blocked": "Waiting for the budget to allow new work. The task starts automatically when it does.", + [UNPRICED_USAGE_WAIT]: "Waiting because recorded usage has no price. Check Costs, or allow unpriced usage in the budget policy.", }; /** Recognize old persisted failures without displaying arbitrary run error text. */ @@ -349,7 +361,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i // A new message must not hide an existing Retry's admission explanation. db.selectDistinctOn([wakeIssue, isRetryWait], { issueId: wakeIssue, id: agentWakeupRequests.id, requestedAt: agentWakeupRequests.requestedAt, sourceRunId: boundedJsonString(sql`${agentWakeupRequests.payload}#>'{taskRetry,sourceRunId}'`, 160), - executionWaitReason: boundedJsonString(sql`${agentWakeupRequests.payload}#>'{executionWait,reason}'`, 80), + executionWaitReason: boundedJsonString(sql`to_jsonb(${unpricedUsageWait(sql`${agentWakeupRequests.payload}#>>'{executionWait,reason}'`, + sql`${agentWakeupRequests.payload}#>>'{executionWait,message}'`)})`, 80), preparationStatus: chatActions.status, preparationRetryAt: chatActionDeadline("failed_run_retry"), preparationUpdatedAt: chatActions.updatedAt }) .from(agentWakeupRequests).leftJoin(chatActions, and(eq(chatActions.companyId, agentWakeupRequests.companyId), @@ -377,7 +390,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i db.select({ status: companies.status, pauseReason: companies.pauseReason }).from(companies).where(eq(companies.id, companyId)), // Only consulted for tasks without a run: the latest wake says whether admission refused it. startingIssueIds.length ? db.selectDistinctOn([wakeIssue], { issueId: wakeIssue, id: agentWakeupRequests.id, - status: agentWakeupRequests.status, reason: agentWakeupRequests.reason, agentId: agentWakeupRequests.agentId }) + status: agentWakeupRequests.status, reason: unpricedUsageWait(sql`${agentWakeupRequests.reason}`, sql`${agentWakeupRequests.error}`), + agentId: agentWakeupRequests.agentId }) .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), inArray(wakeIssue, startingIssueIds))) .orderBy(wakeIssue, desc(agentWakeupRequests.requestedAt), desc(agentWakeupRequests.id)) : [], ]);