diff --git a/doc/observability.md b/doc/observability.md index de9d0666744..e2a8d6f8746 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1126,6 +1126,12 @@ evidence, unknown binding reasons, secret-provider failures, ambiguous missing secret-definition lookups remain reportable. This filter does not change task recovery, credentials, or execution policy. +Native `native_provider_usage_limit` failures also stay on the task when their +typed provider failure is `quota_exhausted` or `rate_limit`. The task keeps its +blocked state and quota cooldown. Setup failures, interrupted processes, missing +or malformed provider evidence, and other failure categories still report. +`all` mode also reports typed native usage limits. + AI account selection also stays local when the selection service proves a missing responsible user, personal default, or selected connection; an incompatible selection; an account that needs reconnection; or an active diff --git a/server/src/services/__tests__/run-failure-report-native-usage-limit.test.ts b/server/src/services/__tests__/run-failure-report-native-usage-limit.test.ts new file mode 100644 index 00000000000..ddc866a02d9 --- /dev/null +++ b/server/src/services/__tests__/run-failure-report-native-usage-limit.test.ts @@ -0,0 +1,141 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "../../__tests__/helpers/embedded-postgres.js"; + +const mockCaptureRunFailure = vi.hoisted(() => vi.fn()); + +vi.mock("../../sentry.js", () => ({ + captureRunFailure: mockCaptureRunFailure, +})); + +import { reportRunFailure } from "../run-failure-report.js"; + +const testDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL; +const embeddedPostgresSupport = testDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +const limitFailure = { + schema: "paperclip.provider-failure.v1", + provider: "anthropic", + phase: "turn", + category: "quota_exhausted", + message: "ACP agent reported a terminal limit failure.\nYou've hit your weekly limit · resets Nov 10, 3pm (UTC)", + retryability: "requires_action", +}; + +describeEmbeddedPostgres("reportRunFailure native provider usage limits", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + let companyId!: string; + let agentId!: string; + let inheritedEnv: NodeJS.ProcessEnv; + + beforeAll(async () => { + if (testDatabaseUrl) { + await applyPendingMigrations(testDatabaseUrl); + db = createDb(testDatabaseUrl); + } else { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-failure-native-usage-limit-"); + db = createDb(tempDb.connectionString); + } + }, 20_000); + + beforeEach(async () => { + inheritedEnv = process.env; + process.env = Object.fromEntries( + ["PATH", "HOME", "USER", "USERNAME", "LOGNAME", "USERPROFILE", "TMPDIR", "TEMP", "TMP"] + .flatMap((key) => inheritedEnv[key] === undefined ? [] : [[key, inheritedEnv[key]]]), + ); + process.env.PAPERCLIP_SENTRY_RUN_FAILURES = "unexpected"; + companyId = randomUUID(); + agentId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: "responsible-user", + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Runner", + role: "engineer", + status: "active", + adapterType: "paperclip_runner", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + }); + + afterEach(async () => { + vi.clearAllMocks(); + vi.unstubAllEnvs(); + process.env = inheritedEnv; + await db.delete(heartbeatRuns); + await db.delete(agents); + await db.delete(companies); + }); + + afterAll(async () => { + if (testDatabaseUrl) await closeRegisteredClients(testDatabaseUrl); + await tempDb?.cleanup(); + }); + + /** The persisted shape of a native ACP turn that hit an exhausted provider allowance. */ + function nativeUsageLimitRun( + overrides: Partial = {}, + ): typeof heartbeatRuns.$inferSelect { + return { + id: randomUUID(), + companyId, + agentId, + status: "failed", + error: "ACP agent reported a terminal limit failure.", + errorCode: "native_provider_usage_limit", + runtimeMode: "native", + nativePhase: "terminal_failure", + nativeIssueId: randomUUID(), + exitCode: null, + signal: null, + contextSnapshot: null, + resultJson: { providerFailure: limitFailure }, + ...overrides, + } as unknown as typeof heartbeatRuns.$inferSelect; + } + + it.each(["quota_exhausted", "rate_limit"])("keeps a native %s usage limit on the task", async (category) => { + const run = nativeUsageLimitRun({ resultJson: { providerFailure: { ...limitFailure, category } } }); + const before = structuredClone(run); + for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, run, { phase }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + expect(run).toEqual(before); + }); + + it.each([ + { status: "timed_out" }, { signal: "SIGTERM" }, { runtimeMode: "legacy" }, + { errorCode: "native_provider_terminal_failed" }, { errorCode: "native_session_retry_exhausted" }, + { resultJson: {} }, { resultJson: null }, + { resultJson: { providerFailure: { ...limitFailure, category: "unknown", retryability: "unknown" } } }, + { resultJson: { providerFailure: { category: "quota_exhausted" } } }, + ])("reports a native usage limit outcome without typed limit evidence: %j", async (overrides) => { + await reportRunFailure(db, nativeUsageLimitRun(overrides), { phase: "execute" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }); + + it("reports a native usage limit during setup", async () => { + await reportRunFailure(db, nativeUsageLimitRun(), { phase: "setup" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }); + + it("reports native usage limits when every run failure is reported", async () => { + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all"); + await reportRunFailure(db, nativeUsageLimitRun(), { phase: "execute" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/services/native-runtime/native-provider-limit-cooldown.integration.test.ts b/server/src/services/native-runtime/native-provider-limit-cooldown.integration.test.ts new file mode 100644 index 00000000000..55ea60a445e --- /dev/null +++ b/server/src/services/native-runtime/native-provider-limit-cooldown.integration.test.ts @@ -0,0 +1,84 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { agentProviderLimitCooldown } from "./native-startup-retry.js"; + +const HOUR_MS = 60 * 60_000; +const DAY_MS = 24 * HOUR_MS; + +/** Claude ACP wording for an exhausted weekly allowance, such as "resets Nov 10, 3pm (UTC)". */ +function weeklyLimitMessage(at: Date) { + const month = at.toLocaleString("en-US", { month: "short", timeZone: "UTC" }); + const hour = at.getUTCHours(); + return `ACP agent reported a terminal limit failure.\nYou've hit your weekly limit · resets ${month} ${at.getUTCDate()}, ${ + hour % 12 || 12}${hour < 12 ? "am" : "pm"} (UTC)`; +} + +describe("native provider limit cooldown", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = randomUUID(); + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-provider-limit-cooldown-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Native provider limit", issuePrefix: "NPL" }); + }); + + afterAll(async () => { + await temporary?.cleanup(); + }); + + /** An agent whose latest native run failed on an exhausted weekly allowance. */ + async function limitedAgent(resetAt: Date) { + const agentId = randomUUID(); + await db.insert(agents).values({ + id: agentId, + companyId, + name: `Limited agent ${agentId.slice(0, 8)}`, + status: "active", + adapterType: "paperclip_runner", + adapterConfig: { provider: "acpx", agent: "claude" }, + }); + const limitedAt = new Date(Date.now() - 5 * 60_000); + await db.insert(heartbeatRuns).values({ + id: randomUUID(), + companyId, + agentId, + runtimeMode: "native", + invocationSource: "assignment", + triggerDetail: "system", + status: "failed", + startedAt: new Date(limitedAt.getTime() - 60_000), + finishedAt: limitedAt, + errorCode: "native_provider_usage_limit", + resultJson: { + providerFailure: { + schema: "paperclip.provider-failure.v1", + provider: "anthropic", + phase: "turn", + category: "quota_exhausted", + message: weeklyLimitMessage(resetAt), + retryability: "requires_action", + }, + }, + }); + return agentId; + } + + it("holds an agent until a dated weekly-limit reset instead of the one-hour default", async () => { + const resetAt = new Date(Math.ceil((Date.now() + 20 * HOUR_MS) / HOUR_MS) * HOUR_MS); + const agentId = await limitedAgent(resetAt); + expect(await agentProviderLimitCooldown(db, { companyId, agentId, now: new Date() })) + .toMatchObject({ resetAt, resetReported: true, category: "quota_exhausted" }); + }); + + it("re-checks a dated reset days away within one day", async () => { + const agentId = await limitedAgent(new Date(Math.ceil((Date.now() + 3 * DAY_MS) / HOUR_MS) * HOUR_MS)); + const now = new Date(); + const cooldown = await agentProviderLimitCooldown(db, { companyId, agentId, now }); + expect(cooldown).toMatchObject({ resetReported: true }); + expect(cooldown!.resetAt.getTime()).toBe(now.getTime() + DAY_MS); + }); +}); diff --git a/server/src/services/provider-quota-reset.test.ts b/server/src/services/provider-quota-reset.test.ts index f3ba512145b..96b81178886 100644 --- a/server/src/services/provider-quota-reset.test.ts +++ b/server/src/services/provider-quota-reset.test.ts @@ -41,3 +41,68 @@ describe("parseProviderQuotaClockReset", () => { expect(parseProviderQuotaClockReset("Provider quota exceeded for this model.", now)).toBeNull(); }); }); + +const weeklyLimit = "ACP agent reported a terminal limit failure.\nYou've hit your weekly limit · resets Nov 10, 3pm (UTC)"; + +describe("dated provider quota reset compatibility", () => { + it("reads a dated reset as that day, not as an unparsed limit", () => { + expect(parseProviderQuotaClockReset(weeklyLimit, new Date("2026-11-07T00:50:37Z"))) + .toEqual(new Date("2026-11-10T15:00:00Z")); + }); + + it.each([ + ["resets Nov 10 at 3:30pm (UTC)", "2026-11-10T15:30:00Z"], + ["resets November 10, 3 PM UTC", "2026-11-10T15:00:00Z"], + ["resets Dec. 1, 9am (UTC)", "2026-12-01T09:00:00Z"], + ["resets Nov 10th, 2026 3pm", "2026-11-10T15:00:00Z"], + ["try again at Nov 10th, 2026 2:04 AM.", "2026-11-10T02:04:00Z"], + ["resets Nov 10, 15:00 (UTC)", "2026-11-10T15:00:00Z"], + ["resets Nov 10, 3pm (America/Chicago)", "2026-11-10T21:00:00Z"], + ])("reads %s", (wording, expected) => { + expect(parseProviderQuotaClockReset(`You've hit your weekly limit · ${wording}`, new Date("2026-11-07T00:50:37Z"))) + .toEqual(new Date(expected)); + }); + + it("rolls a dated reset without a year into the next year at the year boundary", () => { + expect(parseProviderQuotaClockReset("resets Jan 2, 3pm (UTC)", new Date("2026-12-30T12:00:00Z"))) + .toEqual(new Date("2027-01-02T15:00:00Z")); + }); + + it("keeps a recently passed dated reset in the past instead of a year later", () => { + expect(parseProviderQuotaClockReset("resets Nov 6, 3pm (UTC)", new Date("2026-11-07T00:50:37Z"))) + .toEqual(new Date("2026-11-06T15:00:00Z")); + }); + + it("keeps stale calendar dates expired and rejects a far-future explicit year", () => { + const now = new Date("2026-11-07T00:50:37Z"); + expect(parseProviderQuotaClockReset("resets Sept 1, 9am (UTC)", now)).toEqual(new Date("2026-09-01T09:00:00Z")); + expect(parseProviderQuotaClockReset("resets Nov 10, 2027 3pm (UTC)", now)).toBeNull(); + }); + + it.each([ + "resets Apr 31, 3pm (UTC)", + "resets Feb 29, 2027 3pm (UTC)", + "resets Nov 10, 2026", + "resets Nov 10, 2026 25pm (UTC)", + "resets Nov 0, 3pm (UTC)", + "resets Nov 10, 13pm (UTC)", + "resets Nov 10, 3:60pm (UTC)", + "resets Novembork 10, 3pm (UTC)", + "resets Nov 10, 3pm (Not/AZone)", + ])("rejects the invalid dated reset %s", (wording) => { + expect(parseProviderQuotaClockReset(wording, new Date("2026-11-07T00:50:37Z"))).toBeNull(); + }); + + it("keeps clock-only resets on the next matching wall clock", () => { + const now = new Date("2026-11-07T00:50:37Z"); + expect(parseProviderQuotaClockReset("You've hit your session limit · resets 2am (UTC)", now)) + .toEqual(new Date("2026-11-07T02:00:00Z")); + expect(parseProviderQuotaClockReset("try again at 4:30 PM (America/Chicago)", now)) + .toEqual(new Date("2026-11-07T22:30:00Z")); + expect(parseProviderQuotaClockReset("resets 12am", now)).toEqual(new Date("2026-11-08T00:00:00Z")); + }); + + it("returns null when no reset is stated", () => { + expect(parseProviderQuotaClockReset("ACP agent reported a terminal limit failure.", new Date())).toBeNull(); + }); +}); diff --git a/server/src/services/provider-quota-reset.ts b/server/src/services/provider-quota-reset.ts index 25e275ce9e0..c158244175a 100644 --- a/server/src/services/provider-quota-reset.ts +++ b/server/src/services/provider-quota-reset.ts @@ -1,9 +1,11 @@ const MONTHS = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"]; +const MONTH_NAMES = "jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?"; +const MAX_DATED_RESET_AHEAD_MS = 45 * 24 * 60 * 60_000; const RESET_PATTERN = new RegExp( "(?:try again at|resets?(?:\\s+(?:at|on))?)\\s+" + // Optional calendar date, as in "resets Nov 12, 6pm (UTC)" for a weekly limit. - `(?:(${MONTHS.join("|")})[a-z]*\\.?\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+(?:at\\s+)?)?` + - "(\\d{1,2})(?::(\\d{2}))?\\s*(?:([ap])\\.?\\s*m\\.?)?(?:\\s*\\(([^)]+)\\)|\\s+([A-Z]{2,5}))?", + `(?:(${MONTH_NAMES})\\b\\.?\\s+(\\d{1,2})(?:st|nd|rd|th)?(?:,?\\s+(\\d{4}))?,?\\s+(?:at\\s+)?)?` + + "(\\d{1,2})(?::(\\d{2}))?\\s*(?:([ap])\\.?\\s*m\\.?)?(?![\\d:])(?:\\s*\\(([^)]+)\\)|\\s+([A-Z]{2,5}))?", "i", ); @@ -20,10 +22,11 @@ export function parseProviderQuotaClockReset(error: string, now: Date) { const monthIndex = match[1] ? MONTHS.indexOf(match[1].slice(0, 3).toLowerCase()) : null; const dayOfMonth = match[2] ? Number.parseInt(match[2], 10) : null; + const statedYear = match[3] ? Number.parseInt(match[3], 10) : null; if (monthIndex !== null && (monthIndex < 0 || !dayOfMonth || dayOfMonth < 1 || dayOfMonth > 31)) return null; - const hourValue = Number.parseInt(match[3] ?? "", 10); - const minute = Number.parseInt(match[4] ?? "0", 10); - const meridiem = (match[5] ?? "").toLowerCase(); + const hourValue = Number.parseInt(match[4] ?? "", 10); + const minute = Number.parseInt(match[5] ?? "0", 10); + const meridiem = (match[6] ?? "").toLowerCase(); if (!Number.isInteger(hourValue)) return null; if ( meridiem ? hourValue < 1 || hourValue > 12 : hourValue < 0 || hourValue > 23 @@ -33,7 +36,7 @@ export function parseProviderQuotaClockReset(error: string, now: Date) { let hour = meridiem ? hourValue % 12 : hourValue; if (meridiem === "p") hour += 12; - const timeZone = (match[6] ?? match[7])?.trim() || "UTC"; + const timeZone = (match[7] ?? match[8])?.trim() || "UTC"; try { const wallClock = (date: Date) => @@ -79,12 +82,13 @@ export function parseProviderQuotaClockReset(error: string, now: Date) { // Dec 30 is next year's. A date read after it passed stays in the past, // so a caller reading an old message sees an expired reset rather than // one a year away. - const candidates = [year - 1, year, year + 1] + const candidates = (statedYear === null ? [year - 1, year, year + 1] : [statedYear]) .map((candidateYear) => instant(candidateYear, monthIndex, dayOfMonth)) .filter((candidate): candidate is Date => candidate !== null); if (!candidates.length) return null; - return candidates.reduce((nearest, candidate) => + const nearest = candidates.reduce((nearest, candidate) => Math.abs(candidate.getTime() - now.getTime()) < Math.abs(nearest.getTime() - now.getTime()) ? candidate : nearest); + return nearest.getTime() > now.getTime() + MAX_DATED_RESET_AHEAD_MS ? null : nearest; } const month = Number(nowParts.month) - 1; const day = Number(nowParts.day); diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index 1e0ceb7d4cc..79b8b2805aa 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -21,6 +21,7 @@ import { PLUGIN_RUNNER_AI_CONNECTION_REQUIRED } from "./plugin-runner-ai-connect import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-runtime/native-provider-failure.js"; import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js"; import { REVIEWED_CHAT_BINDING_NOT_READY } from "./reviewed-chat-binding-diagnostics.js"; +import { readProviderFailure } from "@paperclipai/shared"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; @@ -259,6 +260,14 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOpti options.phase !== "setup" && run.signal == null; } +/** A typed native allowance failure stays on the task and its cooldown path. */ +function isNativeProviderUsageLimitFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean { + if (run.status !== "failed" || run.runtimeMode !== "native" || + run.errorCode !== "native_provider_usage_limit" || options.phase === "setup" || run.signal != null) return false; + const category = readProviderFailure(asRecord(run.resultJson)?.providerFailure)?.category; + return category === "quota_exhausted" || category === "rate_limit"; +} + function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean { const result = asRecord(run.resultJson); return run.status === "failed" && run.errorCode === "codex_transient_upstream" && @@ -364,6 +373,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR isProjectAccessDenialBlocker(run, options) || isGitConnectionFailure(run, options) || isProviderQuotaFailure(run, options) || + isNativeProviderUsageLimitFailure(run, options) || isCodexAcpModelCapacityFailure(run, options) || isAcpProviderAccessFailure(run, options) || isScheduledReviewedChatRetry(run, options) ||