fix(native): wait for a dated provider limit reset and keep it out of Sentry (#19775)

Keep native quota cooldowns tied to valid dated reset messages. Parse explicit years and strict month names while preserving already-expired dates. Keep typed native quota and rate-limit failures on their tasks in unexpected Sentry reporting mode. Preserve setup, process, malformed and all-mode reporting.

Verified current-head CI, Greptile 5/5 at Medium impact, local parser/report red-green tests and PostgreSQL integration coverage.

Co-Authored-By: Devin Foley <devin@devinfoley.com>
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-11-12 17:06:22 -08:00
1 parent 2dcfdabc79
commit ad320aeb71
6 files changed
+318 -8

No files matched your search

+6
View File
@@ -1126,6 +1126,12 @@ evidence, unknown binding reasons, secret-provider failures, ambiguous missing
secret-definition lookups remain reportable. This filter does not change task
recovery, credentials, or execution policy.
Native `native_provider_usage_limit` failures also stay on the task when their
typed provider failure is `quota_exhausted` or `rate_limit`. The task keeps its
blocked state and quota cooldown. Setup failures, interrupted processes, missing
or malformed provider evidence, and other failure categories still report.
`all` mode also reports typed native usage limits.
AI account selection also stays local when the selection service proves a
missing responsible user, personal default, or selected connection; an
incompatible selection; an account that needs reconnection; or an active
@@ -0,0 +1,141 @@
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "../../__tests__/helpers/embedded-postgres.js";
const mockCaptureRunFailure = vi.hoisted(() => vi.fn());
vi.mock("../../sentry.js", () => ({
captureRunFailure: mockCaptureRunFailure,
}));
import { reportRunFailure } from "../run-failure-report.js";
const testDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL;
const embeddedPostgresSupport = testDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
const limitFailure = {
schema: "paperclip.provider-failure.v1",
provider: "anthropic",
phase: "turn",
category: "quota_exhausted",
message: "ACP agent reported a terminal limit failure.\nYou've hit your weekly limit · resets Nov 10, 3pm (UTC)",
retryability: "requires_action",
};
describeEmbeddedPostgres("reportRunFailure native provider usage limits", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let companyId!: string;
let agentId!: string;
let inheritedEnv: NodeJS.ProcessEnv;
beforeAll(async () => {
if (testDatabaseUrl) {
await applyPendingMigrations(testDatabaseUrl);
db = createDb(testDatabaseUrl);
} else {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-failure-native-usage-limit-");
db = createDb(tempDb.connectionString);
}
}, 20_000);
beforeEach(async () => {
inheritedEnv = process.env;
process.env = Object.fromEntries(
["PATH", "HOME", "USER", "USERNAME", "LOGNAME", "USERPROFILE", "TMPDIR", "TEMP", "TMP"]
.flatMap((key) => inheritedEnv[key] === undefined ? [] : [[key, inheritedEnv[key]]]),
);
process.env.PAPERCLIP_SENTRY_RUN_FAILURES = "unexpected";
companyId = randomUUID();
agentId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
defaultResponsibleUserId: "responsible-user",
});
await db.insert(agents).values({
id: agentId,
companyId,
name: "Runner",
role: "engineer",
status: "active",
adapterType: "paperclip_runner",
adapterConfig: {},
runtimeConfig: {},
permissions: {},
});
});
afterEach(async () => {
vi.clearAllMocks();
vi.unstubAllEnvs();
process.env = inheritedEnv;
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(companies);
});
afterAll(async () => {
if (testDatabaseUrl) await closeRegisteredClients(testDatabaseUrl);
await tempDb?.cleanup();
});
/** The persisted shape of a native ACP turn that hit an exhausted provider allowance. */
function nativeUsageLimitRun(
overrides: Partial<typeof heartbeatRuns.$inferSelect> = {},
): typeof heartbeatRuns.$inferSelect {
return {
id: randomUUID(),
companyId,
agentId,
status: "failed",
error: "ACP agent reported a terminal limit failure.",
errorCode: "native_provider_usage_limit",
runtimeMode: "native",
nativePhase: "terminal_failure",
nativeIssueId: randomUUID(),
exitCode: null,
signal: null,
contextSnapshot: null,
resultJson: { providerFailure: limitFailure },
...overrides,
} as unknown as typeof heartbeatRuns.$inferSelect;
}
it.each(["quota_exhausted", "rate_limit"])("keeps a native %s usage limit on the task", async (category) => {
const run = nativeUsageLimitRun({ resultJson: { providerFailure: { ...limitFailure, category } } });
const before = structuredClone(run);
for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, run, { phase });
expect(mockCaptureRunFailure).not.toHaveBeenCalled();
expect(run).toEqual(before);
});
it.each([
{ status: "timed_out" }, { signal: "SIGTERM" }, { runtimeMode: "legacy" },
{ errorCode: "native_provider_terminal_failed" }, { errorCode: "native_session_retry_exhausted" },
{ resultJson: {} }, { resultJson: null },
{ resultJson: { providerFailure: { ...limitFailure, category: "unknown", retryability: "unknown" } } },
{ resultJson: { providerFailure: { category: "quota_exhausted" } } },
])("reports a native usage limit outcome without typed limit evidence: %j", async (overrides) => {
await reportRunFailure(db, nativeUsageLimitRun(overrides), { phase: "execute" });
expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
});
it("reports a native usage limit during setup", async () => {
await reportRunFailure(db, nativeUsageLimitRun(), { phase: "setup" });
expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
});
it("reports native usage limits when every run failure is reported", async () => {
vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
await reportRunFailure(db, nativeUsageLimitRun(), { phase: "execute" });
expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,84 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { agentProviderLimitCooldown } from "./native-startup-retry.js";
const HOUR_MS = 60 * 60_000;
const DAY_MS = 24 * HOUR_MS;
/** Claude ACP wording for an exhausted weekly allowance, such as "resets Nov 10, 3pm (UTC)". */
function weeklyLimitMessage(at: Date) {
const month = at.toLocaleString("en-US", { month: "short", timeZone: "UTC" });
const hour = at.getUTCHours();
return `ACP agent reported a terminal limit failure.\nYou've hit your weekly limit · resets ${month} ${at.getUTCDate()}, ${
hour % 12 || 12}${hour < 12 ? "am" : "pm"} (UTC)`;
}
describe("native provider limit cooldown", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
const companyId = randomUUID();
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("native-provider-limit-cooldown-");
db = createDb(temporary.connectionString);
await db.insert(companies).values({ id: companyId, name: "Native provider limit", issuePrefix: "NPL" });
});
afterAll(async () => {
await temporary?.cleanup();
});
/** An agent whose latest native run failed on an exhausted weekly allowance. */
async function limitedAgent(resetAt: Date) {
const agentId = randomUUID();
await db.insert(agents).values({
id: agentId,
companyId,
name: `Limited agent ${agentId.slice(0, 8)}`,
status: "active",
adapterType: "paperclip_runner",
adapterConfig: { provider: "acpx", agent: "claude" },
});
const limitedAt = new Date(Date.now() - 5 * 60_000);
await db.insert(heartbeatRuns).values({
id: randomUUID(),
companyId,
agentId,
runtimeMode: "native",
invocationSource: "assignment",
triggerDetail: "system",
status: "failed",
startedAt: new Date(limitedAt.getTime() - 60_000),
finishedAt: limitedAt,
errorCode: "native_provider_usage_limit",
resultJson: {
providerFailure: {
schema: "paperclip.provider-failure.v1",
provider: "anthropic",
phase: "turn",
category: "quota_exhausted",
message: weeklyLimitMessage(resetAt),
retryability: "requires_action",
},
},
});
return agentId;
}
it("holds an agent until a dated weekly-limit reset instead of the one-hour default", async () => {
const resetAt = new Date(Math.ceil((Date.now() + 20 * HOUR_MS) / HOUR_MS) * HOUR_MS);
const agentId = await limitedAgent(resetAt);
expect(await agentProviderLimitCooldown(db, { companyId, agentId, now: new Date() }))
.toMatchObject({ resetAt, resetReported: true, category: "quota_exhausted" });
});
it("re-checks a dated reset days away within one day", async () => {
const agentId = await limitedAgent(new Date(Math.ceil((Date.now() + 3 * DAY_MS) / HOUR_MS) * HOUR_MS));
const now = new Date();
const cooldown = await agentProviderLimitCooldown(db, { companyId, agentId, now });
expect(cooldown).toMatchObject({ resetReported: true });
expect(cooldown!.resetAt.getTime()).toBe(now.getTime() + DAY_MS);
});
});
@@ -41,3 +41,68 @@ describe("parseProviderQuotaClockReset", () => {
expect(parseProviderQuotaClockReset("Provider quota exceeded for this model.", now)).toBeNull();
});
});
const weeklyLimit = "ACP agent reported a terminal limit failure.\nYou've hit your weekly limit · resets Nov 10, 3pm (UTC)";
describe("dated provider quota reset compatibility", () => {
it("reads a dated reset as that day, not as an unparsed limit", () => {
expect(parseProviderQuotaClockReset(weeklyLimit, new Date("2026-11-07T00:50:37Z")))
.toEqual(new Date("2026-11-10T15:00:00Z"));
});
it.each([
["resets Nov 10 at 3:30pm (UTC)", "2026-11-10T15:30:00Z"],
["resets November 10, 3 PM UTC", "2026-11-10T15:00:00Z"],
["resets Dec. 1, 9am (UTC)", "2026-12-01T09:00:00Z"],
["resets Nov 10th, 2026 3pm", "2026-11-10T15:00:00Z"],
["try again at Nov 10th, 2026 2:04 AM.", "2026-11-10T02:04:00Z"],
["resets Nov 10, 15:00 (UTC)", "2026-11-10T15:00:00Z"],
["resets Nov 10, 3pm (America/Chicago)", "2026-11-10T21:00:00Z"],
])("reads %s", (wording, expected) => {
expect(parseProviderQuotaClockReset(`You've hit your weekly limit · ${wording}`, new Date("2026-11-07T00:50:37Z")))
.toEqual(new Date(expected));
});
it("rolls a dated reset without a year into the next year at the year boundary", () => {
expect(parseProviderQuotaClockReset("resets Jan 2, 3pm (UTC)", new Date("2026-12-30T12:00:00Z")))
.toEqual(new Date("2027-01-02T15:00:00Z"));
});
it("keeps a recently passed dated reset in the past instead of a year later", () => {
expect(parseProviderQuotaClockReset("resets Nov 6, 3pm (UTC)", new Date("2026-11-07T00:50:37Z")))
.toEqual(new Date("2026-11-06T15:00:00Z"));
});
it("keeps stale calendar dates expired and rejects a far-future explicit year", () => {
const now = new Date("2026-11-07T00:50:37Z");
expect(parseProviderQuotaClockReset("resets Sept 1, 9am (UTC)", now)).toEqual(new Date("2026-09-01T09:00:00Z"));
expect(parseProviderQuotaClockReset("resets Nov 10, 2027 3pm (UTC)", now)).toBeNull();
});
it.each([
"resets Apr 31, 3pm (UTC)",
"resets Feb 29, 2027 3pm (UTC)",
"resets Nov 10, 2026",
"resets Nov 10, 2026 25pm (UTC)",
"resets Nov 0, 3pm (UTC)",
"resets Nov 10, 13pm (UTC)",
"resets Nov 10, 3:60pm (UTC)",
"resets Novembork 10, 3pm (UTC)",
"resets Nov 10, 3pm (Not/AZone)",
])("rejects the invalid dated reset %s", (wording) => {
expect(parseProviderQuotaClockReset(wording, new Date("2026-11-07T00:50:37Z"))).toBeNull();
});
it("keeps clock-only resets on the next matching wall clock", () => {
const now = new Date("2026-11-07T00:50:37Z");
expect(parseProviderQuotaClockReset("You've hit your session limit · resets 2am (UTC)", now))
.toEqual(new Date("2026-11-07T02:00:00Z"));
expect(parseProviderQuotaClockReset("try again at 4:30 PM (America/Chicago)", now))
.toEqual(new Date("2026-11-07T22:30:00Z"));
expect(parseProviderQuotaClockReset("resets 12am", now)).toEqual(new Date("2026-11-08T00:00:00Z"));
});
it("returns null when no reset is stated", () => {
expect(parseProviderQuotaClockReset("ACP agent reported a terminal limit failure.", new Date())).toBeNull();
});
});
+12 -8
View File
@@ -1,9 +1,11 @@
const MONTHS = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"];
const MONTH_NAMES = "jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?";
const MAX_DATED_RESET_AHEAD_MS = 45 * 24 * 60 * 60_000;
const RESET_PATTERN = new RegExp(
"(?:try again at|resets?(?:\\s+(?:at|on))?)\\s+" +
// Optional calendar date, as in "resets Nov 12, 6pm (UTC)" for a weekly limit.
`(?:(${MONTHS.join("|")})[a-z]*\\.?\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+(?:at\\s+)?)?` +
"(\\d{1,2})(?::(\\d{2}))?\\s*(?:([ap])\\.?\\s*m\\.?)?(?:\\s*\\(([^)]+)\\)|\\s+([A-Z]{2,5}))?",
`(?:(${MONTH_NAMES})\\b\\.?\\s+(\\d{1,2})(?:st|nd|rd|th)?(?:,?\\s+(\\d{4}))?,?\\s+(?:at\\s+)?)?` +
"(\\d{1,2})(?::(\\d{2}))?\\s*(?:([ap])\\.?\\s*m\\.?)?(?![\\d:])(?:\\s*\\(([^)]+)\\)|\\s+([A-Z]{2,5}))?",
"i",
);
@@ -20,10 +22,11 @@ export function parseProviderQuotaClockReset(error: string, now: Date) {
const monthIndex = match[1] ? MONTHS.indexOf(match[1].slice(0, 3).toLowerCase()) : null;
const dayOfMonth = match[2] ? Number.parseInt(match[2], 10) : null;
const statedYear = match[3] ? Number.parseInt(match[3], 10) : null;
if (monthIndex !== null && (monthIndex < 0 || !dayOfMonth || dayOfMonth < 1 || dayOfMonth > 31)) return null;
const hourValue = Number.parseInt(match[3] ?? "", 10);
const minute = Number.parseInt(match[4] ?? "0", 10);
const meridiem = (match[5] ?? "").toLowerCase();
const hourValue = Number.parseInt(match[4] ?? "", 10);
const minute = Number.parseInt(match[5] ?? "0", 10);
const meridiem = (match[6] ?? "").toLowerCase();
if (!Number.isInteger(hourValue)) return null;
if (
meridiem ? hourValue < 1 || hourValue > 12 : hourValue < 0 || hourValue > 23
@@ -33,7 +36,7 @@ export function parseProviderQuotaClockReset(error: string, now: Date) {
let hour = meridiem ? hourValue % 12 : hourValue;
if (meridiem === "p") hour += 12;
const timeZone = (match[6] ?? match[7])?.trim() || "UTC";
const timeZone = (match[7] ?? match[8])?.trim() || "UTC";
try {
const wallClock = (date: Date) =>
@@ -79,12 +82,13 @@ export function parseProviderQuotaClockReset(error: string, now: Date) {
// Dec 30 is next year's. A date read after it passed stays in the past,
// so a caller reading an old message sees an expired reset rather than
// one a year away.
const candidates = [year - 1, year, year + 1]
const candidates = (statedYear === null ? [year - 1, year, year + 1] : [statedYear])
.map((candidateYear) => instant(candidateYear, monthIndex, dayOfMonth))
.filter((candidate): candidate is Date => candidate !== null);
if (!candidates.length) return null;
return candidates.reduce((nearest, candidate) =>
const nearest = candidates.reduce((nearest, candidate) =>
Math.abs(candidate.getTime() - now.getTime()) < Math.abs(nearest.getTime() - now.getTime()) ? candidate : nearest);
return nearest.getTime() > now.getTime() + MAX_DATED_RESET_AHEAD_MS ? null : nearest;
}
const month = Number(nowParts.month) - 1;
const day = Number(nowParts.day);
+10
View File
@@ -21,6 +21,7 @@ import { PLUGIN_RUNNER_AI_CONNECTION_REQUIRED } from "./plugin-runner-ai-connect
import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-runtime/native-provider-failure.js";
import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js";
import { REVIEWED_CHAT_BINDING_NOT_READY } from "./reviewed-chat-binding-diagnostics.js";
import { readProviderFailure } from "@paperclipai/shared";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
@@ -259,6 +260,14 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOpti
options.phase !== "setup" && run.signal == null;
}
/** A typed native allowance failure stays on the task and its cooldown path. */
function isNativeProviderUsageLimitFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean {
if (run.status !== "failed" || run.runtimeMode !== "native" ||
run.errorCode !== "native_provider_usage_limit" || options.phase === "setup" || run.signal != null) return false;
const category = readProviderFailure(asRecord(run.resultJson)?.providerFailure)?.category;
return category === "quota_exhausted" || category === "rate_limit";
}
function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean {
const result = asRecord(run.resultJson);
return run.status === "failed" && run.errorCode === "codex_transient_upstream" &&
@@ -364,6 +373,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR
isProjectAccessDenialBlocker(run, options) ||
isGitConnectionFailure(run, options) ||
isProviderQuotaFailure(run, options) ||
isNativeProviderUsageLimitFailure(run, options) ||
isCodexAcpModelCapacityFailure(run, options) ||
isAcpProviderAccessFailure(run, options) ||
isScheduledReviewedChatRetry(run, options) ||