diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index 01a7dae4263..6c203e2549d 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -122,6 +122,7 @@ describeEmbeddedPostgres("reportRunFailure", () => { errorCode: "process_lost", nativeIssueId: randomUUID(), contextSnapshot: null, + createdAt: new Date(), ...overrides, } as unknown as typeof heartbeatRuns.$inferSelect; } @@ -530,6 +531,79 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).toHaveBeenCalledTimes(3); }); + async function nativeToolReceiptLimitStop(input: { + run?: Partial; + retry?: Partial | null; + } = {}) { + await seedCompanyAndAgent({ adapterType: "paperclip_runner" }); + const run = buildRun({ + status: "cancelled", error: "Native session cancelled", errorCode: "cancelled", + runtimeMode: "native", nativePhase: "committed", exitCode: 1, signal: null, + startedAt: new Date(0), finishedAt: new Date(1000), + resultJson: { + stopReason: "cancelled", finalizationPhase: "committed", finalizationReasonCode: "semantic_tool_turn_receipt_limit", + cancellation: { source: "unknown", expected: false, initiator: { type: "system" }, + reason: "Execution stopped without a recorded cancellation request: Native session cancelled", + recordedAt: new Date(1000).toISOString() }, + }, + ...input.run, + }); + const [stored] = await db.insert(heartbeatRuns).values(run).returning(); + if (input.retry !== null) { + await db.insert(heartbeatRuns).values({ + companyId, agentId, invocationSource: "automation", triggerDetail: "system", status: "scheduled_retry", + retryOfRunId: run.id, scheduledRetryReason: "semantic_tool_turn_receipt_limit", scheduledRetryAttempt: 1, + scheduledRetryAt: new Date(Date.now() + 5_000), errorCode: "semantic_tool_turn_receipt_limit", + contextSnapshot: { issueId: run.nativeIssueId, retryOfRunId: run.id }, ...input.retry, + }); + } + return stored!; + } + + it("keeps a native tool-receipt-limit stop local once its fresh-session continuation is scheduled", async () => { + const run = await nativeToolReceiptLimitStop(); + await reportRunFailure(db, run); + await reportRunFailure(db, run, { phase: "execute" }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it("keeps a failed native run local when the finalizer scheduled its receipt-limit continuation", async () => { + const run = await nativeToolReceiptLimitStop({ run: { status: "failed", errorCode: "native_run_failed", + error: "The provider turn failed", resultJson: { finalizationPhase: "committed" } } }); + await reportRunFailure(db, run); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it.each([ + ["the continuations are exhausted", { retry: null }], + ["the successor belongs to another retry lane", { retry: { scheduledRetryReason: "native_safe_replacement" } }], + ["the successor belongs to another task", { retry: { contextSnapshot: { issueId: randomUUID() } } }], + ["the run has no native task", { run: { nativeIssueId: null } }], + ["the run is legacy", { run: { runtimeMode: "legacy" } }], + ["the run has a signal", { run: { signal: "SIGTERM" } }], + ["the run timed out", { run: { status: "timed_out" } }], + ] as const)("reports a native tool-receipt-limit stop when %s", async (_label, input) => { + const run = await nativeToolReceiptLimitStop(input as Parameters[0]); + await reportRunFailure(db, run); + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + }); + + it("reports a native tool-receipt-limit stop during setup, in all mode, or when its continuation cannot be read", async () => { + const run = await nativeToolReceiptLimitStop(); + await reportRunFailure(db, run, { phase: "setup" }); + const select = db.select.bind(db); + const spy = vi.spyOn(db, "select").mockImplementationOnce(select as typeof db.select) + .mockImplementation(() => { throw new Error("fixture continuation read failed"); }); + try { + await reportRunFailure(db, run); + } finally { + spy.mockRestore(); + } + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all"); + await reportRunFailure(db, run); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(3); + }); + function providerAccessRun(overrides: Partial = {}) { return buildRun({ error: "ACP agent reported a terminal access failure.\nInvalid API key ยท Fix external API key", diff --git a/server/src/services/native-runtime/native-tool-receipt-limit.integration.test.ts b/server/src/services/native-runtime/native-tool-receipt-limit.integration.test.ts index 594a1ee0262..8a37785b7c3 100644 --- a/server/src/services/native-runtime/native-tool-receipt-limit.integration.test.ts +++ b/server/src/services/native-runtime/native-tool-receipt-limit.integration.test.ts @@ -8,6 +8,10 @@ import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/ const provider = vi.hoisted(() => ({ execute: vi.fn() })); const receiptLimitRead = vi.hoisted(() => ({ after: null as null | (() => Promise) })); +const sentry = vi.hoisted(() => ({ captureRunFailure: vi.fn((_event: { runId: string }) => true) })); +vi.mock("../../sentry.js", async (importOriginal) => ({ + ...(await importOriginal()), captureRunFailure: sentry.captureRunFailure, +})); vi.mock("./native-tool-receipt-limit.js", async (importOriginal) => { const original = await importOriginal(); return { ...original, readPersistedNativeToolReceiptLimitStop: async (...args: Parameters) => { @@ -33,24 +37,30 @@ import { createReleaseIssueExecution } from "../../modules/wake-queue/applicatio import { taskInterruptionPresentation } from "../task-interruption-presentation.js"; import { materializeNativeToolReceiptLimitContinuation } from "./native-provider-capacity-retry.js"; import { NativeStatusRaceError } from "./status-decision-committer.js"; +import { waitForPendingRunFailureReports } from "../run-failure-report.js"; describe("native turn stopped at its tool receipt limit", () => { let temporary: Awaited>; let db: ReturnType; const companyId = randomUUID(), agentId = randomUUID(); beforeAll(async () => { + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "unexpected"); temporary = await startEmbeddedPostgresTestDatabase("native-tool-receipt-limit-"); db = createDb(temporary.connectionString); await db.insert(companies).values({ id: companyId, name: "Receipt limit regression", issuePrefix: "RCL" }); await db.insert(agents).values({ id: agentId, companyId, name: "Receipt worker", status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "test-model" } }); }); - afterAll(async () => { await temporary?.cleanup(); }); + afterAll(async () => { vi.unstubAllEnvs(); await temporary?.cleanup(); }); + async function sentryReports(runId: string) { + await waitForPendingRunFailureReports(); + return sentry.captureRunFailure.mock.calls.filter(([event]) => event.runId === runId); + } let finalTurnId: string | null = null; async function stopAtReceiptLimit(runId: string, issueId: string, diagnosticCode: string, beforeFinalize?: () => Promise, diagnosticTurnId?: string) { const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); - const [run] = await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativeIssueId: issueId, error: null, errorCode: null }).where(eq(heartbeatRuns.id, runId)).returning(); + const [run] = await db.update(heartbeatRuns).set({ status: "running", startedAt: new Date(), runtimeMode: "native", nativeIssueId: issueId, error: null, errorCode: null }).where(eq(heartbeatRuns.id, runId)).returning(); await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); @@ -115,6 +125,7 @@ describe("native turn stopped at its tool receipt limit", () => { contextSnapshot: { issueId, forceFreshSession: true, livenessContinuationSourceRunId: runId, livenessContinuationReason: NATIVE_TOOL_RECEIPT_LIMIT_CODE, livenessContinuationInstruction: NATIVE_TOOL_RECEIPT_LIMIT_HINT } }); expect(retry!.contextSnapshot).not.toHaveProperty("wakeCommentId"); + expect(await sentryReports(runId)).toHaveLength(0); expect(retry!.scheduledRetryAt!.getTime() - Date.now()).toBeLessThanOrEqual(NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_DELAY_MS); const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); expect(decision).toMatchObject({ reasonCode: NATIVE_TOOL_RECEIPT_LIMIT_CODE }); @@ -170,6 +181,7 @@ describe("native turn stopped at its tool receipt limit", () => { expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); expect(decision).toMatchObject({ reasonCode: "cancellation_run_only" }); + expect(await sentryReports(runId)).toEqual([[expect.objectContaining({ runStatus: "cancelled" })]]); }); it("refuses to create the continuation under the run lock once a person's Stop is saved", async () => { diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index 50d1a074ded..a514a340bd8 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -20,6 +20,7 @@ import { RESTRICTED_WORKSPACE_SOURCE_REASON } from "./workspace-source-restricti import { PLUGIN_RUNNER_AI_CONNECTION_REQUIRED } from "./plugin-runner-ai-connection.js"; import { PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE } from "./project-storage-environment-reason.js"; import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-runtime/native-provider-failure.js"; +import { NATIVE_TOOL_RECEIPT_LIMIT_CODE } from "./native-runtime/native-tool-receipt-limit.js"; import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js"; import { REVIEWED_CHAT_BINDING_NOT_READY } from "./reviewed-chat-binding-diagnostics.js"; import { readProviderFailure } from "@paperclipai/shared"; @@ -313,14 +314,20 @@ function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureRe * carries the task in its context, so the company/context-issue index bounds * the read to that task's runs created since this one. */ -async function isScheduledNativeCapacityRetry(db: Db, run: HeartbeatRun, options: RunFailureReportOptions): Promise { - if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== NATIVE_PROVIDER_OVERLOADED_CODE || - run.nativeIssueId == null || options.phase === "setup" || run.signal != null) return false; +function scheduledNativeRetryReason(run: HeartbeatRun, options: RunFailureReportOptions): string | null { + if (run.runtimeMode !== "native" || run.nativeIssueId == null || options.phase === "setup" || run.signal != null) return null; + if (run.status === "failed" && run.errorCode === NATIVE_PROVIDER_OVERLOADED_CODE) return NATIVE_PROVIDER_OVERLOADED_CODE; + return run.status === "failed" || run.status === "cancelled" ? NATIVE_TOOL_RECEIPT_LIMIT_CODE : null; +} + +async function isScheduledNativeRetry(db: Db, run: HeartbeatRun, options: RunFailureReportOptions): Promise { + const reason = scheduledNativeRetryReason(run, options); + if (!reason) return false; const [retry] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, run.companyId), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${run.nativeIssueId}`, gte(heartbeatRuns.createdAt, run.createdAt), eq(heartbeatRuns.agentId, run.agentId), - eq(heartbeatRuns.retryOfRunId, run.id), eq(heartbeatRuns.scheduledRetryReason, NATIVE_PROVIDER_OVERLOADED_CODE), + eq(heartbeatRuns.retryOfRunId, run.id), eq(heartbeatRuns.scheduledRetryReason, reason), )).limit(1); return retry !== undefined; } @@ -446,11 +453,11 @@ async function captureTerminalRunFailure( return false; }); if (rejectedExternally) return; - const capacityRetry = await isScheduledNativeCapacityRetry(db, run, options).catch((err) => { - logger.warn({ err, runId: run.id }, "failed to read native capacity retry for run failure report"); + const scheduledRetry = await isScheduledNativeRetry(db, run, options).catch((err) => { + logger.warn({ err, runId: run.id }, "failed to read native scheduled retry for run failure report"); return false; }); - if (capacityRetry) return; + if (scheduledRetry) return; const restartEndedUsageLimit = await isRestartEndedNativeUsageLimit(db, run, options).catch((err) => { logger.warn({ err, runId: run.id }, "failed to read native usage-limit terminal for run failure report"); return false;