diff --git a/packages/db/src/migrations/0346_cloudy_whiplash.sql b/packages/db/src/migrations/0346_cloudy_whiplash.sql new file mode 100644 index 00000000000..c9018ab00a3 --- /dev/null +++ b/packages/db/src/migrations/0346_cloudy_whiplash.sql @@ -0,0 +1,2 @@ +-- Find an assignment's work wakes without scanning the company's dispatch history. +CREATE INDEX "grok_dispatch_assignment_kind_idx" ON "grok_bot_dispatches" USING btree ("assignment_id","kind"); diff --git a/packages/db/src/migrations/meta/0341_snapshot.json b/packages/db/src/migrations/meta/0346_snapshot.json similarity index 98% rename from packages/db/src/migrations/meta/0341_snapshot.json rename to packages/db/src/migrations/meta/0346_snapshot.json index 1c38823f95a..f57eebd88db 100644 --- a/packages/db/src/migrations/meta/0341_snapshot.json +++ b/packages/db/src/migrations/meta/0346_snapshot.json @@ -1,6 +1,6 @@ { - "id": "927d7532-059b-4a4f-846a-5e3445a81891", - "prevId": "b32fa655-79c7-4793-a021-22258b6c369c", + "id": "40048887-b6ac-4334-b77b-6b8fc4b52701", + "prevId": "3bd7b1df-1434-4fac-87d6-53b90a9eb9e1", "version": "7", "dialect": "postgresql", "tables": { @@ -4098,6 +4098,12 @@ "primaryKey": false, "notNull": false }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -16852,6 +16858,53 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.company_skill_import_budgets": { + "name": "company_skill_import_budgets", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "available_bytes": { + "name": "available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 536870912 + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "company_skill_import_budgets_company_id_companies_id_fk": { + "name": "company_skill_import_budgets_company_id_companies_id_fk", + "tableFrom": "company_skill_import_budgets", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_skill_source_entries": { "name": "company_skill_source_entries", "schema": "", @@ -24927,6 +24980,13 @@ "primaryKey": false, "notNull": false }, + "provider_kind": { + "name": "provider_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'openai_dot'" + }, "generation": { "name": "generation", "type": "integer", @@ -25554,6 +25614,18 @@ "primaryKey": false, "notNull": false }, + "pending_returned_at": { + "name": "pending_returned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, "source_event_id": { "name": "source_event_id", "type": "text", @@ -30638,6 +30710,495 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.grok_bot_dispatches": { + "name": "grok_bot_dispatches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_id": { + "name": "binding_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_generation": { + "name": "binding_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "routine_revision": { + "name": "routine_revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "assignment_id": { + "name": "assignment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "source_event_id": { + "name": "source_event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "challenge_hash": { + "name": "challenge_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'prepared'" + }, + "http_status": { + "name": "http_status", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "acknowledged_at": { + "name": "acknowledged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "grok_dispatch_event_uq": { + "name": "grok_dispatch_event_uq", + "columns": [ + { + "expression": "binding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "routine_revision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "grok_dispatch_company_status_idx": { + "name": "grok_dispatch_company_status_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "grok_dispatch_assignment_kind_idx": { + "name": "grok_dispatch_assignment_kind_idx", + "columns": [ + { + "expression": "assignment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "grok_bot_dispatches_company_id_companies_id_fk": { + "name": "grok_bot_dispatches_company_id_companies_id_fk", + "tableFrom": "grok_bot_dispatches", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grok_bot_dispatches_routine_id_grok_bot_routines_id_fk": { + "name": "grok_bot_dispatches_routine_id_grok_bot_routines_id_fk", + "tableFrom": "grok_bot_dispatches", + "tableTo": "grok_bot_routines", + "columnsFrom": [ + "routine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grok_bot_dispatches_binding_id_dot_agent_bindings_id_fk": { + "name": "grok_bot_dispatches_binding_id_dot_agent_bindings_id_fk", + "tableFrom": "grok_bot_dispatches", + "tableTo": "dot_agent_bindings", + "columnsFrom": [ + "binding_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grok_bot_dispatches_grant_id_mcp_oauth_grants_id_fk": { + "name": "grok_bot_dispatches_grant_id_mcp_oauth_grants_id_fk", + "tableFrom": "grok_bot_dispatches", + "tableTo": "mcp_oauth_grants", + "columnsFrom": [ + "grant_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grok_bot_dispatches_assignment_id_dot_runner_assignments_id_fk": { + "name": "grok_bot_dispatches_assignment_id_dot_runner_assignments_id_fk", + "tableFrom": "grok_bot_dispatches", + "tableTo": "dot_runner_assignments", + "columnsFrom": [ + "assignment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grok_bot_routines": { + "name": "grok_bot_routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_id": { + "name": "binding_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_generation": { + "name": "binding_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bot_name": { + "name": "bot_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_id": { + "name": "secret_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "secret_version": { + "name": "secret_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "verified_grant_id": { + "name": "verified_grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_contact_at": { + "name": "last_contact_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "recovery_required_at": { + "name": "recovery_required_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "recovery_reason": { + "name": "recovery_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "grok_routines_binding_uq": { + "name": "grok_routines_binding_uq", + "columns": [ + { + "expression": "binding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "grok_routines_company_agent_idx": { + "name": "grok_routines_company_agent_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "grok_bot_routines_company_id_companies_id_fk": { + "name": "grok_bot_routines_company_id_companies_id_fk", + "tableFrom": "grok_bot_routines", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grok_bot_routines_agent_id_agents_id_fk": { + "name": "grok_bot_routines_agent_id_agents_id_fk", + "tableFrom": "grok_bot_routines", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grok_bot_routines_binding_id_dot_agent_bindings_id_fk": { + "name": "grok_bot_routines_binding_id_dot_agent_bindings_id_fk", + "tableFrom": "grok_bot_routines", + "tableTo": "dot_agent_bindings", + "columnsFrom": [ + "binding_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grok_bot_routines_secret_id_company_secrets_id_fk": { + "name": "grok_bot_routines_secret_id_company_secrets_id_fk", + "tableFrom": "grok_bot_routines", + "tableTo": "company_secrets", + "columnsFrom": [ + "secret_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "grok_bot_routines_verified_grant_id_mcp_oauth_grants_id_fk": { + "name": "grok_bot_routines_verified_grant_id_mcp_oauth_grants_id_fk", + "tableFrom": "grok_bot_routines", + "tableTo": "mcp_oauth_grants", + "columnsFrom": [ + "verified_grant_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.heartbeat_run_events": { "name": "heartbeat_run_events", "schema": "", @@ -31967,6 +32528,54 @@ "concurrently": false, "method": "btree", "with": {} + }, + "heartbeat_runs_company_ctx_paperclip_issue_created_idx": { + "name": "heartbeat_runs_company_ctx_paperclip_issue_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"context_snapshot\" -> 'paperclipIssue' ->> 'id')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": true, + "method": "btree", + "with": {} + }, + "heartbeat_runs_company_origin_issue_idx": { + "name": "heartbeat_runs_company_origin_issue_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\n \"native_issue_id\"::text,\n nullif(\"context_snapshot\"->>'issueId', ''),\n nullif(\"context_snapshot\"->>'taskId', ''),\n nullif(\"context_snapshot\"->>'taskKey', '')\n )", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": true, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -58385,6 +58994,18 @@ "name": "mcp_attachment_uploads", "schema": "", "columns": { + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "work_session_id": { + "name": "work_session_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "id": { "name": "id", "type": "uuid", @@ -58529,6 +59150,19 @@ } }, "foreignKeys": { + "mcp_attachment_uploads_grant_id_mcp_oauth_grants_id_fk": { + "name": "mcp_attachment_uploads_grant_id_mcp_oauth_grants_id_fk", + "tableFrom": "mcp_attachment_uploads", + "tableTo": "mcp_oauth_grants", + "columnsFrom": [ + "grant_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, "mcp_attachment_uploads_company_id_companies_id_fk": { "name": "mcp_attachment_uploads_company_id_companies_id_fk", "tableFrom": "mcp_attachment_uploads", @@ -59005,6 +59639,12 @@ "name": "mcp_file_tickets", "schema": "", "columns": { + "work_session_id": { + "name": "work_session_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "token_hash": { "name": "token_hash", "type": "text", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 1fd5c5fb5a8..00a90de13cc 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2409,6 +2409,13 @@ "when": 1791978393174, "tag": "0345_luxuriant_carmella_unuscione", "breakpoints": true + }, + { + "idx": 346, + "version": "7", + "when": 1791989860950, + "tag": "0346_cloudy_whiplash", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/grok_bot.ts b/packages/db/src/schema/grok_bot.ts index 1ec3f1d98c1..da978d3a629 100644 --- a/packages/db/src/schema/grok_bot.ts +++ b/packages/db/src/schema/grok_bot.ts @@ -50,4 +50,5 @@ export const grokBotDispatches = pgTable("grok_bot_dispatches", { }, t => [ uniqueIndex("grok_dispatch_event_uq").on(t.bindingId, t.bindingGeneration, t.routineRevision, t.sourceEventId), index("grok_dispatch_company_status_idx").on(t.companyId, t.status), + index("grok_dispatch_assignment_kind_idx").on(t.assignmentId, t.kind), ]); diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index c4f5271e206..9c6f143e9e6 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -1,7 +1,9 @@ import { randomUUID } from "node:crypto"; import os from "node:os"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { agents, applyPendingMigrations, authUsers, closeRegisteredClients, companies, companySecrets, createDb, dotAgentBindings, + dotRunnerAssignments, grokBotDispatches, grokBotRoutines, heartbeatRuns, issues, mcpOauthClients, mcpOauthGrants, nativeRunFinalizations, + type Db } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -67,9 +69,15 @@ describeEmbeddedPostgres("reportRunFailure", () => { vi.clearAllMocks(); vi.unstubAllEnvs(); process.env = inheritedEnv; + await db.delete(grokBotRoutines); + await db.delete(companySecrets); + await db.delete(nativeRunFinalizations); await db.delete(heartbeatRuns); + await db.delete(issues); await db.delete(agents); await db.delete(companies); + await db.delete(mcpOauthClients); + await db.delete(authUsers); }); afterAll(async () => { @@ -335,6 +343,80 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); }); + async function expiredGrokOffer(input: { + wakes: Array<{ status: "rejected" | "accepted" | "acknowledged" | "unknown"; httpStatus: number | null }>; + acceptedAt?: Date | null; + originalFailureCode?: string; + finalization?: boolean; + offers?: number; + }) { + await seedCompanyAndAgent({ adapterType: "paperclip_runner", adapterConfig: { provider: "grok_bot" } }); + const userId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name: "Operator", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(mcpOauthClients).values({ id: userId, name: "Grok", redirectUris: ["https://client.example/callback"] }); + const [grant] = await db.insert(mcpOauthGrants).values({ companyId, userId, clientId: userId, resource: "https://paperclip.example/mcp/runner", + purpose: "agent", agentId, scopes: ["paperclip:agent"] }).returning(); + const [binding] = await db.insert(dotAgentBindings).values({ companyId, agentId, operatorId: userId, grantId: grant!.id, + providerKind: "grok_bot", status: "ready" }).returning(); + const [secret] = await db.insert(companySecrets).values({ companyId, key: `grok-${binding!.id}`, name: "Grok routine" }).returning(); + const [routine] = await db.insert(grokBotRoutines).values({ companyId, agentId, bindingId: binding!.id, bindingGeneration: 1, + botName: "Grok", secretId: secret!.id, secretVersion: 1 }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Grok task", assigneeAgentId: agentId, status: "in_progress" }).returning(); + const run = buildRun({ errorCode: "native_provider_terminal_failed", error: "Dot assignment acceptance or execution authority expired.", + nativeIssueId: issue!.id, runtimeMode: "native", executionStage: "preparing", nativePhase: "terminal_failure", exitCode: null, signal: null }); + await db.insert(heartbeatRuns).values(run); + if (input.finalization !== false) { + await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId, issueId: run.nativeIssueId!, phase: "terminal_failure", + failureCode: "native_provider_terminal_failed", failureDetail: { originalFailureCode: input.originalFailureCode ?? "dot_assignment_expired" } }); + } + const offers = await db.insert(dotRunnerAssignments).values(Array.from({ length: input.offers ?? 1 }, () => ({ companyId, agentId, + bindingId: binding!.id, bindingGeneration: 1, runId: run.id, normalizedSessionId: randomUUID(), turnId: randomUUID(), controllerGeneration: 1, + catalogDigest: "fixture", projection: {}, status: "fenced" as const, acceptBy: new Date(Date.now() - 1000), + expiresAt: new Date(Date.now() + 3_600_000), acceptedAt: input.acceptedAt ?? null }))).returning(); + const assignment = offers[0]; + for (const wake of input.wakes) { + await db.insert(grokBotDispatches).values({ companyId, routineId: routine!.id, bindingId: binding!.id, bindingGeneration: 1, + grantId: grant!.id, routineRevision: 1, assignmentId: assignment!.id, sourceEventId: randomUUID(), kind: "mailbox_updated", + payload: {}, status: wake.status, httpStatus: wake.httpStatus, expiresAt: new Date(Date.now() + 600_000) }); + } + return run; + } + + it("keeps a Grok offer local when the endpoint rejected every work wake with HTTP 4xx", async () => { + const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: 400 }] }); + await reportRunFailure(db, run, { phase: "execute" }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it.each([ + ["no wake was sent", { wakes: [] }], + ["a wake was rejected locally without HTTP evidence", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: null }] }], + ["the endpoint accepted a wake", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "acknowledged", httpStatus: 200 }] }], + ["delivery is uncertain", { wakes: [{ status: "unknown", httpStatus: 503 }] }], + ["the bot accepted the work", { wakes: [{ status: "rejected", httpStatus: 400 }], acceptedAt: new Date() }], + ["the run failed for another provider reason", { wakes: [{ status: "rejected", httpStatus: 400 }], originalFailureCode: "provider_failed" }], + ["the run has no finalization", { wakes: [{ status: "rejected", httpStatus: 400 }], finalization: false }], + ["the run has more than one offer", { wakes: [{ status: "rejected", httpStatus: 400 }], offers: 2 }], + ] as const)("keeps an expired Grok offer reportable when %s", async (_label, input) => { + const run = await expiredGrokOffer({ ...input, wakes: [...input.wakes] }); + await reportRunFailure(db, run, { phase: "execute" }); + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + }); + + it("keeps an expired Grok offer reportable when its receipts cannot be read", async () => { + const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }] }); + const select = db.select.bind(db); + // The agent lookup succeeds; the receipt reads fail. + const spy = vi.spyOn(db, "select").mockImplementationOnce(select as typeof db.select) + .mockImplementation(() => { throw new Error("fixture receipt read failed"); }); + try { + await reportRunFailure(db, run, { phase: "execute" }); + } finally { + spy.mockRestore(); + } + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + }); + function aiSelectionRun(overrides: Partial = {}) { return buildRun({ errorCode: "configuration_incomplete", diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index 60e6309e541..e738dfe67a9 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -1,5 +1,5 @@ import { and, eq } from "drizzle-orm"; -import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { agents, dotRunnerAssignments, grokBotDispatches, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { captureRunFailure, type RunFailureStatus } from "../sentry.js"; import { collectRunFailureDiagnostics, @@ -174,6 +174,27 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOpti options.phase !== "setup" && run.signal == null; } +/** + * The Grok endpoint refused every work wake with an HTTP 4xx, so the offer + * expired unaccepted. Durable delivery receipts prove the external cause. A + * local rejection (no HTTP status), any delivered wake, or accepted work stays + * reportable. + */ +async function isRejectedExternalBotWake(db: Db, run: HeartbeatRun): Promise { + if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== "native_provider_terminal_failed") return false; + const [finalization] = await db.select({ failureDetail: nativeRunFinalizations.failureDetail }).from(nativeRunFinalizations) + .where(and(eq(nativeRunFinalizations.runId, run.id), eq(nativeRunFinalizations.companyId, run.companyId))); + if (finalization?.failureDetail?.originalFailureCode !== "dot_assignment_expired") return false; + const offers = await db.select({ id: dotRunnerAssignments.id, acceptedAt: dotRunnerAssignments.acceptedAt }).from(dotRunnerAssignments) + .where(and(eq(dotRunnerAssignments.runId, run.id), eq(dotRunnerAssignments.companyId, run.companyId))); + if (offers.length !== 1 || offers[0]!.acceptedAt) return false; + const wakes = await db.select({ status: grokBotDispatches.status, httpStatus: grokBotDispatches.httpStatus }).from(grokBotDispatches) + .where(and(eq(grokBotDispatches.assignmentId, offers[0]!.id), eq(grokBotDispatches.companyId, run.companyId), + eq(grokBotDispatches.kind, "mailbox_updated"))); + return wakes.length > 0 && wakes.every((wake) => + wake.status === "rejected" && wake.httpStatus != null && wake.httpStatus >= 400 && wake.httpStatus < 500); +} + /** * Report a terminal run failure to Sentry. Returns at once for any status * other than failures and unexpected started cancellations. Never throws — a Sentry failure or a @@ -222,6 +243,12 @@ async function captureTerminalRunFailure( .then((rows) => rows[0] ?? null); if (isHermesConnectionFailure(run, options, agent?.adapterType)) return; + // An unreadable receipt proves nothing; keep the failure reportable. + const rejectedExternally = await isRejectedExternalBotWake(db, run).catch((err) => { + logger.warn({ err, runId: run.id }, "failed to read bot wake receipts for run failure report"); + return false; + }); + if (rejectedExternally) return; const taskId = readTaskId(run); if (!taskId) {