diff --git a/server/src/__tests__/tool-gateway-connector-telemetry.test.ts b/server/src/__tests__/tool-gateway-connector-telemetry.test.ts index 1674b84e906..d34f10dbf43 100644 --- a/server/src/__tests__/tool-gateway-connector-telemetry.test.ts +++ b/server/src/__tests__/tool-gateway-connector-telemetry.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, @@ -304,7 +304,7 @@ describeEmbeddedPostgres("gateway connector invocation telemetry", () => { async function fixture(options: Parameters[1] = {}) { const company = await createCompany(db); const agent = await createAgent(db, company.id); - const { run } = await createIssueAndRun(db, company.id, agent.id); + const { issue, run } = await createIssueAndRun(db, company.id, agent.id); const remoteTool = await createRemoteMcpTool(db, company.id); await allowAllToolsForAgent(db, company.id, agent.id); const gateway = createToolGatewayService(db, { @@ -321,7 +321,7 @@ describeEmbeddedPostgres("gateway connector invocation telemetry", () => { (candidate) => candidate.connectionId === remoteTool.connection.id, ); expect(tool).toBeDefined(); - return { company, agent, run, remoteTool, gateway, session, tool: tool! }; + return { company, agent, issue, run, remoteTool, gateway, session, tool: tool! }; } it("emits exactly one succeeded event for a successful agent execution", async () => { @@ -345,6 +345,55 @@ describeEmbeddedPostgres("gateway connector invocation telemetry", () => { expect(scopesOf("connection.invoked")).toEqual([f.company.id]); }, 30_000); + it("keeps the result and its audit row when the audit insert loses a deadlock to an issue-then-run locker", async () => { + const locker = createDb(tempDb!.connectionString); + let startLocker!: () => void, issueHeld!: () => void, lockRun!: () => void; + const lockerStart = new Promise(resolve => { startLocker = resolve; }); + const issueLocked = new Promise(resolve => { issueHeld = resolve; }); + const runGate = new Promise(resolve => { lockRun = resolve; }); + const f = await fixture({ + remoteHttpRequest: async (_url, init) => { + if (JSON.parse(String(init.body ?? "{}"))?.method === "tools/call") { + startLocker(); + await issueLocked; + } + return mcpSuccessResponse(init); + }, + }); + const lockerDone = lockerStart.then(() => locker.transaction(async tx => { + await tx.execute(sql`set local deadlock_timeout = '20s'`); + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, f.issue.id)).for("update"); + issueHeld(); + await runGate; + await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update"); + })); + try { + const call = f.gateway.executeTool({ + sessionToken: f.session.token, + tool: f.tool.name, + parameters: { key: "a", value: "1" }, + }).then(value => ({ value }), error => ({ error })); + await vi.waitFor(async () => { + const [waiting] = await locker.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where wait_event_type = 'Lock' and query like 'insert into "tool_call_events"%'`); + expect(waiting?.count).toBe(1); + }, { timeout: 10_000, interval: 50 }); + lockRun(); + await lockerDone; + const outcome = await call; + expect(outcome).not.toHaveProperty("error"); + expect(outcome).toMatchObject({ value: { status: "completed" } }); + expect(await db.select({ id: toolCallEvents.id }).from(toolCallEvents).where(and( + eq(toolCallEvents.runId, f.run.id), eq(toolCallEvents.eventType, "call_completed")))).toHaveLength(1); + } finally { + startLocker(); + issueHeld(); + lockRun(); + await lockerDone.catch(() => undefined); + await locker.$client.end({ timeout: 1 }); + } + }, 30_000); + it("emits a single failed event when post-success audit work throws (no contradictory pair)", async () => { const f = await fixture(); failLogActivityForAction.current = "tool_gateway.call_completed"; diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index 2fd02c61f25..af153b9703d 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -7,6 +7,7 @@ import { type AwsMcpClients, } from "./connectors/aws/mcp-credentials.js"; import { workloadIdentityService } from "./workload-identity.js"; +import { retryTransientLockContention } from "./transient-lock-contention.js"; import { isNewHumanIntentAfterToolRejection } from "./tool-action-replay-intent.js"; import { approvalSnapshotsMatch, approvalTargetChangeMessage, oauthApprovalConfig, approvalGrantTenantIdentity, externalCredentialApprovalIdentity } from "./tool-approval-target.js"; @@ -2666,7 +2667,7 @@ export function createToolGatewayService( tool?: ToolGatewayDescriptor | null; }) { const metadata = input.tool ? toolAuditMetadata(input.tool) : {}; - await db.insert(toolCallEvents).values({ + await retryTransientLockContention(() => db.insert(toolCallEvents).values({ companyId: input.session.companyId, invocationId: input.invocationId ?? null, actionRequestId: input.actionRequestId ?? null, @@ -2709,7 +2710,7 @@ export function createToolGatewayService( ...(input.detectorMetadata ? { promptInjection: input.detectorMetadata } : {}), } : null, - }); + })); } async function reflectToolActionInteractionLifecycle(input: {