diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md index 24db42b22ac..fe1b460ff8a 100644 --- a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md +++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md @@ -415,12 +415,23 @@ from event-driven normal paths and route only real deadlines through the interna scheduler. Preserve durable delivery, startup recovery, and domain-level claims. Longer polling intervals alone do not complete an item. -- [ ] **JOB-01 — Chat reconciliation.** Current: every **1s**, with multiple - runtime/delivery/publication lanes and SQL even without connectors. Change: - disarm empty lanes and activate on committed events or deadlines. **Complete - when:** a tenant with no chat obligations issues no chat SQL and new inbound, - publication, and recovery work still runs. Sources: +- [ ] **JOB-01 — Chat reconciliation.** Partially migrated on 2026-10-10. + Inbound deliveries, outbound publications, and Slack upload receipt recovery + now use transaction-aware commit wakes and durable retry/claim deadlines. + Empty queues disarm after startup. Publication deadlines use the dispatch + eligibility predicate so paused endpoints, unknown outcomes, blocked followers, + and receipts owned by a live sender do not cause empty scans. Completion wakes + refill free endpoint slots without waiting for slow peers. No new tables. + Remaining **1s** maintenance: provider runtimes, provider-specific action + outboxes, run milestone/wake-notice projection, GitHub operation recovery, + Teams protocol maintenance, and Slack session sync. GitHub pre-ingress recovery + must still inspect provider history: a lost callback has no local queue row. + This slice does **not** yet make the whole chat subsystem quiet. Independent + DB roots/direct SQL require startup recovery, as with the other migrated queues. + **Complete when:** all remaining lanes are demand/deadline-driven and a tenant + with no chat obligations issues no chat SQL. Sources: [app.ts](../../server/src/app.ts), + [chat-delivery-work.ts](../../server/src/services/chat-delivery-work.ts), [chat-channels.ts](../../server/src/services/chat-channels.ts). - [x] **JOB-02 — Email reconciliation.** Replaced the unconditional **1s** diff --git a/server/src/__tests__/app-private-hostname-gate.test.ts b/server/src/__tests__/app-private-hostname-gate.test.ts index 3e688f40f1a..2b26ebc7d16 100644 --- a/server/src/__tests__/app-private-hostname-gate.test.ts +++ b/server/src/__tests__/app-private-hostname-gate.test.ts @@ -90,17 +90,18 @@ describe("createChatReconciliationCoordinator", () => { expect(ordinary).not.toContain("processFailedGitHubWebhookDeliveries()"); }); - it("wires periodic publication reconciliation to bounded scheduled refill rather than awaiting provider sends", () => { + it("wires bounded publication refill to commit workers and isolates periodic maintenance", () => { const source = readFileSync(new URL("../app.ts", import.meta.url), "utf8"); - const flush = source.slice( - source.indexOf("const flushChatPublications ="), - source.indexOf("const flushChatPublications =") + 600, + expect(source).toContain("registerChatDeliveryWork(deliveryWork, chatChannels,"); + const maintenance = source.slice( + source.indexOf("const reconcileChatPublicationMaintenance ="), + source.indexOf("const chatReconciliation ="), ); - expect(flush).toContain("await chatChannels.schedulePendingPublications()"); - expect(flush).not.toContain("chatChannels.processPendingPublications()"); - expect( - flush.slice(0, flush.indexOf("const chatReconciliation")), - ).not.toContain("await enqueueChatRunMilestones"); + expect(maintenance).toContain("await chatChannels.processPublicationMaintenance()"); + expect(maintenance).not.toContain("chatChannels.schedulePendingPublications()"); + expect(maintenance).not.toContain("chatChannels.processPendingPublications()"); + const workers = readFileSync(new URL("../services/chat-delivery-work.ts", import.meta.url), "utf8"); + expect(workers).toContain("run: service.scheduleQueuedPublications"); // The service integration tests hold real publication workers while this // scheduled method returns; app shutdown must also join those workers. expect(source).toContain("await chatChannels.shutdown()"); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index f1608aac12f..c237fdeba3d 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1,3 +1,7 @@ +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerChatDeliveryWork } from "../services/chat-delivery-work.js"; +import { notifyChatPublicationWork } from "../services/chat-work-notifications.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; import { chatSlackRegistrations, toolOauthStates } from "@paperclipai/db"; import { buildSlackAppManifest } from "@paperclipai/shared"; import { toolAccessService } from "../services/tool-access.js"; @@ -7345,7 +7349,12 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const countBefore = postFetch.mock.calls.length; await executeConnectorTool(db, binding, "slack_post_message", uncertain); expect(postFetch.mock.calls).toHaveLength(countBefore); - await expect(executeSlackTool(db, binding, "slack_delivery", { actionId: uncertainAction.id }, postFetch as typeof fetch)).resolves.toMatchObject({ state: "delivered", receipt: { reconciled: true } }); + const receiptSettled = vi.fn(); + const unsubscribeReceipt = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatPublications, receiptSettled); + try { + await expect(executeSlackTool(db, binding, "slack_delivery", { actionId: uncertainAction.id }, postFetch as typeof fetch)).resolves.toMatchObject({ state: "delivered", receipt: { reconciled: true } }); + expect(receiptSettled).toHaveBeenCalledTimes(1); + } finally { unsubscribeReceipt(); } await expect(executeConnectorTool(db, binding, "slack_create_channel", { name: "approval-required", is_private: true, idempotencyKey: randomUUID() })).rejects.toMatchObject({ reasonCode: expect.stringMatching(/approval/) }); expect(postFetch.mock.calls.some(call => String(call[0]).endsWith("/conversations.create"))).toBe(false); const [approval] = await db.select().from(toolActionRequests).where(eq(toolActionRequests.companyId, fixture.companyId)); @@ -14482,6 +14491,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .onDiscordGatewayEvent; if (!onGatewayEvent) throw new Error("Expected Discord Gateway callback"); + const recovered = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatPublications, recovered); const disconnect = Promise.resolve( onGatewayEvent({ endpointId: endpoint.id, @@ -14505,7 +14516,10 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await Promise.resolve(); expect(recoveryFinished).toBe(false); releaseDisconnect(); - await Promise.all([disconnect, recovery]); + try { + await Promise.all([disconnect, recovery]); + expect(recovered).toHaveBeenCalled(); + } finally { unsubscribe(); } await expect(context.service.get(endpoint.id)).resolves.toMatchObject({ status: "active", healthMessage: "Connected", @@ -17360,10 +17374,22 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const fixture = await seedCompany(); const { endpoint, service } = await configuredSlackEndpoint(fixture); const configuredEndpoint = await service.get(endpoint.id); + const originalTransaction = db.transaction.bind(db); const transaction = vi.spyOn(db, "transaction"); - transaction.mockRejectedValueOnce( - new Error("injected lifecycle persistence failure"), - ); + let injected = false; + transaction.mockImplementation(async (callback, options) => { + const [delivery] = await db.select({ state: chatDeliveries.state }).from(chatDeliveries).where(and( + eq(chatDeliveries.endpointId, endpoint.id), + eq(chatDeliveries.providerEventId, "lifecycle:Ev-lifecycle-retry"), + )); + // Fail the lifecycle effect after durable admission and claim, regardless + // of how many transaction boundaries those writes require. + if (!injected && delivery?.state === "processing") { + injected = true; + throw new Error("injected lifecycle persistence failure"); + } + return originalTransaction(callback, options); + }); await expect( service.handleWebhook( endpoint.publicId, @@ -32754,6 +32780,158 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { return { ...fixture, storage, runtime, service, lanes, enqueue, cleanup }; } + it("restores chat queue deadlines without polling blocked followers or receipt owners", async () => { + const f = await publicationLaneFixture(1); + const { service, lanes, enqueue } = f; + const lane = lanes[0]!; + try { + const head = await enqueue(0, "Head"); + const tail = await enqueue(0, "Follower", 1); + const retryAt = new Date(Date.now() + 120_000); + await db.update(chatPublications).set({ state: "retry", nextAttemptAt: retryAt }).where(eq(chatPublications.id, head.id)); + expect(await service.nextPublicationAt()).toBe(retryAt.getTime()); + await db.update(chatPublications).set({ state: "delivery_unknown", nextAttemptAt: null }).where(eq(chatPublications.id, head.id)); + expect(await service.nextPublicationAt()).toBeNull(); + const leaseUntil = new Date(Date.now() + 180_000); + await db.update(chatPublications).set({ state: "streaming", updatedAt: new Date() }).where(eq(chatPublications.id, head.id)); + await db.insert(chatEndpointLeases).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + leaseKey: `publication:${head.id}:0`, token: randomUUID(), expiresAt: leaseUntil }); + expect(await service.nextPublicationAt()).toBe(leaseUntil.getTime()); + await db.insert(chatActions).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + kind: "slack_file_upload_receipt", providerActionId: randomUUID(), status: "received", + payload: { publicationId: head.id, publicationAttempt: 0 } }); + expect(await service.nextSlackReceiptAt()).toBeNull(); + await db.delete(chatEndpointLeases).where(eq(chatEndpointLeases.endpointId, lane.endpoint.id)); + await db.update(chatPublications).set({ updatedAt: new Date(Date.now() - 70_000) }).where(eq(chatPublications.id, head.id)); + const wake = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatReceipts, wake); + try { + await service.scheduleQueuedPublications(); + expect(wake).toHaveBeenCalled(); + expect(await service.nextSlackReceiptAt()).toBeLessThanOrEqual(Date.now()); + await service.processPendingSlackFileUploadReceipts(); // Invalid retained payload is closed, not retried. + expect(await service.nextSlackReceiptAt()).toBeNull(); + } finally { unsubscribe(); } + await db.update(chatPublications).set({ state: "cancelled" }).where(eq(chatPublications.id, tail.id)); + expect(await service.nextPublicationAt()).toBeNull(); + } finally { await f.cleanup(); } + }); + + it("restores inbound retry and abandoned wake deadlines and excludes paused queues", async () => { + const f = await publicationLaneFixture(1); + const { service, lanes } = f; + const lane = lanes[0]!; + try { + const retryAt = new Date(Date.now() + 120_000); + const [delivery] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + conversationId: lane.conversation.id, providerEventId: randomUUID(), deduplicationKey: randomUUID(), + eventKind: "message", normalizedEvent: { conversation: { externalThreadId: lane.conversation.externalThreadId } }, + receivedAt: new Date(Date.now() - 1000), state: "retry", nextAttemptAt: retryAt }).returning(); + const [follower] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + conversationId: lane.conversation.id, providerEventId: randomUUID(), deduplicationKey: randomUUID(), + eventKind: "message", normalizedEvent: delivery!.normalizedEvent, state: "received" }).returning(); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + const [edit] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "message_updated", state: "received", + receivedAt: new Date(Date.now() - 2000), normalizedEvent: { ...delivery!.normalizedEvent, + message: { targetProviderEventId: delivery!.providerEventId } } }).returning(); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + await db.update(chatDeliveries).set({ state: "filtered" }).where(eq(chatDeliveries.id, edit!.id)); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + await db.update(chatDeliveries).set({ state: "processed" }).where(eq(chatDeliveries.id, delivery!.id)); + const [action] = await db.insert(chatActions).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + deliveryId: delivery!.id, kind: "inbound_wakeup", providerActionId: randomUUID(), status: "issued", + result: { retryAt: retryAt.toISOString() } }).returning(); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + const claimedAt = new Date(); + await db.update(chatActions).set({ status: "processing", updatedAt: claimedAt }).where(eq(chatActions.id, action!.id)); + expect(await service.nextInboundDeliveryAt()).toBe(claimedAt.getTime() + 60_000); + await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, lane.endpoint.id)); + expect(await service.nextInboundDeliveryAt()).toBeNull(); + await db.update(chatActions).set({ status: "processed" }).where(eq(chatActions.id, action!.id)); + await db.update(chatDeliveries).set({ state: "filtered" }).where(eq(chatDeliveries.id, follower!.id)); + await db.update(chatEndpoints).set({ status: "active" }).where(eq(chatEndpoints.id, lane.endpoint.id)); + expect(await service.nextInboundDeliveryAt()).toBeNull(); + await db.update(chatEndpoints).set({ status: "attention" }).where(eq(chatEndpoints.id, lane.endpoint.id)); + await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "installation", state: "retry", + nextAttemptAt: retryAt, normalizedEvent: { lifecycle: { kind: "endpoint", provider: "slack", + providerEventId: randomUUID(), availability: "available", reason: "Recovered" } } }); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + } finally { await f.cleanup(); } + }); + + it("sleeps behind an unresolved Slack tool send and wakes when its receipt settles", async () => { + const f = await publicationLaneFixture(1); + const lane = f.lanes[0]!; + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + try { + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "succeeded" }).returning(); + const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: lane.conversation.issueId, + authorAgentId: f.assignedAgentId, createdByRunId: run!.id, body: "Already sent" }).returning(); + const publication = await f.enqueue(0, "Already sent"); + await db.update(chatPublications).set({ commentId: comment!.id }).where(eq(chatPublications.id, publication.id)); + const [send] = await db.insert(chatActions).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + conversationId: lane.conversation.id, kind: "slack_tool_write", providerActionId: randomUUID(), status: "processing", + payload: { name: "slack_post_message", binding: { runId: run!.id }, args: { + channel: lane.conversation.externalConversationId.replace(/^slack:/, ""), + thread_ts: lane.conversation.externalThreadId.split(":").at(-1), text: " Already sent\n" } } }).returning(); + await registerChatDeliveryWork(coordinator, f.service, () => true).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 }); + expect(await f.service.nextPublicationAt()).toBeNull(); + for (const status of ["uncertain", "processed"]) { + await db.transaction(async tx => { + await notifyChatPublicationWork(tx); + await tx.update(chatActions).set({ status }).where(eq(chatActions.id, send!.id)); + }); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 }); + } + await vi.waitFor(async () => { + const [row] = await db.select().from(chatPublications).where(eq(chatPublications.id, publication.id)); + expect(row!.state).toBe("cancelled"); + expect(coordinator.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + expect(lane.providerRuntime.posts).toHaveLength(0); + expect(onError).not.toHaveBeenCalled(); + } finally { await coordinator.stop(); await f.cleanup(); } + }); + + it("wakes chat dispatch after commit and refills free endpoint slots without the maintenance loop", async () => { + const f = await publicationLaneFixture(6); + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + f.lanes[0]!.providerRuntime.postHook = () => gate; + try { + await registerChatDeliveryWork(coordinator, f.service, () => true).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 }); + let settle!: () => void; + const commitGate = new Promise(resolve => { settle = resolve; }); + let staged!: () => void; + const stagedGate = new Promise(resolve => { staged = resolve; }); + const writing = db.transaction(async tx => { + await notifyChatPublicationWork(tx); + await tx.insert(chatPublications).values(f.lanes.map((lane, index) => ({ companyId: f.companyId, + endpointId: lane.endpoint.id, conversationId: lane.conversation.id, issueId: lane.conversation.issueId, + idempotencyKey: `work-test:${randomUUID()}`, payload: { text: `Queued ${index}` }, state: "pending" as const }))); + staged(); await commitGate; + }); + await stagedGate; + expect(f.lanes.every(lane => lane.providerRuntime.posts.length === 0)).toBe(true); + settle(); await writing; + await vi.waitFor(() => expect(f.lanes.slice(1).every(lane => lane.providerRuntime.posts.length === 1)).toBe(true), { timeout: 10_000 }); + release(); + await vi.waitFor(async () => { + const rows = await db.select().from(chatPublications).where(eq(chatPublications.companyId, f.companyId)); + expect(rows.every(row => row.state === "published")).toBe(true); + expect(coordinator.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + expect(onError).not.toHaveBeenCalled(); + } finally { release(); await coordinator.stop(); await f.cleanup(); } + }); + it("refills four tracked publication endpoint lanes after a scheduled budget and joins shutdown", async () => { const fixture = await publicationLaneFixture(6); const { service, lanes, enqueue, storage } = fixture; diff --git a/server/src/__tests__/chat-delivery-work.test.ts b/server/src/__tests__/chat-delivery-work.test.ts new file mode 100644 index 00000000000..fe2c9bc68a0 --- /dev/null +++ b/server/src/__tests__/chat-delivery-work.test.ts @@ -0,0 +1,112 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerChatDeliveryWork } from "../services/chat-delivery-work.js"; +import { notifyChatDeliveryWork, notifyChatPublicationWork } from "../services/chat-work-notifications.js"; + +const coordinators: ReturnType[] = []; +afterEach(async () => { + for (const coordinator of coordinators.splice(0)) await coordinator.stop(); + vi.useRealTimers(); +}); +function setup() { + vi.useFakeTimers(); + const execute = vi.fn(async () => [{ xid: "42", status: "committed" }]); + const owner = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise) { + return callback({ execute }); + } }); + const enabled = vi.fn(() => true), primary = vi.fn(() => true), onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner, + canRun: () => enabled() && primary(), canReconcile: primary, onError, + }); + coordinators.push(coordinator); + const service = { + processQueuedDeliveries: vi.fn(async () => {}), + scheduleQueuedPublications: vi.fn(async () => {}), + processPendingSlackFileUploadReceipts: vi.fn(async () => {}), + nextInboundDeliveryAt: vi.fn(async (): Promise => null), + nextPublicationAt: vi.fn(async (): Promise => null), + nextSlackReceiptAt: vi.fn(async (): Promise => null), + onPublicationsSettled: vi.fn(), + }; + const start = () => registerChatDeliveryWork(coordinator, service, enabled); + return { owner, service, coordinator, start, enabled, primary, onError }; +} + +describe("chat delivery work", () => { + it("does one startup pass and no further queue scans when empty", async () => { + const s = setup(); + await s.start().ready; + await vi.advanceTimersByTimeAsync(3_600_000); + for (const run of [s.service.processQueuedDeliveries, s.service.scheduleQueuedPublications, s.service.processPendingSlackFileUploadReceipts]) expect(run).toHaveBeenCalledTimes(1); + expect(s.coordinator.nextWakeAt()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + it("coalesces commits, wakes receipt recovery after publication changes, and refills freed slots", async () => { + const s = setup(); + await s.start().ready; + await s.owner.transaction(tx => notifyChatPublicationWork(tx)); + await s.owner.transaction(tx => notifyChatPublicationWork(tx)); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(2); + expect(s.service.processPendingSlackFileUploadReceipts).toHaveBeenCalledTimes(2); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(1); + s.service.onPublicationsSettled.mock.calls[0]![0](); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(3); + expect(vi.getTimerCount()).toBe(0); + }); + it("keeps lanes independent and retains commits received during a blocked drain", async () => { + const s = setup(); + let release!: () => void; + s.service.processQueuedDeliveries.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + const worker = s.start(); + await vi.advanceTimersByTimeAsync(1); + await s.owner.transaction(tx => notifyChatDeliveryWork(tx)); + await s.owner.transaction(tx => notifyChatPublicationWork(tx)); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(2); + release(); + await worker.ready; + await vi.advanceTimersByTimeAsync(1); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(2); + }); + it("restores future retries and stops when the last retry finishes", async () => { + const s = setup(); + const due = Date.now() + 60_000; + s.service.nextPublicationAt.mockResolvedValue(due); + await s.start().ready; + expect(s.coordinator.nextWakeAt()).toBe(due); + await vi.advanceTimersByTimeAsync(59_999); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(1); + s.service.nextPublicationAt.mockResolvedValue(null); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }); + it("does no queue SQL while standby or drain defers a first commit", async () => { + const s = setup(); + s.primary.mockReturnValue(false); + await s.start().ready; + await vi.advanceTimersByTimeAsync(2000); + s.primary.mockReturnValue(true); s.enabled.mockReturnValue(false); + await s.owner.transaction(tx => notifyChatDeliveryWork(tx)); + await vi.advanceTimersByTimeAsync(2000); + expect(s.service.processQueuedDeliveries).not.toHaveBeenCalled(); + expect(s.service.nextInboundDeliveryAt).not.toHaveBeenCalled(); + s.enabled.mockReturnValue(true); + await vi.advanceTimersByTimeAsync(1000); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }); + it("retries failed selection without losing its startup recovery", async () => { + const s = setup(); + s.service.nextSlackReceiptAt.mockRejectedValueOnce(new Error("connection lost")); + await s.start().ready; + expect(s.onError).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1000); + expect(s.service.processPendingSlackFileUploadReceipts).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/server/src/__tests__/chat-interaction-publications.test.ts b/server/src/__tests__/chat-interaction-publications.test.ts index 6c4c5d5f8ee..eab650d3927 100644 --- a/server/src/__tests__/chat-interaction-publications.test.ts +++ b/server/src/__tests__/chat-interaction-publications.test.ts @@ -20,7 +20,7 @@ import { toolConnections, } from "@paperclipai/db"; import type { ChatProvider } from "@paperclipai/shared"; -import { enqueueTerminalIssueInteractionChatPublications } from "../services/chat-interaction-publications.js"; +import { enqueueIssueInteractionChatPublications, enqueueTerminalIssueInteractionChatPublications } from "../services/chat-interaction-publications.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { getEmbeddedPostgresTestSupport, @@ -226,6 +226,7 @@ describeEmbeddedPostgres( wrapQuery(Reflect.apply(db.select, db, args))) as typeof db.select, insert: db.insert.bind(db), update: db.update.bind(db), + transaction: db.transaction.bind(db), }; } @@ -312,6 +313,35 @@ describeEmbeddedPostgres( ).toBe(true); }); + it("rolls back an interaction publication if its callback tokens cannot be stored", async () => { + const fixture = await seedBoundIssue(["telegram"]); + const interaction = await issueThreadInteractionService(db).create( + { id: fixture.issueId, companyId: fixture.companyId }, + { kind: "request_confirmation", continuationPolicy: "wake_assignee", payload: { version: 1, prompt: "Proceed?" } }, + { agentId: fixture.agentId }, + ); + expect(await publicationsForInteraction(fixture.companyId, interaction.id)).toHaveLength(1); + await db.delete(chatActions).where(eq(chatActions.companyId, fixture.companyId)); + await db.delete(chatPublications).where(eq(chatPublications.companyId, fixture.companyId)); + const failTokens = (insert: typeof db.insert): typeof db.insert => + ((table: Parameters[0]) => { + if (table === chatActions) throw new Error("Token write failed"); + return insert(table); + }) as typeof db.insert; + const failingDb: Parameters[0] = { + select: db.select.bind(db), + update: db.update.bind(db), + insert: failTokens(db.insert.bind(db)), + transaction: callback => db.transaction(async tx => { + tx.insert = failTokens(tx.insert.bind(tx)); + return callback(tx); + }), + }; + await expect(enqueueIssueInteractionChatPublications(failingDb, interaction)) + .rejects.toThrow("Token write failed"); + expect(await publicationsForInteraction(fixture.companyId, interaction.id)).toEqual([]); + }); + it("keeps unsupported governance interactions authoritative in Paperclip", async () => { const fixture = await seedBoundIssue(); const interaction = await issueThreadInteractionService(db).create( diff --git a/server/src/app.ts b/server/src/app.ts index c622f62e27c..041de40844a 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -9,6 +9,7 @@ import { customerSuccessRoutes } from "./routes/customer-success.js"; import { cloudWarmStandbyMiddleware } from "./middleware/cloud-warm-standby.js"; import type { CloudWarmStandby } from "./services/cloud-warm-standby.js"; import { browserUseRoutes } from "./routes/browser-use.js"; +import { registerChatDeliveryWork } from "./services/chat-delivery-work.js"; import { registerBrowserUseCleanup } from "./services/browser-use-work.js"; import { browserUseService } from "./services/browser-use.js"; import { slackToolRoutes } from "./routes/slack-tools.js"; @@ -319,7 +320,7 @@ export function createChatReconciliationCoordinator(input: { processFailedGitHubWebhookDeliveries?: () => Promise; projectRunMilestones: () => Promise; flushPublications: () => Promise; - processPendingSlackFileUploadReceipts: () => Promise; + processPendingSlackFileUploadReceipts?: () => Promise; processPendingSlackSessionSyncs: () => Promise; onError: (lane: ChatReconciliationLane, error: unknown) => void; }) { @@ -366,7 +367,7 @@ export function createChatReconciliationCoordinator(input: { } milestoneReconciliation.poll(); publicationReconciliation.poll(); - start("Slack file receipts", input.processPendingSlackFileUploadReceipts); + if (input.processPendingSlackFileUploadReceipts) start("Slack file receipts", input.processPendingSlackFileUploadReceipts); start("Slack session status", input.processPendingSlackSessionSyncs); }, notifyPublications() { @@ -1232,25 +1233,24 @@ export async function createApp( hasPending: () => opts.feedbackExportService!.hasPendingFeedbackTraces(), }); } + registerChatDeliveryWork(deliveryWork, chatChannels, () => !isIdleTaskDrainActive()); emailChannels.start(); - const flushChatPublications = async () => { - await chatChannels.schedulePendingPublications(); + const reconcileChatPublicationMaintenance = async () => { + await chatChannels.processPublicationMaintenance(); }; const chatReconciliation = createChatReconciliationCoordinator({ reconcileProviderRuntimes: async () => { await chatChannels.reconcileProviderRuntimes(); await chatChannels.voice.reconcile(); }, - processPendingDeliveries: () => chatChannels.processPendingDeliveries(), + processPendingDeliveries: () => chatChannels.processPendingChatMaintenance(), processFailedGitHubWebhookDeliveries: () => chatChannels.processFailedGitHubWebhookDeliveries(), projectRunMilestones: () => enqueueChatRunMilestones(db, { publicBaseUrl: opts.authPublicBaseUrl, }), - flushPublications: () => flushChatPublications(), - processPendingSlackFileUploadReceipts: () => - chatChannels.processPendingSlackFileUploadReceipts(), + flushPublications: () => reconcileChatPublicationMaintenance(), processPendingSlackSessionSyncs: () => chatChannels.processPendingSlackSessionSyncs(), onError: (lane, err) => { @@ -1263,6 +1263,8 @@ export async function createApp( chatReconciliation.notifyPublications(); }, ); + // Provider/action maintenance and milestone projection still use this cadence. + // Inbox, outbox, and file receipt queues have independent commit/deadline wakes. let chatPublicationTimer: ReturnType | null = setInterval( () => { if (!isWarmStandby() && !isIdleTaskDrainActive()) chatReconciliation.reconcile(); diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index 51c4b86af2f..59d0d9f4e4a 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -1,3 +1,4 @@ +import { notifyChatDeliveryWork, notifyChatPublicationWork, notifyChatEndpointWork } from "./chat-work-notifications.js"; import { syncSpekoVoiceTools } from "./voice/speko-agent-tools.js"; import { configureSpekoSessionTools } from "./voice/speko-tool-setup.js"; import { voiceSessionService } from "./voice/voice-session-service.js"; @@ -22,7 +23,7 @@ import { withSlackBoardLease } from "./slack-board-lease.js"; import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js"; import { authorizeSlackBoardPublication } from "./slack-board-authority.js"; import { assertSlackBoardWorkAllowed } from "./slack-board-resume.js"; -import { slackExplicitPublicationDuplicate } from "./connectors/slack-publication.js"; +import { slackExplicitPublicationDuplicate, unresolvedSlackPublicationCondition } from "./connectors/slack-publication.js"; import { rememberVerifiedSlackSearchEvent, slackSearchActionToken } from "./connectors/slack-search-context.js"; import { slackAuthorizationRevision } from "./connectors/slack-revision.js"; import { slackPublicationAllowed } from "./connectors/slack-access.js"; @@ -4632,6 +4633,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { principalId: string; }, ) { + await notifyChatPublicationWork(tx); const [inserted] = await tx .insert(chatPublications) .values({ @@ -4749,6 +4751,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatActions.id }); if (!ownedAction) return false; if (input.payload.settleDelivery && input.action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -4852,6 +4855,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .returning({ id: chatActions.id }); if (quarantined && payload?.settleDelivery && action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -5116,6 +5120,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatActions.id }); if (!cancelled) return false; if (action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -5402,6 +5407,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } if (payload.settleDelivery && action!.deliveryId) { + await notifyChatDeliveryWork(tx); const [settled] = await tx .update(chatDeliveries) .set({ @@ -5750,6 +5756,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (current.status === "attention" && !gatewayOwnedAttention) return; const status = current.setup.step === "complete" ? "active" : "verifying"; + if (status !== current.status) await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -7850,6 +7857,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (!current) return; const connection = await tx.select().from(toolConnections).where(eq(toolConnections.id, current.connectionId)).then((rows) => rows[0]); if (!connection?.enabled) throw new Error("Photon endpoint requires operator recovery"); + await notifyChatEndpointWork(tx); await tx.update(chatEndpoints).set({ status: current.setup.step === "complete" ? "active" : "verifying", healthMessage: "Photon receiver connected", lastError: null, updatedAt: new Date() }).where(eq(chatEndpoints.id, endpoint.id)); }); }, @@ -8473,6 +8481,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { status: "processed", }) .onConflictDoNothing(); + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -9061,6 +9070,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_endpoint_not_active", }); } + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9076,6 +9086,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .update(toolConnections) .set({ status: "disabled", enabled: false, updatedAt: new Date() }) .where(eq(toolConnections.id, endpoint.connectionId)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -9178,6 +9189,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // Fail closed for any legacy or racing delivery that remained open // while this endpoint was paused. Resume must never execute traffic // that Paperclip acknowledged during the inactive interval. + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -9194,6 +9206,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { inArray(chatDeliveries.state, ["received", "retry"]), ), ); + await notifyChatEndpointWork(tx); [activatedEndpoint] = await tx .update(chatEndpoints) .set({ @@ -9240,6 +9253,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const failedAt = new Date(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9267,6 +9281,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { updatedAt: failedAt, }) .where(eq(toolConnections.id, endpoint.connectionId)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -9341,6 +9356,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_endpoint_already_removed", }); } + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9574,6 +9590,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { try { const claimed = await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointWork(tx); const rows = await tx .update(chatEndpoints) .set({ @@ -9657,6 +9674,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const preserveSlackVerification = current.setup.slackSetupMethod === "automatic" && (current.setup as InternalSetupState).slackVerificationSigningFingerprint === createHash("sha256").update(credentials.signingSecret ?? "").digest("hex") && Boolean(observedSlackUrl && slackCallbackMatchesPublicUrl(observedSlackUrl, `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/slack`)); + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9848,6 +9866,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { await invalidateRuntime(endpoint.id).catch(() => undefined); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -10130,6 +10149,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { )).limit(1).then((rows) => rows[0]); if (!linked || !(await lockCurrentPrincipalAuthorization(tx, endpoint, linked.principalId)).allowed) throw forbidden("Your connected account is no longer authorized"); } + await notifyChatEndpointWork(tx); const [activated] = await tx .update(chatEndpoints) .set({ @@ -11222,6 +11242,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }, runtimeContext: effectContext, }); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -11414,6 +11435,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { message: { providerMessageId: event.messageId }, filtering: { contentRetained: false, providerThreadCreated: false }, }; + await notifyChatDeliveryWork(tx); await tx .insert(chatDeliveries) .values({ @@ -11526,6 +11548,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ["verifying", "active"], ); if (!current) return false; + await notifyChatDeliveryWork(tx); const [row] = await tx .update(chatDeliveries) .set({ @@ -13542,6 +13565,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { current.status === "deferred_issue_execution" ? "queued" : "not_started"; + await notifyChatPublicationWork(tx); return ( await tx .insert(chatPublications) @@ -13981,6 +14005,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return 0; if (prior && !(await inboundQueueNoticeStillVisible(tx, prior))) return 0; + await notifyChatPublicationWork(tx); const rows = await tx .insert(chatPublications) .values({ @@ -14146,6 +14171,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { existingReceipt, ); } + await notifyChatDeliveryWork(tx); const accepted = await tx .update(chatDeliveries) .set({ @@ -14168,6 +14194,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // An operator can repair a failed delivery ledger after its wake was // already committed. Keep the immutable receipt and never admit twice. if (action.status !== "preparing") return; + await notifyChatDeliveryWork(tx); const issued = await tx .update(chatActions) .set({ @@ -14260,24 +14287,28 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { Date.parse(candidate.result.retryAt) > now.getTime()) ) return false; - const claimed = await db - .update(chatActions) - .set({ status: "processing", updatedAt: now }) - .where( - and( - eq(chatActions.id, candidate.id), - eq(chatActions.status, candidate.status), - eq(chatActions.updatedAt, candidate.updatedAt), - ), - ) - .returning() - .then((rows) => rows[0] ?? null); + const claimed = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatActions) + .set({ status: "processing", updatedAt: now }) + .where( + and( + eq(chatActions.id, candidate.id), + eq(chatActions.status, candidate.status), + eq(chatActions.updatedAt, candidate.updatedAt), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); if (!claimed) return false; const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1; const receiptDeclined = (row: typeof agentWakeupRequests.$inferSelect) => ["skipped", "cancelled", "failed"].includes(row.status); const settle = async (status: string, result: Record) => { const changed = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); const changed = await tx .update(chatActions) .set({ status, result, updatedAt: new Date() }) @@ -15008,6 +15039,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((rows) => rows[0] ?? null) : null; if (!admittedDeliveryId) { + await notifyChatDeliveryWork(tx); const [delivery] = await tx .insert(chatDeliveries) .values({ @@ -15072,6 +15104,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { existingDelivery.state, ) && deliveryContentWasRedacted(existingDelivery.normalizedEvent); + await notifyChatDeliveryWork(tx); [candidate] = await tx .update(chatDeliveries) .set({ @@ -15148,6 +15181,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { !accepting && ["received", "retry", "processing"].includes(candidate.state) ) { + await notifyChatDeliveryWork(tx); [candidate] = await tx .update(chatDeliveries) .set({ @@ -15237,23 +15271,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // The adapter invokes the ordinary message callback only after Discord // has returned the created thread. Promote the pre-admission receipt // atomically so the normal ordered drain can begin immediately. - const [promoted] = await db - .update(chatDeliveries) - .set({ - state: "received", - normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, - nextAttemptAt: scheduledAt ? new Date(scheduledAt) : null, - redactedError: null, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, candidate.id), - eq(chatDeliveries.state, "retry"), - sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, - ), - ) - .returning(); + const candidateId = candidate.id; + const [promoted] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "received", + normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, + nextAttemptAt: scheduledAt ? new Date(scheduledAt) : null, + redactedError: null, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, candidateId), + eq(chatDeliveries.state, "retry"), + sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, + ), + ) + .returning(); + }); if (promoted) candidate = promoted; } const now = new Date(); @@ -15298,17 +15336,20 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ]; if (candidate.state === "processing") claimConditions.push(lte(chatDeliveries.updatedAt, staleBefore)); - const claimed = await db - .update(chatDeliveries) - .set({ - state: "processing", - attempts: candidate.attempts + 1, - nextAttemptAt: null, - redactedError: null, - updatedAt: now, - }) - .where(and(...claimConditions)) - .returning(); + const claimed = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "processing", + attempts: candidate.attempts + 1, + nextAttemptAt: null, + redactedError: null, + updatedAt: now, + }) + .where(and(...claimConditions)) + .returning(); + }); const activeDelivery = claimed[0]; if (!activeDelivery) return; @@ -15317,29 +15358,32 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const resolutionAt = new Date(); const waitForSetupRoot = activeDelivery.attempts <= ORPHAN_FOLLOW_UP_MAX_ATTEMPTS; - await db - .update(chatDeliveries) - .set( - waitForSetupRoot - ? { - state: "retry", - nextAttemptAt: new Date( - resolutionAt.getTime() + ORPHAN_FOLLOW_UP_GRACE_MS, - ), - redactedError: "Waiting briefly for an earlier root mention", - updatedAt: resolutionAt, - } - : { - state: "filtered", - normalizedEvent: redactedDestinationNormalized, - principalId: null, - nextAttemptAt: null, - processedAt: resolutionAt, - redactedError: "Destination is not enabled in Paperclip", - updatedAt: resolutionAt, - }, - ) - .where(eq(chatDeliveries.id, activeDelivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set( + waitForSetupRoot + ? { + state: "retry", + nextAttemptAt: new Date( + resolutionAt.getTime() + ORPHAN_FOLLOW_UP_GRACE_MS, + ), + redactedError: "Waiting briefly for an earlier root mention", + updatedAt: resolutionAt, + } + : { + state: "filtered", + normalizedEvent: redactedDestinationNormalized, + principalId: null, + nextAttemptAt: null, + processedAt: resolutionAt, + redactedError: "Destination is not enabled in Paperclip", + updatedAt: resolutionAt, + }, + ) + .where(eq(chatDeliveries.id, activeDelivery.id)); + }); return; } // A committed inbound message link proves that a prior attempt completed @@ -15693,20 +15737,23 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // the durable thread drain can sort again if a delayed root arrives. // A standalone unaddressed message is filtered after the bounded // retention window. - await db - .update(chatDeliveries) - .set({ - state: "retry", - nextAttemptAt: new Date(Date.now() + ORPHAN_FOLLOW_UP_GRACE_MS), - redactedError: "Waiting briefly for an earlier root mention", - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, activeDelivery.id), - eq(chatDeliveries.state, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "retry", + nextAttemptAt: new Date(Date.now() + ORPHAN_FOLLOW_UP_GRACE_MS), + redactedError: "Waiting briefly for an earlier root mention", + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, activeDelivery.id), + eq(chatDeliveries.state, "processing"), + ), + ); + }); return; } if (!allowed) { @@ -15721,18 +15768,21 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ? "Endpoint sponsor can no longer authorize external guests" : "External identity must be linked to a Paperclip account" : "Message did not address the agent or an active task thread"; - await db - .update(chatDeliveries) - .set({ - state: "filtered", - normalizedEvent: redactedDestinationNormalized, - principalId: null, - nextAttemptAt: null, - redactedError: filteredReason, - processedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, activeDelivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + normalizedEvent: redactedDestinationNormalized, + principalId: null, + nextAttemptAt: null, + redactedError: filteredReason, + processedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, activeDelivery.id)); + }); return; } @@ -15790,17 +15840,20 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .limit(1) .then((rows) => rows[0]); if (!source || source.conversationId !== existingConversation?.id) { - await db - .update(chatDeliveries) - .set({ - state: "filtered", - processedAt: new Date(), - nextAttemptAt: null, - redactedError: - "Quoted control does not belong to the current task generation", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, activeDelivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + processedAt: new Date(), + nextAttemptAt: null, + redactedError: + "Quoted control does not belong to the current task generation", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, activeDelivery.id)); + }); return; } } @@ -15836,6 +15889,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .update(chatEndpoints) .set({ lastEventAt: new Date(), updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -15944,6 +15998,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .set({ lastEventAt: new Date(), updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); if (await filterPreControlSource(tx)) return false; + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -16403,6 +16458,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { !currentPrincipalAuthorization.allowed ) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -16567,25 +16623,28 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { providerEffectAmbiguous || isExternalActionAuthorizationChange(error) || activeDelivery.attempts >= 5; - await db - .update(chatDeliveries) - .set({ - state: terminal ? "failed" : "retry", - nextAttemptAt: terminal - ? null - : new Date( - Date.now() + - Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), - ), - redactedError: redactError(error), - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, activeDelivery.id), - eq(chatDeliveries.state, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: terminal ? "failed" : "retry", + nextAttemptAt: terminal + ? null + : new Date( + Date.now() + + Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), + ), + redactedError: redactError(error), + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, activeDelivery.id), + eq(chatDeliveries.state, "processing"), + ), + ); + }); if (terminal) await settleRejectedInboundWakeups(activeDelivery.id); throw error; } @@ -16704,6 +16763,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ["verifying", "active"], ); if (!current) return false; + await notifyChatDeliveryWork(tx); const [promoted] = await tx .update(chatDeliveries) .set({ @@ -16789,6 +16849,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -16815,7 +16876,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } } - function normalizedDeliveryThreadId(delivery: DeliveryRow): string | null { + function normalizedDeliveryThreadId(delivery: Pick): string | null { const normalized = delivery.normalizedEvent as { conversation?: { externalThreadId?: unknown }; }; @@ -16825,7 +16886,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } function normalizedLifecycleTargetEventId( - delivery: DeliveryRow, + delivery: Pick, ): string | null { if ( delivery.eventKind !== "message_updated" && @@ -16846,7 +16907,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } function normalizedLifecycleEffect( - delivery: DeliveryRow, + delivery: Pick, ): ChatProviderLifecycleEffect | null { const normalized = delivery.normalizedEvent as { lifecycle?: unknown }; const value = normalized.lifecycle; @@ -16901,6 +16962,48 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } + function conversationDeliveryOrder() { + return [ + asc( + sql`coalesce(nullif(${chatDeliveries.normalizedEvent}->'message'->>'providerSentAt', '')::timestamptz, ${chatDeliveries.receivedAt})`, + ), + // GitHub timestamps and Telegram message dates have one-second + // resolution. GitHub's numeric comment id and Telegram's message_id + // are monotonic within one conversation, so use them before receipt + // order. The Telegram update_id is a final provider-native tie-breaker + // for unusual payloads that lack a usable message_id. + asc(sql`coalesce( + case + when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence' ~ '^[0-9]+$' + then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence')::numeric + else null + end, + case + when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' ~ '^[0-9]+$' + then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId')::numeric + else null + end + )`), + asc(sql`case + when ${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId' ~ '^[0-9]+$' + then (${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId')::numeric + else null + end`), + // GitHub exposes no sortable webhook sequence. When an edit and delete + // share its whole-second updated_at value, preserve the only valid + // lifecycle state transition: update before delete. Provider-native + // update ids (Telegram) remain the stronger preceding key. + asc(sql`case ${chatDeliveries.eventKind} + when 'message_updated' then 1 + when 'message_deleted' then 2 + when 'message_restored' then 3 + else 0 + end`), + asc(chatDeliveries.receivedAt), + asc(chatDeliveries.id), + ]; + } + async function earliestOpenConversationDelivery( endpointId: string, threadId: string, @@ -16926,45 +17029,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { externalThreadIdentityCondition(externalThreadId, threadId), ), ) - .orderBy( - asc( - sql`coalesce(nullif(${chatDeliveries.normalizedEvent}->'message'->>'providerSentAt', '')::timestamptz, ${chatDeliveries.receivedAt})`, - ), - // GitHub timestamps and Telegram message dates have one-second - // resolution. GitHub's numeric comment id and Telegram's message_id - // are monotonic within one conversation, so use them before receipt - // order. The Telegram update_id is a final provider-native tie-breaker - // for unusual payloads that lack a usable message_id. - asc(sql`coalesce( - case - when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence' ~ '^[0-9]+$' - then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence')::numeric - else null - end, - case - when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' ~ '^[0-9]+$' - then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId')::numeric - else null - end - )`), - asc(sql`case - when ${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId' ~ '^[0-9]+$' - then (${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId')::numeric - else null - end`), - // GitHub exposes no sortable webhook sequence. When an edit and delete - // share its whole-second updated_at value, preserve the only valid - // lifecycle state transition: update before delete. Provider-native - // update ids (Telegram) remain the stronger preceding key. - asc(sql`case ${chatDeliveries.eventKind} - when 'message_updated' then 1 - when 'message_deleted' then 2 - when 'message_restored' then 3 - else 0 - end`), - asc(chatDeliveries.receivedAt), - asc(chatDeliveries.id), - ) + .orderBy(...conversationDeliveryOrder()) .limit(1) .then((rows) => rows[0] ?? null); if (!candidate) return null; @@ -17376,6 +17441,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { !telegramDeliveryHasZeroMessageId(current, endpoint.provider) ) return false; + await notifyChatDeliveryWork(tx); const changed = await tx .update(chatDeliveries) .set({ @@ -17403,15 +17469,18 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { continue; } if (endpoint.status === "archived" || endpoint.status === "revoked") { - await db - .update(chatDeliveries) - .set({ - state: "filtered", - redactedError: "Connection was removed before processing", - processedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + redactedError: "Connection was removed before processing", + processedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); liveInboundMessages.delete(delivery.id); continue; } @@ -17457,27 +17526,30 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } else { if (deliveryContentWasRedacted(delivery.normalizedEvent)) { const filteredAt = new Date(); - await db - .update(chatDeliveries) - .set({ - state: "filtered", - principalId: null, - nextAttemptAt: null, - processedAt: filteredAt, - redactedError: - "Provisional Teams setup reply could not be hydrated from a current provider event", - updatedAt: filteredAt, - }) - .where( - and( - eq(chatDeliveries.id, delivery.id), - inArray(chatDeliveries.state, [ - "received", - "retry", - "processing", - ]), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + principalId: null, + nextAttemptAt: null, + processedAt: filteredAt, + redactedError: + "Provisional Teams setup reply could not be hydrated from a current provider event", + updatedAt: filteredAt, + }) + .where( + and( + eq(chatDeliveries.id, delivery.id), + inArray(chatDeliveries.state, [ + "received", + "retry", + "processing", + ]), + ), + ); + }); continue; } const endpointRuntime = await runtimeFor(endpoint); @@ -17492,14 +17564,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { endpointRuntime, ); if (!reconstructed) { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Normalized delivery is incomplete", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Normalized delivery is incomplete", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); continue; } await processMessage( @@ -17728,6 +17803,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { conversationDrainKey(input.endpointId, input.threadId), ) ?? Date.now() + reorderWindow) : null; + await notifyChatDeliveryWork(tx); const [delivery] = await tx .insert(chatDeliveries) .values({ @@ -17886,14 +17962,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ): Promise { const lifecycle = lifecycleMessageFromDelivery(candidate); if (!lifecycle) { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Normalized lifecycle delivery is incomplete", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, candidate.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Normalized lifecycle delivery is incomplete", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, candidate.id)); + }); return; } @@ -17905,17 +17984,20 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ]; if (candidate.state === "processing") claimConditions.push(lte(chatDeliveries.updatedAt, staleBefore)); - const [activeDelivery] = await db - .update(chatDeliveries) - .set({ - state: "processing", - attempts: candidate.attempts + 1, - nextAttemptAt: null, - redactedError: null, - updatedAt: now, - }) - .where(and(...claimConditions)) - .returning(); + const [activeDelivery] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "processing", + attempts: candidate.attempts + 1, + nextAttemptAt: null, + redactedError: null, + updatedAt: now, + }) + .where(and(...claimConditions)) + .returning(); + }); if (!activeDelivery) return; try { @@ -17931,6 +18013,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null; if (!admittedRuntimeContext || !currentEndpoint) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -17989,6 +18072,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { linkedMessages.some((row) => row.direction === "inbound"); if (outbound || (lifecycle.isBotMessage && !githubBotSourceEdit)) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18093,6 +18177,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { editPredatesRestore ) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18182,6 +18267,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { originalIsOpen || activeDelivery.attempts <= ORPHAN_FOLLOW_UP_MAX_ATTEMPTS; const resolutionAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set( @@ -18261,6 +18347,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // Provider authentication proves the exact source changed. The // editor's current Paperclip rights govern admitting new content, // not whether a later regrant can resurrect the stale old source. + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18369,6 +18456,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) return; const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18420,6 +18508,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { (activeDelivery.eventKind === "message_deleted" || (activeDelivery.eventKind === "message_updated" && lifecyclePrincipalId !== null)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18479,6 +18568,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { { authorType: "system" }, tx, ); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18498,25 +18588,28 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } catch (error) { const terminal = activeDelivery.attempts >= 5; - await db - .update(chatDeliveries) - .set({ - state: terminal ? "failed" : "retry", - nextAttemptAt: terminal - ? null - : new Date( - Date.now() + - Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), - ), - redactedError: redactError(error), - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, activeDelivery.id), - eq(chatDeliveries.state, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: terminal ? "failed" : "retry", + nextAttemptAt: terminal + ? null + : new Date( + Date.now() + + Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), + ), + redactedError: redactError(error), + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, activeDelivery.id), + eq(chatDeliveries.state, "processing"), + ), + ); + }); throw error; } } @@ -18966,6 +19059,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .limit(1) .then((rows) => rows.length > 0); + await notifyChatDeliveryWork(tx); const reconciled = hasOutboundLink ? await tx .update(chatDeliveries) @@ -18992,6 +19086,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatDeliveries.id }) : []; if (reconciled.length > 0) return true; + await notifyChatDeliveryWork(tx); const expired = await tx .update(chatDeliveries) .set({ @@ -19018,6 +19113,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatDeliveries.id }); if (expired.length > 0) return false; } + await notifyChatDeliveryWork(tx); const inserted = await tx .insert(chatDeliveries) .values({ @@ -19141,6 +19237,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { safelyKnown.principalId ?? null, ) : null; + await notifyChatDeliveryWork(tx); const [inserted] = await tx .insert(chatDeliveries) .values({ @@ -19268,6 +19365,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { let invalidateEndpointRuntime = false; try { effect = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); const [inserted] = await tx .insert(chatDeliveries) .values({ @@ -19729,6 +19827,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ), ); + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -20460,6 +20559,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((rows) => rows[0] ?? null); if (occupied && occupied.publicationId !== current.id) return null; const reconciledAt = new Date(); + await notifyChatPublicationWork(tx); const [reconciled] = await tx .update(chatPublications) .set({ @@ -21181,6 +21281,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { answeredQuestionCount: 1, }, }); + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -23697,31 +23798,34 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .digest("hex"); const eventKind: ChatEventKind = effect.availability === "available" ? "installation" : "uninstallation"; - const [inserted] = await db - .insert(chatDeliveries) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - providerEventId, - deduplicationKey, - eventKind, - normalizedEvent: { + const [inserted] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .insert(chatDeliveries) + .values({ + companyId: endpoint.companyId, + endpointId: endpoint.id, providerEventId, - kind: eventKind, - lifecycle: effect, - ...(runtimeContext - ? { - runtimeContext: { - credentialFingerprint: runtimeContext.credentialFingerprint, - generation: runtimeContext.generation, - }, - } - : {}), - }, - state: "received", - }) - .onConflictDoNothing() - .returning(); + deduplicationKey, + eventKind, + normalizedEvent: { + providerEventId, + kind: eventKind, + lifecycle: effect, + ...(runtimeContext + ? { + runtimeContext: { + credentialFingerprint: runtimeContext.credentialFingerprint, + generation: runtimeContext.generation, + }, + } + : {}), + }, + state: "received", + }) + .onConflictDoNothing() + .returning(); + }); const candidate = inserted ?? (await db @@ -23736,28 +23840,31 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((rows) => rows[0] ?? null)); if (!candidate || candidate.state === "processed") return false; const staleBefore = new Date(Date.now() - DELIVERY_PROCESSING_STALE_MS); - const [claimed] = await db - .update(chatDeliveries) - .set({ - state: "processing", - attempts: candidate.attempts + 1, - redactedError: null, - nextAttemptAt: null, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, candidate.id), - or( - inArray(chatDeliveries.state, ["received", "retry"]), - and( - eq(chatDeliveries.state, "processing"), - lte(chatDeliveries.updatedAt, staleBefore), + const [claimed] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "processing", + attempts: candidate.attempts + 1, + redactedError: null, + nextAttemptAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, candidate.id), + or( + inArray(chatDeliveries.state, ["received", "retry"]), + and( + eq(chatDeliveries.state, "processing"), + lte(chatDeliveries.updatedAt, staleBefore), + ), ), ), - ), - ) - .returning({ id: chatDeliveries.id }); + ) + .returning({ id: chatDeliveries.id }); + }); if (!claimed) return false; let refreshRuntimeAfterLifecycle = false; @@ -23781,6 +23888,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const filteredAt = new Date(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -23839,6 +23947,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); const revokedAt = new Date(); if (!current) { + await notifyChatDeliveryWork(tx); const [filtered] = await tx .update(chatDeliveries) .set({ @@ -23914,6 +24023,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { inArray(chatConversations.state, ["active", "waiting"]), ), ); + await notifyChatDeliveryWork(tx); const [processed] = await tx .update(chatDeliveries) .set({ @@ -24075,6 +24185,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { eq(toolConnections.id, canonical.endpoint.connectionId), ); } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24170,6 +24281,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { inArray(chatConversations.state, ["active", "waiting"]), ), ); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24221,6 +24333,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .update(chatEndpoints) .set({ lastEventAt: now, updatedAt: now }) .where(eq(chatEndpoints.id, currentEndpoint.id)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24413,6 +24526,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : currentEndpoint.status === "attention" ? "verifying" : currentEndpoint.status; + if (status !== currentEndpoint.status) await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -24492,6 +24606,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24523,22 +24638,25 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } if (!lifecycleFailurePersisted) { const terminal = candidate.attempts + 1 >= 5; - await db - .update(chatDeliveries) - .set({ - state: terminal ? "failed" : "retry", - redactedError: redactError(error), - nextAttemptAt: terminal ? null : new Date(), - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, claimed.id), - eq(chatDeliveries.state, "processing"), - eq(chatDeliveries.attempts, candidate.attempts + 1), - ), - ) - .catch(() => undefined); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: terminal ? "failed" : "retry", + redactedError: redactError(error), + nextAttemptAt: terminal ? null : new Date(), + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, claimed.id), + eq(chatDeliveries.state, "processing"), + eq(chatDeliveries.attempts, candidate.attempts + 1), + ), + ) + .catch(() => undefined); + }); } throw error; } @@ -25014,6 +25132,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : action === "deleted" ? "message_deleted" : "message"; + await notifyChatDeliveryWork(tx); await tx .insert(chatDeliveries) .values({ @@ -27625,21 +27744,24 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const reaction = normalizedPendingReaction(candidate); if (!reaction || !candidate.principalId) { const now = new Date(); - await db - .update(chatDeliveries) - .set({ - state: "failed", - nextAttemptAt: null, - processedAt: now, - redactedError: "Normalized reaction delivery is incomplete", - updatedAt: now, - }) - .where( - and( - eq(chatDeliveries.id, candidate.id), - inArray(chatDeliveries.state, ["received", "retry"]), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + nextAttemptAt: null, + processedAt: now, + redactedError: "Normalized reaction delivery is incomplete", + updatedAt: now, + }) + .where( + and( + eq(chatDeliveries.id, candidate.id), + inArray(chatDeliveries.state, ["received", "retry"]), + ), + ); + }); return; } const principalId = candidate.principalId; @@ -27766,6 +27888,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { 2 ** Math.min(Math.max(locked.attempts, 0), 5), REACTION_LINK_MAX_DELAY_MS, ); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -27796,6 +27919,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const processedAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set( @@ -27903,6 +28027,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { async function cancelWork(message: string) { await lease.commit(async (tx) => { await tx.update(chatActions).set({ status: "cancelled", result: { code: "slack_board_work_not_authorized", message }, updatedAt: new Date() }).where(eq(chatActions.id, action.id)); + await notifyChatPublicationWork(tx); await tx.update(chatPublications).set({ state: "cancelled", redactedError: message, nextAttemptAt: null, updatedAt: new Date() }).where(and( eq(chatPublications.companyId, action.companyId), eq(chatPublications.endpointId, action.endpointId), eq(chatPublications.conversationId, action.conversationId!), eq(chatPublications.commentId, commentId), @@ -27948,6 +28073,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { contextSnapshot: { issueId, taskId: issueId, taskKey: issue.identifier ?? issueId, wakeCommentId: commentId, source: "issue.comment", resumeIntent: true, followUpRequested: true }, }); await lease.commit(async (tx) => { + await notifyChatPublicationWork(tx); await tx.update(chatActions).set({ status: "processed", updatedAt: new Date() }).where(eq(chatActions.id, action.id)); }); } catch (error) { @@ -27961,28 +28087,30 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } } - async function processPendingDeliveries(limit = 25, onlyDeliveryId?: string) { + async function processPendingChatMaintenance(limit = 25) { + return Promise.allSettled([ + processPendingSlackBoardMessages(limit), + processPendingGitHubWebhookIngress(limit), + processPendingProviderEffects(limit), + slackRegistration.processPendingVerificationMessages(limit), + processPendingReceiptReactions(limit), + processPendingSlackSessionStops(limit), + processPendingTelegramMaintenance(limit), + // Slack task starts are an action-backed outbox. Queued work may + // post once; provider-confirmed rows perform Paperclip-only + // admission; a stale in-flight post is quarantined as unknown. + processPendingSlackTaskStarts(limit), + processFailedChatRunRetries(limit), + ]); + } + + async function processPendingDeliveries(limit = 25, onlyDeliveryId?: string, maintenance = true) { await settleRejectedInboundWakeups(onlyDeliveryId); // Provider-visible effects that are not backed by a task publication use // chat_actions as their outbox. Reconcile them before inbound deliveries // so a crashed processing claim is quarantined before the delivery worker // could otherwise replay it. - const actionRecovery = onlyDeliveryId - ? null - : Promise.allSettled([ - processPendingSlackBoardMessages(limit), - processPendingGitHubWebhookIngress(limit), - processPendingProviderEffects(limit), - slackRegistration.processPendingVerificationMessages(limit), - processPendingReceiptReactions(limit), - processPendingSlackSessionStops(limit), - processPendingTelegramMaintenance(limit), - // Slack task starts are an action-backed outbox. Queued work may - // post once; provider-confirmed rows perform Paperclip-only - // admission; a stale in-flight post is quarantined as unknown. - processPendingSlackTaskStarts(limit), - processFailedChatRunRetries(limit), - ]); + const actionRecovery = onlyDeliveryId || !maintenance ? null : processPendingChatMaintenance(limit); const reactionRecovery = processPendingReactionDeliveries( limit, onlyDeliveryId, @@ -28046,14 +28174,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (lifecycleEffect) { const record = await endpointRecord(delivery.endpointId); if (!record || record.endpoint.status === "archived") { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Chat endpoint is no longer available", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Chat endpoint is no longer available", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); continue; } try { @@ -28075,14 +28206,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const externalThreadId = normalizedDeliveryThreadId(delivery); if (!externalThreadId) { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Normalized delivery is incomplete", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Normalized delivery is incomplete", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); continue; } const key = conversationDrainKey(delivery.endpointId, externalThreadId); @@ -28854,6 +28988,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const current = await runtimeCallbackEndpoint(tx, endpoint.id, runtimeContext, ["verifying", "active"]); if (!current) return; const providerEventId = `github:policy_ingress:${createHash("sha256").update(deliveryId).digest("hex")}`; + await notifyChatDeliveryWork(tx); await tx.insert(chatDeliveries).values({ companyId: current.companyId, endpointId: current.id, providerEventId, deduplicationKey: providerEventId, eventKind: "message", state: "filtered", processedAt: new Date(), @@ -29357,6 +29492,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); const claimed = await tx .update(chatDeliveries) .set({ @@ -29496,6 +29632,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_file_transfer_resolution_required", }, ); + await notifyChatPublicationWork(tx); return tx .update(chatPublications) .set({ @@ -29701,6 +29838,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .onConflictDoNothing(); } const now = new Date(); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set( @@ -29941,6 +30079,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }, ); } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -30096,6 +30235,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } if (payload.settleDelivery && action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set( @@ -30747,24 +30887,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .then((rows) => rows[0] ?? null); if (!comment) throw notFound("Task comment not found"); - const [publication] = await db - .insert(chatPublications) - .values({ - companyId: conversation.companyId, - endpointId, - conversationId, - issueId: conversation.issueId, - commentId, - idempotencyKey: `explicit:${commentId}:${endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "explicit_board_send", - text: comment.body, - }), - state: "pending", - }) - .onConflictDoNothing() - .returning(); + const [publication] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: conversation.companyId, + endpointId, + conversationId, + issueId: conversation.issueId, + commentId, + idempotencyKey: `explicit:${commentId}:${endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "explicit_board_send", + text: comment.body, + }), + state: "pending", + }) + .onConflictDoNothing() + .returning(); + }); await processPendingPublications(); const batch = await db .select() @@ -31019,6 +31162,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { (attachmentId) => attachedFileById.get(attachmentId)!, ); const publicationCreatedAt = new Date(); + await notifyChatPublicationWork(tx); const [created] = await tx .insert(chatPublications) .values({ @@ -31043,6 +31187,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const attachmentCreatedAt = new Date( publicationCreatedAt.getTime() + index + 1, ); + await notifyChatPublicationWork(tx); const [attachmentPublication] = await tx .insert(chatPublications) .values({ @@ -32635,6 +32780,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { "Slack file upload ownership changed before receipt persistence", ); } + await notifyChatPublicationWork(tx); const inserted = await tx .insert(chatActions) .values({ @@ -33199,6 +33345,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { updatedAt: stoppedAt, }) .where(eq(chatActions.id, action.id)); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -34359,6 +34506,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (preparedText !== null) { // Persist the exact provider text before I/O. A retry must reuse this // body, not accumulate links or regenerate it from changed settings. + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -35389,6 +35537,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // enqueued later for this old generation stays internal even if its run // finishes after /new or /close. A racing insert that commits after this // transaction is rejected by the completed-conversation send guard. + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -35488,6 +35637,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : disposition.kind === "resource_unavailable" ? "cancelled" : "failed"; + await notifyChatPublicationWork(tx); const [ownedPublication] = await tx .update(chatPublications) .set({ @@ -35650,6 +35800,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ? "Paperclip could not send the response attachment. The complete response remains on its Paperclip task for an operator to retry." : "Paperclip could not send an attachment. The file remains on its Paperclip task for an operator to retry."; const idempotencyKey = `${ATTACHMENT_FAILURE_NOTICE_PREFIX}${publication.id}:${attachmentFailureRuntimeContext.generation}:${attachmentFailureRuntimeContext.credentialFingerprint}`; + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -35775,10 +35926,12 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return payload; }; const rootPayload = payloadFor(0); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: rootPayload, updatedAt }) .where(eq(chatPublications.id, current.id)); + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values( currentParts.slice(1).map((_part, offset) => ({ companyId: current.companyId, @@ -35851,6 +36004,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ...currentPayload, transportPart: transportPart(0, 1, "discord_markdown_attachment"), }; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: attachmentPayload, updatedAt }) @@ -35864,6 +36018,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(0, 2, "inline"), }; delete handoffPayload.attachmentIds; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: handoffPayload, updatedAt }) @@ -35874,6 +36029,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(1, 2, "discord_markdown_attachment"), }; delete attachmentPayload.progressState; + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values({ companyId: current.companyId, endpointId: current.endpointId, @@ -35948,6 +36104,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ...currentPayload, transportPart: transportPart(0, 1, "telegram_markdown_attachment"), }; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: attachmentPayload, updatedAt }) @@ -35961,6 +36118,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(0, 2, "inline"), }; delete handoffPayload.attachmentIds; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: handoffPayload, updatedAt }) @@ -35971,6 +36129,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(1, 2, "telegram_markdown_attachment"), }; delete attachmentPayload.progressState; + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values({ companyId: current.companyId, endpointId: current.endpointId, @@ -36023,6 +36182,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { text: currentParts[0]!, transportPart: transportPart(0), }; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: rootPayload, updatedAt }) @@ -36030,6 +36190,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const { progressState: _progressState, ...nonProgressPayload } = currentPayload; + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values( currentParts.slice(1).map((text, offset) => { const index = offset + 1; @@ -36070,6 +36231,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ? classified.retryAfterMs : Math.min(60_000, 2 ** Math.max(0, attempts) * 1_000); const result = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); const [settled] = await tx .update(chatPublications) .set({ @@ -36291,50 +36453,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .select() .from(chatActions) .where( - and( - eq(chatActions.kind, "slack_file_upload_receipt"), - notExists( - db - .select({ id: chatEndpoints.id }) - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.id, chatActions.endpointId), - inArray(chatEndpoints.status, ["paused", "attention"]), - ), - ), - ), - // The normal publication owner is still polling Slack. Receipt - // recovery must neither contend with it nor let its not-yet-due - // action occupy the bounded recovery page. - notExists( - db - .select({ id: chatPublications.id }) - .from(chatPublications) - .where( - and( - sql`${chatPublications.id}::text = ${chatActions.payload}->>'publicationId'`, - sql`${chatPublications.attempts}::text = ${chatActions.payload}->>'publicationAttempt'`, - eq(chatPublications.state, "streaming"), - ), - ), - ), - or( - eq(chatActions.status, "received"), - and( - eq(chatActions.status, "failed"), - sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`, - sql`(${chatActions.result}->>'retryAt')::timestamptz <= ${selectedAt.toISOString()}::timestamptz`, - ), - and( - eq(chatActions.status, "processing"), - lte( - chatActions.updatedAt, - new Date(selectedAt.getTime() - SLACK_FILE_RECEIPT_STALE_MS), - ), - ), - ), - ), + slackReceiptEligibility(selectedAt), ) .orderBy(asc(chatActions.updatedAt), asc(chatActions.id)) .limit(limit); @@ -36342,49 +36461,55 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { for (const selected of actions) { const payload = slackFileUploadReceiptPayload(selected.payload); if (!payload) { - await db - .update(chatActions) - .set({ - status: "failed", - result: { - code: "slack_file_upload_receipt_payload_invalid", - retryable: false, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, selected.id), - eq(chatActions.kind, selected.kind), - eq(chatActions.status, selected.status), - sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, - selected.result === null - ? isNull(chatActions.result) - : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + code: "slack_file_upload_receipt_payload_invalid", + retryable: false, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, selected.id), + eq(chatActions.kind, selected.kind), + eq(chatActions.status, selected.status), + sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, + selected.result === null + ? isNull(chatActions.result) + : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, + ), + ); + }); continue; } const record = await endpointRecord(selected.endpointId); if (!record) { - await db - .update(chatActions) - .set({ - status: "cancelled", - result: { code: "slack_file_upload_receipt_endpoint_removed" }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, selected.id), - eq(chatActions.kind, selected.kind), - eq(chatActions.status, selected.status), - sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, - selected.result === null - ? isNull(chatActions.result) - : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: "cancelled", + result: { code: "slack_file_upload_receipt_endpoint_removed" }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, selected.id), + eq(chatActions.kind, selected.kind), + eq(chatActions.status, selected.status), + sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, + selected.result === null + ? isNull(chatActions.result) + : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, + ), + ); + }); continue; } if (["paused", "attention"].includes(record.endpoint.status)) continue; @@ -36422,6 +36547,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (locked && !eligible) return null; if (locked?.mode === "defer") { if (locked.action.status === "processing") { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36437,6 +36563,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return null; } if (locked?.mode === "settled") { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36457,6 +36584,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { SLACK_FILE_RECEIPT_MAX_AGE_MS || attempt > SLACK_FILE_RECEIPT_MAX_ATTEMPTS ) { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36488,6 +36616,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) { return null; } + await notifyChatPublicationWork(tx); const [claimed] = await tx .update(chatActions) .set({ @@ -36530,6 +36659,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (!providerMessageId) { await db.transaction(async (tx) => { await lease.assertOwned(tx); + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36571,6 +36701,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { locked.action.result?.ownerToken !== ownerToken ) { if (!locked) { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36589,6 +36720,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ); } else if (locked.mode === "defer") { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36607,6 +36739,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ); } else { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36638,6 +36771,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { providerMessageId, })) ) { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36653,6 +36787,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return; } if (locked.mode === "delivery_unknown") { + await notifyChatPublicationWork(tx); const [settled] = await tx .update(chatPublications) .set({ @@ -36692,6 +36827,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), }); } else { + await notifyChatPublicationWork(tx); const [enriched] = await tx .update(chatPublications) .set({ providerMessageId }) @@ -36710,6 +36846,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } } + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36739,31 +36876,34 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { disposition.kind === "retry" ? disposition.retryAfterMs : Math.min(60_000, 1_000 * 2 ** Math.min(attempt, 8)); - await db - .update(chatActions) - .set({ - status: retryable ? "failed" : "cancelled", - result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify( - { - attempts: attempt, - code: retryable - ? "slack_file_upload_receipt_lookup_retry" - : "slack_file_upload_receipt_lookup_rejected", - retryable, - ...(retryable - ? { retryAt: new Date(Date.now() + retryMs).toISOString() } - : {}), - }, - )}::jsonb`, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, selected.id), - eq(chatActions.status, "processing"), - sql`${chatActions.result}->>'ownerToken' = ${ownerToken}`, - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: retryable ? "failed" : "cancelled", + result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify( + { + attempts: attempt, + code: retryable + ? "slack_file_upload_receipt_lookup_retry" + : "slack_file_upload_receipt_lookup_rejected", + retryable, + ...(retryable + ? { retryAt: new Date(Date.now() + retryMs).toISOString() } + : {}), + }, + )}::jsonb`, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, selected.id), + eq(chatActions.status, "processing"), + sql`${chatActions.result}->>'ownerToken' = ${ownerToken}`, + ), + ); + }); } } return processed; @@ -37163,6 +37303,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .then((rows) => rows[0]); if (transfer) return false; + await notifyChatPublicationWork(tx); const [row] = await tx .update(chatPublications) .set({ @@ -37290,6 +37431,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // cards must not wait behind this notice in the conversation FIFO. await db.transaction(async tx => { await guard?.assertOwned(tx); + await notifyChatPublicationWork(tx); await tx.update(chatPublications).set({ state: "cancelled", nextAttemptAt: null, redactedError: "Automatic fallback suppressed while the GitHub tool reply remains unresolved", updatedAt: new Date() }) .where(and(eq(chatPublications.id, publication.id), guard @@ -37305,6 +37447,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } else if (publication.payload.progressState === "failed") return false; const actionIds = await db.transaction(async tx => { await guard?.assertOwned(tx); + await notifyChatPublicationWork(tx); const [cancelled] = await tx.update(chatPublications).set({ state: "cancelled", nextAttemptAt: null, redactedError: "GitHub replies are sent through task-scoped tools", updatedAt: new Date() }) .where(and(eq(chatPublications.id, publication.id), eq(chatPublications.companyId, publication.companyId), @@ -37357,7 +37500,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return; } if (explicitSlackDelivery === "delivered") { - await db.update(chatPublications).set({ state: "cancelled", redactedError: "Identical response already delivered by Slack tool", updatedAt: new Date() }).where(and(eq(chatPublications.id, publication.id), inArray(chatPublications.state, ["pending", "retry"]))); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx.update(chatPublications).set({ state: "cancelled", redactedError: "Identical response already delivered by Slack tool", updatedAt: new Date() }).where(and(eq(chatPublications.id, publication.id), inArray(chatPublications.state, ["pending", "retry"]))) +; + }); return; } const earlierOpenPublication = await db @@ -37447,24 +37594,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { (await hasCommittedTaskControlCompletion(conversation.id)); if (endpoint.status === "paused" || endpoint.status === "attention") { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "pending", - attempts: publication.attempts, - // Pause can race the eligibility read. The query excludes - // inactive endpoints on the next pass, so retain only the - // original provider deadline rather than delaying Resume. - nextAttemptAt: publication.nextAttemptAt, - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "pending", + attempts: publication.attempts, + // Pause can race the eligibility read. The query excludes + // inactive endpoints on the next pass, so retain only the + // original provider deadline rather than delaying Resume. + nextAttemptAt: publication.nextAttemptAt, + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } // The setup conversation is a real end-to-end test: once provider @@ -37477,32 +37627,15 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { providerVisibleCompletion ) { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - redactedError: providerVisibleCompletion - ? "Conversation was completed before delivery" - : "External destination is no longer active", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); - return; - } - if (conversation.isDirectMessage) { - if (!endpoint.allowDirectMessages) { - await publicationLease.assertOwned(); - await db + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx .update(chatPublications) .set({ state: "cancelled", - redactedError: "Direct messages are disabled in Paperclip", + redactedError: providerVisibleCompletion + ? "Conversation was completed before delivery" + : "External destination is no longer active", updatedAt: new Date(), }) .where( @@ -37512,6 +37645,29 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { eq(chatPublications.attempts, publication.attempts + 1), ), ); + }); + return; + } + if (conversation.isDirectMessage) { + if (!endpoint.allowDirectMessages) { + await publicationLease.assertOwned(); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + redactedError: "Direct messages are disabled in Paperclip", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } } else { @@ -37530,20 +37686,23 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null; if (!nonDirectDestinationAllowed(endpoint, resource)) { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - redactedError: "Destination is disabled in Paperclip", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + redactedError: "Destination is disabled in Paperclip", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } } @@ -37566,22 +37725,25 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((result) => result[0] ?? null); if (!currentInteraction || currentInteraction.status !== "pending") { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - attempts: publication.attempts, - nextAttemptAt: null, - redactedError: "Interaction resolved before provider delivery", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + attempts: publication.attempts, + nextAttemptAt: null, + redactedError: "Interaction resolved before provider delivery", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } } @@ -37595,22 +37757,25 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null); if (progressSupersessionReason) { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - attempts: publication.attempts, - nextAttemptAt: null, - redactedError: progressSupersessionReason, - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + attempts: publication.attempts, + nextAttemptAt: null, + redactedError: progressSupersessionReason, + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } // Status is sampled when it reaches the head of the provider lane, @@ -37665,6 +37830,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (!authorizationClaim) { await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -37812,6 +37978,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); const committedAt = new Date(); + await notifyChatPublicationWork(tx); const [completedPublication] = await tx .update(chatPublications) .set({ @@ -38032,35 +38199,61 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } - async function processPendingPublications( - limit = 25, - { waitForCompletion = true }: { waitForCompletion?: boolean } = {}, - ) { - if (shuttingDown) return 0; - await enqueueInboundWakeupPublications(); - await reconcileTerminalConfirmationActions(limit); - const now = new Date(); - const staleBefore = new Date(now.getTime() - 60_000); - // A process that disappears after the provider accepted a post but before - // Paperclip persisted its message id leaves an ambiguous delivery. Never - // resend it automatically; an operator can explicitly replay after - // checking the provider conversation. - const quarantinedPublications = await db - .update(chatPublications) - .set({ - state: "delivery_unknown", - nextAttemptAt: null, - redactedError: - "Provider delivery could not be confirmed after the worker stopped. Check the external conversation before replaying.", - updatedAt: now, - }) - .where( + function slackReceiptEligibility(selectedAt?: Date) { + return and( + eq(chatActions.kind, "slack_file_upload_receipt"), + notExists( + db + .select({ id: chatEndpoints.id }) + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.id, chatActions.endpointId), + inArray(chatEndpoints.status, ["paused", "attention"]), + ), + ), + ), + // The normal publication owner is still polling Slack. Receipt + // recovery must neither contend with it nor let its not-yet-due + // action occupy the bounded recovery page. + notExists( + db + .select({ id: chatPublications.id }) + .from(chatPublications) + .where( + and( + sql`${chatPublications.id}::text = ${chatActions.payload}->>'publicationId'`, + sql`${chatPublications.attempts}::text = ${chatActions.payload}->>'publicationAttempt'`, + eq(chatPublications.state, "streaming"), + ), + ), + ), + or( + eq(chatActions.status, "received"), + and( + eq(chatActions.status, "failed"), + sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`, + selectedAt ? sql`(${chatActions.result}->>'retryAt')::timestamptz <= ${selectedAt.toISOString()}::timestamptz` : sql`${chatActions.result}->>'retryAt' is not null`, + ), + and( + eq(chatActions.status, "processing"), + selectedAt ? lte(chatActions.updatedAt, new Date(selectedAt.getTime() - SLACK_FILE_RECEIPT_STALE_MS)) : undefined, + ), + ), + ); + } + + function publicationEligibility({ now, busyEndpointIds = [], coolingTeamsPublicationIds = [], attemptedIds = [] }: { now?: Date; busyEndpointIds?: string[]; coolingTeamsPublicationIds?: string[]; attemptedIds?: string[] } = {}) { + const earlierPublication = alias(chatPublications, "earlier_chat_publications"); + return and( + sql`not (${unresolvedSlackPublicationCondition()})`, + // An explicit Board send promises both delivery and work. Keep + // its text/files pending until the durable wake has been accepted. + sql`not exists (select 1 from chat_actions a where a.company_id = ${chatPublications.companyId} and a.endpoint_id = ${chatPublications.endpointId} and a.conversation_id = ${chatPublications.conversationId} and a.kind = 'slack_board_message' and a.status = 'received' and a.payload->>'commentId' = ${chatPublications.commentId}::text)`, + or( and( sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, - eq(chatPublications.state, "streaming"), - lte(chatPublications.updatedAt, staleBefore), - // Teams owns separate staged I/O intents and a longer attempt lease. - // Only that protocol may recover/quarantine its in-flight stages. + inArray(chatPublications.state, ["pending", "retry"]), notExists( db .select({ id: chatTeamsFileTransfers.id }) @@ -38071,29 +38264,184 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { chatTeamsFileTransfers.companyId, chatPublications.companyId, ), - eq(chatTeamsFileTransfers.publicationId, chatPublications.id), - ), - ), - ), - notExists( - db - .select({ id: chatEndpointLeases.id }) - .from(chatEndpointLeases) - .where( - and( - eq(chatEndpointLeases.companyId, chatPublications.companyId), eq( - chatEndpointLeases.endpointId, - chatPublications.endpointId, + chatTeamsFileTransfers.publicationId, + chatPublications.id, ), - sql`${chatEndpointLeases.leaseKey} = 'publication:' || ${chatPublications.id}::text || ':' || ${chatPublications.attempts}::text`, - gt(chatEndpointLeases.expiresAt, now), ), ), ), ), - ) - .returning({ id: chatPublications.id }); + // Dedicated protocol work shares the endpoint concurrency cap, + // never the ordinary publication claim or resend implementation. + sql`exists (${db + .select({ id: chatTeamsFileTransfers.id }) + .from(chatTeamsFileTransfers) + .where( + and( + eq( + chatTeamsFileTransfers.companyId, + chatPublications.companyId, + ), + eq( + chatTeamsFileTransfers.endpointId, + chatPublications.endpointId, + ), + eq( + chatTeamsFileTransfers.conversationId, + chatPublications.conversationId, + ), + eq( + chatTeamsFileTransfers.publicationId, + chatPublications.id, + ), + sql`exists (${db + .select({ id: chatEndpoints.id }) + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.id, chatPublications.endpointId), + eq( + chatEndpoints.companyId, + chatPublications.companyId, + ), + eq(chatEndpoints.provider, "microsoft-teams"), + eq(chatEndpoints.status, "active"), + ), + )})`, + or( + inArray(chatTeamsFileTransfers.phase, [ + "consent_pending", + "upload_pending", + "file_info_pending", + ]), + and( + eq(chatTeamsFileTransfers.phase, "consent_unknown"), + isNotNull(chatTeamsFileTransfers.responseActivityId), + sql`${chatTeamsFileTransfers.privateState}->'response' is not null`, + ), + ), + ), + )})`, + ), + busyEndpointIds.length > 0 + ? notInArray(chatPublications.endpointId, busyEndpointIds) + : undefined, + coolingTeamsPublicationIds.length > 0 + ? notInArray(chatPublications.id, coolingTeamsPublicationIds) + : undefined, + // Inactive endpoints must not consume the eligibility page or + // acquire an artificial retry deadline. When an operator resumes + // them, due work is immediately eligible; genuine provider retry + // deadlines below remain unchanged. + notExists( + db + .select({ id: chatEndpoints.id }) + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.id, chatPublications.endpointId), + inArray(chatEndpoints.status, ["paused", "attention"]), + ), + ), + ), + now ? or(isNull(chatPublications.nextAttemptAt), lte(chatPublications.nextAttemptAt, now)) : undefined, + attemptedIds.length > 0 + ? notInArray(chatPublications.id, attemptedIds) + : undefined, + notExists( + db + .select({ id: earlierPublication.id }) + .from(earlierPublication) + .where( + and( + eq( + earlierPublication.conversationId, + chatPublications.conversationId, + ), + publicationPrecedes(earlierPublication, chatPublications), + inArray(earlierPublication.state, [ + "pending", + "retry", + "streaming", + "delivery_unknown", + ]), + ), + ), + ), + ); + } + + async function processPendingPublications( + limit = 25, + { waitForCompletion = true, maintenance = true }: { waitForCompletion?: boolean; maintenance?: boolean } = {}, + ) { + if (shuttingDown) return 0; + if (maintenance) { + await enqueueInboundWakeupPublications(); + await reconcileTerminalConfirmationActions(limit); + } + const now = new Date(); + const staleBefore = new Date(now.getTime() - 60_000); + // A process that disappears after the provider accepted a post but before + // Paperclip persisted its message id leaves an ambiguous delivery. Never + // resend it automatically; an operator can explicitly replay after + // checking the provider conversation. + const quarantinedPublications = await db.transaction(async tx => { + const condition = and( + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, + eq(chatPublications.state, "streaming"), + lte(chatPublications.updatedAt, staleBefore), + // Teams owns separate staged I/O intents and a longer attempt lease. + // Only that protocol may recover/quarantine its in-flight stages. + notExists( + db + .select({ id: chatTeamsFileTransfers.id }) + .from(chatTeamsFileTransfers) + .where( + and( + eq( + chatTeamsFileTransfers.companyId, + chatPublications.companyId, + ), + eq(chatTeamsFileTransfers.publicationId, chatPublications.id), + ), + ), + ), + notExists( + db + .select({ id: chatEndpointLeases.id }) + .from(chatEndpointLeases) + .where( + and( + eq(chatEndpointLeases.companyId, chatPublications.companyId), + eq( + chatEndpointLeases.endpointId, + chatPublications.endpointId, + ), + sql`${chatEndpointLeases.leaseKey} = 'publication:' || ${chatPublications.id}::text || ':' || ${chatPublications.attempts}::text`, + gt(chatEndpointLeases.expiresAt, now), + ), + ), + ), + ); + const candidates = await tx.select({ id: chatPublications.id }).from(chatPublications).where(condition).for("update", { skipLocked: true }); + if (!candidates.length) return []; + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "delivery_unknown", + nextAttemptAt: null, + redactedError: + "Provider delivery could not be confirmed after the worker stopped. Check the external conversation before replaying.", + updatedAt: now, + }) + .where( + and(condition, inArray(chatPublications.id, candidates.map(row => row.id))), + ) + .returning({ id: chatPublications.id }); + }); if (quarantinedPublications.length > 0) { await db .update(chatActions) @@ -38118,10 +38466,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ); } - const earlierPublication = alias( - chatPublications, - "earlier_chat_publications", - ); const attemptedIds: string[] = []; const ownedTasks = new Set>(); let failed = false; @@ -38144,133 +38488,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .select() .from(chatPublications) .where( - and( - // An explicit Board send promises both delivery and work. Keep - // its text/files pending until the durable wake has been accepted. - sql`not exists (select 1 from chat_actions a where a.company_id = ${chatPublications.companyId} and a.endpoint_id = ${chatPublications.endpointId} and a.conversation_id = ${chatPublications.conversationId} and a.kind = 'slack_board_message' and a.status = 'received' and a.payload->>'commentId' = ${chatPublications.commentId}::text)`, - or( - and( - sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, - inArray(chatPublications.state, ["pending", "retry"]), - notExists( - db - .select({ id: chatTeamsFileTransfers.id }) - .from(chatTeamsFileTransfers) - .where( - and( - eq( - chatTeamsFileTransfers.companyId, - chatPublications.companyId, - ), - eq( - chatTeamsFileTransfers.publicationId, - chatPublications.id, - ), - ), - ), - ), - ), - // Dedicated protocol work shares the endpoint concurrency cap, - // never the ordinary publication claim or resend implementation. - sql`exists (${db - .select({ id: chatTeamsFileTransfers.id }) - .from(chatTeamsFileTransfers) - .where( - and( - eq( - chatTeamsFileTransfers.companyId, - chatPublications.companyId, - ), - eq( - chatTeamsFileTransfers.endpointId, - chatPublications.endpointId, - ), - eq( - chatTeamsFileTransfers.conversationId, - chatPublications.conversationId, - ), - eq( - chatTeamsFileTransfers.publicationId, - chatPublications.id, - ), - sql`exists (${db - .select({ id: chatEndpoints.id }) - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.id, chatPublications.endpointId), - eq( - chatEndpoints.companyId, - chatPublications.companyId, - ), - eq(chatEndpoints.provider, "microsoft-teams"), - eq(chatEndpoints.status, "active"), - ), - )})`, - or( - inArray(chatTeamsFileTransfers.phase, [ - "consent_pending", - "upload_pending", - "file_info_pending", - ]), - and( - eq(chatTeamsFileTransfers.phase, "consent_unknown"), - isNotNull(chatTeamsFileTransfers.responseActivityId), - sql`${chatTeamsFileTransfers.privateState}->'response' is not null`, - ), - ), - ), - )})`, - ), - busyEndpointIds.length > 0 - ? notInArray(chatPublications.endpointId, busyEndpointIds) - : undefined, - coolingTeamsPublicationIds.length > 0 - ? notInArray(chatPublications.id, coolingTeamsPublicationIds) - : undefined, - // Inactive endpoints must not consume the eligibility page or - // acquire an artificial retry deadline. When an operator resumes - // them, due work is immediately eligible; genuine provider retry - // deadlines below remain unchanged. - notExists( - db - .select({ id: chatEndpoints.id }) - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.id, chatPublications.endpointId), - inArray(chatEndpoints.status, ["paused", "attention"]), - ), - ), - ), - or( - isNull(chatPublications.nextAttemptAt), - lte(chatPublications.nextAttemptAt, now), - ), - attemptedIds.length > 0 - ? notInArray(chatPublications.id, attemptedIds) - : undefined, - notExists( - db - .select({ id: earlierPublication.id }) - .from(earlierPublication) - .where( - and( - eq( - earlierPublication.conversationId, - chatPublications.conversationId, - ), - publicationPrecedes(earlierPublication, chatPublications), - inArray(earlierPublication.state, [ - "pending", - "retry", - "streaming", - "delivery_unknown", - ]), - ), - ), - ), - ), + publicationEligibility({ now, busyEndpointIds, coolingTeamsPublicationIds, attemptedIds }), ) .orderBy( asc(chatPublications.createdAt), @@ -38334,6 +38552,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { publicationEndpointTasks.delete(selectedPublication.endpointId); } ownedTasks.delete(task); + wakePublications(); }); publicationEndpointTasks.set(selectedPublication.endpointId, task); ownedTasks.add(task); @@ -38356,7 +38575,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (waitForCompletion) await Promise.all(ownedTasks); } if (failed) throw firstError; - await processPendingInteractionWakeups(limit); + if (maintenance) await processPendingInteractionWakeups(limit); return attemptedIds.length; } @@ -38376,12 +38595,103 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } - // Cron refills free endpoint slots from the durable outbox each second. - // Work remains tracked by this service and is joined before runtime shutdown. - async function schedulePendingPublications(limit = 25) { + async function processPublicationMaintenance(limit = 25) { scheduleTeamsFileMaintenance(); scheduleGitHubMaintenance(); - return processPendingPublications(limit, { waitForCompletion: false }); + await enqueueInboundWakeupPublications(); + await reconcileTerminalConfirmationActions(limit); + await processPendingInteractionWakeups(limit); + } + + // The app refills free endpoint slots on commits and durable deadlines. + // Work remains tracked by this service and is joined before runtime shutdown. + async function schedulePendingPublications(limit = 25) { + await processPublicationMaintenance(limit); + return processPendingPublications(limit, { waitForCompletion: false, maintenance: false }); + } + + let wakePublications = () => {}; + + async function nextInboundDeliveryAt(): Promise { + const rows = await db.select({ delivery: { + id: chatDeliveries.id, endpointId: chatDeliveries.endpointId, providerEventId: chatDeliveries.providerEventId, + state: chatDeliveries.state, eventKind: chatDeliveries.eventKind, + // Retain scheduling metadata only, not message bodies or attachment data. + normalizedEvent: sql`jsonb_build_object( + 'conversation', ${chatDeliveries.normalizedEvent}->'conversation', + 'lifecycle', ${chatDeliveries.normalizedEvent}->'lifecycle', + 'message', jsonb_build_object( + 'targetProviderEventId', ${chatDeliveries.normalizedEvent}->'message'->'targetProviderEventId', + 'admissionDependencyEventId', ${chatDeliveries.normalizedEvent}->'message'->'admissionDependencyEventId'))`, + }, at: sql`case + when ${chatDeliveries.state} = 'processing' then ${chatDeliveries.updatedAt} + ${DELIVERY_PROCESSING_STALE_MS} * interval '1 millisecond' + when ${chatDeliveries.state} = 'processed' then ( + select min(case when a.status = 'processing' then a.updated_at + ${DELIVERY_PROCESSING_STALE_MS} * interval '1 millisecond' + else coalesce((a.result->>'retryAt')::timestamptz, now()) end) + from chat_actions a where a.delivery_id = chat_deliveries.id + and a.company_id = chat_deliveries.company_id and a.endpoint_id = chat_deliveries.endpoint_id + and a.kind = 'inbound_wakeup' and a.status in ('issued', 'processing')) + else coalesce(${chatDeliveries.nextAttemptAt}, now()) end` }) + .from(chatDeliveries).where(and( + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.provider = 'agentmail')`, + // Provider lifecycle receipts can restore an unavailable endpoint. + // The ordinary conversation drain still waits for operator resume. + or(sql`${chatDeliveries.normalizedEvent}->'lifecycle' is not null`, + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.status in ('paused', 'attention'))`), + or(inArray(chatDeliveries.state, ["received", "retry", "processing"]), + and(eq(chatDeliveries.state, "processed"), pendingInboundWakeupCondition())), + )).orderBy(...conversationDeliveryOrder()); + // Only an ordered conversation head can run. Reaction and provider + // lifecycle receipts are independent; lifecycle edits may select their + // exact admission dependency before the chronologically earliest row. + const byEvent = new Map(rows.map(row => [`${row.delivery.endpointId}:${row.delivery.providerEventId}`, row])); + const conversations = new Set(); + let next: number | null = null; + for (let row of rows) { + const delivery = row.delivery; + const threadId = normalizedDeliveryThreadId(delivery); + if (threadId && !normalizedLifecycleEffect(delivery) && !["reaction_added", "reaction_removed"].includes(delivery.eventKind)) { + const key = conversationDrainKey(delivery.endpointId, threadId); + if (conversations.has(key)) continue; + conversations.add(key); + const dependency = normalizedLifecycleTargetEventId(delivery); + const target = dependency ? byEvent.get(`${delivery.endpointId}:${dependency}`) : undefined; + if (target && ["received", "retry", "processing"].includes(target.delivery.state)) row = target; + } + if (row.at !== null) { + const at = new Date(row.at).getTime(); + next = next === null ? at : Math.min(next, at); + } + } + return next; + } + + async function nextSlackReceiptAt(): Promise { + const [row] = await db.select({ at: sql`min(case + when ${chatActions.status} = 'processing' then ${chatActions.updatedAt} + ${SLACK_FILE_RECEIPT_STALE_MS} * interval '1 millisecond' + when ${chatActions.status} = 'failed' then (${chatActions.result}->>'retryAt')::timestamptz + else now() end)` }).from(chatActions).where(slackReceiptEligibility()); + return row?.at == null ? null : new Date(row.at).getTime(); + } + + async function nextPublicationAt(): Promise { + const cooldowns = Object.fromEntries([...teamsFileRetryAfter].map(([id, at]) => [id, new Date(at).toISOString()])); + const [queued, streaming] = await Promise.all([ + db.select({ at: sql`min(greatest(coalesce(${chatPublications.nextAttemptAt}, now()), + (${JSON.stringify(cooldowns)}::jsonb->>${chatPublications.id}::text)::timestamptz))` }) + .from(chatPublications).where(publicationEligibility({ busyEndpointIds: [...publicationEndpointTasks.keys()] })), + db.select({ at: sql`min(greatest(${chatPublications.updatedAt} + interval '60 seconds', + (select max(l.expires_at) from chat_endpoint_leases l + where l.company_id = chat_publications.company_id and l.endpoint_id = chat_publications.endpoint_id + and l.lease_key = 'publication:' || chat_publications.id::text || ':' || chat_publications.attempts::text)))` }) + .from(chatPublications).where(and( + eq(chatPublications.state, "streaming"), + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, + sql`not exists (select 1 from ${chatTeamsFileTransfers} f where f.company_id = ${chatPublications.companyId} and f.publication_id = ${chatPublications.id})`, + )), + ]); + const deadlines = [queued[0]?.at, streaming[0]?.at].filter(at => at != null).map(at => new Date(at!).getTime()); + return deadlines.length ? Math.min(...deadlines) : null; } async function getIssueBinding( @@ -38759,6 +39069,14 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { enqueueInboundWakeupPublications, schedulePendingPublications, processPendingDeliveries, + processQueuedDeliveries: () => processPendingDeliveries(25, undefined, false), + scheduleQueuedPublications: () => processPendingPublications(25, { waitForCompletion: false, maintenance: false }), + processPendingChatMaintenance, + processPublicationMaintenance, + nextInboundDeliveryAt, + nextPublicationAt, + nextSlackReceiptAt, + onPublicationsSettled: (wake: () => void) => { wakePublications = wake; }, prepareFailedChatRunRetry, authorizeFailedChatRunRetry, processFailedChatRunRetry, diff --git a/server/src/services/chat-delivery-work.ts b/server/src/services/chat-delivery-work.ts new file mode 100644 index 00000000000..1872e3b6899 --- /dev/null +++ b/server/src/services/chat-delivery-work.ts @@ -0,0 +1,44 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; + +type ChatDeliveryService = { + processQueuedDeliveries(): Promise; + scheduleQueuedPublications(): Promise; + processPendingSlackFileUploadReceipts(): Promise; + nextInboundDeliveryAt(): Promise; + nextPublicationAt(): Promise; + nextSlackReceiptAt(): Promise; + onPublicationsSettled(wake: () => void): void; +}; + +/** Independent queue workers share the app's scheduler and DB transaction owner. */ +export function registerChatDeliveryWork( + coordinator: ReturnType, + service: ChatDeliveryService, + canRun: () => boolean, +) { + const tasks = [ + { queue: DELIVERY_QUEUES.chatInbound, run: service.processQueuedDeliveries, next: service.nextInboundDeliveryAt }, + { queue: DELIVERY_QUEUES.chatPublications, run: service.scheduleQueuedPublications, next: service.nextPublicationAt }, + { queue: DELIVERY_QUEUES.chatReceipts, run: service.processPendingSlackFileUploadReceipts, next: service.nextSlackReceiptAt }, + ]; + const workers = tasks.map(task => { + let next: number | null = null; + const worker = coordinator.register(task.queue, { + retryMs: 1000, + run: () => task.run(), + hasPending: async () => { + next = canRun() ? await task.next() : Date.now() + 1000; + return false; + }, + nextRunAt: () => next === null ? null : Math.max(Date.now() + 1000, next), + }); + if (task.queue === DELIVERY_QUEUES.chatPublications) { + // Dispatch returns after reserving endpoint slots. Refill them when a + // provider task ends, including failures before its final durable write. + service.onPublicationsSettled(worker.wake); + } + return worker; + }); + return { ready: Promise.all(workers.map(worker => worker.ready)) }; +} diff --git a/server/src/services/chat-interaction-publications.ts b/server/src/services/chat-interaction-publications.ts index cddec16dab7..c1d2574b95f 100644 --- a/server/src/services/chat-interaction-publications.ts +++ b/server/src/services/chat-interaction-publications.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { nativePhotonInteraction } from "./photon/interactions.js"; import { randomBytes } from "node:crypto"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -30,7 +31,7 @@ const QUESTION_ACTION_TOKEN_BYTES = 16; export const CHAT_QUESTION_ACTION_TOKEN_TTL_MS = 7 * 24 * 60 * 60 * 1_000; export const TELEGRAM_CALLBACK_DATA_LIMIT_BYTES = 64; -type ChatPublicationDb = Pick; +type ChatPublicationDb = Pick; function terminalNativeInteractionCopy( interaction: IssueThreadInteraction, @@ -408,83 +409,87 @@ export async function enqueueIssueInteractionChatPublications( }, }, }); - const rows = await db - .insert(chatPublications) - .values({ - companyId: interaction.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - issueId: interaction.issueId, - idempotencyKey: `interaction:${interaction.id}:${endpoint.id}`, - payload, - state: "pending", - }) - .onConflictDoNothing() - .returning(); - const publication = rows[0]; - if (publication && endpoint.provider === "imessage-photon" && nativePhotonInteraction(interaction)) { - const reference = randomBytes(9).toString("base64url"); - await db.insert(chatActions).values({ companyId: interaction.companyId, endpointId: endpoint.id, conversationId: conversation.id, - kind: "photon_interaction", providerActionId: `photon:${reference}`, - payload: { version: 1, reference, interactionId: interaction.id, publicationId: publication.id, sessionGeneration: conversation.sessionGeneration, - expiresAt: new Date(publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS).toISOString() }, status: "issued" }); - } else if (publication && formDraft) { - await db.insert(chatActions).values( - chatQuestionFormActionRecords(formDraft, { + const rows = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + const rows = await tx + .insert(chatPublications) + .values({ companyId: interaction.companyId, endpointId: endpoint.id, conversationId: conversation.id, - publicationId: publication.id, - }), - ); - } else if (publication && question && questionActionTokens.length > 0) { - const expiresAt = new Date( - publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, - ).toISOString(); - await db.insert(chatActions).values( - questionActionTokens.map(({ actionId, option }) => ({ - companyId: interaction.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - kind: "question_answer", - providerActionId: actionId, - payload: { - version: 1, + issueId: interaction.issueId, + idempotencyKey: `interaction:${interaction.id}:${endpoint.id}`, + payload, + state: "pending", + }) + .onConflictDoNothing() + .returning(); + const publication = rows[0]; + if (publication && endpoint.provider === "imessage-photon" && nativePhotonInteraction(interaction)) { + const reference = randomBytes(9).toString("base64url"); + await tx.insert(chatActions).values({ companyId: interaction.companyId, endpointId: endpoint.id, conversationId: conversation.id, + kind: "photon_interaction", providerActionId: `photon:${reference}`, + payload: { version: 1, reference, interactionId: interaction.id, publicationId: publication.id, sessionGeneration: conversation.sessionGeneration, + expiresAt: new Date(publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS).toISOString() }, status: "issued" }); + } else if (publication && formDraft) { + await tx.insert(chatActions).values( + chatQuestionFormActionRecords(formDraft, { + companyId: interaction.companyId, + endpointId: endpoint.id, + conversationId: conversation.id, publicationId: publication.id, - interactionId: interaction.id, - questionId: question.id, - optionId: option.id, - expiresAt, - }, - status: "issued", - })), - ); - } else if ( - publication && - confirmation && - confirmationActionTokens.length > 0 - ) { - const expiresAt = new Date( - publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, - ).toISOString(); - await db.insert(chatActions).values( - confirmationActionTokens.map(({ actionId, decision }) => ({ - companyId: interaction.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - kind: "confirmation_response", - providerActionId: actionId, - payload: { - version: 1, - publicationId: publication.id, - interactionId: interaction.id, - decision, - expiresAt, - }, - status: "issued", - })), - ); - } + }), + ); + } else if (publication && question && questionActionTokens.length > 0) { + const expiresAt = new Date( + publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, + ).toISOString(); + await tx.insert(chatActions).values( + questionActionTokens.map(({ actionId, option }) => ({ + companyId: interaction.companyId, + endpointId: endpoint.id, + conversationId: conversation.id, + kind: "question_answer", + providerActionId: actionId, + payload: { + version: 1, + publicationId: publication.id, + interactionId: interaction.id, + questionId: question.id, + optionId: option.id, + expiresAt, + }, + status: "issued", + })), + ); + } else if ( + publication && + confirmation && + confirmationActionTokens.length > 0 + ) { + const expiresAt = new Date( + publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, + ).toISOString(); + await tx.insert(chatActions).values( + confirmationActionTokens.map(({ actionId, decision }) => ({ + companyId: interaction.companyId, + endpointId: endpoint.id, + conversationId: conversation.id, + kind: "confirmation_response", + providerActionId: actionId, + payload: { + version: 1, + publicationId: publication.id, + interactionId: interaction.id, + decision, + expiresAt, + }, + status: "issued", + })), + ); + } + return rows; + }); inserted.push(...rows); } return inserted; @@ -560,20 +565,23 @@ export async function enqueueTerminalIssueInteractionChatPublications( ) .map((publication) => publication.id); if (unsentIds.length > 0) { - await db - .update(chatPublications) - .set({ - state: "cancelled", - nextAttemptAt: null, - redactedError: "Interaction was resolved before provider publication", - updatedAt: new Date(), - }) - .where( - and( - inArray(chatPublications.id, unsentIds), - inArray(chatPublications.state, ["pending", "retry"]), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + nextAttemptAt: null, + redactedError: "Interaction was resolved before provider publication", + updatedAt: new Date(), + }) + .where( + and( + inArray(chatPublications.id, unsentIds), + inArray(chatPublications.state, ["pending", "retry"]), + ), + ); + }); } const originalIds = originals.map((publication) => publication.id); @@ -706,33 +714,37 @@ export async function enqueueTerminalIssueInteractionChatPublications( if (!copy) return []; const inserted: Array = []; for (const original of providerVisibleOriginals) { - if (!original.payload.card) continue; - const rows = await db - .insert(chatPublications) - .values({ - companyId: interaction.companyId, - endpointId: original.endpointId, - conversationId: original.conversationId, - issueId: interaction.issueId, - idempotencyKey: `interaction-resolution:${interaction.id}:${original.endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "issue_interaction", - text: copy.text, - interaction: { - id: interaction.id, - card: { - kind: original.payload.card.kind, - title: original.payload.card.title, - body: copy.body, - actions: [], + const card = original.payload.card; + if (!card) continue; + const rows = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: interaction.companyId, + endpointId: original.endpointId, + conversationId: original.conversationId, + issueId: interaction.issueId, + idempotencyKey: `interaction-resolution:${interaction.id}:${original.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "issue_interaction", + text: copy.text, + interaction: { + id: interaction.id, + card: { + kind: card.kind, + title: card.title, + body: copy.body, + actions: [], + }, }, - }, - }), - state: "pending", - }) - .onConflictDoNothing() - .returning(); + }), + state: "pending", + }) + .onConflictDoNothing() + .returning(); + }); inserted.push(...rows); } return inserted; @@ -771,23 +783,26 @@ export async function cancelPendingIssueInteractionChatPublications( ]), ), ); - return db - .update(chatPublications) - .set({ - state: "cancelled", - nextAttemptAt: null, - redactedError: "Interaction was superseded before publication", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.companyId, input.companyId), - eq(chatPublications.issueId, input.issueId), - inArray(chatPublications.state, ["pending", "retry"]), - inArray(sql`${chatPublications.payload}->>'interactionId'`, [ - ...input.interactionIds, - ]), - ), - ) - .returning(); + return db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + nextAttemptAt: null, + redactedError: "Interaction was superseded before publication", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.companyId, input.companyId), + eq(chatPublications.issueId, input.issueId), + inArray(chatPublications.state, ["pending", "retry"]), + inArray(sql`${chatPublications.payload}->>'interactionId'`, [ + ...input.interactionIds, + ]), + ), + ) + .returning(); + }); } diff --git a/server/src/services/chat-run-publications.ts b/server/src/services/chat-run-publications.ts index 8ec337f76b5..f8c4575c2b3 100644 --- a/server/src/services/chat-run-publications.ts +++ b/server/src/services/chat-run-publications.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { and, asc, @@ -623,6 +624,7 @@ async function enqueueSafeNativeChatProgress( .limit(1); if (explicitlyAuthoredFinal.length > 0) return 0; + await notifyChatPublicationWork(tx); const insertedRows = await tx .insert(chatPublications) .values({ @@ -997,30 +999,33 @@ export async function enqueueChatRunMilestones( .limit(1); if (explicitlyAuthoredPublication.length > 0) continue; } - const result = await db - .insert(chatPublications) - .values({ - companyId: row.companyId, - endpointId: row.endpointId, - conversationId: row.conversationId, - issueId: row.issueId, - idempotencyKey: `run:${row.runId}:${milestone}:${row.endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "safe_milestone", - text: safeMilestoneText({ - agentName: row.agentName, - errorCode: row.runErrorCode, - milestone, - issueId: row.issueId, - publicBaseUrl: input.publicBaseUrl, + const result = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: row.companyId, + endpointId: row.endpointId, + conversationId: row.conversationId, + issueId: row.issueId, + idempotencyKey: `run:${row.runId}:${milestone}:${row.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "safe_milestone", + text: safeMilestoneText({ + agentName: row.agentName, + errorCode: row.runErrorCode, + milestone, + issueId: row.issueId, + publicBaseUrl: input.publicBaseUrl, + }), + progressState: milestone, }), - progressState: milestone, - }), - state: "pending", - }) - .onConflictDoNothing() - .returning({ id: chatPublications.id }); + state: "pending", + }) + .onConflictDoNothing() + .returning({ id: chatPublications.id }); + }); inserted += result.length; } if (rows.length < pageSize) break; diff --git a/server/src/services/chat-teams-file-publication.ts b/server/src/services/chat-teams-file-publication.ts index af3328dce32..0f33df89716 100644 --- a/server/src/services/chat-teams-file-publication.ts +++ b/server/src/services/chat-teams-file-publication.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { z } from "zod"; import { CHAT_FILE_TRANSFER_PHASES } from "@paperclipai/shared"; @@ -375,6 +376,7 @@ export async function projectTeamsFilePublication( !sameDate(publication.nextAttemptAt, nextAttemptAt) || !sameDate(publication.publishedAt, publishedAt) ) { + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ diff --git a/server/src/services/chat-work-notifications.ts b/server/src/services/chat-work-notifications.ts new file mode 100644 index 00000000000..f6caa18e0b9 --- /dev/null +++ b/server/src/services/chat-work-notifications.ts @@ -0,0 +1,16 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export async function notifyChatDeliveryWork(tx: object): Promise { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatInbound); +} + +export async function notifyChatPublicationWork(tx: object): Promise { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatPublications); + // Receipt recovery waits for the publication owner to release its claim. + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatReceipts); +} + +export async function notifyChatEndpointWork(tx: object): Promise { + await notifyChatDeliveryWork(tx); + await notifyChatPublicationWork(tx); +} diff --git a/server/src/services/connectors/slack-delivery.ts b/server/src/services/connectors/slack-delivery.ts index 4fcf08b8320..afbc4fbff5b 100644 --- a/server/src/services/connectors/slack-delivery.ts +++ b/server/src/services/connectors/slack-delivery.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "../chat-work-notifications.js"; import { and, eq, sql, inArray } from "drizzle-orm"; import { chatActions, type Db } from "@paperclipai/db"; import { forbidden } from "../../errors.js"; @@ -40,17 +41,20 @@ export async function slackDelivery( action.status === "processing" && Date.now() - action.updatedAt.getTime() > 5 * 60_000 ) { - const [stale] = await db - .update(chatActions) - .set({ status: "uncertain", updatedAt: new Date() }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - eq(chatActions.updatedAt, action.updatedAt), - ), - ) - .returning(); + const [stale] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "uncertain", updatedAt: new Date() }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + eq(chatActions.updatedAt, action.updatedAt), + ), + ) + .returning(); + }); if (stale) action = stale; } const args = object(action.payload.args); @@ -107,16 +111,19 @@ export async function slackDelivery( }; } if (receipt) { - const [settled] = await db - .update(chatActions) - .set({ status: "processed", result: receipt, updatedAt: new Date() }) - .where( - and( - eq(chatActions.id, action.id), - inArray(chatActions.status, ["uncertain", "processing"]), - ), - ) - .returning(); + const [settled] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "processed", result: receipt, updatedAt: new Date() }) + .where( + and( + eq(chatActions.id, action.id), + inArray(chatActions.status, ["uncertain", "processing"]), + ), + ) + .returning(); + }); if (settled) action = settled; } } diff --git a/server/src/services/connectors/slack-publication.ts b/server/src/services/connectors/slack-publication.ts index e855fb81714..661d581894b 100644 --- a/server/src/services/connectors/slack-publication.ts +++ b/server/src/services/connectors/slack-publication.ts @@ -1,81 +1,54 @@ -import { and, eq, sql, inArray } from "drizzle-orm"; -import { - chatActions, - chatConversations, - issueComments, - type chatPublications, - type Db, -} from "@paperclipai/db"; -import { object } from "./slack-client.js"; +import { sql, type SQL } from "drizzle-orm"; +import { chatPublications, type Db } from "@paperclipai/db"; -/** Explicit sends use a separate durable receipt. Suppress an identical automatic - * final, but retain distinct summaries, questions, blockers and progress. */ +// ECMAScript String.trim whitespace, matching tool-send text comparison. +const trimCharacters = "\u0009\u000a\u000b\u000c\u000d\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"; +type PublicationMatch = { + companyId: SQL; endpointId: SQL; conversationId: SQL; + issueId: SQL; commentId: SQL; payload: SQL; +}; + +function matchingSlackSend(publication: PublicationMatch) { + const p = publication; + return sql`from chat_actions send + join issue_comments comment on comment.id = ${p.commentId} + and comment.company_id = ${p.companyId} and comment.issue_id = ${p.issueId} + join chat_conversations conversation on conversation.id = ${p.conversationId} + and conversation.company_id = ${p.companyId} + where send.company_id = ${p.companyId} and send.endpoint_id = ${p.endpointId} + and send.conversation_id = ${p.conversationId} and send.kind = 'slack_tool_write' + and send.payload->'binding'->>'runId' = coalesce(comment.created_by_run_id, comment.derived_created_by_run_id)::text + and send.payload->>'name' = 'slack_post_message' + and coalesce(${p.payload}->>'interactionId', '') = '' + and coalesce(${p.payload}->>'progressState', '') = '' + and (case when jsonb_typeof(${p.payload}->'attachmentIds') = 'array' + then jsonb_array_length(${p.payload}->'attachmentIds') else 0 end) = 0 + and send.payload->'args'->>'channel' = regexp_replace(conversation.external_conversation_id, '^slack:', '') + and send.payload->'args'->>'thread_ts' = regexp_replace(conversation.external_thread_id, '^.*:', '') + and btrim(send.payload->'args'->>'text', ${trimCharacters}) = btrim(${p.payload}->>'text', ${trimCharacters})`; +} + +/** Dispatch and deadlines must agree about pending explicit Slack sends. */ +export function unresolvedSlackPublicationCondition(): SQL { + return sql`exists (select 1 ${matchingSlackSend({ + companyId: sql`${chatPublications.companyId}`, endpointId: sql`${chatPublications.endpointId}`, + conversationId: sql`${chatPublications.conversationId}`, issueId: sql`${chatPublications.issueId}`, + commentId: sql`${chatPublications.commentId}`, payload: sql`${chatPublications.payload}`, + })} and send.status in ('received', 'processing', 'uncertain'))`; +} + +/** Explicit sends suppress identical automatic finals, but not distinct content. */ export async function slackExplicitPublicationDuplicate( db: Db, publication: typeof chatPublications.$inferSelect, ) { - if ( - !publication.commentId || - publication.payload.interactionId || - publication.payload.progressState || - publication.payload.attachmentIds?.length - ) - return null; - const [comment] = await db - .select({ - runId: sql`coalesce(${issueComments.createdByRunId}, ${issueComments.derivedCreatedByRunId})`, - }) - .from(issueComments) - .where( - and( - eq(issueComments.id, publication.commentId), - eq(issueComments.companyId, publication.companyId), - eq(issueComments.issueId, publication.issueId), - ), - ); - if (!comment?.runId) return null; - const [conversation] = await db - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.id, publication.conversationId), - eq(chatConversations.companyId, publication.companyId), - ), - ); - if (!conversation) return null; - const sends = await db - .select({ payload: chatActions.payload, status: chatActions.status }) - .from(chatActions) - .where( - and( - eq(chatActions.companyId, publication.companyId), - eq(chatActions.endpointId, publication.endpointId), - eq(chatActions.conversationId, publication.conversationId), - eq(chatActions.kind, "slack_tool_write"), - inArray(chatActions.status, [ - "processed", - "received", - "processing", - "uncertain", - ]), - sql`${chatActions.payload}->'binding'->>'runId' = ${comment.runId}`, - sql`${chatActions.payload}->>'name' = 'slack_post_message'`, - ), - ); - const match = sends.find(({ payload }) => { - const args = object(payload.args); - const thread = conversation.externalThreadId.split(":").at(-1); - return ( - args.channel === - conversation.externalConversationId.replace(/^slack:/, "") && - args.thread_ts === thread && - String(args.text).trim() === publication.payload.text.trim() - ); - }); - return match - ? match.status === "processed" - ? "delivered" - : "unresolved" - : null; + const p = publication; + if (!p.commentId || p.payload.interactionId || p.payload.progressState || p.payload.attachmentIds?.length) return null; + const rows = await db.execute<{ status: string }>(sql`select send.status ${matchingSlackSend({ + companyId: sql`${p.companyId}::uuid`, endpointId: sql`${p.endpointId}::uuid`, + conversationId: sql`${p.conversationId}::uuid`, issueId: sql`${p.issueId}::uuid`, + commentId: sql`${p.commentId}::uuid`, payload: sql`${JSON.stringify(p.payload)}::jsonb`, + })} and send.status in ('processed', 'received', 'processing', 'uncertain') + order by (send.status = 'processed') asc limit 1`); + return rows[0] ? rows[0].status === "processed" ? "delivered" : "unresolved" : null; } diff --git a/server/src/services/connectors/slack-retry.ts b/server/src/services/connectors/slack-retry.ts index cea980e5d32..be07562aa87 100644 --- a/server/src/services/connectors/slack-retry.ts +++ b/server/src/services/connectors/slack-retry.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "../chat-work-notifications.js"; import { and, eq, sql, inArray } from "drizzle-orm"; import { chatActions, toolInvocations, type Db } from "@paperclipai/db"; import { @@ -52,6 +53,7 @@ export async function claimSlackRateLimitRetry( !(Date.parse(action.result.retryAt) <= Date.now()) ) return false; + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ status: "received", updatedAt: new Date() }) diff --git a/server/src/services/connectors/slack-writes.ts b/server/src/services/connectors/slack-writes.ts index 6426ef877da..6b48d5a72ed 100644 --- a/server/src/services/connectors/slack-writes.ts +++ b/server/src/services/connectors/slack-writes.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "../chat-work-notifications.js"; import { authorizeSlackDocument } from "./slack-document.js"; import { slackMessage } from "./slack-message.js"; import { createHash } from "node:crypto"; @@ -100,28 +101,31 @@ export async function executeSlackWrite( .update(JSON.stringify({ name, args })) .digest("hex"); const key = `slack-tool:${binding.issueId}:${args.idempotencyKey}`; - const [created] = await db - .insert(chatActions) - .values({ - companyId: binding.companyId, - endpointId: authority.endpoint.id, - conversationId: authority.conversation?.id ?? null, - principalId: authority.principalId, - kind: "slack_tool_write", - providerActionId: key, - status: "received", - payload: { - name, - args, - hash, - binding, - userId: authority.userId, - revision: authority.revision, - invocationId, - }, - }) - .onConflictDoNothing() - .returning(); + const [created] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatActions) + .values({ + companyId: binding.companyId, + endpointId: authority.endpoint.id, + conversationId: authority.conversation?.id ?? null, + principalId: authority.principalId, + kind: "slack_tool_write", + providerActionId: key, + status: "received", + payload: { + name, + args, + hash, + binding, + userId: authority.userId, + revision: authority.revision, + invocationId, + }, + }) + .onConflictDoNothing() + .returning(); + }); const action = created ?? ( @@ -151,13 +155,16 @@ export async function executeSlackWrite( instruction: "Do not retry with another key. Inspect delivery before reconciling an uncertain result.", }; - const [claimed] = await db - .update(chatActions) - .set({ status: "processing", updatedAt: new Date() }) - .where( - and(eq(chatActions.id, action.id), eq(chatActions.status, "received")), - ) - .returning(); + const [claimed] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "processing", updatedAt: new Date() }) + .where( + and(eq(chatActions.id, action.id), eq(chatActions.status, "received")), + ) + .returning(); + }); if (!claimed) return { actionId: action.id, state: "processing" }; try { authority = await resolveSlackTaskAuthority(db, binding); @@ -252,13 +259,16 @@ export async function executeSlackWrite( ) throw forbidden("Slack returned an unsupported upload destination"); // Store the provider receipt before transport. An uncertain finish is never blindly retried. - await db - .update(chatActions) - .set({ - result: { fileId: upload.file_id, phase: "upload_allocated" }, - updatedAt: new Date(), - }) - .where(eq(chatActions.id, action.id)); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + result: { fileId: upload.file_id, phase: "upload_allocated" }, + updatedAt: new Date(), + }) + .where(eq(chatActions.id, action.id)); + }); const uploaded = await fetchImpl(url, { method: "POST", body: bytes, @@ -436,10 +446,13 @@ export async function executeSlackWrite( } : {}), }; - await db - .update(chatActions) - .set({ status: "processed", result: receipt, updatedAt: new Date() }) - .where(eq(chatActions.id, action.id)); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "processed", result: receipt, updatedAt: new Date() }) + .where(eq(chatActions.id, action.id)); + }); await logActivity(db, { companyId: binding.companyId, actorType: "agent", @@ -468,19 +481,22 @@ export async function executeSlackWrite( "slack_request_timeout", "slack_service_unavailable", ].includes(details.code))); - await db - .update(chatActions) - .set({ - status: definite ? "failed" : "uncertain", - result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify({ code: details.code ?? "slack_delivery_uncertain", ...(error instanceof HttpError && error.status === 429 ? { retryAt: new Date(Date.now() + Number(details.retryAfterSeconds ?? 60) * 1000).toISOString() } : {}) })}::jsonb`, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: definite ? "failed" : "uncertain", + result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify({ code: details.code ?? "slack_delivery_uncertain", ...(error instanceof HttpError && error.status === 429 ? { retryAt: new Date(Date.now() + Number(details.retryAfterSeconds ?? 60) * 1000).toISOString() } : {}) })}::jsonb`, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + ), + ); + }); if (error instanceof HttpError) error.details = { ...details, diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 42bed21ae4d..36747099889 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,9 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + chatInbound: "chat-inbound", + chatPublications: "chat-publications", + chatReceipts: "chat-slack-receipts", browser: "browser-use", email: "email-channels", feedback: "feedback-exports", diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 400593dace3..70af5005448 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { recordChatHandoff, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js"; import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js"; import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js"; @@ -13047,49 +13048,55 @@ export function issueService(db: Db) { ); const publicationCreatedAt = new Date(); for (const binding of bindings) { - await dbOrTx - .insert(chatPublications) - .values({ - companyId: binding.companyId, - endpointId: binding.endpointId, - conversationId: binding.conversationId, - issueId, - commentId: comment.id, - idempotencyKey: `comment:${comment.id}:${binding.endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "agent_comment", - text: redactedBody, - }), - state: "pending", - createdAt: publicationCreatedAt, - updatedAt: publicationCreatedAt, - }) - .onConflictDoNothing(); - for (const [index, attachment] of boundAttachments.entries()) { - const attachmentCreatedAt = new Date( - publicationCreatedAt.getTime() + index + 1, - ); - await dbOrTx + await dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx .insert(chatPublications) .values({ companyId: binding.companyId, endpointId: binding.endpointId, conversationId: binding.conversationId, issueId, - commentId: attachment.commentId, - idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + commentId: comment.id, + idempotencyKey: `comment:${comment.id}:${binding.endpointId}`, payload: projectSafeChatPublication({ classification: "external", source: "agent_comment", - text: `Shared ${attachment.originalFilename ?? "a file"}.`, - attachmentIds: [attachment.id], + text: redactedBody, }), state: "pending", - createdAt: attachmentCreatedAt, - updatedAt: attachmentCreatedAt, + createdAt: publicationCreatedAt, + updatedAt: publicationCreatedAt, }) .onConflictDoNothing(); + }); + for (const [index, attachment] of boundAttachments.entries()) { + const attachmentCreatedAt = new Date( + publicationCreatedAt.getTime() + index + 1, + ); + await dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: binding.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId, + commentId: attachment.commentId, + idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: `Shared ${attachment.originalFilename ?? "a file"}.`, + attachmentIds: [attachment.id], + }), + state: "pending", + createdAt: attachmentCreatedAt, + updatedAt: attachmentCreatedAt, + }) + .onConflictDoNothing(); + }); } } } @@ -13333,6 +13340,7 @@ export function issueService(db: Db) { registeredRunId, ); for (const binding of bindings) { + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ diff --git a/server/src/services/slack-board-messages.ts b/server/src/services/slack-board-messages.ts index 756961c7f0a..b8fc4ddd8f3 100644 --- a/server/src/services/slack-board-messages.ts +++ b/server/src/services/slack-board-messages.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; import { assets, @@ -121,6 +122,7 @@ export async function mirrorSlackBoardComment( .onConflictDoNothing() .returning(); if (!receipt) continue; + await notifyChatPublicationWork(db); const [publication] = await db .insert(chatPublications) .values({ @@ -161,7 +163,7 @@ export async function mirrorSlackBoardComment( byId.has(id) ? [byId.get(id)!] : [], ) : files; - for (const [index, file] of orderedFiles.entries()) + for (const [index, file] of orderedFiles.entries()) { await db .insert(chatPublications) .values({ @@ -181,6 +183,7 @@ export async function mirrorSlackBoardComment( createdAt: new Date(Date.now() + index + 1), }) .onConflictDoNothing(); + } await logActivity(db as Db, { companyId: comment.companyId, actorType: "user", diff --git a/server/src/services/voice/voice-session-store.ts b/server/src/services/voice/voice-session-store.ts index 21a864c3730..7a0ce846acc 100644 --- a/server/src/services/voice/voice-session-store.ts +++ b/server/src/services/voice/voice-session-store.ts @@ -1,3 +1,4 @@ +import { notifyChatDeliveryWork } from "../chat-work-notifications.js"; import { environmentService } from "../environments.js"; import { logger } from "../../middleware/logger.js"; import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; @@ -210,6 +211,7 @@ export function voiceSessionStore(db: Db, options: { allowLocalBoard: boolean; n const threadId = conversation.externalThreadId; const providerEventId = `${threadId}:${session.id}:${toolId}`; const trigger = prior.length ? "subscribed_message" : "direct_message"; + await notifyChatDeliveryWork(tx); const [delivery] = await tx.insert(chatDeliveries).values({ companyId: session.companyId, endpointId: session.endpointId, conversationId: session.conversationId, providerEventId, deduplicationKey: createHash("sha256").update(providerEventId).digest("hex"),