diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 1e9be2453b8..f58ef608b00 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1358,8 +1358,16 @@ Native selection retains the row lock that serializes it with cancellation and controller lease renewal. The executor receives an explicit selection result; lifecycle and ownership callbacks update its teardown state immediately, including when preparation later throws. Public sandbox lifecycle and model fallback helpers -remain re-exported by `heartbeat.ts`. Provider dispatch and final cleanup stay in -the executor. +remain re-exported by `heartbeat.ts`. + +Runtime execution is in `server/src/services/heartbeat/runtime-execution.ts`. +`executeHeartbeatRuntime` dispatches a prepared native session or legacy adapter, +captures usage, finalizes workspace operations, and releases gateway tokens and +instruction copies. The executor supplies the atomic dispatch gate and retains +admission, terminal run status, retry handling, and outer lease cleanup. An explicit +dispatch result preserves early cancellation returns; ownership callbacks update +teardown state before errors propagate. Instruction getters retain updates made +during provider execution. Keep policy changes separate from this extraction. ## Wake Context Delivery diff --git a/doc/composer-stop.md b/doc/composer-stop.md index 5579d55f10e..bd0165a19ca 100644 --- a/doc/composer-stop.md +++ b/doc/composer-stop.md @@ -159,7 +159,9 @@ pnpm exec playwright test --config tests/e2e/playwright-composer-stop.config.ts The suite boots a disposable local-trusted instance on port 3199 (override with `PAPERCLIP_E2E_PORT`). It never attaches to an existing server. Native coverage is explicitly skipped without the fixture; it must not use a logged-in provider -as a fallback. Test companies are archived during cleanup. +as a fallback. The acceptance server pins its Codex command resolver to the fixture +and uses an empty credential home, because an installed Codex dependency takes +precedence over `PATH`. Test companies are archived during cleanup. For each runner, the journey starts a parent, child, and unrelated task, plus a terminal child. It sends while running and verifies the durable queue, clicks diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md index 24db42b22ac..fe1b460ff8a 100644 --- a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md +++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md @@ -415,12 +415,23 @@ from event-driven normal paths and route only real deadlines through the interna scheduler. Preserve durable delivery, startup recovery, and domain-level claims. Longer polling intervals alone do not complete an item. -- [ ] **JOB-01 — Chat reconciliation.** Current: every **1s**, with multiple - runtime/delivery/publication lanes and SQL even without connectors. Change: - disarm empty lanes and activate on committed events or deadlines. **Complete - when:** a tenant with no chat obligations issues no chat SQL and new inbound, - publication, and recovery work still runs. Sources: +- [ ] **JOB-01 — Chat reconciliation.** Partially migrated on 2026-10-10. + Inbound deliveries, outbound publications, and Slack upload receipt recovery + now use transaction-aware commit wakes and durable retry/claim deadlines. + Empty queues disarm after startup. Publication deadlines use the dispatch + eligibility predicate so paused endpoints, unknown outcomes, blocked followers, + and receipts owned by a live sender do not cause empty scans. Completion wakes + refill free endpoint slots without waiting for slow peers. No new tables. + Remaining **1s** maintenance: provider runtimes, provider-specific action + outboxes, run milestone/wake-notice projection, GitHub operation recovery, + Teams protocol maintenance, and Slack session sync. GitHub pre-ingress recovery + must still inspect provider history: a lost callback has no local queue row. + This slice does **not** yet make the whole chat subsystem quiet. Independent + DB roots/direct SQL require startup recovery, as with the other migrated queues. + **Complete when:** all remaining lanes are demand/deadline-driven and a tenant + with no chat obligations issues no chat SQL. Sources: [app.ts](../../server/src/app.ts), + [chat-delivery-work.ts](../../server/src/services/chat-delivery-work.ts), [chat-channels.ts](../../server/src/services/chat-channels.ts). - [x] **JOB-02 — Email reconciliation.** Replaced the unconditional **1s** diff --git a/server/src/__tests__/app-private-hostname-gate.test.ts b/server/src/__tests__/app-private-hostname-gate.test.ts index 3e688f40f1a..2b26ebc7d16 100644 --- a/server/src/__tests__/app-private-hostname-gate.test.ts +++ b/server/src/__tests__/app-private-hostname-gate.test.ts @@ -90,17 +90,18 @@ describe("createChatReconciliationCoordinator", () => { expect(ordinary).not.toContain("processFailedGitHubWebhookDeliveries()"); }); - it("wires periodic publication reconciliation to bounded scheduled refill rather than awaiting provider sends", () => { + it("wires bounded publication refill to commit workers and isolates periodic maintenance", () => { const source = readFileSync(new URL("../app.ts", import.meta.url), "utf8"); - const flush = source.slice( - source.indexOf("const flushChatPublications ="), - source.indexOf("const flushChatPublications =") + 600, + expect(source).toContain("registerChatDeliveryWork(deliveryWork, chatChannels,"); + const maintenance = source.slice( + source.indexOf("const reconcileChatPublicationMaintenance ="), + source.indexOf("const chatReconciliation ="), ); - expect(flush).toContain("await chatChannels.schedulePendingPublications()"); - expect(flush).not.toContain("chatChannels.processPendingPublications()"); - expect( - flush.slice(0, flush.indexOf("const chatReconciliation")), - ).not.toContain("await enqueueChatRunMilestones"); + expect(maintenance).toContain("await chatChannels.processPublicationMaintenance()"); + expect(maintenance).not.toContain("chatChannels.schedulePendingPublications()"); + expect(maintenance).not.toContain("chatChannels.processPendingPublications()"); + const workers = readFileSync(new URL("../services/chat-delivery-work.ts", import.meta.url), "utf8"); + expect(workers).toContain("run: service.scheduleQueuedPublications"); // The service integration tests hold real publication workers while this // scheduled method returns; app shutdown must also join those workers. expect(source).toContain("await chatChannels.shutdown()"); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index f1608aac12f..c237fdeba3d 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1,3 +1,7 @@ +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerChatDeliveryWork } from "../services/chat-delivery-work.js"; +import { notifyChatPublicationWork } from "../services/chat-work-notifications.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; import { chatSlackRegistrations, toolOauthStates } from "@paperclipai/db"; import { buildSlackAppManifest } from "@paperclipai/shared"; import { toolAccessService } from "../services/tool-access.js"; @@ -7345,7 +7349,12 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const countBefore = postFetch.mock.calls.length; await executeConnectorTool(db, binding, "slack_post_message", uncertain); expect(postFetch.mock.calls).toHaveLength(countBefore); - await expect(executeSlackTool(db, binding, "slack_delivery", { actionId: uncertainAction.id }, postFetch as typeof fetch)).resolves.toMatchObject({ state: "delivered", receipt: { reconciled: true } }); + const receiptSettled = vi.fn(); + const unsubscribeReceipt = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatPublications, receiptSettled); + try { + await expect(executeSlackTool(db, binding, "slack_delivery", { actionId: uncertainAction.id }, postFetch as typeof fetch)).resolves.toMatchObject({ state: "delivered", receipt: { reconciled: true } }); + expect(receiptSettled).toHaveBeenCalledTimes(1); + } finally { unsubscribeReceipt(); } await expect(executeConnectorTool(db, binding, "slack_create_channel", { name: "approval-required", is_private: true, idempotencyKey: randomUUID() })).rejects.toMatchObject({ reasonCode: expect.stringMatching(/approval/) }); expect(postFetch.mock.calls.some(call => String(call[0]).endsWith("/conversations.create"))).toBe(false); const [approval] = await db.select().from(toolActionRequests).where(eq(toolActionRequests.companyId, fixture.companyId)); @@ -14482,6 +14491,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .onDiscordGatewayEvent; if (!onGatewayEvent) throw new Error("Expected Discord Gateway callback"); + const recovered = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatPublications, recovered); const disconnect = Promise.resolve( onGatewayEvent({ endpointId: endpoint.id, @@ -14505,7 +14516,10 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await Promise.resolve(); expect(recoveryFinished).toBe(false); releaseDisconnect(); - await Promise.all([disconnect, recovery]); + try { + await Promise.all([disconnect, recovery]); + expect(recovered).toHaveBeenCalled(); + } finally { unsubscribe(); } await expect(context.service.get(endpoint.id)).resolves.toMatchObject({ status: "active", healthMessage: "Connected", @@ -17360,10 +17374,22 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const fixture = await seedCompany(); const { endpoint, service } = await configuredSlackEndpoint(fixture); const configuredEndpoint = await service.get(endpoint.id); + const originalTransaction = db.transaction.bind(db); const transaction = vi.spyOn(db, "transaction"); - transaction.mockRejectedValueOnce( - new Error("injected lifecycle persistence failure"), - ); + let injected = false; + transaction.mockImplementation(async (callback, options) => { + const [delivery] = await db.select({ state: chatDeliveries.state }).from(chatDeliveries).where(and( + eq(chatDeliveries.endpointId, endpoint.id), + eq(chatDeliveries.providerEventId, "lifecycle:Ev-lifecycle-retry"), + )); + // Fail the lifecycle effect after durable admission and claim, regardless + // of how many transaction boundaries those writes require. + if (!injected && delivery?.state === "processing") { + injected = true; + throw new Error("injected lifecycle persistence failure"); + } + return originalTransaction(callback, options); + }); await expect( service.handleWebhook( endpoint.publicId, @@ -32754,6 +32780,158 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { return { ...fixture, storage, runtime, service, lanes, enqueue, cleanup }; } + it("restores chat queue deadlines without polling blocked followers or receipt owners", async () => { + const f = await publicationLaneFixture(1); + const { service, lanes, enqueue } = f; + const lane = lanes[0]!; + try { + const head = await enqueue(0, "Head"); + const tail = await enqueue(0, "Follower", 1); + const retryAt = new Date(Date.now() + 120_000); + await db.update(chatPublications).set({ state: "retry", nextAttemptAt: retryAt }).where(eq(chatPublications.id, head.id)); + expect(await service.nextPublicationAt()).toBe(retryAt.getTime()); + await db.update(chatPublications).set({ state: "delivery_unknown", nextAttemptAt: null }).where(eq(chatPublications.id, head.id)); + expect(await service.nextPublicationAt()).toBeNull(); + const leaseUntil = new Date(Date.now() + 180_000); + await db.update(chatPublications).set({ state: "streaming", updatedAt: new Date() }).where(eq(chatPublications.id, head.id)); + await db.insert(chatEndpointLeases).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + leaseKey: `publication:${head.id}:0`, token: randomUUID(), expiresAt: leaseUntil }); + expect(await service.nextPublicationAt()).toBe(leaseUntil.getTime()); + await db.insert(chatActions).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + kind: "slack_file_upload_receipt", providerActionId: randomUUID(), status: "received", + payload: { publicationId: head.id, publicationAttempt: 0 } }); + expect(await service.nextSlackReceiptAt()).toBeNull(); + await db.delete(chatEndpointLeases).where(eq(chatEndpointLeases.endpointId, lane.endpoint.id)); + await db.update(chatPublications).set({ updatedAt: new Date(Date.now() - 70_000) }).where(eq(chatPublications.id, head.id)); + const wake = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatReceipts, wake); + try { + await service.scheduleQueuedPublications(); + expect(wake).toHaveBeenCalled(); + expect(await service.nextSlackReceiptAt()).toBeLessThanOrEqual(Date.now()); + await service.processPendingSlackFileUploadReceipts(); // Invalid retained payload is closed, not retried. + expect(await service.nextSlackReceiptAt()).toBeNull(); + } finally { unsubscribe(); } + await db.update(chatPublications).set({ state: "cancelled" }).where(eq(chatPublications.id, tail.id)); + expect(await service.nextPublicationAt()).toBeNull(); + } finally { await f.cleanup(); } + }); + + it("restores inbound retry and abandoned wake deadlines and excludes paused queues", async () => { + const f = await publicationLaneFixture(1); + const { service, lanes } = f; + const lane = lanes[0]!; + try { + const retryAt = new Date(Date.now() + 120_000); + const [delivery] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + conversationId: lane.conversation.id, providerEventId: randomUUID(), deduplicationKey: randomUUID(), + eventKind: "message", normalizedEvent: { conversation: { externalThreadId: lane.conversation.externalThreadId } }, + receivedAt: new Date(Date.now() - 1000), state: "retry", nextAttemptAt: retryAt }).returning(); + const [follower] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + conversationId: lane.conversation.id, providerEventId: randomUUID(), deduplicationKey: randomUUID(), + eventKind: "message", normalizedEvent: delivery!.normalizedEvent, state: "received" }).returning(); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + const [edit] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "message_updated", state: "received", + receivedAt: new Date(Date.now() - 2000), normalizedEvent: { ...delivery!.normalizedEvent, + message: { targetProviderEventId: delivery!.providerEventId } } }).returning(); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + await db.update(chatDeliveries).set({ state: "filtered" }).where(eq(chatDeliveries.id, edit!.id)); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + await db.update(chatDeliveries).set({ state: "processed" }).where(eq(chatDeliveries.id, delivery!.id)); + const [action] = await db.insert(chatActions).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + deliveryId: delivery!.id, kind: "inbound_wakeup", providerActionId: randomUUID(), status: "issued", + result: { retryAt: retryAt.toISOString() } }).returning(); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + const claimedAt = new Date(); + await db.update(chatActions).set({ status: "processing", updatedAt: claimedAt }).where(eq(chatActions.id, action!.id)); + expect(await service.nextInboundDeliveryAt()).toBe(claimedAt.getTime() + 60_000); + await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, lane.endpoint.id)); + expect(await service.nextInboundDeliveryAt()).toBeNull(); + await db.update(chatActions).set({ status: "processed" }).where(eq(chatActions.id, action!.id)); + await db.update(chatDeliveries).set({ state: "filtered" }).where(eq(chatDeliveries.id, follower!.id)); + await db.update(chatEndpoints).set({ status: "active" }).where(eq(chatEndpoints.id, lane.endpoint.id)); + expect(await service.nextInboundDeliveryAt()).toBeNull(); + await db.update(chatEndpoints).set({ status: "attention" }).where(eq(chatEndpoints.id, lane.endpoint.id)); + await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "installation", state: "retry", + nextAttemptAt: retryAt, normalizedEvent: { lifecycle: { kind: "endpoint", provider: "slack", + providerEventId: randomUUID(), availability: "available", reason: "Recovered" } } }); + expect(await service.nextInboundDeliveryAt()).toBe(retryAt.getTime()); + } finally { await f.cleanup(); } + }); + + it("sleeps behind an unresolved Slack tool send and wakes when its receipt settles", async () => { + const f = await publicationLaneFixture(1); + const lane = f.lanes[0]!; + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + try { + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "succeeded" }).returning(); + const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: lane.conversation.issueId, + authorAgentId: f.assignedAgentId, createdByRunId: run!.id, body: "Already sent" }).returning(); + const publication = await f.enqueue(0, "Already sent"); + await db.update(chatPublications).set({ commentId: comment!.id }).where(eq(chatPublications.id, publication.id)); + const [send] = await db.insert(chatActions).values({ companyId: f.companyId, endpointId: lane.endpoint.id, + conversationId: lane.conversation.id, kind: "slack_tool_write", providerActionId: randomUUID(), status: "processing", + payload: { name: "slack_post_message", binding: { runId: run!.id }, args: { + channel: lane.conversation.externalConversationId.replace(/^slack:/, ""), + thread_ts: lane.conversation.externalThreadId.split(":").at(-1), text: " Already sent\n" } } }).returning(); + await registerChatDeliveryWork(coordinator, f.service, () => true).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 }); + expect(await f.service.nextPublicationAt()).toBeNull(); + for (const status of ["uncertain", "processed"]) { + await db.transaction(async tx => { + await notifyChatPublicationWork(tx); + await tx.update(chatActions).set({ status }).where(eq(chatActions.id, send!.id)); + }); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 }); + } + await vi.waitFor(async () => { + const [row] = await db.select().from(chatPublications).where(eq(chatPublications.id, publication.id)); + expect(row!.state).toBe("cancelled"); + expect(coordinator.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + expect(lane.providerRuntime.posts).toHaveLength(0); + expect(onError).not.toHaveBeenCalled(); + } finally { await coordinator.stop(); await f.cleanup(); } + }); + + it("wakes chat dispatch after commit and refills free endpoint slots without the maintenance loop", async () => { + const f = await publicationLaneFixture(6); + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + f.lanes[0]!.providerRuntime.postHook = () => gate; + try { + await registerChatDeliveryWork(coordinator, f.service, () => true).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 }); + let settle!: () => void; + const commitGate = new Promise(resolve => { settle = resolve; }); + let staged!: () => void; + const stagedGate = new Promise(resolve => { staged = resolve; }); + const writing = db.transaction(async tx => { + await notifyChatPublicationWork(tx); + await tx.insert(chatPublications).values(f.lanes.map((lane, index) => ({ companyId: f.companyId, + endpointId: lane.endpoint.id, conversationId: lane.conversation.id, issueId: lane.conversation.issueId, + idempotencyKey: `work-test:${randomUUID()}`, payload: { text: `Queued ${index}` }, state: "pending" as const }))); + staged(); await commitGate; + }); + await stagedGate; + expect(f.lanes.every(lane => lane.providerRuntime.posts.length === 0)).toBe(true); + settle(); await writing; + await vi.waitFor(() => expect(f.lanes.slice(1).every(lane => lane.providerRuntime.posts.length === 1)).toBe(true), { timeout: 10_000 }); + release(); + await vi.waitFor(async () => { + const rows = await db.select().from(chatPublications).where(eq(chatPublications.companyId, f.companyId)); + expect(rows.every(row => row.state === "published")).toBe(true); + expect(coordinator.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + expect(onError).not.toHaveBeenCalled(); + } finally { release(); await coordinator.stop(); await f.cleanup(); } + }); + it("refills four tracked publication endpoint lanes after a scheduled budget and joins shutdown", async () => { const fixture = await publicationLaneFixture(6); const { service, lanes, enqueue, storage } = fixture; diff --git a/server/src/__tests__/chat-delivery-work.test.ts b/server/src/__tests__/chat-delivery-work.test.ts new file mode 100644 index 00000000000..fe2c9bc68a0 --- /dev/null +++ b/server/src/__tests__/chat-delivery-work.test.ts @@ -0,0 +1,112 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerChatDeliveryWork } from "../services/chat-delivery-work.js"; +import { notifyChatDeliveryWork, notifyChatPublicationWork } from "../services/chat-work-notifications.js"; + +const coordinators: ReturnType[] = []; +afterEach(async () => { + for (const coordinator of coordinators.splice(0)) await coordinator.stop(); + vi.useRealTimers(); +}); +function setup() { + vi.useFakeTimers(); + const execute = vi.fn(async () => [{ xid: "42", status: "committed" }]); + const owner = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise) { + return callback({ execute }); + } }); + const enabled = vi.fn(() => true), primary = vi.fn(() => true), onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner, + canRun: () => enabled() && primary(), canReconcile: primary, onError, + }); + coordinators.push(coordinator); + const service = { + processQueuedDeliveries: vi.fn(async () => {}), + scheduleQueuedPublications: vi.fn(async () => {}), + processPendingSlackFileUploadReceipts: vi.fn(async () => {}), + nextInboundDeliveryAt: vi.fn(async (): Promise => null), + nextPublicationAt: vi.fn(async (): Promise => null), + nextSlackReceiptAt: vi.fn(async (): Promise => null), + onPublicationsSettled: vi.fn(), + }; + const start = () => registerChatDeliveryWork(coordinator, service, enabled); + return { owner, service, coordinator, start, enabled, primary, onError }; +} + +describe("chat delivery work", () => { + it("does one startup pass and no further queue scans when empty", async () => { + const s = setup(); + await s.start().ready; + await vi.advanceTimersByTimeAsync(3_600_000); + for (const run of [s.service.processQueuedDeliveries, s.service.scheduleQueuedPublications, s.service.processPendingSlackFileUploadReceipts]) expect(run).toHaveBeenCalledTimes(1); + expect(s.coordinator.nextWakeAt()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + it("coalesces commits, wakes receipt recovery after publication changes, and refills freed slots", async () => { + const s = setup(); + await s.start().ready; + await s.owner.transaction(tx => notifyChatPublicationWork(tx)); + await s.owner.transaction(tx => notifyChatPublicationWork(tx)); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(2); + expect(s.service.processPendingSlackFileUploadReceipts).toHaveBeenCalledTimes(2); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(1); + s.service.onPublicationsSettled.mock.calls[0]![0](); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(3); + expect(vi.getTimerCount()).toBe(0); + }); + it("keeps lanes independent and retains commits received during a blocked drain", async () => { + const s = setup(); + let release!: () => void; + s.service.processQueuedDeliveries.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + const worker = s.start(); + await vi.advanceTimersByTimeAsync(1); + await s.owner.transaction(tx => notifyChatDeliveryWork(tx)); + await s.owner.transaction(tx => notifyChatPublicationWork(tx)); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(2); + release(); + await worker.ready; + await vi.advanceTimersByTimeAsync(1); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(2); + }); + it("restores future retries and stops when the last retry finishes", async () => { + const s = setup(); + const due = Date.now() + 60_000; + s.service.nextPublicationAt.mockResolvedValue(due); + await s.start().ready; + expect(s.coordinator.nextWakeAt()).toBe(due); + await vi.advanceTimersByTimeAsync(59_999); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(1); + s.service.nextPublicationAt.mockResolvedValue(null); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.scheduleQueuedPublications).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }); + it("does no queue SQL while standby or drain defers a first commit", async () => { + const s = setup(); + s.primary.mockReturnValue(false); + await s.start().ready; + await vi.advanceTimersByTimeAsync(2000); + s.primary.mockReturnValue(true); s.enabled.mockReturnValue(false); + await s.owner.transaction(tx => notifyChatDeliveryWork(tx)); + await vi.advanceTimersByTimeAsync(2000); + expect(s.service.processQueuedDeliveries).not.toHaveBeenCalled(); + expect(s.service.nextInboundDeliveryAt).not.toHaveBeenCalled(); + s.enabled.mockReturnValue(true); + await vi.advanceTimersByTimeAsync(1000); + expect(s.service.processQueuedDeliveries).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }); + it("retries failed selection without losing its startup recovery", async () => { + const s = setup(); + s.service.nextSlackReceiptAt.mockRejectedValueOnce(new Error("connection lost")); + await s.start().ready; + expect(s.onError).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1000); + expect(s.service.processPendingSlackFileUploadReceipts).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/server/src/__tests__/chat-interaction-publications.test.ts b/server/src/__tests__/chat-interaction-publications.test.ts index 6c4c5d5f8ee..eab650d3927 100644 --- a/server/src/__tests__/chat-interaction-publications.test.ts +++ b/server/src/__tests__/chat-interaction-publications.test.ts @@ -20,7 +20,7 @@ import { toolConnections, } from "@paperclipai/db"; import type { ChatProvider } from "@paperclipai/shared"; -import { enqueueTerminalIssueInteractionChatPublications } from "../services/chat-interaction-publications.js"; +import { enqueueIssueInteractionChatPublications, enqueueTerminalIssueInteractionChatPublications } from "../services/chat-interaction-publications.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { getEmbeddedPostgresTestSupport, @@ -226,6 +226,7 @@ describeEmbeddedPostgres( wrapQuery(Reflect.apply(db.select, db, args))) as typeof db.select, insert: db.insert.bind(db), update: db.update.bind(db), + transaction: db.transaction.bind(db), }; } @@ -312,6 +313,35 @@ describeEmbeddedPostgres( ).toBe(true); }); + it("rolls back an interaction publication if its callback tokens cannot be stored", async () => { + const fixture = await seedBoundIssue(["telegram"]); + const interaction = await issueThreadInteractionService(db).create( + { id: fixture.issueId, companyId: fixture.companyId }, + { kind: "request_confirmation", continuationPolicy: "wake_assignee", payload: { version: 1, prompt: "Proceed?" } }, + { agentId: fixture.agentId }, + ); + expect(await publicationsForInteraction(fixture.companyId, interaction.id)).toHaveLength(1); + await db.delete(chatActions).where(eq(chatActions.companyId, fixture.companyId)); + await db.delete(chatPublications).where(eq(chatPublications.companyId, fixture.companyId)); + const failTokens = (insert: typeof db.insert): typeof db.insert => + ((table: Parameters[0]) => { + if (table === chatActions) throw new Error("Token write failed"); + return insert(table); + }) as typeof db.insert; + const failingDb: Parameters[0] = { + select: db.select.bind(db), + update: db.update.bind(db), + insert: failTokens(db.insert.bind(db)), + transaction: callback => db.transaction(async tx => { + tx.insert = failTokens(tx.insert.bind(tx)); + return callback(tx); + }), + }; + await expect(enqueueIssueInteractionChatPublications(failingDb, interaction)) + .rejects.toThrow("Token write failed"); + expect(await publicationsForInteraction(fixture.companyId, interaction.id)).toEqual([]); + }); + it("keeps unsupported governance interactions authoritative in Paperclip", async () => { const fixture = await seedBoundIssue(); const interaction = await issueThreadInteractionService(db).create( diff --git a/server/src/app.ts b/server/src/app.ts index 7d17b92ce03..32bef274ebe 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -9,6 +9,7 @@ import { customerSuccessRoutes } from "./routes/customer-success.js"; import { cloudWarmStandbyMiddleware } from "./middleware/cloud-warm-standby.js"; import type { CloudWarmStandby } from "./services/cloud-warm-standby.js"; import { browserUseRoutes } from "./routes/browser-use.js"; +import { registerChatDeliveryWork } from "./services/chat-delivery-work.js"; import { registerBrowserUseCleanup } from "./services/browser-use-work.js"; import { computerRoutes } from "./routes/computers.js"; import { browserUseService } from "./services/browser-use.js"; @@ -320,7 +321,7 @@ export function createChatReconciliationCoordinator(input: { processFailedGitHubWebhookDeliveries?: () => Promise; projectRunMilestones: () => Promise; flushPublications: () => Promise; - processPendingSlackFileUploadReceipts: () => Promise; + processPendingSlackFileUploadReceipts?: () => Promise; processPendingSlackSessionSyncs: () => Promise; onError: (lane: ChatReconciliationLane, error: unknown) => void; }) { @@ -367,7 +368,7 @@ export function createChatReconciliationCoordinator(input: { } milestoneReconciliation.poll(); publicationReconciliation.poll(); - start("Slack file receipts", input.processPendingSlackFileUploadReceipts); + if (input.processPendingSlackFileUploadReceipts) start("Slack file receipts", input.processPendingSlackFileUploadReceipts); start("Slack session status", input.processPendingSlackSessionSyncs); }, notifyPublications() { @@ -1234,25 +1235,24 @@ export async function createApp( hasPending: () => opts.feedbackExportService!.hasPendingFeedbackTraces(), }); } + registerChatDeliveryWork(deliveryWork, chatChannels, () => !isIdleTaskDrainActive()); emailChannels.start(); - const flushChatPublications = async () => { - await chatChannels.schedulePendingPublications(); + const reconcileChatPublicationMaintenance = async () => { + await chatChannels.processPublicationMaintenance(); }; const chatReconciliation = createChatReconciliationCoordinator({ reconcileProviderRuntimes: async () => { await chatChannels.reconcileProviderRuntimes(); await chatChannels.voice.reconcile(); }, - processPendingDeliveries: () => chatChannels.processPendingDeliveries(), + processPendingDeliveries: () => chatChannels.processPendingChatMaintenance(), processFailedGitHubWebhookDeliveries: () => chatChannels.processFailedGitHubWebhookDeliveries(), projectRunMilestones: () => enqueueChatRunMilestones(db, { publicBaseUrl: opts.authPublicBaseUrl, }), - flushPublications: () => flushChatPublications(), - processPendingSlackFileUploadReceipts: () => - chatChannels.processPendingSlackFileUploadReceipts(), + flushPublications: () => reconcileChatPublicationMaintenance(), processPendingSlackSessionSyncs: () => chatChannels.processPendingSlackSessionSyncs(), onError: (lane, err) => { @@ -1265,6 +1265,8 @@ export async function createApp( chatReconciliation.notifyPublications(); }, ); + // Provider/action maintenance and milestone projection still use this cadence. + // Inbox, outbox, and file receipt queues have independent commit/deadline wakes. let chatPublicationTimer: ReturnType | null = setInterval( () => { if (!isWarmStandby() && !isIdleTaskDrainActive()) chatReconciliation.reconcile(); diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index 51c4b86af2f..59d0d9f4e4a 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -1,3 +1,4 @@ +import { notifyChatDeliveryWork, notifyChatPublicationWork, notifyChatEndpointWork } from "./chat-work-notifications.js"; import { syncSpekoVoiceTools } from "./voice/speko-agent-tools.js"; import { configureSpekoSessionTools } from "./voice/speko-tool-setup.js"; import { voiceSessionService } from "./voice/voice-session-service.js"; @@ -22,7 +23,7 @@ import { withSlackBoardLease } from "./slack-board-lease.js"; import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js"; import { authorizeSlackBoardPublication } from "./slack-board-authority.js"; import { assertSlackBoardWorkAllowed } from "./slack-board-resume.js"; -import { slackExplicitPublicationDuplicate } from "./connectors/slack-publication.js"; +import { slackExplicitPublicationDuplicate, unresolvedSlackPublicationCondition } from "./connectors/slack-publication.js"; import { rememberVerifiedSlackSearchEvent, slackSearchActionToken } from "./connectors/slack-search-context.js"; import { slackAuthorizationRevision } from "./connectors/slack-revision.js"; import { slackPublicationAllowed } from "./connectors/slack-access.js"; @@ -4632,6 +4633,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { principalId: string; }, ) { + await notifyChatPublicationWork(tx); const [inserted] = await tx .insert(chatPublications) .values({ @@ -4749,6 +4751,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatActions.id }); if (!ownedAction) return false; if (input.payload.settleDelivery && input.action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -4852,6 +4855,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .returning({ id: chatActions.id }); if (quarantined && payload?.settleDelivery && action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -5116,6 +5120,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatActions.id }); if (!cancelled) return false; if (action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -5402,6 +5407,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } if (payload.settleDelivery && action!.deliveryId) { + await notifyChatDeliveryWork(tx); const [settled] = await tx .update(chatDeliveries) .set({ @@ -5750,6 +5756,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (current.status === "attention" && !gatewayOwnedAttention) return; const status = current.setup.step === "complete" ? "active" : "verifying"; + if (status !== current.status) await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -7850,6 +7857,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (!current) return; const connection = await tx.select().from(toolConnections).where(eq(toolConnections.id, current.connectionId)).then((rows) => rows[0]); if (!connection?.enabled) throw new Error("Photon endpoint requires operator recovery"); + await notifyChatEndpointWork(tx); await tx.update(chatEndpoints).set({ status: current.setup.step === "complete" ? "active" : "verifying", healthMessage: "Photon receiver connected", lastError: null, updatedAt: new Date() }).where(eq(chatEndpoints.id, endpoint.id)); }); }, @@ -8473,6 +8481,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { status: "processed", }) .onConflictDoNothing(); + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -9061,6 +9070,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_endpoint_not_active", }); } + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9076,6 +9086,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .update(toolConnections) .set({ status: "disabled", enabled: false, updatedAt: new Date() }) .where(eq(toolConnections.id, endpoint.connectionId)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -9178,6 +9189,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // Fail closed for any legacy or racing delivery that remained open // while this endpoint was paused. Resume must never execute traffic // that Paperclip acknowledged during the inactive interval. + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -9194,6 +9206,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { inArray(chatDeliveries.state, ["received", "retry"]), ), ); + await notifyChatEndpointWork(tx); [activatedEndpoint] = await tx .update(chatEndpoints) .set({ @@ -9240,6 +9253,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const failedAt = new Date(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9267,6 +9281,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { updatedAt: failedAt, }) .where(eq(toolConnections.id, endpoint.connectionId)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -9341,6 +9356,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_endpoint_already_removed", }); } + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9574,6 +9590,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { try { const claimed = await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointWork(tx); const rows = await tx .update(chatEndpoints) .set({ @@ -9657,6 +9674,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const preserveSlackVerification = current.setup.slackSetupMethod === "automatic" && (current.setup as InternalSetupState).slackVerificationSigningFingerprint === createHash("sha256").update(credentials.signingSecret ?? "").digest("hex") && Boolean(observedSlackUrl && slackCallbackMatchesPublicUrl(observedSlackUrl, `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/slack`)); + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -9848,6 +9866,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { await invalidateRuntime(endpoint.id).catch(() => undefined); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -10130,6 +10149,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { )).limit(1).then((rows) => rows[0]); if (!linked || !(await lockCurrentPrincipalAuthorization(tx, endpoint, linked.principalId)).allowed) throw forbidden("Your connected account is no longer authorized"); } + await notifyChatEndpointWork(tx); const [activated] = await tx .update(chatEndpoints) .set({ @@ -11222,6 +11242,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }, runtimeContext: effectContext, }); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -11414,6 +11435,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { message: { providerMessageId: event.messageId }, filtering: { contentRetained: false, providerThreadCreated: false }, }; + await notifyChatDeliveryWork(tx); await tx .insert(chatDeliveries) .values({ @@ -11526,6 +11548,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ["verifying", "active"], ); if (!current) return false; + await notifyChatDeliveryWork(tx); const [row] = await tx .update(chatDeliveries) .set({ @@ -13542,6 +13565,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { current.status === "deferred_issue_execution" ? "queued" : "not_started"; + await notifyChatPublicationWork(tx); return ( await tx .insert(chatPublications) @@ -13981,6 +14005,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return 0; if (prior && !(await inboundQueueNoticeStillVisible(tx, prior))) return 0; + await notifyChatPublicationWork(tx); const rows = await tx .insert(chatPublications) .values({ @@ -14146,6 +14171,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { existingReceipt, ); } + await notifyChatDeliveryWork(tx); const accepted = await tx .update(chatDeliveries) .set({ @@ -14168,6 +14194,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // An operator can repair a failed delivery ledger after its wake was // already committed. Keep the immutable receipt and never admit twice. if (action.status !== "preparing") return; + await notifyChatDeliveryWork(tx); const issued = await tx .update(chatActions) .set({ @@ -14260,24 +14287,28 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { Date.parse(candidate.result.retryAt) > now.getTime()) ) return false; - const claimed = await db - .update(chatActions) - .set({ status: "processing", updatedAt: now }) - .where( - and( - eq(chatActions.id, candidate.id), - eq(chatActions.status, candidate.status), - eq(chatActions.updatedAt, candidate.updatedAt), - ), - ) - .returning() - .then((rows) => rows[0] ?? null); + const claimed = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatActions) + .set({ status: "processing", updatedAt: now }) + .where( + and( + eq(chatActions.id, candidate.id), + eq(chatActions.status, candidate.status), + eq(chatActions.updatedAt, candidate.updatedAt), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); if (!claimed) return false; const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1; const receiptDeclined = (row: typeof agentWakeupRequests.$inferSelect) => ["skipped", "cancelled", "failed"].includes(row.status); const settle = async (status: string, result: Record) => { const changed = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); const changed = await tx .update(chatActions) .set({ status, result, updatedAt: new Date() }) @@ -15008,6 +15039,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((rows) => rows[0] ?? null) : null; if (!admittedDeliveryId) { + await notifyChatDeliveryWork(tx); const [delivery] = await tx .insert(chatDeliveries) .values({ @@ -15072,6 +15104,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { existingDelivery.state, ) && deliveryContentWasRedacted(existingDelivery.normalizedEvent); + await notifyChatDeliveryWork(tx); [candidate] = await tx .update(chatDeliveries) .set({ @@ -15148,6 +15181,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { !accepting && ["received", "retry", "processing"].includes(candidate.state) ) { + await notifyChatDeliveryWork(tx); [candidate] = await tx .update(chatDeliveries) .set({ @@ -15237,23 +15271,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // The adapter invokes the ordinary message callback only after Discord // has returned the created thread. Promote the pre-admission receipt // atomically so the normal ordered drain can begin immediately. - const [promoted] = await db - .update(chatDeliveries) - .set({ - state: "received", - normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, - nextAttemptAt: scheduledAt ? new Date(scheduledAt) : null, - redactedError: null, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, candidate.id), - eq(chatDeliveries.state, "retry"), - sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, - ), - ) - .returning(); + const candidateId = candidate.id; + const [promoted] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "received", + normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, + nextAttemptAt: scheduledAt ? new Date(scheduledAt) : null, + redactedError: null, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, candidateId), + eq(chatDeliveries.state, "retry"), + sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, + ), + ) + .returning(); + }); if (promoted) candidate = promoted; } const now = new Date(); @@ -15298,17 +15336,20 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ]; if (candidate.state === "processing") claimConditions.push(lte(chatDeliveries.updatedAt, staleBefore)); - const claimed = await db - .update(chatDeliveries) - .set({ - state: "processing", - attempts: candidate.attempts + 1, - nextAttemptAt: null, - redactedError: null, - updatedAt: now, - }) - .where(and(...claimConditions)) - .returning(); + const claimed = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "processing", + attempts: candidate.attempts + 1, + nextAttemptAt: null, + redactedError: null, + updatedAt: now, + }) + .where(and(...claimConditions)) + .returning(); + }); const activeDelivery = claimed[0]; if (!activeDelivery) return; @@ -15317,29 +15358,32 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const resolutionAt = new Date(); const waitForSetupRoot = activeDelivery.attempts <= ORPHAN_FOLLOW_UP_MAX_ATTEMPTS; - await db - .update(chatDeliveries) - .set( - waitForSetupRoot - ? { - state: "retry", - nextAttemptAt: new Date( - resolutionAt.getTime() + ORPHAN_FOLLOW_UP_GRACE_MS, - ), - redactedError: "Waiting briefly for an earlier root mention", - updatedAt: resolutionAt, - } - : { - state: "filtered", - normalizedEvent: redactedDestinationNormalized, - principalId: null, - nextAttemptAt: null, - processedAt: resolutionAt, - redactedError: "Destination is not enabled in Paperclip", - updatedAt: resolutionAt, - }, - ) - .where(eq(chatDeliveries.id, activeDelivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set( + waitForSetupRoot + ? { + state: "retry", + nextAttemptAt: new Date( + resolutionAt.getTime() + ORPHAN_FOLLOW_UP_GRACE_MS, + ), + redactedError: "Waiting briefly for an earlier root mention", + updatedAt: resolutionAt, + } + : { + state: "filtered", + normalizedEvent: redactedDestinationNormalized, + principalId: null, + nextAttemptAt: null, + processedAt: resolutionAt, + redactedError: "Destination is not enabled in Paperclip", + updatedAt: resolutionAt, + }, + ) + .where(eq(chatDeliveries.id, activeDelivery.id)); + }); return; } // A committed inbound message link proves that a prior attempt completed @@ -15693,20 +15737,23 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // the durable thread drain can sort again if a delayed root arrives. // A standalone unaddressed message is filtered after the bounded // retention window. - await db - .update(chatDeliveries) - .set({ - state: "retry", - nextAttemptAt: new Date(Date.now() + ORPHAN_FOLLOW_UP_GRACE_MS), - redactedError: "Waiting briefly for an earlier root mention", - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, activeDelivery.id), - eq(chatDeliveries.state, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "retry", + nextAttemptAt: new Date(Date.now() + ORPHAN_FOLLOW_UP_GRACE_MS), + redactedError: "Waiting briefly for an earlier root mention", + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, activeDelivery.id), + eq(chatDeliveries.state, "processing"), + ), + ); + }); return; } if (!allowed) { @@ -15721,18 +15768,21 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ? "Endpoint sponsor can no longer authorize external guests" : "External identity must be linked to a Paperclip account" : "Message did not address the agent or an active task thread"; - await db - .update(chatDeliveries) - .set({ - state: "filtered", - normalizedEvent: redactedDestinationNormalized, - principalId: null, - nextAttemptAt: null, - redactedError: filteredReason, - processedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, activeDelivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + normalizedEvent: redactedDestinationNormalized, + principalId: null, + nextAttemptAt: null, + redactedError: filteredReason, + processedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, activeDelivery.id)); + }); return; } @@ -15790,17 +15840,20 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .limit(1) .then((rows) => rows[0]); if (!source || source.conversationId !== existingConversation?.id) { - await db - .update(chatDeliveries) - .set({ - state: "filtered", - processedAt: new Date(), - nextAttemptAt: null, - redactedError: - "Quoted control does not belong to the current task generation", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, activeDelivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + processedAt: new Date(), + nextAttemptAt: null, + redactedError: + "Quoted control does not belong to the current task generation", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, activeDelivery.id)); + }); return; } } @@ -15836,6 +15889,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .update(chatEndpoints) .set({ lastEventAt: new Date(), updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -15944,6 +15998,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .set({ lastEventAt: new Date(), updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); if (await filterPreControlSource(tx)) return false; + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -16403,6 +16458,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { !currentPrincipalAuthorization.allowed ) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -16567,25 +16623,28 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { providerEffectAmbiguous || isExternalActionAuthorizationChange(error) || activeDelivery.attempts >= 5; - await db - .update(chatDeliveries) - .set({ - state: terminal ? "failed" : "retry", - nextAttemptAt: terminal - ? null - : new Date( - Date.now() + - Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), - ), - redactedError: redactError(error), - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, activeDelivery.id), - eq(chatDeliveries.state, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: terminal ? "failed" : "retry", + nextAttemptAt: terminal + ? null + : new Date( + Date.now() + + Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), + ), + redactedError: redactError(error), + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, activeDelivery.id), + eq(chatDeliveries.state, "processing"), + ), + ); + }); if (terminal) await settleRejectedInboundWakeups(activeDelivery.id); throw error; } @@ -16704,6 +16763,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ["verifying", "active"], ); if (!current) return false; + await notifyChatDeliveryWork(tx); const [promoted] = await tx .update(chatDeliveries) .set({ @@ -16789,6 +16849,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -16815,7 +16876,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } } - function normalizedDeliveryThreadId(delivery: DeliveryRow): string | null { + function normalizedDeliveryThreadId(delivery: Pick): string | null { const normalized = delivery.normalizedEvent as { conversation?: { externalThreadId?: unknown }; }; @@ -16825,7 +16886,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } function normalizedLifecycleTargetEventId( - delivery: DeliveryRow, + delivery: Pick, ): string | null { if ( delivery.eventKind !== "message_updated" && @@ -16846,7 +16907,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } function normalizedLifecycleEffect( - delivery: DeliveryRow, + delivery: Pick, ): ChatProviderLifecycleEffect | null { const normalized = delivery.normalizedEvent as { lifecycle?: unknown }; const value = normalized.lifecycle; @@ -16901,6 +16962,48 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } + function conversationDeliveryOrder() { + return [ + asc( + sql`coalesce(nullif(${chatDeliveries.normalizedEvent}->'message'->>'providerSentAt', '')::timestamptz, ${chatDeliveries.receivedAt})`, + ), + // GitHub timestamps and Telegram message dates have one-second + // resolution. GitHub's numeric comment id and Telegram's message_id + // are monotonic within one conversation, so use them before receipt + // order. The Telegram update_id is a final provider-native tie-breaker + // for unusual payloads that lack a usable message_id. + asc(sql`coalesce( + case + when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence' ~ '^[0-9]+$' + then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence')::numeric + else null + end, + case + when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' ~ '^[0-9]+$' + then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId')::numeric + else null + end + )`), + asc(sql`case + when ${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId' ~ '^[0-9]+$' + then (${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId')::numeric + else null + end`), + // GitHub exposes no sortable webhook sequence. When an edit and delete + // share its whole-second updated_at value, preserve the only valid + // lifecycle state transition: update before delete. Provider-native + // update ids (Telegram) remain the stronger preceding key. + asc(sql`case ${chatDeliveries.eventKind} + when 'message_updated' then 1 + when 'message_deleted' then 2 + when 'message_restored' then 3 + else 0 + end`), + asc(chatDeliveries.receivedAt), + asc(chatDeliveries.id), + ]; + } + async function earliestOpenConversationDelivery( endpointId: string, threadId: string, @@ -16926,45 +17029,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { externalThreadIdentityCondition(externalThreadId, threadId), ), ) - .orderBy( - asc( - sql`coalesce(nullif(${chatDeliveries.normalizedEvent}->'message'->>'providerSentAt', '')::timestamptz, ${chatDeliveries.receivedAt})`, - ), - // GitHub timestamps and Telegram message dates have one-second - // resolution. GitHub's numeric comment id and Telegram's message_id - // are monotonic within one conversation, so use them before receipt - // order. The Telegram update_id is a final provider-native tie-breaker - // for unusual payloads that lack a usable message_id. - asc(sql`coalesce( - case - when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence' ~ '^[0-9]+$' - then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageSequence')::numeric - else null - end, - case - when ${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' ~ '^[0-9]+$' - then (${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId')::numeric - else null - end - )`), - asc(sql`case - when ${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId' ~ '^[0-9]+$' - then (${chatDeliveries.normalizedEvent}->'message'->>'providerUpdateId')::numeric - else null - end`), - // GitHub exposes no sortable webhook sequence. When an edit and delete - // share its whole-second updated_at value, preserve the only valid - // lifecycle state transition: update before delete. Provider-native - // update ids (Telegram) remain the stronger preceding key. - asc(sql`case ${chatDeliveries.eventKind} - when 'message_updated' then 1 - when 'message_deleted' then 2 - when 'message_restored' then 3 - else 0 - end`), - asc(chatDeliveries.receivedAt), - asc(chatDeliveries.id), - ) + .orderBy(...conversationDeliveryOrder()) .limit(1) .then((rows) => rows[0] ?? null); if (!candidate) return null; @@ -17376,6 +17441,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { !telegramDeliveryHasZeroMessageId(current, endpoint.provider) ) return false; + await notifyChatDeliveryWork(tx); const changed = await tx .update(chatDeliveries) .set({ @@ -17403,15 +17469,18 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { continue; } if (endpoint.status === "archived" || endpoint.status === "revoked") { - await db - .update(chatDeliveries) - .set({ - state: "filtered", - redactedError: "Connection was removed before processing", - processedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + redactedError: "Connection was removed before processing", + processedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); liveInboundMessages.delete(delivery.id); continue; } @@ -17457,27 +17526,30 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } else { if (deliveryContentWasRedacted(delivery.normalizedEvent)) { const filteredAt = new Date(); - await db - .update(chatDeliveries) - .set({ - state: "filtered", - principalId: null, - nextAttemptAt: null, - processedAt: filteredAt, - redactedError: - "Provisional Teams setup reply could not be hydrated from a current provider event", - updatedAt: filteredAt, - }) - .where( - and( - eq(chatDeliveries.id, delivery.id), - inArray(chatDeliveries.state, [ - "received", - "retry", - "processing", - ]), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "filtered", + principalId: null, + nextAttemptAt: null, + processedAt: filteredAt, + redactedError: + "Provisional Teams setup reply could not be hydrated from a current provider event", + updatedAt: filteredAt, + }) + .where( + and( + eq(chatDeliveries.id, delivery.id), + inArray(chatDeliveries.state, [ + "received", + "retry", + "processing", + ]), + ), + ); + }); continue; } const endpointRuntime = await runtimeFor(endpoint); @@ -17492,14 +17564,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { endpointRuntime, ); if (!reconstructed) { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Normalized delivery is incomplete", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Normalized delivery is incomplete", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); continue; } await processMessage( @@ -17728,6 +17803,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { conversationDrainKey(input.endpointId, input.threadId), ) ?? Date.now() + reorderWindow) : null; + await notifyChatDeliveryWork(tx); const [delivery] = await tx .insert(chatDeliveries) .values({ @@ -17886,14 +17962,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ): Promise { const lifecycle = lifecycleMessageFromDelivery(candidate); if (!lifecycle) { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Normalized lifecycle delivery is incomplete", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, candidate.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Normalized lifecycle delivery is incomplete", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, candidate.id)); + }); return; } @@ -17905,17 +17984,20 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ]; if (candidate.state === "processing") claimConditions.push(lte(chatDeliveries.updatedAt, staleBefore)); - const [activeDelivery] = await db - .update(chatDeliveries) - .set({ - state: "processing", - attempts: candidate.attempts + 1, - nextAttemptAt: null, - redactedError: null, - updatedAt: now, - }) - .where(and(...claimConditions)) - .returning(); + const [activeDelivery] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "processing", + attempts: candidate.attempts + 1, + nextAttemptAt: null, + redactedError: null, + updatedAt: now, + }) + .where(and(...claimConditions)) + .returning(); + }); if (!activeDelivery) return; try { @@ -17931,6 +18013,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null; if (!admittedRuntimeContext || !currentEndpoint) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -17989,6 +18072,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { linkedMessages.some((row) => row.direction === "inbound"); if (outbound || (lifecycle.isBotMessage && !githubBotSourceEdit)) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18093,6 +18177,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { editPredatesRestore ) { const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18182,6 +18267,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { originalIsOpen || activeDelivery.attempts <= ORPHAN_FOLLOW_UP_MAX_ATTEMPTS; const resolutionAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set( @@ -18261,6 +18347,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // Provider authentication proves the exact source changed. The // editor's current Paperclip rights govern admitting new content, // not whether a later regrant can resurrect the stale old source. + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18369,6 +18456,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) return; const filteredAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18420,6 +18508,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { (activeDelivery.eventKind === "message_deleted" || (activeDelivery.eventKind === "message_updated" && lifecyclePrincipalId !== null)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18479,6 +18568,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { { authorType: "system" }, tx, ); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -18498,25 +18588,28 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } catch (error) { const terminal = activeDelivery.attempts >= 5; - await db - .update(chatDeliveries) - .set({ - state: terminal ? "failed" : "retry", - nextAttemptAt: terminal - ? null - : new Date( - Date.now() + - Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), - ), - redactedError: redactError(error), - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, activeDelivery.id), - eq(chatDeliveries.state, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: terminal ? "failed" : "retry", + nextAttemptAt: terminal + ? null + : new Date( + Date.now() + + Math.min(60_000, 1000 * 2 ** activeDelivery.attempts), + ), + redactedError: redactError(error), + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, activeDelivery.id), + eq(chatDeliveries.state, "processing"), + ), + ); + }); throw error; } } @@ -18966,6 +19059,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .limit(1) .then((rows) => rows.length > 0); + await notifyChatDeliveryWork(tx); const reconciled = hasOutboundLink ? await tx .update(chatDeliveries) @@ -18992,6 +19086,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatDeliveries.id }) : []; if (reconciled.length > 0) return true; + await notifyChatDeliveryWork(tx); const expired = await tx .update(chatDeliveries) .set({ @@ -19018,6 +19113,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .returning({ id: chatDeliveries.id }); if (expired.length > 0) return false; } + await notifyChatDeliveryWork(tx); const inserted = await tx .insert(chatDeliveries) .values({ @@ -19141,6 +19237,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { safelyKnown.principalId ?? null, ) : null; + await notifyChatDeliveryWork(tx); const [inserted] = await tx .insert(chatDeliveries) .values({ @@ -19268,6 +19365,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { let invalidateEndpointRuntime = false; try { effect = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); const [inserted] = await tx .insert(chatDeliveries) .values({ @@ -19729,6 +19827,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ), ); + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -20460,6 +20559,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((rows) => rows[0] ?? null); if (occupied && occupied.publicationId !== current.id) return null; const reconciledAt = new Date(); + await notifyChatPublicationWork(tx); const [reconciled] = await tx .update(chatPublications) .set({ @@ -21181,6 +21281,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { answeredQuestionCount: 1, }, }); + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -23697,31 +23798,34 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .digest("hex"); const eventKind: ChatEventKind = effect.availability === "available" ? "installation" : "uninstallation"; - const [inserted] = await db - .insert(chatDeliveries) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - providerEventId, - deduplicationKey, - eventKind, - normalizedEvent: { + const [inserted] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .insert(chatDeliveries) + .values({ + companyId: endpoint.companyId, + endpointId: endpoint.id, providerEventId, - kind: eventKind, - lifecycle: effect, - ...(runtimeContext - ? { - runtimeContext: { - credentialFingerprint: runtimeContext.credentialFingerprint, - generation: runtimeContext.generation, - }, - } - : {}), - }, - state: "received", - }) - .onConflictDoNothing() - .returning(); + deduplicationKey, + eventKind, + normalizedEvent: { + providerEventId, + kind: eventKind, + lifecycle: effect, + ...(runtimeContext + ? { + runtimeContext: { + credentialFingerprint: runtimeContext.credentialFingerprint, + generation: runtimeContext.generation, + }, + } + : {}), + }, + state: "received", + }) + .onConflictDoNothing() + .returning(); + }); const candidate = inserted ?? (await db @@ -23736,28 +23840,31 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((rows) => rows[0] ?? null)); if (!candidate || candidate.state === "processed") return false; const staleBefore = new Date(Date.now() - DELIVERY_PROCESSING_STALE_MS); - const [claimed] = await db - .update(chatDeliveries) - .set({ - state: "processing", - attempts: candidate.attempts + 1, - redactedError: null, - nextAttemptAt: null, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, candidate.id), - or( - inArray(chatDeliveries.state, ["received", "retry"]), - and( - eq(chatDeliveries.state, "processing"), - lte(chatDeliveries.updatedAt, staleBefore), + const [claimed] = await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "processing", + attempts: candidate.attempts + 1, + redactedError: null, + nextAttemptAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, candidate.id), + or( + inArray(chatDeliveries.state, ["received", "retry"]), + and( + eq(chatDeliveries.state, "processing"), + lte(chatDeliveries.updatedAt, staleBefore), + ), ), ), - ), - ) - .returning({ id: chatDeliveries.id }); + ) + .returning({ id: chatDeliveries.id }); + }); if (!claimed) return false; let refreshRuntimeAfterLifecycle = false; @@ -23781,6 +23888,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const filteredAt = new Date(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -23839,6 +23947,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); const revokedAt = new Date(); if (!current) { + await notifyChatDeliveryWork(tx); const [filtered] = await tx .update(chatDeliveries) .set({ @@ -23914,6 +24023,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { inArray(chatConversations.state, ["active", "waiting"]), ), ); + await notifyChatDeliveryWork(tx); const [processed] = await tx .update(chatDeliveries) .set({ @@ -24075,6 +24185,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { eq(toolConnections.id, canonical.endpoint.connectionId), ); } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24170,6 +24281,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { inArray(chatConversations.state, ["active", "waiting"]), ), ); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24221,6 +24333,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .update(chatEndpoints) .set({ lastEventAt: now, updatedAt: now }) .where(eq(chatEndpoints.id, currentEndpoint.id)); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24413,6 +24526,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : currentEndpoint.status === "attention" ? "verifying" : currentEndpoint.status; + if (status !== currentEndpoint.status) await notifyChatEndpointWork(tx); await tx .update(chatEndpoints) .set({ @@ -24492,6 +24606,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -24523,22 +24638,25 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } if (!lifecycleFailurePersisted) { const terminal = candidate.attempts + 1 >= 5; - await db - .update(chatDeliveries) - .set({ - state: terminal ? "failed" : "retry", - redactedError: redactError(error), - nextAttemptAt: terminal ? null : new Date(), - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, claimed.id), - eq(chatDeliveries.state, "processing"), - eq(chatDeliveries.attempts, candidate.attempts + 1), - ), - ) - .catch(() => undefined); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: terminal ? "failed" : "retry", + redactedError: redactError(error), + nextAttemptAt: terminal ? null : new Date(), + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, claimed.id), + eq(chatDeliveries.state, "processing"), + eq(chatDeliveries.attempts, candidate.attempts + 1), + ), + ) + .catch(() => undefined); + }); } throw error; } @@ -25014,6 +25132,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : action === "deleted" ? "message_deleted" : "message"; + await notifyChatDeliveryWork(tx); await tx .insert(chatDeliveries) .values({ @@ -27625,21 +27744,24 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const reaction = normalizedPendingReaction(candidate); if (!reaction || !candidate.principalId) { const now = new Date(); - await db - .update(chatDeliveries) - .set({ - state: "failed", - nextAttemptAt: null, - processedAt: now, - redactedError: "Normalized reaction delivery is incomplete", - updatedAt: now, - }) - .where( - and( - eq(chatDeliveries.id, candidate.id), - inArray(chatDeliveries.state, ["received", "retry"]), - ), - ); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + nextAttemptAt: null, + processedAt: now, + redactedError: "Normalized reaction delivery is incomplete", + updatedAt: now, + }) + .where( + and( + eq(chatDeliveries.id, candidate.id), + inArray(chatDeliveries.state, ["received", "retry"]), + ), + ); + }); return; } const principalId = candidate.principalId; @@ -27766,6 +27888,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { 2 ** Math.min(Math.max(locked.attempts, 0), 5), REACTION_LINK_MAX_DELAY_MS, ); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -27796,6 +27919,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const processedAt = new Date(); + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set( @@ -27903,6 +28027,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { async function cancelWork(message: string) { await lease.commit(async (tx) => { await tx.update(chatActions).set({ status: "cancelled", result: { code: "slack_board_work_not_authorized", message }, updatedAt: new Date() }).where(eq(chatActions.id, action.id)); + await notifyChatPublicationWork(tx); await tx.update(chatPublications).set({ state: "cancelled", redactedError: message, nextAttemptAt: null, updatedAt: new Date() }).where(and( eq(chatPublications.companyId, action.companyId), eq(chatPublications.endpointId, action.endpointId), eq(chatPublications.conversationId, action.conversationId!), eq(chatPublications.commentId, commentId), @@ -27948,6 +28073,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { contextSnapshot: { issueId, taskId: issueId, taskKey: issue.identifier ?? issueId, wakeCommentId: commentId, source: "issue.comment", resumeIntent: true, followUpRequested: true }, }); await lease.commit(async (tx) => { + await notifyChatPublicationWork(tx); await tx.update(chatActions).set({ status: "processed", updatedAt: new Date() }).where(eq(chatActions.id, action.id)); }); } catch (error) { @@ -27961,28 +28087,30 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } } - async function processPendingDeliveries(limit = 25, onlyDeliveryId?: string) { + async function processPendingChatMaintenance(limit = 25) { + return Promise.allSettled([ + processPendingSlackBoardMessages(limit), + processPendingGitHubWebhookIngress(limit), + processPendingProviderEffects(limit), + slackRegistration.processPendingVerificationMessages(limit), + processPendingReceiptReactions(limit), + processPendingSlackSessionStops(limit), + processPendingTelegramMaintenance(limit), + // Slack task starts are an action-backed outbox. Queued work may + // post once; provider-confirmed rows perform Paperclip-only + // admission; a stale in-flight post is quarantined as unknown. + processPendingSlackTaskStarts(limit), + processFailedChatRunRetries(limit), + ]); + } + + async function processPendingDeliveries(limit = 25, onlyDeliveryId?: string, maintenance = true) { await settleRejectedInboundWakeups(onlyDeliveryId); // Provider-visible effects that are not backed by a task publication use // chat_actions as their outbox. Reconcile them before inbound deliveries // so a crashed processing claim is quarantined before the delivery worker // could otherwise replay it. - const actionRecovery = onlyDeliveryId - ? null - : Promise.allSettled([ - processPendingSlackBoardMessages(limit), - processPendingGitHubWebhookIngress(limit), - processPendingProviderEffects(limit), - slackRegistration.processPendingVerificationMessages(limit), - processPendingReceiptReactions(limit), - processPendingSlackSessionStops(limit), - processPendingTelegramMaintenance(limit), - // Slack task starts are an action-backed outbox. Queued work may - // post once; provider-confirmed rows perform Paperclip-only - // admission; a stale in-flight post is quarantined as unknown. - processPendingSlackTaskStarts(limit), - processFailedChatRunRetries(limit), - ]); + const actionRecovery = onlyDeliveryId || !maintenance ? null : processPendingChatMaintenance(limit); const reactionRecovery = processPendingReactionDeliveries( limit, onlyDeliveryId, @@ -28046,14 +28174,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (lifecycleEffect) { const record = await endpointRecord(delivery.endpointId); if (!record || record.endpoint.status === "archived") { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Chat endpoint is no longer available", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Chat endpoint is no longer available", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); continue; } try { @@ -28075,14 +28206,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const externalThreadId = normalizedDeliveryThreadId(delivery); if (!externalThreadId) { - await db - .update(chatDeliveries) - .set({ - state: "failed", - redactedError: "Normalized delivery is incomplete", - updatedAt: new Date(), - }) - .where(eq(chatDeliveries.id, delivery.id)); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + redactedError: "Normalized delivery is incomplete", + updatedAt: new Date(), + }) + .where(eq(chatDeliveries.id, delivery.id)); + }); continue; } const key = conversationDrainKey(delivery.endpointId, externalThreadId); @@ -28854,6 +28988,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const current = await runtimeCallbackEndpoint(tx, endpoint.id, runtimeContext, ["verifying", "active"]); if (!current) return; const providerEventId = `github:policy_ingress:${createHash("sha256").update(deliveryId).digest("hex")}`; + await notifyChatDeliveryWork(tx); await tx.insert(chatDeliveries).values({ companyId: current.companyId, endpointId: current.id, providerEventId, deduplicationKey: providerEventId, eventKind: "message", state: "filtered", processedAt: new Date(), @@ -29357,6 +29492,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); const claimed = await tx .update(chatDeliveries) .set({ @@ -29496,6 +29632,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_file_transfer_resolution_required", }, ); + await notifyChatPublicationWork(tx); return tx .update(chatPublications) .set({ @@ -29701,6 +29838,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .onConflictDoNothing(); } const now = new Date(); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set( @@ -29941,6 +30079,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }, ); } + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set({ @@ -30096,6 +30235,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } if (payload.settleDelivery && action.deliveryId) { + await notifyChatDeliveryWork(tx); await tx .update(chatDeliveries) .set( @@ -30747,24 +30887,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .then((rows) => rows[0] ?? null); if (!comment) throw notFound("Task comment not found"); - const [publication] = await db - .insert(chatPublications) - .values({ - companyId: conversation.companyId, - endpointId, - conversationId, - issueId: conversation.issueId, - commentId, - idempotencyKey: `explicit:${commentId}:${endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "explicit_board_send", - text: comment.body, - }), - state: "pending", - }) - .onConflictDoNothing() - .returning(); + const [publication] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: conversation.companyId, + endpointId, + conversationId, + issueId: conversation.issueId, + commentId, + idempotencyKey: `explicit:${commentId}:${endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "explicit_board_send", + text: comment.body, + }), + state: "pending", + }) + .onConflictDoNothing() + .returning(); + }); await processPendingPublications(); const batch = await db .select() @@ -31019,6 +31162,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { (attachmentId) => attachedFileById.get(attachmentId)!, ); const publicationCreatedAt = new Date(); + await notifyChatPublicationWork(tx); const [created] = await tx .insert(chatPublications) .values({ @@ -31043,6 +31187,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const attachmentCreatedAt = new Date( publicationCreatedAt.getTime() + index + 1, ); + await notifyChatPublicationWork(tx); const [attachmentPublication] = await tx .insert(chatPublications) .values({ @@ -32635,6 +32780,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { "Slack file upload ownership changed before receipt persistence", ); } + await notifyChatPublicationWork(tx); const inserted = await tx .insert(chatActions) .values({ @@ -33199,6 +33345,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { updatedAt: stoppedAt, }) .where(eq(chatActions.id, action.id)); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -34359,6 +34506,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (preparedText !== null) { // Persist the exact provider text before I/O. A retry must reuse this // body, not accumulate links or regenerate it from changed settings. + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -35389,6 +35537,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // enqueued later for this old generation stays internal even if its run // finishes after /new or /close. A racing insert that commits after this // transaction is rejected by the completed-conversation send guard. + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -35488,6 +35637,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : disposition.kind === "resource_unavailable" ? "cancelled" : "failed"; + await notifyChatPublicationWork(tx); const [ownedPublication] = await tx .update(chatPublications) .set({ @@ -35650,6 +35800,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ? "Paperclip could not send the response attachment. The complete response remains on its Paperclip task for an operator to retry." : "Paperclip could not send an attachment. The file remains on its Paperclip task for an operator to retry."; const idempotencyKey = `${ATTACHMENT_FAILURE_NOTICE_PREFIX}${publication.id}:${attachmentFailureRuntimeContext.generation}:${attachmentFailureRuntimeContext.credentialFingerprint}`; + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ @@ -35775,10 +35926,12 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return payload; }; const rootPayload = payloadFor(0); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: rootPayload, updatedAt }) .where(eq(chatPublications.id, current.id)); + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values( currentParts.slice(1).map((_part, offset) => ({ companyId: current.companyId, @@ -35851,6 +36004,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ...currentPayload, transportPart: transportPart(0, 1, "discord_markdown_attachment"), }; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: attachmentPayload, updatedAt }) @@ -35864,6 +36018,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(0, 2, "inline"), }; delete handoffPayload.attachmentIds; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: handoffPayload, updatedAt }) @@ -35874,6 +36029,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(1, 2, "discord_markdown_attachment"), }; delete attachmentPayload.progressState; + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values({ companyId: current.companyId, endpointId: current.endpointId, @@ -35948,6 +36104,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ...currentPayload, transportPart: transportPart(0, 1, "telegram_markdown_attachment"), }; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: attachmentPayload, updatedAt }) @@ -35961,6 +36118,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(0, 2, "inline"), }; delete handoffPayload.attachmentIds; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: handoffPayload, updatedAt }) @@ -35971,6 +36129,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { transportPart: transportPart(1, 2, "telegram_markdown_attachment"), }; delete attachmentPayload.progressState; + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values({ companyId: current.companyId, endpointId: current.endpointId, @@ -36023,6 +36182,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { text: currentParts[0]!, transportPart: transportPart(0), }; + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ payload: rootPayload, updatedAt }) @@ -36030,6 +36190,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const { progressState: _progressState, ...nonProgressPayload } = currentPayload; + await notifyChatPublicationWork(tx); await tx.insert(chatPublications).values( currentParts.slice(1).map((text, offset) => { const index = offset + 1; @@ -36070,6 +36231,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ? classified.retryAfterMs : Math.min(60_000, 2 ** Math.max(0, attempts) * 1_000); const result = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); const [settled] = await tx .update(chatPublications) .set({ @@ -36291,50 +36453,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .select() .from(chatActions) .where( - and( - eq(chatActions.kind, "slack_file_upload_receipt"), - notExists( - db - .select({ id: chatEndpoints.id }) - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.id, chatActions.endpointId), - inArray(chatEndpoints.status, ["paused", "attention"]), - ), - ), - ), - // The normal publication owner is still polling Slack. Receipt - // recovery must neither contend with it nor let its not-yet-due - // action occupy the bounded recovery page. - notExists( - db - .select({ id: chatPublications.id }) - .from(chatPublications) - .where( - and( - sql`${chatPublications.id}::text = ${chatActions.payload}->>'publicationId'`, - sql`${chatPublications.attempts}::text = ${chatActions.payload}->>'publicationAttempt'`, - eq(chatPublications.state, "streaming"), - ), - ), - ), - or( - eq(chatActions.status, "received"), - and( - eq(chatActions.status, "failed"), - sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`, - sql`(${chatActions.result}->>'retryAt')::timestamptz <= ${selectedAt.toISOString()}::timestamptz`, - ), - and( - eq(chatActions.status, "processing"), - lte( - chatActions.updatedAt, - new Date(selectedAt.getTime() - SLACK_FILE_RECEIPT_STALE_MS), - ), - ), - ), - ), + slackReceiptEligibility(selectedAt), ) .orderBy(asc(chatActions.updatedAt), asc(chatActions.id)) .limit(limit); @@ -36342,49 +36461,55 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { for (const selected of actions) { const payload = slackFileUploadReceiptPayload(selected.payload); if (!payload) { - await db - .update(chatActions) - .set({ - status: "failed", - result: { - code: "slack_file_upload_receipt_payload_invalid", - retryable: false, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, selected.id), - eq(chatActions.kind, selected.kind), - eq(chatActions.status, selected.status), - sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, - selected.result === null - ? isNull(chatActions.result) - : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + code: "slack_file_upload_receipt_payload_invalid", + retryable: false, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, selected.id), + eq(chatActions.kind, selected.kind), + eq(chatActions.status, selected.status), + sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, + selected.result === null + ? isNull(chatActions.result) + : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, + ), + ); + }); continue; } const record = await endpointRecord(selected.endpointId); if (!record) { - await db - .update(chatActions) - .set({ - status: "cancelled", - result: { code: "slack_file_upload_receipt_endpoint_removed" }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, selected.id), - eq(chatActions.kind, selected.kind), - eq(chatActions.status, selected.status), - sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, - selected.result === null - ? isNull(chatActions.result) - : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: "cancelled", + result: { code: "slack_file_upload_receipt_endpoint_removed" }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, selected.id), + eq(chatActions.kind, selected.kind), + eq(chatActions.status, selected.status), + sql`${chatActions.payload} = ${JSON.stringify(selected.payload)}::jsonb`, + selected.result === null + ? isNull(chatActions.result) + : sql`${chatActions.result} = ${JSON.stringify(selected.result)}::jsonb`, + ), + ); + }); continue; } if (["paused", "attention"].includes(record.endpoint.status)) continue; @@ -36422,6 +36547,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (locked && !eligible) return null; if (locked?.mode === "defer") { if (locked.action.status === "processing") { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36437,6 +36563,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return null; } if (locked?.mode === "settled") { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36457,6 +36584,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { SLACK_FILE_RECEIPT_MAX_AGE_MS || attempt > SLACK_FILE_RECEIPT_MAX_ATTEMPTS ) { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36488,6 +36616,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) { return null; } + await notifyChatPublicationWork(tx); const [claimed] = await tx .update(chatActions) .set({ @@ -36530,6 +36659,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (!providerMessageId) { await db.transaction(async (tx) => { await lease.assertOwned(tx); + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36571,6 +36701,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { locked.action.result?.ownerToken !== ownerToken ) { if (!locked) { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36589,6 +36720,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ); } else if (locked.mode === "defer") { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36607,6 +36739,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ); } else { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36638,6 +36771,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { providerMessageId, })) ) { + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36653,6 +36787,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return; } if (locked.mode === "delivery_unknown") { + await notifyChatPublicationWork(tx); const [settled] = await tx .update(chatPublications) .set({ @@ -36692,6 +36827,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), }); } else { + await notifyChatPublicationWork(tx); const [enriched] = await tx .update(chatPublications) .set({ providerMessageId }) @@ -36710,6 +36846,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } } + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ @@ -36739,31 +36876,34 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { disposition.kind === "retry" ? disposition.retryAfterMs : Math.min(60_000, 1_000 * 2 ** Math.min(attempt, 8)); - await db - .update(chatActions) - .set({ - status: retryable ? "failed" : "cancelled", - result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify( - { - attempts: attempt, - code: retryable - ? "slack_file_upload_receipt_lookup_retry" - : "slack_file_upload_receipt_lookup_rejected", - retryable, - ...(retryable - ? { retryAt: new Date(Date.now() + retryMs).toISOString() } - : {}), - }, - )}::jsonb`, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, selected.id), - eq(chatActions.status, "processing"), - sql`${chatActions.result}->>'ownerToken' = ${ownerToken}`, - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: retryable ? "failed" : "cancelled", + result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify( + { + attempts: attempt, + code: retryable + ? "slack_file_upload_receipt_lookup_retry" + : "slack_file_upload_receipt_lookup_rejected", + retryable, + ...(retryable + ? { retryAt: new Date(Date.now() + retryMs).toISOString() } + : {}), + }, + )}::jsonb`, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, selected.id), + eq(chatActions.status, "processing"), + sql`${chatActions.result}->>'ownerToken' = ${ownerToken}`, + ), + ); + }); } } return processed; @@ -37163,6 +37303,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ) .then((rows) => rows[0]); if (transfer) return false; + await notifyChatPublicationWork(tx); const [row] = await tx .update(chatPublications) .set({ @@ -37290,6 +37431,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // cards must not wait behind this notice in the conversation FIFO. await db.transaction(async tx => { await guard?.assertOwned(tx); + await notifyChatPublicationWork(tx); await tx.update(chatPublications).set({ state: "cancelled", nextAttemptAt: null, redactedError: "Automatic fallback suppressed while the GitHub tool reply remains unresolved", updatedAt: new Date() }) .where(and(eq(chatPublications.id, publication.id), guard @@ -37305,6 +37447,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } else if (publication.payload.progressState === "failed") return false; const actionIds = await db.transaction(async tx => { await guard?.assertOwned(tx); + await notifyChatPublicationWork(tx); const [cancelled] = await tx.update(chatPublications).set({ state: "cancelled", nextAttemptAt: null, redactedError: "GitHub replies are sent through task-scoped tools", updatedAt: new Date() }) .where(and(eq(chatPublications.id, publication.id), eq(chatPublications.companyId, publication.companyId), @@ -37357,7 +37500,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return; } if (explicitSlackDelivery === "delivered") { - await db.update(chatPublications).set({ state: "cancelled", redactedError: "Identical response already delivered by Slack tool", updatedAt: new Date() }).where(and(eq(chatPublications.id, publication.id), inArray(chatPublications.state, ["pending", "retry"]))); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx.update(chatPublications).set({ state: "cancelled", redactedError: "Identical response already delivered by Slack tool", updatedAt: new Date() }).where(and(eq(chatPublications.id, publication.id), inArray(chatPublications.state, ["pending", "retry"]))) +; + }); return; } const earlierOpenPublication = await db @@ -37447,24 +37594,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { (await hasCommittedTaskControlCompletion(conversation.id)); if (endpoint.status === "paused" || endpoint.status === "attention") { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "pending", - attempts: publication.attempts, - // Pause can race the eligibility read. The query excludes - // inactive endpoints on the next pass, so retain only the - // original provider deadline rather than delaying Resume. - nextAttemptAt: publication.nextAttemptAt, - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "pending", + attempts: publication.attempts, + // Pause can race the eligibility read. The query excludes + // inactive endpoints on the next pass, so retain only the + // original provider deadline rather than delaying Resume. + nextAttemptAt: publication.nextAttemptAt, + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } // The setup conversation is a real end-to-end test: once provider @@ -37477,32 +37627,15 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { providerVisibleCompletion ) { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - redactedError: providerVisibleCompletion - ? "Conversation was completed before delivery" - : "External destination is no longer active", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); - return; - } - if (conversation.isDirectMessage) { - if (!endpoint.allowDirectMessages) { - await publicationLease.assertOwned(); - await db + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx .update(chatPublications) .set({ state: "cancelled", - redactedError: "Direct messages are disabled in Paperclip", + redactedError: providerVisibleCompletion + ? "Conversation was completed before delivery" + : "External destination is no longer active", updatedAt: new Date(), }) .where( @@ -37512,6 +37645,29 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { eq(chatPublications.attempts, publication.attempts + 1), ), ); + }); + return; + } + if (conversation.isDirectMessage) { + if (!endpoint.allowDirectMessages) { + await publicationLease.assertOwned(); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + redactedError: "Direct messages are disabled in Paperclip", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } } else { @@ -37530,20 +37686,23 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null; if (!nonDirectDestinationAllowed(endpoint, resource)) { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - redactedError: "Destination is disabled in Paperclip", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + redactedError: "Destination is disabled in Paperclip", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } } @@ -37566,22 +37725,25 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((result) => result[0] ?? null); if (!currentInteraction || currentInteraction.status !== "pending") { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - attempts: publication.attempts, - nextAttemptAt: null, - redactedError: "Interaction resolved before provider delivery", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + attempts: publication.attempts, + nextAttemptAt: null, + redactedError: "Interaction resolved before provider delivery", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } } @@ -37595,22 +37757,25 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null); if (progressSupersessionReason) { await publicationLease.assertOwned(); - await db - .update(chatPublications) - .set({ - state: "cancelled", - attempts: publication.attempts, - nextAttemptAt: null, - redactedError: progressSupersessionReason, - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.id, publication.id), - eq(chatPublications.state, "streaming"), - eq(chatPublications.attempts, publication.attempts + 1), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + attempts: publication.attempts, + nextAttemptAt: null, + redactedError: progressSupersessionReason, + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.id, publication.id), + eq(chatPublications.state, "streaming"), + eq(chatPublications.attempts, publication.attempts + 1), + ), + ); + }); return; } // Status is sampled when it reaches the head of the provider lane, @@ -37665,6 +37830,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (!authorizationClaim) { await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ @@ -37812,6 +37978,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); const committedAt = new Date(); + await notifyChatPublicationWork(tx); const [completedPublication] = await tx .update(chatPublications) .set({ @@ -38032,35 +38199,61 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } - async function processPendingPublications( - limit = 25, - { waitForCompletion = true }: { waitForCompletion?: boolean } = {}, - ) { - if (shuttingDown) return 0; - await enqueueInboundWakeupPublications(); - await reconcileTerminalConfirmationActions(limit); - const now = new Date(); - const staleBefore = new Date(now.getTime() - 60_000); - // A process that disappears after the provider accepted a post but before - // Paperclip persisted its message id leaves an ambiguous delivery. Never - // resend it automatically; an operator can explicitly replay after - // checking the provider conversation. - const quarantinedPublications = await db - .update(chatPublications) - .set({ - state: "delivery_unknown", - nextAttemptAt: null, - redactedError: - "Provider delivery could not be confirmed after the worker stopped. Check the external conversation before replaying.", - updatedAt: now, - }) - .where( + function slackReceiptEligibility(selectedAt?: Date) { + return and( + eq(chatActions.kind, "slack_file_upload_receipt"), + notExists( + db + .select({ id: chatEndpoints.id }) + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.id, chatActions.endpointId), + inArray(chatEndpoints.status, ["paused", "attention"]), + ), + ), + ), + // The normal publication owner is still polling Slack. Receipt + // recovery must neither contend with it nor let its not-yet-due + // action occupy the bounded recovery page. + notExists( + db + .select({ id: chatPublications.id }) + .from(chatPublications) + .where( + and( + sql`${chatPublications.id}::text = ${chatActions.payload}->>'publicationId'`, + sql`${chatPublications.attempts}::text = ${chatActions.payload}->>'publicationAttempt'`, + eq(chatPublications.state, "streaming"), + ), + ), + ), + or( + eq(chatActions.status, "received"), + and( + eq(chatActions.status, "failed"), + sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`, + selectedAt ? sql`(${chatActions.result}->>'retryAt')::timestamptz <= ${selectedAt.toISOString()}::timestamptz` : sql`${chatActions.result}->>'retryAt' is not null`, + ), + and( + eq(chatActions.status, "processing"), + selectedAt ? lte(chatActions.updatedAt, new Date(selectedAt.getTime() - SLACK_FILE_RECEIPT_STALE_MS)) : undefined, + ), + ), + ); + } + + function publicationEligibility({ now, busyEndpointIds = [], coolingTeamsPublicationIds = [], attemptedIds = [] }: { now?: Date; busyEndpointIds?: string[]; coolingTeamsPublicationIds?: string[]; attemptedIds?: string[] } = {}) { + const earlierPublication = alias(chatPublications, "earlier_chat_publications"); + return and( + sql`not (${unresolvedSlackPublicationCondition()})`, + // An explicit Board send promises both delivery and work. Keep + // its text/files pending until the durable wake has been accepted. + sql`not exists (select 1 from chat_actions a where a.company_id = ${chatPublications.companyId} and a.endpoint_id = ${chatPublications.endpointId} and a.conversation_id = ${chatPublications.conversationId} and a.kind = 'slack_board_message' and a.status = 'received' and a.payload->>'commentId' = ${chatPublications.commentId}::text)`, + or( and( sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, - eq(chatPublications.state, "streaming"), - lte(chatPublications.updatedAt, staleBefore), - // Teams owns separate staged I/O intents and a longer attempt lease. - // Only that protocol may recover/quarantine its in-flight stages. + inArray(chatPublications.state, ["pending", "retry"]), notExists( db .select({ id: chatTeamsFileTransfers.id }) @@ -38071,29 +38264,184 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { chatTeamsFileTransfers.companyId, chatPublications.companyId, ), - eq(chatTeamsFileTransfers.publicationId, chatPublications.id), - ), - ), - ), - notExists( - db - .select({ id: chatEndpointLeases.id }) - .from(chatEndpointLeases) - .where( - and( - eq(chatEndpointLeases.companyId, chatPublications.companyId), eq( - chatEndpointLeases.endpointId, - chatPublications.endpointId, + chatTeamsFileTransfers.publicationId, + chatPublications.id, ), - sql`${chatEndpointLeases.leaseKey} = 'publication:' || ${chatPublications.id}::text || ':' || ${chatPublications.attempts}::text`, - gt(chatEndpointLeases.expiresAt, now), ), ), ), ), - ) - .returning({ id: chatPublications.id }); + // Dedicated protocol work shares the endpoint concurrency cap, + // never the ordinary publication claim or resend implementation. + sql`exists (${db + .select({ id: chatTeamsFileTransfers.id }) + .from(chatTeamsFileTransfers) + .where( + and( + eq( + chatTeamsFileTransfers.companyId, + chatPublications.companyId, + ), + eq( + chatTeamsFileTransfers.endpointId, + chatPublications.endpointId, + ), + eq( + chatTeamsFileTransfers.conversationId, + chatPublications.conversationId, + ), + eq( + chatTeamsFileTransfers.publicationId, + chatPublications.id, + ), + sql`exists (${db + .select({ id: chatEndpoints.id }) + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.id, chatPublications.endpointId), + eq( + chatEndpoints.companyId, + chatPublications.companyId, + ), + eq(chatEndpoints.provider, "microsoft-teams"), + eq(chatEndpoints.status, "active"), + ), + )})`, + or( + inArray(chatTeamsFileTransfers.phase, [ + "consent_pending", + "upload_pending", + "file_info_pending", + ]), + and( + eq(chatTeamsFileTransfers.phase, "consent_unknown"), + isNotNull(chatTeamsFileTransfers.responseActivityId), + sql`${chatTeamsFileTransfers.privateState}->'response' is not null`, + ), + ), + ), + )})`, + ), + busyEndpointIds.length > 0 + ? notInArray(chatPublications.endpointId, busyEndpointIds) + : undefined, + coolingTeamsPublicationIds.length > 0 + ? notInArray(chatPublications.id, coolingTeamsPublicationIds) + : undefined, + // Inactive endpoints must not consume the eligibility page or + // acquire an artificial retry deadline. When an operator resumes + // them, due work is immediately eligible; genuine provider retry + // deadlines below remain unchanged. + notExists( + db + .select({ id: chatEndpoints.id }) + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.id, chatPublications.endpointId), + inArray(chatEndpoints.status, ["paused", "attention"]), + ), + ), + ), + now ? or(isNull(chatPublications.nextAttemptAt), lte(chatPublications.nextAttemptAt, now)) : undefined, + attemptedIds.length > 0 + ? notInArray(chatPublications.id, attemptedIds) + : undefined, + notExists( + db + .select({ id: earlierPublication.id }) + .from(earlierPublication) + .where( + and( + eq( + earlierPublication.conversationId, + chatPublications.conversationId, + ), + publicationPrecedes(earlierPublication, chatPublications), + inArray(earlierPublication.state, [ + "pending", + "retry", + "streaming", + "delivery_unknown", + ]), + ), + ), + ), + ); + } + + async function processPendingPublications( + limit = 25, + { waitForCompletion = true, maintenance = true }: { waitForCompletion?: boolean; maintenance?: boolean } = {}, + ) { + if (shuttingDown) return 0; + if (maintenance) { + await enqueueInboundWakeupPublications(); + await reconcileTerminalConfirmationActions(limit); + } + const now = new Date(); + const staleBefore = new Date(now.getTime() - 60_000); + // A process that disappears after the provider accepted a post but before + // Paperclip persisted its message id leaves an ambiguous delivery. Never + // resend it automatically; an operator can explicitly replay after + // checking the provider conversation. + const quarantinedPublications = await db.transaction(async tx => { + const condition = and( + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, + eq(chatPublications.state, "streaming"), + lte(chatPublications.updatedAt, staleBefore), + // Teams owns separate staged I/O intents and a longer attempt lease. + // Only that protocol may recover/quarantine its in-flight stages. + notExists( + db + .select({ id: chatTeamsFileTransfers.id }) + .from(chatTeamsFileTransfers) + .where( + and( + eq( + chatTeamsFileTransfers.companyId, + chatPublications.companyId, + ), + eq(chatTeamsFileTransfers.publicationId, chatPublications.id), + ), + ), + ), + notExists( + db + .select({ id: chatEndpointLeases.id }) + .from(chatEndpointLeases) + .where( + and( + eq(chatEndpointLeases.companyId, chatPublications.companyId), + eq( + chatEndpointLeases.endpointId, + chatPublications.endpointId, + ), + sql`${chatEndpointLeases.leaseKey} = 'publication:' || ${chatPublications.id}::text || ':' || ${chatPublications.attempts}::text`, + gt(chatEndpointLeases.expiresAt, now), + ), + ), + ), + ); + const candidates = await tx.select({ id: chatPublications.id }).from(chatPublications).where(condition).for("update", { skipLocked: true }); + if (!candidates.length) return []; + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "delivery_unknown", + nextAttemptAt: null, + redactedError: + "Provider delivery could not be confirmed after the worker stopped. Check the external conversation before replaying.", + updatedAt: now, + }) + .where( + and(condition, inArray(chatPublications.id, candidates.map(row => row.id))), + ) + .returning({ id: chatPublications.id }); + }); if (quarantinedPublications.length > 0) { await db .update(chatActions) @@ -38118,10 +38466,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ), ); } - const earlierPublication = alias( - chatPublications, - "earlier_chat_publications", - ); const attemptedIds: string[] = []; const ownedTasks = new Set>(); let failed = false; @@ -38144,133 +38488,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .select() .from(chatPublications) .where( - and( - // An explicit Board send promises both delivery and work. Keep - // its text/files pending until the durable wake has been accepted. - sql`not exists (select 1 from chat_actions a where a.company_id = ${chatPublications.companyId} and a.endpoint_id = ${chatPublications.endpointId} and a.conversation_id = ${chatPublications.conversationId} and a.kind = 'slack_board_message' and a.status = 'received' and a.payload->>'commentId' = ${chatPublications.commentId}::text)`, - or( - and( - sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, - inArray(chatPublications.state, ["pending", "retry"]), - notExists( - db - .select({ id: chatTeamsFileTransfers.id }) - .from(chatTeamsFileTransfers) - .where( - and( - eq( - chatTeamsFileTransfers.companyId, - chatPublications.companyId, - ), - eq( - chatTeamsFileTransfers.publicationId, - chatPublications.id, - ), - ), - ), - ), - ), - // Dedicated protocol work shares the endpoint concurrency cap, - // never the ordinary publication claim or resend implementation. - sql`exists (${db - .select({ id: chatTeamsFileTransfers.id }) - .from(chatTeamsFileTransfers) - .where( - and( - eq( - chatTeamsFileTransfers.companyId, - chatPublications.companyId, - ), - eq( - chatTeamsFileTransfers.endpointId, - chatPublications.endpointId, - ), - eq( - chatTeamsFileTransfers.conversationId, - chatPublications.conversationId, - ), - eq( - chatTeamsFileTransfers.publicationId, - chatPublications.id, - ), - sql`exists (${db - .select({ id: chatEndpoints.id }) - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.id, chatPublications.endpointId), - eq( - chatEndpoints.companyId, - chatPublications.companyId, - ), - eq(chatEndpoints.provider, "microsoft-teams"), - eq(chatEndpoints.status, "active"), - ), - )})`, - or( - inArray(chatTeamsFileTransfers.phase, [ - "consent_pending", - "upload_pending", - "file_info_pending", - ]), - and( - eq(chatTeamsFileTransfers.phase, "consent_unknown"), - isNotNull(chatTeamsFileTransfers.responseActivityId), - sql`${chatTeamsFileTransfers.privateState}->'response' is not null`, - ), - ), - ), - )})`, - ), - busyEndpointIds.length > 0 - ? notInArray(chatPublications.endpointId, busyEndpointIds) - : undefined, - coolingTeamsPublicationIds.length > 0 - ? notInArray(chatPublications.id, coolingTeamsPublicationIds) - : undefined, - // Inactive endpoints must not consume the eligibility page or - // acquire an artificial retry deadline. When an operator resumes - // them, due work is immediately eligible; genuine provider retry - // deadlines below remain unchanged. - notExists( - db - .select({ id: chatEndpoints.id }) - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.id, chatPublications.endpointId), - inArray(chatEndpoints.status, ["paused", "attention"]), - ), - ), - ), - or( - isNull(chatPublications.nextAttemptAt), - lte(chatPublications.nextAttemptAt, now), - ), - attemptedIds.length > 0 - ? notInArray(chatPublications.id, attemptedIds) - : undefined, - notExists( - db - .select({ id: earlierPublication.id }) - .from(earlierPublication) - .where( - and( - eq( - earlierPublication.conversationId, - chatPublications.conversationId, - ), - publicationPrecedes(earlierPublication, chatPublications), - inArray(earlierPublication.state, [ - "pending", - "retry", - "streaming", - "delivery_unknown", - ]), - ), - ), - ), - ), + publicationEligibility({ now, busyEndpointIds, coolingTeamsPublicationIds, attemptedIds }), ) .orderBy( asc(chatPublications.createdAt), @@ -38334,6 +38552,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { publicationEndpointTasks.delete(selectedPublication.endpointId); } ownedTasks.delete(task); + wakePublications(); }); publicationEndpointTasks.set(selectedPublication.endpointId, task); ownedTasks.add(task); @@ -38356,7 +38575,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (waitForCompletion) await Promise.all(ownedTasks); } if (failed) throw firstError; - await processPendingInteractionWakeups(limit); + if (maintenance) await processPendingInteractionWakeups(limit); return attemptedIds.length; } @@ -38376,12 +38595,103 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } - // Cron refills free endpoint slots from the durable outbox each second. - // Work remains tracked by this service and is joined before runtime shutdown. - async function schedulePendingPublications(limit = 25) { + async function processPublicationMaintenance(limit = 25) { scheduleTeamsFileMaintenance(); scheduleGitHubMaintenance(); - return processPendingPublications(limit, { waitForCompletion: false }); + await enqueueInboundWakeupPublications(); + await reconcileTerminalConfirmationActions(limit); + await processPendingInteractionWakeups(limit); + } + + // The app refills free endpoint slots on commits and durable deadlines. + // Work remains tracked by this service and is joined before runtime shutdown. + async function schedulePendingPublications(limit = 25) { + await processPublicationMaintenance(limit); + return processPendingPublications(limit, { waitForCompletion: false, maintenance: false }); + } + + let wakePublications = () => {}; + + async function nextInboundDeliveryAt(): Promise { + const rows = await db.select({ delivery: { + id: chatDeliveries.id, endpointId: chatDeliveries.endpointId, providerEventId: chatDeliveries.providerEventId, + state: chatDeliveries.state, eventKind: chatDeliveries.eventKind, + // Retain scheduling metadata only, not message bodies or attachment data. + normalizedEvent: sql`jsonb_build_object( + 'conversation', ${chatDeliveries.normalizedEvent}->'conversation', + 'lifecycle', ${chatDeliveries.normalizedEvent}->'lifecycle', + 'message', jsonb_build_object( + 'targetProviderEventId', ${chatDeliveries.normalizedEvent}->'message'->'targetProviderEventId', + 'admissionDependencyEventId', ${chatDeliveries.normalizedEvent}->'message'->'admissionDependencyEventId'))`, + }, at: sql`case + when ${chatDeliveries.state} = 'processing' then ${chatDeliveries.updatedAt} + ${DELIVERY_PROCESSING_STALE_MS} * interval '1 millisecond' + when ${chatDeliveries.state} = 'processed' then ( + select min(case when a.status = 'processing' then a.updated_at + ${DELIVERY_PROCESSING_STALE_MS} * interval '1 millisecond' + else coalesce((a.result->>'retryAt')::timestamptz, now()) end) + from chat_actions a where a.delivery_id = chat_deliveries.id + and a.company_id = chat_deliveries.company_id and a.endpoint_id = chat_deliveries.endpoint_id + and a.kind = 'inbound_wakeup' and a.status in ('issued', 'processing')) + else coalesce(${chatDeliveries.nextAttemptAt}, now()) end` }) + .from(chatDeliveries).where(and( + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.provider = 'agentmail')`, + // Provider lifecycle receipts can restore an unavailable endpoint. + // The ordinary conversation drain still waits for operator resume. + or(sql`${chatDeliveries.normalizedEvent}->'lifecycle' is not null`, + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.status in ('paused', 'attention'))`), + or(inArray(chatDeliveries.state, ["received", "retry", "processing"]), + and(eq(chatDeliveries.state, "processed"), pendingInboundWakeupCondition())), + )).orderBy(...conversationDeliveryOrder()); + // Only an ordered conversation head can run. Reaction and provider + // lifecycle receipts are independent; lifecycle edits may select their + // exact admission dependency before the chronologically earliest row. + const byEvent = new Map(rows.map(row => [`${row.delivery.endpointId}:${row.delivery.providerEventId}`, row])); + const conversations = new Set(); + let next: number | null = null; + for (let row of rows) { + const delivery = row.delivery; + const threadId = normalizedDeliveryThreadId(delivery); + if (threadId && !normalizedLifecycleEffect(delivery) && !["reaction_added", "reaction_removed"].includes(delivery.eventKind)) { + const key = conversationDrainKey(delivery.endpointId, threadId); + if (conversations.has(key)) continue; + conversations.add(key); + const dependency = normalizedLifecycleTargetEventId(delivery); + const target = dependency ? byEvent.get(`${delivery.endpointId}:${dependency}`) : undefined; + if (target && ["received", "retry", "processing"].includes(target.delivery.state)) row = target; + } + if (row.at !== null) { + const at = new Date(row.at).getTime(); + next = next === null ? at : Math.min(next, at); + } + } + return next; + } + + async function nextSlackReceiptAt(): Promise { + const [row] = await db.select({ at: sql`min(case + when ${chatActions.status} = 'processing' then ${chatActions.updatedAt} + ${SLACK_FILE_RECEIPT_STALE_MS} * interval '1 millisecond' + when ${chatActions.status} = 'failed' then (${chatActions.result}->>'retryAt')::timestamptz + else now() end)` }).from(chatActions).where(slackReceiptEligibility()); + return row?.at == null ? null : new Date(row.at).getTime(); + } + + async function nextPublicationAt(): Promise { + const cooldowns = Object.fromEntries([...teamsFileRetryAfter].map(([id, at]) => [id, new Date(at).toISOString()])); + const [queued, streaming] = await Promise.all([ + db.select({ at: sql`min(greatest(coalesce(${chatPublications.nextAttemptAt}, now()), + (${JSON.stringify(cooldowns)}::jsonb->>${chatPublications.id}::text)::timestamptz))` }) + .from(chatPublications).where(publicationEligibility({ busyEndpointIds: [...publicationEndpointTasks.keys()] })), + db.select({ at: sql`min(greatest(${chatPublications.updatedAt} + interval '60 seconds', + (select max(l.expires_at) from chat_endpoint_leases l + where l.company_id = chat_publications.company_id and l.endpoint_id = chat_publications.endpoint_id + and l.lease_key = 'publication:' || chat_publications.id::text || ':' || chat_publications.attempts::text)))` }) + .from(chatPublications).where(and( + eq(chatPublications.state, "streaming"), + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`, + sql`not exists (select 1 from ${chatTeamsFileTransfers} f where f.company_id = ${chatPublications.companyId} and f.publication_id = ${chatPublications.id})`, + )), + ]); + const deadlines = [queued[0]?.at, streaming[0]?.at].filter(at => at != null).map(at => new Date(at!).getTime()); + return deadlines.length ? Math.min(...deadlines) : null; } async function getIssueBinding( @@ -38759,6 +39069,14 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { enqueueInboundWakeupPublications, schedulePendingPublications, processPendingDeliveries, + processQueuedDeliveries: () => processPendingDeliveries(25, undefined, false), + scheduleQueuedPublications: () => processPendingPublications(25, { waitForCompletion: false, maintenance: false }), + processPendingChatMaintenance, + processPublicationMaintenance, + nextInboundDeliveryAt, + nextPublicationAt, + nextSlackReceiptAt, + onPublicationsSettled: (wake: () => void) => { wakePublications = wake; }, prepareFailedChatRunRetry, authorizeFailedChatRunRetry, processFailedChatRunRetry, diff --git a/server/src/services/chat-delivery-work.ts b/server/src/services/chat-delivery-work.ts new file mode 100644 index 00000000000..1872e3b6899 --- /dev/null +++ b/server/src/services/chat-delivery-work.ts @@ -0,0 +1,44 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; + +type ChatDeliveryService = { + processQueuedDeliveries(): Promise; + scheduleQueuedPublications(): Promise; + processPendingSlackFileUploadReceipts(): Promise; + nextInboundDeliveryAt(): Promise; + nextPublicationAt(): Promise; + nextSlackReceiptAt(): Promise; + onPublicationsSettled(wake: () => void): void; +}; + +/** Independent queue workers share the app's scheduler and DB transaction owner. */ +export function registerChatDeliveryWork( + coordinator: ReturnType, + service: ChatDeliveryService, + canRun: () => boolean, +) { + const tasks = [ + { queue: DELIVERY_QUEUES.chatInbound, run: service.processQueuedDeliveries, next: service.nextInboundDeliveryAt }, + { queue: DELIVERY_QUEUES.chatPublications, run: service.scheduleQueuedPublications, next: service.nextPublicationAt }, + { queue: DELIVERY_QUEUES.chatReceipts, run: service.processPendingSlackFileUploadReceipts, next: service.nextSlackReceiptAt }, + ]; + const workers = tasks.map(task => { + let next: number | null = null; + const worker = coordinator.register(task.queue, { + retryMs: 1000, + run: () => task.run(), + hasPending: async () => { + next = canRun() ? await task.next() : Date.now() + 1000; + return false; + }, + nextRunAt: () => next === null ? null : Math.max(Date.now() + 1000, next), + }); + if (task.queue === DELIVERY_QUEUES.chatPublications) { + // Dispatch returns after reserving endpoint slots. Refill them when a + // provider task ends, including failures before its final durable write. + service.onPublicationsSettled(worker.wake); + } + return worker; + }); + return { ready: Promise.all(workers.map(worker => worker.ready)) }; +} diff --git a/server/src/services/chat-interaction-publications.ts b/server/src/services/chat-interaction-publications.ts index cddec16dab7..c1d2574b95f 100644 --- a/server/src/services/chat-interaction-publications.ts +++ b/server/src/services/chat-interaction-publications.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { nativePhotonInteraction } from "./photon/interactions.js"; import { randomBytes } from "node:crypto"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -30,7 +31,7 @@ const QUESTION_ACTION_TOKEN_BYTES = 16; export const CHAT_QUESTION_ACTION_TOKEN_TTL_MS = 7 * 24 * 60 * 60 * 1_000; export const TELEGRAM_CALLBACK_DATA_LIMIT_BYTES = 64; -type ChatPublicationDb = Pick; +type ChatPublicationDb = Pick; function terminalNativeInteractionCopy( interaction: IssueThreadInteraction, @@ -408,83 +409,87 @@ export async function enqueueIssueInteractionChatPublications( }, }, }); - const rows = await db - .insert(chatPublications) - .values({ - companyId: interaction.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - issueId: interaction.issueId, - idempotencyKey: `interaction:${interaction.id}:${endpoint.id}`, - payload, - state: "pending", - }) - .onConflictDoNothing() - .returning(); - const publication = rows[0]; - if (publication && endpoint.provider === "imessage-photon" && nativePhotonInteraction(interaction)) { - const reference = randomBytes(9).toString("base64url"); - await db.insert(chatActions).values({ companyId: interaction.companyId, endpointId: endpoint.id, conversationId: conversation.id, - kind: "photon_interaction", providerActionId: `photon:${reference}`, - payload: { version: 1, reference, interactionId: interaction.id, publicationId: publication.id, sessionGeneration: conversation.sessionGeneration, - expiresAt: new Date(publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS).toISOString() }, status: "issued" }); - } else if (publication && formDraft) { - await db.insert(chatActions).values( - chatQuestionFormActionRecords(formDraft, { + const rows = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + const rows = await tx + .insert(chatPublications) + .values({ companyId: interaction.companyId, endpointId: endpoint.id, conversationId: conversation.id, - publicationId: publication.id, - }), - ); - } else if (publication && question && questionActionTokens.length > 0) { - const expiresAt = new Date( - publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, - ).toISOString(); - await db.insert(chatActions).values( - questionActionTokens.map(({ actionId, option }) => ({ - companyId: interaction.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - kind: "question_answer", - providerActionId: actionId, - payload: { - version: 1, + issueId: interaction.issueId, + idempotencyKey: `interaction:${interaction.id}:${endpoint.id}`, + payload, + state: "pending", + }) + .onConflictDoNothing() + .returning(); + const publication = rows[0]; + if (publication && endpoint.provider === "imessage-photon" && nativePhotonInteraction(interaction)) { + const reference = randomBytes(9).toString("base64url"); + await tx.insert(chatActions).values({ companyId: interaction.companyId, endpointId: endpoint.id, conversationId: conversation.id, + kind: "photon_interaction", providerActionId: `photon:${reference}`, + payload: { version: 1, reference, interactionId: interaction.id, publicationId: publication.id, sessionGeneration: conversation.sessionGeneration, + expiresAt: new Date(publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS).toISOString() }, status: "issued" }); + } else if (publication && formDraft) { + await tx.insert(chatActions).values( + chatQuestionFormActionRecords(formDraft, { + companyId: interaction.companyId, + endpointId: endpoint.id, + conversationId: conversation.id, publicationId: publication.id, - interactionId: interaction.id, - questionId: question.id, - optionId: option.id, - expiresAt, - }, - status: "issued", - })), - ); - } else if ( - publication && - confirmation && - confirmationActionTokens.length > 0 - ) { - const expiresAt = new Date( - publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, - ).toISOString(); - await db.insert(chatActions).values( - confirmationActionTokens.map(({ actionId, decision }) => ({ - companyId: interaction.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - kind: "confirmation_response", - providerActionId: actionId, - payload: { - version: 1, - publicationId: publication.id, - interactionId: interaction.id, - decision, - expiresAt, - }, - status: "issued", - })), - ); - } + }), + ); + } else if (publication && question && questionActionTokens.length > 0) { + const expiresAt = new Date( + publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, + ).toISOString(); + await tx.insert(chatActions).values( + questionActionTokens.map(({ actionId, option }) => ({ + companyId: interaction.companyId, + endpointId: endpoint.id, + conversationId: conversation.id, + kind: "question_answer", + providerActionId: actionId, + payload: { + version: 1, + publicationId: publication.id, + interactionId: interaction.id, + questionId: question.id, + optionId: option.id, + expiresAt, + }, + status: "issued", + })), + ); + } else if ( + publication && + confirmation && + confirmationActionTokens.length > 0 + ) { + const expiresAt = new Date( + publication.createdAt.getTime() + CHAT_QUESTION_ACTION_TOKEN_TTL_MS, + ).toISOString(); + await tx.insert(chatActions).values( + confirmationActionTokens.map(({ actionId, decision }) => ({ + companyId: interaction.companyId, + endpointId: endpoint.id, + conversationId: conversation.id, + kind: "confirmation_response", + providerActionId: actionId, + payload: { + version: 1, + publicationId: publication.id, + interactionId: interaction.id, + decision, + expiresAt, + }, + status: "issued", + })), + ); + } + return rows; + }); inserted.push(...rows); } return inserted; @@ -560,20 +565,23 @@ export async function enqueueTerminalIssueInteractionChatPublications( ) .map((publication) => publication.id); if (unsentIds.length > 0) { - await db - .update(chatPublications) - .set({ - state: "cancelled", - nextAttemptAt: null, - redactedError: "Interaction was resolved before provider publication", - updatedAt: new Date(), - }) - .where( - and( - inArray(chatPublications.id, unsentIds), - inArray(chatPublications.state, ["pending", "retry"]), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + nextAttemptAt: null, + redactedError: "Interaction was resolved before provider publication", + updatedAt: new Date(), + }) + .where( + and( + inArray(chatPublications.id, unsentIds), + inArray(chatPublications.state, ["pending", "retry"]), + ), + ); + }); } const originalIds = originals.map((publication) => publication.id); @@ -706,33 +714,37 @@ export async function enqueueTerminalIssueInteractionChatPublications( if (!copy) return []; const inserted: Array = []; for (const original of providerVisibleOriginals) { - if (!original.payload.card) continue; - const rows = await db - .insert(chatPublications) - .values({ - companyId: interaction.companyId, - endpointId: original.endpointId, - conversationId: original.conversationId, - issueId: interaction.issueId, - idempotencyKey: `interaction-resolution:${interaction.id}:${original.endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "issue_interaction", - text: copy.text, - interaction: { - id: interaction.id, - card: { - kind: original.payload.card.kind, - title: original.payload.card.title, - body: copy.body, - actions: [], + const card = original.payload.card; + if (!card) continue; + const rows = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: interaction.companyId, + endpointId: original.endpointId, + conversationId: original.conversationId, + issueId: interaction.issueId, + idempotencyKey: `interaction-resolution:${interaction.id}:${original.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "issue_interaction", + text: copy.text, + interaction: { + id: interaction.id, + card: { + kind: card.kind, + title: card.title, + body: copy.body, + actions: [], + }, }, - }, - }), - state: "pending", - }) - .onConflictDoNothing() - .returning(); + }), + state: "pending", + }) + .onConflictDoNothing() + .returning(); + }); inserted.push(...rows); } return inserted; @@ -771,23 +783,26 @@ export async function cancelPendingIssueInteractionChatPublications( ]), ), ); - return db - .update(chatPublications) - .set({ - state: "cancelled", - nextAttemptAt: null, - redactedError: "Interaction was superseded before publication", - updatedAt: new Date(), - }) - .where( - and( - eq(chatPublications.companyId, input.companyId), - eq(chatPublications.issueId, input.issueId), - inArray(chatPublications.state, ["pending", "retry"]), - inArray(sql`${chatPublications.payload}->>'interactionId'`, [ - ...input.interactionIds, - ]), - ), - ) - .returning(); + return db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatPublications) + .set({ + state: "cancelled", + nextAttemptAt: null, + redactedError: "Interaction was superseded before publication", + updatedAt: new Date(), + }) + .where( + and( + eq(chatPublications.companyId, input.companyId), + eq(chatPublications.issueId, input.issueId), + inArray(chatPublications.state, ["pending", "retry"]), + inArray(sql`${chatPublications.payload}->>'interactionId'`, [ + ...input.interactionIds, + ]), + ), + ) + .returning(); + }); } diff --git a/server/src/services/chat-run-publications.ts b/server/src/services/chat-run-publications.ts index 8ec337f76b5..f8c4575c2b3 100644 --- a/server/src/services/chat-run-publications.ts +++ b/server/src/services/chat-run-publications.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { and, asc, @@ -623,6 +624,7 @@ async function enqueueSafeNativeChatProgress( .limit(1); if (explicitlyAuthoredFinal.length > 0) return 0; + await notifyChatPublicationWork(tx); const insertedRows = await tx .insert(chatPublications) .values({ @@ -997,30 +999,33 @@ export async function enqueueChatRunMilestones( .limit(1); if (explicitlyAuthoredPublication.length > 0) continue; } - const result = await db - .insert(chatPublications) - .values({ - companyId: row.companyId, - endpointId: row.endpointId, - conversationId: row.conversationId, - issueId: row.issueId, - idempotencyKey: `run:${row.runId}:${milestone}:${row.endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "safe_milestone", - text: safeMilestoneText({ - agentName: row.agentName, - errorCode: row.runErrorCode, - milestone, - issueId: row.issueId, - publicBaseUrl: input.publicBaseUrl, + const result = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: row.companyId, + endpointId: row.endpointId, + conversationId: row.conversationId, + issueId: row.issueId, + idempotencyKey: `run:${row.runId}:${milestone}:${row.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "safe_milestone", + text: safeMilestoneText({ + agentName: row.agentName, + errorCode: row.runErrorCode, + milestone, + issueId: row.issueId, + publicBaseUrl: input.publicBaseUrl, + }), + progressState: milestone, }), - progressState: milestone, - }), - state: "pending", - }) - .onConflictDoNothing() - .returning({ id: chatPublications.id }); + state: "pending", + }) + .onConflictDoNothing() + .returning({ id: chatPublications.id }); + }); inserted += result.length; } if (rows.length < pageSize) break; diff --git a/server/src/services/chat-teams-file-publication.ts b/server/src/services/chat-teams-file-publication.ts index af3328dce32..0f33df89716 100644 --- a/server/src/services/chat-teams-file-publication.ts +++ b/server/src/services/chat-teams-file-publication.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { z } from "zod"; import { CHAT_FILE_TRANSFER_PHASES } from "@paperclipai/shared"; @@ -375,6 +376,7 @@ export async function projectTeamsFilePublication( !sameDate(publication.nextAttemptAt, nextAttemptAt) || !sameDate(publication.publishedAt, publishedAt) ) { + await notifyChatPublicationWork(tx); await tx .update(chatPublications) .set({ diff --git a/server/src/services/chat-work-notifications.ts b/server/src/services/chat-work-notifications.ts new file mode 100644 index 00000000000..f6caa18e0b9 --- /dev/null +++ b/server/src/services/chat-work-notifications.ts @@ -0,0 +1,16 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export async function notifyChatDeliveryWork(tx: object): Promise { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatInbound); +} + +export async function notifyChatPublicationWork(tx: object): Promise { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatPublications); + // Receipt recovery waits for the publication owner to release its claim. + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatReceipts); +} + +export async function notifyChatEndpointWork(tx: object): Promise { + await notifyChatDeliveryWork(tx); + await notifyChatPublicationWork(tx); +} diff --git a/server/src/services/connectors/slack-delivery.ts b/server/src/services/connectors/slack-delivery.ts index 4fcf08b8320..afbc4fbff5b 100644 --- a/server/src/services/connectors/slack-delivery.ts +++ b/server/src/services/connectors/slack-delivery.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "../chat-work-notifications.js"; import { and, eq, sql, inArray } from "drizzle-orm"; import { chatActions, type Db } from "@paperclipai/db"; import { forbidden } from "../../errors.js"; @@ -40,17 +41,20 @@ export async function slackDelivery( action.status === "processing" && Date.now() - action.updatedAt.getTime() > 5 * 60_000 ) { - const [stale] = await db - .update(chatActions) - .set({ status: "uncertain", updatedAt: new Date() }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - eq(chatActions.updatedAt, action.updatedAt), - ), - ) - .returning(); + const [stale] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "uncertain", updatedAt: new Date() }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + eq(chatActions.updatedAt, action.updatedAt), + ), + ) + .returning(); + }); if (stale) action = stale; } const args = object(action.payload.args); @@ -107,16 +111,19 @@ export async function slackDelivery( }; } if (receipt) { - const [settled] = await db - .update(chatActions) - .set({ status: "processed", result: receipt, updatedAt: new Date() }) - .where( - and( - eq(chatActions.id, action.id), - inArray(chatActions.status, ["uncertain", "processing"]), - ), - ) - .returning(); + const [settled] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "processed", result: receipt, updatedAt: new Date() }) + .where( + and( + eq(chatActions.id, action.id), + inArray(chatActions.status, ["uncertain", "processing"]), + ), + ) + .returning(); + }); if (settled) action = settled; } } diff --git a/server/src/services/connectors/slack-publication.ts b/server/src/services/connectors/slack-publication.ts index e855fb81714..661d581894b 100644 --- a/server/src/services/connectors/slack-publication.ts +++ b/server/src/services/connectors/slack-publication.ts @@ -1,81 +1,54 @@ -import { and, eq, sql, inArray } from "drizzle-orm"; -import { - chatActions, - chatConversations, - issueComments, - type chatPublications, - type Db, -} from "@paperclipai/db"; -import { object } from "./slack-client.js"; +import { sql, type SQL } from "drizzle-orm"; +import { chatPublications, type Db } from "@paperclipai/db"; -/** Explicit sends use a separate durable receipt. Suppress an identical automatic - * final, but retain distinct summaries, questions, blockers and progress. */ +// ECMAScript String.trim whitespace, matching tool-send text comparison. +const trimCharacters = "\u0009\u000a\u000b\u000c\u000d\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"; +type PublicationMatch = { + companyId: SQL; endpointId: SQL; conversationId: SQL; + issueId: SQL; commentId: SQL; payload: SQL; +}; + +function matchingSlackSend(publication: PublicationMatch) { + const p = publication; + return sql`from chat_actions send + join issue_comments comment on comment.id = ${p.commentId} + and comment.company_id = ${p.companyId} and comment.issue_id = ${p.issueId} + join chat_conversations conversation on conversation.id = ${p.conversationId} + and conversation.company_id = ${p.companyId} + where send.company_id = ${p.companyId} and send.endpoint_id = ${p.endpointId} + and send.conversation_id = ${p.conversationId} and send.kind = 'slack_tool_write' + and send.payload->'binding'->>'runId' = coalesce(comment.created_by_run_id, comment.derived_created_by_run_id)::text + and send.payload->>'name' = 'slack_post_message' + and coalesce(${p.payload}->>'interactionId', '') = '' + and coalesce(${p.payload}->>'progressState', '') = '' + and (case when jsonb_typeof(${p.payload}->'attachmentIds') = 'array' + then jsonb_array_length(${p.payload}->'attachmentIds') else 0 end) = 0 + and send.payload->'args'->>'channel' = regexp_replace(conversation.external_conversation_id, '^slack:', '') + and send.payload->'args'->>'thread_ts' = regexp_replace(conversation.external_thread_id, '^.*:', '') + and btrim(send.payload->'args'->>'text', ${trimCharacters}) = btrim(${p.payload}->>'text', ${trimCharacters})`; +} + +/** Dispatch and deadlines must agree about pending explicit Slack sends. */ +export function unresolvedSlackPublicationCondition(): SQL { + return sql`exists (select 1 ${matchingSlackSend({ + companyId: sql`${chatPublications.companyId}`, endpointId: sql`${chatPublications.endpointId}`, + conversationId: sql`${chatPublications.conversationId}`, issueId: sql`${chatPublications.issueId}`, + commentId: sql`${chatPublications.commentId}`, payload: sql`${chatPublications.payload}`, + })} and send.status in ('received', 'processing', 'uncertain'))`; +} + +/** Explicit sends suppress identical automatic finals, but not distinct content. */ export async function slackExplicitPublicationDuplicate( db: Db, publication: typeof chatPublications.$inferSelect, ) { - if ( - !publication.commentId || - publication.payload.interactionId || - publication.payload.progressState || - publication.payload.attachmentIds?.length - ) - return null; - const [comment] = await db - .select({ - runId: sql`coalesce(${issueComments.createdByRunId}, ${issueComments.derivedCreatedByRunId})`, - }) - .from(issueComments) - .where( - and( - eq(issueComments.id, publication.commentId), - eq(issueComments.companyId, publication.companyId), - eq(issueComments.issueId, publication.issueId), - ), - ); - if (!comment?.runId) return null; - const [conversation] = await db - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.id, publication.conversationId), - eq(chatConversations.companyId, publication.companyId), - ), - ); - if (!conversation) return null; - const sends = await db - .select({ payload: chatActions.payload, status: chatActions.status }) - .from(chatActions) - .where( - and( - eq(chatActions.companyId, publication.companyId), - eq(chatActions.endpointId, publication.endpointId), - eq(chatActions.conversationId, publication.conversationId), - eq(chatActions.kind, "slack_tool_write"), - inArray(chatActions.status, [ - "processed", - "received", - "processing", - "uncertain", - ]), - sql`${chatActions.payload}->'binding'->>'runId' = ${comment.runId}`, - sql`${chatActions.payload}->>'name' = 'slack_post_message'`, - ), - ); - const match = sends.find(({ payload }) => { - const args = object(payload.args); - const thread = conversation.externalThreadId.split(":").at(-1); - return ( - args.channel === - conversation.externalConversationId.replace(/^slack:/, "") && - args.thread_ts === thread && - String(args.text).trim() === publication.payload.text.trim() - ); - }); - return match - ? match.status === "processed" - ? "delivered" - : "unresolved" - : null; + const p = publication; + if (!p.commentId || p.payload.interactionId || p.payload.progressState || p.payload.attachmentIds?.length) return null; + const rows = await db.execute<{ status: string }>(sql`select send.status ${matchingSlackSend({ + companyId: sql`${p.companyId}::uuid`, endpointId: sql`${p.endpointId}::uuid`, + conversationId: sql`${p.conversationId}::uuid`, issueId: sql`${p.issueId}::uuid`, + commentId: sql`${p.commentId}::uuid`, payload: sql`${JSON.stringify(p.payload)}::jsonb`, + })} and send.status in ('processed', 'received', 'processing', 'uncertain') + order by (send.status = 'processed') asc limit 1`); + return rows[0] ? rows[0].status === "processed" ? "delivered" : "unresolved" : null; } diff --git a/server/src/services/connectors/slack-retry.ts b/server/src/services/connectors/slack-retry.ts index cea980e5d32..be07562aa87 100644 --- a/server/src/services/connectors/slack-retry.ts +++ b/server/src/services/connectors/slack-retry.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "../chat-work-notifications.js"; import { and, eq, sql, inArray } from "drizzle-orm"; import { chatActions, toolInvocations, type Db } from "@paperclipai/db"; import { @@ -52,6 +53,7 @@ export async function claimSlackRateLimitRetry( !(Date.parse(action.result.retryAt) <= Date.now()) ) return false; + await notifyChatPublicationWork(tx); await tx .update(chatActions) .set({ status: "received", updatedAt: new Date() }) diff --git a/server/src/services/connectors/slack-writes.ts b/server/src/services/connectors/slack-writes.ts index 6426ef877da..6b48d5a72ed 100644 --- a/server/src/services/connectors/slack-writes.ts +++ b/server/src/services/connectors/slack-writes.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "../chat-work-notifications.js"; import { authorizeSlackDocument } from "./slack-document.js"; import { slackMessage } from "./slack-message.js"; import { createHash } from "node:crypto"; @@ -100,28 +101,31 @@ export async function executeSlackWrite( .update(JSON.stringify({ name, args })) .digest("hex"); const key = `slack-tool:${binding.issueId}:${args.idempotencyKey}`; - const [created] = await db - .insert(chatActions) - .values({ - companyId: binding.companyId, - endpointId: authority.endpoint.id, - conversationId: authority.conversation?.id ?? null, - principalId: authority.principalId, - kind: "slack_tool_write", - providerActionId: key, - status: "received", - payload: { - name, - args, - hash, - binding, - userId: authority.userId, - revision: authority.revision, - invocationId, - }, - }) - .onConflictDoNothing() - .returning(); + const [created] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatActions) + .values({ + companyId: binding.companyId, + endpointId: authority.endpoint.id, + conversationId: authority.conversation?.id ?? null, + principalId: authority.principalId, + kind: "slack_tool_write", + providerActionId: key, + status: "received", + payload: { + name, + args, + hash, + binding, + userId: authority.userId, + revision: authority.revision, + invocationId, + }, + }) + .onConflictDoNothing() + .returning(); + }); const action = created ?? ( @@ -151,13 +155,16 @@ export async function executeSlackWrite( instruction: "Do not retry with another key. Inspect delivery before reconciling an uncertain result.", }; - const [claimed] = await db - .update(chatActions) - .set({ status: "processing", updatedAt: new Date() }) - .where( - and(eq(chatActions.id, action.id), eq(chatActions.status, "received")), - ) - .returning(); + const [claimed] = await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "processing", updatedAt: new Date() }) + .where( + and(eq(chatActions.id, action.id), eq(chatActions.status, "received")), + ) + .returning(); + }); if (!claimed) return { actionId: action.id, state: "processing" }; try { authority = await resolveSlackTaskAuthority(db, binding); @@ -252,13 +259,16 @@ export async function executeSlackWrite( ) throw forbidden("Slack returned an unsupported upload destination"); // Store the provider receipt before transport. An uncertain finish is never blindly retried. - await db - .update(chatActions) - .set({ - result: { fileId: upload.file_id, phase: "upload_allocated" }, - updatedAt: new Date(), - }) - .where(eq(chatActions.id, action.id)); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + result: { fileId: upload.file_id, phase: "upload_allocated" }, + updatedAt: new Date(), + }) + .where(eq(chatActions.id, action.id)); + }); const uploaded = await fetchImpl(url, { method: "POST", body: bytes, @@ -436,10 +446,13 @@ export async function executeSlackWrite( } : {}), }; - await db - .update(chatActions) - .set({ status: "processed", result: receipt, updatedAt: new Date() }) - .where(eq(chatActions.id, action.id)); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ status: "processed", result: receipt, updatedAt: new Date() }) + .where(eq(chatActions.id, action.id)); + }); await logActivity(db, { companyId: binding.companyId, actorType: "agent", @@ -468,19 +481,22 @@ export async function executeSlackWrite( "slack_request_timeout", "slack_service_unavailable", ].includes(details.code))); - await db - .update(chatActions) - .set({ - status: definite ? "failed" : "uncertain", - result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify({ code: details.code ?? "slack_delivery_uncertain", ...(error instanceof HttpError && error.status === 429 ? { retryAt: new Date(Date.now() + Number(details.retryAfterSeconds ?? 60) * 1000).toISOString() } : {}) })}::jsonb`, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - ), - ); + await db.transaction(async (tx) => { + await notifyChatPublicationWork(tx); + return tx + .update(chatActions) + .set({ + status: definite ? "failed" : "uncertain", + result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || ${JSON.stringify({ code: details.code ?? "slack_delivery_uncertain", ...(error instanceof HttpError && error.status === 429 ? { retryAt: new Date(Date.now() + Number(details.retryAfterSeconds ?? 60) * 1000).toISOString() } : {}) })}::jsonb`, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + ), + ); + }); if (error instanceof HttpError) error.details = { ...details, diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 42bed21ae4d..36747099889 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,9 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + chatInbound: "chat-inbound", + chatPublications: "chat-publications", + chatReceipts: "chat-slack-receipts", browser: "browser-use", email: "email-channels", feedback: "feedback-exports", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 66ec5d28d70..904a5dcf17d 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js"; import { selectHeartbeatRuntime } from "./heartbeat/runtime-selection.js"; import { readNativeComputerWorkspaceReference } from "./native-runtime/native-workspace-sync.js"; import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js"; @@ -10,7 +11,6 @@ export { import { createHeartbeatScheduling, formatIssueIdentifierLink, - parseNativeSessionGoalControl, } from "./heartbeat/scheduling.js"; import { agentExecutionsHaveStopped } from "./agent-execution-stop.js"; import { @@ -121,12 +121,6 @@ import { applyRunScopedMentionedSkillKeys, MANAGED_GITHUB_TOKEN_KEYS, configuredPaperclipApiBaseUrl, - buildPaperclipRuntimeMcpServers, - createAdapterRuntimeToolAccess, - paperclipApiBaseUrl, - createAdapterRuntimeMcpAccess, - createManagedMcpRunConfig, - revokeHeartbeatRunGatewayTokens, createHeartbeatRunPreparation, } from "./heartbeat/run-preparation.js"; export { @@ -169,9 +163,6 @@ import { buildRunWorkspaceHints, buildReferencedProjectRunObservability, assertGitSensitiveAdapterWorkspaceValid, - isWorkspaceValidationFailure, - fingerprintFinalizeWorkspaceBranchValidation, - attachPaperclipSessionMetadataToSessionParams, type EffectiveRunSessionConfigMetadata, createHeartbeatWorkspaceResolver, } from "./heartbeat/workspaces.js"; @@ -246,19 +237,14 @@ export { import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js"; export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js"; -import { recordLegacyWorkspaceRestoreFailure } from "./legacy-execution-recovery.js"; -import { - configuredEnvironmentProjection, -} from "../vendor/paperclip-runner/index.js"; import { decisionModelService } from "./decision-models.js"; import { TASK_QUESTION_GUIDANCE } from "./issue-question-context.js"; import { createAgentIdentityRedactor } from "./agent-identity-redaction.js"; import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-identity.js"; -import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils"; import { prepareConnectionInstructionDelivery } from "./connection-instructions.js"; import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/assigned-mcp-tools.js"; -import { externalObjectService } from "./external-objects.js"; + import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { @@ -271,22 +257,14 @@ import { import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./agent-directory-working-copies.js"; import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; -import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js"; import { - withNativeWorkspaceFinalizationOwnership, NativeWorkspaceFinalizationBusyError, NativeWorkspaceFinalizationOwnershipLostError, - type NativeWorkspaceFinalizationOwnership, } from "./native-runtime/native-workspace-finalization-ownership.js"; import { reserveRunBudget } from "./budget-reservations.js"; import { accountRunCost, createCostAccountingReconciler } from "./run-cost-accounting.js"; -import { createRunUsageRecorder } from "./usage-receipts.js"; -import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result"; -import { - hasWorkspaceRestoreFailure, -} from "@paperclipai/shared"; import { toolActionDeliveryService } from "./tool-action-delivery.js"; import { liveVoiceExecutionGuidance } from "./voice/voice-execution-guidance.js"; @@ -308,9 +286,6 @@ import { watchLegacyControllerLease, } from "./legacy-controller-lease.js"; -import { - remoteExecutionHasStopped, -} from "./remote-execution-termination.js"; import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js"; import { connectionIntentService } from "./connection-intents.js"; @@ -337,7 +312,6 @@ import { import { adapterExecutionControls, createAdapterExecutionControl, - registerAdapterExecutionControl, } from "./adapter-execution-control.js"; import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; @@ -359,9 +333,6 @@ import { } from "@paperclipai/adapter-utils/execution-target"; import { agentInstructionWorkingCopyService, collectStoppedInstructionCopyWithRetries, instructionWorkingCopyGuidance } from "./agent-instruction-working-copies.js"; import { preparePersistentAgentExecutionHome } from "./agent-file-store.js"; -import { - resolveManagedOpenAiBilling, -} from "@paperclipai/adapter-utils"; import fs from "node:fs/promises"; import path from "node:path"; import { execFile as execFileCallback } from "node:child_process"; @@ -400,11 +371,9 @@ import { nativeRunFinalizations, projects, projectWorkspaces, - workspaceOperations, } from "@paperclipai/db"; -import { conflict, HttpError, notFound } from "../errors.js"; +import { HttpError, notFound } from "../errors.js"; import { - getStartupTraceContext, getStartupTracer, } from "../instrumentation.js"; import { createHostDuplexObservabilityRecorder } from "./duplex-observability-recorder.js"; @@ -427,27 +396,19 @@ import { import { getTaskPlanContext } from "./task-plan-context.js"; import { - buildNativeProviderEnvironment, claimWarmNativeInstructionCopy, nativeSessionWorkspaceScope, reserveWarmNativeInstructionDirectory, dispatchNativeSessionResumptions, - executePaperclipNativeSession, - finalizeNativeRun, - isRunnerIngressAuthorized, - materializeLegacyQuestionResponseWakeProjection, NativeCancellationPendingRecoveryError, NativeControllerDetachedForRestartError, prepareNativeWorkspaceSync, readNativeWorkspaceSyncReference, - recordNativeFinalizationFailure, type NativeRestartRecoveryClaim, resolveHeartbeatNativeRuntimeMode, } from "./native-runtime/index.js"; import { - buildNativeHeartbeatPreparationSpans, - buildNativeWakeIngressSpan, recordFailedSkillPreparation, type NativeRunHistoricalSpan, } from "./native-runtime/native-run-trace.js"; @@ -461,19 +422,16 @@ import { createNativeSessionHandoffLoader } from "./native-runtime/native-sessio import { getRunLogStore, type RunLogHandle } from "./run-log-store.js"; import { providerTraceStore, - PROVIDER_TRACE_MAX_BYTES, } from "./provider-trace-store.js"; import { getServerAdapter, runningProcesses } from "../adapters/index.js"; import type { - AdapterExecutionResult, AdapterInvocationMeta, AdapterRuntimeEvent, } from "../adapters/index.js"; -import { createLocalAgentJwt } from "../agent-auth-jwt.js"; + import { createRuntimeToolsToken } from "../runtime-tools-token.js"; import { parseObject, - asNumber, } from "../adapters/utils.js"; import { @@ -525,11 +483,8 @@ import { buildWorkspaceReadyMetadata, buildWorkspaceReadyPresentation, cleanupExecutionWorkspaceArtifacts, - ensureGitWorktreeBranchCoherent, ensurePersistedExecutionWorkspaceAvailable, ensureRuntimeServicesForRun, - formatManagedGitWorktreeBranchInspection, - inspectManagedGitWorktreeBranch, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, @@ -544,10 +499,7 @@ import { } from "./workspace-instance-cleanup.js"; import { issueService } from "./issues.js"; import { - blockRunnerGoalRecovery, failRunnerGoalAction, - isRunnerGoalActionCompleted, - settleLiveRunnerGoalBeforeInterrupt, } from "./runner-goals.js"; import { @@ -963,25 +915,6 @@ export interface HeartbeatServiceOptions { }) => Promise; } -class NativeSessionResumeScheduledError extends Error { - constructor(readonly original: unknown) { - super("Native session recovery has been scheduled for the same run."); - this.name = "NativeSessionResumeScheduledError"; - } -} - -class NativeWorkspaceFinalizeScheduledError extends Error { - constructor( - readonly original: unknown, - readonly terminalFailure: boolean, - readonly reasonCode: - "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable", - ) { - super("Native workspace finalization recovery has been scheduled."); - this.name = "NativeWorkspaceFinalizeScheduledError"; - } -} - type WorkspaceReadyCommentWriter = { addComment: ( issueId: string, @@ -5682,929 +5615,106 @@ export function heartbeatService( if (!runtimeSelection.selected) return; const { nativeExecution, nativeRunnerInstanceId, getNativeFreshSessionHandoff } = runtimeSelection; nativeWorkspaceSync = runtimeSelection.nativeWorkspaceSync; - const localAgentJwtScope = - issueRef?.workMode === "skill_test" - ? { kind: "skill_test" as const, issueId: issueRef.id } - : { kind: "standard" as const }; - const authToken = - nativeRuntimeResolution.kind === "legacy" && - adapter.supportsLocalAgentJwt - ? createLocalAgentJwt( - agent.id, - agent.companyId, - agent.adapterType, - run.id, - run.responsibleUserId, - localAgentJwtScope, - ) - : null; - if ( - nativeRuntimeResolution.kind === "legacy" && - adapter.supportsLocalAgentJwt && - !authToken - ) { - logger.warn( - { - companyId: agent.companyId, - agentId: agent.id, - runId: run.id, - adapterType: agent.adapterType, - }, - "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY", - ); - } - let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; - const inspectFinalizeWorkspaceBranch = async () => { - // A remote pathname must never be inspected as a controller Git tree. - if (executionTarget?.kind === "remote" && executionTarget.transport === "computer") { - if (executionWorkspace.strategy === "git_worktree" && executionWorkspace.branchName) { - const branch = await executionTarget.runner!.execute({ command: "git", args: ["symbolic-ref", "--quiet", "--short", "HEAD"], - cwd: executionTarget.remoteCwd, timeoutMs: 10_000, bypassSession: true }); - if (branch.exitCode !== 0 || branch.timedOut || branch.stdout.trim() !== executionWorkspace.branchName) { - throw new Error("computer_workspace_branch_mismatch"); - } - } - return null; - } - const workspaceRecord = persistedExecutionWorkspace?.id - ? await executionWorkspacesSvc.getById( - persistedExecutionWorkspace.id, - ) - : persistedExecutionWorkspace; - if (workspaceRecord?.strategyType !== "git_worktree") return null; - - const worktreePath = - readNonEmptyString(workspaceRecord.providerRef) ?? - readNonEmptyString(workspaceRecord.cwd) ?? - readNonEmptyString(executionWorkspace.worktreePath) ?? - readNonEmptyString(executionWorkspace.cwd); - const expectedBranchName = - readNonEmptyString(workspaceRecord.branchName) ?? - readNonEmptyString(executionWorkspace.branchName); - if (!worktreePath || !expectedBranchName) return null; - - const inspection = await inspectManagedGitWorktreeBranch({ - worktreePath, - expectedBranchName, - }); - return { workspaceRecord, inspection }; - }; - const recordWorkspaceFinalize = async ( - status: "succeeded" | "failed", - metadata?: Record, - ) => { - if (adapterFinalizeOutcome) return; - let finalizeBranchMetadata: Record | null = null; - let finalizeBranchRepairMetadata: Record | null = - null; - if (status === "succeeded") { - const branchInspection = await inspectFinalizeWorkspaceBranch(); - if (branchInspection) { - let inspection = branchInspection.inspection; - const initialManagedGitWorktreeBranch = - formatManagedGitWorktreeBranchInspection(inspection); - if ( - !inspection.valid && - inspection.reasonCode === "branch_mismatch" && - inspection.repoRoot - ) { - let repairedExpectedBranchName = inspection.expectedBranchName; - try { - const coherence = await ensureGitWorktreeBranchCoherent({ - db, - repoRoot: inspection.repoRoot, - worktreePath: inspection.worktreePath, - expectedBranchName: inspection.expectedBranchName, - actualBranchName: inspection.actualBranchName, - sourceIssue: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - workMode: issueRef.workMode, - } - : null, - executionWorkspaceId: branchInspection.workspaceRecord.id, - heartbeatRunId: run.id, - enableWorkspaceBranchReconcileForward: - resolvedInstanceSettings.experimental - .enableWorkspaceBranchReconcileForward, - enableWorkspaceDirtyQuarantineRepair: - resolvedInstanceSettings.experimental - .enableWorkspaceDirtyQuarantineRepair, - persistForwardReconcile: false, - reconcileOperationPhase: "workspace_finalize", - recorder: workspaceOperationRecorder, - }); - if ( - coherence.branchName && - coherence.branchName !== - branchInspection.workspaceRecord.branchName - ) { - repairedExpectedBranchName = coherence.branchName; - executionWorkspace.branchName = coherence.branchName; - executionWorkspace.warnings.push(...coherence.warnings); - } - } catch (repairErr) { - const workspaceValidationFailure = - isWorkspaceValidationFailure(repairErr) ? repairErr : null; - finalizeBranchMetadata = { - executionWorkspaceId: branchInspection.workspaceRecord.id, - ...initialManagedGitWorktreeBranch, - }; - finalizeBranchRepairMetadata = { - attempted: true, - succeeded: false, - initial: initialManagedGitWorktreeBranch, - reason: - repairErr instanceof Error - ? repairErr.message - : String(repairErr), - }; - await workspaceOperationRecorder.recordOperation({ - phase: "workspace_finalize", - cwd: executionWorkspace.cwd, - metadata: { - adapterType: agent.adapterType, - executionTargetKind: executionTarget?.kind ?? "local", - ...metadata, - managedGitWorktreeBranch: finalizeBranchMetadata, - managedGitWorktreeBranchRepair: - finalizeBranchRepairMetadata, - ...(workspaceValidationFailure?.resultJson - ? { - workspaceValidation: - workspaceValidationFailure.resultJson - .workspaceValidation ?? - workspaceValidationFailure.resultJson, - } - : {}), - }, - run: async () => ({ - status: "failed", - stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`, - }), - }); - adapterFinalizeOutcome = "failed"; - throw repairErr; - } - - const repairedInspection = - await inspectManagedGitWorktreeBranch({ - worktreePath: inspection.worktreePath, - expectedBranchName: repairedExpectedBranchName, - repoRoot: inspection.repoRoot, - }); - finalizeBranchRepairMetadata = { - attempted: true, - succeeded: repairedInspection.valid, - initial: initialManagedGitWorktreeBranch, - repaired: - formatManagedGitWorktreeBranchInspection( - repairedInspection, - ), - }; - inspection = repairedInspection; - } - - const managedGitWorktreeBranch = - formatManagedGitWorktreeBranchInspection(inspection); - finalizeBranchMetadata = { - executionWorkspaceId: branchInspection.workspaceRecord.id, - ...managedGitWorktreeBranch, - }; - if (!inspection.valid) { - const workspaceValidationFingerprint = - fingerprintFinalizeWorkspaceBranchValidation({ - issueId: issueRef?.id ?? null, - executionWorkspaceId: branchInspection.workspaceRecord.id, - inspection: managedGitWorktreeBranch, - }); - await workspaceOperationRecorder.recordOperation({ - phase: "workspace_finalize", - cwd: executionWorkspace.cwd, - metadata: { - adapterType: agent.adapterType, - executionTargetKind: executionTarget?.kind ?? "local", - ...metadata, - managedGitWorktreeBranch: finalizeBranchMetadata, - ...(finalizeBranchRepairMetadata - ? { - managedGitWorktreeBranchRepair: - finalizeBranchRepairMetadata, - } - : {}), - }, - run: async () => ({ - status: "failed", - stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`, - }), - }); - adapterFinalizeOutcome = "failed"; - throw new WorkspaceValidationFailure( - `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`, - { - workspaceValidation: { - reason: "git_worktree_branch_incoherence", - fingerprint: workspaceValidationFingerprint, - adapterType: agent.adapterType, - issueId: issueRef?.id ?? null, - issueIdentifier: issueRef?.identifier ?? null, - persistedExecutionWorkspaceId: - branchInspection.workspaceRecord.id, - executionWorkspaceCwd: executionWorkspace.cwd, - managedGitWorktreeBranch: finalizeBranchMetadata, - }, - }, - ); - } - } - } - await workspaceOperationRecorder.recordOperation({ - phase: "workspace_finalize", - cwd: executionWorkspace.cwd, - metadata: { - adapterType: agent.adapterType, - executionTargetKind: executionTarget?.kind ?? "local", - ...metadata, - ...(finalizeBranchMetadata - ? { managedGitWorktreeBranch: finalizeBranchMetadata } - : {}), - ...(finalizeBranchRepairMetadata - ? { - managedGitWorktreeBranchRepair: - finalizeBranchRepairMetadata, - } - : {}), - }, - run: async () => ({ status }), - }); - // Only mark the outcome after the row landed, so a transient write - // failure on the succeeded path can still be recovered by recording - // finalize=failed from the catch path below. - adapterFinalizeOutcome = status; - }; - - const usageRecorder = await createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }); - persistUsageCaptureFailure = usageRecorder.persistFailure; - let adapterResult: AdapterExecutionResult; - const runGoalControlRequestId = readNonEmptyString( - context.goalControlRequestId, - ); - try { - if (nativeRuntimeResolution.kind === "native") { - if (!nativeExecution || !nativeRunnerInstanceId) - throw new Error("native_runtime_selection_not_persisted"); - const expectedNativeMcpDigest = - "runtimeContext" in nativeExecution && - nativeExecution.runtimeContext.mcp.bindingId - ? nativeExecution.runtimeContext.mcp.digest - : null; - const nativeMcpServers = await buildPaperclipRuntimeMcpServers({ - db, - agent, - runId: run.id, - expectedAssignmentDigest: expectedNativeMcpDigest, - }); - if ("runtimeContext" in nativeExecution) { - if (nativeMcpServers.length > 1) - throw new Error( - "native MCP realization must produce one aggregate gateway", - ); - const server = nativeMcpServers[0] ?? null; - const digest = server?.connectionId.startsWith("assignment:") - ? server.connectionId.slice("assignment:".length) - : null; - if (digest && digest !== expectedNativeMcpDigest) { - throw new Error("native MCP assignment digest mismatch"); - } - } - const nativeMcpServer = nativeMcpServers[0] ?? null; - let sessionGoalControl = parseNativeSessionGoalControl( - context.runnerGoalControl, - ); - if (runGoalControlRequestId && !sessionGoalControl) { - throw new Error("session_goal_control_payload_invalid"); - } - // A hard restart replays the heartbeat context, not a new user - // action. Do not repeat a completed create/replace/edit (which - // could reactivate or clear a goal that finished while detached). - const completedGoalControl = - sessionGoalControl !== null && - taskKey !== null && - (await isRunnerGoalActionCompleted( - db, - { - companyId: agent.companyId, - agentId: agent.id, - issueId: taskKey, - }, - sessionGoalControl.requestId, - )); - if (completedGoalControl) sessionGoalControl = null; - const nativeDispatchAtMs = Date.now(); - const runCreatedAtMs = run.createdAt.getTime(); - const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime(); - const wakeComments = Array.isArray( - parseObject(context.paperclipWake).comments, - ) - ? (parseObject(context.paperclipWake).comments as unknown[]) - : []; - const wakeIngressSpan = buildNativeWakeIngressSpan({ - runCreatedAtMs, - wakeComments, - attestedQuestionResponseAtMs, - }); - if (wakeIngressSpan) - nativeRunnerPreparationSpans.unshift(wakeIngressSpan); - nativeRunnerPreparationSpans.push( - ...buildNativeHeartbeatPreparationSpans({ - runCreatedAtMs, - runStartedAtMs, - attemptStartedAtMs, - environmentAcquireStartedAtMs, - environmentRealizeEndedAtMs, - nativeDispatchAtMs, - }), - ); - const guardedDispatch = - await dispatchResolvedInteractionContinuationWithAtomicGate( - (markDispatchStarted) => { - return executePaperclipNativeSession({ - db, - execution: nativeExecution, - getFreshSessionHandoff: getNativeFreshSessionHandoff, - refreshTools: context.refreshTools === true, - conversationMode: isConversation(issueContext), - turnTimeoutMs: Math.max(0, asNumber(runtimeConfig.timeoutSec, 0)) * 1_000, - runnerInstanceId: nativeRunnerInstanceId, - leaseOwner: runOptions.nativeLeaseOwner, - restartRecovery: runOptions.nativeRestartRecovery, - backend: - options.nativeSessionBackendFactory?.(nativeExecution), - useRunnerd: agent.adapterType === "paperclip_runner", - adapterType: agent.adapterType, - sessionGoalControl, - resumeSessionGoalHeartbeat: - context.resumeSessionGoalHeartbeat === true || - completedGoalControl, - onGoalCheckpoint: async (snapshot) => { - if (!taskKey) return; - const params = - attachPaperclipSessionMetadataToSessionParams( - { - ...runtimeSessionParamsForAdapter, - sessionId: snapshot.identity.sessionId, - cwd: executionWorkspace.cwd, - }, - configuredModel, - sessionConfigMetadata, - )!; - const displayId = - snapshot.providerSessionId ?? snapshot.sessionId; - await upsertTaskSession({ - companyId: agent.companyId, - agentId: agent.id, - adapterType: agent.adapterType, - taskKey, - sessionParamsJson: params, - sessionDisplayId: displayId, - lastRunId: run.id, - lastError: null, - }); - goalCheckpointSession.current = { params, displayId }; - }, - onLog, - onEvent: onAdapterEvent, - instructionWorkingCopy: nativeInstructionWorkingCopy(), - - onUsage: async receipt => { await usageRecorder.capture(receipt); }, - preparationSpans: nativeRunnerPreparationSpans, - // Bootstrap with executable/home discovery while keeping - // configured provider values and the server-selected - // workspace boundary authoritative. - managedGitHub: !useHostGitHub && githubSelection.configured, - billingIdentity: managedAiRuntime ? { provider: managedAiRuntime.attribution.provider, biller: managedAiRuntime.attribution.provider === "openai" ? resolveManagedOpenAiBilling(managedAiRuntime.config.managedAiRouting)?.biller ?? managedAiRuntime.attribution.provider : managedAiRuntime.attribution.provider, billingType: managedAiRuntime.attribution.method === "subscription" ? "subscription_included" : "metered_api" } : undefined, - managedAiCredentialIdentity: managedAiRuntime?.identity, - managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record).CODEX_HOME) : undefined, - dotWorkspaceRoot: nativeExecution.provider.kind === "openai_dot" && resolvedConfig.dotWorkspaceAccess === true ? executionWorkspace.cwd : undefined, - runnerEnvironment: { - ...configuredEnvironmentProjection(configuredTaskEnvironment), - ...buildNativeProviderEnvironment( - adapterEnv, - process.env, - executionWorkspace.cwd, - ), - ...buildAgentIdentityEnv(agentIdentity), - ...(instructionCopy && isAgentDirectoryCopy(instructionCopy) ? { AGENT_HOME: instructionCopy.executionRoot } : {}), - ...(nativeMcpServer - ? { - PAPERCLIP_NATIVE_MCP_NAME: nativeMcpServer.name, - PAPERCLIP_NATIVE_MCP_URL: nativeMcpServer.url, - PAPERCLIP_NATIVE_MCP_TOKEN: nativeMcpServer.token, - } - : {}), - ...(providerTraceCapture - ? { - PAPERCLIP_PROVIDER_TRACE_PATH: - providerTraceCapture.path, - PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String( - PROVIDER_TRACE_MAX_BYTES, - ), - } - : {}), - }, - runnerExecutionTarget: executionTarget, - runnerIngressAuthorized: isRunnerIngressAuthorized( - nativeRuntimeResolution, - ), - runnerPublicUrl: - runtimeEnv.PAPERCLIP_RUNNER_PUBLIC_URL?.trim() || null, - runnerCaBundlePath: - runtimeEnv.PAPERCLIP_RUNNER_CA_BUNDLE_PATH?.trim() || - null, - runnerRemoteBinaryPath: - runtimeEnv.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH?.trim() || - null, - runnerRemoteCodexPath: - runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH?.trim() || - null, - runnerRemoteCodexNpmSpec: - runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC?.trim() || - null, - runnerRemoteProviderPackPath: - runtimeEnv.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim() || - null, - stopTaskForReassignment: async (target) => { - await settleLiveRunnerGoalBeforeInterrupt(db, target); - if (!target.runId) return; - const prior = await getRun(target.runId); - if (!prior || prior.companyId !== target.companyId || prior.agentId !== target.agentId) { - throw conflict("Reassignment run binding changed"); - } - const stopped = await cancelRunInternal(target.runId, "Cancelled for task reassignment", { - errorCode: "issue_reassigned", suppressImmediateRecovery: true, - resultJson: { reassignmentStopConfirmed: true }, - }); - if (stopped && ["running", "queued", "scheduled_retry"].includes(stopped.status)) { - throw conflict("The previous run did not stop; reassignment was not applied"); - } - }, - enqueueWakeup, - syncIssueExternalObjects: externalObjectService(db, { - pluginWorkerManager: options.pluginWorkerManager, - enabled: async () => (await instanceSettings.getExperimental()).enableExternalObjects === true, - }).syncIssueSafely, - onSpawn: async (meta) => { - markDispatchStarted(); - await persistRunProcessMetadata(run.id, { ...meta, targetKind: executionTarget?.kind ?? "local" }); - }, - }); - }, - ); - if (!guardedDispatch.dispatched) return; - nativeDispatchStarted = true; - adapterResult = await guardedDispatch.resultPromise; - } else { - const interactionId = readNonEmptyString(context.interactionId); - const legacyQuestionResponse = - issueRef && - interactionId && - readNonEmptyString(context.interactionKind) === - "ask_user_questions" && - readNonEmptyString(context.interactionStatus) === "answered" - ? await materializeLegacyQuestionResponseWakeProjection({ - db, - companyId: agent.companyId, - issueId: issueRef.id, - runId: run.id, - agentId: agent.id, - interactionId, - }) - : null; - // Do not write the answer projection back to `context`: legacy - // adapters need it in their prompt, but the authoritative answers - // remain on the interaction instead of being duplicated in the - // heartbeat run snapshot. - const adapterContext: Record = { - ...context, - ...(legacyQuestionResponse - ? { - [PAPERCLIP_WAKE_PAYLOAD_KEY]: { - ...parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]), - questionResponse: legacyQuestionResponse, - }, - } - : {}), - }; - const runtimeTools = createAdapterRuntimeToolAccess({ - agentId: agent.id, - companyId: agent.companyId, - runId: run.id, - responsibleUserId: run.responsibleUserId, - }); - if (!runtimeTools) { - logger.warn( - { - companyId: agent.companyId, - agentId: agent.id, - runId: run.id, - }, - "runtime connection tools could not be delivered", - ); - } - const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({ - db, - agent, - runId: run.id, - }); - const runtimeToolDelivery = - adapter.runtimeToolDelivery ?? "invocation_context"; - if (runtimeTools && runtimeToolDelivery === "native_mcp") { - runtimeMcpServers.unshift({ - name: "Paperclip connections", - url: runtimeTools.mcpEndpoint, - token: runtimeTools.bearerToken, - connectionId: "paperclip-runtime-tools", - }); - } - if (authToken && configuredPaperclipApiBaseUrl() && issueRef) { - runtimeMcpServers.unshift({ name: "Paperclip projects", url: `${paperclipApiBaseUrl()}/api/mcp/project-tools`, - token: authToken, connectionId: "paperclip-project-tools" }); - } - const runtimeMcp = createAdapterRuntimeMcpAccess(runtimeMcpServers); - if (runtimeTools && runtimeToolDelivery === "invocation_context") { - adapterContext.paperclipRuntimeTools = runtimeTools; - } - const managedMcpConfig = await createManagedMcpRunConfig({ - db, - agent, - runId: run.id, - config: runtimeConfig, - projectId: issueRef?.projectId ?? null, - issueId: issueRef?.id ?? null, - }); - if (managedMcpConfig) { - adapterContext.paperclipManagedMcp = managedMcpConfig; - } - const guardedDispatch = - await dispatchResolvedInteractionContinuationWithAtomicGate( - (markDispatchStarted) => { - legacyAdapterEntered = true; - return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({ - getFreshSessionHandoff, - agentIdentity, - runId: run.id, - agent, - runtime: runtimeForAdapter, - config: runtimeConfig, - context: adapterContext, - executionContinuation: executionContinuation ?? null, - runtimeCommandSpec: - adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null, - executionTarget, - executionTransport: remoteExecution - ? { - remoteExecution: remoteExecution as unknown as Record< - string, - unknown - >, - } - : undefined, - runtimeMcp, - runtimeTools, - onLog, - onMeta: onAdapterMeta, - onEvent: onAdapterEvent, - onUsage: async receipt => { await usageRecorder.capture(receipt); }, - onExecutionPhase: executionControl.phases.enter, - startupTraceContext: getStartupTraceContext(), - onRuntimeProgress: async (progress) => { - await recordCurrentHeartbeatRunRuntimeProgress( - run, - progress, - issueId, - ); - }, - onProviderStopped: collectStoppedInstructions, - onDispatch: markDispatchStarted, - signal: executionControl.controller.signal, - ...(executionTarget?.kind === "remote" && (executionTarget.transport === "sandbox" || executionTarget.transport === "computer") ? { - stopRemoteStartup: async () => { - // Scope comes from the running host invocation, never agent - // config. Keep adapter ownership until setup has unwound. - if (!executionControl.controller.signal.aborted) { - throw new Error("Remote startup stop requires a cancelled run"); - } - const release = await envOrchestrator.releaseForRun({ - heartbeatRunId: run.id, - companyId: agent.companyId, - agentId: agent.id, - status: "released", - providerResourceDisposition: "stop_and_retain", - cancelActiveWork: true, - }); - if (release.errors.length || !await remoteExecutionHasStopped(db, agent.companyId, run.id)) { - throw new Error("Could not verify remote startup stopped"); - } - }, - } : {}), - onCancellationReady: async () => { - await registerAdapterExecutionControl(run.id, executionControl); - const current = await getRun(run.id); - if (!current || isHeartbeatRunTerminalStatus(current.status)) { - executionControl.controller.abort(new Error("Run stopped before provider startup")); - } - }, - onSpawn: async (meta) => { - markDispatchStarted(); - await persistRunProcessMetadata(run.id, { - pid: meta.pid, - processGroupId: - "processGroupId" in meta && - typeof meta.processGroupId === "number" - ? meta.processGroupId - : null, - startedAt: meta.startedAt, - }); - }, - authToken: authToken ?? undefined, - })); - }, - ); - if (!guardedDispatch.dispatched) return; - adapterResult = await guardedDispatch.resultPromise; - } - adapterResult = identityRedactor.redact(adapterResult); - if (run.runtimeMode === "legacy" && hasWorkspaceRestoreFailure(adapterResult.resultJson) - && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox") { - requiredWorkspaceRestoreEvidence = { - workspaceRestoreFailure: adapterResult.resultJson!.workspaceRestoreFailure, - ...(adapterResult.resultJson?.workspaceRestoreDiagnostic ? { workspaceRestoreDiagnostic: adapterResult.resultJson.workspaceRestoreDiagnostic } : {}), - }; - // Retention is the fallback even if recording this receipt fails. - providerResourceDispositionForRun = "stop_and_retain"; - await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource); - } - for (const stream of ["stdout", "stderr"] as const) { - const tail = identityRedactor.finish(stream); - if (tail) await appendIdentityRedactedLog(stream, tail); - } - if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave }; - - if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) { - const captured = await usageRecorder.complete(adapterResult); - adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete }; - } else { - // Stop may already own the terminal result. Preserve its metadata - // while durably recording the proof needed to release admission. - await db.update(heartbeatRuns).set({ costAccountingPending: true, - usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`, - }).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt))); - } - adapterResult = applyWorkspaceRestoreFailure(adapterResult); - // A returned result can include a failed restore. Keep the workspace - // barrier closed until required files have been restored. - // If recording the barrier itself fails, propagate as a run failure - // rather than silently leaving dependents stranded behind a missing - // finalize row. - const completeWorkspace = async (ownership?: NativeWorkspaceFinalizationOwnership) => { - try { - if (nativeWorkspaceSync) { - const exported = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( - eq(workspaceOperations.companyId, run.companyId), - eq(workspaceOperations.heartbeatRunId, run.id), - eq(workspaceOperations.phase, "workspace_finalize"), - eq(workspaceOperations.status, "succeeded"), - )).limit(1); - if (exported.length) adapterFinalizeOutcome = "succeeded"; - else await restoreNativeWorkspaceBestEffort({ - db, runId: run.id, assertOwnership: ownership?.assertHeld, - restore: () => nativeWorkspaceSync!.restoreWorkspace(ownership?.assertHeld), - }); - } - await ownership?.assertHeld(); - await db - .update(heartbeatRuns) - .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) - .where( - and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.status, "running"), - ), - ); - const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; - await recordWorkspaceFinalize(workspaceFinalizeStatus); - if (adapterResult.nativeFinalization) { - adapterResult.nativeFinalization.workspaceFinalizeStatus = - workspaceFinalizeStatus; - try { - const finalized = await finalizeNativeRun({ - db, - runId: run.id, - workspaceFinalizeStatus, - preserveProviderAttempt: Boolean(nativeWorkspaceSync), - }); - await dispatchPendingNativeStatusWakeups({ - companyId: run.companyId, - }); - if (finalized.phase === "committed") { - await nativeWorkspaceSync?.cleanup(); - } - } catch (finalizeErr) { - logger.warn( - { err: finalizeErr, runId: run.id }, - "native result persisted but finalization did not apply; the reconciliation loop will retry", - ); - } - } - } catch (error) { - if (ownership) { - await ownership.assertHeld(); - await recordWorkspaceFinalize("failed"); - } - throw error; - } - }; - if (nativeWorkspaceSync) { - const owned = await withNativeWorkspaceFinalizationOwnership({ - db, companyId: run.companyId, runId: run.id, - }, completeWorkspace); - if (!owned.acquired) throw new NativeWorkspaceFinalizationBusyError(); - } else { - await completeWorkspace(); - } - } catch (adapterErr) { - if (adapterErr instanceof NativeCancellationPendingRecoveryError) { - // Durable cancellation is settled by the outer recovery handler; - // it does not imply a failed workspace or a persisted run result. - throw adapterErr; - } - if (adapterErr instanceof NativeWorkspaceFinalizationBusyError - || adapterErr instanceof NativeWorkspaceFinalizationOwnershipLostError) { - nativeWorkspaceFinalizeScheduled = true; - throw adapterErr; - } - if (adapterErr instanceof NativeControllerDetachedForRestartError) { - // Preserve the provider and its run for the new controller. This - // also keeps generic teardown from terminalizing/releasing its lease. - nativeSessionResumeScheduled = true; - throw adapterErr; - } - if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) { - nativeOwnershipHeld = true; - throw adapterErr; - } - await db - .update(heartbeatRuns) - .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) - .where( - and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.status, "running"), - ), - ); - if ( - issueRef && - context.resumeSessionGoalHeartbeat === true && - !runGoalControlRequestId - ) { - await blockRunnerGoalRecovery( - db, - { - companyId: run.companyId, - issueId: issueRef.id, - agentId: agent.id, - adapterType: agent.adapterType, - }, - "provider_session_goal_recovery_failed", - ).catch(() => undefined); - } - if (issueRef && runGoalControlRequestId) { - await failRunnerGoalAction( - db, - { - companyId: run.companyId, - issueId: issueRef.id, - agentId: agent.id, - adapterType: agent.adapterType, - }, - runGoalControlRequestId, - adapterErr instanceof Error - ? adapterErr.message - : "session_goal_control_failed", - ).catch(() => undefined); - } - const nativeResumeScheduled = - nativeRuntimeResolution.kind === "native" - ? await db - .select({ - phase: nativeRunFinalizations.phase, - resultId: nativeRunFinalizations.resultId, - }) - .from(nativeRunFinalizations) - .where(eq(nativeRunFinalizations.runId, run.id)) - .limit(1) - .then( - (rows) => - rows[0]?.phase === "retryable_failure" && - rows[0]?.resultId === null, - ) - : false; - if (nativeResumeScheduled) { - nativeSessionResumeScheduled = true; - throw new NativeSessionResumeScheduledError(adapterErr); - } - // Adapter (or its restore finally) threw — or the finalize record - // write itself threw. Either way the workspace may be in a partial - // state. Best-effort record finalize=failed so the dependent readiness - // check keeps the gate closed instead of waking on stale local state, - // and surface the original error to the caller. - try { - await recordWorkspaceFinalize("failed", { - errorMessage: - adapterErr instanceof Error - ? adapterErr.message - : String(adapterErr), - }); - } catch (recordErr) { - logger.warn( - { - err: recordErr, - runId: run.id, - executionWorkspaceId: persistedExecutionWorkspace?.id ?? null, - }, - "failed to record workspace_finalize=failed operation; dependents may remain gated", - ); - } - if (nativeRuntimeResolution.kind === "native") { - const proposedResult = await db - .select({ resultId: nativeRunFinalizations.resultId }) - .from(nativeRunFinalizations) - .where(eq(nativeRunFinalizations.runId, run.id)) - .limit(1) - .then((rows) => rows[0]?.resultId ?? null); - if (proposedResult && nativeWorkspaceSync) { - const workspaceFailureMessage = - adapterErr instanceof Error ? adapterErr.message : ""; - const unrecoverable = - workspaceFailureMessage === - "workspace_sync_out_unrecoverable" || - workspaceFailureMessage.includes("daytona_sandbox_not_found"); - const failure = await recordNativeFinalizationFailure({ - db, - runId: run.id, - error: new Error( - unrecoverable - ? "native_workspace_sync_out_unrecoverable" - : "native_workspace_sync_out_failed", - ), - projectRunStatus: true, - failureScope: "workspace", - permanent: unrecoverable, - }); - nativeWorkspaceFinalizeScheduled = true; - throw new NativeWorkspaceFinalizeScheduledError( - adapterErr, - failure.phase === "terminal_failure", - unrecoverable - ? "workspace_sync_out_unrecoverable" - : "workspace_sync_out_failed", - ); - } - try { - await finalizeNativeRun({ - db, - runId: run.id, - workspaceFinalizeStatus: "failed", - }); - await dispatchPendingNativeStatusWakeups({ - companyId: run.companyId, - }); - } catch (finalizeErr) { - logger.warn( - { err: finalizeErr, runId: run.id }, - "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results", - ); - } - } - throw adapterErr; - } finally { - try { - await revokeHeartbeatRunGatewayTokens({ - db, - companyId: agent.companyId, - runId: run.id, - }); - } catch (revokeErr) { - logger.warn( - { err: revokeErr, runId: run.id, companyId: agent.companyId }, - "failed to revoke heartbeat-run MCP gateway tokens", - ); - } - await nativeInstructionReservation?.release(); - await withAdapterExecutionPhase(executionPhaseContext, "instruction_cleanup", releaseInstructionCopy); - } + const execution = await executeHeartbeatRuntime(db, { + run, agent, options, + task: { + issueRef, + issueContext, + context, + issueId, + taskKey, + executionContinuation, + }, + runtime: { + nativeRuntimeResolution, + nativeExecution, + nativeRunnerInstanceId, + adapter, + runtimeForAdapter, + getNativeFreshSessionHandoff, + getFreshSessionHandoff, + runOptions, + }, + workspace: { + persistedExecutionWorkspace, + executionWorkspace, + executionTarget, + workspaceOperationRecorder, + nativeWorkspaceSync, + workspaceRestoreSource, + remoteExecution, + }, + config: { + runtimeConfig, + resolvedConfig, + resolvedInstanceSettings, + configuredTaskEnvironment, + adapterEnv, + agentIdentity, + managedAiRuntime, + runtimeEnv, + useHostGitHub, + githubSelection, + }, + session: { + runtimeSessionParamsForAdapter, + configuredModel, + sessionConfigMetadata, + goalCheckpointSession, + }, + instructions: { + nativeInstructionWorkingCopy, + collectStoppedInstructions, + nativeInstructionReservation, + releaseInstructionCopy, + get instructionCopy() { return instructionCopy; }, + getInstructionSave: () => instructionSave, + }, + trace: { + providerTraceCapture, + nativeRunnerPreparationSpans, + attestedQuestionResponseAtMs, + attemptStartedAtMs, + environmentAcquireStartedAtMs, + environmentRealizeEndedAtMs, + }, + control: { + executionControl, + executionPhaseContext, + dispatchResolvedInteractionContinuationWithAtomicGate, + }, + output: { + identityRedactor, + appendIdentityRedactedLog, + onLog, + onAdapterEvent, + onAdapterMeta, + }, + services: { + executionWorkspacesSvc, + instanceSettings, + envOrchestrator, + upsertTaskSession, + getRun, + cancelRunInternal, + enqueueWakeup, + persistRunProcessMetadata, + recordCurrentHeartbeatRunRuntimeProgress, + dispatchPendingNativeStatusWakeups, + }, + effects: { + onUsageCaptureReady: (value) => { persistUsageCaptureFailure = value; }, + onNativeDispatchStarted: () => { nativeDispatchStarted = true; }, + onLegacyAdapterEntered: () => { legacyAdapterEntered = true; }, + onNativeWorkspaceFinalizeScheduled: () => { nativeWorkspaceFinalizeScheduled = true; }, + onNativeSessionResumeScheduled: () => { nativeSessionResumeScheduled = true; }, + onNativeOwnershipHeld: () => { nativeOwnershipHeld = true; }, + onProviderResourceDisposition: (value) => { providerResourceDispositionForRun = value; }, + onWorkspaceRestoreFailure: (evidence) => { requiredWorkspaceRestoreEvidence = evidence; }, + }, + }); + if (!execution.dispatched) return; + const { adapterResult } = execution; // Reconcile the referenced-project set against the real remote staging outcome. A referenced // project can pass authorization and clone locally at run prep, then fail to stage into the // sandbox during execution. The run-prep observability above counts such a project as synced, diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts new file mode 100644 index 00000000000..be39cf56883 --- /dev/null +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -0,0 +1,291 @@ +import { randomUUID } from "node:crypto"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import * as legacyRecovery from "../legacy-execution-recovery.js"; +import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { createAdapterExecutionControl } from "../adapter-execution-control.js"; +import { executionWorkspaceService } from "../execution-workspaces.js"; +import { workspaceOperationService } from "../workspace-operations.js"; +import { instanceSettingsService } from "../instance-settings.js"; +import { executePaperclipNativeSession, NativeCancellationPendingRecoveryError, NativeControllerDetachedForRestartError } from "../native-runtime/native-session-executor.js"; +import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js"; +import { NativeWorkspaceFinalizationBusyError, NativeWorkspaceFinalizationOwnershipLostError } from "../native-runtime/native-workspace-finalization-ownership.js"; +import { resolveHeartbeatNativeRuntimeMode } from "../native-runtime/runtime-mode.js"; +import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js"; +import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js"; +import { createHeartbeatRunState } from "./run-state.js"; +import { buildEffectiveRunSessionConfigMetadata } from "./workspaces.js"; +import { revokeHeartbeatRunGatewayTokens } from "./run-preparation.js"; +import { executeHeartbeatRuntime, type HeartbeatRuntimeExecutionInput } from "./runtime-execution.js"; + +// Keep real accounting, workspace barriers, and task-session persistence. Providers +// and gateway delivery are external boundaries; browser acceptance covers their wiring. +vi.mock("../native-runtime/native-session-executor.js", async original => ({ + ...await original(), + executePaperclipNativeSession: vi.fn(), +})); +vi.mock("./run-preparation.js", async original => ({ + ...await original(), + buildPaperclipRuntimeMcpServers: vi.fn(async () => []), + createAdapterRuntimeToolAccess: vi.fn(() => null), + createManagedMcpRunConfig: vi.fn(async () => null), + revokeHeartbeatRunGatewayTokens: vi.fn(async () => {}), +})); + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () => { + let database: Awaited>; + let db: Db; + let home: string; + beforeAll(async () => { + home = await mkdtemp(join(tmpdir(), "paperclip-runtime-execution-")); + vi.stubEnv("PAPERCLIP_HOME", home); + database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-execution"); + db = createDb(database.connectionString); + await db.execute(sql`set client_min_messages = warning`); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); }); + beforeEach(async () => { + await db.execute(sql`truncate table companies restart identity cascade`); + vi.clearAllMocks(); + vi.mocked(executePaperclipNativeSession).mockResolvedValue({ exitCode: 0, signal: null, timedOut: false }); + }); + + async function fixture(native = false) { + const [company] = await db.insert(companies).values({ name: "Runtime execution", issuePrefix: "EXE" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Executing agent", status: "running", adapterType: native ? "paperclip_runner" : "process", adapterConfig: { provider: "codex" } }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Complete this task", assigneeAgentId: agent.id }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: native ? "native" : "legacy" }).returning(); + const state = createHeartbeatRunState(db); + const executionControl = createAdapterExecutionControl(); + const adapter = { type: "process", label: "Test process", testEnvironment: vi.fn(), execute: vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false })) }; + let instructionSave: Record | null = null; + const nativeExecution = native ? buildNativeExecutionInput({ + companyId: company.id, agentId: agent.id, runId: run.id, issue, + taskPrompt: issue.title, workspace: { id: run.id, cwd: home, repoUrl: null, repoRef: null, branchName: null }, + normalizedSessionId: null, provider: "codex", runtimeContext: nativeRuntimeContextFixture(), + completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: issue.title, criteria: [{ id: "output", requirement: issue.title }] } }, + }) : null; + const input: HeartbeatRuntimeExecutionInput = { + run, agent, options: {}, + task: { issueRef: issue, issueContext: null, context: {}, issueId: issue.id, taskKey: issue.id, executionContinuation: null }, + runtime: { nativeExecution, nativeRunnerInstanceId: native ? randomUUID() : null, getNativeFreshSessionHandoff: undefined, + nativeRuntimeResolution: resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: native, runtimeConfig: {}, adapterConfig: agent.adapterConfig, agent, issue, target: null, workspaceId: null }), + adapter, runtimeForAdapter: { taskKey: issue.id, sessionId: null, sessionParams: null, sessionDisplayId: null }, getFreshSessionHandoff: undefined, runOptions: {}, + }, + workspace: { persistedExecutionWorkspace: null, executionWorkspace: { baseCwd: home, cwd: home, source: "agent_home", projectId: null, workspaceId: null, repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false }, + executionTarget: null, nativeWorkspaceSync: null, workspaceRestoreSource: undefined, remoteExecution: null, + workspaceOperationRecorder: workspaceOperationService(db).createRecorder({ companyId: company.id, heartbeatRunId: run.id, issueId: issue.id }), + }, + config: { runtimeConfig: {}, resolvedConfig: {}, managedAiRuntime: undefined, + resolvedInstanceSettings: { experimental: { enableWorkspaceBranchReconcileForward: false, enableWorkspaceDirtyQuarantineRepair: false } }, + configuredTaskEnvironment: {}, adapterEnv: {}, agentIdentity: undefined, runtimeEnv: {}, useHostGitHub: false, githubSelection: { configured: false }, + }, + session: { runtimeSessionParamsForAdapter: null, configuredModel: null, + sessionConfigMetadata: await buildEffectiveRunSessionConfigMetadata({ adapterType: agent.adapterType, effectiveAdapterConfig: {}, agentRuntimeConfig: {}, issueOverrides: null, workspaceConfig: null, environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: [] }), goalCheckpointSession: { current: null }, + }, + instructions: { instructionCopy: null, getInstructionSave: () => instructionSave, nativeInstructionWorkingCopy: () => undefined, + collectStoppedInstructions: async () => { instructionSave = { state: "saved", revisionId: "during-execution" }; }, + nativeInstructionReservation: null, releaseInstructionCopy: vi.fn(async () => {}), + }, + trace: { providerTraceCapture: null, nativeRunnerPreparationSpans: [], attestedQuestionResponseAtMs: null, attemptStartedAtMs: Date.now(), environmentAcquireStartedAtMs: Date.now(), environmentRealizeEndedAtMs: Date.now() }, + control: { executionControl, executionPhaseContext: { onExecutionPhase: executionControl.phases.enter }, + dispatchResolvedInteractionContinuationWithAtomicGate: async dispatch => ({ dispatched: true, resultPromise: dispatch(vi.fn()) }), + }, + output: { identityRedactor: createAgentIdentityRedactor(), appendIdentityRedactedLog: vi.fn(async () => {}), onLog: vi.fn(async () => {}), onAdapterEvent: vi.fn(async () => {}), onAdapterMeta: vi.fn(async () => {}) }, + services: { ...state, executionWorkspacesSvc: executionWorkspaceService(db), instanceSettings: instanceSettingsService(db), + envOrchestrator: { releaseForRun: vi.fn() }, cancelRunInternal: vi.fn(), enqueueWakeup: vi.fn(), + persistRunProcessMetadata: vi.fn(), recordCurrentHeartbeatRunRuntimeProgress: vi.fn(), dispatchPendingNativeStatusWakeups: vi.fn(), + }, + effects: { onUsageCaptureReady: vi.fn(), onNativeDispatchStarted: vi.fn(), onLegacyAdapterEntered: vi.fn(), onWorkspaceRestoreFailure: vi.fn(), onProviderResourceDisposition: vi.fn(), onNativeWorkspaceFinalizeScheduled: vi.fn(), onNativeSessionResumeScheduled: vi.fn(), onNativeOwnershipHeld: vi.fn() }, + }; + return { input, adapter, run }; + } + const read = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!); + const finalizations = () => db.select().from(workspaceOperations).where(eq(workspaceOperations.phase, "workspace_finalize")); + function expectCleanup(input: HeartbeatRuntimeExecutionInput) { + expect(revokeHeartbeatRunGatewayTokens).toHaveBeenCalledWith({ db, companyId: input.agent.companyId, runId: input.run.id }); + expect(input.instructions.releaseInstructionCopy).toHaveBeenCalledOnce(); + } + + it.each([false, true])("preserves the dispatch gate's early return and cleanup (native: %s)", async native => { + const { input, adapter } = await fixture(native); + input.control.dispatchResolvedInteractionContinuationWithAtomicGate = async () => ({ dispatched: false }); + expect(await executeHeartbeatRuntime(db, input)).toEqual({ dispatched: false }); + expect(adapter.execute).not.toHaveBeenCalled(); + expect(executePaperclipNativeSession).not.toHaveBeenCalled(); + expect(input.effects.onNativeDispatchStarted).not.toHaveBeenCalled(); + expect(input.effects.onLegacyAdapterEntered).not.toHaveBeenCalled(); + expect(await finalizations()).toHaveLength(0); + expectCleanup(input); + }); + + it("captures usage and live instruction saves before finalizing a legacy workspace", async () => { + const { input, adapter, run } = await fixture(); + adapter.execute.mockImplementationOnce(async ctx => { + expect(input.effects.onLegacyAdapterEntered).toHaveBeenCalledOnce(); + expect(input.effects.onUsageCaptureReady).toHaveBeenCalledOnce(); + await ctx.onProviderStopped?.(); + return { exitCode: 0, signal: null, timedOut: false, usage: { inputTokens: 10, outputTokens: 4 }, costUsd: 0.01, provider: "test" }; + }); + expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true, adapterResult: { usageComplete: true, resultJson: { instructionSave: { state: "saved", revisionId: "during-execution" } } } }); + expect(await read(run.id)).toMatchObject({ status: "running", costAccountingPending: true, usageJson: { inputTokens: 10, outputTokens: 4, accountingReceiptReady: true } }); + expect(await db.select().from(runUsageReceipts)).toHaveLength(1); + expect(await finalizations()).toMatchObject([{ status: "succeeded" }]); + expectCleanup(input); + }); + + it("preserves Stop metadata and marks no-work accounting without manufacturing a receipt", async () => { + const { input, adapter, run } = await fixture(); + await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { stoppedByUser: true } }).where(eq(heartbeatRuns.id, run.id)); + adapter.execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }); + await executeHeartbeatRuntime(db, input); + expect(await read(run.id)).toMatchObject({ status: "cancelled", resultJson: { stoppedByUser: true }, usageJson: { accountingProviderWorkStarted: false } }); + expect(await db.select().from(runUsageReceipts)).toHaveLength(0); + expectCleanup(input); + }); + + it("records the failed workspace and keeps the original provider error", async () => { + const { input, adapter } = await fixture(); + const error = new Error("provider failed"); + adapter.execute.mockRejectedValueOnce(error); + await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); + expect(await finalizations()).toMatchObject([{ status: "failed", metadata: { errorMessage: "provider failed" } }]); + expect(input.effects.onLegacyAdapterEntered).toHaveBeenCalledOnce(); + expectCleanup(input); + }); + + it.each([ + [new NativeCancellationPendingRecoveryError(), null], + [new NativeControllerDetachedForRestartError(), "onNativeSessionResumeScheduled"], + [new NativeRunnerOwnershipUnverifiedError(), "onNativeOwnershipHeld"], + [new NativeWorkspaceFinalizationBusyError(), "onNativeWorkspaceFinalizeScheduled"], + [new NativeWorkspaceFinalizationOwnershipLostError(), "onNativeWorkspaceFinalizeScheduled"], + ] as const)("transfers recovery ownership for %s before cleanup", async (error, effect) => { + const { input } = await fixture(true); + vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); + input.instructions.releaseInstructionCopy = vi.fn(async () => { + if (effect) expect(input.effects[effect]).toHaveBeenCalledOnce(); + }); + await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); + expect(input.effects.onNativeDispatchStarted).toHaveBeenCalledOnce(); + expect(input.effects.onLegacyAdapterEntered).not.toHaveBeenCalled(); + expect(await finalizations()).toHaveLength(0); + expectCleanup(input); + }); + + it("dispatches native execution with the selected session and persists its usage", async () => { + const { input, adapter, run } = await fixture(true); + vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => { + expect(native.execution).toBe(input.runtime.nativeExecution); + expect(native.runnerInstanceId).toBe(input.runtime.nativeRunnerInstanceId); + await native.onUsage?.({ usage: { inputTokens: 7, outputTokens: 3 }, complete: true }); + return { exitCode: 0, signal: null, timedOut: false, usageComplete: false }; + }); + expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true, adapterResult: { usage: { inputTokens: 7, outputTokens: 3 }, usageComplete: true } }); + expect(adapter.execute).not.toHaveBeenCalled(); + expect(input.effects.onNativeDispatchStarted).toHaveBeenCalledOnce(); + expect(await read(run.id)).toMatchObject({ status: "running", usageJson: { inputTokens: 7, outputTokens: 3, accountingReceiptReady: true } }); + expect(await finalizations()).toMatchObject([{ status: "succeeded" }]); + expectCleanup(input); + }); + + it("retains the same-run retry signal without recording a failed workspace", async () => { + const { input, run } = await fixture(true); + await db.update(heartbeatRuns).set({ nativeIssueId: input.task.issueId }).where(eq(heartbeatRuns.id, run.id)); + await db.insert(nativeRunFinalizations).values({ companyId: run.companyId, issueId: input.task.issueId!, runId: run.id, phase: "retryable_failure" }); + const error = new Error("session connection lost"); + vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); + await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ + name: "NativeSessionResumeScheduledError", original: error, + }); + expect(input.effects.onNativeSessionResumeScheduled).toHaveBeenCalledOnce(); + expect(await finalizations()).toHaveLength(0); + expectCleanup(input); + }); + + it("records a failed barrier if the successful finalization write fails", async () => { + const { input } = await fixture(); + const error = new Error("finalization write failed"); + vi.spyOn(input.workspace.workspaceOperationRecorder, "recordOperation").mockRejectedValueOnce(error); + await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); + expect(await finalizations()).toMatchObject([{ status: "failed" }]); + expectCleanup(input); + }); + + it.each([true, false])("validates computer worktrees remotely before finalization (matching branch: %s)", async matches => { + const { input, adapter } = await fixture(); + const remoteCwd = "/home/user/paperclip/company/projects/project/task"; + const runner: CommandManagedRuntimeRunner = { + execute: vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString(), + stdout: matches ? "paperclip/task\n" : "other-task\n", stderr: "" })), + }; + input.workspace.executionTarget = { + kind: "remote", transport: "computer", remoteCwd, runner, processRunner: runner, + listenerPort: 43100, + resourceAuthority: { kind: "computer-owner", computerId: randomUUID(), ownerId: randomUUID(), generation: 1 }, + fileAuthority: { kind: "remote-persistent", placementId: randomUUID(), root: remoteCwd, agentHome: "/home/user/paperclip/company/agents/agent" }, + launch: vi.fn(), inspectProcess: vi.fn(async () => ({ running: false, claim: null })), + retainWarm: vi.fn(), retire: vi.fn(async () => true), computerTool: { command: "computer", args: [] }, + }; + Object.assign(input.workspace.executionWorkspace, { + cwd: remoteCwd, worktreePath: remoteCwd, strategy: "git_worktree", branchName: "paperclip/task", + }); + adapter.execute.mockImplementationOnce(async ctx => { + expect(ctx.stopRemoteStartup).toBeTypeOf("function"); + await expect(ctx.stopRemoteStartup!()).rejects.toThrow("Remote startup stop requires a cancelled run"); + expect(input.services.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + return { exitCode: 0, signal: null, timedOut: false }; + }); + if (matches) { + await expect(executeHeartbeatRuntime(db, input)).resolves.toMatchObject({ dispatched: true }); + expect(await finalizations()).toMatchObject([{ status: "succeeded" }]); + } else { + await expect(executeHeartbeatRuntime(db, input)).rejects.toThrow("computer_workspace_branch_mismatch"); + expect(await finalizations()).toMatchObject([{ status: "failed" }]); + } + expect(runner.execute).toHaveBeenCalledOnce(); + expect(runner.execute).toHaveBeenCalledWith(expect.objectContaining({ + command: "git", args: ["symbolic-ref", "--quiet", "--short", "HEAD"], cwd: remoteCwd, bypassSession: true, + })); + expectCleanup(input); + }); + + it("retains a failed sandbox restore even when persisting its receipt throws", async () => { + const { input, adapter, run } = await fixture(); + input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home }; + const evidence = { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: { message: "copy-back failed" } }; + adapter.execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, resultJson: evidence }); + const order: string[] = []; + input.effects.onWorkspaceRestoreFailure = vi.fn(() => { order.push("restore evidence"); }); + input.effects.onProviderResourceDisposition = vi.fn(() => { order.push("retain sandbox"); }); + input.instructions.releaseInstructionCopy = vi.fn(async () => { order.push("cleanup"); }); + const error = new Error("restore receipt write failed"); + const record = vi.spyOn(legacyRecovery, "recordLegacyWorkspaceRestoreFailure").mockImplementationOnce(async () => { + order.push("receipt write"); + throw error; + }); + try { + await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); + expect(record).toHaveBeenCalledWith(db, run, evidence, input.workspace.workspaceRestoreSource); + expect(input.effects.onWorkspaceRestoreFailure).toHaveBeenCalledWith(evidence); + expect(input.effects.onProviderResourceDisposition).toHaveBeenCalledWith("stop_and_retain"); + expect(order).toEqual(["restore evidence", "retain sandbox", "receipt write", "cleanup"]); + expect(await finalizations()).toMatchObject([{ status: "failed" }]); + expectCleanup(input); + } finally { record.mockRestore(); } + }); + + it("runs instruction cleanup even when gateway revocation fails", async () => { + const { input } = await fixture(); + vi.mocked(revokeHeartbeatRunGatewayTokens).mockRejectedValueOnce(new Error("gateway unavailable")); + expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true }); + expectCleanup(input); + }); +}); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts new file mode 100644 index 00000000000..40c4d16f1ed --- /dev/null +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -0,0 +1,1166 @@ +import { parseNativeSessionGoalControl } from "./scheduling.js"; +import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js"; +import { + PAPERCLIP_WAKE_PAYLOAD_KEY, + configuredPaperclipApiBaseUrl, + buildPaperclipRuntimeMcpServers, + createAdapterRuntimeToolAccess, + paperclipApiBaseUrl, + createAdapterRuntimeMcpAccess, + createManagedMcpRunConfig, + revokeHeartbeatRunGatewayTokens, +} from "./run-preparation.js"; +import { + WorkspaceValidationFailure, + isWorkspaceValidationFailure, + fingerprintFinalizeWorkspaceBranchValidation, + attachPaperclipSessionMetadataToSessionParams, +} from "./workspaces.js"; +import { recordLegacyWorkspaceRestoreFailure } from "../legacy-execution-recovery.js"; +import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; +import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils"; +import { externalObjectService } from "../external-objects.js"; +import { isAgentDirectoryCopy } from "../agent-directory-working-copies.js"; +import { restoreNativeWorkspaceBestEffort } from "../native-runtime/native-workspace-best-effort.js"; +import { + withNativeWorkspaceFinalizationOwnership, + NativeWorkspaceFinalizationBusyError, + NativeWorkspaceFinalizationOwnershipLostError, + type NativeWorkspaceFinalizationOwnership, +} from "../native-runtime/native-workspace-finalization-ownership.js"; +import { createRunUsageRecorder } from "../usage-receipts.js"; +import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result"; +import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; +import { isConversation } from "../agent-conversations.js"; +import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-phase"; +import { remoteExecutionHasStopped } from "../remote-execution-termination.js"; +import { registerAdapterExecutionControl } from "../adapter-execution-control.js"; +import { resolveManagedOpenAiBilling } from "@paperclipai/adapter-utils"; +import { and, eq, isNull, sql } from "drizzle-orm"; +import { heartbeatRuns, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db"; +import { conflict } from "../../errors.js"; +import { getStartupTraceContext } from "../../instrumentation.js"; +import { logger } from "../../middleware/logger.js"; +import { + buildNativeProviderEnvironment, + executePaperclipNativeSession, + finalizeNativeRun, + isRunnerIngressAuthorized, + materializeLegacyQuestionResponseWakeProjection, + NativeCancellationPendingRecoveryError, + NativeControllerDetachedForRestartError, + recordNativeFinalizationFailure, +} from "../native-runtime/index.js"; +import { + buildNativeHeartbeatPreparationSpans, + buildNativeWakeIngressSpan, +} from "../native-runtime/native-run-trace.js"; +import { PROVIDER_TRACE_MAX_BYTES } from "../provider-trace-store.js"; +import type { AdapterExecutionResult } from "../../adapters/index.js"; +import { createLocalAgentJwt } from "../../agent-auth-jwt.js"; +import { parseObject, asNumber } from "../../adapters/utils.js"; +import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js"; +import { + ensureGitWorktreeBranchCoherent, + formatManagedGitWorktreeBranchInspection, + inspectManagedGitWorktreeBranch, +} from "../workspace-runtime.js"; +import { + blockRunnerGoalRecovery, + failRunnerGoalAction, + isRunnerGoalActionCompleted, + settleLiveRunnerGoalBeforeInterrupt, +} from "../runner-goals.js"; +import type { Db, agents, executionWorkspaces } from "@paperclipai/db"; +import type { AdapterExecutionContext, ServerAdapterModule } from "../../adapters/index.js"; +import type { HeartbeatRuntimeSelectionInput, HeartbeatRuntimeSelectionResult } from "./runtime-selection.js"; +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; +import type { createHeartbeatRunControl } from "./run-control.js"; +import type { createHeartbeatQueue } from "./queue.js"; +import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import type { createAdapterExecutionControl } from "../adapter-execution-control.js"; +import type { executionWorkspaceService } from "../execution-workspaces.js"; +import type { workspaceOperationService } from "../workspace-operations.js"; +import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; +import type { instanceSettingsService } from "../instance-settings.js"; +import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js"; +import type { ProviderResourceDisposition } from "../environment-runtime.js"; +import type { NativeExecutionInput, NativeSessionBackend } from "../../vendor/paperclip-runner/index.js"; +import type { PluginWorkerManager } from "../plugin-worker-manager.js"; +import type { RealizedExecutionWorkspace } from "../workspace-runtime.js"; +import type { reserveWarmNativeInstructionDirectory } from "../native-runtime/index.js"; + +type Selection = HeartbeatRuntimeSelectionInput; +type Selected = Extract; +type NativeSessionInput = Parameters[0]; +type RunState = ReturnType; +type Lifecycle = ReturnType; + +/** Dispatches a prepared run; admission and terminal run status remain owned by the executor. */ +export interface HeartbeatRuntimeExecutionInput { + run: typeof heartbeatRuns.$inferSelect; + agent: typeof agents.$inferSelect; + options: { + nativeSessionBackendFactory?: (execution: NativeExecutionInput) => NativeSessionBackend; + pluginWorkerManager?: PluginWorkerManager; + }; + task: Pick & { + issueRef: (NonNullable & { projectId: string | null }) | null; + issueId: string | null; + taskKey: string | null; + }; + runtime: Pick & { + nativeRuntimeResolution: Selection["nativeRuntimeResolution"]; + adapter: ServerAdapterModule; + runtimeForAdapter: AdapterExecutionContext["runtime"]; + getFreshSessionHandoff: AdapterExecutionContext["getFreshSessionHandoff"]; + runOptions: Selection["recovery"]; + }; + workspace: { + persistedExecutionWorkspace: typeof executionWorkspaces.$inferSelect | null; + executionWorkspace: RealizedExecutionWorkspace; + executionTarget: Selection["workspace"]["executionTarget"]; + workspaceOperationRecorder: ReturnType["createRecorder"]>; + nativeWorkspaceSync: Selected["nativeWorkspaceSync"]; + workspaceRestoreSource: Parameters[3]; + remoteExecution: unknown; + }; + config: Pick & { + resolvedInstanceSettings: { experimental: Pick["getExperimental"]>>, "enableWorkspaceBranchReconcileForward" | "enableWorkspaceDirtyQuarantineRepair"> }; + configuredTaskEnvironment: Parameters[0]; + adapterEnv: Parameters[0]; + agentIdentity: AdapterExecutionContext["agentIdentity"]; + runtimeEnv: Selection["runtimeEnv"]; + useHostGitHub: boolean; + githubSelection: { configured: boolean }; + }; + session: { + runtimeSessionParamsForAdapter: Record | null; + configuredModel: string | null; + sessionConfigMetadata: EffectiveRunSessionConfigMetadata; + goalCheckpointSession: { current: { params: Record; displayId: string } | null }; + }; + instructions: { + // Instruction callbacks can replace the copy and receipt while a provider is running. + readonly instructionCopy: Selection["config"]["instructionCopy"]; + getInstructionSave: () => Record | null; + nativeInstructionWorkingCopy: () => NativeSessionInput["instructionWorkingCopy"]; + collectStoppedInstructions: AdapterExecutionContext["onProviderStopped"]; + nativeInstructionReservation: Awaited>; + releaseInstructionCopy: () => Promise; + }; + trace: { + providerTraceCapture: { path: string } | null; + nativeRunnerPreparationSpans: NonNullable; + attestedQuestionResponseAtMs: Parameters[0]["attestedQuestionResponseAtMs"]; + attemptStartedAtMs: number; + environmentAcquireStartedAtMs: number; + environmentRealizeEndedAtMs: number; + }; + control: { + executionControl: ReturnType; + executionPhaseContext: Parameters[0]; + dispatchResolvedInteractionContinuationWithAtomicGate: (dispatch: (markDispatchStarted: () => void) => Promise) => Promise< + { dispatched: true; resultPromise: Promise } | { dispatched: false } + >; + }; + output: Pick & { + onAdapterEvent: NonNullable; + onAdapterMeta: AdapterExecutionContext["onMeta"]; + identityRedactor: ReturnType; + appendIdentityRedactedLog: AdapterExecutionContext["onLog"]; + }; + services: Pick & + Pick & + Pick, "cancelRunInternal"> & + Pick, "enqueueWakeup" | "dispatchPendingNativeStatusWakeups"> & { + executionWorkspacesSvc: Pick, "getById">; + instanceSettings: Pick, "getExperimental">; + envOrchestrator: Pick, "releaseForRun">; + }; + // Teardown must observe these updates immediately, including when execution later throws. + effects: { + onUsageCaptureReady: (persistFailure: () => Promise) => void; + onNativeDispatchStarted: () => void; + onLegacyAdapterEntered: () => void; + onWorkspaceRestoreFailure: (evidence: Record) => void; + onProviderResourceDisposition: (disposition: ProviderResourceDisposition) => void; + onNativeWorkspaceFinalizeScheduled: () => void; + onNativeSessionResumeScheduled: () => void; + onNativeOwnershipHeld: () => void; + }; +} + +export type HeartbeatRuntimeExecutionResult = { dispatched: false } | { + dispatched: true; + adapterResult: AdapterExecutionResult; +}; + +export class NativeSessionResumeScheduledError extends Error { + constructor(readonly original: unknown) { + super("Native session recovery has been scheduled for the same run."); + this.name = "NativeSessionResumeScheduledError"; + } +} + +export class NativeWorkspaceFinalizeScheduledError extends Error { + constructor( + readonly original: unknown, + readonly terminalFailure: boolean, + readonly reasonCode: + "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable", + ) { + super("Native workspace finalization recovery has been scheduled."); + this.name = "NativeWorkspaceFinalizeScheduledError"; + } +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExecutionInput): Promise { + const { run, agent, options } = input; + const { issueRef, issueContext, context, issueId, taskKey, executionContinuation } = input.task; + const { nativeRuntimeResolution, nativeExecution, nativeRunnerInstanceId, adapter, runtimeForAdapter, getNativeFreshSessionHandoff, getFreshSessionHandoff, runOptions } = input.runtime; + const { persistedExecutionWorkspace, executionWorkspace, executionTarget, workspaceOperationRecorder, nativeWorkspaceSync, workspaceRestoreSource, remoteExecution } = input.workspace; + const { runtimeConfig, resolvedConfig, resolvedInstanceSettings, configuredTaskEnvironment, adapterEnv, agentIdentity, managedAiRuntime, runtimeEnv, useHostGitHub, githubSelection } = input.config; + const { runtimeSessionParamsForAdapter, configuredModel, sessionConfigMetadata, goalCheckpointSession } = input.session; + const { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions; + const { providerTraceCapture, nativeRunnerPreparationSpans, attestedQuestionResponseAtMs, attemptStartedAtMs, environmentAcquireStartedAtMs, environmentRealizeEndedAtMs } = input.trace; + const { executionControl, executionPhaseContext, dispatchResolvedInteractionContinuationWithAtomicGate } = input.control; + const { identityRedactor, appendIdentityRedactedLog, onLog, onAdapterEvent, onAdapterMeta } = input.output; + const { executionWorkspacesSvc, instanceSettings, envOrchestrator, upsertTaskSession, getRun, cancelRunInternal, enqueueWakeup, persistRunProcessMetadata, recordCurrentHeartbeatRunRuntimeProgress, dispatchPendingNativeStatusWakeups } = input.services; + const localAgentJwtScope = + issueRef?.workMode === "skill_test" + ? { kind: "skill_test" as const, issueId: issueRef.id } + : { kind: "standard" as const }; + const authToken = + nativeRuntimeResolution.kind === "legacy" && + adapter.supportsLocalAgentJwt + ? createLocalAgentJwt( + agent.id, + agent.companyId, + agent.adapterType, + run.id, + run.responsibleUserId, + localAgentJwtScope, + ) + : null; + if ( + nativeRuntimeResolution.kind === "legacy" && + adapter.supportsLocalAgentJwt && + !authToken + ) { + logger.warn( + { + companyId: agent.companyId, + agentId: agent.id, + runId: run.id, + adapterType: agent.adapterType, + }, + "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY", + ); + } + let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; + const inspectFinalizeWorkspaceBranch = async () => { + // A remote pathname must never be inspected as a controller Git tree. + if (executionTarget?.kind === "remote" && executionTarget.transport === "computer") { + if (executionWorkspace.strategy === "git_worktree" && executionWorkspace.branchName) { + const branch = await executionTarget.runner!.execute({ + command: "git", + args: ["symbolic-ref", "--quiet", "--short", "HEAD"], + cwd: executionTarget.remoteCwd, + timeoutMs: 10_000, + bypassSession: true, + }); + if (branch.exitCode !== 0 || branch.timedOut || branch.stdout.trim() !== executionWorkspace.branchName) { + throw new Error("computer_workspace_branch_mismatch"); + } + } + return null; + } + const workspaceRecord = persistedExecutionWorkspace?.id + ? await executionWorkspacesSvc.getById( + persistedExecutionWorkspace.id, + ) + : persistedExecutionWorkspace; + if (workspaceRecord?.strategyType !== "git_worktree") return null; + + const worktreePath = + readNonEmptyString(workspaceRecord.providerRef) ?? + readNonEmptyString(workspaceRecord.cwd) ?? + readNonEmptyString(executionWorkspace.worktreePath) ?? + readNonEmptyString(executionWorkspace.cwd); + const expectedBranchName = + readNonEmptyString(workspaceRecord.branchName) ?? + readNonEmptyString(executionWorkspace.branchName); + if (!worktreePath || !expectedBranchName) return null; + + const inspection = await inspectManagedGitWorktreeBranch({ + worktreePath, + expectedBranchName, + }); + return { workspaceRecord, inspection }; + }; + const recordWorkspaceFinalize = async ( + status: "succeeded" | "failed", + metadata?: Record, + ) => { + if (adapterFinalizeOutcome) return; + let finalizeBranchMetadata: Record | null = null; + let finalizeBranchRepairMetadata: Record | null = + null; + if (status === "succeeded") { + const branchInspection = await inspectFinalizeWorkspaceBranch(); + if (branchInspection) { + let inspection = branchInspection.inspection; + const initialManagedGitWorktreeBranch = + formatManagedGitWorktreeBranchInspection(inspection); + if ( + !inspection.valid && + inspection.reasonCode === "branch_mismatch" && + inspection.repoRoot + ) { + let repairedExpectedBranchName = inspection.expectedBranchName; + try { + const coherence = await ensureGitWorktreeBranchCoherent({ + db, + repoRoot: inspection.repoRoot, + worktreePath: inspection.worktreePath, + expectedBranchName: inspection.expectedBranchName, + actualBranchName: inspection.actualBranchName, + sourceIssue: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + workMode: issueRef.workMode, + } + : null, + executionWorkspaceId: branchInspection.workspaceRecord.id, + heartbeatRunId: run.id, + enableWorkspaceBranchReconcileForward: + resolvedInstanceSettings.experimental + .enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental + .enableWorkspaceDirtyQuarantineRepair, + persistForwardReconcile: false, + reconcileOperationPhase: "workspace_finalize", + recorder: workspaceOperationRecorder, + }); + if ( + coherence.branchName && + coherence.branchName !== + branchInspection.workspaceRecord.branchName + ) { + repairedExpectedBranchName = coherence.branchName; + executionWorkspace.branchName = coherence.branchName; + executionWorkspace.warnings.push(...coherence.warnings); + } + } catch (repairErr) { + const workspaceValidationFailure = + isWorkspaceValidationFailure(repairErr) ? repairErr : null; + finalizeBranchMetadata = { + executionWorkspaceId: branchInspection.workspaceRecord.id, + ...initialManagedGitWorktreeBranch, + }; + finalizeBranchRepairMetadata = { + attempted: true, + succeeded: false, + initial: initialManagedGitWorktreeBranch, + reason: + repairErr instanceof Error + ? repairErr.message + : String(repairErr), + }; + await workspaceOperationRecorder.recordOperation({ + phase: "workspace_finalize", + cwd: executionWorkspace.cwd, + metadata: { + adapterType: agent.adapterType, + executionTargetKind: executionTarget?.kind ?? "local", + ...metadata, + managedGitWorktreeBranch: finalizeBranchMetadata, + managedGitWorktreeBranchRepair: + finalizeBranchRepairMetadata, + ...(workspaceValidationFailure?.resultJson + ? { + workspaceValidation: + workspaceValidationFailure.resultJson + .workspaceValidation ?? + workspaceValidationFailure.resultJson, + } + : {}), + }, + run: async () => ({ + status: "failed", + stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`, + }), + }); + adapterFinalizeOutcome = "failed"; + throw repairErr; + } + + const repairedInspection = + await inspectManagedGitWorktreeBranch({ + worktreePath: inspection.worktreePath, + expectedBranchName: repairedExpectedBranchName, + repoRoot: inspection.repoRoot, + }); + finalizeBranchRepairMetadata = { + attempted: true, + succeeded: repairedInspection.valid, + initial: initialManagedGitWorktreeBranch, + repaired: + formatManagedGitWorktreeBranchInspection( + repairedInspection, + ), + }; + inspection = repairedInspection; + } + + const managedGitWorktreeBranch = + formatManagedGitWorktreeBranchInspection(inspection); + finalizeBranchMetadata = { + executionWorkspaceId: branchInspection.workspaceRecord.id, + ...managedGitWorktreeBranch, + }; + if (!inspection.valid) { + const workspaceValidationFingerprint = + fingerprintFinalizeWorkspaceBranchValidation({ + issueId: issueRef?.id ?? null, + executionWorkspaceId: branchInspection.workspaceRecord.id, + inspection: managedGitWorktreeBranch, + }); + await workspaceOperationRecorder.recordOperation({ + phase: "workspace_finalize", + cwd: executionWorkspace.cwd, + metadata: { + adapterType: agent.adapterType, + executionTargetKind: executionTarget?.kind ?? "local", + ...metadata, + managedGitWorktreeBranch: finalizeBranchMetadata, + ...(finalizeBranchRepairMetadata + ? { + managedGitWorktreeBranchRepair: + finalizeBranchRepairMetadata, + } + : {}), + }, + run: async () => ({ + status: "failed", + stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`, + }), + }); + adapterFinalizeOutcome = "failed"; + throw new WorkspaceValidationFailure( + `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`, + { + workspaceValidation: { + reason: "git_worktree_branch_incoherence", + fingerprint: workspaceValidationFingerprint, + adapterType: agent.adapterType, + issueId: issueRef?.id ?? null, + issueIdentifier: issueRef?.identifier ?? null, + persistedExecutionWorkspaceId: + branchInspection.workspaceRecord.id, + executionWorkspaceCwd: executionWorkspace.cwd, + managedGitWorktreeBranch: finalizeBranchMetadata, + }, + }, + ); + } + } + } + await workspaceOperationRecorder.recordOperation({ + phase: "workspace_finalize", + cwd: executionWorkspace.cwd, + metadata: { + adapterType: agent.adapterType, + executionTargetKind: executionTarget?.kind ?? "local", + ...metadata, + ...(finalizeBranchMetadata + ? { managedGitWorktreeBranch: finalizeBranchMetadata } + : {}), + ...(finalizeBranchRepairMetadata + ? { + managedGitWorktreeBranchRepair: + finalizeBranchRepairMetadata, + } + : {}), + }, + run: async () => ({ status }), + }); + // Only mark the outcome after the row landed, so a transient write + // failure on the succeeded path can still be recovered by recording + // finalize=failed from the catch path below. + adapterFinalizeOutcome = status; + }; + + const usageRecorder = await createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }); + input.effects.onUsageCaptureReady(usageRecorder.persistFailure); + let adapterResult: AdapterExecutionResult; + const runGoalControlRequestId = readNonEmptyString( + context.goalControlRequestId, + ); + try { + if (nativeRuntimeResolution.kind === "native") { + if (!nativeExecution || !nativeRunnerInstanceId) + throw new Error("native_runtime_selection_not_persisted"); + const expectedNativeMcpDigest = + "runtimeContext" in nativeExecution && + nativeExecution.runtimeContext.mcp.bindingId + ? nativeExecution.runtimeContext.mcp.digest + : null; + const nativeMcpServers = await buildPaperclipRuntimeMcpServers({ + db, + agent, + runId: run.id, + expectedAssignmentDigest: expectedNativeMcpDigest, + }); + if ("runtimeContext" in nativeExecution) { + if (nativeMcpServers.length > 1) + throw new Error( + "native MCP realization must produce one aggregate gateway", + ); + const server = nativeMcpServers[0] ?? null; + const digest = server?.connectionId.startsWith("assignment:") + ? server.connectionId.slice("assignment:".length) + : null; + if (digest && digest !== expectedNativeMcpDigest) { + throw new Error("native MCP assignment digest mismatch"); + } + } + const nativeMcpServer = nativeMcpServers[0] ?? null; + let sessionGoalControl = parseNativeSessionGoalControl( + context.runnerGoalControl, + ); + if (runGoalControlRequestId && !sessionGoalControl) { + throw new Error("session_goal_control_payload_invalid"); + } + // A hard restart replays the heartbeat context, not a new user + // action. Do not repeat a completed create/replace/edit (which + // could reactivate or clear a goal that finished while detached). + const completedGoalControl = + sessionGoalControl !== null && + taskKey !== null && + (await isRunnerGoalActionCompleted( + db, + { + companyId: agent.companyId, + agentId: agent.id, + issueId: taskKey, + }, + sessionGoalControl.requestId, + )); + if (completedGoalControl) sessionGoalControl = null; + const nativeDispatchAtMs = Date.now(); + const runCreatedAtMs = run.createdAt.getTime(); + const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime(); + const wakeComments = Array.isArray( + parseObject(context.paperclipWake).comments, + ) + ? (parseObject(context.paperclipWake).comments as unknown[]) + : []; + const wakeIngressSpan = buildNativeWakeIngressSpan({ + runCreatedAtMs, + wakeComments, + attestedQuestionResponseAtMs, + }); + if (wakeIngressSpan) + nativeRunnerPreparationSpans.unshift(wakeIngressSpan); + nativeRunnerPreparationSpans.push( + ...buildNativeHeartbeatPreparationSpans({ + runCreatedAtMs, + runStartedAtMs, + attemptStartedAtMs, + environmentAcquireStartedAtMs, + environmentRealizeEndedAtMs, + nativeDispatchAtMs, + }), + ); + const guardedDispatch = + await dispatchResolvedInteractionContinuationWithAtomicGate( + (markDispatchStarted) => { + return executePaperclipNativeSession({ + db, + execution: nativeExecution, + getFreshSessionHandoff: getNativeFreshSessionHandoff, + refreshTools: context.refreshTools === true, + conversationMode: isConversation(issueContext), + turnTimeoutMs: Math.max(0, asNumber(runtimeConfig.timeoutSec, 0)) * 1_000, + runnerInstanceId: nativeRunnerInstanceId, + leaseOwner: runOptions.nativeLeaseOwner, + restartRecovery: runOptions.nativeRestartRecovery, + backend: + options.nativeSessionBackendFactory?.(nativeExecution), + useRunnerd: agent.adapterType === "paperclip_runner", + adapterType: agent.adapterType, + sessionGoalControl, + resumeSessionGoalHeartbeat: + context.resumeSessionGoalHeartbeat === true || + completedGoalControl, + onGoalCheckpoint: async (snapshot) => { + if (!taskKey) return; + const params = + attachPaperclipSessionMetadataToSessionParams( + { + ...runtimeSessionParamsForAdapter, + sessionId: snapshot.identity.sessionId, + cwd: executionWorkspace.cwd, + }, + configuredModel, + sessionConfigMetadata, + )!; + const displayId = + snapshot.providerSessionId ?? snapshot.sessionId; + await upsertTaskSession({ + companyId: agent.companyId, + agentId: agent.id, + adapterType: agent.adapterType, + taskKey, + sessionParamsJson: params, + sessionDisplayId: displayId, + lastRunId: run.id, + lastError: null, + }); + goalCheckpointSession.current = { params, displayId }; + }, + onLog, + onEvent: onAdapterEvent, + instructionWorkingCopy: nativeInstructionWorkingCopy(), + + onUsage: async receipt => { await usageRecorder.capture(receipt); }, + preparationSpans: nativeRunnerPreparationSpans, + // Bootstrap with executable/home discovery while keeping + // configured provider values and the server-selected + // workspace boundary authoritative. + managedGitHub: !useHostGitHub && githubSelection.configured, + billingIdentity: managedAiRuntime ? { provider: managedAiRuntime.attribution.provider, biller: managedAiRuntime.attribution.provider === "openai" ? resolveManagedOpenAiBilling(managedAiRuntime.config.managedAiRouting)?.biller ?? managedAiRuntime.attribution.provider : managedAiRuntime.attribution.provider, billingType: managedAiRuntime.attribution.method === "subscription" ? "subscription_included" : "metered_api" } : undefined, + managedAiCredentialIdentity: managedAiRuntime?.identity, + managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record).CODEX_HOME) : undefined, + dotWorkspaceRoot: nativeExecution.provider.kind === "openai_dot" && resolvedConfig.dotWorkspaceAccess === true ? executionWorkspace.cwd : undefined, + runnerEnvironment: { + ...configuredEnvironmentProjection(configuredTaskEnvironment), + ...buildNativeProviderEnvironment( + adapterEnv, + process.env, + executionWorkspace.cwd, + ), + ...buildAgentIdentityEnv(agentIdentity), + ...(input.instructions.instructionCopy && isAgentDirectoryCopy(input.instructions.instructionCopy) ? { AGENT_HOME: input.instructions.instructionCopy.executionRoot } : {}), + ...(nativeMcpServer + ? { + PAPERCLIP_NATIVE_MCP_NAME: nativeMcpServer.name, + PAPERCLIP_NATIVE_MCP_URL: nativeMcpServer.url, + PAPERCLIP_NATIVE_MCP_TOKEN: nativeMcpServer.token, + } + : {}), + ...(providerTraceCapture + ? { + PAPERCLIP_PROVIDER_TRACE_PATH: + providerTraceCapture.path, + PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String( + PROVIDER_TRACE_MAX_BYTES, + ), + } + : {}), + }, + runnerExecutionTarget: executionTarget, + runnerIngressAuthorized: isRunnerIngressAuthorized( + nativeRuntimeResolution, + ), + runnerPublicUrl: + runtimeEnv.PAPERCLIP_RUNNER_PUBLIC_URL?.trim() || null, + runnerCaBundlePath: + runtimeEnv.PAPERCLIP_RUNNER_CA_BUNDLE_PATH?.trim() || + null, + runnerRemoteBinaryPath: + runtimeEnv.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH?.trim() || + null, + runnerRemoteCodexPath: + runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH?.trim() || + null, + runnerRemoteCodexNpmSpec: + runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC?.trim() || + null, + runnerRemoteProviderPackPath: + runtimeEnv.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim() || + null, + stopTaskForReassignment: async (target) => { + await settleLiveRunnerGoalBeforeInterrupt(db, target); + if (!target.runId) return; + const prior = await getRun(target.runId); + if (!prior || prior.companyId !== target.companyId || prior.agentId !== target.agentId) { + throw conflict("Reassignment run binding changed"); + } + const stopped = await cancelRunInternal(target.runId, "Cancelled for task reassignment", { + errorCode: "issue_reassigned", suppressImmediateRecovery: true, + resultJson: { reassignmentStopConfirmed: true }, + }); + if (stopped && ["running", "queued", "scheduled_retry"].includes(stopped.status)) { + throw conflict("The previous run did not stop; reassignment was not applied"); + } + }, + enqueueWakeup, + syncIssueExternalObjects: externalObjectService(db, { + pluginWorkerManager: options.pluginWorkerManager, + enabled: async () => (await instanceSettings.getExperimental()).enableExternalObjects === true, + }).syncIssueSafely, + onSpawn: async (meta) => { + markDispatchStarted(); + await persistRunProcessMetadata(run.id, { ...meta, targetKind: executionTarget?.kind ?? "local" }); + }, + }); + }, + ); + if (!guardedDispatch.dispatched) return { dispatched: false }; + input.effects.onNativeDispatchStarted(); + adapterResult = await guardedDispatch.resultPromise; + } else { + const interactionId = readNonEmptyString(context.interactionId); + const legacyQuestionResponse = + issueRef && + interactionId && + readNonEmptyString(context.interactionKind) === + "ask_user_questions" && + readNonEmptyString(context.interactionStatus) === "answered" + ? await materializeLegacyQuestionResponseWakeProjection({ + db, + companyId: agent.companyId, + issueId: issueRef.id, + runId: run.id, + agentId: agent.id, + interactionId, + }) + : null; + // Do not write the answer projection back to `context`: legacy + // adapters need it in their prompt, but the authoritative answers + // remain on the interaction instead of being duplicated in the + // heartbeat run snapshot. + const adapterContext: Record = { + ...context, + ...(legacyQuestionResponse + ? { + [PAPERCLIP_WAKE_PAYLOAD_KEY]: { + ...parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]), + questionResponse: legacyQuestionResponse, + }, + } + : {}), + }; + const runtimeTools = createAdapterRuntimeToolAccess({ + agentId: agent.id, + companyId: agent.companyId, + runId: run.id, + responsibleUserId: run.responsibleUserId, + }); + if (!runtimeTools) { + logger.warn( + { + companyId: agent.companyId, + agentId: agent.id, + runId: run.id, + }, + "runtime connection tools could not be delivered", + ); + } + const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({ + db, + agent, + runId: run.id, + }); + const runtimeToolDelivery = + adapter.runtimeToolDelivery ?? "invocation_context"; + if (runtimeTools && runtimeToolDelivery === "native_mcp") { + runtimeMcpServers.unshift({ + name: "Paperclip connections", + url: runtimeTools.mcpEndpoint, + token: runtimeTools.bearerToken, + connectionId: "paperclip-runtime-tools", + }); + } + if (authToken && configuredPaperclipApiBaseUrl() && issueRef) { + runtimeMcpServers.unshift({ name: "Paperclip projects", url: `${paperclipApiBaseUrl()}/api/mcp/project-tools`, + token: authToken, connectionId: "paperclip-project-tools" }); + } + const runtimeMcp = createAdapterRuntimeMcpAccess(runtimeMcpServers); + if (runtimeTools && runtimeToolDelivery === "invocation_context") { + adapterContext.paperclipRuntimeTools = runtimeTools; + } + const managedMcpConfig = await createManagedMcpRunConfig({ + db, + agent, + runId: run.id, + config: runtimeConfig, + projectId: issueRef?.projectId ?? null, + issueId: issueRef?.id ?? null, + }); + if (managedMcpConfig) { + adapterContext.paperclipManagedMcp = managedMcpConfig; + } + const guardedDispatch = + await dispatchResolvedInteractionContinuationWithAtomicGate( + (markDispatchStarted) => { + input.effects.onLegacyAdapterEntered(); + return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({ + getFreshSessionHandoff, + agentIdentity, + runId: run.id, + agent, + runtime: runtimeForAdapter, + config: runtimeConfig, + context: adapterContext, + executionContinuation: executionContinuation ?? null, + runtimeCommandSpec: + adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null, + executionTarget, + executionTransport: remoteExecution + ? { + remoteExecution: remoteExecution as unknown as Record< + string, + unknown + >, + } + : undefined, + runtimeMcp, + runtimeTools, + onLog, + onMeta: onAdapterMeta, + onEvent: onAdapterEvent, + onUsage: async receipt => { await usageRecorder.capture(receipt); }, + onExecutionPhase: executionControl.phases.enter, + startupTraceContext: getStartupTraceContext(), + onRuntimeProgress: async (progress) => { + await recordCurrentHeartbeatRunRuntimeProgress( + run, + progress, + issueId, + ); + }, + onProviderStopped: collectStoppedInstructions, + onDispatch: markDispatchStarted, + signal: executionControl.controller.signal, + ...(executionTarget?.kind === "remote" && (executionTarget.transport === "sandbox" || executionTarget.transport === "computer") ? { + stopRemoteStartup: async () => { + // Scope comes from the running host invocation, never agent + // config. Keep adapter ownership until setup has unwound. + if (!executionControl.controller.signal.aborted) { + throw new Error("Remote startup stop requires a cancelled run"); + } + const release = await envOrchestrator.releaseForRun({ + heartbeatRunId: run.id, + companyId: agent.companyId, + agentId: agent.id, + status: "released", + providerResourceDisposition: "stop_and_retain", + cancelActiveWork: true, + }); + if (release.errors.length || !await remoteExecutionHasStopped(db, agent.companyId, run.id)) { + throw new Error("Could not verify remote startup stopped"); + } + }, + } : {}), + onCancellationReady: async () => { + await registerAdapterExecutionControl(run.id, executionControl); + const current = await getRun(run.id); + if (!current || isHeartbeatRunTerminalStatus(current.status)) { + executionControl.controller.abort(new Error("Run stopped before provider startup")); + } + }, + onSpawn: async (meta) => { + markDispatchStarted(); + await persistRunProcessMetadata(run.id, { + pid: meta.pid, + processGroupId: + "processGroupId" in meta && + typeof meta.processGroupId === "number" + ? meta.processGroupId + : null, + startedAt: meta.startedAt, + }); + }, + authToken: authToken ?? undefined, + })); + }, + ); + if (!guardedDispatch.dispatched) return { dispatched: false }; + adapterResult = await guardedDispatch.resultPromise; + } + adapterResult = identityRedactor.redact(adapterResult); + if (run.runtimeMode === "legacy" && hasWorkspaceRestoreFailure(adapterResult.resultJson) + && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox") { + const requiredWorkspaceRestoreEvidence = { + workspaceRestoreFailure: adapterResult.resultJson!.workspaceRestoreFailure, + ...(adapterResult.resultJson?.workspaceRestoreDiagnostic ? { workspaceRestoreDiagnostic: adapterResult.resultJson.workspaceRestoreDiagnostic } : {}), + }; + input.effects.onWorkspaceRestoreFailure(requiredWorkspaceRestoreEvidence); + // Retention is the fallback even if recording this receipt fails. + input.effects.onProviderResourceDisposition("stop_and_retain"); + await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource); + } + for (const stream of ["stdout", "stderr"] as const) { + const tail = identityRedactor.finish(stream); + if (tail) await appendIdentityRedactedLog(stream, tail); + } + const instructionSave = input.instructions.getInstructionSave(); + if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave }; + + if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) { + const captured = await usageRecorder.complete(adapterResult); + adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete }; + } else { + // Stop may already own the terminal result. Preserve its metadata + // while durably recording the proof needed to release admission. + await db.update(heartbeatRuns).set({ costAccountingPending: true, + usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`, + }).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt))); + } + adapterResult = applyWorkspaceRestoreFailure(adapterResult); + // A returned result can include a failed restore. Keep the workspace + // barrier closed until required files have been restored. + // If recording the barrier itself fails, propagate as a run failure + // rather than silently leaving dependents stranded behind a missing + // finalize row. + const completeWorkspace = async (ownership?: NativeWorkspaceFinalizationOwnership) => { + try { + if (nativeWorkspaceSync) { + const exported = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, run.companyId), + eq(workspaceOperations.heartbeatRunId, run.id), + eq(workspaceOperations.phase, "workspace_finalize"), + eq(workspaceOperations.status, "succeeded"), + )).limit(1); + if (exported.length) adapterFinalizeOutcome = "succeeded"; + else await restoreNativeWorkspaceBestEffort({ + db, runId: run.id, assertOwnership: ownership?.assertHeld, + restore: () => nativeWorkspaceSync!.restoreWorkspace(ownership?.assertHeld), + }); + } + await ownership?.assertHeld(); + await db + .update(heartbeatRuns) + .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + ), + ); + const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; + await recordWorkspaceFinalize(workspaceFinalizeStatus); + if (adapterResult.nativeFinalization) { + adapterResult.nativeFinalization.workspaceFinalizeStatus = + workspaceFinalizeStatus; + try { + const finalized = await finalizeNativeRun({ + db, + runId: run.id, + workspaceFinalizeStatus, + preserveProviderAttempt: Boolean(nativeWorkspaceSync), + }); + await dispatchPendingNativeStatusWakeups({ + companyId: run.companyId, + }); + if (finalized.phase === "committed") { + await nativeWorkspaceSync?.cleanup(); + } + } catch (finalizeErr) { + logger.warn( + { err: finalizeErr, runId: run.id }, + "native result persisted but finalization did not apply; the reconciliation loop will retry", + ); + } + } + } catch (error) { + if (ownership) { + await ownership.assertHeld(); + await recordWorkspaceFinalize("failed"); + } + throw error; + } + }; + if (nativeWorkspaceSync) { + const owned = await withNativeWorkspaceFinalizationOwnership({ + db, companyId: run.companyId, runId: run.id, + }, completeWorkspace); + if (!owned.acquired) throw new NativeWorkspaceFinalizationBusyError(); + } else { + await completeWorkspace(); + } + } catch (adapterErr) { + if (adapterErr instanceof NativeCancellationPendingRecoveryError) { + // Durable cancellation is settled by the outer recovery handler; + // it does not imply a failed workspace or a persisted run result. + throw adapterErr; + } + if (adapterErr instanceof NativeWorkspaceFinalizationBusyError + || adapterErr instanceof NativeWorkspaceFinalizationOwnershipLostError) { + input.effects.onNativeWorkspaceFinalizeScheduled(); + throw adapterErr; + } + if (adapterErr instanceof NativeControllerDetachedForRestartError) { + // Preserve the provider and its run for the new controller. This + // also keeps generic teardown from terminalizing/releasing its lease. + input.effects.onNativeSessionResumeScheduled(); + throw adapterErr; + } + if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) { + input.effects.onNativeOwnershipHeld(); + throw adapterErr; + } + await db + .update(heartbeatRuns) + .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + ), + ); + if ( + issueRef && + context.resumeSessionGoalHeartbeat === true && + !runGoalControlRequestId + ) { + await blockRunnerGoalRecovery( + db, + { + companyId: run.companyId, + issueId: issueRef.id, + agentId: agent.id, + adapterType: agent.adapterType, + }, + "provider_session_goal_recovery_failed", + ).catch(() => undefined); + } + if (issueRef && runGoalControlRequestId) { + await failRunnerGoalAction( + db, + { + companyId: run.companyId, + issueId: issueRef.id, + agentId: agent.id, + adapterType: agent.adapterType, + }, + runGoalControlRequestId, + adapterErr instanceof Error + ? adapterErr.message + : "session_goal_control_failed", + ).catch(() => undefined); + } + const nativeResumeScheduled = + nativeRuntimeResolution.kind === "native" + ? await db + .select({ + phase: nativeRunFinalizations.phase, + resultId: nativeRunFinalizations.resultId, + }) + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)) + .limit(1) + .then( + (rows) => + rows[0]?.phase === "retryable_failure" && + rows[0]?.resultId === null, + ) + : false; + if (nativeResumeScheduled) { + input.effects.onNativeSessionResumeScheduled(); + throw new NativeSessionResumeScheduledError(adapterErr); + } + // Adapter (or its restore finally) threw — or the finalize record + // write itself threw. Either way the workspace may be in a partial + // state. Best-effort record finalize=failed so the dependent readiness + // check keeps the gate closed instead of waking on stale local state, + // and surface the original error to the caller. + try { + await recordWorkspaceFinalize("failed", { + errorMessage: + adapterErr instanceof Error + ? adapterErr.message + : String(adapterErr), + }); + } catch (recordErr) { + logger.warn( + { + err: recordErr, + runId: run.id, + executionWorkspaceId: persistedExecutionWorkspace?.id ?? null, + }, + "failed to record workspace_finalize=failed operation; dependents may remain gated", + ); + } + if (nativeRuntimeResolution.kind === "native") { + const proposedResult = await db + .select({ resultId: nativeRunFinalizations.resultId }) + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)) + .limit(1) + .then((rows) => rows[0]?.resultId ?? null); + if (proposedResult && nativeWorkspaceSync) { + const workspaceFailureMessage = + adapterErr instanceof Error ? adapterErr.message : ""; + const unrecoverable = + workspaceFailureMessage === + "workspace_sync_out_unrecoverable" || + workspaceFailureMessage.includes("daytona_sandbox_not_found"); + const failure = await recordNativeFinalizationFailure({ + db, + runId: run.id, + error: new Error( + unrecoverable + ? "native_workspace_sync_out_unrecoverable" + : "native_workspace_sync_out_failed", + ), + projectRunStatus: true, + failureScope: "workspace", + permanent: unrecoverable, + }); + input.effects.onNativeWorkspaceFinalizeScheduled(); + throw new NativeWorkspaceFinalizeScheduledError( + adapterErr, + failure.phase === "terminal_failure", + unrecoverable + ? "workspace_sync_out_unrecoverable" + : "workspace_sync_out_failed", + ); + } + try { + await finalizeNativeRun({ + db, + runId: run.id, + workspaceFinalizeStatus: "failed", + }); + await dispatchPendingNativeStatusWakeups({ + companyId: run.companyId, + }); + } catch (finalizeErr) { + logger.warn( + { err: finalizeErr, runId: run.id }, + "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results", + ); + } + } + throw adapterErr; + } finally { + try { + await revokeHeartbeatRunGatewayTokens({ + db, + companyId: agent.companyId, + runId: run.id, + }); + } catch (revokeErr) { + logger.warn( + { err: revokeErr, runId: run.id, companyId: agent.companyId }, + "failed to revoke heartbeat-run MCP gateway tokens", + ); + } + await nativeInstructionReservation?.release(); + await withAdapterExecutionPhase(executionPhaseContext, "instruction_cleanup", releaseInstructionCopy); + } + return { dispatched: true, adapterResult }; +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 400593dace3..70af5005448 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { recordChatHandoff, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js"; import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js"; import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js"; @@ -13047,49 +13048,55 @@ export function issueService(db: Db) { ); const publicationCreatedAt = new Date(); for (const binding of bindings) { - await dbOrTx - .insert(chatPublications) - .values({ - companyId: binding.companyId, - endpointId: binding.endpointId, - conversationId: binding.conversationId, - issueId, - commentId: comment.id, - idempotencyKey: `comment:${comment.id}:${binding.endpointId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "agent_comment", - text: redactedBody, - }), - state: "pending", - createdAt: publicationCreatedAt, - updatedAt: publicationCreatedAt, - }) - .onConflictDoNothing(); - for (const [index, attachment] of boundAttachments.entries()) { - const attachmentCreatedAt = new Date( - publicationCreatedAt.getTime() + index + 1, - ); - await dbOrTx + await dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx .insert(chatPublications) .values({ companyId: binding.companyId, endpointId: binding.endpointId, conversationId: binding.conversationId, issueId, - commentId: attachment.commentId, - idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + commentId: comment.id, + idempotencyKey: `comment:${comment.id}:${binding.endpointId}`, payload: projectSafeChatPublication({ classification: "external", source: "agent_comment", - text: `Shared ${attachment.originalFilename ?? "a file"}.`, - attachmentIds: [attachment.id], + text: redactedBody, }), state: "pending", - createdAt: attachmentCreatedAt, - updatedAt: attachmentCreatedAt, + createdAt: publicationCreatedAt, + updatedAt: publicationCreatedAt, }) .onConflictDoNothing(); + }); + for (const [index, attachment] of boundAttachments.entries()) { + const attachmentCreatedAt = new Date( + publicationCreatedAt.getTime() + index + 1, + ); + await dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: binding.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId, + commentId: attachment.commentId, + idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: `Shared ${attachment.originalFilename ?? "a file"}.`, + attachmentIds: [attachment.id], + }), + state: "pending", + createdAt: attachmentCreatedAt, + updatedAt: attachmentCreatedAt, + }) + .onConflictDoNothing(); + }); } } } @@ -13333,6 +13340,7 @@ export function issueService(db: Db) { registeredRunId, ); for (const binding of bindings) { + await notifyChatPublicationWork(tx); await tx .insert(chatPublications) .values({ diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 842b31faa74..52f2f12606c 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -24,7 +24,6 @@ import { nativeRunResults, } from "@paperclipai/db"; import { describe, expect, it, vi } from "vitest"; -import * as codexCommandResolver from "../../../../packages/paperclip-runner/src/drivers/codex/codex-command.js"; import { CodexAppServerDriver, HarnessDriverBackend, @@ -746,6 +745,9 @@ const recoveryFakeCodex = resolve( }; // The production resolver prefers installed Codex over PATH. Explicitly // bind this disposable fixture so it can never invoke a real provider. + const codexCommandResolver = await vi.importActual<{ resolveCodexCommand: () => string }>( + "../../../../packages/paperclip-runner/src/drivers/codex/codex-command.js", + ); const resolveFixtureCodex = vi.spyOn(codexCommandResolver, "resolveCodexCommand") .mockReturnValue(join(bin, "codex")); process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; diff --git a/server/src/services/slack-board-messages.ts b/server/src/services/slack-board-messages.ts index 756961c7f0a..b8fc4ddd8f3 100644 --- a/server/src/services/slack-board-messages.ts +++ b/server/src/services/slack-board-messages.ts @@ -1,3 +1,4 @@ +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; import { assets, @@ -121,6 +122,7 @@ export async function mirrorSlackBoardComment( .onConflictDoNothing() .returning(); if (!receipt) continue; + await notifyChatPublicationWork(db); const [publication] = await db .insert(chatPublications) .values({ @@ -161,7 +163,7 @@ export async function mirrorSlackBoardComment( byId.has(id) ? [byId.get(id)!] : [], ) : files; - for (const [index, file] of orderedFiles.entries()) + for (const [index, file] of orderedFiles.entries()) { await db .insert(chatPublications) .values({ @@ -181,6 +183,7 @@ export async function mirrorSlackBoardComment( createdAt: new Date(Date.now() + index + 1), }) .onConflictDoNothing(); + } await logActivity(db as Db, { companyId: comment.companyId, actorType: "user", diff --git a/server/src/services/voice/voice-session-store.ts b/server/src/services/voice/voice-session-store.ts index 21a864c3730..7a0ce846acc 100644 --- a/server/src/services/voice/voice-session-store.ts +++ b/server/src/services/voice/voice-session-store.ts @@ -1,3 +1,4 @@ +import { notifyChatDeliveryWork } from "../chat-work-notifications.js"; import { environmentService } from "../environments.js"; import { logger } from "../../middleware/logger.js"; import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; @@ -210,6 +211,7 @@ export function voiceSessionStore(db: Db, options: { allowLocalBoard: boolean; n const threadId = conversation.externalThreadId; const providerEventId = `${threadId}:${session.id}:${toolId}`; const trigger = prior.length ? "subscribed_message" : "direct_message"; + await notifyChatDeliveryWork(tx); const [delivery] = await tx.insert(chatDeliveries).values({ companyId: session.companyId, endpointId: session.endpointId, conversationId: session.conversationId, providerEventId, deduplicationKey: createHash("sha256").update(providerEventId).digest("hex"), diff --git a/tests/e2e/fixtures/composer-stop-codex-loader.mjs b/tests/e2e/fixtures/composer-stop-codex-loader.mjs new file mode 100644 index 00000000000..0815a0c62bf --- /dev/null +++ b/tests/e2e/fixtures/composer-stop-codex-loader.mjs @@ -0,0 +1,23 @@ +// The installed Codex dependency takes precedence over PATH. Pin only this +// acceptance server's command resolver to its deterministic protocol fixture. +import { accessSync, constants } from "node:fs"; +import { registerHooks } from "node:module"; +import { isAbsolute } from "node:path"; + +const command = process.env.PAPERCLIP_STOP_CODEX_COMMAND; +if (process.env.NODE_ENV !== "test" || !command || !isAbsolute(command)) { + throw new Error("Composer Stop requires an absolute fixture command in NODE_ENV=test"); +} +accessSync(command, constants.X_OK); +registerHooks({ + load(url, context, nextLoad) { + if (/\/packages\/paperclip-runner\/(src|dist)\/drivers\/codex\/codex-command\.(js|ts)$/.test(url)) { + return { + format: "module", + shortCircuit: true, + source: `export const resolveCodexCommand = () => ${JSON.stringify(command)}; export const resolvePinnedCodexCommand = resolveCodexCommand;`, + }; + } + return nextLoad(url, context); + }, +}); diff --git a/tests/e2e/fixtures/composer-stop-codex-loader.test.mjs b/tests/e2e/fixtures/composer-stop-codex-loader.test.mjs new file mode 100644 index 00000000000..35a0de116ee --- /dev/null +++ b/tests/e2e/fixtures/composer-stop-codex-loader.test.mjs @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; + +const loader = fileURLToPath(new URL("./composer-stop-codex-loader.mjs", import.meta.url)); +const resolver = new URL("../../../packages/paperclip-runner/src/drivers/codex/codex-command.ts", import.meta.url).href; +function launch(env, source = "") { + return spawnSync(process.execPath, ["--import", loader, "--input-type=module", "-e", source], { + encoding: "utf8", env: { ...process.env, NODE_ENV: "test", PAPERCLIP_STOP_CODEX_COMMAND: "", ...env }, + }); +} + +test("pins the fixture even when a bundled Codex dependency is installed", () => { + const dir = mkdtempSync(join(tmpdir(), "composer-stop-loader-")); + const command = join(dir, "codex"); + try { + writeFileSync(command, "#!/bin/sh\nprintf 'composer-stop-fixture'\n", { mode: 0o755 }); + const result = launch({ PAPERCLIP_STOP_CODEX_COMMAND: command }, ` + import { resolveCodexCommand, resolvePinnedCodexCommand } from ${JSON.stringify(resolver)}; + import { execFileSync } from "node:child_process"; + import assert from "node:assert/strict"; + assert.equal(resolveCodexCommand(), ${JSON.stringify(command)}); + assert.equal(resolvePinnedCodexCommand(), ${JSON.stringify(command)}); + process.stdout.write(execFileSync(resolveCodexCommand(), ["--version"])); + `); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, "composer-stop-fixture"); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +for (const env of [{}, { PAPERCLIP_STOP_CODEX_COMMAND: "codex" }, { NODE_ENV: "production", PAPERCLIP_STOP_CODEX_COMMAND: process.execPath }]) { + test(`rejects a missing or unsafe fixture configuration: ${JSON.stringify(env)}`, () => { + const result = launch(env); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Composer Stop requires an absolute fixture command/); + }); +} diff --git a/tests/e2e/playwright-composer-stop.config.ts b/tests/e2e/playwright-composer-stop.config.ts index 41d03194680..a58f8507034 100644 --- a/tests/e2e/playwright-composer-stop.config.ts +++ b/tests/e2e/playwright-composer-stop.config.ts @@ -19,6 +19,7 @@ if (fixture) { throw new Error( "PAPERCLIP_STOP_FAKE_CODEX must name the built fake-codex-app-server binary", ); + fs.mkdirSync(path.join(fixtureDir, "codex-home")); const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; fs.writeFileSync( path.join(fixtureDir, "codex"), @@ -37,6 +38,14 @@ export default defineConfig({ ...server?.env, HEARTBEAT_SCHEDULER_INTERVAL_MS: "10000", PATH: `${fixtureDir}${path.delimiter}${process.env.PATH ?? ""}`, + ...(fixture ? { + // Provider selection prefers the installed dependency over PATH. + // Scope this override and its empty credential home to the test server. + PAPERCLIP_STOP_CODEX_COMMAND: path.join(fixtureDir, "codex"), + CODEX_HOME: path.join(fixtureDir, "codex-home"), + OPENAI_API_KEY: "composer-stop-fixture-key", + NODE_OPTIONS: `${server?.env?.NODE_OPTIONS ?? ""} --import=${path.resolve(import.meta.dirname, "fixtures/composer-stop-codex-loader.mjs")}`, + } : {}), }, }, });