fix(server): wake failed retry notices from receipt changes

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/chat-channels.ts

@@ -3354,9 +3354,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
 
   async function processPublicationMaintenance(limit = 25) {
     scheduleGitHubMaintenance();
-    // Normal inbound notices are commit-driven. Failed-run retries retain their
-    // separate maintenance owner until its own producer audit is complete.
-    await enqueueFailedChatRetryPublications(50);
     await reconcileTerminalConfirmationActions(limit);
     await processPendingInteractionWakeups(limit);
   }

server/src/services/chat-channels/failed-retry-notices.ts

@@ -17,8 +17,6 @@ type Options = {
   isShuttingDown: () => boolean;
 };
 
-/** Retry authorization belongs to the retry service. This projection owns only
- * the queued/not-started notice, selected from its existing durable receipt. */
 export function createFailedRetryNotices(db: Db, options: Options) {
   let manualCursor: FailedRetryNoticeCursor | null = null;
   async function candidates(limit: number, cursor: FailedRetryNoticeCursor | null, scope?: FailedRetryNoticeScope) {

server/src/services/chat-channels/failed-run-retry.ts

@@ -1,3 +1,4 @@
+import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
 import { createFailedRetryNotices } from "./failed-retry-notices.js";
 import { continuingTeamsFilePublication, type ContinuingTeamsFileTransfer } from "./microsoft-teams/file-transfer-source.js";
 import { readTaskChatRetryIntent, taskChatRetryCanAwaitRecovery, taskChatRetryExecutionStopped } from "./task-retry-authority.js";
@@ -1647,6 +1648,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
                 typeof claimed.payload.taskRetryIntentId === "string" ? claimed.payload.taskRetryIntentId : undefined,
                 true, // The durable wake callback validates admission; later phases recheck containment.
               );
+              await notifyChatInboundWakeNotices(tx, { companyId: scope.companyId, issueId: scope.issueId });
             },
           }),
         });
@@ -1684,12 +1686,17 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
             receipt.payload?.taskRetryRequestId === claimed.payload.taskRetryIntentId) {
           // Receipt existence proves the earlier wait, not current authority.
           // Retire only this unadmitted wake; an admitted run keeps its history.
-          await db.update(agentWakeupRequests).set({
-            status: "failed", error: "chat_failed_run_retry_not_authorized",
-            finishedAt: new Date(), updatedAt: new Date(),
-          }).where(and(eq(agentWakeupRequests.id, receipt.id),
-            eq(agentWakeupRequests.companyId, claimed.companyId),
-            eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId)));
+          await db.transaction(async tx => {
+            await notifyChatInboundWakeNotices(tx, {
+              companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
+            });
+            await tx.update(agentWakeupRequests).set({
+              status: "failed", error: "chat_failed_run_retry_not_authorized",
+              finishedAt: new Date(), updatedAt: new Date(),
+            }).where(and(eq(agentWakeupRequests.id, receipt.id),
+              eq(agentWakeupRequests.companyId, claimed.companyId),
+              eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId)));
+          });
           return settle("failed", { code: "failed_chat_retry_not_authorized", attemptCount });
         }
         return settle(["skipped", "failed", "cancelled"].includes(receipt.status) ? "failed" : "processed", {

server/src/services/chat-failed-retry-notice-work.ts

@@ -0,0 +1,39 @@
+import type { ChatChannelService } from "./chat-channels.js";
+import type { FailedRetryNoticeCursor, FailedRetryNoticeScope } from "./chat-channels/failed-retry-notices.js";
+import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
+import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
+import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
+
+type Entry = {
+  id: string; companyId: string; issueId: string; dueAt: number;
+  cursor: FailedRetryNoticeCursor | null; contended: boolean;
+};
+const batchSize = 100;
+
+export function chatFailedRetryNoticeWork(service: Pick<ChatChannelService,
+  "loadFailedRetryNoticeScopes" | "reconcileFailedRetryNotices">) {
+  async function load(scope?: FailedRetryNoticeScope, signal?: AbortSignal): Promise<Entry[]> {
+    return (await service.loadFailedRetryNoticeScopes(scope, signal)).map(row => ({
+      ...row, id: `${row.companyId}:${row.issueId}`, dueAt: 0, cursor: null, contended: false,
+    }));
+  }
+  async function check(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
+    const result = await service.reconcileFailedRetryNotices({
+      scope: { companyId: entry.companyId, issueId: entry.issueId },
+      limit: batchSize, cursor: entry.cursor, signal,
+    });
+    const contended = entry.contended || result.contended;
+    if (result.nextCursor) return { ...entry, cursor: result.nextCursor, contended, dueAt: Date.now() };
+    return contended ? { ...entry, cursor: null, contended: false, dueAt: Date.now() + 1000 } : null;
+  }
+  function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
+    return registerScopedDeadlineWork(coordinator, {
+      queue: DELIVERY_QUEUES.chatFailedRetryNotices,
+      decodeScope: (hint: string): FailedRetryNoticeScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
+      matches: (entry: Entry, scope?: FailedRetryNoticeScope) => !scope ||
+        (entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId)),
+      load, check,
+    }, canRun);
+  }
+  return { load, check, register };
+}

server/src/services/chat-inbound-wake-notice-notifications.ts

@@ -1,7 +1,8 @@
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import type { InboundWakeNoticeScope } from "./chat-channels/inbound/wakeup-notices.js";
 
-export function notifyChatInboundWakeNotices(tx: object, scope: InboundWakeNoticeScope) {
-  return notifyDeliveryWork(tx, DELIVERY_QUEUES.chatInboundWakeNotices,
-    JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
+export async function notifyChatInboundWakeNotices(tx: object, scope: InboundWakeNoticeScope) {
+  const hint = JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId });
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatInboundWakeNotices, hint);
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatFailedRetryNotices, hint);
 }

server/src/services/delivery-work-notifications.ts

@@ -56,6 +56,7 @@ export const DELIVERY_QUEUES = {
   chatPublications: "chat-publications",
   chatNativeProgress: "chat-native-progress",
   chatInboundWakeNotices: "chat-inbound-wake-notices",
+  chatFailedRetryNotices: "chat-failed-retry-notices",
   chatRunMilestones: "chat-run-milestones",
   chatReceipts: "chat-slack-receipts",
   browser: "browser-use",

server/src/services/heartbeat/queue.ts

@@ -1,3 +1,4 @@
+import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
 import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js";
 import { authorizationService } from "../authorization.js";
 import { assertTaskRetryConversationAllowed, readTaskRetryIntent, retireSupersededTaskRetry } from "../task-retry.js";
@@ -2462,12 +2463,24 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
         const waitIssueId = issueId;
         return db.transaction(async (tx) => {
           await tx.execute(sql`select id from issues where id = ${waitIssueId} and company_id = ${agent.companyId} for update`);
+          if (durableRequest) await notifyChatInboundWakeNotices(tx, {
+            companyId: durableRequest.companyId, issueId: durableRequest.issueId,
+          });
           return recordExecutionWait(tx as unknown as Db, {
             issueId: waitIssueId, request, condition: waitCondition, coalesce: coalesceExecutionWait,
           });
         });
       }
-      await db.insert(agentWakeupRequests).values(request);
+      if (durableRequest) {
+        await db.transaction(async tx => {
+          await notifyChatInboundWakeNotices(tx, {
+            companyId: durableRequest.companyId, issueId: durableRequest.issueId,
+          });
+          await tx.insert(agentWakeupRequests).values(request);
+        });
+      } else {
+        await db.insert(agentWakeupRequests).values(request);
+      }
       return { created: true };
     };
     const writeSkippedHeartbeatRequest = async (