fix(server): wake Discord registration from committed changes and deadlines

Replace command maintenance inside runtime polling with scoped producer wakes, saved verification deadlines, and fair bounded dispatch. Preserve current credential and namespace authority, foreground setup, runtime continuity, and shutdown settlement.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/access.ts

@@ -1,3 +1,4 @@
+import { notifyChatEndpointWork } from "./chat-work-notifications.js";
 import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js";
 import { and, eq, inArray, ne, notInArray, sql } from "drizzle-orm";
 import type { Db } from "@paperclipai/db";
@@ -130,6 +131,7 @@ export function accessService(db: Db) {
     }> = [];
     let connectionRefs: Array<{
       id: string;
+      transport: typeof toolConnections.$inferSelect.transport;
       credentialRefs: typeof toolConnections.$inferSelect.credentialRefs;
       credentialSecretRefs: typeof toolConnections.$inferSelect.credentialSecretRefs;
     }> = [];
@@ -168,6 +170,7 @@ export function accessService(db: Db) {
         }).from(connectionGrants).where(eq(connectionGrants.companyId, companyId)),
         tx.select({
           id: toolConnections.id,
+          transport: toolConnections.transport,
           credentialRefs: toolConnections.credentialRefs,
           credentialSecretRefs: toolConnections.credentialSecretRefs,
         }).from(toolConnections).where(eq(toolConnections.companyId, companyId)),
@@ -324,6 +327,7 @@ export function accessService(db: Db) {
             && grant.credentialSecretRefs.length > 0
             && grant.credentialSecretRefs.every((ref) => !connectionSecretIdsToRemove.has(ref.secretId)),
           );
+          if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id);
           if (!hasUnaffectedActiveGrant) await notifyToolConnectionHealth(tx, companyId);
           await tx.update(toolConnections).set({
             credentialRefs,

server/src/services/chat-channels.ts

@@ -1,3 +1,7 @@
+import { notifyChatCommandRegistration } from "./chat-channels/command-work-notifications.js";
+import { registerDiscordCommandWork } from "./chat-channels/discord/command-work.js";
+import { discordCommandSchedule } from "./chat-channels/discord/command-schedule.js";
+import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
 import { notifyChatConnectionHealth } from "./chat-channels/connection-health-notifications.js";
 import { createChatSdkMessageReceiver } from "./chat-channels/runtime/sdk-messages.js";
 import { admitDiscordSdkMessage } from "./chat-channels/discord/messages.js";
@@ -889,7 +893,15 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
   const { authenticateGitHubWebhook } = createGitHubWebhookAuthentication(db, { endpointRecord, runtimeContextForRecord, withCredentialMutationLease, resolveCredentials, reconcileGitHubWebhookRepository, stageGitHubWebhookIngress, processGitHubWebhookIngress, githubIngressAcceptedResponse, scheduleMessageProcessing, backgroundMessageTasks, redactError, get githubWebhookAuthenticationBarrier() { return options.githubWebhookAuthenticationBarrier; }, get deferWebhookProcessing() { return options.deferWebhookProcessing; } });
   const { inspectSlackWebhookScope, recordSlackWebhookCallback } = createSlackWebhookCallbacks(db, { resolveCredentials, runtimeCallbackEndpoint, getWebhookPublicBaseUrl, scheduleMessageProcessing, slackRegistration: { notifyVerified: (endpointId) => slackRegistration.notifyVerified(endpointId) } });
 
-  const { reconcileDiscordRuntime } = createDiscordRuntimeReconciliation(db, { reconcileDiscordCommands, endpointRecord, runtimeContexts, invalidateRuntime, runtimeFor, runtime, runtimeCallbackEndpoint, credentialFingerprint, discordGatewayRuntimeIsCurrent });
+  const { reconcileDiscordRuntime } = createDiscordRuntimeReconciliation(db, { endpointRecord, runtimeContexts, invalidateRuntime, runtimeFor, runtime, runtimeCallbackEndpoint, credentialFingerprint, discordGatewayRuntimeIsCurrent });
+
+  function registerCommandWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
+    return registerDiscordCommandWork(coordinator, discordCommandSchedule(db, async endpointId => {
+      await reconcileDiscordCommands(endpointId);
+      const instance = runtime.get(endpointId);
+      if (instance) await reconcileDiscordRuntime(endpointId, instance);
+    }), () => !shuttingDown && canRun());
+  }
 
   async function runtimeFor(
     endpoint: EndpointRow,
@@ -2203,6 +2215,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
               ? "Started a conversation by mentioning the agent."
               : "Sent an empty message.");
         let comment!: Awaited<ReturnType<typeof issuesSvc.addComment>>;
+        await notifyChatCommandRegistration(taskTx, taskEndpoint);
         await taskTx
           .update(chatEndpoints)
           .set({
@@ -3461,6 +3474,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
   const { prepareTeamsBoardSend } = createTeamsBoardSendPreparation(db, { endpointRecord, runtimeFor });
   const { getPublicationBatchStatus, publishComment, publishBoardMessage } = createChatPublicationRequests(db, { endpointRecord, stageTeamsBoardFileIntent, processPendingSlackBoardMessages, processPendingPublications, prepareTeamsBoardSend });
 return {
+    registerCommandWork,
     voice,
     authorizeFastResponse,
     slackRegistration,

server/src/services/chat-channels/command-work-notifications.ts

@@ -0,0 +1,18 @@
+import type { ChatProvider, ToolConnectionTransport } from "@paperclipai/shared";
+import { notifyDiscordCommandRegistration } from "./discord/command-work-notifications.js";
+
+export async function notifyChatCommandRegistration(transaction: object, endpoint: { id: string; provider: ChatProvider }) {
+  if (endpoint.provider === "discord") {
+    await notifyDiscordCommandRegistration(transaction, { kind: "endpoint", id: endpoint.id });
+  }
+}
+
+export async function notifyChatConnectionCommandRegistration(transaction: object, connectionId: string, transport: ToolConnectionTransport = "chat_sdk") {
+  if (transport === "chat_sdk") {
+    await notifyDiscordCommandRegistration(transaction, { kind: "connection", id: connectionId });
+  }
+}
+
+export async function notifyChatCompanyCommandRegistration(transaction: object, companyId: string) {
+  await notifyDiscordCommandRegistration(transaction, { kind: "company", id: companyId });
+}

server/src/services/chat-channels/credentials.ts

@@ -1,3 +1,4 @@
+import { notifyChatCommandRegistration } from "./command-work-notifications.js";
 import { prepareSlackInventory } from "./slack/inventory.js";
 import { prepareGitHubInventory } from "./github/inventory.js";
 import { prepareDiscordInventory } from "./discord/inventory.js";
@@ -315,6 +316,7 @@ export function createChatCredentials(db: Db, options: ChatCredentialsOptions) {
             code: "chat_endpoint_credentials_changed",
           });
         }
+        await notifyChatCommandRegistration(tx, endpoint);
         await tx
           .delete(companySecretBindings)
           .where(
@@ -411,6 +413,7 @@ export function createChatCredentials(db: Db, options: ChatCredentialsOptions) {
         .then((rows) => rows[0] ?? null);
       if (!current) throw notFound("Chat connection not found");
       refs = current.refs;
+      await notifyChatCommandRegistration(tx, endpoint);
       await tx
         .delete(companySecretBindings)
         .where(

server/src/services/chat-channels/discord/command-work.ts

@@ -1,3 +1,4 @@
+import { takeCompanyWorkBatch } from "../../company-work-batch.js";
 import type { createDeliveryWorkCoordinator } from "../../delivery-work-coordinator.js";
 import { DELIVERY_QUEUES } from "../../delivery-work-notifications.js";
 import type { CommandRegistrationDeadline, discordCommandSchedule } from "./command-schedule.js";
@@ -27,6 +28,7 @@ export function registerDiscordCommandWork(
   const failures = new Map<string, { count: number; retryAt: number }>();
   const retryMs = 30_000;
   let earliest = 0;
+  let lastCompany: string | null = null;
   function next() {
     let at: number | null = dirty.size ? earliest : null;
     for (const entry of deadlines.values()) {
@@ -66,8 +68,10 @@ export function registerDiscordCommandWork(
       const due = [...deadlines.values()]
         .filter(entry => !unavailable.some(scope => matches(entry, scope)) &&
           Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())
-        .sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id)).slice(0, 25);
-      for (const entry of due) {
+        .sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id));
+      const batch = takeCompanyWorkBatch(due, 25, lastCompany);
+      lastCompany = batch.lastCompany;
+      for (const entry of batch.entries) {
         if (signal.aborted || !canRun()) break;
         try {
           const current = await service.check(entry.id, signal);

server/src/services/chat-channels/discord/gateway-events.ts

@@ -1,3 +1,4 @@
+import { notifyDiscordCommandRegistration } from "./command-work-notifications.js";
 import type { EndpointRow, DbTransaction, RuntimeContext, LifecycleRuntimeFence, InternalSetupState } from "../runtime-types.js";
 
 import { notifyChatEndpointWork } from "../../chat-work-notifications.js";
@@ -129,6 +130,7 @@ export function createDiscordGatewayEvents(db: Db, options: DiscordGatewayEvents
       }
       const now = new Date();
       if (event.type === "connecting") {
+        if (current.status === "attention") await notifyDiscordCommandRegistration(tx, { kind: "endpoint", id: current.id });
         await tx
           .update(chatEndpoints)
           .set({
@@ -157,6 +159,7 @@ export function createDiscordGatewayEvents(db: Db, options: DiscordGatewayEvents
         if (current.status === "attention" && !gatewayOwnedAttention) return;
         const status =
           current.setup.step === "complete" ? "active" : "verifying";
+        await notifyDiscordCommandRegistration(tx, { kind: "endpoint", id: current.id });
         if (status !== current.status) await notifyChatEndpointWork(tx, current.connectionId);
         await tx
           .update(chatEndpoints)
@@ -186,6 +189,7 @@ export function createDiscordGatewayEvents(db: Db, options: DiscordGatewayEvents
         return;
       }
 
+      await notifyDiscordCommandRegistration(tx, { kind: "endpoint", id: current.id });
       const detail = discordGatewayFailureDetail(event);
       const transientGuildOutage = event.type === "guild_unavailable";
       const terminal =

server/src/services/chat-channels/discord/runtime-reconciliation.ts

@@ -7,11 +7,9 @@ import type { Db } from "@paperclipai/db";
 import type { EndpointRow, EndpointRecord, RuntimeContext } from "../runtime-types.js";
 import type { ChatRuntime } from "../../chat-runtime.js";
 import type { ChatEndpointRuntime } from "../../voice/voice-runtime.js";
-import type { createDiscordCommands } from "./commands.js";
 import type { createChatMessageLifecycle } from "../inbound/lifecycle.js";
 import type { ToolCredentialSecretRef } from "@paperclipai/shared";
 type ReconciliationOptions = {
-  reconcileDiscordCommands: ReturnType<typeof createDiscordCommands>["reconcileDiscordCommands"];
   endpointRecord: (endpointId: string) => Promise<EndpointRecord | null>;
   runtimeContexts: WeakMap<object, RuntimeContext>;
   invalidateRuntime: (endpointId: string) => Promise<boolean>;
@@ -22,9 +20,8 @@ type ReconciliationOptions = {
   discordGatewayRuntimeIsCurrent: (endpointId: string, context: RuntimeContext) => boolean;
 };
 export function createDiscordRuntimeReconciliation(db: Db, options: ReconciliationOptions) {
-  const { reconcileDiscordCommands, endpointRecord, runtimeContexts, invalidateRuntime, runtimeFor, runtime, runtimeCallbackEndpoint, credentialFingerprint, discordGatewayRuntimeIsCurrent } = options;
+  const { endpointRecord, runtimeContexts, invalidateRuntime, runtimeFor, runtime, runtimeCallbackEndpoint, credentialFingerprint, discordGatewayRuntimeIsCurrent } = options;
   async function reconcileDiscordRuntime(endpointId: string, instance: ChatEndpointRuntime) {
-    await reconcileDiscordCommands(endpointId);
     const afterCommands = await endpointRecord(endpointId);
     const desiredCommands =
       afterCommands?.endpoint.capabilities.slashCommands &&

server/src/services/chat-channels/interactions/callbacks.ts

@@ -1,3 +1,4 @@
+import { notifyChatCommandRegistration } from "../command-work-notifications.js";
 import { notifyChatConnectionHealth } from "../connection-health-notifications.js";
 import { createDiscordQuestionForms } from "../discord/question-forms.js";
 
@@ -411,6 +412,7 @@ export function createChatInteractionCallbacks(db: Db, options: InteractionCallb
             ["active"],
           );
           if (currentEndpoint) {
+            await notifyChatCommandRegistration(tx, currentEndpoint);
             await tx
               .update(chatEndpoints)
               .set({

server/src/services/chat-channels/provider-effects.ts

@@ -1,3 +1,4 @@
+import { notifyChatCommandRegistration } from "./command-work-notifications.js";
 import { notifyChatConnectionHealth } from "./connection-health-notifications.js";
 import { createTelegramCallbackNoticeAuthority } from "./telegram/callback-notices.js";
 
@@ -317,6 +318,7 @@ export function createProviderEffects(db: Db, options: ProviderEffectsOptions) {
           );
       }
       if (input.disposition.kind === "endpoint_attention" && currentEndpoint) {
+        await notifyChatCommandRegistration(tx, currentEndpoint);
         await tx
           .update(chatEndpoints)
           .set({

server/src/services/chat-channels/publications/failures.ts

@@ -1,3 +1,4 @@
+import { notifyChatCommandRegistration } from "../command-work-notifications.js";
 import { notifyChatConnectionHealth } from "../connection-health-notifications.js";
 import { isOutboundAttachmentHydrationError, isOutboundAttachmentValidationError } from "../outbound-attachments.js";
 
@@ -189,6 +190,7 @@ export function createChatPublicationFailures(db: Db, options: PublicationFailur
           : null;
 
       if (disposition.kind === "endpoint_attention" && currentEndpoint) {
+        await notifyChatCommandRegistration(tx, currentEndpoint);
         await tx
           .update(chatEndpoints)
           .set({

server/src/services/chat-channels/resource-reconciliation.ts

@@ -1,3 +1,4 @@
+import { notifyChatCommandRegistration } from "./command-work-notifications.js";
 import { notifyChatConnectionHealth } from "./connection-health-notifications.js";
 import { imessageResourceParticipants } from "./imessage-photon/resources.js";
 import { createGitHubResources, reconcileGitHubRepositoryCoordinates } from "./github/resources.js";
@@ -509,6 +510,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc
                     : currentEndpoint.status === "attention"
                       ? "verifying"
                       : currentEndpoint.status;
+                await notifyChatCommandRegistration(tx, currentEndpoint);
                 if (status !== currentEndpoint.status) await notifyChatEndpointWork(tx, currentEndpoint.connectionId);
                 await tx
                   .update(chatEndpoints)
@@ -538,6 +540,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc
                   .where(eq(toolConnections.id, currentEndpoint.connectionId));
               }
             } else {
+              await notifyChatCommandRegistration(tx, currentEndpoint);
               await tx
                 .update(chatEndpoints)
                 .set({

server/src/services/chat-discord-command-registration-store.test.ts

@@ -805,4 +805,40 @@ suite("Discord command ownership store (real PostgreSQL, no network)", () => {
     expect(coordinator!.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
   });
 
+  it("retains a committed eligibility change while the provider attempt is in flight", async () => {
+    const f = await fixture(); await f.reconcile(); await f.due();
+    let release!: () => void;
+    const held = new Promise<void>(resolve => { release = resolve; });
+    f.beforeResponse(async method => { if (method === "GET") await held; });
+    const starting = scheduled(f.scope.companyId, async () => { await f.reconcile(); });
+    try {
+      await vi.waitFor(() => expect(f.calls).toEqual(["GET", "POST", "GET"]));
+      await db.transaction(async tx => {
+        await notifyDiscordCommandRegistration(tx, { kind: "endpoint", id: f.scope.endpointId });
+        await tx.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.scope.endpointId));
+      });
+    } finally { release(); }
+    const { errors } = await starting;
+    await vi.advanceTimersByTimeAsync(31_000);
+    await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull());
+    await sqlQuiet(120_000);
+    expect(f.calls).toEqual(["GET", "POST", "GET"]); expect(errors).toEqual([]);
+  });
+
+  it("checks a quiet company's due command in the first batch despite another company's backlog", async () => {
+    await reconcileDatabaseWork(db, DELIVERY_QUEUES.chatCommandRegistration);
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
+    const entries = Array.from({ length: 30 }, (_, index) => ({
+      id: `busy-${index.toString().padStart(2, "0")}`, companyId: "busy", connectionId: "busy-connection", dueAt: 0,
+    }));
+    entries.push({ id: "quiet", companyId: "quiet", connectionId: "quiet-connection", dueAt: 1 });
+    const check = vi.fn(async (id: string) => ({ ...entries.find(entry => entry.id === id)!, dueAt: Date.now() + 300_000 }));
+    const errors: unknown[] = [];
+    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
+    await registerDiscordCommandWork(coordinator, { load: async () => entries, check }, () => true).ready;
+    expect(check).toHaveBeenCalledTimes(25);
+    expect(check.mock.calls.map(call => call[0])).toContain("quiet");
+    expect(errors).toEqual([]);
+  });
+
 });

server/src/services/chat-work-notifications.ts

@@ -1,3 +1,4 @@
+import { notifyChatConnectionCommandRegistration } from "./chat-channels/command-work-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 
 export async function notifyChatDeliveryWork(tx: object): Promise<void> {
@@ -37,4 +38,5 @@ export async function notifyChatEndpointWork(tx: object, connectionId: string):
   for (const queue of Object.values(CHAT_ACTION_QUEUES)) await notifyDeliveryWork(tx, queue);
   await notifyChatVerificationWork(tx);
   await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatTelegramSubscriptions, connectionId);
+  await notifyChatConnectionCommandRegistration(tx, connectionId);
 }

server/src/services/company-deletion.ts

@@ -1,3 +1,4 @@
+import { notifyChatCompanyCommandRegistration } from "./chat-channels/command-work-notifications.js";
 import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js";
 import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
 import { notifyDecisionTargetWork } from "./decision-target-work.js";
@@ -119,6 +120,7 @@ export async function deleteCompany(db: Db, id: string) {
         await tx.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, id));
         await tx.delete(projects).where(eq(projects.companyId, id));
         await agentLifecycleCompanyDeletion.deleteCompanyData(tx, id);
+        await notifyChatCompanyCommandRegistration(tx, id);
         await notifyToolConnectionHealth(tx, id);
         const rows = await tx
           .delete(companies)

server/src/services/tool-access/catalog/health.ts

@@ -1,3 +1,4 @@
+import { notifyChatConnectionCommandRegistration } from "../../chat-channels/command-work-notifications.js";
 import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js";
 import { assertSupportedConnection } from "../connections/setup-policy.js";
 import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js";
@@ -47,8 +48,9 @@ export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, d
     message: string | null,
   ) {
     const now = new Date();
-    const [updated] = await writeToolConnectionHealthChange(db, connection.companyId, tx => tx
-      .update(toolConnections)
+    const [updated] = await writeToolConnectionHealthChange(db, connection.companyId, async tx => {
+      await notifyChatConnectionCommandRegistration(tx, connection.id, connection.transport);
+      return tx.update(toolConnections)
       .set({
         healthStatus: status,
         healthMessage: message,
@@ -58,7 +60,8 @@ export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, d
         updatedAt: now,
       })
       .where(eq(toolConnections.id, connection.id))
-      .returning());
+      .returning();
+    });
     if (connection.transport === "local_stdio") {
       await db
         .update(toolRuntimeSlots)

server/src/services/tool-access/connections/removal.ts

@@ -1,3 +1,4 @@
+import { notifyChatEndpointWork } from "../../chat-work-notifications.js";
 import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js";
 import { toConnection } from "./projection.js";
 import { actorBinding } from "../actor.js";
@@ -537,6 +538,7 @@ async function removeConnection(
       .returning({ id: connectionTokenIssuances.id });
 
     const archived = await db.transaction(async (tx) => {
+      if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id);
       await notifyToolConnectionHealth(tx, connection.companyId);
       await tx.select({ id: toolApplications.id }).from(toolApplications)
         .where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, connection.companyId)))