From a2ac5c27157ab033a2e0e2731b1b8e35bf20e7b9 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:03:23 -0500 Subject: [PATCH] refactor(chat): isolate Telegram, Slack, GitHub and Discord workflows (#15976) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip manages AI agents and their work. > - Chat channels connect that work to external conversations. > - The shared chat service still contains provider-specific workflows. > - Those workflows are hard to inspect in a 34,000-line file. > - This stack extracts Telegram, Slack, GitHub and Discord workflows into their channel folders. > - Shared authorization and service lifetime remain explicit. ## Linked Issues or Issue Description Refs #15823, #15834, #15836, #15844 and #15845. Root of the second behavior-preserving channel-folder refactor stack. The dependent PRs #15979, #15981, #15985 and #15987 have merged into this branch; this PR lands all five extractions on master. ## What Changed - Move Telegram maintenance and subscription repair into `chat-channels/telegram/maintenance.ts`. - Move Slack session stopping and task starts into `chat-channels/slack/session-stop.ts` and `task-start.ts`. - Move GitHub webhook intake and recovery into `chat-channels/github/webhook-ingress.ts`. - Move Discord command registration and handling into `chat-channels/discord/commands.ts`. - Share type-only runtime records while preserving shared authorization, transactions, live lease checks and worker ownership. - Reduce the original service from 34,603 to 29,897 lines, removing 4,706 lines (13.6%). ## Verification - All 205 selected integration cases passed before and after their respective extractions: Telegram 41, Slack session stop 2 multi-step scenarios, GitHub 84, Slack task start 25 and Discord 53. Each dependent PR records its focused reproduction command. - Server typecheck, module boundaries and whitespace checks passed. - Per-slice syntax comparisons passed for moved declarations and remaining service statements, allowing only the documented live callback substitutions. - Tests exercise the public service with disposable databases and fixture provider transports. They do not claim live third-party execution. - Cumulative workspace typecheck and build passed on the final stack. - Current-head CI passed on the combined stack commit `d5b37dbe512e3e199932b6608cdc17639eda555a`; Greptile is 5/5 with no unresolved threads. - The [original serialized shard](https://github.com/paperclipai/paperclip/actions/runs/38266518817/job/114870022249) timed out in unchanged `adapter-auth-signal-routes.test.ts`. The exact case passed locally without changes, and the [single job retry](https://github.com/paperclipai/paperclip/actions/runs/38266518817/job/114874019933) passed. The original timeout remains recorded. Reproduce the original Telegram slice’s focused suite: ```sh pnpm exec vitest run server/src/__tests__/chat-channels.integration.test.ts -t 'Telegram durable private draft Stop|Telegram maintenance|Telegram webhook|Telegram.*(pause|remove)|Telegram.*commands' ``` - Cumulative local `pnpm test:run` exited 1: the first server group had **18,770 passed, 105 skipped, and one timeout** in unchanged `server/src/services/workspace-git-snapshot-streaming.test.ts:56` (300,000 ms on macOS). The fail-fast runner did not reach its later workspace and serialized groups. An unchanged isolated rerun passed using the existing timeout; this does not turn the original full run green. The test and its Git utility code are outside this refactor; all current-head CI jobs passed. - Reproduce that diagnostic alone: `pnpm exec vitest run server/src/services/workspace-git-snapshot-streaming.test.ts`. ## Risks - Credential lease ownership and provider-confirmation ordering must remain unchanged. Existing rotation, lease-loss, recovery and durable Stop cases pass. - The channel factories create no workers or timers and preserve live runtime, shutdown and gateway-ownership reads. ## Model Used OpenAI Codex, GPT-6 family, with reasoning and repository tools. The exact runtime model ID and context-window size are not exposed in this session. No separate agent was used. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- ...6-10-12-chat-channel-workflows-refactor.md | 119 + server/src/services/chat-channels.ts | 4736 +---------------- .../chat-channels/discord/commands.ts | 639 +++ .../chat-channels/github/webhook-ingress.ts | 1579 ++++++ .../services/chat-channels/runtime-types.ts | 38 + .../chat-channels/slack/session-stop.ts | 858 +++ .../chat-channels/slack/task-start.ts | 1075 ++++ .../chat-channels/telegram/maintenance.ts | 931 ++++ 8 files changed, 5254 insertions(+), 4721 deletions(-) create mode 100644 doc/plans/2026-10-12-chat-channel-workflows-refactor.md create mode 100644 server/src/services/chat-channels/discord/commands.ts create mode 100644 server/src/services/chat-channels/github/webhook-ingress.ts create mode 100644 server/src/services/chat-channels/runtime-types.ts create mode 100644 server/src/services/chat-channels/slack/session-stop.ts create mode 100644 server/src/services/chat-channels/slack/task-start.ts create mode 100644 server/src/services/chat-channels/telegram/maintenance.ts diff --git a/doc/plans/2026-10-12-chat-channel-workflows-refactor.md b/doc/plans/2026-10-12-chat-channel-workflows-refactor.md new file mode 100644 index 00000000000..840b95ebf8b --- /dev/null +++ b/doc/plans/2026-10-12-chat-channel-workflows-refactor.md @@ -0,0 +1,119 @@ +# Chat channel workflow refactor, batch two + +Base: `37913e4907324838ae2c8c94dc8155002b6d4223` (fresh master after the previous stack merged). The previous five PRs +(#15823, #15834, #15836, #15844, #15845) are merged. The service remains +34,603 lines. This batch creates five dependent PRs; merging is separate work. + +| Order / responsibility | Destination | Approximate lines moved | Contract and ownership | Verification | +| --- | --- | --- | --- | --- | +| 1. Telegram maintenance and Stop subscription repair | `chat-channels/telegram/maintenance.ts` | 850 | Endpoint lookup, credential lease, credential resolution, runtime fence, live webhook URL, fetch. Own the per-service subscription scan cursor. Preserve claim/confirmation/settlement ordering. | Existing maintenance, rotation, lost-lease, subscription recovery and durable draft Stop integration cases before/after. | +| 2. Slack session stopping | `chat-channels/slack/session-stop.ts` | 750 | Stage and consume durable stop receipts; call shared principal authorization and task-control publication. Preserve captured target, event chronology and transaction locks. | Existing Slack stop, stale-session, authorization and failure-recovery cases before/after. | +| 3. GitHub durable webhook ingress and recovery | `chat-channels/github/webhook-ingress.ts` | 1,400 | Persist verified ingress, replay through the shared webhook entry point, recover provider failures. Keep persistence lease, live shutdown check and current endpoint/credential fences. | Existing signed webhook, restart/replay, recovery and stale-source integration cases before/after. | +| 4. Slack task-start admission | `chat-channels/slack/task-start.ts` | 920 | Recover task starter receipt, revalidate principal/resource/runtime and admit via the shared message path. Preserve lease ownership, synthetic message identity and claim barrier. | Existing slash-task concurrency, partial-post and recovery cases before/after. | +| 5. Discord native commands | `chat-channels/discord/commands.ts` | 550 | Reconcile command registration and admit native controls; shared service retains gateway ownership and authorization. Read gateway ownership live. | Existing command registration, stale runtime, gateway ownership and native command cases before/after. | + +Keep message admission, shared control authority, provider runtime startup, +publication transport and service shutdown together. Each factory receives named, +typed dependencies. Do not copy mutable maps or snapshot live values. Shared +runtime record types may move to a type-only sibling module; do not introduce a +runtime import back to the original service. + +For each slice: baseline existing public-service tests, mechanical extraction, +syntax/body parity, same tests afterward, server typecheck and module boundaries, +then PR checks and current-head review. Run broad local gates once at final +handoff. Record failures honestly; a retry does not erase the original failure. + +## Slice 1: Telegram maintenance + +- Service: 34,603 lines / 1,337,182 bytes → 33,718 lines / 1,306,801 bytes. +- `telegram/maintenance.ts`: 931 lines / 32,792 bytes; type-only shared runtime + records: 38 lines / 1,200 bytes. +- Twelve moved declarations and six shared types preserve token-level syntax; + all remaining service statements are unchanged. The module owns the existing + scan cursor and creates no timers or background work. Its live callbacks retain + credential-lease checks and runtime fencing around every provider mutation. +- The same 41 existing Telegram public-service cases passed before and after. + Server typecheck, module boundaries and whitespace checks passed. Full local + repository gates are deferred to the cumulative handoff; remote CI/review will + be recorded in this slice's PR. +- Reassessment: keep Telegram draft Stop admission and publication transport in + the orchestrator. Slack durable session stopping remains the next cohesive cut. + +## Slice 2: Slack session stopping + +- Service: 33,718 lines / 1,306,801 bytes → 32,928 lines / 1,279,554 bytes. +- `slack/session-stop.ts`: 858 lines / 30,166 bytes. +- All six moved declarations and all remaining service statements preserve + token-level syntax. A getter retains the original live heartbeat property + access. Shared principal authorization and publication staging stay in the + caller; the entire captured-target claim and settlement transactions move. +- Both existing public-service scenarios passed before and after. They cover + exact-run cancellation, retry/deduplication, guest denial, delayed events, + queued wakeups, a wakeup promoted after Stop, and cancellation losing to run + completion. Server typecheck passes. This factory owns no mutable state. +- Reassessment: GitHub ingress/recovery has a coherent durable inbox and replay + contract; preserve its shutdown getter and persistence compare-and-set lease. + +## Slice 3: GitHub webhook ingress and recovery + +- Service: 32,928 lines / 1,279,554 bytes → 31,438 lines / 1,225,374 bytes. +- `github/webhook-ingress.ts`: 1,579 lines / 58,233 bytes. +- All 26 moved declarations and remaining statements preserve syntax after the + explicit shutdown getter substitution. The replay barrier remains a live + getter. Persistence and credential leases remain the same service instances. +- The same 84 existing public-service cases passed before and after, including + authentication before retention, disabled-repository filtering, restart, + recovery epochs, concurrent scans, ambiguous redelivery, operator retries, + canonical-source changes and stale-runtime replay. Server typecheck passes. +- Reassessment: Slack task starter confirmation, recovery and failure settlement + form one boundary. Keep normal message admission shared and pass its exact + existing call contract; do not duplicate provider-visible posts during recovery. + +## Slice 4: Slack task starts + +- Service: 31,438 lines / 1,225,374 bytes → 30,449 lines / 1,190,643 bytes. +- `slack/task-start.ts`: 1,075 lines / 39,262 bytes. +- Nine moved declarations and all remaining statements preserve syntax. The + existing positional message-admission contract is unchanged. The claim barrier + remains live through a getter; no worker or mutable per-service state moves. +- The same 25 public-service cases passed before and after, covering slash-task + admission, provider-confirmed recovery without reposting, reclaimed admission, + stale credential ownership, ambiguous sends and explicit operator retries. + Server typecheck passes. +- Reassessment: Discord command registration and native control admission can + share a channel module. Expose only a live lease-key/token lookup; retain the + gateway ownership map, renewal, partial acquisition and shutdown in the service. + +## Slice 5: Discord commands + +- Service: 30,449 lines / 1,190,643 bytes → 29,897 lines / 1,170,080 bytes. +- `discord/commands.ts`: 639 lines / 25,036 bytes. +- Both moved function bodies and all remaining statements preserve syntax after + replacing the map read with a live lease-key/token callback. Gateway map, + renewal, runtime identity, partial startup and shutdown ownership stay shared. +- The same 53 registration and native-command integration cases passed before + and after. They cover registration failure, stale/runtime/gateway authority, + command receipts, exact target replay and current principal/resource access. + Server typecheck passes. Network work remains outside command transactions. + +## Reassessment after batch two + +The service falls from 34,603 lines / 1,337,182 bytes to 29,897 lines / +1,170,080 bytes: 4,706 lines and 167,102 bytes removed (13.6% of source lines). +The five provider modules total 5,082 lines, plus 38 shared type lines. + +The service function itself spans 27,835 lines. Its 212 nested functions account +for 27,146 lines; 51 state/wiring/registration/public API statements account for +435 lines. Imports account for 423 lines. This batch reduces provider workflow +mixing; it does not resolve the large shared admission functions. + +The largest remaining functions are `processMessage` (2,166 lines), +`handleAction` (1,074), `configureWithCredentialLease` (868), +`processSelectedPublication` (729), `handleModalSubmit` (708), `handleWebhook` +(685), `processLifecycleDelivery` (657), and `applyProviderLifecycleEffect` (593). +Future candidates include Teams file-transfer authority/protocol and shared +inbound persistence/drain ownership. Preserve the shared transaction and live +message contracts before attempting those cuts. Stop this batch at five PRs. + +Cumulative verification: `pnpm -r typecheck` and `pnpm build` passed. Full local +test suite results and current-head CI/review are recorded in the final PR. diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index fa5c85cea27..0cfbfa5fdbf 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -1,8 +1,14 @@ +import { createDiscordCommands } from "./chat-channels/discord/commands.js"; +import { createSlackTaskStart, slackSlashTaskRecoveryPayload, slackTaskStarterThreadId } from "./chat-channels/slack/task-start.js"; +import { createGitHubWebhookIngress, GITHUB_RECOVERY_STATE_KEY, type GitHubIngressStageResult } from "./chat-channels/github/webhook-ingress.js"; +import { createSlackSessionStop } from "./chat-channels/slack/session-stop.js"; +import { createTelegramMaintenance } from "./chat-channels/telegram/maintenance.js"; +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence, InboundRuntimeContext } from "./chat-channels/runtime-types.js"; import { createChatFailedRunRetry, type ChatFailedRetrySource as FailedChatRetrySource } from "./chat-channels/failed-run-retry.js"; import { createChatPublicationReplacement } from "./chat-channels/publication-replacement.js"; import { createChatIdentityLinks } from "./chat-channels/identity-links.js"; import { createChatResourceReconciliation } from "./chat-channels/resource-reconciliation.js"; -import { githubRepositoryInventoryItemFromPayload } from "./chat-channels/github/resources.js"; + import { baseTeamsConversationId } from "./chat-channels/microsoft-teams/resources.js"; import { slackResourceLabelIsFallback } from "./chat-channels/slack/resources.js"; import { telegramResourceLabelIsFallback } from "./chat-channels/telegram/resources.js"; @@ -37,7 +43,7 @@ import { type GitHubLifecycleEvent, githubLifecycleEventFromRequest, } from "./chat-channels/github/messages.js"; -import { dueChatAction, nextChatActionAt, SLACK_COMMAND_POST_STALE_MS, SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS, SLACK_COMMAND_ADMISSION_STALE_MS, PROVIDER_EFFECT_STALE_MS } from "./chat-action-work.js"; +import { dueChatAction, nextChatActionAt, SLACK_COMMAND_POST_STALE_MS, SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS, PROVIDER_EFFECT_STALE_MS } from "./chat-action-work.js"; import type { ScheduledChatAction } from "./chat-work-notifications.js"; import { notifyChatDeliveryWork, notifyChatPublicationWork, notifyChatEndpointWork, notifyChatActionWork, notifyChatVerificationWork } from "./chat-work-notifications.js"; import { syncSpekoVoiceTools } from "./voice/speko-agent-tools.js"; @@ -82,7 +88,6 @@ import { githubAutomaticIssueEvent, githubAutomaticAdmission, githubPreviousAssessment, - githubBodyMentionsBot, githubExplicitMentionEvent, githubMessageAddressesAnotherBot, } from "./chat-github-events.js"; @@ -258,10 +263,7 @@ import { type TelegramDraftControl, type TelegramGenerationStoppedProof, } from "./chat-telegram-draft-stop.js"; -import { - telegramStopSubscriptionPlan, - telegramStopSubscriptionConfirmed, -} from "./chat-telegram-stop-subscription.js"; + import { parseChatWebhookPublicBaseUrl } from "../chat-webhook-public-url.js"; import { @@ -287,16 +289,11 @@ import { type ChatSdkProvider, type ChatSdkRuntime, type DiscordGatewayCallbackEvent, - type DiscordNativeCommandResponse, type DiscordRootMentionAdmissionEvent, type ResolvedChatSdkProviderConfig, type SlackFileUploadAcceptedReceipt, } from "./chat-sdk-runtime.js"; -import { - parseDiscordNativeCommand, - parseDiscordNativeCommandReceipt, - type DiscordNativeCommandTarget, -} from "./chat-discord-native-commands.js"; + import type { ChatSdkStateCompareAndSetInput, ChatSdkStateDeleteInput, @@ -323,12 +320,7 @@ import { safeChatTaskUrl } from "./chat-task-url.js"; import { resyncGitHubAppWebhook, listGitHubAppWebhookDeliveries, - getGitHubAppWebhookDelivery, - readGitHubAppWebhookConfig, requestGitHubAppWebhookRedelivery, - getGitHubRecoveryComment, - type GitHubAppWebhookDelivery, - type GitHubAppWebhookDeliveryDetail, } from "./chat-github-webhook-config.js"; import { GITHUB_ATTACHMENT_BATCH_TIMEOUT_MS, @@ -350,7 +342,6 @@ import { } from "./chat-publication-text-parts.js"; import { readRegisteredDiscordCommandRegistration, - reconcileStoredDiscordCommandRegistration, } from "./chat-discord-command-registration-store.js"; import { deleteDiscordQuestionFormCorrection, @@ -365,7 +356,6 @@ import { parseSlackSessionStop, setSlackSessionStatus, slackSessionStatusForPublication, - type SlackSessionStop, } from "./chat-slack-sessions.js"; import { slackSessionSyncPayload, @@ -385,9 +375,7 @@ import { import { chatProviderConversationUrl } from "./chat-provider-links.js"; import { classifyChatPublicationError } from "./chat-publication-errors.js"; import { - enqueueChatRunMilestones, githubRunReplyState, - safeMilestoneText, } from "./chat-run-publications.js"; import { inboundWakePublicationKey, @@ -829,13 +817,6 @@ const REQUIRED_GITHUB_PERMISSIONS = { pull_requests: "write", } as const; -const TELEGRAM_COMMANDS = [ - { command: "task", description: "Start or continue a Paperclip task" }, - { command: "status", description: "Show the active Paperclip task" }, - { command: "new", description: "Start a new task after the current one" }, - { command: "close", description: "Close the active chat conversation" }, -] as const; - const UNAVOIDABLE_GITHUB_EVENTS = [ "github_app_authorization", "installation", @@ -912,11 +893,7 @@ const INGRESS_REORDER_WINDOW_MS: Partial> = { // non-sliding window without serializing separate chats. telegram: 750, }; - -type EndpointRow = typeof chatEndpoints.$inferSelect; type ResourceRow = typeof chatEndpointResources.$inferSelect; -type DbTransaction = Parameters[0]>[0]; -type DbOrTransaction = Db | DbTransaction; type SlackCallbackSurface = keyof ChatEndpointCallbackSurfaces; type SlackCallbackObservation = { url: string; observedAt: string }; type InternalSetupState = ChatEndpointSetupState & { @@ -928,24 +905,6 @@ type InternalSetupState = ChatEndpointSetupState & { Record >; }; -type RuntimeContext = { - credentialFingerprint: string; - discordGatewayEventQueue?: Promise; - discordGatewaySequence?: number; - discordGatewayOwned?: boolean; - /** Cache identity only; durable command admission rechecks ownership. */ - discordCommandId?: string; - endpointRuntime?: ChatEndpointRuntime; - generation: number; - localEpoch: number; - version: string; -}; -type LifecycleRuntimeFence = Pick< - RuntimeContext, - "credentialFingerprint" | "generation" ->; -type InboundRuntimeContext = LifecycleRuntimeFence & - Pick; type VerifiedProviderIdentity = { githubOwnerType?: "personal" | "organization"; githubOwnerLogin?: string; @@ -956,38 +915,6 @@ type VerifiedProviderIdentity = { botLabel?: string | null; }; -const GITHUB_RECOVERY_STATE_KEY = "paperclip:github-webhook-recovery:v1"; -const GITHUB_RECOVERY_INTERVAL_MS = 60_000; -const GITHUB_RECOVERY_WINDOW_MS = 60 * 60_000; -const GITHUB_RECOVERY_SCAN_LEASE_MS = 5 * 60_000; -type GitHubRecoveryWindow = LifecycleRuntimeFence & { - appId: string; - webhookUrl: string; - floor: string; - nextScanAt: string; -}; -type GitHubRecoveryReceipt = GitHubRecoveryWindow & { - original: GitHubAppWebhookDeliveryDetail; -}; - -function githubRecoveryWindow(value: unknown): GitHubRecoveryWindow | null { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const item = value as GitHubRecoveryWindow; - return Number.isSafeInteger(item.generation) && - item.generation >= 0 && - typeof item.credentialFingerprint === "string" && - /^[a-f0-9]{64}$/.test(item.credentialFingerprint) && - typeof item.appId === "string" && - /^\d+$/.test(item.appId) && - typeof item.webhookUrl === "string" && - typeof item.floor === "string" && - Number.isFinite(Date.parse(item.floor)) && - typeof item.nextScanAt === "string" && - Number.isFinite(Date.parse(item.nextScanAt)) - ? item - : null; -} - function runtimeGeneration(setup: ChatEndpointSetupState): number { const value = Number((setup as InternalSetupState).runtimeGeneration ?? 0); return Number.isSafeInteger(value) && value >= 0 ? value : 0; @@ -1230,97 +1157,6 @@ function sameSlackFileUploadReceipt( ); } -type GitHubWebhookIngressPayload = { - version: 1; - body: string; - bodySha256: string; - contentType: string; - deliveryId: string; - eventType: string; - runtimeGeneration: number; - credentialFingerprint: string; -}; - -type RedactedGitHubWebhookIngressPayload = Omit< - GitHubWebhookIngressPayload, - "body" | "contentType" -> & { - redacted: true; -}; - -type TelegramMaintenancePayload = { - version: 1; - operation: "register_commands" | "remove_endpoint" | "stop_subscription"; - runtimeGeneration: number; - credentialFingerprint: string; - webhookUrlSha256?: string; - botUserId?: string; -}; - -function telegramMaintenancePayload( - payload: Record, -): TelegramMaintenancePayload | null { - if ( - payload.version !== 1 || - (payload.operation !== "register_commands" && - payload.operation !== "remove_endpoint" && - payload.operation !== "stop_subscription") || - typeof payload.runtimeGeneration !== "number" || - !Number.isSafeInteger(payload.runtimeGeneration) || - typeof payload.credentialFingerprint !== "string" || - (payload.operation === "stop_subscription" && - (typeof payload.webhookUrlSha256 !== "string" || - !/^[a-f0-9]{64}$/.test(payload.webhookUrlSha256) || - typeof payload.botUserId !== "string" || - !/^[1-9][0-9]{0,15}$/.test(payload.botUserId))) - ) { - return null; - } - return payload as TelegramMaintenancePayload; -} - -function githubWebhookIngressPayload( - payload: Record, -): GitHubWebhookIngressPayload | null { - if ( - payload.version !== 1 || - typeof payload.body !== "string" || - typeof payload.bodySha256 !== "string" || - !/^[a-f0-9]{64}$/.test(payload.bodySha256) || - typeof payload.contentType !== "string" || - typeof payload.deliveryId !== "string" || - !payload.deliveryId || - typeof payload.eventType !== "string" || - !payload.eventType || - typeof payload.runtimeGeneration !== "number" || - !Number.isSafeInteger(payload.runtimeGeneration) || - typeof payload.credentialFingerprint !== "string" - ) { - return null; - } - if ( - createHash("sha256").update(payload.body).digest("hex") !== - payload.bodySha256 - ) { - return null; - } - return payload as GitHubWebhookIngressPayload; -} - -function redactedGitHubWebhookIngressPayload( - payload: GitHubWebhookIngressPayload, -): RedactedGitHubWebhookIngressPayload { - return { - version: 1, - bodySha256: payload.bodySha256, - deliveryId: payload.deliveryId, - eventType: payload.eventType, - runtimeGeneration: payload.runtimeGeneration, - credentialFingerprint: payload.credentialFingerprint, - redacted: true, - }; -} - function isSlackReceiptReactionAlreadyApplied(error: unknown): boolean { const pending: Array<{ depth: number; value: unknown }> = [ { depth: 0, value: error }, @@ -1734,110 +1570,6 @@ function safeTitle(text: string, fallback: string): string { return (line || fallback).slice(0, 160); } -type SlackSlashTaskRecoveryPayload = { - channelId: string; - command: string; - syntheticMessageId: string; - taskText: string; -}; - -type SlackSessionStopTarget = - { id: string; kind: "run" } | { id: string; kind: "wakeup" }; - -type SlackSessionStopPayload = { - version: 1; - assignedAgentId: string; - eventTimestamp: string; - issueId: string | null; - occurredAt: string; - sessionGeneration: number | null; - target: SlackSessionStopTarget | null; - threadId: string; - userId: string; -}; - -function slackSessionStopPayload( - value: Record, -): SlackSessionStopPayload | null { - const target = value.target; - const parsedTarget = - target === null - ? null - : target && - typeof target === "object" && - !Array.isArray(target) && - ((target as { kind?: unknown }).kind === "run" || - (target as { kind?: unknown }).kind === "wakeup") && - typeof (target as { id?: unknown }).id === "string" && - isUuidLike((target as { id: string }).id) - ? (target as SlackSessionStopTarget) - : undefined; - if ( - value.version !== 1 || - typeof value.assignedAgentId !== "string" || - !isUuidLike(value.assignedAgentId) || - typeof value.eventTimestamp !== "string" || - !value.eventTimestamp || - (value.issueId !== null && - (typeof value.issueId !== "string" || !isUuidLike(value.issueId))) || - typeof value.occurredAt !== "string" || - !Number.isFinite(new Date(value.occurredAt).getTime()) || - (value.sessionGeneration !== null && - (typeof value.sessionGeneration !== "number" || - !Number.isSafeInteger(value.sessionGeneration) || - value.sessionGeneration < 1)) || - (parsedTarget !== null && - (value.issueId === null || value.sessionGeneration === null)) || - parsedTarget === undefined || - typeof value.threadId !== "string" || - !value.threadId.startsWith("slack:") || - typeof value.userId !== "string" || - !value.userId - ) { - return null; - } - return { - version: 1, - assignedAgentId: value.assignedAgentId, - eventTimestamp: value.eventTimestamp, - issueId: value.issueId, - occurredAt: value.occurredAt, - sessionGeneration: value.sessionGeneration, - target: parsedTarget, - threadId: value.threadId, - userId: value.userId, - }; -} - -function slackSlashTaskRecoveryPayload( - payload: Record, -): SlackSlashTaskRecoveryPayload | null { - const channelId = payload.channelId; - const command = payload.command; - const syntheticMessageId = payload.syntheticMessageId; - const taskText = payload.taskText; - if ( - typeof channelId !== "string" || - !channelId.trim() || - typeof command !== "string" || - !command.trim() || - typeof syntheticMessageId !== "string" || - !syntheticMessageId.trim() || - typeof taskText !== "string" || - !taskText.trim() - ) { - return null; - } - return { channelId, command, syntheticMessageId, taskText }; -} - -function slackTaskStarterThreadId( - channelId: string, - providerMessageId: string, -): string { - return `slack:${channelId.replace(/^slack:/, "")}:${providerMessageId}`; -} - function slackThreadChannelId(threadId: string): string | null { if (!threadId.startsWith("slack:")) return null; const channelAndThread = threadId.slice("slack:".length); @@ -1881,42 +1613,6 @@ function slackTaskStartActivityStatus(action: { return action.status; } -function confirmedSlackTaskStartResult( - result: Record | null, -): { - authorizedUserId?: string | null; - providerMessageId: string; - threadId: string; -} | null { - if ( - !result || - typeof result.providerMessageId !== "string" || - !result.providerMessageId || - typeof result.threadId !== "string" || - !result.threadId - ) { - return null; - } - const hasAuthorizedUserId = Object.prototype.hasOwnProperty.call( - result, - "authorizedUserId", - ); - if ( - hasAuthorizedUserId && - result.authorizedUserId !== null && - (typeof result.authorizedUserId !== "string" || !result.authorizedUserId) - ) { - return null; - } - return { - ...(hasAuthorizedUserId - ? { authorizedUserId: result.authorizedUserId as string | null } - : {}), - providerMessageId: result.providerMessageId, - threadId: result.threadId, - }; -} - function sanitizeFilename(value: string | undefined): string | null { if (!value) return null; const leaf = value.replaceAll("\\", "/").split("/").pop()?.trim(); @@ -2375,7 +2071,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // Scheduling-only cooldown for rejected/contended staged work. Do not alter // an unverified transfer or its publication merely because a worker saw it. const teamsFileRetryAfter = new Map(); - let telegramSubscriptionScanAfter: string | null = null; const runtimeContexts = new WeakMap(); const runtimeInitializations = new Map< string, @@ -3424,539 +3119,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return actions.length; } - async function stageTelegramMaintenance( - endpoint: EndpointRow, - operation: TelegramMaintenancePayload["operation"], - ) { - const record = await endpointRecord(endpoint.id); - if (!record) throw notFound("Chat endpoint not found"); - const context = runtimeContextForRecord(record); - const subscriptionScope = - operation === "stop_subscription" - ? telegramStopSubscriptionScope(record.endpoint, context) - : null; - if (operation === "stop_subscription" && !subscriptionScope) return null; - const operationGeneration = subscriptionScope - ? `${context.generation}:${context.credentialFingerprint}:${subscriptionScope.botUserId}:${subscriptionScope.webhookUrlSha256}` - : operation === "register_commands" - ? `${context.generation}:${record.endpoint.setup.testStartedAt ?? record.endpoint.updatedAt.toISOString()}` - : String(context.generation); - const providerActionId = `telegram_maintenance:${operation}:${operationGeneration}`; - const [inserted] = await db - .insert(chatActions) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - kind: "telegram_maintenance", - providerActionId, - payload: { - version: 1, - operation, - runtimeGeneration: context.generation, - credentialFingerprint: context.credentialFingerprint, - ...(subscriptionScope - ? { - webhookUrlSha256: subscriptionScope.webhookUrlSha256, - botUserId: subscriptionScope.botUserId, - } - : {}), - } satisfies TelegramMaintenancePayload, - status: "received", - }) - .onConflictDoNothing() - .returning(); - return ( - inserted ?? - (await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, endpoint.id), - eq(chatActions.providerActionId, providerActionId), - ), - ) - .then((rows) => rows[0] ?? null)) - ); - } - - async function telegramMaintenanceRequest( - botToken: string, - method: "deleteWebhook" | "deleteMyCommands" | "setMyCommands", - body: Record, - ): Promise { - const response = await fetchImpl( - `https://api.telegram.org/bot${encodeURIComponent(botToken)}/${method}`, - { - method: "POST", - signal: AbortSignal.timeout(PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS), - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - }, - ); - const result = (await response.json()) as { - ok?: boolean; - description?: string; - parameters?: { retry_after?: number }; - }; - if (response.ok && result.ok) return; - throw Object.assign( - new Error( - `Telegram ${method} failed: ${result.description ?? response.status}`, - ), - response.status === 429 - ? { - adapter: "telegram", - code: "RATE_LIMITED", - name: "AdapterRateLimitError", - retryAfter: result.parameters?.retry_after, - response: { status: response.status, headers: response.headers }, - } - : { - adapter: "telegram", - status: response.status, - response: { status: response.status, headers: response.headers }, - }, - ); - } - - async function processTelegramMaintenance( - actionId: string, - force = false, - credentialLease?: CredentialMutationLeaseGuard, - ): Promise<"processed" | "pending" | "failed"> { - if (!credentialLease) { - const initialAction = await db - .select({ endpointId: chatActions.endpointId }) - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "telegram_maintenance"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!initialAction) return "failed"; - const initialEndpoint = await endpointRecord(initialAction.endpointId); - if (!initialEndpoint) return "failed"; - // Recovery sweeps run independently from endpoint configuration. Hold - // the same durable credential lease as reconnect, pause, and removal so - // a stale bot token cannot mutate Telegram after its final runtime fence - // but before transport. - return withCredentialMutationLease(initialEndpoint.endpoint, (lease) => - processTelegramMaintenance(actionId, force, lease), - ); - } - let action = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "telegram_maintenance"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!action) return "failed"; - if (action.status === "processed") return "processed"; - const payload = telegramMaintenancePayload(action.payload); - if (!payload) { - await db - .update(chatActions) - .set({ - status: "failed", - result: { - code: "telegram_maintenance_payload_invalid", - retryable: false, - }, - updatedAt: new Date(), - }) - .where(eq(chatActions.id, action.id)); - return "failed"; - } - if (action.status === "processing") { - if (action.updatedAt > new Date(Date.now() - PROVIDER_EFFECT_STALE_MS)) - return "pending"; - await db - .update(chatActions) - .set({ - status: "failed", - result: { - ...(action.result ?? {}), - retryable: true, - retryAt: new Date().toISOString(), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - eq(chatActions.updatedAt, action.updatedAt), - ), - ); - action = await db - .select() - .from(chatActions) - .where(eq(chatActions.id, action.id)) - .then((rows) => rows[0] ?? null); - if (!action) return "failed"; - } - if (action.status === "failed") { - if (action.result?.retryable !== true && !force) return "failed"; - const retryAt = - typeof action.result?.retryAt === "string" - ? new Date(action.result.retryAt) - : null; - if (!force && retryAt && retryAt > new Date()) return "pending"; - } else if (action.status !== "received") { - return "failed"; - } - const attempt = - (typeof action.result?.attempts === "number" - ? action.result.attempts - : 0) + 1; - let providerConfirmed = action.result?.providerConfirmed === true; - const [claimed] = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const rows = await tx - .update(chatActions) - .set({ - status: "processing", - result: { attempts: attempt, providerConfirmed }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, action!.status), - ), - ) - .returning(); - await credentialLease.assertOwned(tx); - return rows; - }); - if (!claimed) return "pending"; - - try { - const record = await endpointRecord(action.endpointId); - if (!record || record.endpoint.provider !== "telegram") { - throw Object.assign(new Error("Telegram endpoint is unavailable"), { - code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED", - }); - } - if ( - payload.operation === "remove_endpoint" && - record.credentialSecretRefs.length === 0 - ) { - // Recovery may observe the narrow crash window after provider cleanup - // and credential clearing but before the action receipt was committed. - // Empty refs prove removal cleanup crossed its final local boundary; - // settle the idempotent action instead of misclassifying it as a - // credential rotation. - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await tx - .update(chatActions) - .set({ - status: "processed", - result: { attempts: attempt, providerConfirmed: true }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ); - await credentialLease.assertOwned(tx); - }); - return "processed"; - } - if ( - credentialFingerprint(record.credentialSecretRefs) !== - payload.credentialFingerprint || - (payload.operation !== "remove_endpoint" && - runtimeGeneration(record.endpoint.setup) !== - payload.runtimeGeneration) - ) { - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await tx - .update(chatActions) - .set({ - status: "cancelled", - result: { - attempts: attempt, - code: "telegram_maintenance_credentials_superseded", - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ); - await credentialLease.assertOwned(tx); - }); - return "failed"; - } - if (!providerConfirmed || payload.operation === "stop_subscription") { - const credentials = await resolveCredentials(record.endpoint); - if (!credentials.botToken) { - throw Object.assign(new Error("Telegram bot token is unavailable"), { - code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED", - }); - } - if (payload.operation === "stop_subscription") { - await repairTelegramStopSubscription( - record.endpoint, - payload, - credentialLease, - ); - providerConfirmed = true; - } else if (payload.operation === "register_commands") { - await credentialLease.assertOwned(); - await telegramMaintenanceRequest( - credentials.botToken, - "setMyCommands", - { commands: TELEGRAM_COMMANDS }, - ); - providerConfirmed = true; - } else { - await credentialLease.assertOwned(); - await telegramMaintenanceRequest( - credentials.botToken, - "deleteWebhook", - { drop_pending_updates: false }, - ); - await credentialLease.assertOwned(); - await telegramMaintenanceRequest( - credentials.botToken, - "deleteMyCommands", - {}, - ); - providerConfirmed = true; - } - await credentialLease.assertOwned(); - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const confirmed = await tx - .update(chatActions) - .set({ - result: { attempts: attempt, providerConfirmed: true }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!confirmed.length) { - throw new Error( - "Telegram maintenance ownership changed before provider confirmation", - ); - } - await credentialLease.assertOwned(tx); - }); - } - if (payload.operation === "remove_endpoint") { - await credentialLease.assertOwned(); - await clearCredentials(record.endpoint, credentialLease); - } - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const processed = await tx - .update(chatActions) - .set({ - status: "processed", - result: { attempts: attempt, providerConfirmed: true }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!processed.length) { - throw new Error( - "Telegram maintenance ownership changed before success settlement", - ); - } - await credentialLease.assertOwned(tx); - }); - await logActivity(db, { - companyId: record.endpoint.companyId, - actorType: "system", - actorId: "chat-maintenance", - action: `chat_endpoint.telegram_${payload.operation}_completed`, - entityType: "tool_connection", - entityId: record.endpoint.connectionId, - details: { endpointId: record.endpoint.id, attempt }, - }); - return "processed"; - } catch (error) { - const disposition = classifyChatPublicationError(error, attempt); - // Webhook/menu mutations are idempotent provider operations. A lost - // response is safe to retry, and an explicit provider Retry-After must - // remain durable rather than occupying an in-memory timer. - const retryable = - disposition.kind === "retry" || disposition.kind === "delivery_unknown"; - const retryAfterMs = - disposition.kind === "retry" - ? disposition.retryAfterMs - : Math.min(60_000, 1_000 * 2 ** Math.max(0, attempt - 1)); - const retryAt = retryable - ? new Date(Date.now() + retryAfterMs).toISOString() - : null; - const diagnostic = redactError(error); - await db - .update(chatActions) - .set({ - status: "failed", - result: { - attempts: attempt, - code: `telegram_maintenance_${disposition.kind}`, - providerConfirmed: - providerConfirmed || action.result?.providerConfirmed === true, - redactedError: diagnostic, - retryable, - retryAt, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ); - await logActivity(db, { - companyId: action.companyId, - actorType: "system", - actorId: "chat-maintenance", - action: `chat_endpoint.telegram_${payload.operation}_deferred`, - entityType: "tool_connection", - entityId: - (await endpointRecord(action.endpointId))?.endpoint.connectionId ?? - action.endpointId, - details: { - endpointId: action.endpointId, - attempt, - disposition: disposition.kind, - retryAt, - }, - }).catch((auditError) => { - logger.error( - { endpointId: action!.endpointId, error: redactError(auditError) }, - "could not persist Telegram maintenance deferral audit", - ); - }); - logger.warn( - { - endpointId: action.endpointId, - operation: payload.operation, - retryAt, - }, - "Telegram maintenance was deferred", - ); - return retryable ? "pending" : "failed"; - } - } - - async function processPendingTelegramMaintenance(limit = 25) { - // Keyset scanning bounds each sweep without starving endpoints behind a - // page whose subscriptions were already confirmed. No provider I/O here. - const candidates = await db - .select({ endpoint: chatEndpoints }) - .from(chatEndpoints) - .innerJoin( - toolConnections, - and( - eq(toolConnections.id, chatEndpoints.connectionId), - eq(toolConnections.companyId, chatEndpoints.companyId), - ), - ) - .where( - and( - eq(chatEndpoints.provider, "telegram"), - eq(chatEndpoints.status, "active"), - eq(toolConnections.status, "active"), - eq(toolConnections.enabled, true), - telegramSubscriptionScanAfter - ? gt(chatEndpoints.id, telegramSubscriptionScanAfter) - : undefined, - ), - ) - .orderBy(asc(chatEndpoints.id)) - .limit(Math.max(1, Math.min(limit, 100))); - telegramSubscriptionScanAfter = - candidates.length >= Math.max(1, Math.min(limit, 100)) - ? candidates.at(-1)!.endpoint.id - : null; - for (const { endpoint } of candidates) { - try { - const record = await endpointRecord(endpoint.id); - if ( - record && - !(await hasTelegramStopSubscription( - record.endpoint, - runtimeContextForRecord(record), - )) - ) - await stageTelegramMaintenance(record.endpoint, "stop_subscription"); - } catch { - logger.warn( - { endpointId: endpoint.id }, - "Telegram subscription maintenance could not be staged", - ); - } - } - const now = new Date(); - const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); - const actions = await db - .select({ id: chatActions.id }) - .from(chatActions) - .where( - and( - eq(chatActions.kind, "telegram_maintenance"), - sql`(${chatActions.payload}->>'operation' <> 'stop_subscription' or exists ( - select 1 from ${chatEndpoints} inner join ${toolConnections} - on ${toolConnections.id} = ${chatEndpoints.connectionId} and ${toolConnections.companyId} = ${chatEndpoints.companyId} - where ${chatEndpoints.id} = ${chatActions.endpointId} and ${chatEndpoints.companyId} = ${chatActions.companyId} - and ${chatEndpoints.provider} = 'telegram' and ${chatEndpoints.status} = 'active' - and ${toolConnections.status} = 'active' and ${toolConnections.enabled} = true - ))`, - or( - eq(chatActions.status, "received"), - and( - eq(chatActions.status, "processing"), - lte(chatActions.updatedAt, staleBefore), - ), - and( - eq(chatActions.status, "failed"), - sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`, - sql`(${chatActions.result}->>'retryAt' is null or (${chatActions.result}->>'retryAt')::timestamptz <= ${now.toISOString()}::timestamptz)`, - ), - ), - ), - ) - .orderBy(asc(chatActions.createdAt)) - .limit(limit); - for (const action of actions) await processTelegramMaintenance(action.id); - return actions.length; - } - async function stageProviderEffect( database: DbOrTransaction, input: { @@ -6186,6 +5348,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const withCredentialMutationLease = chatCredentialMutationLease(db, options); + const { reconcileDiscordCommands, handleDiscordNativeCommand } = createDiscordCommands(db, { endpointRecord, runtimeContextForRecord, runtimeGeneration, credentialFingerprint, resolveCredentialRefs, fetchImpl, withCredentialMutationLease, runtimeCallbackRecord, ensurePrincipal, runtimeCallbackEndpoint, nonDirectDestinationAllowed, lockCurrentPrincipalAuthorization, getDiscordGatewayOwnership: (endpointId) => discordGatewayOwnerships.get(endpointId), discordGatewayRuntimeIsCurrent, stageAuthorizedTaskControlPublication, scheduleMessageProcessing, processPendingPublications }); + const { latestSlackDmControlThreadId, finalizeSlackTaskStartFailure, admitConfirmedSlackTaskStart, processSlackTaskStart, processPendingSlackTaskStarts } = createSlackTaskStart(db, { endpointRecord, runtimeGeneration, credentialFingerprint, invalidateRuntime, runtimeFor, processMessage, runtimeContextForRecord, redactError, withCredentialMutationLease, sponsorAllowsGuest, nonDirectDestinationAllowed, runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, get slackTaskAdmissionClaimBarrier() { return options.slackTaskAdmissionClaimBarrier; }, maxErrorText: MAX_ERROR_TEXT }); + const { stageGitHubWebhookIngress, githubIngressAcceptedResponse, processGitHubWebhookIngress, processFailedGitHubWebhookDeliveries, processPendingGitHubWebhookIngress } = createGitHubWebhookIngress(db, { endpointRecord, runtimeContextForRecord, resolveCredentialRefs, runtimeGeneration, credentialFingerprint, getWebhookPublicBaseUrl, redactError, handleWebhook, runtimeCallbackEndpoint, fetchImpl, persistence, withCredentialMutationLease, isShuttingDown: () => shuttingDown, get githubWebhookReplayBarrier() { return options.githubWebhookReplayBarrier; }, supportedWebhookEvents: SUPPORTED_GITHUB_WEBHOOK_EVENTS }); + const { stageSlackSessionStop, processSlackSessionStop, processPendingSlackSessionStops } = createSlackSessionStop(db, { runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, nonDirectDestinationAllowed, stageAuthorizedTaskControlPublication, getPublicBaseUrl, get heartbeat() { return options.heartbeat; }, redactError }); + const { stageTelegramMaintenance, processTelegramMaintenance, processPendingTelegramMaintenance, hasTelegramStopSubscription, recordTelegramStopSubscription } = createTelegramMaintenance(db, { endpointRecord, runtimeContextForRecord, withCredentialMutationLease, resolveCredentials, clearCredentials, runtimeCallbackEndpoint, getWebhookPublicBaseUrl, fetchImpl, runtimeGeneration, credentialFingerprint, redactError, credentialCheckTimeoutMs: PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS }); const { reconcileGitHubWebhookRepository, reconcileProviderResourceRows, listResources, listGitHubRepositories, replaceResources, toggleAllGitHubRepositories, hasNewerProcessedProviderLifecycleEffect, applyProviderAvailability } = createChatResourceReconciliation(db, { endpointRecord, withCredentialMutationLease, runtimeGeneration, maxErrorText: MAX_ERROR_TEXT }); async function generateSetupSecret( @@ -6652,165 +5819,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }; } - async function reconcileDiscordCommands( - endpointId: string, - existingLease?: CredentialMutationLeaseGuard, - force = false, - ): Promise { - const initial = await endpointRecord(endpointId); - if (!initial || initial.endpoint.provider !== "discord") return; - const reconcile = async (lease: CredentialMutationLeaseGuard) => { - const record = await endpointRecord(endpointId); - if ( - !record || - record.endpoint.provider !== "discord" || - !["verifying", "active", "attention"].includes( - record.endpoint.status, - ) || - !record.endpoint.botExternalId || - !record.endpoint.providerAccountId - ) - return; - const scope = { - companyId: record.endpoint.companyId, - endpointId, - applicationId: record.endpoint.botExternalId, - guildId: record.endpoint.providerAccountId, - }; - const fence = runtimeContextForRecord(record); - const authorize = async (tx: DbTransaction) => { - const endpoint = await tx - .select() - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.companyId, scope.companyId), - eq(chatEndpoints.id, scope.endpointId), - ), - ) - .for("no key update") - .then((rows) => rows[0]); - if ( - !endpoint || - endpoint.provider !== "discord" || - !["verifying", "active", "attention"].includes(endpoint.status) || - endpoint.botExternalId !== scope.applicationId || - endpoint.providerAccountId !== scope.guildId || - runtimeGeneration(endpoint.setup) !== fence.generation - ) - throw conflict("Discord command registration authority changed"); - const connection = await tx - .select() - .from(toolConnections) - .where( - and( - eq(toolConnections.companyId, scope.companyId), - eq(toolConnections.id, endpoint.connectionId), - ), - ) - .for("no key update") - .then((rows) => rows[0]); - if ( - !connection?.enabled || - connection.status !== "active" || - credentialFingerprint(connection.credentialSecretRefs) !== - fence.credentialFingerprint - ) - throw conflict("Discord command registration authority changed"); - await lease.assertOwned(tx); - }; - const setCapability = async (tx: DbTransaction, registered: boolean) => { - const current = await tx - .select() - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.companyId, scope.companyId), - eq(chatEndpoints.id, endpointId), - ), - ) - .then((rows) => rows[0]); - if (!current) throw notFound("Chat endpoint not found"); - if ( - current.capabilities.slashCommands === registered && - current.capabilities.ephemeralMessages === registered - ) - return; - await tx - .update(chatEndpoints) - .set({ - capabilities: { - ...current.capabilities, - slashCommands: registered, - ephemeralMessages: registered, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatEndpoints.companyId, scope.companyId), - eq(chatEndpoints.id, endpointId), - ), - ); - await logActivity(tx as unknown as Db, { - companyId: scope.companyId, - actorType: "system", - actorId: "system", - action: registered - ? "chat_endpoint.commands_registered" - : "chat_endpoint.commands_unavailable", - entityType: "tool_connection", - entityId: current.connectionId, - details: { endpointId, provider: "discord" }, - }); - }; - await lease.assertOwned(); - const credentials = await resolveCredentialRefs( - record.endpoint, - record.credentialSecretRefs, - ); - if ( - credentials.applicationId !== scope.applicationId || - credentials.guildId !== scope.guildId - ) - throw conflict("Discord command registration identity changed"); - const result = await reconcileStoredDiscordCommandRegistration(db, { - scope, - runtimeFence: { - generation: fence.generation, - credentialFingerprint: fence.credentialFingerprint, - }, - botToken: credentials.botToken, - fetch: fetchImpl, - authorize, - force, - onState: async (tx, state) => - setCapability(tx, state.phase === "registered"), - }); - if (result.kind === "deferred") return; - await db.transaction(async (tx) => { - await authorize(tx); - const registered = - result.kind === "registered" - ? await readRegisteredDiscordCommandRegistration(tx, scope) - : null; - await setCapability(tx, registered !== null); - await lease.assertOwned(tx); - }); - }; - try { - if (existingLease) await reconcile(existingLease); - else await withCredentialMutationLease(initial.endpoint, reconcile); - } catch { - // Commands are additive. Do not fail an otherwise working mention/thread - // connection or expose provider errors/tokens when registration is busy. - logger.warn( - { endpointId }, - "Discord command registration remains unconfirmed", - ); - } - } - async function runtimeFor( endpoint: EndpointRow, optionsForRuntime: { @@ -19473,1314 +18481,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return acceptedResponse(); } - async function latestSlackDmControlThreadId( - endpointId: string, - rawChannelId: string, - ): Promise { - const conversationIds = [rawChannelId, `slack:${rawChannelId}`]; - return db - .select({ externalThreadId: chatConversations.externalThreadId }) - .from(chatConversations) - .innerJoin( - issues, - and( - eq(issues.companyId, chatConversations.companyId), - eq(issues.id, chatConversations.issueId), - ), - ) - .where( - and( - eq(chatConversations.endpointId, endpointId), - inArray(chatConversations.externalConversationId, conversationIds), - eq(chatConversations.isDirectMessage, true), - inArray(chatConversations.state, ["active", "waiting"]), - notInArray(issues.status, ["done", "cancelled"]), - ), - ) - .orderBy( - desc(chatConversations.lastActivityAt), - desc(chatConversations.createdAt), - ) - .limit(1) - .then((rows) => rows[0]?.externalThreadId ?? null); - } - - async function finalizeSlackTaskStartFailure(input: { - actionId: string; - actionStatus: "received" | "validating" | "resolving"; - attempt?: number; - connectionId: string; - credentialLease: CredentialMutationLeaseGuard; - endpoint: EndpointRow; - error: unknown; - providerAttempted?: boolean; - resource: ResourceRow | null; - runtimeFence: LifecycleRuntimeFence; - }): Promise { - const attempt = input.attempt ?? 1; - const classified = classifyChatPublicationError(input.error, attempt); - // Runtime construction, lease acquisition, and authorization validation - // happen before the transport fence. Even a socket-shaped error in that - // phase is known not to have sent a Slack message and remains safely - // retryable instead of entering the ambiguous-delivery quarantine. - const classifiedDisposition = - input.providerAttempted === false && - classified.kind === "delivery_unknown" - ? { - kind: "retry" as const, - retryAfterMs: Math.min(60_000, 2 ** Math.max(0, attempt) * 1000), - reason: classified.reason, - } - : classified; - await input.credentialLease.assertOwned(); - const currentRecord = await endpointRecord(input.endpoint.id); - const runtimeFenceCurrent = - currentRecord !== null && - runtimeGeneration(currentRecord.endpoint.setup) === - input.runtimeFence.generation && - credentialFingerprint(currentRecord.credentialSecretRefs) === - input.runtimeFence.credentialFingerprint; - // A provider rejection from credentials that were superseded while this - // worker lost its lease cannot quarantine the successor endpoint or its - // resources. Preserve genuinely ambiguous provider acceptance, but make a - // definite old-runtime rejection safely retryable so the normal worker can - // re-authorize it against the current endpoint before any new Slack call. - const disposition = - !runtimeFenceCurrent && classifiedDisposition.kind !== "delivery_unknown" - ? { - kind: "retry" as const, - retryAfterMs: 1_000, - reason: "Slack runtime changed before failure settlement", - } - : classifiedDisposition; - const failure = redactSensitiveText(disposition.reason).slice( - 0, - MAX_ERROR_TEXT, - ); - const finalized = await db.transaction(async (tx) => { - await input.credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "slash_task_start"); - const [ownedAction] = await tx - .update(chatActions) - .set({ - status: - disposition.kind === "delivery_unknown" - ? "delivery_unknown" - : disposition.kind === "retry" - ? "queued" - : "failed", - result: { - code: runtimeFenceCurrent - ? `slash_task_${disposition.kind}` - : disposition.kind === "delivery_unknown" - ? "slash_task_delivery_unknown" - : "slash_task_runtime_superseded", - redactedError: failure, - retryable: disposition.kind === "retry", - attemptCount: attempt, - ...(disposition.kind === "retry" - ? { - retryAt: new Date( - Date.now() + disposition.retryAfterMs, - ).toISOString(), - } - : {}), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, input.actionId), - eq(chatActions.status, input.actionStatus), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!ownedAction) return false; - if (runtimeFenceCurrent && disposition.kind === "endpoint_attention") { - await tx - .update(chatEndpoints) - .set({ - status: "attention", - healthMessage: "Provider credentials or permissions need attention", - lastError: failure, - updatedAt: new Date(), - }) - .where(eq(chatEndpoints.id, input.endpoint.id)); - await tx - .update(toolConnections) - .set({ - status: "disabled", - enabled: false, - healthStatus: "degraded", - healthMessage: "Provider credentials or permissions need attention", - lastError: failure, - healthCheckedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(toolConnections.id, input.connectionId)); - } else if ( - runtimeFenceCurrent && - disposition.kind === "resource_unavailable" && - input.resource - ) { - await tx - .update(chatEndpointResources) - .set({ availability: "unavailable", updatedAt: new Date() }) - .where(eq(chatEndpointResources.id, input.resource.id)); - } - await input.credentialLease.assertOwned(tx); - return true; - }); - if ( - finalized && - runtimeFenceCurrent && - disposition.kind === "endpoint_attention" - ) { - // Runtime removal mutates process-local provider state after the durable - // transaction. Prove this worker still owns the credential fence at the - // last await boundary so a reclaimer cannot have installed a successor - // runtime that this obsolete failure would then remove. - await input.credentialLease.assertOwned(); - await invalidateRuntime(input.endpoint.id).catch(() => undefined); - } - return finalized; - } - - async function admitConfirmedSlackTaskStart( - actionId: string, - ingressOnly = true, - ): Promise { - const admissionClaimId = randomUUID(); - const staleAdmission = new Date( - Date.now() - SLACK_COMMAND_ADMISSION_STALE_MS, - ); - const [action] = await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "admitting", - result: sql`coalesce(${chatActions.result}, '{}'::jsonb) - || jsonb_build_object('admissionClaimId', ${admissionClaimId}::text)`, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "slash_task_start"), - or( - eq(chatActions.status, "provider_confirmed"), - and( - eq(chatActions.status, "admitting"), - lte(chatActions.updatedAt, staleAdmission), - ), - ), - ), - ) - .returning(); - }); - if (!action) return false; - await options.slackTaskAdmissionClaimBarrier?.({ - actionId: action.id, - claimId: admissionClaimId, - }); - - const recovery = slackSlashTaskRecoveryPayload(action.payload); - const confirmed = confirmedSlackTaskStartResult(action.result); - const ownsAdmission = () => - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "admitting"), - sql`${chatActions.result}->>'admissionClaimId' = ${admissionClaimId}`, - ); - const failAdmission = async (code: string, detail: string) => { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { - ...(confirmed ?? {}), - code, - redactedError: redactSensitiveText(detail).slice(0, MAX_ERROR_TEXT), - }, - updatedAt: new Date(), - }) - .where(ownsAdmission()); - }); - }; - if (!recovery || !confirmed || !action.principalId) { - await failAdmission( - "slash_task_admission_context_missing", - "Confirmed Slack task start lacks durable admission context", - ); - return true; - } - - const record = await endpointRecord(action.endpointId); - if (!record || record.endpoint.provider !== "slack") { - await failAdmission( - "slash_task_endpoint_missing", - "Slack endpoint is no longer available", - ); - return true; - } - if (!["verifying", "active"].includes(record.endpoint.status)) { - if (record.endpoint.status === "archived") { - await failAdmission( - "slash_task_endpoint_archived", - "Slack endpoint was archived before task admission completed", - ); - } else { - // The provider-visible send is already confirmed. Wait for an - // operator to repair/resume the endpoint, then continue Paperclip-only - // admission without sending another Slack message. - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "provider_confirmed", - result: confirmed, - updatedAt: new Date(), - }) - .where(ownsAdmission()); - }); - } - return true; - } - const principal = await db - .select() - .from(chatExternalPrincipals) - .where( - and( - eq(chatExternalPrincipals.id, action.principalId), - eq(chatExternalPrincipals.companyId, record.endpoint.companyId), - eq(chatExternalPrincipals.provider, "slack"), - eq( - chatExternalPrincipals.providerAccountId, - record.endpoint.providerAccountId ?? "unknown", - ), - ), - ) - .then((rows) => rows[0] ?? null); - if (!principal || principal.isBot || principal.kind !== "user") { - await failAdmission( - "slash_task_principal_missing", - "Original Slack account is no longer available", - ); - return true; - } - - const author = { - userId: principal.externalId, - userName: principal.handle ?? principal.externalId, - fullName: - principal.displayName ?? principal.handle ?? principal.externalId, - isBot: false, - isMe: false, - isSystem: false, - } satisfies Author; - try { - const endpointRuntime = await runtimeFor(record.endpoint); - const thread = endpointRuntime.thread(confirmed.threadId); - await processMessage( - record.endpoint, - thread, - { - id: recovery.syntheticMessageId, - threadId: thread.id, - text: recovery.taskText, - formatted: { type: "root", children: [] }, - raw: { recoveredFromActionId: action.id }, - author, - metadata: { dateSent: action.createdAt, edited: false }, - attachments: [], - links: [], - isMention: true, - } as unknown as Message, - "mention", - ingressOnly, - null, - { ...runtimeContextForRecord(record), endpointRuntime }, - undefined, - null, - false, - undefined, - false, - true, - ); - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ status: "processed", result: confirmed, updatedAt: new Date() }) - .where(ownsAdmission()); - }); - return true; - } catch (error) { - // The provider post is already confirmed, so this retry is strictly a - // Paperclip admission retry. Preserve the root tuple and let the durable - // reconciler resume without ever posting to Slack again. - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "provider_confirmed", - result: { - ...confirmed, - code: "slash_task_admission_retry", - retryAt: new Date(Date.now() + 1_000).toISOString(), - redactedError: redactError(error), - }, - updatedAt: new Date(), - }) - .where(ownsAdmission()); - }); - throw error; - } - } - - async function processSlackTaskStart( - actionId: string, - ingressOnly = true, - ): Promise { - let action = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "slash_task_start"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!action) return false; - - if ( - action.status === "provider_confirmed" || - action.status === "admitting" - ) { - return await admitConfirmedSlackTaskStart(action.id, ingressOnly); - } - if (action.status === "resolving") { - if ( - action.updatedAt.getTime() <= - Date.now() - SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS - ) { - const staleRecord = await endpointRecord(action.endpointId); - if (!staleRecord) return false; - await withCredentialMutationLease( - staleRecord.endpoint, - async (credentialLease) => { - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "slash_task_start"); - await tx - .update(chatActions) - .set({ - status: "delivery_unknown", - result: { - ...(action.result ?? {}), - code: "slash_task_delivery_unknown", - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "resolving"), - eq(chatActions.updatedAt, action.updatedAt), - lte( - chatActions.updatedAt, - new Date( - Date.now() - SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS, - ), - ), - ), - ); - await credentialLease.assertOwned(tx); - }); - }, - ); - } - return false; - } - if (action.status === "validating") { - if ( - action.updatedAt.getTime() <= - Date.now() - SLACK_COMMAND_POST_STALE_MS - ) { - const staleRecord = await endpointRecord(action.endpointId); - if (!staleRecord) return false; - await withCredentialMutationLease( - staleRecord.endpoint, - async (credentialLease) => { - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "slash_task_start"); - await tx - .update(chatActions) - .set({ status: "queued", updatedAt: new Date() }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "validating"), - eq(chatActions.updatedAt, action.updatedAt), - lte( - chatActions.updatedAt, - new Date(Date.now() - SLACK_COMMAND_POST_STALE_MS), - ), - ), - ); - await credentialLease.assertOwned(tx); - }); - }, - ); - } - return false; - } - if (action.status !== "queued") return false; - const retryAt = - typeof action.result?.retryAt === "string" - ? new Date(action.result.retryAt) - : null; - if ( - retryAt && - !Number.isNaN(retryAt.getTime()) && - retryAt.getTime() > Date.now() - ) { - return false; - } - const attempt = - (typeof action.result?.attemptCount === "number" && - Number.isSafeInteger(action.result.attemptCount) - ? Math.max(0, action.result.attemptCount) - : 0) + 1; - - const [claimed] = await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "validating", - result: { attemptCount: attempt }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.kind, "slash_task_start"), - eq(chatActions.status, action.status), - ), - ) - .returning(); - }); - if (!claimed) return false; - action = claimed; - - const recovery = slackSlashTaskRecoveryPayload(action.payload); - const initialRecord = await endpointRecord(action.endpointId); - if ( - !recovery || - !initialRecord || - initialRecord.endpoint.provider !== "slack" - ) { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { - code: "slash_task_provider_context_missing", - retryable: false, - attemptCount: attempt, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "validating"), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ); - }); - return true; - } - let providerPostStarted = false; - let transportClaimed = false; - let failureResource: ResourceRow | null = null; - let failureRuntimeFence = runtimeContextForRecord(initialRecord); - let failureSettledUnderLease = false; - let confirmedActionId: string | null = null; - try { - const processed = await withCredentialMutationLease( - initialRecord.endpoint, - async (credentialLease) => { - const record = await endpointRecord(action.endpointId); - if (!record || record.endpoint.provider !== "slack") { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { - code: "slash_task_provider_context_missing", - retryable: false, - attemptCount: attempt, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "validating"), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ); - }); - return true; - } - failureRuntimeFence = runtimeContextForRecord(record); - if (!["verifying", "active"].includes(record.endpoint.status)) { - const terminallyArchived = record.endpoint.status === "archived"; - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: terminallyArchived ? "cancelled" : "queued", - result: { - code: terminallyArchived - ? "slash_task_endpoint_archived" - : "slash_task_endpoint_waiting", - attemptCount: attempt - 1, - ...(terminallyArchived - ? {} - : { - retryAt: new Date(Date.now() + 5_000).toISOString(), - }), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "validating"), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ); - }); - return false; - } - - const principal = action.principalId - ? await db - .select() - .from(chatExternalPrincipals) - .where( - and( - eq(chatExternalPrincipals.id, action.principalId), - eq( - chatExternalPrincipals.companyId, - record.endpoint.companyId, - ), - eq(chatExternalPrincipals.provider, "slack"), - eq( - chatExternalPrincipals.providerAccountId, - record.endpoint.providerAccountId ?? "unknown", - ), - ), - ) - .then((rows) => rows[0] ?? null) - : null; - let principalAllowed = false; - if (principal && !principal.isBot && principal.kind === "user") { - const link = await db - .select({ - paperclipUserId: chatIdentityLinks.paperclipUserId, - status: chatIdentityLinks.status, - }) - .from(chatIdentityLinks) - .where( - and( - eq(chatIdentityLinks.endpointId, record.endpoint.id), - eq(chatIdentityLinks.principalId, principal.id), - ), - ) - .then((rows) => rows[0] ?? null); - if (link?.status === "linked" && link.paperclipUserId) { - principalAllowed = await db - .select({ id: companyMemberships.id }) - .from(companyMemberships) - .where( - and( - eq(companyMemberships.companyId, record.endpoint.companyId), - eq(companyMemberships.principalType, "user"), - eq(companyMemberships.principalId, link.paperclipUserId), - eq(companyMemberships.status, "active"), - ne(companyMemberships.membershipRole, "viewer"), - ), - ) - .then((rows) => rows.length > 0); - } else if (record.endpoint.allowUnlinkedPeople) { - principalAllowed = await sponsorAllowsGuest(record.endpoint); - } - } - - const rawChannelId = recovery.channelId.replace(/^slack:/, ""); - const isDirectMessage = /^D[A-Z0-9]+$/i.test(rawChannelId); - const resource = await db - .select() - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.endpointId, record.endpoint.id), - eq(chatEndpointResources.providerResourceId, rawChannelId), - ), - ) - .then((rows) => rows[0] ?? null); - failureResource = resource; - const destinationAllowed = isDirectMessage - ? record.endpoint.allowDirectMessages - : nonDirectDestinationAllowed(record.endpoint, resource); - if (!principalAllowed || !destinationAllowed) { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "cancelled", - result: { - code: "slash_task_no_longer_authorized", - attemptCount: attempt, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "validating"), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ); - }); - return true; - } - - const endpointRuntime = await runtimeFor(record.endpoint); - const baseThread = endpointRuntime.thread(`slack:${rawChannelId}:`); - const authorizationSnapshot = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const currentEndpoint = await runtimeCallbackEndpoint( - tx, - record.endpoint.id, - runtimeContextForRecord(record), - ["verifying", "active"], - ); - const currentResource = currentEndpoint - ? await tx - .select() - .from(chatEndpointResources) - .where( - and( - eq( - chatEndpointResources.companyId, - currentEndpoint.companyId, - ), - eq(chatEndpointResources.endpointId, currentEndpoint.id), - eq( - chatEndpointResources.providerResourceId, - rawChannelId, - ), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - const currentAuthorization = - currentEndpoint && action!.principalId - ? await lockCurrentPrincipalAuthorization( - tx, - currentEndpoint, - action!.principalId, - ) - : null; - const currentDestinationAllowed = currentEndpoint - ? isDirectMessage - ? currentEndpoint.allowDirectMessages - : nonDirectDestinationAllowed(currentEndpoint, currentResource) - : false; - if ( - !currentEndpoint || - !currentAuthorization?.allowed || - !currentDestinationAllowed - ) { - await notifyChatActionWork(tx, "slash_task_start"); - await tx - .update(chatActions) - .set({ - status: "cancelled", - result: { - code: "slash_task_no_longer_authorized", - attemptCount: attempt, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, "validating"), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ); - return null; - } - - failureResource = currentResource; - await notifyChatActionWork(tx, "slash_task_start"); - const [transportClaim] = await tx - .update(chatActions) - .set({ - status: "resolving", - result: { - attemptCount: attempt, - authorizedUserId: currentAuthorization.userId, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, "validating"), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - return transportClaim - ? { authorizedUserId: currentAuthorization.userId } - : null; - }); - if (!authorizationSnapshot) return false; - - // `resolving` is the durable, single-owner transport claim. The - // authorization snapshot above commits and releases every row lock - // before Slack I/O starts. A crash after Slack accepts the post is - // quarantined as delivery_unknown and is never replayed implicitly. - transportClaimed = true; - providerPostStarted = true; - let starter: Awaited>; - try { - starter = await baseThread.post("Starting a task…"); - } catch (error) { - try { - await finalizeSlackTaskStartFailure({ - actionId: action.id, - actionStatus: "resolving", - attempt, - connectionId: record.endpoint.connectionId, - credentialLease, - endpoint: record.endpoint, - error, - providerAttempted: true, - resource: failureResource, - runtimeFence: failureRuntimeFence, - }); - failureSettledUnderLease = true; - } catch { - // If ownership was reclaimed while Slack was in flight, the - // outer recovery path reacquires the endpoint lease and settles - // the attempt without applying stale health state. - } - throw error; - } - const threadId = slackTaskStarterThreadId(rawChannelId, starter.id); - const [providerConfirmed] = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "slash_task_start"); - return tx - .update(chatActions) - .set({ - status: "provider_confirmed", - result: { - attemptCount: attempt, - authorizedUserId: authorizationSnapshot.authorizedUserId, - threadId, - providerMessageId: starter.id, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "resolving"), - sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, - ), - ) - .returning(); - }); - if (!providerConfirmed) { - throw new Error( - "Slack task-start ownership changed after provider acceptance", - ); - } - // Provider transport and its durable confirmation remain fenced by - // the credential lease. Paperclip-only task admission happens after - // this callback returns and releases that lease: normal admission - // stages a receipt reaction, whose independently fenced worker must - // acquire the same lease and would otherwise self-deadlock here. - confirmedActionId = providerConfirmed.id; - return true; - }, - ); - if (typeof confirmedActionId === "string") { - await admitConfirmedSlackTaskStart(confirmedActionId, ingressOnly); - } - return processed; - } catch (error) { - if (!failureSettledUnderLease) { - await withCredentialMutationLease( - initialRecord.endpoint, - async (credentialLease) => { - await finalizeSlackTaskStartFailure({ - actionId: action.id, - actionStatus: transportClaimed ? "resolving" : "validating", - attempt, - connectionId: initialRecord.endpoint.connectionId, - credentialLease, - endpoint: initialRecord.endpoint, - error, - providerAttempted: providerPostStarted, - resource: failureResource, - runtimeFence: failureRuntimeFence, - }); - }, - ); - } - throw error; - } - } - - async function processPendingSlackTaskStarts(limit = 25) { - const actions = await db.select({ id: chatActions.id }).from(chatActions) - .leftJoin(chatEndpoints, and(eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.companyId, chatActions.companyId))) - .where(dueChatAction("slash_task_start")) - .orderBy(asc(chatActions.createdAt)).limit(limit); - let processed = 0; - for (let offset = 0; offset < actions.length; offset += 4) { - const results = await Promise.all( - actions.slice(offset, offset + 4).map(async (action) => { - try { - return await processSlackTaskStart(action.id); - } catch (error) { - logger.warn( - { actionId: action.id, error: redactError(error) }, - "Slack task start will retry or await explicit resolution", - ); - return false; - } - }), - ); - processed += results.filter(Boolean).length; - } - return processed; - } - - async function handleDiscordNativeCommand( - event: ChatSdkCallbackEvent, - runtimeContext: RuntimeContext, - ): Promise { - const denied: DiscordNativeCommandResponse = { kind: "denied" }; - const record = await runtimeCallbackRecord( - event.endpointId, - runtimeContext, - ["active"], - ); - if ( - !record || - record.endpoint.provider !== "discord" || - record.endpoint.capabilities.slashCommands !== true || - event.transport !== "discord_gateway" || - runtimeContext.discordGatewayOwned !== true - ) - return denied; - const scope = { - companyId: record.endpoint.companyId, - endpointId: record.endpoint.id, - applicationId: record.endpoint.botExternalId ?? "", - guildId: record.endpoint.providerAccountId ?? "", - }; - const invocation = parseDiscordNativeCommand(event, scope); - if (!invocation) return denied; - const initialRegistration = await readRegisteredDiscordCommandRegistration( - db, - scope, - ); - if ( - initialRegistration?.receipt.commandId !== invocation.registeredCommandId - ) - return denied; - const principal = await ensurePrincipal( - record.endpoint, - event.event.user, - event.event.raw, - ); - const authorityChanged = new Error("Discord command authority changed"); - let publish = false; - try { - const response = await db.transaction( - async (tx): Promise => { - const endpoint = await runtimeCallbackEndpoint( - tx, - scope.endpointId, - runtimeContext, - ["active"], - ); - if ( - !endpoint || - endpoint.companyId !== scope.companyId || - endpoint.provider !== "discord" || - endpoint.botExternalId !== scope.applicationId || - endpoint.providerAccountId !== scope.guildId || - endpoint.capabilities.slashCommands !== true - ) - return denied; - const connection = await tx - .select() - .from(toolConnections) - .where( - and( - eq(toolConnections.companyId, scope.companyId), - eq(toolConnections.id, endpoint.connectionId), - ), - ) - .for("no key update") - .then((rows) => rows[0] ?? null); - if ( - !connection || - !connection.enabled || - connection.status !== "active" || - credentialFingerprint(connection.credentialSecretRefs) !== - runtimeContext.credentialFingerprint - ) - return denied; - const registration = await readRegisteredDiscordCommandRegistration( - tx, - scope, - true, - ); - if ( - registration?.receipt.commandId !== invocation.registeredCommandId - ) - return denied; - const providerActionId = `discord-native-command:${invocation.interactionId}`; - const prior = await tx - .select() - .from(chatActions) - .where( - and( - eq(chatActions.companyId, scope.companyId), - eq(chatActions.endpointId, scope.endpointId), - eq(chatActions.providerActionId, providerActionId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - const receipt = prior - ? parseDiscordNativeCommandReceipt(prior, invocation, scope) - : null; - if ( - prior && - (!receipt || - receipt.principalId !== principal.principal.id || - receipt.runtimeFence.generation !== runtimeContext.generation || - receipt.runtimeFence.credentialFingerprint !== - runtimeContext.credentialFingerprint) - ) - return denied; - // A replay selects only its original target, including a completed - // generation. Never reinterpret an old close as a new task's command. - const conversation = receipt - ? receipt.target - ? await tx - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.companyId, scope.companyId), - eq(chatConversations.endpointId, scope.endpointId), - eq(chatConversations.id, receipt.target.conversationId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null - : invocation.sourceKind !== "guild_channel" - ? await tx - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.companyId, scope.companyId), - eq(chatConversations.endpointId, scope.endpointId), - eq( - chatConversations.externalConversationId, - invocation.channelId, - ), - eq( - chatConversations.externalThreadId, - invocation.threadId, - ), - ), - ) - .orderBy(desc(chatConversations.sessionGeneration)) - .limit(1) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - if ( - (receipt?.target && !conversation) || - (conversation && - (conversation.externalConversationId !== invocation.channelId || - conversation.externalThreadId !== invocation.threadId || - conversation.isDirectMessage !== - (invocation.sourceKind === "direct_message") || - !["active", "waiting", "completed"].includes( - conversation.state, - ) || - (receipt?.target && - (conversation.issueId !== receipt.target.issueId || - conversation.sessionGeneration !== - receipt.target.sessionGeneration)))) - ) - return denied; - const resource = - invocation.sourceKind === "direct_message" - ? null - : await tx - .select() - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.companyId, scope.companyId), - eq(chatEndpointResources.endpointId, scope.endpointId), - eq( - chatEndpointResources.providerResourceId, - invocation.providerResourceId, - ), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if ( - invocation.sourceKind === "direct_message" - ? !endpoint.allowDirectMessages - : !nonDirectDestinationAllowed(endpoint, resource) || - (conversation && conversation.resourceId !== resource?.id) - ) - return denied; - const authorization = await lockCurrentPrincipalAuthorization( - tx, - endpoint, - principal.principal.id, - ); - if ( - !authorization.allowed || - authorization.userId !== principal.userId - ) - return denied; - const issue = conversation - ? await tx - .select() - .from(issues) - .where( - and( - eq(issues.companyId, scope.companyId), - eq(issues.id, conversation.issueId), - ), - ) - .for("share") - .then((rows) => rows[0] ?? null) - : null; - if (conversation && !issue) return denied; - const ownership = discordGatewayOwnerships.get(scope.endpointId); - if ( - !ownership || - !discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext) - ) - return denied; - const lease = await tx - .select() - .from(chatEndpointLeases) - .where( - and( - eq(chatEndpointLeases.companyId, scope.companyId), - eq(chatEndpointLeases.endpointId, scope.endpointId), - eq(chatEndpointLeases.leaseKey, ownership.leaseKey), - eq(chatEndpointLeases.token, ownership.token), - gt(chatEndpointLeases.expiresAt, new Date()), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if ( - !lease || - !discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext) - ) - return denied; - if (receipt) - return { kind: "accepted", content: receipt.result.content }; - const target: DiscordNativeCommandTarget | null = conversation - ? { - conversationId: conversation.id, - issueId: conversation.issueId, - sessionGeneration: conversation.sessionGeneration, - } - : null; - const active = - conversation && - issue && - ["active", "waiting"].includes(conversation.state) && - (invocation.sourceKind !== "direct_message" || - !["done", "cancelled"].includes(issue.status)); - let content: string; - let publicationId: string | null = null; - if (invocation.command === "status") { - content = - issue && (invocation.sourceKind !== "direct_message" || active) - ? `${issue.identifier}: ${issue.title} — ${issue.status}${conversation?.state === "completed" ? " (conversation closed)" : ""}`.slice( - 0, - 2000, - ) - : invocation.sourceKind === "guild_channel" - ? "Open the Discord task thread to view its Paperclip status." - : "No task is active in this conversation."; - } else if ( - invocation.command === "new" && - invocation.sourceKind !== "direct_message" - ) { - content = - "Open a new Discord thread by mentioning this agent in a new channel message. This thread stays bound to its current Paperclip task."; - } else if (!active || !conversation || !issue) { - content = - invocation.sourceKind === "direct_message" - ? "No task is active. Send a message to start a new Paperclip task." - : "No active task is bound here. Open its Discord thread to manage it."; - } else if ( - (await readChatControlChronology( - tx, - record.endpoint, - { - id: conversation.externalThreadId, - channelId: conversation.externalConversationId, - }, - { - receivedAt: new Date(), - normalizedEvent: { - message: { providerMessageId: invocation.interactionId }, - }, - }, - )) === "before_or_unproven" - ) { - content = - "This command predates an already completed chat control. Send a new command for the current conversation."; - } else { - const publicText = - invocation.command === "new" - ? "Send your request to start a new Paperclip task." - : invocation.sourceKind === "direct_message" - ? "This chat conversation is closed. Send another message to start a new task." - : "This chat conversation is closed. A later message here will continue the same Paperclip task."; - const publication = await stageAuthorizedTaskControlPublication( - tx, - { - companyId: scope.companyId, - endpointId: scope.endpointId, - conversationId: conversation.id, - issueId: issue.id, - principalId: principal.principal.id, - idempotencyKey: `control:${invocation.command}:discord:${scope.endpointId}:${invocation.interactionId}`, - payload: projectSafeChatPublication({ - classification: "external", - source: "task_control", - text: publicText, - }), - }, - ); - publicationId = publication.id; - content = `${invocation.command === "new" ? "New-task" : "Close"} request recorded. Watch this conversation for confirmation.`; - } - const inserted = await tx - .insert(chatActions) - .values({ - companyId: scope.companyId, - endpointId: scope.endpointId, - conversationId: target?.conversationId ?? null, - principalId: principal.principal.id, - kind: "discord_native_command", - providerActionId, - payload: { - version: 1, - invocation, - runtimeFence: { - generation: runtimeContext.generation, - credentialFingerprint: runtimeContext.credentialFingerprint, - }, - target, - }, - status: "processed", - result: { - kind: "discord_native_command_recorded", - content, - publicationId, - }, - }) - .onConflictDoNothing() - .returning({ id: chatActions.id }); - if (!inserted.length) throw authorityChanged; - await logActivity(tx as unknown as Db, { - companyId: scope.companyId, - actorType: authorization.userId ? "user" : "system", - actorId: authorization.userId ?? `chat:${principal.principal.id}`, - action: "chat.discord_command_recorded", - entityType: "chat_endpoint", - entityId: scope.endpointId, - details: { - command: invocation.command, - interactionId: invocation.interactionId, - conversationId: target?.conversationId ?? null, - sessionGeneration: target?.sessionGeneration ?? null, - publicationId, - }, - }); - // No network work is performed under these locks. A stop while any DB - // await was in flight rolls back the command and its staged outbox row. - if ( - !discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext) || - lease.expiresAt.getTime() <= Date.now() - ) - throw authorityChanged; - publish = publicationId !== null; - return { kind: "accepted", content }; - }, - ); - if (publish) - scheduleMessageProcessing(async () => { - await processPendingPublications(); - }); - return response; - } catch (error) { - if (error === authorityChanged) return denied; - throw error; - } - } - async function handleSlashCommand( event: ChatSdkCallbackEvent, runtimeContext: RuntimeContext, @@ -21825,2121 +19525,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } } - type GitHubIngressStageResult = - | { kind: "invalid_signature" } - | { kind: "ignored" } - | { actionId: string; kind: "staged" }; - - function githubSignatureIsValid( - body: string, - signature: string | null, - webhookSecret: string | undefined, - ): boolean { - if (!signature || !webhookSecret) return false; - const expected = `sha256=${createHmac("sha256", webhookSecret).update(body).digest("hex")}`; - try { - return timingSafeEqual(Buffer.from(signature), Buffer.from(expected)); - } catch { - return false; - } - } - - function githubWebhookContainsUserContent(eventType: string): boolean { - return ( - eventType === "issue_comment" || - eventType === "issues" || - eventType === "pull_request" || - eventType === "pull_request_review" || - eventType === "pull_request_review_comment" - ); - } - - function githubWebhookAddressesBot( - payload: unknown, - botUsername: string | null, - ): boolean { - if (!botUsername || !payload || typeof payload !== "object") return false; - const comment = (payload as { comment?: unknown }).comment; - if (!comment || typeof comment !== "object") return false; - const body = (comment as { body?: unknown }).body; - if (typeof body !== "string") return false; - return githubBodyMentionsBot(body, botUsername); - } - - function githubIngressAllowedInEndpointState( - status: EndpointRow["status"], - eventType: string, - ): boolean { - if (status === "verifying" || status === "active") return true; - if (eventType === "ping" && (status === "draft" || status === "attention")) - return true; - if ( - status === "attention" && - [ - "github_app_authorization", - "installation", - "installation_repositories", - ].includes(eventType) - ) - return true; - return status === "revoked" && eventType === "installation"; - } - - async function stageGitHubWebhookIngress( - endpoint: EndpointRow, - request: Request, - ): Promise { - const body = await request.text(); - const signature = request.headers.get("x-hub-signature-256"); - const eventType = request.headers.get("x-github-event")?.trim() ?? ""; - const providerDeliveryId = - request.headers.get("x-github-delivery")?.trim() || - `body:${createHash("sha256").update(body).digest("hex")}`; - if (!SUPPORTED_GITHUB_WEBHOOK_EVENTS.has(eventType)) - return { kind: "ignored" }; - - let parsedPayload: unknown = null; - try { - parsedPayload = JSON.parse(body) as unknown; - } catch { - // A valid signature is still required before a malformed body may be - // retained. The native adapter will settle the authenticated envelope. - } - - // Resolve from a captured ref set, then prove that exact ref set and - // generation are still authoritative in the insertion transaction. This - // keeps webhook authentication outside the potentially long credential - // mutation lease without letting an obsolete secret stage new work. - for (let attempt = 0; attempt < 3; attempt += 1) { - const record = await endpointRecord(endpoint.id); - if (!record || record.endpoint.provider !== "github") - return { kind: "ignored" }; - if ( - !githubIngressAllowedInEndpointState(record.endpoint.status, eventType) - ) { - return { kind: "ignored" }; - } - const context = runtimeContextForRecord(record); - let credentials: Record; - try { - credentials = await resolveCredentialRefs( - record.endpoint, - record.credentialSecretRefs, - ); - } catch (error) { - const latest = await endpointRecord(endpoint.id); - if ( - latest && - (runtimeGeneration(latest.endpoint.setup) !== context.generation || - credentialFingerprint(latest.credentialSecretRefs) !== - context.credentialFingerprint) - ) { - continue; - } - throw error; - } - if (!githubSignatureIsValid(body, signature, credentials.webhookSecret)) { - const latest = await endpointRecord(endpoint.id); - if ( - latest && - (runtimeGeneration(latest.endpoint.setup) !== context.generation || - credentialFingerprint(latest.credentialSecretRefs) !== - context.credentialFingerprint) - ) { - continue; - } - return { kind: "invalid_signature" }; - } - - const incomingInstallationId = - parsedPayload && - typeof parsedPayload === "object" && - !Array.isArray(parsedPayload) - ? (parsedPayload as { installation?: { id?: unknown } }).installation - ?.id - : undefined; - if ( - incomingInstallationId !== undefined && - String(incomingInstallationId) !== credentials.installationId && - eventType !== "installation" - ) { - return { kind: "ignored" }; - } - - const bodySha256 = createHash("sha256").update(body).digest("hex"); - const staged = await db.transaction(async (tx) => { - const currentEndpoint = await tx - .select() - .from(chatEndpoints) - .where(eq(chatEndpoints.id, endpoint.id)) - .for("update") - .then((rows) => rows[0] ?? null); - const currentConnection = await tx - .select({ refs: toolConnections.credentialSecretRefs }) - .from(toolConnections) - .where( - and( - eq(toolConnections.companyId, record.endpoint.companyId), - eq(toolConnections.id, record.endpoint.connectionId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if (!currentEndpoint || !currentConnection) return "retry" as const; - if ( - runtimeGeneration(currentEndpoint.setup) !== context.generation || - credentialFingerprint(currentConnection.refs) !== - context.credentialFingerprint - ) { - return "retry" as const; - } - if ( - !githubIngressAllowedInEndpointState( - currentEndpoint.status, - eventType, - ) - ) { - return "ignored" as const; - } - - // Recovery history is denial-only provenance. A genuine callback - // queued by GitHub before pause/reconnect must not be authenticated - // into a later generation, even if the provider signs with its current - // secret. Include archived endpoints so reusing an App cannot move a - // delayed callback into another endpoint/company. - const recoveryReceipts = credentials.appId - ? await tx - .select({ - endpointId: chatActions.endpointId, - payload: chatActions.payload, - status: chatActions.status, - }) - .from(chatActions) - .innerJoin( - chatEndpoints, - eq(chatEndpoints.id, chatActions.endpointId), - ) - .where( - and( - eq(chatEndpoints.provider, "github"), - eq(chatEndpoints.botExternalId, credentials.appId), - eq(chatActions.kind, "github_webhook_recovery"), - eq( - chatActions.providerActionId, - `github_webhook_recovery:${providerDeliveryId}`, - ), - ), - ) - : []; - for (const receipt of recoveryReceipts) { - const fence = githubRecoveryWindow(receipt.payload); - const original = (receipt.payload as GitHubRecoveryReceipt).original; - const incoming = parsedPayload as { - action?: unknown; - installation?: { id?: unknown }; - repository?: { id?: unknown; full_name?: unknown }; - issue?: { id?: unknown; number?: unknown }; - pull_request?: { id?: unknown; number?: unknown }; - sender?: { id?: unknown }; - comment?: { - id?: unknown; - body?: unknown; - user?: { id?: unknown }; - created_at?: unknown; - updated_at?: unknown; - }; - } | null; - const comment = incoming?.comment; - if ( - !fence || - receipt.status === "cancelled" || - receipt.endpointId !== currentEndpoint.id || - fence.generation !== context.generation || - fence.credentialFingerprint !== context.credentialFingerprint || - fence.webhookUrl !== - `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${currentEndpoint.publicId}/github` || - !original?.payload?.comment || - original.event !== eventType || - incoming?.action !== "created" || - String(incoming?.installation?.id) !== - original.payload.installationId || - String(incoming?.repository?.id) !== - original.payload.repositoryId || - incoming?.repository?.full_name !== - original.payload.repositoryFullName || - String(incoming?.sender?.id) !== original.payload.senderId || - (eventType === "issue_comment" - ? String(incoming?.issue?.id) !== original.payload.issueId || - String(incoming?.issue?.number) !== original.payload.issueNumber - : String(incoming?.pull_request?.id) !== - original.payload.pullRequestId || - String(incoming?.pull_request?.number) !== - original.payload.pullRequestNumber) || - String(comment?.id) !== original.payload.comment.id || - String(comment?.user?.id) !== original.payload.comment.userId || - comment?.created_at !== original.payload.comment.createdAt || - comment?.updated_at !== original.payload.comment.updatedAt || - typeof comment?.body !== "string" || - createHash("sha256").update(comment.body).digest("hex") !== - original.payload.comment.bodySha256 - ) { - return "ignored" as const; - } - } - - // Signed App webhooks can cover every installed repository. Do not - // retain comment bodies for destinations the operator did not enable. - if (githubWebhookContainsUserContent(eventType)) { - const item = githubRepositoryInventoryItemFromPayload(parsedPayload); - if (!item) return "ignored" as const; - const providerRepositoryId = - typeof item.metadata?.providerRepositoryId === "string" - ? item.metadata.providerRepositoryId - : null; - const matchingResource = await tx - .select({ - enabled: chatEndpointResources.enabled, - id: chatEndpointResources.id, - }) - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.companyId, currentEndpoint.companyId), - eq(chatEndpointResources.endpointId, currentEndpoint.id), - eq(chatEndpointResources.type, "repository"), - eq(chatEndpointResources.availability, "available"), - or( - eq( - chatEndpointResources.providerResourceId, - item.providerResourceId, - ), - providerRepositoryId - ? sql`${chatEndpointResources.metadata}->>'providerRepositoryId' = ${providerRepositoryId}` - : undefined, - ), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - if (!matchingResource) return "ignored" as const; - if (!matchingResource.enabled) { - const action = - parsedPayload && - typeof parsedPayload === "object" && - !Array.isArray(parsedPayload) - ? (parsedPayload as { action?: unknown }).action - : null; - const ignoreDisabledResource = async () => { - // Record only an authenticated, content-free admission decision. - // Do not make operators infer a working callback from silence, - // or retain the comment/actor/thread from a disabled repository. - const providerEventId = `github:filtered_ingress:${createHash( - "sha256", - ) - .update(providerDeliveryId) - .digest("hex")}`; - const eventKind = - action === "edited" - ? "message_updated" - : action === "deleted" - ? "message_deleted" - : "message"; - await notifyChatDeliveryWork(tx); - await tx - .insert(chatDeliveries) - .values({ - companyId: currentEndpoint.companyId, - endpointId: currentEndpoint.id, - providerEventId, - deduplicationKey: providerEventId, - eventKind, - state: "filtered", - processedAt: new Date(), - redactedError: "Destination is not enabled in Paperclip", - normalizedEvent: { - providerEventId, - kind: eventKind, - filtering: { - contentRetained: false, - reason: "destination_not_enabled", - resourceId: matchingResource.id, - }, - }, - }) - .onConflictDoNothing(); - return "ignored" as const; - }; - if ( - currentEndpoint.status !== "verifying" || - action !== "created" || - !githubWebhookAddressesBot( - parsedPayload, - currentEndpoint.botUsername, - ) - ) { - return ignoreDisabledResource(); - } - const enabledResourceCount = await tx - .select({ count: sql`count(*)::int` }) - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.endpointId, currentEndpoint.id), - eq(chatEndpointResources.type, "repository"), - eq(chatEndpointResources.enabled, true), - eq(chatEndpointResources.availability, "available"), - ), - ) - .then((rows) => rows[0]?.count ?? 0); - if (enabledResourceCount !== 0) return ignoreDisabledResource(); - } - } - - const payload: GitHubWebhookIngressPayload = { - version: 1, - body, - bodySha256, - contentType: - request.headers.get("content-type") ?? "application/json", - deliveryId: providerDeliveryId, - eventType, - runtimeGeneration: context.generation, - credentialFingerprint: context.credentialFingerprint, - }; - await notifyChatActionWork(tx, "github_webhook_ingress"); - const inserted = await tx - .insert(chatActions) - .values({ - companyId: currentEndpoint.companyId, - endpointId: currentEndpoint.id, - kind: "github_webhook_ingress", - providerActionId: `github_webhook_ingress:${providerDeliveryId}`, - payload, - status: "received", - }) - .onConflictDoNothing() - .returning({ id: chatActions.id }); - if (inserted[0]?.id) { - return { actionId: inserted[0].id, kind: "staged" } as const; - } - - // An automatic callback is never authorization to reset the local - // worker's terminal retry budget. The original may have arrived in the - // interval between our remote scan and GitHub's asynchronous callback. - if (recoveryReceipts.length > 0) return "ignored" as const; - - // GitHub preserves X-GitHub-Delivery when an operator uses Recent - // Deliveries to retry a failed callback. Re-arm only the exact same, - // currently authenticated body after a terminal processing failure. - // Successful, cancelled, retryable, and in-flight receipts remain - // immutable deduplication fences. - await notifyChatActionWork(tx, "github_webhook_ingress"); - const [rearmed] = await tx - .update(chatActions) - .set({ - payload, - // Keep the lifetime attempt counter monotonic. Besides bounding - // repeated manual redeliveries, this prevents a late settlement - // from an older processing claim from matching a fresh claim - // after re-arm (an attempts=1 ABA). - result: sql`jsonb_build_object( - 'attempts', - coalesce((${chatActions.result}->>'attempts')::int, 0) - )`, - status: "received", - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.endpointId, currentEndpoint.id), - eq( - chatActions.providerActionId, - `github_webhook_ingress:${providerDeliveryId}`, - ), - eq(chatActions.kind, "github_webhook_ingress"), - eq(chatActions.status, "failed"), - sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'false'`, - sql`${chatActions.payload}->>'bodySha256' = ${bodySha256}`, - sql`${chatActions.payload}->>'eventType' = ${eventType}`, - ), - ) - .returning({ id: chatActions.id }); - const actionId = - rearmed?.id ?? - (await tx - .select({ id: chatActions.id }) - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, currentEndpoint.id), - eq( - chatActions.providerActionId, - `github_webhook_ingress:${providerDeliveryId}`, - ), - ), - ) - .then((rows) => rows[0]?.id ?? null)); - return actionId - ? ({ actionId, kind: "staged" } as const) - : ("retry" as const); - }); - if (staged === "retry") continue; - if (staged === "ignored") return { kind: "ignored" }; - return staged; - } - throw conflict("GitHub credentials changed during webhook authentication", { - code: "chat_endpoint_credentials_changed", - }); - } - - function githubIngressAcceptedResponse() { - return new Response("accepted", { status: 202 }); - } - - async function settleGitHubWebhookIngress( - actionId: string, - input: { - attempts: number; - payload: GitHubWebhookIngressPayload; - response?: Response; - error?: unknown; - cancelledCode?: string; - }, - ): Promise { - const terminalResponse = - input.response !== undefined && input.response.status < 500; - const terminalFailure = input.attempts >= 5; - const cancelled = Boolean(input.cancelledCode); - const processed = Boolean(input.response?.ok); - const terminal = - processed || terminalResponse || terminalFailure || cancelled; - const retryAt = new Date( - Date.now() + - Math.min(60_000, 1_000 * 2 ** Math.max(0, input.attempts - 1)), - ); - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "github_webhook_ingress"); - return tx - .update(chatActions) - .set({ - status: cancelled ? "cancelled" : processed ? "processed" : "failed", - payload: terminal - ? redactedGitHubWebhookIngressPayload(input.payload) - : input.payload, - result: { - attempts: input.attempts, - ...(input.cancelledCode ? { code: input.cancelledCode } : {}), - ...(input.response - ? { httpStatus: input.response.status } - : { error: redactError(input.error) }), - retryable: !terminal, - ...(!terminal ? { retryAt: retryAt.toISOString() } : {}), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "github_webhook_ingress"), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${input.attempts}`, - ), - ); - }); - } - - async function processGitHubWebhookIngress( - actionId: string, - propagateProcessingFailure = false, - ): Promise { - let action = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "github_webhook_ingress"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!action) return null; - if (["processed", "cancelled"].includes(action.status)) - return githubIngressAcceptedResponse(); - if (action.status === "failed" && action.result?.retryable !== true) - return githubIngressAcceptedResponse(); - const payload = githubWebhookIngressPayload(action.payload); - if (!payload) { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "github_webhook_ingress"); - return tx - .update(chatActions) - .set({ - status: "failed", - payload: {}, - result: { - code: "github_webhook_ingress_payload_invalid", - retryable: false, - }, - updatedAt: new Date(), - }) - .where(eq(chatActions.id, action.id)); - }); - return githubIngressAcceptedResponse(); - } - const now = new Date(); - const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); - if (action.status === "processing" && action.updatedAt > staleBefore) { - return null; - } - if (action.status === "failed") { - const retryAt = - typeof action.result?.retryAt === "string" - ? new Date(action.result.retryAt) - : null; - if (retryAt && retryAt > now) return null; - } else if (action.status !== "received" && action.status !== "processing") { - return null; - } - const attempts = - (typeof action.result?.attempts === "number" - ? action.result.attempts - : 0) + 1; - const previousAttempts = attempts - 1; - const claimGuard = - action.status === "processing" - ? and( - eq(chatActions.status, "processing"), - lte(chatActions.updatedAt, staleBefore), - sql`coalesce((${chatActions.result}->>'attempts')::int, 0) = ${previousAttempts}`, - ) - : action.status === "failed" - ? and( - eq(chatActions.status, "failed"), - sql`coalesce((${chatActions.result}->>'attempts')::int, 0) = ${previousAttempts}`, - ) - : eq(chatActions.status, "received"); - const [claimed] = await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "github_webhook_ingress"); - return tx - .update(chatActions) - .set({ - status: "processing", - result: { attempts }, - updatedAt: now, - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.kind, "github_webhook_ingress"), - claimGuard, - ), - ) - .returning(); - }); - if (!claimed) return null; - action = claimed; - - const record = await endpointRecord(action.endpointId); - if ( - !record || - record.endpoint.provider !== "github" || - !githubIngressAllowedInEndpointState( - record.endpoint.status, - payload.eventType, - ) || - runtimeGeneration(record.endpoint.setup) !== payload.runtimeGeneration || - credentialFingerprint(record.credentialSecretRefs) !== - payload.credentialFingerprint - ) { - await settleGitHubWebhookIngress(action.id, { - attempts, - payload, - cancelledCode: "github_webhook_ingress_runtime_superseded", - }); - return githubIngressAcceptedResponse(); - } - - try { - const credentials = await resolveCredentialRefs( - record.endpoint, - record.credentialSecretRefs, - ); - const recovery = await db - .select({ payload: chatActions.payload, status: chatActions.status }) - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, record.endpoint.id), - eq(chatActions.kind, "github_webhook_recovery"), - eq( - chatActions.providerActionId, - `github_webhook_recovery:${payload.deliveryId}`, - ), - ), - ) - .then((rows) => rows[0] ?? null); - if ( - recovery && - (recovery.status === "cancelled" || - !(await githubRecoverySourceIsCurrent( - (recovery.payload as GitHubRecoveryReceipt).original, - credentials, - ))) - ) { - await settleGitHubWebhookIngress(action.id, { - attempts, - payload, - cancelledCode: "github_webhook_recovery_source_changed", - }); - return githubIngressAcceptedResponse(); - } - const signature = `sha256=${createHmac( - "sha256", - credentials.webhookSecret, - ) - .update(payload.body) - .digest("hex")}`; - await options.githubWebhookReplayBarrier?.(); - const replayContext = { - githubIngressActionId: action.id, - githubIngressFence: { - generation: payload.runtimeGeneration, - credentialFingerprint: payload.credentialFingerprint, - }, - superseded: false, - }; - const response = await handleWebhook( - record.endpoint.publicId, - "github", - new Request( - `https://paperclip.internal/api/chat-webhooks/${record.endpoint.publicId}/github`, - { - method: "POST", - headers: { - "content-type": payload.contentType, - "x-github-delivery": payload.deliveryId, - "x-github-event": payload.eventType, - "x-hub-signature-256": signature, - }, - body: payload.body, - }, - ), - replayContext, - ); - if (replayContext.superseded) { - await settleGitHubWebhookIngress(action.id, { - attempts, - payload, - cancelledCode: "github_webhook_ingress_runtime_superseded", - }); - return githubIngressAcceptedResponse(); - } - if (response.status === 401) { - const latest = await endpointRecord(action.endpointId); - if ( - !latest || - runtimeGeneration(latest.endpoint.setup) !== - payload.runtimeGeneration || - credentialFingerprint(latest.credentialSecretRefs) !== - payload.credentialFingerprint - ) { - await settleGitHubWebhookIngress(action.id, { - attempts, - payload, - cancelledCode: "github_webhook_ingress_runtime_superseded", - }); - return githubIngressAcceptedResponse(); - } - } - await settleGitHubWebhookIngress(action.id, { - attempts, - payload, - response, - }); - return response.status >= 500 && !propagateProcessingFailure - ? githubIngressAcceptedResponse() - : response; - } catch (error) { - await settleGitHubWebhookIngress(action.id, { - attempts, - payload, - error, - }); - logger.warn( - { - actionId: action.id, - endpointId: action.endpointId, - error: redactError(error), - }, - "durable GitHub webhook processing deferred", - ); - if (propagateProcessingFailure) throw error; - return githubIngressAcceptedResponse(); - } - } - - async function githubRecoveryResource( - database: DbOrTransaction, - endpoint: EndpointRow, - repositoryId: string, - repositoryFullName?: string | null, - ) { - return database - .select({ id: chatEndpointResources.id }) - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.companyId, endpoint.companyId), - eq(chatEndpointResources.endpointId, endpoint.id), - eq(chatEndpointResources.type, "repository"), - eq(chatEndpointResources.enabled, true), - eq(chatEndpointResources.availability, "available"), - sql`${chatEndpointResources.metadata}->>'providerRepositoryId' = ${repositoryId}`, - repositoryFullName - ? eq( - chatEndpointResources.providerResourceId, - repositoryFullName.toLowerCase(), - ) - : undefined, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - } - - async function githubRecoveryAlreadyReceived( - database: DbOrTransaction, - endpoint: EndpointRow, - guid: string, - ) { - const action = await database - .select({ id: chatActions.id }) - .from(chatActions) - .where( - and( - eq(chatActions.companyId, endpoint.companyId), - eq(chatActions.endpointId, endpoint.id), - eq(chatActions.providerActionId, `github_webhook_ingress:${guid}`), - ), - ) - .limit(1) - .then((rows) => rows[0]); - if (action) return true; - // Disabled destinations intentionally retain no ingress body. Their - // content-free filtered receipt must also remain an admission fence. - return database - .select({ id: chatDeliveries.id }) - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.companyId, endpoint.companyId), - eq(chatDeliveries.endpointId, endpoint.id), - eq( - chatDeliveries.providerEventId, - `github:filtered_ingress:${createHash("sha256").update(guid).digest("hex")}`, - ), - ), - ) - .limit(1) - .then((rows) => rows.length > 0); - } - - async function rememberGitHubRecoverySkip( - endpoint: EndpointRow, - context: LifecycleRuntimeFence, - window: GitHubRecoveryWindow, - original: GitHubAppWebhookDelivery, - code: string, - ) { - await db.transaction(async (tx) => { - const current = await runtimeCallbackEndpoint(tx, endpoint.id, context, [ - "active", - ]); - if ( - !current || - !original.repositoryId || - !(await githubRecoveryResource(tx, current, original.repositoryId)) - ) - return; - // Only the deny tombstone is needed, not the rejected detail or source - // contents. Remembering it lets the next bounded scan reach older valid - // messages instead of repeatedly inspecting the same five bot callbacks. - await tx - .insert(chatActions) - .values({ - companyId: current.companyId, - endpointId: current.id, - kind: "github_webhook_recovery", - providerActionId: `github_webhook_recovery:${original.guid}`, - payload: { - ...window, - original: { id: original.id, guid: original.guid }, - }, - status: "cancelled", - result: { attempts: 0, code }, - }) - .onConflictDoUpdate({ - target: [chatActions.endpointId, chatActions.providerActionId], - set: { - status: "cancelled", - updatedAt: new Date(), - result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || jsonb_build_object('code', ${code}::text)`, - }, - setWhere: and( - eq(chatActions.kind, "github_webhook_recovery"), - sql`(${chatActions.payload}->>'generation')::int = ${context.generation}`, - sql`${chatActions.payload}->>'credentialFingerprint' = ${context.credentialFingerprint}`, - ), - }); - }); - } - - async function githubRecoverySourceIsCurrent( - detail: GitHubAppWebhookDeliveryDetail, - credentials: Record, - ): Promise { - const comment = detail?.payload?.comment; - if ( - !comment || - !detail.payload.repositoryFullName || - !["issue_comment", "pull_request_review_comment"].includes(detail.event) - ) - return false; - const current = await getGitHubRecoveryComment({ - fetch: fetchImpl, - appToken: githubAppJwt(credentials.appId, credentials.privateKey), - installationId: credentials.installationId, - repositoryFullName: detail.payload.repositoryFullName, - event: detail.event as "issue_comment" | "pull_request_review_comment", - commentId: comment.id, - }).catch((error) => { - if (error?.statusCode === 404 || error?.statusCode === 410) return null; - throw error; - }); - if (!current) return false; - return ( - current.id === comment.id && - current.userId === comment.userId && - current.userType === "User" && - current.bodySha256 === comment.bodySha256 && - current.createdAt === comment.createdAt && - current.updatedAt === comment.updatedAt && - (detail.event === "issue_comment" - ? current.issueNumber === detail.payload.issueNumber - : current.pullRequestNumber === detail.payload.pullRequestNumber) - ); - } - - /** - * GitHub does not retry callbacks that fail before our durable inbox. Ask - * GitHub to redeliver a bounded recent created-comment envelope, never turn - * delivery-history JSON into a locally signed or directly admitted request. - * A missing/changed epoch starts at now; historical messages are not adopted - * into a new credential/generation merely because the App can still see them. - */ - async function processFailedGitHubWebhookDeliveries( - limit = 5, - onlyEndpointId?: string, - ) { - if (shuttingDown || !getWebhookPublicBaseUrl()?.startsWith("https://")) return 0; - const now = new Date(); - const rows = await db - .select({ endpoint: chatEndpoints }) - .from(chatEndpoints) - .innerJoin( - toolConnections, - and( - eq(toolConnections.id, chatEndpoints.connectionId), - eq(toolConnections.companyId, chatEndpoints.companyId), - ), - ) - .leftJoin( - chatSdkState, - and( - eq(chatSdkState.endpointId, chatEndpoints.id), - eq(chatSdkState.stateKey, GITHUB_RECOVERY_STATE_KEY), - ), - ) - .where( - and( - eq(chatEndpoints.provider, "github"), - eq(chatEndpoints.status, "active"), - eq(toolConnections.enabled, true), - eq(toolConnections.status, "active"), - onlyEndpointId ? eq(chatEndpoints.id, onlyEndpointId) : undefined, - or( - isNull(chatSdkState.id), - // Backoff belongs to one runtime epoch. Reconnect or callback - // changes must establish a fresh floor immediately, even when - // the superseded epoch was rate limited for several hours. - sql`${chatSdkState.value}->>'generation' is distinct from coalesce(${chatEndpoints.setup}->>'runtimeGeneration', '0')`, - sql`${chatSdkState.value}->>'appId' is distinct from ${chatEndpoints.botExternalId}`, - sql`${chatSdkState.value}->>'webhookUrl' is distinct from (${getWebhookPublicBaseUrl()} || '/api/chat-webhooks/' || ${chatEndpoints.publicId} || '/github')`, - sql`(${chatSdkState.value}->>'nextScanAt')::timestamptz <= ${now.toISOString()}::timestamptz`, - ), - ), - ) - .orderBy(asc(chatSdkState.updatedAt), asc(chatEndpoints.id)) - .limit(Math.min(5, Math.max(1, limit))); - for (const { endpoint } of rows) { - const record = await endpointRecord(endpoint.id); - if (!record || !record.endpoint.botExternalId) continue; - const context = runtimeContextForRecord(record); - const webhookUrl = `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/github`; - const scope = { companyId: endpoint.companyId, endpointId: endpoint.id }; - const stored = await persistence.read(scope, GITHUB_RECOVERY_STATE_KEY); - const previous = githubRecoveryWindow(stored?.value); - const epochMatches = - previous && - previous.generation === context.generation && - previous.credentialFingerprint === context.credentialFingerprint && - previous.webhookUrl === webhookUrl && - previous.appId === record.endpoint.botExternalId; - if (epochMatches && Date.parse(previous.nextScanAt) > now.getTime()) - continue; - const window: GitHubRecoveryWindow = { - generation: context.generation, - credentialFingerprint: context.credentialFingerprint, - appId: record.endpoint.botExternalId, - webhookUrl, - floor: epochMatches ? previous.floor : now.toISOString(), - nextScanAt: new Date( - now.getTime() + GITHUB_RECOVERY_SCAN_LEASE_MS, - ).toISOString(), - }; - if ( - !(await persistence.compareAndSet({ - ...scope, - key: GITHUB_RECOVERY_STATE_KEY, - expectedVersion: stored?.version ?? null, - value: window, - expiresAt: null, - })) - ) - continue; - const claimedVersion = (stored?.version ?? 0) + 1; - let nextDelay = GITHUB_RECOVERY_INTERVAL_MS; - let outcome = "scanned"; - let requested = 0; - let inspected = 0; - try { - if (!epochMatches) { - outcome = "epoch_initialized"; - continue; - } - const credentials = await resolveCredentialRefs( - record.endpoint, - record.credentialSecretRefs, - ); - if (credentials.appId !== window.appId || !credentials.installationId) - continue; - const appToken = githubAppJwt( - credentials.appId, - credentials.privateKey, - ); - const config = await readGitHubAppWebhookConfig({ - fetch: fetchImpl, - appToken, - }); - if ( - config.url !== webhookUrl || - config.contentType !== "json" || - config.insecureSsl !== "0" - ) { - outcome = "callback_mismatch"; - nextDelay = 5 * 60_000; - continue; - } - const attempts: GitHubAppWebhookDelivery[] = []; - const floor = Math.max( - Date.parse(window.floor), - now.getTime() - GITHUB_RECOVERY_WINDOW_MS, - ); - let cursor: string | undefined; - let truncated = false; - let previousTimestamp = Number.POSITIVE_INFINITY; - for (let page = 0; page < 3; page += 1) { - const result = await listGitHubAppWebhookDeliveries({ - fetch: fetchImpl, - appToken, - cursor, - }); - // GitHub's official recovery recipe uses newest-first pagination - // and stops at the age floor. Reject observed ordering drift rather - // than assuming unseen pages cannot contain a successful sibling. - for (const attempt of result.deliveries) { - const timestamp = Date.parse(attempt.deliveredAt); - if (timestamp > previousTimestamp) - throw new Error("GitHub recovery history ordering changed"); - previousTimestamp = timestamp; - if (timestamp >= floor) attempts.push(attempt); - } - if (previousTimestamp < floor) break; - if (!result.nextCursor) break; - if (result.nextCursor === cursor) - throw new Error("GitHub recovery pagination did not advance"); - cursor = result.nextCursor; - truncated = page === 2; - } - // All pages in this time window are needed to rule out a successful - // sibling; do not act on a partial window after hitting the work cap. - if (truncated) { - outcome = "history_limit_reached"; - nextDelay = 5 * 60_000; - continue; - } - const groups = new Map(); - for (const attempt of attempts) { - const group = groups.get(attempt.guid) ?? []; - group.push(attempt); - groups.set(attempt.guid, group); - } - for (const [guid, group] of groups) { - if ( - inspected >= 5 || - shuttingDown || - Date.now() >= Date.parse(window.nextScanAt) - ) - break; - if ( - group.some( - (item) => - item.statusCode !== null && - item.statusCode >= 200 && - item.statusCode < 400, - ) - ) - continue; - const originals = group.filter((item) => !item.redelivery); - if (originals.length !== 1) continue; - const original = originals[0]!; - if ( - Date.parse(original.deliveredAt) < floor || - Date.parse(original.deliveredAt) > now.getTime() - 10_000 || - original.statusCode === null || - original.statusCode < 500 || - original.action !== "created" || - !["issue_comment", "pull_request_review_comment"].includes( - original.event, - ) || - original.installationId !== credentials.installationId || - !original.repositoryId || - !(await githubRecoveryResource(db, endpoint, original.repositoryId)) - ) - continue; - const recoveryId = `github_webhook_recovery:${guid}`; - if (await githubRecoveryAlreadyReceived(db, endpoint, guid)) continue; - const receipt = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, endpoint.id), - eq(chatActions.providerActionId, recoveryId), - ), - ) - .then((items) => items[0] ?? null); - const count = - typeof receipt?.result?.attempts === "number" - ? receipt.result.attempts - : 0; - const receiptWindow = githubRecoveryWindow(receipt?.payload); - const latest = [...group].sort( - (left, right) => - Date.parse(right.deliveredAt) - Date.parse(left.deliveredAt), - )[0]!; - if ( - latest.statusCode === null || - latest.statusCode < 500 || - latest.throttledAt !== null || - Date.parse(latest.deliveredAt) > now.getTime() - 10_000 - ) - continue; - if ( - receipt && - (receipt.status === "cancelled" || - !receiptWindow || - receiptWindow.generation !== window.generation || - receiptWindow.credentialFingerprint !== - window.credentialFingerprint || - receiptWindow.webhookUrl !== window.webhookUrl || - count >= 3 || - receipt.result?.latestAttemptId === latest.id || - !latest.redelivery || - latest.statusCode === null || - latest.statusCode < 500 || - Date.parse(latest.deliveredAt) < - Date.parse(String(receipt.result?.requestedAt)) || - Date.parse(String(receipt.result?.retryAt)) > now.getTime()) - ) - continue; - inspected += 1; - const detail = await getGitHubAppWebhookDelivery({ - fetch: fetchImpl, - appToken, - deliveryId: original.id, - }); - const comment = detail.payload.comment; - if ( - detail.id !== original.id || - detail.guid !== guid || - detail.redelivery || - detail.url !== webhookUrl || - detail.event !== original.event || - detail.deliveredAt !== original.deliveredAt || - detail.statusCode !== original.statusCode || - detail.installationId !== original.installationId || - detail.repositoryId !== original.repositoryId || - detail.action !== "created" || - detail.payload.action !== "created" || - detail.payload.installationId !== credentials.installationId || - detail.payload.repositoryId !== original.repositoryId || - !comment || - comment.userType !== "User" || - detail.payload.senderId !== comment.userId || - Date.parse(comment.createdAt) < floor || - comment.createdAt !== comment.updatedAt || - !(await githubRecoveryResource( - db, - endpoint, - original.repositoryId, - detail.payload.repositoryFullName, - )) - ) { - await rememberGitHubRecoverySkip( - endpoint, - context, - window, - original, - "not_eligible_for_recovery", - ); - continue; - } - if (!(await githubRecoverySourceIsCurrent(detail, credentials))) { - await rememberGitHubRecoverySkip( - endpoint, - context, - window, - original, - "source_changed_or_unavailable", - ); - continue; - } - await withCredentialMutationLease(endpoint, async (lease) => { - await lease.assertOwned(); - const currentConfig = await readGitHubAppWebhookConfig({ - fetch: fetchImpl, - appToken, - }); - if ( - currentConfig.url !== webhookUrl || - currentConfig.contentType !== "json" || - currentConfig.insecureSsl !== "0" - ) - return; - const claimed = await db.transaction(async (tx) => { - await lease.assertOwned(tx); - const current = await runtimeCallbackEndpoint( - tx, - endpoint.id, - context, - ["active"], - ); - if ( - !current || - !(await githubRecoveryResource( - tx, - current, - original.repositoryId!, - detail.payload.repositoryFullName, - )) - ) - return null; - const connection = await tx - .select({ id: toolConnections.id }) - .from(toolConnections) - .where( - and( - eq(toolConnections.id, current.connectionId), - eq(toolConnections.companyId, current.companyId), - eq(toolConnections.enabled, true), - eq(toolConnections.status, "active"), - ), - ) - .then((items) => items[0]); - const scan = await tx - .select({ id: chatSdkState.id }) - .from(chatSdkState) - .where( - and( - eq(chatSdkState.endpointId, endpoint.id), - eq(chatSdkState.stateKey, GITHUB_RECOVERY_STATE_KEY), - eq(chatSdkState.version, claimedVersion), - ), - ) - .then((items) => items[0]); - const local = await githubRecoveryAlreadyReceived( - tx, - current, - guid, - ); - if ( - !connection || - !scan || - local || - shuttingDown || - Date.now() >= Date.parse(window.nextScanAt) - ) - return null; - const result = { - attempts: count + 1, - latestAttemptId: latest.id, - requestedAt: new Date().toISOString(), - retryAt: new Date( - Date.now() + 60_000 * 5 ** count, - ).toISOString(), - }; - const payload: GitHubRecoveryReceipt = { - ...window, - original: detail, - }; - const changes = receipt - ? await tx - .update(chatActions) - .set({ - status: "processing", - result, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, receipt.id), - eq(chatActions.updatedAt, receipt.updatedAt), - ), - ) - .returning({ id: chatActions.id }) - : await tx - .insert(chatActions) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - kind: "github_webhook_recovery", - providerActionId: recoveryId, - payload, - status: "processing", - result, - }) - .onConflictDoNothing() - .returning({ id: chatActions.id }); - return changes[0]?.id ?? null; - }); - if (!claimed) return; - requested += 1; - // Commit the immutable denial fence before transport. Even an - // ambiguous POST is not reattempted without a distinct failed - // provider delivery. A stale scanner cannot overwrite a successor. - await lease.assertOwned(); - try { - await requestGitHubAppWebhookRedelivery({ - fetch: fetchImpl, - appToken, - deliveryId: original.id, - }); - await db - .update(chatActions) - .set({ status: "processed", updatedAt: new Date() }) - .where( - and( - eq(chatActions.id, claimed), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${count + 1}`, - ), - ); - } catch (error) { - await db - .update(chatActions) - .set({ status: "delivery_unknown", updatedAt: new Date() }) - .where( - and( - eq(chatActions.id, claimed), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${count + 1}`, - ), - ); - throw error; - } - }); - } - } catch (error) { - outcome = "scan_failed"; - nextDelay = Math.max( - 5 * 60_000, - Math.min( - 24 * 60 * 60_000, - Number((error as { retryAfterMs?: unknown })?.retryAfterMs) || 0, - ), - ); - logger.warn( - { endpointId: endpoint.id, error: redactError(error) }, - "GitHub webhook recovery deferred", - ); - } finally { - await persistence.compareAndSet({ - ...scope, - key: GITHUB_RECOVERY_STATE_KEY, - expectedVersion: claimedVersion, - value: { - ...window, - nextScanAt: new Date(Date.now() + nextDelay).toISOString(), - outcome, - requested, - }, - expiresAt: null, - }); - } - } - return rows.length; - } - - async function processPendingGitHubWebhookIngress( - limit = 25, - onlyActionId?: string, - ) { - const now = new Date(); - const actions = await db - .select({ id: chatActions.id }) - .from(chatActions) - .where(and( - onlyActionId ? eq(chatActions.id, onlyActionId) : undefined, - dueChatAction("github_webhook_ingress", now.getTime()), - )) - .orderBy(asc(chatActions.createdAt)) - .limit(limit); - for (const action of actions) await processGitHubWebhookIngress(action.id); - return actions.length; - } - - async function stageSlackSessionStop( - endpoint: EndpointRow, - event: SlackSessionStop, - runtimeContext: RuntimeContext, - ): Promise { - return db.transaction(async (tx) => { - const currentEndpoint = await runtimeCallbackEndpoint( - tx, - endpoint.id, - runtimeContext, - ["active"], - ); - if (!currentEndpoint || currentEndpoint.provider !== "slack") return null; - - const conversation = await tx - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.companyId, currentEndpoint.companyId), - eq(chatConversations.endpointId, currentEndpoint.id), - eq(chatConversations.externalThreadId, event.threadId), - inArray(chatConversations.state, ["active", "waiting"]), - ), - ) - .orderBy(desc(chatConversations.sessionGeneration)) - .limit(1) - .for("update") - .then((rows) => rows[0] ?? null); - const principal = await tx - .select() - .from(chatExternalPrincipals) - .where( - and( - eq(chatExternalPrincipals.companyId, currentEndpoint.companyId), - eq(chatExternalPrincipals.provider, "slack"), - eq( - chatExternalPrincipals.providerAccountId, - currentEndpoint.providerAccountId ?? "unknown", - ), - eq(chatExternalPrincipals.externalId, event.userId), - eq(chatExternalPrincipals.kind, "user"), - eq(chatExternalPrincipals.isBot, false), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - const issue = conversation - ? await tx - .select() - .from(issues) - .where( - and( - eq(issues.companyId, currentEndpoint.companyId), - eq(issues.id, conversation.issueId), - eq(issues.assigneeAgentId, currentEndpoint.assignedAgentId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - - // Slack carries fractional event timestamps. Treat that exact instant as - // the cancellation boundary so a run admitted even milliseconds after - // Stop was clicked can never be mistaken for the stopped session. - const eventCutoff = event.occurredAt; - const futureEvent = event.occurredAt.getTime() > Date.now() + 60_000; - let target: SlackSessionStopTarget | null = null; - if (issue && !futureEvent && issue.executionRunId) { - const run = await tx - .select({ - agentId: heartbeatRuns.agentId, - companyId: heartbeatRuns.companyId, - contextSnapshot: heartbeatRuns.contextSnapshot, - createdAt: heartbeatRuns.createdAt, - id: heartbeatRuns.id, - startedAt: heartbeatRuns.startedAt, - }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.id, issue.executionRunId), - eq(heartbeatRuns.companyId, currentEndpoint.companyId), - eq(heartbeatRuns.agentId, currentEndpoint.assignedAgentId), - inArray(heartbeatRuns.status, [ - "queued", - "running", - "scheduled_retry", - ]), - lte(heartbeatRuns.createdAt, eventCutoff), - or( - isNull(heartbeatRuns.startedAt), - lte(heartbeatRuns.startedAt, eventCutoff), - ), - or( - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, - sql`${heartbeatRuns.contextSnapshot}->>'taskId' = ${issue.id}`, - ), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if (run) target = { id: run.id, kind: "run" }; - } else if (issue && !futureEvent && !issue.executionRunId) { - const wakeup = await tx - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, currentEndpoint.companyId), - eq(agentWakeupRequests.agentId, currentEndpoint.assignedAgentId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - isNull(agentWakeupRequests.runId), - lte(agentWakeupRequests.requestedAt, eventCutoff), - or( - sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, - sql`${agentWakeupRequests.payload}->>'taskId' = ${issue.id}`, - sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId' = ${issue.id}`, - sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId' = ${issue.id}`, - ), - ), - ) - .orderBy(desc(agentWakeupRequests.requestedAt)) - .limit(1) - .for("update") - .then((rows) => rows[0] ?? null); - if (wakeup) target = { id: wakeup.id, kind: "wakeup" }; - } - - const payload: SlackSessionStopPayload = { - version: 1, - assignedAgentId: currentEndpoint.assignedAgentId, - eventTimestamp: event.eventTimestamp, - issueId: issue?.id ?? null, - occurredAt: event.occurredAt.toISOString(), - sessionGeneration: conversation?.sessionGeneration ?? null, - target, - threadId: event.threadId, - userId: event.userId, - }; - await notifyChatActionWork(tx, "slack_session_stop"); - const [inserted] = await tx - .insert(chatActions) - .values({ - companyId: currentEndpoint.companyId, - endpointId: currentEndpoint.id, - conversationId: conversation?.id ?? null, - principalId: principal?.id ?? null, - kind: "slack_session_stop", - providerActionId: `slack_session_stop:${event.providerEventId}`, - payload, - status: "received", - }) - .onConflictDoNothing() - .returning({ id: chatActions.id }); - return ( - inserted?.id ?? - (await tx - .select({ id: chatActions.id }) - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, currentEndpoint.id), - eq( - chatActions.providerActionId, - `slack_session_stop:${event.providerEventId}`, - ), - eq(chatActions.kind, "slack_session_stop"), - ), - ) - .then((rows) => rows[0]?.id ?? null)) - ); - }); - } - - async function processSlackSessionStop(actionId: string): Promise { - const now = new Date(); - const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); - const action = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "slack_session_stop"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!action || ["processed", "cancelled"].includes(action.status)) return; - if (action.status === "processing" && action.updatedAt > staleBefore) - return; - if (action.status === "failed") { - if (action.result?.retryable !== true) return; - const retryAt = - typeof action.result.retryAt === "string" - ? new Date(action.result.retryAt) - : null; - if (retryAt && retryAt > now) return; - } else if (action.status !== "received" && action.status !== "processing") { - return; - } - const payload = slackSessionStopPayload(action.payload); - const previousAttempts = - typeof action.result?.attempts === "number" && - Number.isSafeInteger(action.result.attempts) - ? Math.max(0, action.result.attempts) - : 0; - const attempts = previousAttempts + 1; - if (!payload) { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slack_session_stop"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { - attempts, - code: "slack_session_stop_payload_invalid", - retryable: false, - }, - updatedAt: now, - }) - .where(eq(chatActions.id, action.id)); - }); - return; - } - - type StopClaim = - | { kind: "cancel_run"; runId: string; userId: string } - | { kind: "settled" } - | { kind: "retry" }; - const claim = await db.transaction(async (tx): Promise => { - const current = await tx - .select() - .from(chatActions) - .where(eq(chatActions.id, action.id)) - .for("update") - .then((rows) => rows[0] ?? null); - const currentRetryAt = - typeof current?.result?.retryAt === "string" - ? new Date(current.result.retryAt) - : null; - const currentRetryIsDue = - currentRetryAt === null || - (Number.isFinite(currentRetryAt.getTime()) && currentRetryAt <= now); - const eligible = - current?.status === "received" || - (current?.status === "processing" && - current.updatedAt <= staleBefore && - (current.result?.attempts ?? 0) === previousAttempts) || - (current?.status === "failed" && - current.result?.retryable === true && - currentRetryIsDue && - (current.result?.attempts ?? 0) === previousAttempts); - if (!current || !eligible) return { kind: "settled" }; - - const settle = async ( - code: string, - actorUserId: string | null, - targetKind: "run" | "wakeup" | "none", - ): Promise => { - await notifyChatActionWork(tx, "slack_session_stop"); - await tx - .update(chatActions) - .set({ - status: - code === "slack_session_stop_cancelled" - ? "processed" - : "cancelled", - result: { attempts, code, retryable: false }, - updatedAt: now, - }) - .where(eq(chatActions.id, current.id)); - await logActivity(tx as unknown as Db, { - companyId: current.companyId, - actorType: actorUserId ? "user" : "system", - actorId: actorUserId ?? "slack-session-stop", - action: - code === "slack_session_stop_cancelled" - ? "chat.slack_session_stopped" - : "chat.slack_session_stop_filtered", - entityType: "chat_action", - entityId: current.id, - details: { - endpointId: current.endpointId, - conversationId: current.conversationId, - eventTimestamp: payload.eventTimestamp, - targetKind, - resultCode: code, - }, - }); - return { kind: "settled" }; - }; - - if ( - !payload.target || - !payload.issueId || - !payload.sessionGeneration || - !current.conversationId || - !current.principalId - ) { - return settle("slack_session_stop_no_current_work", null, "none"); - } - const endpoint = await tx - .select() - .from(chatEndpoints) - .where( - and( - eq(chatEndpoints.id, current.endpointId), - eq(chatEndpoints.companyId, current.companyId), - eq(chatEndpoints.provider, "slack"), - eq(chatEndpoints.status, "active"), - eq(chatEndpoints.assignedAgentId, payload.assignedAgentId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - const conversation = endpoint - ? await tx - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.id, current.conversationId), - eq(chatConversations.companyId, current.companyId), - eq(chatConversations.endpointId, current.endpointId), - eq(chatConversations.issueId, payload.issueId), - eq(chatConversations.externalThreadId, payload.threadId), - eq( - chatConversations.sessionGeneration, - payload.sessionGeneration, - ), - inArray(chatConversations.state, ["active", "waiting"]), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - const resource = - endpoint && conversation?.resourceId - ? await tx - .select() - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.id, conversation.resourceId), - eq(chatEndpointResources.companyId, current.companyId), - eq(chatEndpointResources.endpointId, current.endpointId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - const principal = endpoint - ? await tx - .select() - .from(chatExternalPrincipals) - .where( - and( - eq(chatExternalPrincipals.id, current.principalId), - eq(chatExternalPrincipals.companyId, current.companyId), - eq(chatExternalPrincipals.provider, "slack"), - eq( - chatExternalPrincipals.providerAccountId, - endpoint.providerAccountId ?? "unknown", - ), - eq(chatExternalPrincipals.externalId, payload.userId), - eq(chatExternalPrincipals.kind, "user"), - eq(chatExternalPrincipals.isBot, false), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - const authorization = - endpoint && principal - ? await lockCurrentPrincipalAuthorization(tx, endpoint, principal.id) - : null; - const destinationAllowed = - endpoint && conversation - ? conversation.isDirectMessage - ? endpoint.allowDirectMessages - : nonDirectDestinationAllowed(endpoint, resource) - : false; - const issue = - endpoint && conversation - ? await tx - .select() - .from(issues) - .where( - and( - eq(issues.id, payload.issueId), - eq(issues.companyId, current.companyId), - eq(issues.assigneeAgentId, payload.assignedAgentId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - if ( - !endpoint || - !conversation || - !principal || - !authorization?.allowed || - !authorization.userId || - !destinationAllowed || - !issue - ) { - return settle( - "slack_session_stop_no_longer_authorized", - null, - payload.target.kind, - ); - } - - const eventCutoff = new Date(payload.occurredAt); - let runId: string | null = null; - if (payload.target.kind === "run") { - runId = payload.target.id; - } else { - const wakeup = await tx - .select() - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.id, payload.target.id), - eq(agentWakeupRequests.companyId, current.companyId), - eq(agentWakeupRequests.agentId, payload.assignedAgentId), - lte(agentWakeupRequests.requestedAt, eventCutoff), - or( - sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, - sql`${agentWakeupRequests.payload}->>'taskId' = ${issue.id}`, - sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId' = ${issue.id}`, - sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId' = ${issue.id}`, - ), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if ( - wakeup && - !wakeup.runId && - ["queued", "deferred_issue_execution"].includes(wakeup.status) && - !issue.executionRunId - ) { - await tx - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: "Cancelled from the bound Slack agent session", - updatedAt: now, - }) - .where( - and( - eq(agentWakeupRequests.id, wakeup.id), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - isNull(agentWakeupRequests.runId), - ), - ); - const agentName = await tx - .select({ name: agents.name }) - .from(agents) - .where( - and( - eq(agents.companyId, current.companyId), - eq(agents.id, payload.assignedAgentId), - ), - ) - .then((rows) => rows[0]?.name ?? "Paperclip agent"); - await stageAuthorizedTaskControlPublication(tx, { - companyId: current.companyId, - conversationId: conversation.id, - endpointId: current.endpointId, - idempotencyKey: `control:stop:${current.id}`, - issueId: issue.id, - payload: projectSafeChatPublication({ - classification: "external", - progressState: "failed", - source: "safe_milestone", - text: safeMilestoneText({ - agentName, - errorCode: "slack_session_stopped", - issueId: issue.id, - milestone: "failed", - publicBaseUrl: getPublicBaseUrl(), - }), - }), - principalId: principal.id, - }); - return settle( - "slack_session_stop_cancelled", - authorization.userId, - "wakeup", - ); - } - runId = wakeup?.runId ?? null; - if (!runId && wakeup?.status === "claimed") { - await notifyChatActionWork(tx, "slack_session_stop"); - await tx - .update(chatActions) - .set({ - status: "failed", - result: { - attempts, - code: "slack_session_stop_wakeup_in_flight", - retryable: true, - retryAt: new Date(Date.now() + 1_000).toISOString(), - }, - updatedAt: now, - }) - .where(eq(chatActions.id, current.id)); - return { kind: "retry" }; - } - } - - const run = runId - ? await tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.id, runId), - eq(heartbeatRuns.companyId, current.companyId), - eq(heartbeatRuns.agentId, payload.assignedAgentId), - payload.target.kind === "wakeup" - ? eq(heartbeatRuns.wakeupRequestId, payload.target.id) - : and( - lte(heartbeatRuns.createdAt, eventCutoff), - or( - isNull(heartbeatRuns.startedAt), - lte(heartbeatRuns.startedAt, eventCutoff), - ), - ), - or( - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, - sql`${heartbeatRuns.contextSnapshot}->>'taskId' = ${issue.id}`, - ), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - if (!run) { - return settle( - "slack_session_stop_target_superseded", - authorization.userId, - payload.target.kind, - ); - } - if (run.status === "cancelled") { - return settle( - "slack_session_stop_cancelled", - authorization.userId, - "run", - ); - } - if ( - issue.executionRunId !== run.id || - !["queued", "running", "scheduled_retry"].includes(run.status) - ) { - return settle( - "slack_session_stop_target_superseded", - authorization.userId, - "run", - ); - } - await notifyChatActionWork(tx, "slack_session_stop"); - await tx - .update(chatActions) - .set({ - status: "processing", - result: { - attempts, - authorizedUserId: authorization.userId, - runId: run.id, - }, - updatedAt: now, - }) - .where(eq(chatActions.id, current.id)); - return { - kind: "cancel_run", - runId: run.id, - userId: authorization.userId, - }; - }); - if (claim.kind !== "cancel_run") return; - - try { - if (!options.heartbeat.cancelRun) { - throw new Error("Heartbeat cancellation is unavailable"); - } - await options.heartbeat.cancelRun( - claim.runId, - "Stopped from the bound Slack agent session", - { - errorCode: "slack_session_stopped", - eventMessage: "run cancelled from Slack", - resultJson: { - cancelledByActorType: "user", - cancelledByUserId: claim.userId, - }, - eventPayload: { - endpointId: action.endpointId, - conversationId: action.conversationId, - provider: "slack", - }, - }, - ); - await enqueueChatRunMilestones(db, { publicBaseUrl: getPublicBaseUrl() }); - const authoritativeRun = await db - .select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, claim.runId)) - .then((rows) => rows[0] ?? null); - const cancellationWon = authoritativeRun?.status === "cancelled"; - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slack_session_stop"); - const [settled] = await tx - .update(chatActions) - .set({ - status: cancellationWon ? "processed" : "cancelled", - result: { - attempts, - code: cancellationWon - ? "slack_session_stop_cancelled" - : "slack_session_stop_target_superseded", - runId: claim.runId, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempts}`, - sql`${chatActions.result}->>'runId' = ${claim.runId}`, - ), - ) - .returning({ id: chatActions.id }); - if (!settled) return; - await logActivity(tx as unknown as Db, { - companyId: action.companyId, - actorType: "user", - actorId: claim.userId, - action: cancellationWon - ? "chat.slack_session_stopped" - : "chat.slack_session_stop_filtered", - entityType: "chat_action", - entityId: action.id, - details: { - endpointId: action.endpointId, - conversationId: action.conversationId, - eventTimestamp: payload.eventTimestamp, - targetKind: "run", - resultCode: cancellationWon - ? "slack_session_stop_cancelled" - : "slack_session_stop_target_superseded", - }, - }); - }); - } catch (error) { - const retryable = attempts < 5; - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "slack_session_stop"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { - attempts, - code: "slack_session_stop_cancellation_failed", - retryable, - ...(retryable - ? { - retryAt: new Date( - Date.now() + Math.min(60_000, 1_000 * 2 ** (attempts - 1)), - ).toISOString(), - } - : {}), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempts}`, - sql`${chatActions.result}->>'runId' = ${claim.runId}`, - ), - ); - }); - logger.warn( - { - actionId: action.id, - endpointId: action.endpointId, - attempts, - error: redactError(error), - }, - "Slack session stop cancellation deferred", - ); - } - } - - async function processPendingSlackSessionStops( - limit = 25, - onlyActionId?: string, - ) { - const actions = await db.select({ id: chatActions.id }).from(chatActions) - .where(and(dueChatAction("slack_session_stop"), onlyActionId ? eq(chatActions.id, onlyActionId) : undefined)) - .orderBy(asc(chatActions.createdAt)).limit(limit); - for (const action of actions) await processSlackSessionStop(action.id); - return actions.length; - } - async function verifySlackRequestUrl(endpoint: EndpointRow, request: Request): Promise { const callback = await inspectSlackCallback(request.clone()); if (!callback?.isUrlVerification) return null; @@ -29273,297 +24858,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } } - function telegramStopSubscriptionScope( - endpoint: EndpointRow, - context: LifecycleRuntimeFence, - ) { - if ( - endpoint.provider !== "telegram" || - !endpoint.botExternalId || - !getWebhookPublicBaseUrl() - ) - return null; - const webhookUrlSha256 = createHash("sha256") - .update( - `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/telegram`, - ) - .digest("hex"); - return { - version: 1, - botUserId: endpoint.botExternalId, - runtimeGeneration: context.generation, - credentialFingerprint: context.credentialFingerprint, - webhookUrlSha256, - }; - } - - async function hasTelegramStopSubscription( - endpoint: EndpointRow, - context: LifecycleRuntimeFence, - ): Promise { - const scope = telegramStopSubscriptionScope(endpoint, context); - if (!scope) return false; - const [receipt] = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.companyId, endpoint.companyId), - eq(chatActions.endpointId, endpoint.id), - eq(chatActions.kind, "telegram_stop_subscription"), - eq( - chatActions.providerActionId, - `telegram-stop-subscription:${scope.runtimeGeneration}:${scope.credentialFingerprint}:${scope.webhookUrlSha256}`, - ), - eq(chatActions.status, "processed"), - ), - ) - .limit(1); - return ( - !!receipt && - receipt.result?.code === "telegram_stop_subscription_confirmed" && - Object.keys(receipt.payload).length === Object.keys(scope).length && - Object.entries(scope).every( - ([key, value]) => receipt.payload[key] === value, - ) - ); - } - - async function recordTelegramStopSubscription( - endpoint: EndpointRow, - context: LifecycleRuntimeFence, - lease: CredentialMutationLeaseGuard, - ): Promise { - const scope = telegramStopSubscriptionScope(endpoint, context); - if (!scope) - throw new Error("Telegram Stop subscription scope is unavailable"); - await db.transaction(async (tx) => { - await lease.assertOwned(tx); - const current = await runtimeCallbackEndpoint(tx, endpoint.id, context, [ - "verifying", - "active", - ]); - if ( - !current || - current.botExternalId !== scope.botUserId || - current.publicId !== endpoint.publicId - ) - throw new Error("Telegram Stop subscription ownership changed"); - const [connection] = await tx - .select() - .from(toolConnections) - .where( - and( - eq(toolConnections.id, current.connectionId), - eq(toolConnections.companyId, current.companyId), - ), - ) - .for("share"); - if ( - !connection?.enabled || - connection.status !== "active" || - credentialFingerprint(connection.credentialSecretRefs) !== - context.credentialFingerprint - ) - throw new Error( - "Telegram Stop subscription credential ownership changed", - ); - await tx - .insert(chatActions) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - kind: "telegram_stop_subscription", - providerActionId: `telegram-stop-subscription:${scope.runtimeGeneration}:${scope.credentialFingerprint}:${scope.webhookUrlSha256}`, - payload: scope, - status: "processed", - result: { code: "telegram_stop_subscription_confirmed" }, - }) - .onConflictDoUpdate({ - target: [chatActions.endpointId, chatActions.providerActionId], - set: { - payload: scope, - status: "processed", - result: { code: "telegram_stop_subscription_confirmed" }, - updatedAt: new Date(), - }, - }); - await lease.assertOwned(tx); - }); - } - - async function repairTelegramStopSubscription( - endpoint: EndpointRow, - payload: TelegramMaintenancePayload, - lease: CredentialMutationLeaseGuard, - ): Promise { - const plainObject = (value: unknown): Record => - value !== null && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : {}; - const context = { - generation: payload.runtimeGeneration, - credentialFingerprint: payload.credentialFingerprint, - }; - const scope = telegramStopSubscriptionScope(endpoint, context); - const reject = () => - Object.assign( - new Error( - "Telegram subscription cannot be safely upgraded under current ownership", - ), - { - code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED", - }, - ); - if ( - !scope || - scope.webhookUrlSha256 !== payload.webhookUrlSha256 || - scope.botUserId !== payload.botUserId - ) - throw reject(); - const assertCurrent = async () => - db.transaction(async (tx) => { - await lease.assertOwned(tx); - const current = await runtimeCallbackEndpoint( - tx, - endpoint.id, - context, - ["active"], - ); - if ( - !current || - current.companyId !== endpoint.companyId || - current.botExternalId !== scope.botUserId || - current.publicId !== endpoint.publicId - ) - throw reject(); - const [connection] = await tx - .select() - .from(toolConnections) - .where( - and( - eq(toolConnections.id, current.connectionId), - eq(toolConnections.companyId, current.companyId), - ), - ) - .for("share"); - if ( - !connection?.enabled || - connection.status !== "active" || - credentialFingerprint(connection.credentialSecretRefs) !== - context.credentialFingerprint - ) - throw reject(); - await lease.assertOwned(tx); - }); - await assertCurrent(); - // A recovered providerConfirmed flag is not a subscription capability. - // Only the separately committed exact current receipt can avoid fresh I/O. - if (await hasTelegramStopSubscription(endpoint, context)) return; - const credentials = await resolveCredentials(endpoint); - if ( - !credentials.botToken || - typeof credentials.webhookSecret !== "string" || - !/^[A-Za-z0-9_-]{1,256}$/.test(credentials.webhookSecret) - ) - throw reject(); - const request = async ( - method: "getMe" | "getWebhookInfo" | "setWebhook", - body: Record = {}, - ) => { - await assertCurrent(); - let response: Response; - try { - response = await fetchImpl( - `https://api.telegram.org/bot${encodeURIComponent(credentials.botToken!)}/${method}`, - { - method: "POST", - redirect: "error", - signal: AbortSignal.timeout(PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS), - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - }, - ); - } catch { - // Network errors can embed a token-bearing URL. Keep maintenance - // diagnostics closed rather than forwarding provider/transport prose. - throw new Error( - "Telegram subscription transport did not confirm completion", - ); - } - let value: unknown; - try { - value = await response.json(); - } catch { - throw new Error("Telegram subscription returned an invalid response"); - } - const result = plainObject(value); - if (!response.ok || result.ok !== true) { - const retryAfter = plainObject(result.parameters).retry_after; - throw Object.assign( - new Error("Telegram subscription request was not confirmed"), - response.status === 429 - ? { - adapter: "telegram", - code: "RATE_LIMITED", - name: "AdapterRateLimitError", - retryAfter: - typeof retryAfter === "number" && Number.isFinite(retryAfter) - ? retryAfter - : undefined, - response: { - status: response.status, - headers: response.headers, - }, - } - : { - adapter: "telegram", - status: response.status, - response: { - status: response.status, - headers: response.headers, - }, - }, - ); - } - return result.result; - }; - const identity = plainObject(await request("getMe")); - if ( - identity.is_bot !== true || - !Number.isSafeInteger(identity.id) || - String(identity.id) !== scope.botUserId - ) - throw reject(); - const url = `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/telegram`; - const plan = telegramStopSubscriptionPlan( - await request("getWebhookInfo"), - url, - ); - if (!plan) throw reject(); - // No URL repoint, credential rotation, certificate guess, IP pin, or queue - // dropping. A successful SET proves the stored secret; GET cannot do so. - if ( - (await request("setWebhook", { - url, - secret_token: credentials.webhookSecret, - ...plan, - drop_pending_updates: false, - })) !== true - ) - throw new Error("Telegram subscription mutation was not confirmed"); - if ( - !telegramStopSubscriptionConfirmed( - await request("getWebhookInfo"), - url, - plan, - ) - ) - throw new Error("Telegram subscription verification changed"); - await assertCurrent(); - await recordTelegramStopSubscription(endpoint, context, lease); - } - async function handleTelegramGenerationStopped( callback: ChatSdkCallbackEvent, context: RuntimeContext, diff --git a/server/src/services/chat-channels/discord/commands.ts b/server/src/services/chat-channels/discord/commands.ts new file mode 100644 index 00000000000..1c2bfd1ac92 --- /dev/null +++ b/server/src/services/chat-channels/discord/commands.ts @@ -0,0 +1,639 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence } from "../runtime-types.js"; + +import { chatCredentialMutationLease, type CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js"; + +import { + and, + desc, + eq, + gt, +} from "drizzle-orm"; + +import { + readChatControlChronology, +} from "../../chat-control-chronology.js"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatConversations, + chatEndpointLeases, + chatEndpointResources, + chatEndpoints, + issues, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatEndpointSetupState, + SafeChatPublicationPayload, + ToolCredentialSecretRef, +} from "@paperclipai/shared"; + +import { + conflict, + notFound, +} from "../../../errors.js"; +import { logger } from "../../../middleware/logger.js"; + +import { + type ChatSdkCallbackEvent, + type DiscordNativeCommandResponse, +} from "../../chat-sdk-runtime.js"; +import { + parseDiscordNativeCommand, + parseDiscordNativeCommandReceipt, + type DiscordNativeCommandTarget, +} from "../../chat-discord-native-commands.js"; + +import { + logActivity, +} from "../../activity-log.js"; + +import { projectSafeChatPublication } from "../../chat-publication-projection.js"; + +import { + readRegisteredDiscordCommandRegistration, + reconcileStoredDiscordCommandRegistration, +} from "../../chat-discord-command-registration-store.js"; + +import type { + Author, + SlashCommandEvent, +} from "chat"; + +import type { EndpointRecord } from "../runtime-types.js"; + +type DiscordCommandsOptions = { + endpointRecord: (endpointId: string) => Promise; + runtimeContextForRecord: (record: EndpointRecord) => RuntimeContext; + runtimeGeneration: (setup: ChatEndpointSetupState) => number; + credentialFingerprint: (refs: ToolCredentialSecretRef[]) => string; + resolveCredentialRefs: (endpoint: EndpointRow, refs: ToolCredentialSecretRef[], database?: DbOrTransaction) => Promise>; + fetchImpl: typeof fetch; + withCredentialMutationLease: ReturnType; + runtimeCallbackRecord: (endpointId: string, context: RuntimeContext, statuses: EndpointRow["status"][]) => Promise; + ensurePrincipal: (endpoint: EndpointRow, author: Author, raw?: unknown) => Promise<{ principal: { id: string }; userId: string | null }>; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean; + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; userId: string | null }>; + getDiscordGatewayOwnership: (endpointId: string) => { leaseKey: string; token: string } | undefined; + discordGatewayRuntimeIsCurrent: (endpointId: string, context: RuntimeContext) => boolean; + stageAuthorizedTaskControlPublication: (tx: DbOrTransaction, input: { companyId: string; conversationId: string; endpointId: string; idempotencyKey: string; issueId: string; payload: SafeChatPublicationPayload; principalId: string }) => Promise<{ id: string }>; + scheduleMessageProcessing: (task: () => Promise) => void; + processPendingPublications: () => Promise; +}; + +export function createDiscordCommands(db: Db, options: DiscordCommandsOptions) { + const { endpointRecord, runtimeContextForRecord, runtimeGeneration, credentialFingerprint, resolveCredentialRefs, fetchImpl, withCredentialMutationLease, runtimeCallbackRecord, ensurePrincipal, runtimeCallbackEndpoint, nonDirectDestinationAllowed, lockCurrentPrincipalAuthorization, getDiscordGatewayOwnership, discordGatewayRuntimeIsCurrent, stageAuthorizedTaskControlPublication, scheduleMessageProcessing, processPendingPublications } = options; + + + async function reconcileDiscordCommands( + endpointId: string, + existingLease?: CredentialMutationLeaseGuard, + force = false, + ): Promise { + const initial = await endpointRecord(endpointId); + if (!initial || initial.endpoint.provider !== "discord") return; + const reconcile = async (lease: CredentialMutationLeaseGuard) => { + const record = await endpointRecord(endpointId); + if ( + !record || + record.endpoint.provider !== "discord" || + !["verifying", "active", "attention"].includes( + record.endpoint.status, + ) || + !record.endpoint.botExternalId || + !record.endpoint.providerAccountId + ) + return; + const scope = { + companyId: record.endpoint.companyId, + endpointId, + applicationId: record.endpoint.botExternalId, + guildId: record.endpoint.providerAccountId, + }; + const fence = runtimeContextForRecord(record); + const authorize = async (tx: DbTransaction) => { + const endpoint = await tx + .select() + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.companyId, scope.companyId), + eq(chatEndpoints.id, scope.endpointId), + ), + ) + .for("no key update") + .then((rows) => rows[0]); + if ( + !endpoint || + endpoint.provider !== "discord" || + !["verifying", "active", "attention"].includes(endpoint.status) || + endpoint.botExternalId !== scope.applicationId || + endpoint.providerAccountId !== scope.guildId || + runtimeGeneration(endpoint.setup) !== fence.generation + ) + throw conflict("Discord command registration authority changed"); + const connection = await tx + .select() + .from(toolConnections) + .where( + and( + eq(toolConnections.companyId, scope.companyId), + eq(toolConnections.id, endpoint.connectionId), + ), + ) + .for("no key update") + .then((rows) => rows[0]); + if ( + !connection?.enabled || + connection.status !== "active" || + credentialFingerprint(connection.credentialSecretRefs) !== + fence.credentialFingerprint + ) + throw conflict("Discord command registration authority changed"); + await lease.assertOwned(tx); + }; + const setCapability = async (tx: DbTransaction, registered: boolean) => { + const current = await tx + .select() + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.companyId, scope.companyId), + eq(chatEndpoints.id, endpointId), + ), + ) + .then((rows) => rows[0]); + if (!current) throw notFound("Chat endpoint not found"); + if ( + current.capabilities.slashCommands === registered && + current.capabilities.ephemeralMessages === registered + ) + return; + await tx + .update(chatEndpoints) + .set({ + capabilities: { + ...current.capabilities, + slashCommands: registered, + ephemeralMessages: registered, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatEndpoints.companyId, scope.companyId), + eq(chatEndpoints.id, endpointId), + ), + ); + await logActivity(tx as unknown as Db, { + companyId: scope.companyId, + actorType: "system", + actorId: "system", + action: registered + ? "chat_endpoint.commands_registered" + : "chat_endpoint.commands_unavailable", + entityType: "tool_connection", + entityId: current.connectionId, + details: { endpointId, provider: "discord" }, + }); + }; + await lease.assertOwned(); + const credentials = await resolveCredentialRefs( + record.endpoint, + record.credentialSecretRefs, + ); + if ( + credentials.applicationId !== scope.applicationId || + credentials.guildId !== scope.guildId + ) + throw conflict("Discord command registration identity changed"); + const result = await reconcileStoredDiscordCommandRegistration(db, { + scope, + runtimeFence: { + generation: fence.generation, + credentialFingerprint: fence.credentialFingerprint, + }, + botToken: credentials.botToken, + fetch: fetchImpl, + authorize, + force, + onState: async (tx, state) => + setCapability(tx, state.phase === "registered"), + }); + if (result.kind === "deferred") return; + await db.transaction(async (tx) => { + await authorize(tx); + const registered = + result.kind === "registered" + ? await readRegisteredDiscordCommandRegistration(tx, scope) + : null; + await setCapability(tx, registered !== null); + await lease.assertOwned(tx); + }); + }; + try { + if (existingLease) await reconcile(existingLease); + else await withCredentialMutationLease(initial.endpoint, reconcile); + } catch { + // Commands are additive. Do not fail an otherwise working mention/thread + // connection or expose provider errors/tokens when registration is busy. + logger.warn( + { endpointId }, + "Discord command registration remains unconfirmed", + ); + } + } + + async function handleDiscordNativeCommand( + event: ChatSdkCallbackEvent, + runtimeContext: RuntimeContext, + ): Promise { + const denied: DiscordNativeCommandResponse = { kind: "denied" }; + const record = await runtimeCallbackRecord( + event.endpointId, + runtimeContext, + ["active"], + ); + if ( + !record || + record.endpoint.provider !== "discord" || + record.endpoint.capabilities.slashCommands !== true || + event.transport !== "discord_gateway" || + runtimeContext.discordGatewayOwned !== true + ) + return denied; + const scope = { + companyId: record.endpoint.companyId, + endpointId: record.endpoint.id, + applicationId: record.endpoint.botExternalId ?? "", + guildId: record.endpoint.providerAccountId ?? "", + }; + const invocation = parseDiscordNativeCommand(event, scope); + if (!invocation) return denied; + const initialRegistration = await readRegisteredDiscordCommandRegistration( + db, + scope, + ); + if ( + initialRegistration?.receipt.commandId !== invocation.registeredCommandId + ) + return denied; + const principal = await ensurePrincipal( + record.endpoint, + event.event.user, + event.event.raw, + ); + const authorityChanged = new Error("Discord command authority changed"); + let publish = false; + try { + const response = await db.transaction( + async (tx): Promise => { + const endpoint = await runtimeCallbackEndpoint( + tx, + scope.endpointId, + runtimeContext, + ["active"], + ); + if ( + !endpoint || + endpoint.companyId !== scope.companyId || + endpoint.provider !== "discord" || + endpoint.botExternalId !== scope.applicationId || + endpoint.providerAccountId !== scope.guildId || + endpoint.capabilities.slashCommands !== true + ) + return denied; + const connection = await tx + .select() + .from(toolConnections) + .where( + and( + eq(toolConnections.companyId, scope.companyId), + eq(toolConnections.id, endpoint.connectionId), + ), + ) + .for("no key update") + .then((rows) => rows[0] ?? null); + if ( + !connection || + !connection.enabled || + connection.status !== "active" || + credentialFingerprint(connection.credentialSecretRefs) !== + runtimeContext.credentialFingerprint + ) + return denied; + const registration = await readRegisteredDiscordCommandRegistration( + tx, + scope, + true, + ); + if ( + registration?.receipt.commandId !== invocation.registeredCommandId + ) + return denied; + const providerActionId = `discord-native-command:${invocation.interactionId}`; + const prior = await tx + .select() + .from(chatActions) + .where( + and( + eq(chatActions.companyId, scope.companyId), + eq(chatActions.endpointId, scope.endpointId), + eq(chatActions.providerActionId, providerActionId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const receipt = prior + ? parseDiscordNativeCommandReceipt(prior, invocation, scope) + : null; + if ( + prior && + (!receipt || + receipt.principalId !== principal.principal.id || + receipt.runtimeFence.generation !== runtimeContext.generation || + receipt.runtimeFence.credentialFingerprint !== + runtimeContext.credentialFingerprint) + ) + return denied; + // A replay selects only its original target, including a completed + // generation. Never reinterpret an old close as a new task's command. + const conversation = receipt + ? receipt.target + ? await tx + .select() + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, scope.companyId), + eq(chatConversations.endpointId, scope.endpointId), + eq(chatConversations.id, receipt.target.conversationId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null + : invocation.sourceKind !== "guild_channel" + ? await tx + .select() + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, scope.companyId), + eq(chatConversations.endpointId, scope.endpointId), + eq( + chatConversations.externalConversationId, + invocation.channelId, + ), + eq( + chatConversations.externalThreadId, + invocation.threadId, + ), + ), + ) + .orderBy(desc(chatConversations.sessionGeneration)) + .limit(1) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + if ( + (receipt?.target && !conversation) || + (conversation && + (conversation.externalConversationId !== invocation.channelId || + conversation.externalThreadId !== invocation.threadId || + conversation.isDirectMessage !== + (invocation.sourceKind === "direct_message") || + !["active", "waiting", "completed"].includes( + conversation.state, + ) || + (receipt?.target && + (conversation.issueId !== receipt.target.issueId || + conversation.sessionGeneration !== + receipt.target.sessionGeneration)))) + ) + return denied; + const resource = + invocation.sourceKind === "direct_message" + ? null + : await tx + .select() + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.companyId, scope.companyId), + eq(chatEndpointResources.endpointId, scope.endpointId), + eq( + chatEndpointResources.providerResourceId, + invocation.providerResourceId, + ), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if ( + invocation.sourceKind === "direct_message" + ? !endpoint.allowDirectMessages + : !nonDirectDestinationAllowed(endpoint, resource) || + (conversation && conversation.resourceId !== resource?.id) + ) + return denied; + const authorization = await lockCurrentPrincipalAuthorization( + tx, + endpoint, + principal.principal.id, + ); + if ( + !authorization.allowed || + authorization.userId !== principal.userId + ) + return denied; + const issue = conversation + ? await tx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, scope.companyId), + eq(issues.id, conversation.issueId), + ), + ) + .for("share") + .then((rows) => rows[0] ?? null) + : null; + if (conversation && !issue) return denied; + const ownership = getDiscordGatewayOwnership(scope.endpointId); + if ( + !ownership || + !discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext) + ) + return denied; + const lease = await tx + .select() + .from(chatEndpointLeases) + .where( + and( + eq(chatEndpointLeases.companyId, scope.companyId), + eq(chatEndpointLeases.endpointId, scope.endpointId), + eq(chatEndpointLeases.leaseKey, ownership.leaseKey), + eq(chatEndpointLeases.token, ownership.token), + gt(chatEndpointLeases.expiresAt, new Date()), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if ( + !lease || + !discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext) + ) + return denied; + if (receipt) + return { kind: "accepted", content: receipt.result.content }; + const target: DiscordNativeCommandTarget | null = conversation + ? { + conversationId: conversation.id, + issueId: conversation.issueId, + sessionGeneration: conversation.sessionGeneration, + } + : null; + const active = + conversation && + issue && + ["active", "waiting"].includes(conversation.state) && + (invocation.sourceKind !== "direct_message" || + !["done", "cancelled"].includes(issue.status)); + let content: string; + let publicationId: string | null = null; + if (invocation.command === "status") { + content = + issue && (invocation.sourceKind !== "direct_message" || active) + ? `${issue.identifier}: ${issue.title} — ${issue.status}${conversation?.state === "completed" ? " (conversation closed)" : ""}`.slice( + 0, + 2000, + ) + : invocation.sourceKind === "guild_channel" + ? "Open the Discord task thread to view its Paperclip status." + : "No task is active in this conversation."; + } else if ( + invocation.command === "new" && + invocation.sourceKind !== "direct_message" + ) { + content = + "Open a new Discord thread by mentioning this agent in a new channel message. This thread stays bound to its current Paperclip task."; + } else if (!active || !conversation || !issue) { + content = + invocation.sourceKind === "direct_message" + ? "No task is active. Send a message to start a new Paperclip task." + : "No active task is bound here. Open its Discord thread to manage it."; + } else if ( + (await readChatControlChronology( + tx, + record.endpoint, + { + id: conversation.externalThreadId, + channelId: conversation.externalConversationId, + }, + { + receivedAt: new Date(), + normalizedEvent: { + message: { providerMessageId: invocation.interactionId }, + }, + }, + )) === "before_or_unproven" + ) { + content = + "This command predates an already completed chat control. Send a new command for the current conversation."; + } else { + const publicText = + invocation.command === "new" + ? "Send your request to start a new Paperclip task." + : invocation.sourceKind === "direct_message" + ? "This chat conversation is closed. Send another message to start a new task." + : "This chat conversation is closed. A later message here will continue the same Paperclip task."; + const publication = await stageAuthorizedTaskControlPublication( + tx, + { + companyId: scope.companyId, + endpointId: scope.endpointId, + conversationId: conversation.id, + issueId: issue.id, + principalId: principal.principal.id, + idempotencyKey: `control:${invocation.command}:discord:${scope.endpointId}:${invocation.interactionId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "task_control", + text: publicText, + }), + }, + ); + publicationId = publication.id; + content = `${invocation.command === "new" ? "New-task" : "Close"} request recorded. Watch this conversation for confirmation.`; + } + const inserted = await tx + .insert(chatActions) + .values({ + companyId: scope.companyId, + endpointId: scope.endpointId, + conversationId: target?.conversationId ?? null, + principalId: principal.principal.id, + kind: "discord_native_command", + providerActionId, + payload: { + version: 1, + invocation, + runtimeFence: { + generation: runtimeContext.generation, + credentialFingerprint: runtimeContext.credentialFingerprint, + }, + target, + }, + status: "processed", + result: { + kind: "discord_native_command_recorded", + content, + publicationId, + }, + }) + .onConflictDoNothing() + .returning({ id: chatActions.id }); + if (!inserted.length) throw authorityChanged; + await logActivity(tx as unknown as Db, { + companyId: scope.companyId, + actorType: authorization.userId ? "user" : "system", + actorId: authorization.userId ?? `chat:${principal.principal.id}`, + action: "chat.discord_command_recorded", + entityType: "chat_endpoint", + entityId: scope.endpointId, + details: { + command: invocation.command, + interactionId: invocation.interactionId, + conversationId: target?.conversationId ?? null, + sessionGeneration: target?.sessionGeneration ?? null, + publicationId, + }, + }); + // No network work is performed under these locks. A stop while any DB + // await was in flight rolls back the command and its staged outbox row. + if ( + !discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext) || + lease.expiresAt.getTime() <= Date.now() + ) + throw authorityChanged; + publish = publicationId !== null; + return { kind: "accepted", content }; + }, + ); + if (publish) + scheduleMessageProcessing(async () => { + await processPendingPublications(); + }); + return response; + } catch (error) { + if (error === authorityChanged) return denied; + throw error; + } + } + return { reconcileDiscordCommands, handleDiscordNativeCommand }; +} diff --git a/server/src/services/chat-channels/github/webhook-ingress.ts b/server/src/services/chat-channels/github/webhook-ingress.ts new file mode 100644 index 00000000000..b877445881c --- /dev/null +++ b/server/src/services/chat-channels/github/webhook-ingress.ts @@ -0,0 +1,1579 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence } from "../runtime-types.js"; + +import { githubRepositoryInventoryItemFromPayload } from "./resources.js"; + +import { dueChatAction, PROVIDER_EFFECT_STALE_MS } from "../../chat-action-work.js"; + +import { notifyChatDeliveryWork, notifyChatActionWork } from "../../chat-work-notifications.js"; + +import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; + +import { + githubBodyMentionsBot, +} from "../../chat-github-events.js"; + +import { githubAppJwt } from "../../chat-github-client.js"; + +import { + createHash, + createHmac, + timingSafeEqual, +} from "node:crypto"; +import { + and, + asc, + eq, + isNull, + lte, + or, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatDeliveries, + chatEndpointResources, + chatEndpoints, + chatSdkState, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatEndpointSetupState, + ToolCredentialSecretRef, +} from "@paperclipai/shared"; + +import { + conflict, +} from "../../../errors.js"; +import { logger } from "../../../middleware/logger.js"; + +import { + type ChatSdkProvider, +} from "../../chat-sdk-runtime.js"; + +import type { + ChatSdkStatePersistence, +} from "../../chat-sdk-state.js"; + +import { + listGitHubAppWebhookDeliveries, + getGitHubAppWebhookDelivery, + readGitHubAppWebhookConfig, + requestGitHubAppWebhookRedelivery, + getGitHubRecoveryComment, + type GitHubAppWebhookDelivery, + type GitHubAppWebhookDeliveryDetail, +} from "../../chat-github-webhook-config.js"; + +import type { EndpointRecord } from "../runtime-types.js"; + + export type GitHubIngressStageResult = + | { kind: "invalid_signature" } + | { kind: "ignored" } + | { actionId: string; kind: "staged" }; + +export const GITHUB_RECOVERY_STATE_KEY = "paperclip:github-webhook-recovery:v1"; +export const GITHUB_RECOVERY_INTERVAL_MS = 60_000; +export const GITHUB_RECOVERY_WINDOW_MS = 60 * 60_000; +export const GITHUB_RECOVERY_SCAN_LEASE_MS = 5 * 60_000; +export type GitHubRecoveryWindow = LifecycleRuntimeFence & { + appId: string; + webhookUrl: string; + floor: string; + nextScanAt: string; +}; +export type GitHubRecoveryReceipt = GitHubRecoveryWindow & { + original: GitHubAppWebhookDeliveryDetail; +}; + +export function githubRecoveryWindow(value: unknown): GitHubRecoveryWindow | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const item = value as GitHubRecoveryWindow; + return Number.isSafeInteger(item.generation) && + item.generation >= 0 && + typeof item.credentialFingerprint === "string" && + /^[a-f0-9]{64}$/.test(item.credentialFingerprint) && + typeof item.appId === "string" && + /^\d+$/.test(item.appId) && + typeof item.webhookUrl === "string" && + typeof item.floor === "string" && + Number.isFinite(Date.parse(item.floor)) && + typeof item.nextScanAt === "string" && + Number.isFinite(Date.parse(item.nextScanAt)) + ? item + : null; +} + +export type GitHubWebhookIngressPayload = { + version: 1; + body: string; + bodySha256: string; + contentType: string; + deliveryId: string; + eventType: string; + runtimeGeneration: number; + credentialFingerprint: string; +}; + +export type RedactedGitHubWebhookIngressPayload = Omit< + GitHubWebhookIngressPayload, + "body" | "contentType" +> & { + redacted: true; +}; + +export function githubWebhookIngressPayload( + payload: Record, +): GitHubWebhookIngressPayload | null { + if ( + payload.version !== 1 || + typeof payload.body !== "string" || + typeof payload.bodySha256 !== "string" || + !/^[a-f0-9]{64}$/.test(payload.bodySha256) || + typeof payload.contentType !== "string" || + typeof payload.deliveryId !== "string" || + !payload.deliveryId || + typeof payload.eventType !== "string" || + !payload.eventType || + typeof payload.runtimeGeneration !== "number" || + !Number.isSafeInteger(payload.runtimeGeneration) || + typeof payload.credentialFingerprint !== "string" + ) { + return null; + } + if ( + createHash("sha256").update(payload.body).digest("hex") !== + payload.bodySha256 + ) { + return null; + } + return payload as GitHubWebhookIngressPayload; +} + +export function redactedGitHubWebhookIngressPayload( + payload: GitHubWebhookIngressPayload, +): RedactedGitHubWebhookIngressPayload { + return { + version: 1, + bodySha256: payload.bodySha256, + deliveryId: payload.deliveryId, + eventType: payload.eventType, + runtimeGeneration: payload.runtimeGeneration, + credentialFingerprint: payload.credentialFingerprint, + redacted: true, + }; +} +type GitHubWebhookIngressOptions = { + endpointRecord: (endpointId: string) => Promise; + runtimeContextForRecord: (record: EndpointRecord) => RuntimeContext; + resolveCredentialRefs: (endpoint: EndpointRow, refs: ToolCredentialSecretRef[], database?: DbOrTransaction) => Promise>; + runtimeGeneration: (setup: ChatEndpointSetupState) => number; + credentialFingerprint: (refs: ToolCredentialSecretRef[]) => string; + getWebhookPublicBaseUrl: () => string | null; + redactError: (error: unknown) => string; + handleWebhook: (publicId: string, provider: ChatSdkProvider, request: Request, context?: { githubIngressActionId: string; githubIngressFence: LifecycleRuntimeFence; superseded?: boolean }) => Promise; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + fetchImpl: typeof fetch; + persistence: ChatSdkStatePersistence; + withCredentialMutationLease: ReturnType; + isShuttingDown: () => boolean; + githubWebhookReplayBarrier?: () => Promise; + supportedWebhookEvents: ReadonlySet; +}; + +export function createGitHubWebhookIngress(db: Db, options: GitHubWebhookIngressOptions) { + const { endpointRecord, runtimeContextForRecord, resolveCredentialRefs, runtimeGeneration, credentialFingerprint, getWebhookPublicBaseUrl, redactError, handleWebhook, runtimeCallbackEndpoint, fetchImpl, persistence, withCredentialMutationLease, isShuttingDown, supportedWebhookEvents: SUPPORTED_GITHUB_WEBHOOK_EVENTS } = options; + + + function githubSignatureIsValid( + body: string, + signature: string | null, + webhookSecret: string | undefined, + ): boolean { + if (!signature || !webhookSecret) return false; + const expected = `sha256=${createHmac("sha256", webhookSecret).update(body).digest("hex")}`; + try { + return timingSafeEqual(Buffer.from(signature), Buffer.from(expected)); + } catch { + return false; + } + } + + function githubWebhookContainsUserContent(eventType: string): boolean { + return ( + eventType === "issue_comment" || + eventType === "issues" || + eventType === "pull_request" || + eventType === "pull_request_review" || + eventType === "pull_request_review_comment" + ); + } + + function githubWebhookAddressesBot( + payload: unknown, + botUsername: string | null, + ): boolean { + if (!botUsername || !payload || typeof payload !== "object") return false; + const comment = (payload as { comment?: unknown }).comment; + if (!comment || typeof comment !== "object") return false; + const body = (comment as { body?: unknown }).body; + if (typeof body !== "string") return false; + return githubBodyMentionsBot(body, botUsername); + } + + function githubIngressAllowedInEndpointState( + status: EndpointRow["status"], + eventType: string, + ): boolean { + if (status === "verifying" || status === "active") return true; + if (eventType === "ping" && (status === "draft" || status === "attention")) + return true; + if ( + status === "attention" && + [ + "github_app_authorization", + "installation", + "installation_repositories", + ].includes(eventType) + ) + return true; + return status === "revoked" && eventType === "installation"; + } + + async function stageGitHubWebhookIngress( + endpoint: EndpointRow, + request: Request, + ): Promise { + const body = await request.text(); + const signature = request.headers.get("x-hub-signature-256"); + const eventType = request.headers.get("x-github-event")?.trim() ?? ""; + const providerDeliveryId = + request.headers.get("x-github-delivery")?.trim() || + `body:${createHash("sha256").update(body).digest("hex")}`; + if (!SUPPORTED_GITHUB_WEBHOOK_EVENTS.has(eventType)) + return { kind: "ignored" }; + + let parsedPayload: unknown = null; + try { + parsedPayload = JSON.parse(body) as unknown; + } catch { + // A valid signature is still required before a malformed body may be + // retained. The native adapter will settle the authenticated envelope. + } + + // Resolve from a captured ref set, then prove that exact ref set and + // generation are still authoritative in the insertion transaction. This + // keeps webhook authentication outside the potentially long credential + // mutation lease without letting an obsolete secret stage new work. + for (let attempt = 0; attempt < 3; attempt += 1) { + const record = await endpointRecord(endpoint.id); + if (!record || record.endpoint.provider !== "github") + return { kind: "ignored" }; + if ( + !githubIngressAllowedInEndpointState(record.endpoint.status, eventType) + ) { + return { kind: "ignored" }; + } + const context = runtimeContextForRecord(record); + let credentials: Record; + try { + credentials = await resolveCredentialRefs( + record.endpoint, + record.credentialSecretRefs, + ); + } catch (error) { + const latest = await endpointRecord(endpoint.id); + if ( + latest && + (runtimeGeneration(latest.endpoint.setup) !== context.generation || + credentialFingerprint(latest.credentialSecretRefs) !== + context.credentialFingerprint) + ) { + continue; + } + throw error; + } + if (!githubSignatureIsValid(body, signature, credentials.webhookSecret)) { + const latest = await endpointRecord(endpoint.id); + if ( + latest && + (runtimeGeneration(latest.endpoint.setup) !== context.generation || + credentialFingerprint(latest.credentialSecretRefs) !== + context.credentialFingerprint) + ) { + continue; + } + return { kind: "invalid_signature" }; + } + + const incomingInstallationId = + parsedPayload && + typeof parsedPayload === "object" && + !Array.isArray(parsedPayload) + ? (parsedPayload as { installation?: { id?: unknown } }).installation + ?.id + : undefined; + if ( + incomingInstallationId !== undefined && + String(incomingInstallationId) !== credentials.installationId && + eventType !== "installation" + ) { + return { kind: "ignored" }; + } + + const bodySha256 = createHash("sha256").update(body).digest("hex"); + const staged = await db.transaction(async (tx) => { + const currentEndpoint = await tx + .select() + .from(chatEndpoints) + .where(eq(chatEndpoints.id, endpoint.id)) + .for("update") + .then((rows) => rows[0] ?? null); + const currentConnection = await tx + .select({ refs: toolConnections.credentialSecretRefs }) + .from(toolConnections) + .where( + and( + eq(toolConnections.companyId, record.endpoint.companyId), + eq(toolConnections.id, record.endpoint.connectionId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if (!currentEndpoint || !currentConnection) return "retry" as const; + if ( + runtimeGeneration(currentEndpoint.setup) !== context.generation || + credentialFingerprint(currentConnection.refs) !== + context.credentialFingerprint + ) { + return "retry" as const; + } + if ( + !githubIngressAllowedInEndpointState( + currentEndpoint.status, + eventType, + ) + ) { + return "ignored" as const; + } + + // Recovery history is denial-only provenance. A genuine callback + // queued by GitHub before pause/reconnect must not be authenticated + // into a later generation, even if the provider signs with its current + // secret. Include archived endpoints so reusing an App cannot move a + // delayed callback into another endpoint/company. + const recoveryReceipts = credentials.appId + ? await tx + .select({ + endpointId: chatActions.endpointId, + payload: chatActions.payload, + status: chatActions.status, + }) + .from(chatActions) + .innerJoin( + chatEndpoints, + eq(chatEndpoints.id, chatActions.endpointId), + ) + .where( + and( + eq(chatEndpoints.provider, "github"), + eq(chatEndpoints.botExternalId, credentials.appId), + eq(chatActions.kind, "github_webhook_recovery"), + eq( + chatActions.providerActionId, + `github_webhook_recovery:${providerDeliveryId}`, + ), + ), + ) + : []; + for (const receipt of recoveryReceipts) { + const fence = githubRecoveryWindow(receipt.payload); + const original = (receipt.payload as GitHubRecoveryReceipt).original; + const incoming = parsedPayload as { + action?: unknown; + installation?: { id?: unknown }; + repository?: { id?: unknown; full_name?: unknown }; + issue?: { id?: unknown; number?: unknown }; + pull_request?: { id?: unknown; number?: unknown }; + sender?: { id?: unknown }; + comment?: { + id?: unknown; + body?: unknown; + user?: { id?: unknown }; + created_at?: unknown; + updated_at?: unknown; + }; + } | null; + const comment = incoming?.comment; + if ( + !fence || + receipt.status === "cancelled" || + receipt.endpointId !== currentEndpoint.id || + fence.generation !== context.generation || + fence.credentialFingerprint !== context.credentialFingerprint || + fence.webhookUrl !== + `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${currentEndpoint.publicId}/github` || + !original?.payload?.comment || + original.event !== eventType || + incoming?.action !== "created" || + String(incoming?.installation?.id) !== + original.payload.installationId || + String(incoming?.repository?.id) !== + original.payload.repositoryId || + incoming?.repository?.full_name !== + original.payload.repositoryFullName || + String(incoming?.sender?.id) !== original.payload.senderId || + (eventType === "issue_comment" + ? String(incoming?.issue?.id) !== original.payload.issueId || + String(incoming?.issue?.number) !== original.payload.issueNumber + : String(incoming?.pull_request?.id) !== + original.payload.pullRequestId || + String(incoming?.pull_request?.number) !== + original.payload.pullRequestNumber) || + String(comment?.id) !== original.payload.comment.id || + String(comment?.user?.id) !== original.payload.comment.userId || + comment?.created_at !== original.payload.comment.createdAt || + comment?.updated_at !== original.payload.comment.updatedAt || + typeof comment?.body !== "string" || + createHash("sha256").update(comment.body).digest("hex") !== + original.payload.comment.bodySha256 + ) { + return "ignored" as const; + } + } + + // Signed App webhooks can cover every installed repository. Do not + // retain comment bodies for destinations the operator did not enable. + if (githubWebhookContainsUserContent(eventType)) { + const item = githubRepositoryInventoryItemFromPayload(parsedPayload); + if (!item) return "ignored" as const; + const providerRepositoryId = + typeof item.metadata?.providerRepositoryId === "string" + ? item.metadata.providerRepositoryId + : null; + const matchingResource = await tx + .select({ + enabled: chatEndpointResources.enabled, + id: chatEndpointResources.id, + }) + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.companyId, currentEndpoint.companyId), + eq(chatEndpointResources.endpointId, currentEndpoint.id), + eq(chatEndpointResources.type, "repository"), + eq(chatEndpointResources.availability, "available"), + or( + eq( + chatEndpointResources.providerResourceId, + item.providerResourceId, + ), + providerRepositoryId + ? sql`${chatEndpointResources.metadata}->>'providerRepositoryId' = ${providerRepositoryId}` + : undefined, + ), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!matchingResource) return "ignored" as const; + if (!matchingResource.enabled) { + const action = + parsedPayload && + typeof parsedPayload === "object" && + !Array.isArray(parsedPayload) + ? (parsedPayload as { action?: unknown }).action + : null; + const ignoreDisabledResource = async () => { + // Record only an authenticated, content-free admission decision. + // Do not make operators infer a working callback from silence, + // or retain the comment/actor/thread from a disabled repository. + const providerEventId = `github:filtered_ingress:${createHash( + "sha256", + ) + .update(providerDeliveryId) + .digest("hex")}`; + const eventKind = + action === "edited" + ? "message_updated" + : action === "deleted" + ? "message_deleted" + : "message"; + await notifyChatDeliveryWork(tx); + await tx + .insert(chatDeliveries) + .values({ + companyId: currentEndpoint.companyId, + endpointId: currentEndpoint.id, + providerEventId, + deduplicationKey: providerEventId, + eventKind, + state: "filtered", + processedAt: new Date(), + redactedError: "Destination is not enabled in Paperclip", + normalizedEvent: { + providerEventId, + kind: eventKind, + filtering: { + contentRetained: false, + reason: "destination_not_enabled", + resourceId: matchingResource.id, + }, + }, + }) + .onConflictDoNothing(); + return "ignored" as const; + }; + if ( + currentEndpoint.status !== "verifying" || + action !== "created" || + !githubWebhookAddressesBot( + parsedPayload, + currentEndpoint.botUsername, + ) + ) { + return ignoreDisabledResource(); + } + const enabledResourceCount = await tx + .select({ count: sql`count(*)::int` }) + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.endpointId, currentEndpoint.id), + eq(chatEndpointResources.type, "repository"), + eq(chatEndpointResources.enabled, true), + eq(chatEndpointResources.availability, "available"), + ), + ) + .then((rows) => rows[0]?.count ?? 0); + if (enabledResourceCount !== 0) return ignoreDisabledResource(); + } + } + + const payload: GitHubWebhookIngressPayload = { + version: 1, + body, + bodySha256, + contentType: + request.headers.get("content-type") ?? "application/json", + deliveryId: providerDeliveryId, + eventType, + runtimeGeneration: context.generation, + credentialFingerprint: context.credentialFingerprint, + }; + await notifyChatActionWork(tx, "github_webhook_ingress"); + const inserted = await tx + .insert(chatActions) + .values({ + companyId: currentEndpoint.companyId, + endpointId: currentEndpoint.id, + kind: "github_webhook_ingress", + providerActionId: `github_webhook_ingress:${providerDeliveryId}`, + payload, + status: "received", + }) + .onConflictDoNothing() + .returning({ id: chatActions.id }); + if (inserted[0]?.id) { + return { actionId: inserted[0].id, kind: "staged" } as const; + } + + // An automatic callback is never authorization to reset the local + // worker's terminal retry budget. The original may have arrived in the + // interval between our remote scan and GitHub's asynchronous callback. + if (recoveryReceipts.length > 0) return "ignored" as const; + + // GitHub preserves X-GitHub-Delivery when an operator uses Recent + // Deliveries to retry a failed callback. Re-arm only the exact same, + // currently authenticated body after a terminal processing failure. + // Successful, cancelled, retryable, and in-flight receipts remain + // immutable deduplication fences. + await notifyChatActionWork(tx, "github_webhook_ingress"); + const [rearmed] = await tx + .update(chatActions) + .set({ + payload, + // Keep the lifetime attempt counter monotonic. Besides bounding + // repeated manual redeliveries, this prevents a late settlement + // from an older processing claim from matching a fresh claim + // after re-arm (an attempts=1 ABA). + result: sql`jsonb_build_object( + 'attempts', + coalesce((${chatActions.result}->>'attempts')::int, 0) + )`, + status: "received", + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.endpointId, currentEndpoint.id), + eq( + chatActions.providerActionId, + `github_webhook_ingress:${providerDeliveryId}`, + ), + eq(chatActions.kind, "github_webhook_ingress"), + eq(chatActions.status, "failed"), + sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'false'`, + sql`${chatActions.payload}->>'bodySha256' = ${bodySha256}`, + sql`${chatActions.payload}->>'eventType' = ${eventType}`, + ), + ) + .returning({ id: chatActions.id }); + const actionId = + rearmed?.id ?? + (await tx + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, currentEndpoint.id), + eq( + chatActions.providerActionId, + `github_webhook_ingress:${providerDeliveryId}`, + ), + ), + ) + .then((rows) => rows[0]?.id ?? null)); + return actionId + ? ({ actionId, kind: "staged" } as const) + : ("retry" as const); + }); + if (staged === "retry") continue; + if (staged === "ignored") return { kind: "ignored" }; + return staged; + } + throw conflict("GitHub credentials changed during webhook authentication", { + code: "chat_endpoint_credentials_changed", + }); + } + + function githubIngressAcceptedResponse() { + return new Response("accepted", { status: 202 }); + } + + async function settleGitHubWebhookIngress( + actionId: string, + input: { + attempts: number; + payload: GitHubWebhookIngressPayload; + response?: Response; + error?: unknown; + cancelledCode?: string; + }, + ): Promise { + const terminalResponse = + input.response !== undefined && input.response.status < 500; + const terminalFailure = input.attempts >= 5; + const cancelled = Boolean(input.cancelledCode); + const processed = Boolean(input.response?.ok); + const terminal = + processed || terminalResponse || terminalFailure || cancelled; + const retryAt = new Date( + Date.now() + + Math.min(60_000, 1_000 * 2 ** Math.max(0, input.attempts - 1)), + ); + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "github_webhook_ingress"); + return tx + .update(chatActions) + .set({ + status: cancelled ? "cancelled" : processed ? "processed" : "failed", + payload: terminal + ? redactedGitHubWebhookIngressPayload(input.payload) + : input.payload, + result: { + attempts: input.attempts, + ...(input.cancelledCode ? { code: input.cancelledCode } : {}), + ...(input.response + ? { httpStatus: input.response.status } + : { error: redactError(input.error) }), + retryable: !terminal, + ...(!terminal ? { retryAt: retryAt.toISOString() } : {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "github_webhook_ingress"), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${input.attempts}`, + ), + ); + }); + } + + async function processGitHubWebhookIngress( + actionId: string, + propagateProcessingFailure = false, + ): Promise { + let action = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "github_webhook_ingress"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!action) return null; + if (["processed", "cancelled"].includes(action.status)) + return githubIngressAcceptedResponse(); + if (action.status === "failed" && action.result?.retryable !== true) + return githubIngressAcceptedResponse(); + const payload = githubWebhookIngressPayload(action.payload); + if (!payload) { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "github_webhook_ingress"); + return tx + .update(chatActions) + .set({ + status: "failed", + payload: {}, + result: { + code: "github_webhook_ingress_payload_invalid", + retryable: false, + }, + updatedAt: new Date(), + }) + .where(eq(chatActions.id, action.id)); + }); + return githubIngressAcceptedResponse(); + } + const now = new Date(); + const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); + if (action.status === "processing" && action.updatedAt > staleBefore) { + return null; + } + if (action.status === "failed") { + const retryAt = + typeof action.result?.retryAt === "string" + ? new Date(action.result.retryAt) + : null; + if (retryAt && retryAt > now) return null; + } else if (action.status !== "received" && action.status !== "processing") { + return null; + } + const attempts = + (typeof action.result?.attempts === "number" + ? action.result.attempts + : 0) + 1; + const previousAttempts = attempts - 1; + const claimGuard = + action.status === "processing" + ? and( + eq(chatActions.status, "processing"), + lte(chatActions.updatedAt, staleBefore), + sql`coalesce((${chatActions.result}->>'attempts')::int, 0) = ${previousAttempts}`, + ) + : action.status === "failed" + ? and( + eq(chatActions.status, "failed"), + sql`coalesce((${chatActions.result}->>'attempts')::int, 0) = ${previousAttempts}`, + ) + : eq(chatActions.status, "received"); + const [claimed] = await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "github_webhook_ingress"); + return tx + .update(chatActions) + .set({ + status: "processing", + result: { attempts }, + updatedAt: now, + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.kind, "github_webhook_ingress"), + claimGuard, + ), + ) + .returning(); + }); + if (!claimed) return null; + action = claimed; + + const record = await endpointRecord(action.endpointId); + if ( + !record || + record.endpoint.provider !== "github" || + !githubIngressAllowedInEndpointState( + record.endpoint.status, + payload.eventType, + ) || + runtimeGeneration(record.endpoint.setup) !== payload.runtimeGeneration || + credentialFingerprint(record.credentialSecretRefs) !== + payload.credentialFingerprint + ) { + await settleGitHubWebhookIngress(action.id, { + attempts, + payload, + cancelledCode: "github_webhook_ingress_runtime_superseded", + }); + return githubIngressAcceptedResponse(); + } + + try { + const credentials = await resolveCredentialRefs( + record.endpoint, + record.credentialSecretRefs, + ); + const recovery = await db + .select({ payload: chatActions.payload, status: chatActions.status }) + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, record.endpoint.id), + eq(chatActions.kind, "github_webhook_recovery"), + eq( + chatActions.providerActionId, + `github_webhook_recovery:${payload.deliveryId}`, + ), + ), + ) + .then((rows) => rows[0] ?? null); + if ( + recovery && + (recovery.status === "cancelled" || + !(await githubRecoverySourceIsCurrent( + (recovery.payload as GitHubRecoveryReceipt).original, + credentials, + ))) + ) { + await settleGitHubWebhookIngress(action.id, { + attempts, + payload, + cancelledCode: "github_webhook_recovery_source_changed", + }); + return githubIngressAcceptedResponse(); + } + const signature = `sha256=${createHmac( + "sha256", + credentials.webhookSecret, + ) + .update(payload.body) + .digest("hex")}`; + await options.githubWebhookReplayBarrier?.(); + const replayContext = { + githubIngressActionId: action.id, + githubIngressFence: { + generation: payload.runtimeGeneration, + credentialFingerprint: payload.credentialFingerprint, + }, + superseded: false, + }; + const response = await handleWebhook( + record.endpoint.publicId, + "github", + new Request( + `https://paperclip.internal/api/chat-webhooks/${record.endpoint.publicId}/github`, + { + method: "POST", + headers: { + "content-type": payload.contentType, + "x-github-delivery": payload.deliveryId, + "x-github-event": payload.eventType, + "x-hub-signature-256": signature, + }, + body: payload.body, + }, + ), + replayContext, + ); + if (replayContext.superseded) { + await settleGitHubWebhookIngress(action.id, { + attempts, + payload, + cancelledCode: "github_webhook_ingress_runtime_superseded", + }); + return githubIngressAcceptedResponse(); + } + if (response.status === 401) { + const latest = await endpointRecord(action.endpointId); + if ( + !latest || + runtimeGeneration(latest.endpoint.setup) !== + payload.runtimeGeneration || + credentialFingerprint(latest.credentialSecretRefs) !== + payload.credentialFingerprint + ) { + await settleGitHubWebhookIngress(action.id, { + attempts, + payload, + cancelledCode: "github_webhook_ingress_runtime_superseded", + }); + return githubIngressAcceptedResponse(); + } + } + await settleGitHubWebhookIngress(action.id, { + attempts, + payload, + response, + }); + return response.status >= 500 && !propagateProcessingFailure + ? githubIngressAcceptedResponse() + : response; + } catch (error) { + await settleGitHubWebhookIngress(action.id, { + attempts, + payload, + error, + }); + logger.warn( + { + actionId: action.id, + endpointId: action.endpointId, + error: redactError(error), + }, + "durable GitHub webhook processing deferred", + ); + if (propagateProcessingFailure) throw error; + return githubIngressAcceptedResponse(); + } + } + + async function githubRecoveryResource( + database: DbOrTransaction, + endpoint: EndpointRow, + repositoryId: string, + repositoryFullName?: string | null, + ) { + return database + .select({ id: chatEndpointResources.id }) + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.companyId, endpoint.companyId), + eq(chatEndpointResources.endpointId, endpoint.id), + eq(chatEndpointResources.type, "repository"), + eq(chatEndpointResources.enabled, true), + eq(chatEndpointResources.availability, "available"), + sql`${chatEndpointResources.metadata}->>'providerRepositoryId' = ${repositoryId}`, + repositoryFullName + ? eq( + chatEndpointResources.providerResourceId, + repositoryFullName.toLowerCase(), + ) + : undefined, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + async function githubRecoveryAlreadyReceived( + database: DbOrTransaction, + endpoint: EndpointRow, + guid: string, + ) { + const action = await database + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.companyId, endpoint.companyId), + eq(chatActions.endpointId, endpoint.id), + eq(chatActions.providerActionId, `github_webhook_ingress:${guid}`), + ), + ) + .limit(1) + .then((rows) => rows[0]); + if (action) return true; + // Disabled destinations intentionally retain no ingress body. Their + // content-free filtered receipt must also remain an admission fence. + return database + .select({ id: chatDeliveries.id }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.companyId, endpoint.companyId), + eq(chatDeliveries.endpointId, endpoint.id), + eq( + chatDeliveries.providerEventId, + `github:filtered_ingress:${createHash("sha256").update(guid).digest("hex")}`, + ), + ), + ) + .limit(1) + .then((rows) => rows.length > 0); + } + + async function rememberGitHubRecoverySkip( + endpoint: EndpointRow, + context: LifecycleRuntimeFence, + window: GitHubRecoveryWindow, + original: GitHubAppWebhookDelivery, + code: string, + ) { + await db.transaction(async (tx) => { + const current = await runtimeCallbackEndpoint(tx, endpoint.id, context, [ + "active", + ]); + if ( + !current || + !original.repositoryId || + !(await githubRecoveryResource(tx, current, original.repositoryId)) + ) + return; + // Only the deny tombstone is needed, not the rejected detail or source + // contents. Remembering it lets the next bounded scan reach older valid + // messages instead of repeatedly inspecting the same five bot callbacks. + await tx + .insert(chatActions) + .values({ + companyId: current.companyId, + endpointId: current.id, + kind: "github_webhook_recovery", + providerActionId: `github_webhook_recovery:${original.guid}`, + payload: { + ...window, + original: { id: original.id, guid: original.guid }, + }, + status: "cancelled", + result: { attempts: 0, code }, + }) + .onConflictDoUpdate({ + target: [chatActions.endpointId, chatActions.providerActionId], + set: { + status: "cancelled", + updatedAt: new Date(), + result: sql`coalesce(${chatActions.result}, '{}'::jsonb) || jsonb_build_object('code', ${code}::text)`, + }, + setWhere: and( + eq(chatActions.kind, "github_webhook_recovery"), + sql`(${chatActions.payload}->>'generation')::int = ${context.generation}`, + sql`${chatActions.payload}->>'credentialFingerprint' = ${context.credentialFingerprint}`, + ), + }); + }); + } + + async function githubRecoverySourceIsCurrent( + detail: GitHubAppWebhookDeliveryDetail, + credentials: Record, + ): Promise { + const comment = detail?.payload?.comment; + if ( + !comment || + !detail.payload.repositoryFullName || + !["issue_comment", "pull_request_review_comment"].includes(detail.event) + ) + return false; + const current = await getGitHubRecoveryComment({ + fetch: fetchImpl, + appToken: githubAppJwt(credentials.appId, credentials.privateKey), + installationId: credentials.installationId, + repositoryFullName: detail.payload.repositoryFullName, + event: detail.event as "issue_comment" | "pull_request_review_comment", + commentId: comment.id, + }).catch((error) => { + if (error?.statusCode === 404 || error?.statusCode === 410) return null; + throw error; + }); + if (!current) return false; + return ( + current.id === comment.id && + current.userId === comment.userId && + current.userType === "User" && + current.bodySha256 === comment.bodySha256 && + current.createdAt === comment.createdAt && + current.updatedAt === comment.updatedAt && + (detail.event === "issue_comment" + ? current.issueNumber === detail.payload.issueNumber + : current.pullRequestNumber === detail.payload.pullRequestNumber) + ); + } + + /** + * GitHub does not retry callbacks that fail before our durable inbox. Ask + * GitHub to redeliver a bounded recent created-comment envelope, never turn + * delivery-history JSON into a locally signed or directly admitted request. + * A missing/changed epoch starts at now; historical messages are not adopted + * into a new credential/generation merely because the App can still see them. + */ + async function processFailedGitHubWebhookDeliveries( + limit = 5, + onlyEndpointId?: string, + ) { + if (isShuttingDown() || !getWebhookPublicBaseUrl()?.startsWith("https://")) return 0; + const now = new Date(); + const rows = await db + .select({ endpoint: chatEndpoints }) + .from(chatEndpoints) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, chatEndpoints.connectionId), + eq(toolConnections.companyId, chatEndpoints.companyId), + ), + ) + .leftJoin( + chatSdkState, + and( + eq(chatSdkState.endpointId, chatEndpoints.id), + eq(chatSdkState.stateKey, GITHUB_RECOVERY_STATE_KEY), + ), + ) + .where( + and( + eq(chatEndpoints.provider, "github"), + eq(chatEndpoints.status, "active"), + eq(toolConnections.enabled, true), + eq(toolConnections.status, "active"), + onlyEndpointId ? eq(chatEndpoints.id, onlyEndpointId) : undefined, + or( + isNull(chatSdkState.id), + // Backoff belongs to one runtime epoch. Reconnect or callback + // changes must establish a fresh floor immediately, even when + // the superseded epoch was rate limited for several hours. + sql`${chatSdkState.value}->>'generation' is distinct from coalesce(${chatEndpoints.setup}->>'runtimeGeneration', '0')`, + sql`${chatSdkState.value}->>'appId' is distinct from ${chatEndpoints.botExternalId}`, + sql`${chatSdkState.value}->>'webhookUrl' is distinct from (${getWebhookPublicBaseUrl()} || '/api/chat-webhooks/' || ${chatEndpoints.publicId} || '/github')`, + sql`(${chatSdkState.value}->>'nextScanAt')::timestamptz <= ${now.toISOString()}::timestamptz`, + ), + ), + ) + .orderBy(asc(chatSdkState.updatedAt), asc(chatEndpoints.id)) + .limit(Math.min(5, Math.max(1, limit))); + for (const { endpoint } of rows) { + const record = await endpointRecord(endpoint.id); + if (!record || !record.endpoint.botExternalId) continue; + const context = runtimeContextForRecord(record); + const webhookUrl = `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/github`; + const scope = { companyId: endpoint.companyId, endpointId: endpoint.id }; + const stored = await persistence.read(scope, GITHUB_RECOVERY_STATE_KEY); + const previous = githubRecoveryWindow(stored?.value); + const epochMatches = + previous && + previous.generation === context.generation && + previous.credentialFingerprint === context.credentialFingerprint && + previous.webhookUrl === webhookUrl && + previous.appId === record.endpoint.botExternalId; + if (epochMatches && Date.parse(previous.nextScanAt) > now.getTime()) + continue; + const window: GitHubRecoveryWindow = { + generation: context.generation, + credentialFingerprint: context.credentialFingerprint, + appId: record.endpoint.botExternalId, + webhookUrl, + floor: epochMatches ? previous.floor : now.toISOString(), + nextScanAt: new Date( + now.getTime() + GITHUB_RECOVERY_SCAN_LEASE_MS, + ).toISOString(), + }; + if ( + !(await persistence.compareAndSet({ + ...scope, + key: GITHUB_RECOVERY_STATE_KEY, + expectedVersion: stored?.version ?? null, + value: window, + expiresAt: null, + })) + ) + continue; + const claimedVersion = (stored?.version ?? 0) + 1; + let nextDelay = GITHUB_RECOVERY_INTERVAL_MS; + let outcome = "scanned"; + let requested = 0; + let inspected = 0; + try { + if (!epochMatches) { + outcome = "epoch_initialized"; + continue; + } + const credentials = await resolveCredentialRefs( + record.endpoint, + record.credentialSecretRefs, + ); + if (credentials.appId !== window.appId || !credentials.installationId) + continue; + const appToken = githubAppJwt( + credentials.appId, + credentials.privateKey, + ); + const config = await readGitHubAppWebhookConfig({ + fetch: fetchImpl, + appToken, + }); + if ( + config.url !== webhookUrl || + config.contentType !== "json" || + config.insecureSsl !== "0" + ) { + outcome = "callback_mismatch"; + nextDelay = 5 * 60_000; + continue; + } + const attempts: GitHubAppWebhookDelivery[] = []; + const floor = Math.max( + Date.parse(window.floor), + now.getTime() - GITHUB_RECOVERY_WINDOW_MS, + ); + let cursor: string | undefined; + let truncated = false; + let previousTimestamp = Number.POSITIVE_INFINITY; + for (let page = 0; page < 3; page += 1) { + const result = await listGitHubAppWebhookDeliveries({ + fetch: fetchImpl, + appToken, + cursor, + }); + // GitHub's official recovery recipe uses newest-first pagination + // and stops at the age floor. Reject observed ordering drift rather + // than assuming unseen pages cannot contain a successful sibling. + for (const attempt of result.deliveries) { + const timestamp = Date.parse(attempt.deliveredAt); + if (timestamp > previousTimestamp) + throw new Error("GitHub recovery history ordering changed"); + previousTimestamp = timestamp; + if (timestamp >= floor) attempts.push(attempt); + } + if (previousTimestamp < floor) break; + if (!result.nextCursor) break; + if (result.nextCursor === cursor) + throw new Error("GitHub recovery pagination did not advance"); + cursor = result.nextCursor; + truncated = page === 2; + } + // All pages in this time window are needed to rule out a successful + // sibling; do not act on a partial window after hitting the work cap. + if (truncated) { + outcome = "history_limit_reached"; + nextDelay = 5 * 60_000; + continue; + } + const groups = new Map(); + for (const attempt of attempts) { + const group = groups.get(attempt.guid) ?? []; + group.push(attempt); + groups.set(attempt.guid, group); + } + for (const [guid, group] of groups) { + if ( + inspected >= 5 || + isShuttingDown() || + Date.now() >= Date.parse(window.nextScanAt) + ) + break; + if ( + group.some( + (item) => + item.statusCode !== null && + item.statusCode >= 200 && + item.statusCode < 400, + ) + ) + continue; + const originals = group.filter((item) => !item.redelivery); + if (originals.length !== 1) continue; + const original = originals[0]!; + if ( + Date.parse(original.deliveredAt) < floor || + Date.parse(original.deliveredAt) > now.getTime() - 10_000 || + original.statusCode === null || + original.statusCode < 500 || + original.action !== "created" || + !["issue_comment", "pull_request_review_comment"].includes( + original.event, + ) || + original.installationId !== credentials.installationId || + !original.repositoryId || + !(await githubRecoveryResource(db, endpoint, original.repositoryId)) + ) + continue; + const recoveryId = `github_webhook_recovery:${guid}`; + if (await githubRecoveryAlreadyReceived(db, endpoint, guid)) continue; + const receipt = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, endpoint.id), + eq(chatActions.providerActionId, recoveryId), + ), + ) + .then((items) => items[0] ?? null); + const count = + typeof receipt?.result?.attempts === "number" + ? receipt.result.attempts + : 0; + const receiptWindow = githubRecoveryWindow(receipt?.payload); + const latest = [...group].sort( + (left, right) => + Date.parse(right.deliveredAt) - Date.parse(left.deliveredAt), + )[0]!; + if ( + latest.statusCode === null || + latest.statusCode < 500 || + latest.throttledAt !== null || + Date.parse(latest.deliveredAt) > now.getTime() - 10_000 + ) + continue; + if ( + receipt && + (receipt.status === "cancelled" || + !receiptWindow || + receiptWindow.generation !== window.generation || + receiptWindow.credentialFingerprint !== + window.credentialFingerprint || + receiptWindow.webhookUrl !== window.webhookUrl || + count >= 3 || + receipt.result?.latestAttemptId === latest.id || + !latest.redelivery || + latest.statusCode === null || + latest.statusCode < 500 || + Date.parse(latest.deliveredAt) < + Date.parse(String(receipt.result?.requestedAt)) || + Date.parse(String(receipt.result?.retryAt)) > now.getTime()) + ) + continue; + inspected += 1; + const detail = await getGitHubAppWebhookDelivery({ + fetch: fetchImpl, + appToken, + deliveryId: original.id, + }); + const comment = detail.payload.comment; + if ( + detail.id !== original.id || + detail.guid !== guid || + detail.redelivery || + detail.url !== webhookUrl || + detail.event !== original.event || + detail.deliveredAt !== original.deliveredAt || + detail.statusCode !== original.statusCode || + detail.installationId !== original.installationId || + detail.repositoryId !== original.repositoryId || + detail.action !== "created" || + detail.payload.action !== "created" || + detail.payload.installationId !== credentials.installationId || + detail.payload.repositoryId !== original.repositoryId || + !comment || + comment.userType !== "User" || + detail.payload.senderId !== comment.userId || + Date.parse(comment.createdAt) < floor || + comment.createdAt !== comment.updatedAt || + !(await githubRecoveryResource( + db, + endpoint, + original.repositoryId, + detail.payload.repositoryFullName, + )) + ) { + await rememberGitHubRecoverySkip( + endpoint, + context, + window, + original, + "not_eligible_for_recovery", + ); + continue; + } + if (!(await githubRecoverySourceIsCurrent(detail, credentials))) { + await rememberGitHubRecoverySkip( + endpoint, + context, + window, + original, + "source_changed_or_unavailable", + ); + continue; + } + await withCredentialMutationLease(endpoint, async (lease) => { + await lease.assertOwned(); + const currentConfig = await readGitHubAppWebhookConfig({ + fetch: fetchImpl, + appToken, + }); + if ( + currentConfig.url !== webhookUrl || + currentConfig.contentType !== "json" || + currentConfig.insecureSsl !== "0" + ) + return; + const claimed = await db.transaction(async (tx) => { + await lease.assertOwned(tx); + const current = await runtimeCallbackEndpoint( + tx, + endpoint.id, + context, + ["active"], + ); + if ( + !current || + !(await githubRecoveryResource( + tx, + current, + original.repositoryId!, + detail.payload.repositoryFullName, + )) + ) + return null; + const connection = await tx + .select({ id: toolConnections.id }) + .from(toolConnections) + .where( + and( + eq(toolConnections.id, current.connectionId), + eq(toolConnections.companyId, current.companyId), + eq(toolConnections.enabled, true), + eq(toolConnections.status, "active"), + ), + ) + .then((items) => items[0]); + const scan = await tx + .select({ id: chatSdkState.id }) + .from(chatSdkState) + .where( + and( + eq(chatSdkState.endpointId, endpoint.id), + eq(chatSdkState.stateKey, GITHUB_RECOVERY_STATE_KEY), + eq(chatSdkState.version, claimedVersion), + ), + ) + .then((items) => items[0]); + const local = await githubRecoveryAlreadyReceived( + tx, + current, + guid, + ); + if ( + !connection || + !scan || + local || + isShuttingDown() || + Date.now() >= Date.parse(window.nextScanAt) + ) + return null; + const result = { + attempts: count + 1, + latestAttemptId: latest.id, + requestedAt: new Date().toISOString(), + retryAt: new Date( + Date.now() + 60_000 * 5 ** count, + ).toISOString(), + }; + const payload: GitHubRecoveryReceipt = { + ...window, + original: detail, + }; + const changes = receipt + ? await tx + .update(chatActions) + .set({ + status: "processing", + result, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, receipt.id), + eq(chatActions.updatedAt, receipt.updatedAt), + ), + ) + .returning({ id: chatActions.id }) + : await tx + .insert(chatActions) + .values({ + companyId: endpoint.companyId, + endpointId: endpoint.id, + kind: "github_webhook_recovery", + providerActionId: recoveryId, + payload, + status: "processing", + result, + }) + .onConflictDoNothing() + .returning({ id: chatActions.id }); + return changes[0]?.id ?? null; + }); + if (!claimed) return; + requested += 1; + // Commit the immutable denial fence before transport. Even an + // ambiguous POST is not reattempted without a distinct failed + // provider delivery. A stale scanner cannot overwrite a successor. + await lease.assertOwned(); + try { + await requestGitHubAppWebhookRedelivery({ + fetch: fetchImpl, + appToken, + deliveryId: original.id, + }); + await db + .update(chatActions) + .set({ status: "processed", updatedAt: new Date() }) + .where( + and( + eq(chatActions.id, claimed), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${count + 1}`, + ), + ); + } catch (error) { + await db + .update(chatActions) + .set({ status: "delivery_unknown", updatedAt: new Date() }) + .where( + and( + eq(chatActions.id, claimed), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${count + 1}`, + ), + ); + throw error; + } + }); + } + } catch (error) { + outcome = "scan_failed"; + nextDelay = Math.max( + 5 * 60_000, + Math.min( + 24 * 60 * 60_000, + Number((error as { retryAfterMs?: unknown })?.retryAfterMs) || 0, + ), + ); + logger.warn( + { endpointId: endpoint.id, error: redactError(error) }, + "GitHub webhook recovery deferred", + ); + } finally { + await persistence.compareAndSet({ + ...scope, + key: GITHUB_RECOVERY_STATE_KEY, + expectedVersion: claimedVersion, + value: { + ...window, + nextScanAt: new Date(Date.now() + nextDelay).toISOString(), + outcome, + requested, + }, + expiresAt: null, + }); + } + } + return rows.length; + } + + async function processPendingGitHubWebhookIngress( + limit = 25, + onlyActionId?: string, + ) { + const now = new Date(); + const actions = await db + .select({ id: chatActions.id }) + .from(chatActions) + .where(and( + onlyActionId ? eq(chatActions.id, onlyActionId) : undefined, + dueChatAction("github_webhook_ingress", now.getTime()), + )) + .orderBy(asc(chatActions.createdAt)) + .limit(limit); + for (const action of actions) await processGitHubWebhookIngress(action.id); + return actions.length; + } + return { stageGitHubWebhookIngress, githubIngressAcceptedResponse, processGitHubWebhookIngress, processFailedGitHubWebhookDeliveries, processPendingGitHubWebhookIngress }; +} diff --git a/server/src/services/chat-channels/runtime-types.ts b/server/src/services/chat-channels/runtime-types.ts new file mode 100644 index 00000000000..a764a0297eb --- /dev/null +++ b/server/src/services/chat-channels/runtime-types.ts @@ -0,0 +1,38 @@ +import type { Db, chatEndpoints } from "@paperclipai/db"; +import type { ToolCredentialSecretRef } from "@paperclipai/shared"; +import type { ChatEndpointRuntime } from "../voice/voice-runtime.js"; + +export type EndpointRow = typeof chatEndpoints.$inferSelect; + +export type DbTransaction = Parameters[0]>[0]; + +export type DbOrTransaction = Db | DbTransaction; + +export type RuntimeContext = { + credentialFingerprint: string; + discordGatewayEventQueue?: Promise; + discordGatewaySequence?: number; + discordGatewayOwned?: boolean; + /** Cache identity only; durable command admission rechecks ownership. */ + discordCommandId?: string; + endpointRuntime?: ChatEndpointRuntime; + generation: number; + localEpoch: number; + version: string; +}; + +export type LifecycleRuntimeFence = Pick< + RuntimeContext, + "credentialFingerprint" | "generation" +>; + +export type InboundRuntimeContext = LifecycleRuntimeFence & + Pick; + +export type EndpointRecord = { + endpoint: EndpointRow; + assignedAgentName: string; + connectionName: string; + applicationId: string; + credentialSecretRefs: ToolCredentialSecretRef[]; +}; diff --git a/server/src/services/chat-channels/slack/session-stop.ts b/server/src/services/chat-channels/slack/session-stop.ts new file mode 100644 index 00000000000..bb2e071e7c1 --- /dev/null +++ b/server/src/services/chat-channels/slack/session-stop.ts @@ -0,0 +1,858 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence } from "../runtime-types.js"; + +import { dueChatAction, PROVIDER_EFFECT_STALE_MS } from "../../chat-action-work.js"; + +import { notifyChatActionWork } from "../../chat-work-notifications.js"; + +import { + and, + asc, + desc, + eq, + inArray, + isNull, + lte, + or, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + agentWakeupRequests, + agents, + chatActions, + chatConversations, + chatEndpointResources, + chatEndpoints, + chatExternalPrincipals, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import type { + SafeChatPublicationPayload, +} from "@paperclipai/shared"; +import { + isUuidLike, +} from "@paperclipai/shared"; + +import { logger } from "../../../middleware/logger.js"; + +import { + logActivity, +} from "../../activity-log.js"; + +import { projectSafeChatPublication } from "../../chat-publication-projection.js"; + +import { + type SlackSessionStop, +} from "../../chat-slack-sessions.js"; + +import { + enqueueChatRunMilestones, + safeMilestoneText, +} from "../../chat-run-publications.js"; + +export type SlackSessionStopTarget = + { id: string; kind: "run" } | { id: string; kind: "wakeup" }; + +export type SlackSessionStopPayload = { + version: 1; + assignedAgentId: string; + eventTimestamp: string; + issueId: string | null; + occurredAt: string; + sessionGeneration: number | null; + target: SlackSessionStopTarget | null; + threadId: string; + userId: string; +}; + +export function slackSessionStopPayload( + value: Record, +): SlackSessionStopPayload | null { + const target = value.target; + const parsedTarget = + target === null + ? null + : target && + typeof target === "object" && + !Array.isArray(target) && + ((target as { kind?: unknown }).kind === "run" || + (target as { kind?: unknown }).kind === "wakeup") && + typeof (target as { id?: unknown }).id === "string" && + isUuidLike((target as { id: string }).id) + ? (target as SlackSessionStopTarget) + : undefined; + if ( + value.version !== 1 || + typeof value.assignedAgentId !== "string" || + !isUuidLike(value.assignedAgentId) || + typeof value.eventTimestamp !== "string" || + !value.eventTimestamp || + (value.issueId !== null && + (typeof value.issueId !== "string" || !isUuidLike(value.issueId))) || + typeof value.occurredAt !== "string" || + !Number.isFinite(new Date(value.occurredAt).getTime()) || + (value.sessionGeneration !== null && + (typeof value.sessionGeneration !== "number" || + !Number.isSafeInteger(value.sessionGeneration) || + value.sessionGeneration < 1)) || + (parsedTarget !== null && + (value.issueId === null || value.sessionGeneration === null)) || + parsedTarget === undefined || + typeof value.threadId !== "string" || + !value.threadId.startsWith("slack:") || + typeof value.userId !== "string" || + !value.userId + ) { + return null; + } + return { + version: 1, + assignedAgentId: value.assignedAgentId, + eventTimestamp: value.eventTimestamp, + issueId: value.issueId, + occurredAt: value.occurredAt, + sessionGeneration: value.sessionGeneration, + target: parsedTarget, + threadId: value.threadId, + userId: value.userId, + }; +} +type SlackSessionStopOptions = { + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; userId: string | null }>; + nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean; + stageAuthorizedTaskControlPublication: (tx: DbOrTransaction, input: { companyId: string; conversationId: string; endpointId: string; idempotencyKey: string; issueId: string; payload: SafeChatPublicationPayload; principalId: string }) => Promise<{ id: string }>; + getPublicBaseUrl: () => string | null; + heartbeat: { cancelRun?: (runId: string, reason?: string, options?: { errorCode?: string; eventMessage?: string; eventPayload?: Record; resultJson?: Record }) => Promise }; + redactError: (error: unknown) => string; +}; + +export function createSlackSessionStop(db: Db, options: SlackSessionStopOptions) { + const { runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, nonDirectDestinationAllowed, stageAuthorizedTaskControlPublication, getPublicBaseUrl, redactError } = options; + + + async function stageSlackSessionStop( + endpoint: EndpointRow, + event: SlackSessionStop, + runtimeContext: RuntimeContext, + ): Promise { + return db.transaction(async (tx) => { + const currentEndpoint = await runtimeCallbackEndpoint( + tx, + endpoint.id, + runtimeContext, + ["active"], + ); + if (!currentEndpoint || currentEndpoint.provider !== "slack") return null; + + const conversation = await tx + .select() + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, currentEndpoint.companyId), + eq(chatConversations.endpointId, currentEndpoint.id), + eq(chatConversations.externalThreadId, event.threadId), + inArray(chatConversations.state, ["active", "waiting"]), + ), + ) + .orderBy(desc(chatConversations.sessionGeneration)) + .limit(1) + .for("update") + .then((rows) => rows[0] ?? null); + const principal = await tx + .select() + .from(chatExternalPrincipals) + .where( + and( + eq(chatExternalPrincipals.companyId, currentEndpoint.companyId), + eq(chatExternalPrincipals.provider, "slack"), + eq( + chatExternalPrincipals.providerAccountId, + currentEndpoint.providerAccountId ?? "unknown", + ), + eq(chatExternalPrincipals.externalId, event.userId), + eq(chatExternalPrincipals.kind, "user"), + eq(chatExternalPrincipals.isBot, false), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const issue = conversation + ? await tx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, currentEndpoint.companyId), + eq(issues.id, conversation.issueId), + eq(issues.assigneeAgentId, currentEndpoint.assignedAgentId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + + // Slack carries fractional event timestamps. Treat that exact instant as + // the cancellation boundary so a run admitted even milliseconds after + // Stop was clicked can never be mistaken for the stopped session. + const eventCutoff = event.occurredAt; + const futureEvent = event.occurredAt.getTime() > Date.now() + 60_000; + let target: SlackSessionStopTarget | null = null; + if (issue && !futureEvent && issue.executionRunId) { + const run = await tx + .select({ + agentId: heartbeatRuns.agentId, + companyId: heartbeatRuns.companyId, + contextSnapshot: heartbeatRuns.contextSnapshot, + createdAt: heartbeatRuns.createdAt, + id: heartbeatRuns.id, + startedAt: heartbeatRuns.startedAt, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, issue.executionRunId), + eq(heartbeatRuns.companyId, currentEndpoint.companyId), + eq(heartbeatRuns.agentId, currentEndpoint.assignedAgentId), + inArray(heartbeatRuns.status, [ + "queued", + "running", + "scheduled_retry", + ]), + lte(heartbeatRuns.createdAt, eventCutoff), + or( + isNull(heartbeatRuns.startedAt), + lte(heartbeatRuns.startedAt, eventCutoff), + ), + or( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, + sql`${heartbeatRuns.contextSnapshot}->>'taskId' = ${issue.id}`, + ), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if (run) target = { id: run.id, kind: "run" }; + } else if (issue && !futureEvent && !issue.executionRunId) { + const wakeup = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, currentEndpoint.companyId), + eq(agentWakeupRequests.agentId, currentEndpoint.assignedAgentId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + isNull(agentWakeupRequests.runId), + lte(agentWakeupRequests.requestedAt, eventCutoff), + or( + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->>'taskId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId' = ${issue.id}`, + ), + ), + ) + .orderBy(desc(agentWakeupRequests.requestedAt)) + .limit(1) + .for("update") + .then((rows) => rows[0] ?? null); + if (wakeup) target = { id: wakeup.id, kind: "wakeup" }; + } + + const payload: SlackSessionStopPayload = { + version: 1, + assignedAgentId: currentEndpoint.assignedAgentId, + eventTimestamp: event.eventTimestamp, + issueId: issue?.id ?? null, + occurredAt: event.occurredAt.toISOString(), + sessionGeneration: conversation?.sessionGeneration ?? null, + target, + threadId: event.threadId, + userId: event.userId, + }; + await notifyChatActionWork(tx, "slack_session_stop"); + const [inserted] = await tx + .insert(chatActions) + .values({ + companyId: currentEndpoint.companyId, + endpointId: currentEndpoint.id, + conversationId: conversation?.id ?? null, + principalId: principal?.id ?? null, + kind: "slack_session_stop", + providerActionId: `slack_session_stop:${event.providerEventId}`, + payload, + status: "received", + }) + .onConflictDoNothing() + .returning({ id: chatActions.id }); + return ( + inserted?.id ?? + (await tx + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, currentEndpoint.id), + eq( + chatActions.providerActionId, + `slack_session_stop:${event.providerEventId}`, + ), + eq(chatActions.kind, "slack_session_stop"), + ), + ) + .then((rows) => rows[0]?.id ?? null)) + ); + }); + } + + async function processSlackSessionStop(actionId: string): Promise { + const now = new Date(); + const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); + const action = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "slack_session_stop"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!action || ["processed", "cancelled"].includes(action.status)) return; + if (action.status === "processing" && action.updatedAt > staleBefore) + return; + if (action.status === "failed") { + if (action.result?.retryable !== true) return; + const retryAt = + typeof action.result.retryAt === "string" + ? new Date(action.result.retryAt) + : null; + if (retryAt && retryAt > now) return; + } else if (action.status !== "received" && action.status !== "processing") { + return; + } + const payload = slackSessionStopPayload(action.payload); + const previousAttempts = + typeof action.result?.attempts === "number" && + Number.isSafeInteger(action.result.attempts) + ? Math.max(0, action.result.attempts) + : 0; + const attempts = previousAttempts + 1; + if (!payload) { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slack_session_stop"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + attempts, + code: "slack_session_stop_payload_invalid", + retryable: false, + }, + updatedAt: now, + }) + .where(eq(chatActions.id, action.id)); + }); + return; + } + + type StopClaim = + | { kind: "cancel_run"; runId: string; userId: string } + | { kind: "settled" } + | { kind: "retry" }; + const claim = await db.transaction(async (tx): Promise => { + const current = await tx + .select() + .from(chatActions) + .where(eq(chatActions.id, action.id)) + .for("update") + .then((rows) => rows[0] ?? null); + const currentRetryAt = + typeof current?.result?.retryAt === "string" + ? new Date(current.result.retryAt) + : null; + const currentRetryIsDue = + currentRetryAt === null || + (Number.isFinite(currentRetryAt.getTime()) && currentRetryAt <= now); + const eligible = + current?.status === "received" || + (current?.status === "processing" && + current.updatedAt <= staleBefore && + (current.result?.attempts ?? 0) === previousAttempts) || + (current?.status === "failed" && + current.result?.retryable === true && + currentRetryIsDue && + (current.result?.attempts ?? 0) === previousAttempts); + if (!current || !eligible) return { kind: "settled" }; + + const settle = async ( + code: string, + actorUserId: string | null, + targetKind: "run" | "wakeup" | "none", + ): Promise => { + await notifyChatActionWork(tx, "slack_session_stop"); + await tx + .update(chatActions) + .set({ + status: + code === "slack_session_stop_cancelled" + ? "processed" + : "cancelled", + result: { attempts, code, retryable: false }, + updatedAt: now, + }) + .where(eq(chatActions.id, current.id)); + await logActivity(tx as unknown as Db, { + companyId: current.companyId, + actorType: actorUserId ? "user" : "system", + actorId: actorUserId ?? "slack-session-stop", + action: + code === "slack_session_stop_cancelled" + ? "chat.slack_session_stopped" + : "chat.slack_session_stop_filtered", + entityType: "chat_action", + entityId: current.id, + details: { + endpointId: current.endpointId, + conversationId: current.conversationId, + eventTimestamp: payload.eventTimestamp, + targetKind, + resultCode: code, + }, + }); + return { kind: "settled" }; + }; + + if ( + !payload.target || + !payload.issueId || + !payload.sessionGeneration || + !current.conversationId || + !current.principalId + ) { + return settle("slack_session_stop_no_current_work", null, "none"); + } + const endpoint = await tx + .select() + .from(chatEndpoints) + .where( + and( + eq(chatEndpoints.id, current.endpointId), + eq(chatEndpoints.companyId, current.companyId), + eq(chatEndpoints.provider, "slack"), + eq(chatEndpoints.status, "active"), + eq(chatEndpoints.assignedAgentId, payload.assignedAgentId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const conversation = endpoint + ? await tx + .select() + .from(chatConversations) + .where( + and( + eq(chatConversations.id, current.conversationId), + eq(chatConversations.companyId, current.companyId), + eq(chatConversations.endpointId, current.endpointId), + eq(chatConversations.issueId, payload.issueId), + eq(chatConversations.externalThreadId, payload.threadId), + eq( + chatConversations.sessionGeneration, + payload.sessionGeneration, + ), + inArray(chatConversations.state, ["active", "waiting"]), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const resource = + endpoint && conversation?.resourceId + ? await tx + .select() + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.id, conversation.resourceId), + eq(chatEndpointResources.companyId, current.companyId), + eq(chatEndpointResources.endpointId, current.endpointId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const principal = endpoint + ? await tx + .select() + .from(chatExternalPrincipals) + .where( + and( + eq(chatExternalPrincipals.id, current.principalId), + eq(chatExternalPrincipals.companyId, current.companyId), + eq(chatExternalPrincipals.provider, "slack"), + eq( + chatExternalPrincipals.providerAccountId, + endpoint.providerAccountId ?? "unknown", + ), + eq(chatExternalPrincipals.externalId, payload.userId), + eq(chatExternalPrincipals.kind, "user"), + eq(chatExternalPrincipals.isBot, false), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const authorization = + endpoint && principal + ? await lockCurrentPrincipalAuthorization(tx, endpoint, principal.id) + : null; + const destinationAllowed = + endpoint && conversation + ? conversation.isDirectMessage + ? endpoint.allowDirectMessages + : nonDirectDestinationAllowed(endpoint, resource) + : false; + const issue = + endpoint && conversation + ? await tx + .select() + .from(issues) + .where( + and( + eq(issues.id, payload.issueId), + eq(issues.companyId, current.companyId), + eq(issues.assigneeAgentId, payload.assignedAgentId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + if ( + !endpoint || + !conversation || + !principal || + !authorization?.allowed || + !authorization.userId || + !destinationAllowed || + !issue + ) { + return settle( + "slack_session_stop_no_longer_authorized", + null, + payload.target.kind, + ); + } + + const eventCutoff = new Date(payload.occurredAt); + let runId: string | null = null; + if (payload.target.kind === "run") { + runId = payload.target.id; + } else { + const wakeup = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.id, payload.target.id), + eq(agentWakeupRequests.companyId, current.companyId), + eq(agentWakeupRequests.agentId, payload.assignedAgentId), + lte(agentWakeupRequests.requestedAt, eventCutoff), + or( + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->>'taskId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId' = ${issue.id}`, + ), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if ( + wakeup && + !wakeup.runId && + ["queued", "deferred_issue_execution"].includes(wakeup.status) && + !issue.executionRunId + ) { + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: "Cancelled from the bound Slack agent session", + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, wakeup.id), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + isNull(agentWakeupRequests.runId), + ), + ); + const agentName = await tx + .select({ name: agents.name }) + .from(agents) + .where( + and( + eq(agents.companyId, current.companyId), + eq(agents.id, payload.assignedAgentId), + ), + ) + .then((rows) => rows[0]?.name ?? "Paperclip agent"); + await stageAuthorizedTaskControlPublication(tx, { + companyId: current.companyId, + conversationId: conversation.id, + endpointId: current.endpointId, + idempotencyKey: `control:stop:${current.id}`, + issueId: issue.id, + payload: projectSafeChatPublication({ + classification: "external", + progressState: "failed", + source: "safe_milestone", + text: safeMilestoneText({ + agentName, + errorCode: "slack_session_stopped", + issueId: issue.id, + milestone: "failed", + publicBaseUrl: getPublicBaseUrl(), + }), + }), + principalId: principal.id, + }); + return settle( + "slack_session_stop_cancelled", + authorization.userId, + "wakeup", + ); + } + runId = wakeup?.runId ?? null; + if (!runId && wakeup?.status === "claimed") { + await notifyChatActionWork(tx, "slack_session_stop"); + await tx + .update(chatActions) + .set({ + status: "failed", + result: { + attempts, + code: "slack_session_stop_wakeup_in_flight", + retryable: true, + retryAt: new Date(Date.now() + 1_000).toISOString(), + }, + updatedAt: now, + }) + .where(eq(chatActions.id, current.id)); + return { kind: "retry" }; + } + } + + const run = runId + ? await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.companyId, current.companyId), + eq(heartbeatRuns.agentId, payload.assignedAgentId), + payload.target.kind === "wakeup" + ? eq(heartbeatRuns.wakeupRequestId, payload.target.id) + : and( + lte(heartbeatRuns.createdAt, eventCutoff), + or( + isNull(heartbeatRuns.startedAt), + lte(heartbeatRuns.startedAt, eventCutoff), + ), + ), + or( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, + sql`${heartbeatRuns.contextSnapshot}->>'taskId' = ${issue.id}`, + ), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + if (!run) { + return settle( + "slack_session_stop_target_superseded", + authorization.userId, + payload.target.kind, + ); + } + if (run.status === "cancelled") { + return settle( + "slack_session_stop_cancelled", + authorization.userId, + "run", + ); + } + if ( + issue.executionRunId !== run.id || + !["queued", "running", "scheduled_retry"].includes(run.status) + ) { + return settle( + "slack_session_stop_target_superseded", + authorization.userId, + "run", + ); + } + await notifyChatActionWork(tx, "slack_session_stop"); + await tx + .update(chatActions) + .set({ + status: "processing", + result: { + attempts, + authorizedUserId: authorization.userId, + runId: run.id, + }, + updatedAt: now, + }) + .where(eq(chatActions.id, current.id)); + return { + kind: "cancel_run", + runId: run.id, + userId: authorization.userId, + }; + }); + if (claim.kind !== "cancel_run") return; + + try { + if (!options.heartbeat.cancelRun) { + throw new Error("Heartbeat cancellation is unavailable"); + } + await options.heartbeat.cancelRun( + claim.runId, + "Stopped from the bound Slack agent session", + { + errorCode: "slack_session_stopped", + eventMessage: "run cancelled from Slack", + resultJson: { + cancelledByActorType: "user", + cancelledByUserId: claim.userId, + }, + eventPayload: { + endpointId: action.endpointId, + conversationId: action.conversationId, + provider: "slack", + }, + }, + ); + await enqueueChatRunMilestones(db, { publicBaseUrl: getPublicBaseUrl() }); + const authoritativeRun = await db + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, claim.runId)) + .then((rows) => rows[0] ?? null); + const cancellationWon = authoritativeRun?.status === "cancelled"; + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slack_session_stop"); + const [settled] = await tx + .update(chatActions) + .set({ + status: cancellationWon ? "processed" : "cancelled", + result: { + attempts, + code: cancellationWon + ? "slack_session_stop_cancelled" + : "slack_session_stop_target_superseded", + runId: claim.runId, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempts}`, + sql`${chatActions.result}->>'runId' = ${claim.runId}`, + ), + ) + .returning({ id: chatActions.id }); + if (!settled) return; + await logActivity(tx as unknown as Db, { + companyId: action.companyId, + actorType: "user", + actorId: claim.userId, + action: cancellationWon + ? "chat.slack_session_stopped" + : "chat.slack_session_stop_filtered", + entityType: "chat_action", + entityId: action.id, + details: { + endpointId: action.endpointId, + conversationId: action.conversationId, + eventTimestamp: payload.eventTimestamp, + targetKind: "run", + resultCode: cancellationWon + ? "slack_session_stop_cancelled" + : "slack_session_stop_target_superseded", + }, + }); + }); + } catch (error) { + const retryable = attempts < 5; + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slack_session_stop"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + attempts, + code: "slack_session_stop_cancellation_failed", + retryable, + ...(retryable + ? { + retryAt: new Date( + Date.now() + Math.min(60_000, 1_000 * 2 ** (attempts - 1)), + ).toISOString(), + } + : {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempts}`, + sql`${chatActions.result}->>'runId' = ${claim.runId}`, + ), + ); + }); + logger.warn( + { + actionId: action.id, + endpointId: action.endpointId, + attempts, + error: redactError(error), + }, + "Slack session stop cancellation deferred", + ); + } + } + + async function processPendingSlackSessionStops( + limit = 25, + onlyActionId?: string, + ) { + const actions = await db.select({ id: chatActions.id }).from(chatActions) + .where(and(dueChatAction("slack_session_stop"), onlyActionId ? eq(chatActions.id, onlyActionId) : undefined)) + .orderBy(asc(chatActions.createdAt)).limit(limit); + for (const action of actions) await processSlackSessionStop(action.id); + return actions.length; + } + return { stageSlackSessionStop, processSlackSessionStop, processPendingSlackSessionStops }; +} diff --git a/server/src/services/chat-channels/slack/task-start.ts b/server/src/services/chat-channels/slack/task-start.ts new file mode 100644 index 00000000000..0ccb3793616 --- /dev/null +++ b/server/src/services/chat-channels/slack/task-start.ts @@ -0,0 +1,1075 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence, InboundRuntimeContext } from "../runtime-types.js"; + +import { dueChatAction, SLACK_COMMAND_POST_STALE_MS, SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS, SLACK_COMMAND_ADMISSION_STALE_MS } from "../../chat-action-work.js"; + +import { notifyChatActionWork } from "../../chat-work-notifications.js"; + +import type { ChatEndpointRuntime } from "../../voice/voice-runtime.js"; + +import { chatCredentialMutationLease, type CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js"; + +import { + randomUUID, +} from "node:crypto"; +import { + and, + asc, + desc, + eq, + inArray, + lte, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatConversations, + chatEndpointResources, + chatEndpoints, + chatExternalPrincipals, + chatIdentityLinks, + companyMemberships, + issues, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatEndpointSetupState, + ToolCredentialSecretRef, +} from "@paperclipai/shared"; + +import { logger } from "../../../middleware/logger.js"; +import { redactSensitiveText } from "../../../redaction.js"; + +import { + type ChatSdkMessageCallbackEvent, +} from "../../chat-sdk-runtime.js"; + +import { classifyChatPublicationError } from "../../chat-publication-errors.js"; + +import type { + Author, + Message, + Thread, +} from "chat"; + +import type { EndpointRecord } from "../runtime-types.js"; + +export type SlackSlashTaskRecoveryPayload = { + channelId: string; + command: string; + syntheticMessageId: string; + taskText: string; +}; + +export function slackSlashTaskRecoveryPayload( + payload: Record, +): SlackSlashTaskRecoveryPayload | null { + const channelId = payload.channelId; + const command = payload.command; + const syntheticMessageId = payload.syntheticMessageId; + const taskText = payload.taskText; + if ( + typeof channelId !== "string" || + !channelId.trim() || + typeof command !== "string" || + !command.trim() || + typeof syntheticMessageId !== "string" || + !syntheticMessageId.trim() || + typeof taskText !== "string" || + !taskText.trim() + ) { + return null; + } + return { channelId, command, syntheticMessageId, taskText }; +} + +export function slackTaskStarterThreadId( + channelId: string, + providerMessageId: string, +): string { + return `slack:${channelId.replace(/^slack:/, "")}:${providerMessageId}`; +} + +export function confirmedSlackTaskStartResult( + result: Record | null, +): { + authorizedUserId?: string | null; + providerMessageId: string; + threadId: string; +} | null { + if ( + !result || + typeof result.providerMessageId !== "string" || + !result.providerMessageId || + typeof result.threadId !== "string" || + !result.threadId + ) { + return null; + } + const hasAuthorizedUserId = Object.prototype.hasOwnProperty.call( + result, + "authorizedUserId", + ); + if ( + hasAuthorizedUserId && + result.authorizedUserId !== null && + (typeof result.authorizedUserId !== "string" || !result.authorizedUserId) + ) { + return null; + } + return { + ...(hasAuthorizedUserId + ? { authorizedUserId: result.authorizedUserId as string | null } + : {}), + providerMessageId: result.providerMessageId, + threadId: result.threadId, + }; +} +type ResourceRow = typeof chatEndpointResources.$inferSelect; +type SlackTaskStartOptions = { + endpointRecord: (endpointId: string) => Promise; + runtimeContextForRecord: (record: EndpointRecord) => RuntimeContext; + runtimeGeneration: (setup: ChatEndpointSetupState) => number; + credentialFingerprint: (refs: ToolCredentialSecretRef[]) => string; + invalidateRuntime: (endpointId: string) => Promise; + runtimeFor: (endpoint: EndpointRow) => Promise; + processMessage: (endpoint: EndpointRow, thread: Thread, message: Message, trigger: ChatSdkMessageCallbackEvent["trigger"], ingressOnly?: boolean, recoveredProviderUrl?: string | null, runtimeContext?: InboundRuntimeContext, providerUpdateId?: number, admittedDeliveryId?: string | null, receiptReactionSupported?: boolean, preauthorizedUserId?: string | null, deferDrainUntilFollowup?: boolean, suppressSetupDestinationActivation?: boolean, slackSlashControl?: boolean) => Promise; + redactError: (error: unknown) => string; + withCredentialMutationLease: ReturnType; + sponsorAllowsGuest: (endpoint: EndpointRow) => Promise; + nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: ResourceRow | null | undefined) => boolean; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; userId: string | null }>; + slackTaskAdmissionClaimBarrier?: (input: { actionId: string; claimId: string }) => Promise; + maxErrorText: number; +}; + +export function createSlackTaskStart(db: Db, options: SlackTaskStartOptions) { + const { endpointRecord, runtimeGeneration, credentialFingerprint, invalidateRuntime, runtimeFor, processMessage, runtimeContextForRecord, redactError, withCredentialMutationLease, sponsorAllowsGuest, nonDirectDestinationAllowed, runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, maxErrorText: MAX_ERROR_TEXT } = options; + + + async function latestSlackDmControlThreadId( + endpointId: string, + rawChannelId: string, + ): Promise { + const conversationIds = [rawChannelId, `slack:${rawChannelId}`]; + return db + .select({ externalThreadId: chatConversations.externalThreadId }) + .from(chatConversations) + .innerJoin( + issues, + and( + eq(issues.companyId, chatConversations.companyId), + eq(issues.id, chatConversations.issueId), + ), + ) + .where( + and( + eq(chatConversations.endpointId, endpointId), + inArray(chatConversations.externalConversationId, conversationIds), + eq(chatConversations.isDirectMessage, true), + inArray(chatConversations.state, ["active", "waiting"]), + notInArray(issues.status, ["done", "cancelled"]), + ), + ) + .orderBy( + desc(chatConversations.lastActivityAt), + desc(chatConversations.createdAt), + ) + .limit(1) + .then((rows) => rows[0]?.externalThreadId ?? null); + } + + async function finalizeSlackTaskStartFailure(input: { + actionId: string; + actionStatus: "received" | "validating" | "resolving"; + attempt?: number; + connectionId: string; + credentialLease: CredentialMutationLeaseGuard; + endpoint: EndpointRow; + error: unknown; + providerAttempted?: boolean; + resource: ResourceRow | null; + runtimeFence: LifecycleRuntimeFence; + }): Promise { + const attempt = input.attempt ?? 1; + const classified = classifyChatPublicationError(input.error, attempt); + // Runtime construction, lease acquisition, and authorization validation + // happen before the transport fence. Even a socket-shaped error in that + // phase is known not to have sent a Slack message and remains safely + // retryable instead of entering the ambiguous-delivery quarantine. + const classifiedDisposition = + input.providerAttempted === false && + classified.kind === "delivery_unknown" + ? { + kind: "retry" as const, + retryAfterMs: Math.min(60_000, 2 ** Math.max(0, attempt) * 1000), + reason: classified.reason, + } + : classified; + await input.credentialLease.assertOwned(); + const currentRecord = await endpointRecord(input.endpoint.id); + const runtimeFenceCurrent = + currentRecord !== null && + runtimeGeneration(currentRecord.endpoint.setup) === + input.runtimeFence.generation && + credentialFingerprint(currentRecord.credentialSecretRefs) === + input.runtimeFence.credentialFingerprint; + // A provider rejection from credentials that were superseded while this + // worker lost its lease cannot quarantine the successor endpoint or its + // resources. Preserve genuinely ambiguous provider acceptance, but make a + // definite old-runtime rejection safely retryable so the normal worker can + // re-authorize it against the current endpoint before any new Slack call. + const disposition = + !runtimeFenceCurrent && classifiedDisposition.kind !== "delivery_unknown" + ? { + kind: "retry" as const, + retryAfterMs: 1_000, + reason: "Slack runtime changed before failure settlement", + } + : classifiedDisposition; + const failure = redactSensitiveText(disposition.reason).slice( + 0, + MAX_ERROR_TEXT, + ); + const finalized = await db.transaction(async (tx) => { + await input.credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "slash_task_start"); + const [ownedAction] = await tx + .update(chatActions) + .set({ + status: + disposition.kind === "delivery_unknown" + ? "delivery_unknown" + : disposition.kind === "retry" + ? "queued" + : "failed", + result: { + code: runtimeFenceCurrent + ? `slash_task_${disposition.kind}` + : disposition.kind === "delivery_unknown" + ? "slash_task_delivery_unknown" + : "slash_task_runtime_superseded", + redactedError: failure, + retryable: disposition.kind === "retry", + attemptCount: attempt, + ...(disposition.kind === "retry" + ? { + retryAt: new Date( + Date.now() + disposition.retryAfterMs, + ).toISOString(), + } + : {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, input.actionId), + eq(chatActions.status, input.actionStatus), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!ownedAction) return false; + if (runtimeFenceCurrent && disposition.kind === "endpoint_attention") { + await tx + .update(chatEndpoints) + .set({ + status: "attention", + healthMessage: "Provider credentials or permissions need attention", + lastError: failure, + updatedAt: new Date(), + }) + .where(eq(chatEndpoints.id, input.endpoint.id)); + await tx + .update(toolConnections) + .set({ + status: "disabled", + enabled: false, + healthStatus: "degraded", + healthMessage: "Provider credentials or permissions need attention", + lastError: failure, + healthCheckedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(toolConnections.id, input.connectionId)); + } else if ( + runtimeFenceCurrent && + disposition.kind === "resource_unavailable" && + input.resource + ) { + await tx + .update(chatEndpointResources) + .set({ availability: "unavailable", updatedAt: new Date() }) + .where(eq(chatEndpointResources.id, input.resource.id)); + } + await input.credentialLease.assertOwned(tx); + return true; + }); + if ( + finalized && + runtimeFenceCurrent && + disposition.kind === "endpoint_attention" + ) { + // Runtime removal mutates process-local provider state after the durable + // transaction. Prove this worker still owns the credential fence at the + // last await boundary so a reclaimer cannot have installed a successor + // runtime that this obsolete failure would then remove. + await input.credentialLease.assertOwned(); + await invalidateRuntime(input.endpoint.id).catch(() => undefined); + } + return finalized; + } + + async function admitConfirmedSlackTaskStart( + actionId: string, + ingressOnly = true, + ): Promise { + const admissionClaimId = randomUUID(); + const staleAdmission = new Date( + Date.now() - SLACK_COMMAND_ADMISSION_STALE_MS, + ); + const [action] = await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "admitting", + result: sql`coalesce(${chatActions.result}, '{}'::jsonb) + || jsonb_build_object('admissionClaimId', ${admissionClaimId}::text)`, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "slash_task_start"), + or( + eq(chatActions.status, "provider_confirmed"), + and( + eq(chatActions.status, "admitting"), + lte(chatActions.updatedAt, staleAdmission), + ), + ), + ), + ) + .returning(); + }); + if (!action) return false; + await options.slackTaskAdmissionClaimBarrier?.({ + actionId: action.id, + claimId: admissionClaimId, + }); + + const recovery = slackSlashTaskRecoveryPayload(action.payload); + const confirmed = confirmedSlackTaskStartResult(action.result); + const ownsAdmission = () => + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "admitting"), + sql`${chatActions.result}->>'admissionClaimId' = ${admissionClaimId}`, + ); + const failAdmission = async (code: string, detail: string) => { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + ...(confirmed ?? {}), + code, + redactedError: redactSensitiveText(detail).slice(0, MAX_ERROR_TEXT), + }, + updatedAt: new Date(), + }) + .where(ownsAdmission()); + }); + }; + if (!recovery || !confirmed || !action.principalId) { + await failAdmission( + "slash_task_admission_context_missing", + "Confirmed Slack task start lacks durable admission context", + ); + return true; + } + + const record = await endpointRecord(action.endpointId); + if (!record || record.endpoint.provider !== "slack") { + await failAdmission( + "slash_task_endpoint_missing", + "Slack endpoint is no longer available", + ); + return true; + } + if (!["verifying", "active"].includes(record.endpoint.status)) { + if (record.endpoint.status === "archived") { + await failAdmission( + "slash_task_endpoint_archived", + "Slack endpoint was archived before task admission completed", + ); + } else { + // The provider-visible send is already confirmed. Wait for an + // operator to repair/resume the endpoint, then continue Paperclip-only + // admission without sending another Slack message. + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "provider_confirmed", + result: confirmed, + updatedAt: new Date(), + }) + .where(ownsAdmission()); + }); + } + return true; + } + const principal = await db + .select() + .from(chatExternalPrincipals) + .where( + and( + eq(chatExternalPrincipals.id, action.principalId), + eq(chatExternalPrincipals.companyId, record.endpoint.companyId), + eq(chatExternalPrincipals.provider, "slack"), + eq( + chatExternalPrincipals.providerAccountId, + record.endpoint.providerAccountId ?? "unknown", + ), + ), + ) + .then((rows) => rows[0] ?? null); + if (!principal || principal.isBot || principal.kind !== "user") { + await failAdmission( + "slash_task_principal_missing", + "Original Slack account is no longer available", + ); + return true; + } + + const author = { + userId: principal.externalId, + userName: principal.handle ?? principal.externalId, + fullName: + principal.displayName ?? principal.handle ?? principal.externalId, + isBot: false, + isMe: false, + isSystem: false, + } satisfies Author; + try { + const endpointRuntime = await runtimeFor(record.endpoint); + const thread = endpointRuntime.thread(confirmed.threadId); + await processMessage( + record.endpoint, + thread, + { + id: recovery.syntheticMessageId, + threadId: thread.id, + text: recovery.taskText, + formatted: { type: "root", children: [] }, + raw: { recoveredFromActionId: action.id }, + author, + metadata: { dateSent: action.createdAt, edited: false }, + attachments: [], + links: [], + isMention: true, + } as unknown as Message, + "mention", + ingressOnly, + null, + { ...runtimeContextForRecord(record), endpointRuntime }, + undefined, + null, + false, + undefined, + false, + true, + ); + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ status: "processed", result: confirmed, updatedAt: new Date() }) + .where(ownsAdmission()); + }); + return true; + } catch (error) { + // The provider post is already confirmed, so this retry is strictly a + // Paperclip admission retry. Preserve the root tuple and let the durable + // reconciler resume without ever posting to Slack again. + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "provider_confirmed", + result: { + ...confirmed, + code: "slash_task_admission_retry", + retryAt: new Date(Date.now() + 1_000).toISOString(), + redactedError: redactError(error), + }, + updatedAt: new Date(), + }) + .where(ownsAdmission()); + }); + throw error; + } + } + + async function processSlackTaskStart( + actionId: string, + ingressOnly = true, + ): Promise { + let action = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "slash_task_start"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!action) return false; + + if ( + action.status === "provider_confirmed" || + action.status === "admitting" + ) { + return await admitConfirmedSlackTaskStart(action.id, ingressOnly); + } + if (action.status === "resolving") { + if ( + action.updatedAt.getTime() <= + Date.now() - SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS + ) { + const staleRecord = await endpointRecord(action.endpointId); + if (!staleRecord) return false; + await withCredentialMutationLease( + staleRecord.endpoint, + async (credentialLease) => { + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "slash_task_start"); + await tx + .update(chatActions) + .set({ + status: "delivery_unknown", + result: { + ...(action.result ?? {}), + code: "slash_task_delivery_unknown", + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "resolving"), + eq(chatActions.updatedAt, action.updatedAt), + lte( + chatActions.updatedAt, + new Date( + Date.now() - SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS, + ), + ), + ), + ); + await credentialLease.assertOwned(tx); + }); + }, + ); + } + return false; + } + if (action.status === "validating") { + if ( + action.updatedAt.getTime() <= + Date.now() - SLACK_COMMAND_POST_STALE_MS + ) { + const staleRecord = await endpointRecord(action.endpointId); + if (!staleRecord) return false; + await withCredentialMutationLease( + staleRecord.endpoint, + async (credentialLease) => { + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "slash_task_start"); + await tx + .update(chatActions) + .set({ status: "queued", updatedAt: new Date() }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "validating"), + eq(chatActions.updatedAt, action.updatedAt), + lte( + chatActions.updatedAt, + new Date(Date.now() - SLACK_COMMAND_POST_STALE_MS), + ), + ), + ); + await credentialLease.assertOwned(tx); + }); + }, + ); + } + return false; + } + if (action.status !== "queued") return false; + const retryAt = + typeof action.result?.retryAt === "string" + ? new Date(action.result.retryAt) + : null; + if ( + retryAt && + !Number.isNaN(retryAt.getTime()) && + retryAt.getTime() > Date.now() + ) { + return false; + } + const attempt = + (typeof action.result?.attemptCount === "number" && + Number.isSafeInteger(action.result.attemptCount) + ? Math.max(0, action.result.attemptCount) + : 0) + 1; + + const [claimed] = await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "validating", + result: { attemptCount: attempt }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.kind, "slash_task_start"), + eq(chatActions.status, action.status), + ), + ) + .returning(); + }); + if (!claimed) return false; + action = claimed; + + const recovery = slackSlashTaskRecoveryPayload(action.payload); + const initialRecord = await endpointRecord(action.endpointId); + if ( + !recovery || + !initialRecord || + initialRecord.endpoint.provider !== "slack" + ) { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + code: "slash_task_provider_context_missing", + retryable: false, + attemptCount: attempt, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "validating"), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ); + }); + return true; + } + let providerPostStarted = false; + let transportClaimed = false; + let failureResource: ResourceRow | null = null; + let failureRuntimeFence = runtimeContextForRecord(initialRecord); + let failureSettledUnderLease = false; + let confirmedActionId: string | null = null; + try { + const processed = await withCredentialMutationLease( + initialRecord.endpoint, + async (credentialLease) => { + const record = await endpointRecord(action.endpointId); + if (!record || record.endpoint.provider !== "slack") { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + code: "slash_task_provider_context_missing", + retryable: false, + attemptCount: attempt, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "validating"), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ); + }); + return true; + } + failureRuntimeFence = runtimeContextForRecord(record); + if (!["verifying", "active"].includes(record.endpoint.status)) { + const terminallyArchived = record.endpoint.status === "archived"; + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: terminallyArchived ? "cancelled" : "queued", + result: { + code: terminallyArchived + ? "slash_task_endpoint_archived" + : "slash_task_endpoint_waiting", + attemptCount: attempt - 1, + ...(terminallyArchived + ? {} + : { + retryAt: new Date(Date.now() + 5_000).toISOString(), + }), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "validating"), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ); + }); + return false; + } + + const principal = action.principalId + ? await db + .select() + .from(chatExternalPrincipals) + .where( + and( + eq(chatExternalPrincipals.id, action.principalId), + eq( + chatExternalPrincipals.companyId, + record.endpoint.companyId, + ), + eq(chatExternalPrincipals.provider, "slack"), + eq( + chatExternalPrincipals.providerAccountId, + record.endpoint.providerAccountId ?? "unknown", + ), + ), + ) + .then((rows) => rows[0] ?? null) + : null; + let principalAllowed = false; + if (principal && !principal.isBot && principal.kind === "user") { + const link = await db + .select({ + paperclipUserId: chatIdentityLinks.paperclipUserId, + status: chatIdentityLinks.status, + }) + .from(chatIdentityLinks) + .where( + and( + eq(chatIdentityLinks.endpointId, record.endpoint.id), + eq(chatIdentityLinks.principalId, principal.id), + ), + ) + .then((rows) => rows[0] ?? null); + if (link?.status === "linked" && link.paperclipUserId) { + principalAllowed = await db + .select({ id: companyMemberships.id }) + .from(companyMemberships) + .where( + and( + eq(companyMemberships.companyId, record.endpoint.companyId), + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, link.paperclipUserId), + eq(companyMemberships.status, "active"), + ne(companyMemberships.membershipRole, "viewer"), + ), + ) + .then((rows) => rows.length > 0); + } else if (record.endpoint.allowUnlinkedPeople) { + principalAllowed = await sponsorAllowsGuest(record.endpoint); + } + } + + const rawChannelId = recovery.channelId.replace(/^slack:/, ""); + const isDirectMessage = /^D[A-Z0-9]+$/i.test(rawChannelId); + const resource = await db + .select() + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.endpointId, record.endpoint.id), + eq(chatEndpointResources.providerResourceId, rawChannelId), + ), + ) + .then((rows) => rows[0] ?? null); + failureResource = resource; + const destinationAllowed = isDirectMessage + ? record.endpoint.allowDirectMessages + : nonDirectDestinationAllowed(record.endpoint, resource); + if (!principalAllowed || !destinationAllowed) { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "cancelled", + result: { + code: "slash_task_no_longer_authorized", + attemptCount: attempt, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "validating"), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ); + }); + return true; + } + + const endpointRuntime = await runtimeFor(record.endpoint); + const baseThread = endpointRuntime.thread(`slack:${rawChannelId}:`); + const authorizationSnapshot = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const currentEndpoint = await runtimeCallbackEndpoint( + tx, + record.endpoint.id, + runtimeContextForRecord(record), + ["verifying", "active"], + ); + const currentResource = currentEndpoint + ? await tx + .select() + .from(chatEndpointResources) + .where( + and( + eq( + chatEndpointResources.companyId, + currentEndpoint.companyId, + ), + eq(chatEndpointResources.endpointId, currentEndpoint.id), + eq( + chatEndpointResources.providerResourceId, + rawChannelId, + ), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const currentAuthorization = + currentEndpoint && action!.principalId + ? await lockCurrentPrincipalAuthorization( + tx, + currentEndpoint, + action!.principalId, + ) + : null; + const currentDestinationAllowed = currentEndpoint + ? isDirectMessage + ? currentEndpoint.allowDirectMessages + : nonDirectDestinationAllowed(currentEndpoint, currentResource) + : false; + if ( + !currentEndpoint || + !currentAuthorization?.allowed || + !currentDestinationAllowed + ) { + await notifyChatActionWork(tx, "slash_task_start"); + await tx + .update(chatActions) + .set({ + status: "cancelled", + result: { + code: "slash_task_no_longer_authorized", + attemptCount: attempt, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, "validating"), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ); + return null; + } + + failureResource = currentResource; + await notifyChatActionWork(tx, "slash_task_start"); + const [transportClaim] = await tx + .update(chatActions) + .set({ + status: "resolving", + result: { + attemptCount: attempt, + authorizedUserId: currentAuthorization.userId, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, "validating"), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + return transportClaim + ? { authorizedUserId: currentAuthorization.userId } + : null; + }); + if (!authorizationSnapshot) return false; + + // `resolving` is the durable, single-owner transport claim. The + // authorization snapshot above commits and releases every row lock + // before Slack I/O starts. A crash after Slack accepts the post is + // quarantined as delivery_unknown and is never replayed implicitly. + transportClaimed = true; + providerPostStarted = true; + let starter: Awaited>; + try { + starter = await baseThread.post("Starting a task…"); + } catch (error) { + try { + await finalizeSlackTaskStartFailure({ + actionId: action.id, + actionStatus: "resolving", + attempt, + connectionId: record.endpoint.connectionId, + credentialLease, + endpoint: record.endpoint, + error, + providerAttempted: true, + resource: failureResource, + runtimeFence: failureRuntimeFence, + }); + failureSettledUnderLease = true; + } catch { + // If ownership was reclaimed while Slack was in flight, the + // outer recovery path reacquires the endpoint lease and settles + // the attempt without applying stale health state. + } + throw error; + } + const threadId = slackTaskStarterThreadId(rawChannelId, starter.id); + const [providerConfirmed] = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "slash_task_start"); + return tx + .update(chatActions) + .set({ + status: "provider_confirmed", + result: { + attemptCount: attempt, + authorizedUserId: authorizationSnapshot.authorizedUserId, + threadId, + providerMessageId: starter.id, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "resolving"), + sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`, + ), + ) + .returning(); + }); + if (!providerConfirmed) { + throw new Error( + "Slack task-start ownership changed after provider acceptance", + ); + } + // Provider transport and its durable confirmation remain fenced by + // the credential lease. Paperclip-only task admission happens after + // this callback returns and releases that lease: normal admission + // stages a receipt reaction, whose independently fenced worker must + // acquire the same lease and would otherwise self-deadlock here. + confirmedActionId = providerConfirmed.id; + return true; + }, + ); + if (typeof confirmedActionId === "string") { + await admitConfirmedSlackTaskStart(confirmedActionId, ingressOnly); + } + return processed; + } catch (error) { + if (!failureSettledUnderLease) { + await withCredentialMutationLease( + initialRecord.endpoint, + async (credentialLease) => { + await finalizeSlackTaskStartFailure({ + actionId: action.id, + actionStatus: transportClaimed ? "resolving" : "validating", + attempt, + connectionId: initialRecord.endpoint.connectionId, + credentialLease, + endpoint: initialRecord.endpoint, + error, + providerAttempted: providerPostStarted, + resource: failureResource, + runtimeFence: failureRuntimeFence, + }); + }, + ); + } + throw error; + } + } + + async function processPendingSlackTaskStarts(limit = 25) { + const actions = await db.select({ id: chatActions.id }).from(chatActions) + .leftJoin(chatEndpoints, and(eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.companyId, chatActions.companyId))) + .where(dueChatAction("slash_task_start")) + .orderBy(asc(chatActions.createdAt)).limit(limit); + let processed = 0; + for (let offset = 0; offset < actions.length; offset += 4) { + const results = await Promise.all( + actions.slice(offset, offset + 4).map(async (action) => { + try { + return await processSlackTaskStart(action.id); + } catch (error) { + logger.warn( + { actionId: action.id, error: redactError(error) }, + "Slack task start will retry or await explicit resolution", + ); + return false; + } + }), + ); + processed += results.filter(Boolean).length; + } + return processed; + } + return { latestSlackDmControlThreadId, finalizeSlackTaskStartFailure, admitConfirmedSlackTaskStart, processSlackTaskStart, processPendingSlackTaskStarts }; +} diff --git a/server/src/services/chat-channels/telegram/maintenance.ts b/server/src/services/chat-channels/telegram/maintenance.ts new file mode 100644 index 00000000000..2d468d287f6 --- /dev/null +++ b/server/src/services/chat-channels/telegram/maintenance.ts @@ -0,0 +1,931 @@ +import type { EndpointRow, DbTransaction, LifecycleRuntimeFence } from "../runtime-types.js"; + +import { PROVIDER_EFFECT_STALE_MS } from "../../chat-action-work.js"; + +import type { chatCredentialMutationLease, CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js"; + +import { + createHash, +} from "node:crypto"; +import { + and, + asc, + eq, + gt, + lte, + or, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatEndpoints, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatEndpointSetupState, + ToolCredentialSecretRef, +} from "@paperclipai/shared"; + +import { + telegramStopSubscriptionPlan, + telegramStopSubscriptionConfirmed, +} from "../../chat-telegram-stop-subscription.js"; + +import { + notFound, +} from "../../../errors.js"; +import { logger } from "../../../middleware/logger.js"; + +import { + logActivity, +} from "../../activity-log.js"; + +import { classifyChatPublicationError } from "../../chat-publication-errors.js"; + +import type { EndpointRecord } from "../runtime-types.js"; + +export const TELEGRAM_COMMANDS = [ + { command: "task", description: "Start or continue a Paperclip task" }, + { command: "status", description: "Show the active Paperclip task" }, + { command: "new", description: "Start a new task after the current one" }, + { command: "close", description: "Close the active chat conversation" }, +] as const; + +export type TelegramMaintenancePayload = { + version: 1; + operation: "register_commands" | "remove_endpoint" | "stop_subscription"; + runtimeGeneration: number; + credentialFingerprint: string; + webhookUrlSha256?: string; + botUserId?: string; +}; + +export function telegramMaintenancePayload( + payload: Record, +): TelegramMaintenancePayload | null { + if ( + payload.version !== 1 || + (payload.operation !== "register_commands" && + payload.operation !== "remove_endpoint" && + payload.operation !== "stop_subscription") || + typeof payload.runtimeGeneration !== "number" || + !Number.isSafeInteger(payload.runtimeGeneration) || + typeof payload.credentialFingerprint !== "string" || + (payload.operation === "stop_subscription" && + (typeof payload.webhookUrlSha256 !== "string" || + !/^[a-f0-9]{64}$/.test(payload.webhookUrlSha256) || + typeof payload.botUserId !== "string" || + !/^[1-9][0-9]{0,15}$/.test(payload.botUserId))) + ) { + return null; + } + return payload as TelegramMaintenancePayload; +} +type TelegramMaintenanceOptions = { + endpointRecord: (endpointId: string) => Promise; + runtimeContextForRecord: (record: EndpointRecord) => LifecycleRuntimeFence; + withCredentialMutationLease: ReturnType; + resolveCredentials: (endpoint: EndpointRow) => Promise>; + clearCredentials: (endpoint: EndpointRow, lease: CredentialMutationLeaseGuard) => Promise; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + getWebhookPublicBaseUrl: () => string | null; + fetchImpl: typeof fetch; + runtimeGeneration: (setup: ChatEndpointSetupState) => number; + credentialFingerprint: (refs: ToolCredentialSecretRef[]) => string; + redactError: (error: unknown) => string; + credentialCheckTimeoutMs: number; +}; + +export function createTelegramMaintenance(db: Db, options: TelegramMaintenanceOptions) { + const { endpointRecord, runtimeContextForRecord, withCredentialMutationLease, resolveCredentials, clearCredentials, runtimeCallbackEndpoint, getWebhookPublicBaseUrl, fetchImpl, runtimeGeneration, credentialFingerprint, redactError, credentialCheckTimeoutMs: PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS } = options; + + let telegramSubscriptionScanAfter: string | null = null; + + async function stageTelegramMaintenance( + endpoint: EndpointRow, + operation: TelegramMaintenancePayload["operation"], + ) { + const record = await endpointRecord(endpoint.id); + if (!record) throw notFound("Chat endpoint not found"); + const context = runtimeContextForRecord(record); + const subscriptionScope = + operation === "stop_subscription" + ? telegramStopSubscriptionScope(record.endpoint, context) + : null; + if (operation === "stop_subscription" && !subscriptionScope) return null; + const operationGeneration = subscriptionScope + ? `${context.generation}:${context.credentialFingerprint}:${subscriptionScope.botUserId}:${subscriptionScope.webhookUrlSha256}` + : operation === "register_commands" + ? `${context.generation}:${record.endpoint.setup.testStartedAt ?? record.endpoint.updatedAt.toISOString()}` + : String(context.generation); + const providerActionId = `telegram_maintenance:${operation}:${operationGeneration}`; + const [inserted] = await db + .insert(chatActions) + .values({ + companyId: endpoint.companyId, + endpointId: endpoint.id, + kind: "telegram_maintenance", + providerActionId, + payload: { + version: 1, + operation, + runtimeGeneration: context.generation, + credentialFingerprint: context.credentialFingerprint, + ...(subscriptionScope + ? { + webhookUrlSha256: subscriptionScope.webhookUrlSha256, + botUserId: subscriptionScope.botUserId, + } + : {}), + } satisfies TelegramMaintenancePayload, + status: "received", + }) + .onConflictDoNothing() + .returning(); + return ( + inserted ?? + (await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, endpoint.id), + eq(chatActions.providerActionId, providerActionId), + ), + ) + .then((rows) => rows[0] ?? null)) + ); + } + + async function telegramMaintenanceRequest( + botToken: string, + method: "deleteWebhook" | "deleteMyCommands" | "setMyCommands", + body: Record, + ): Promise { + const response = await fetchImpl( + `https://api.telegram.org/bot${encodeURIComponent(botToken)}/${method}`, + { + method: "POST", + signal: AbortSignal.timeout(PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS), + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }, + ); + const result = (await response.json()) as { + ok?: boolean; + description?: string; + parameters?: { retry_after?: number }; + }; + if (response.ok && result.ok) return; + throw Object.assign( + new Error( + `Telegram ${method} failed: ${result.description ?? response.status}`, + ), + response.status === 429 + ? { + adapter: "telegram", + code: "RATE_LIMITED", + name: "AdapterRateLimitError", + retryAfter: result.parameters?.retry_after, + response: { status: response.status, headers: response.headers }, + } + : { + adapter: "telegram", + status: response.status, + response: { status: response.status, headers: response.headers }, + }, + ); + } + + async function processTelegramMaintenance( + actionId: string, + force = false, + credentialLease?: CredentialMutationLeaseGuard, + ): Promise<"processed" | "pending" | "failed"> { + if (!credentialLease) { + const initialAction = await db + .select({ endpointId: chatActions.endpointId }) + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "telegram_maintenance"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!initialAction) return "failed"; + const initialEndpoint = await endpointRecord(initialAction.endpointId); + if (!initialEndpoint) return "failed"; + // Recovery sweeps run independently from endpoint configuration. Hold + // the same durable credential lease as reconnect, pause, and removal so + // a stale bot token cannot mutate Telegram after its final runtime fence + // but before transport. + return withCredentialMutationLease(initialEndpoint.endpoint, (lease) => + processTelegramMaintenance(actionId, force, lease), + ); + } + let action = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "telegram_maintenance"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!action) return "failed"; + if (action.status === "processed") return "processed"; + const payload = telegramMaintenancePayload(action.payload); + if (!payload) { + await db + .update(chatActions) + .set({ + status: "failed", + result: { + code: "telegram_maintenance_payload_invalid", + retryable: false, + }, + updatedAt: new Date(), + }) + .where(eq(chatActions.id, action.id)); + return "failed"; + } + if (action.status === "processing") { + if (action.updatedAt > new Date(Date.now() - PROVIDER_EFFECT_STALE_MS)) + return "pending"; + await db + .update(chatActions) + .set({ + status: "failed", + result: { + ...(action.result ?? {}), + retryable: true, + retryAt: new Date().toISOString(), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + eq(chatActions.updatedAt, action.updatedAt), + ), + ); + action = await db + .select() + .from(chatActions) + .where(eq(chatActions.id, action.id)) + .then((rows) => rows[0] ?? null); + if (!action) return "failed"; + } + if (action.status === "failed") { + if (action.result?.retryable !== true && !force) return "failed"; + const retryAt = + typeof action.result?.retryAt === "string" + ? new Date(action.result.retryAt) + : null; + if (!force && retryAt && retryAt > new Date()) return "pending"; + } else if (action.status !== "received") { + return "failed"; + } + const attempt = + (typeof action.result?.attempts === "number" + ? action.result.attempts + : 0) + 1; + let providerConfirmed = action.result?.providerConfirmed === true; + const [claimed] = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const rows = await tx + .update(chatActions) + .set({ + status: "processing", + result: { attempts: attempt, providerConfirmed }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, action!.status), + ), + ) + .returning(); + await credentialLease.assertOwned(tx); + return rows; + }); + if (!claimed) return "pending"; + + try { + const record = await endpointRecord(action.endpointId); + if (!record || record.endpoint.provider !== "telegram") { + throw Object.assign(new Error("Telegram endpoint is unavailable"), { + code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED", + }); + } + if ( + payload.operation === "remove_endpoint" && + record.credentialSecretRefs.length === 0 + ) { + // Recovery may observe the narrow crash window after provider cleanup + // and credential clearing but before the action receipt was committed. + // Empty refs prove removal cleanup crossed its final local boundary; + // settle the idempotent action instead of misclassifying it as a + // credential rotation. + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await tx + .update(chatActions) + .set({ + status: "processed", + result: { attempts: attempt, providerConfirmed: true }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ); + await credentialLease.assertOwned(tx); + }); + return "processed"; + } + if ( + credentialFingerprint(record.credentialSecretRefs) !== + payload.credentialFingerprint || + (payload.operation !== "remove_endpoint" && + runtimeGeneration(record.endpoint.setup) !== + payload.runtimeGeneration) + ) { + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await tx + .update(chatActions) + .set({ + status: "cancelled", + result: { + attempts: attempt, + code: "telegram_maintenance_credentials_superseded", + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ); + await credentialLease.assertOwned(tx); + }); + return "failed"; + } + if (!providerConfirmed || payload.operation === "stop_subscription") { + const credentials = await resolveCredentials(record.endpoint); + if (!credentials.botToken) { + throw Object.assign(new Error("Telegram bot token is unavailable"), { + code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED", + }); + } + if (payload.operation === "stop_subscription") { + await repairTelegramStopSubscription( + record.endpoint, + payload, + credentialLease, + ); + providerConfirmed = true; + } else if (payload.operation === "register_commands") { + await credentialLease.assertOwned(); + await telegramMaintenanceRequest( + credentials.botToken, + "setMyCommands", + { commands: TELEGRAM_COMMANDS }, + ); + providerConfirmed = true; + } else { + await credentialLease.assertOwned(); + await telegramMaintenanceRequest( + credentials.botToken, + "deleteWebhook", + { drop_pending_updates: false }, + ); + await credentialLease.assertOwned(); + await telegramMaintenanceRequest( + credentials.botToken, + "deleteMyCommands", + {}, + ); + providerConfirmed = true; + } + await credentialLease.assertOwned(); + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const confirmed = await tx + .update(chatActions) + .set({ + result: { attempts: attempt, providerConfirmed: true }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!confirmed.length) { + throw new Error( + "Telegram maintenance ownership changed before provider confirmation", + ); + } + await credentialLease.assertOwned(tx); + }); + } + if (payload.operation === "remove_endpoint") { + await credentialLease.assertOwned(); + await clearCredentials(record.endpoint, credentialLease); + } + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const processed = await tx + .update(chatActions) + .set({ + status: "processed", + result: { attempts: attempt, providerConfirmed: true }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!processed.length) { + throw new Error( + "Telegram maintenance ownership changed before success settlement", + ); + } + await credentialLease.assertOwned(tx); + }); + await logActivity(db, { + companyId: record.endpoint.companyId, + actorType: "system", + actorId: "chat-maintenance", + action: `chat_endpoint.telegram_${payload.operation}_completed`, + entityType: "tool_connection", + entityId: record.endpoint.connectionId, + details: { endpointId: record.endpoint.id, attempt }, + }); + return "processed"; + } catch (error) { + const disposition = classifyChatPublicationError(error, attempt); + // Webhook/menu mutations are idempotent provider operations. A lost + // response is safe to retry, and an explicit provider Retry-After must + // remain durable rather than occupying an in-memory timer. + const retryable = + disposition.kind === "retry" || disposition.kind === "delivery_unknown"; + const retryAfterMs = + disposition.kind === "retry" + ? disposition.retryAfterMs + : Math.min(60_000, 1_000 * 2 ** Math.max(0, attempt - 1)); + const retryAt = retryable + ? new Date(Date.now() + retryAfterMs).toISOString() + : null; + const diagnostic = redactError(error); + await db + .update(chatActions) + .set({ + status: "failed", + result: { + attempts: attempt, + code: `telegram_maintenance_${disposition.kind}`, + providerConfirmed: + providerConfirmed || action.result?.providerConfirmed === true, + redactedError: diagnostic, + retryable, + retryAt, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ); + await logActivity(db, { + companyId: action.companyId, + actorType: "system", + actorId: "chat-maintenance", + action: `chat_endpoint.telegram_${payload.operation}_deferred`, + entityType: "tool_connection", + entityId: + (await endpointRecord(action.endpointId))?.endpoint.connectionId ?? + action.endpointId, + details: { + endpointId: action.endpointId, + attempt, + disposition: disposition.kind, + retryAt, + }, + }).catch((auditError) => { + logger.error( + { endpointId: action!.endpointId, error: redactError(auditError) }, + "could not persist Telegram maintenance deferral audit", + ); + }); + logger.warn( + { + endpointId: action.endpointId, + operation: payload.operation, + retryAt, + }, + "Telegram maintenance was deferred", + ); + return retryable ? "pending" : "failed"; + } + } + + async function processPendingTelegramMaintenance(limit = 25) { + // Keyset scanning bounds each sweep without starving endpoints behind a + // page whose subscriptions were already confirmed. No provider I/O here. + const candidates = await db + .select({ endpoint: chatEndpoints }) + .from(chatEndpoints) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, chatEndpoints.connectionId), + eq(toolConnections.companyId, chatEndpoints.companyId), + ), + ) + .where( + and( + eq(chatEndpoints.provider, "telegram"), + eq(chatEndpoints.status, "active"), + eq(toolConnections.status, "active"), + eq(toolConnections.enabled, true), + telegramSubscriptionScanAfter + ? gt(chatEndpoints.id, telegramSubscriptionScanAfter) + : undefined, + ), + ) + .orderBy(asc(chatEndpoints.id)) + .limit(Math.max(1, Math.min(limit, 100))); + telegramSubscriptionScanAfter = + candidates.length >= Math.max(1, Math.min(limit, 100)) + ? candidates.at(-1)!.endpoint.id + : null; + for (const { endpoint } of candidates) { + try { + const record = await endpointRecord(endpoint.id); + if ( + record && + !(await hasTelegramStopSubscription( + record.endpoint, + runtimeContextForRecord(record), + )) + ) + await stageTelegramMaintenance(record.endpoint, "stop_subscription"); + } catch { + logger.warn( + { endpointId: endpoint.id }, + "Telegram subscription maintenance could not be staged", + ); + } + } + const now = new Date(); + const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); + const actions = await db + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.kind, "telegram_maintenance"), + sql`(${chatActions.payload}->>'operation' <> 'stop_subscription' or exists ( + select 1 from ${chatEndpoints} inner join ${toolConnections} + on ${toolConnections.id} = ${chatEndpoints.connectionId} and ${toolConnections.companyId} = ${chatEndpoints.companyId} + where ${chatEndpoints.id} = ${chatActions.endpointId} and ${chatEndpoints.companyId} = ${chatActions.companyId} + and ${chatEndpoints.provider} = 'telegram' and ${chatEndpoints.status} = 'active' + and ${toolConnections.status} = 'active' and ${toolConnections.enabled} = true + ))`, + or( + eq(chatActions.status, "received"), + and( + eq(chatActions.status, "processing"), + lte(chatActions.updatedAt, staleBefore), + ), + and( + eq(chatActions.status, "failed"), + sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`, + sql`(${chatActions.result}->>'retryAt' is null or (${chatActions.result}->>'retryAt')::timestamptz <= ${now.toISOString()}::timestamptz)`, + ), + ), + ), + ) + .orderBy(asc(chatActions.createdAt)) + .limit(limit); + for (const action of actions) await processTelegramMaintenance(action.id); + return actions.length; + } + + function telegramStopSubscriptionScope( + endpoint: EndpointRow, + context: LifecycleRuntimeFence, + ) { + if ( + endpoint.provider !== "telegram" || + !endpoint.botExternalId || + !getWebhookPublicBaseUrl() + ) + return null; + const webhookUrlSha256 = createHash("sha256") + .update( + `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/telegram`, + ) + .digest("hex"); + return { + version: 1, + botUserId: endpoint.botExternalId, + runtimeGeneration: context.generation, + credentialFingerprint: context.credentialFingerprint, + webhookUrlSha256, + }; + } + + async function hasTelegramStopSubscription( + endpoint: EndpointRow, + context: LifecycleRuntimeFence, + ): Promise { + const scope = telegramStopSubscriptionScope(endpoint, context); + if (!scope) return false; + const [receipt] = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.companyId, endpoint.companyId), + eq(chatActions.endpointId, endpoint.id), + eq(chatActions.kind, "telegram_stop_subscription"), + eq( + chatActions.providerActionId, + `telegram-stop-subscription:${scope.runtimeGeneration}:${scope.credentialFingerprint}:${scope.webhookUrlSha256}`, + ), + eq(chatActions.status, "processed"), + ), + ) + .limit(1); + return ( + !!receipt && + receipt.result?.code === "telegram_stop_subscription_confirmed" && + Object.keys(receipt.payload).length === Object.keys(scope).length && + Object.entries(scope).every( + ([key, value]) => receipt.payload[key] === value, + ) + ); + } + + async function recordTelegramStopSubscription( + endpoint: EndpointRow, + context: LifecycleRuntimeFence, + lease: CredentialMutationLeaseGuard, + ): Promise { + const scope = telegramStopSubscriptionScope(endpoint, context); + if (!scope) + throw new Error("Telegram Stop subscription scope is unavailable"); + await db.transaction(async (tx) => { + await lease.assertOwned(tx); + const current = await runtimeCallbackEndpoint(tx, endpoint.id, context, [ + "verifying", + "active", + ]); + if ( + !current || + current.botExternalId !== scope.botUserId || + current.publicId !== endpoint.publicId + ) + throw new Error("Telegram Stop subscription ownership changed"); + const [connection] = await tx + .select() + .from(toolConnections) + .where( + and( + eq(toolConnections.id, current.connectionId), + eq(toolConnections.companyId, current.companyId), + ), + ) + .for("share"); + if ( + !connection?.enabled || + connection.status !== "active" || + credentialFingerprint(connection.credentialSecretRefs) !== + context.credentialFingerprint + ) + throw new Error( + "Telegram Stop subscription credential ownership changed", + ); + await tx + .insert(chatActions) + .values({ + companyId: endpoint.companyId, + endpointId: endpoint.id, + kind: "telegram_stop_subscription", + providerActionId: `telegram-stop-subscription:${scope.runtimeGeneration}:${scope.credentialFingerprint}:${scope.webhookUrlSha256}`, + payload: scope, + status: "processed", + result: { code: "telegram_stop_subscription_confirmed" }, + }) + .onConflictDoUpdate({ + target: [chatActions.endpointId, chatActions.providerActionId], + set: { + payload: scope, + status: "processed", + result: { code: "telegram_stop_subscription_confirmed" }, + updatedAt: new Date(), + }, + }); + await lease.assertOwned(tx); + }); + } + + async function repairTelegramStopSubscription( + endpoint: EndpointRow, + payload: TelegramMaintenancePayload, + lease: CredentialMutationLeaseGuard, + ): Promise { + const plainObject = (value: unknown): Record => + value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + const context = { + generation: payload.runtimeGeneration, + credentialFingerprint: payload.credentialFingerprint, + }; + const scope = telegramStopSubscriptionScope(endpoint, context); + const reject = () => + Object.assign( + new Error( + "Telegram subscription cannot be safely upgraded under current ownership", + ), + { + code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED", + }, + ); + if ( + !scope || + scope.webhookUrlSha256 !== payload.webhookUrlSha256 || + scope.botUserId !== payload.botUserId + ) + throw reject(); + const assertCurrent = async () => + db.transaction(async (tx) => { + await lease.assertOwned(tx); + const current = await runtimeCallbackEndpoint( + tx, + endpoint.id, + context, + ["active"], + ); + if ( + !current || + current.companyId !== endpoint.companyId || + current.botExternalId !== scope.botUserId || + current.publicId !== endpoint.publicId + ) + throw reject(); + const [connection] = await tx + .select() + .from(toolConnections) + .where( + and( + eq(toolConnections.id, current.connectionId), + eq(toolConnections.companyId, current.companyId), + ), + ) + .for("share"); + if ( + !connection?.enabled || + connection.status !== "active" || + credentialFingerprint(connection.credentialSecretRefs) !== + context.credentialFingerprint + ) + throw reject(); + await lease.assertOwned(tx); + }); + await assertCurrent(); + // A recovered providerConfirmed flag is not a subscription capability. + // Only the separately committed exact current receipt can avoid fresh I/O. + if (await hasTelegramStopSubscription(endpoint, context)) return; + const credentials = await resolveCredentials(endpoint); + if ( + !credentials.botToken || + typeof credentials.webhookSecret !== "string" || + !/^[A-Za-z0-9_-]{1,256}$/.test(credentials.webhookSecret) + ) + throw reject(); + const request = async ( + method: "getMe" | "getWebhookInfo" | "setWebhook", + body: Record = {}, + ) => { + await assertCurrent(); + let response: Response; + try { + response = await fetchImpl( + `https://api.telegram.org/bot${encodeURIComponent(credentials.botToken!)}/${method}`, + { + method: "POST", + redirect: "error", + signal: AbortSignal.timeout(PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS), + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }, + ); + } catch { + // Network errors can embed a token-bearing URL. Keep maintenance + // diagnostics closed rather than forwarding provider/transport prose. + throw new Error( + "Telegram subscription transport did not confirm completion", + ); + } + let value: unknown; + try { + value = await response.json(); + } catch { + throw new Error("Telegram subscription returned an invalid response"); + } + const result = plainObject(value); + if (!response.ok || result.ok !== true) { + const retryAfter = plainObject(result.parameters).retry_after; + throw Object.assign( + new Error("Telegram subscription request was not confirmed"), + response.status === 429 + ? { + adapter: "telegram", + code: "RATE_LIMITED", + name: "AdapterRateLimitError", + retryAfter: + typeof retryAfter === "number" && Number.isFinite(retryAfter) + ? retryAfter + : undefined, + response: { + status: response.status, + headers: response.headers, + }, + } + : { + adapter: "telegram", + status: response.status, + response: { + status: response.status, + headers: response.headers, + }, + }, + ); + } + return result.result; + }; + const identity = plainObject(await request("getMe")); + if ( + identity.is_bot !== true || + !Number.isSafeInteger(identity.id) || + String(identity.id) !== scope.botUserId + ) + throw reject(); + const url = `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/telegram`; + const plan = telegramStopSubscriptionPlan( + await request("getWebhookInfo"), + url, + ); + if (!plan) throw reject(); + // No URL repoint, credential rotation, certificate guess, IP pin, or queue + // dropping. A successful SET proves the stored secret; GET cannot do so. + if ( + (await request("setWebhook", { + url, + secret_token: credentials.webhookSecret, + ...plan, + drop_pending_updates: false, + })) !== true + ) + throw new Error("Telegram subscription mutation was not confirmed"); + if ( + !telegramStopSubscriptionConfirmed( + await request("getWebhookInfo"), + url, + plan, + ) + ) + throw new Error("Telegram subscription verification changed"); + await assertCurrent(); + await recordTelegramStopSubscription(endpoint, context, lease); + } + return { stageTelegramMaintenance, processTelegramMaintenance, processPendingTelegramMaintenance, hasTelegramStopSubscription, recordTelegramStopSubscription }; +}