diff --git a/doc/plans/2026-10-12-chat-channel-delivery-boundaries-refactor.md b/doc/plans/2026-10-12-chat-channel-delivery-boundaries-refactor.md new file mode 100644 index 00000000000..01a3fb22ef9 --- /dev/null +++ b/doc/plans/2026-10-12-chat-channel-delivery-boundaries-refactor.md @@ -0,0 +1,100 @@ +# Chat channel callback and delivery boundaries, batch four + +Base: 4b766b872993045a2e787404f0475503683e5ded. Batch three (#15995, #15996, #15999, #16001, #16002) +is merged. The service starts at 26,461 lines / 1,046,711 bytes. +This batch may create up to ten stacked PRs. Merging the new batch is separate work. + +| Order / boundary | Destination | Approx. moved / net lines | Contract and state | Risks and existing verification | +| --- | --- | --- | --- | --- | +| 1. Slack callback verification | `slack/callback-verification.ts` | 250 / 230 | Pure signature/workspace/URL inspection plus signed URL verification; explicit vault lookup, URL getter, scheduling and registration callbacks | Preserve signing-secret-only setup, original request, TLS/proxy evidence and workspace checks. Signed verification and callback health cases before/after. | +| 2. Discord Gateway events | `discord/gateway-events.ts` | 200 / 185 | Runtime fence and lease-owner predicates, endpoint lookup and lifecycle callback; original per-runtime event queue and sequence | Preserve stale-owner fencing, event order, transient guild recovery and terminal revocation. Gateway lifecycle cases before/after. | +| 3. Discord root threads | `discord/root-threads.ts` | 500 / 460 | Credential lease, current principal authority, runtime, message admission and conversation drain callbacks | Preserve uncertain thread creation, restart reconciliation, no duplicate starter and pause/credential serialization. Root-thread public-service cases. | +| 4. Receipt reactions | `receipt-reactions.ts` and Slack error classifier beside Slack messages | 545 / 500 | Durable reaction claim/transport/settlement with runtime and credential fences; no scheduling owner moves | Preserve retries, legacy GitHub receipts, terminal-removal supersession and same-attempt settlement. Receipt recovery cases before/after. | +| 5. Inbound reactions | `reactions.ts` | 800 / 740 | Conversation lookup/locks, reaction admission and deferred replay together; current principal/runtime callbacks | Preserve exact event dedupe, completed generation links, bounded replay age and no task wakeup. Reaction race/restart/authorization cases. | +| 6. Inbound attachments | `inbound-attachments.ts` | 600 / 550 | Existing storage and issue service, retained source authorization and live options; return same attachment results | Preserve size limits, source authority around I/O, omissions and text delivery. Provider attachment/restart/stale-authority cases. | +| 7. Outbound attachments | `outbound-attachments.ts`; provider Markdown wrappers beside provider messages | 245 / 220 | Metadata validation and bounded integrity-checked hydration; existing storage object. Keep public hydration export compatible | Preserve stream cleanup, deadlines, byte/digest checks and retry classification. Hydration and publication attachment cases. | +| 8. Provider effects | `provider-effects.ts` | 950 / 850 | Durable reply claim, authorization, transport and settlement; endpoint/runtime/credential/principal callbacks | Preserve uncertain-delivery quarantine, credential fence and transaction ordering. Native callback, task-control and reply recovery cases. Put Telegram-only authority under `telegram/` if its contract is narrow. | +| 9. Conversation and activity reads | `activity.ts` | 540 / 520 | Endpoint lookup and read-only Teams conflict projection; return existing API models | Preserve company/endpoint joins, mixed-source cursor ties, replay eligibility and read-only GETs. Activity, file receipt projection and conversation-state cases. | +| 10. Setup probes | `setup-probes.ts` | 280 / 250 | Credential lease, current identity/principal authority, provider fetch, endpoint projection and live activation hook | Preserve real-message requirement, generation and linked-user activation. Setup-test, reconnect and authorization cases. | + +Each slice must earn its place after reading its complete path and callers. +Reassess before implementation; stop short of ten if the remaining contracts are +not cohesive. Keep provider-specific code in channel folders and common durable +workflows in the shared chat-channels directory. + +Receiver lease ownership stays shared: the existing Discord-named owner is also +used by Photon. Shared lifecycle/application admission, queue-drain ownership, +worker scheduling, and the largest action/message handlers remain in the service. +Do not hide their dependencies in a mutable context object. Preserve live option +hooks, runtime-map identity, partial-failure cleanup, and initialization order. + +Run relevant existing public-service tests before and after each mechanical move. +Compare moved declarations and remaining statements with normalized syntax, then +run server types, module boundaries and whitespace checks. Finish local verification and a fresh +5/5 review before opening the next draft PR. GitHub runner-selection jobs remained +queued without steps for over 25 minutes in this batch. To continue the authorized +stack, publish drafts in review order while tracking CI as pending; CI remains a +merge gate. Run cumulative repository gates at handoff. +Retain failures and skipped/unrun coverage even when an isolated retry passes. +Use fixture-backed assembled service/database tests; do not claim live provider +execution. Remap LOC/bytes and the remaining orchestration after the batch. + +## Slice 1: Slack callback verification + +- Service: 26,461 lines / 1,046,711 bytes → 26,201 lines / 1,035,260 bytes. +- Destination: 299 lines / 13,038 bytes. Eight declarations move without body + changes. Three existing setup types move unchanged into `runtime-types.ts`. +- The verification factory initializes after Slack registration. It receives the + same registration object, credential resolver, live URL getter and scheduler. + Signature/workspace/URL helpers remain callable by shared webhook orchestration. +- Nine public-service cases passed before and after on the updated master base. + Server typecheck, module boundaries, whitespace and normalized syntax parity + passed. Coverage includes signed URL verification, account notification, + workspace rejection, callback surfaces, TLS termination and Cloud host evidence. +- Next: Discord Gateway event sequencing and health transitions. Keep shared + receiver lease ownership and runtime-map lifetime in the service. + +## Batch outcome + +All ten candidates produced cohesive boundaries. Shared receiver ownership, queue +management and worker scheduling remain in the service. The only intentional +control-wiring change is deferred access to `stageProviderEffect` from identity +link construction; typecheck and integration tests caught the initial early read. + +| Slice | Service lines after | Primary module lines | Before/after cases | +| --- | ---: | ---: | ---: | +| 1. Slack signature, workspace and callback url verification | 26,233 | 300 | 9 | +| 2. Discord gateway event sequencing and endpoint health transitions | 26,029 | 253 | 7 | +| 3. Discord root-thread admission, provider creation and durable recovery | 25,527 | 570 | 4 | +| 4. Durable receipt reaction claims, transport, retries and settlement | 24,975 | 586 | 7 | +| 5. Inbound reaction conversation lookup, admission and deferred replay | 24,160 | 891 | 21 | +| 6. Inbound attachment authorization, bounded ingestion, registration and omissions | 23,565 | 674 | 39 | +| 7. Outbound attachment metadata validation and bounded integrity-checked hydration | 23,298 | 294 | 22 | +| 8. Durable provider reply staging, authorization, transport and recovery | 22,331 | 957 | 24 | +| 9. Conversation summaries and paginated activity projections | 21,741 | 616 | 16 | +| 10. Setup message detection and credential-fenced round-trip qualification | 21,460 | 353 | 7 | + +After rebasing on master `8e7b5db822`, the service shrank from +**26,493 lines / 1,048,769 bytes** to **21,460 lines / +862,284 bytes** (5,033 fewer lines, about 19%). Normalized syntax comparison +checks all 69 moved declarations/types and every remaining original statement. +Server types, targeted tests, module boundaries and whitespace checks are the +per-slice gates. Cumulative repository gates and remote reviews are recorded in +the PR descriptions as they complete. + +Orchestration measurement: the service function has 150 nested functions +(19,399 lines) and 67 state/wiring statements (459 lines), versus 191 functions +(23,829 lines) and 54 statements (425 lines) at the start. Imports take 400 lines. +The remaining large paths are `processMessage` (2,166), `handleAction` (1,074), +`configureWithCredentialLease` (868), `processSelectedPublication` (729), +`handleModalSubmit` (708), `handleWebhook` (685), and lifecycle processing (657). + +A future pass should inspect publication admission/transport/settlement as one +workflow, then inbound delivery authorization and drain ownership. Those cuts +need explicit ownership contracts; avoid separating individual checks from their +transactions merely to reduce the remaining line count. + +The rebase preserves the merged managed Slack setup flow, including its shared +`isProvisionedSlackSetup` predicate in the extracted callback verification +module. A fresh syntax audit compares all moved declarations and retained +statement ordering with this updated master. diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index e1077280799..6a302b30fdc 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -1,6 +1,22 @@ import { slackTestMessageService } from "./connectors/slack/test-message.js"; import type { SlackSetupOptions } from "./connectors/slack/setup/registration.js"; import { isProvisionedSlackSetup } from "@paperclipai/shared"; +import { createSetupProbes } from "./chat-channels/setup-probes.js"; +import { createChatActivity, publicationSummary } from "./chat-channels/activity.js"; +import { fileTransferProjectionColumns } from "./chat-channels/microsoft-teams/file-transfer-projection.js"; +import { createProviderEffects, providerEffectPayload, type ProviderEffectTarget, type ProviderEffectPayload } from "./chat-channels/provider-effects.js"; +import { createTelegramCallbackNoticeAuthority } from "./chat-channels/telegram/callback-notices.js"; +import { createOutboundAttachments, isOutboundAttachmentHydrationError, isOutboundAttachmentValidationError } from "./chat-channels/outbound-attachments.js"; +export { hydrateOutboundAttachment } from "./chat-channels/outbound-attachments.js"; +import { discordMarkdownAttachment } from "./chat-channels/discord/messages.js"; +import { telegramMarkdownAttachment } from "./chat-channels/telegram/messages.js"; +import { createInboundAttachments } from "./chat-channels/inbound-attachments.js"; +import { createInboundReactions } from "./chat-channels/reactions.js"; +import { slackThreadChannelId } from "./chat-channels/slack/messages.js"; +import { createReceiptReactions } from "./chat-channels/receipt-reactions.js"; +import { createDiscordRootThreads } from "./chat-channels/discord/root-threads.js"; +import { createDiscordGatewayEvents } from "./chat-channels/discord/gateway-events.js"; +import { createSlackCallbackVerification, slackRequestSignatureIsValid, slackRequestWorkspaceId, canonicalCallbackUrl, slackCallbackMatchesPublicUrl, inspectSlackCallback } from "./chat-channels/slack/callback-verification.js"; import { createTeamsFileAuthority } from "./chat-channels/microsoft-teams/file-transfer-authority.js"; import { createPhotonEvents } from "./chat-channels/imessage-photon/events.js"; import { createTelegramPublicationDrafts } from "./chat-channels/telegram/publication-drafts.js"; @@ -8,10 +24,10 @@ import { createSlackFileReceipts } from "./chat-channels/slack/file-receipts.js" import { createSlackSessionStatus } from "./chat-channels/slack/session-status.js"; import { createDiscordCommands } from "./chat-channels/discord/commands.js"; import { createSlackTaskStart, slackSlashTaskRecoveryPayload, slackTaskStarterThreadId } from "./chat-channels/slack/task-start.js"; -import { createGitHubWebhookIngress, GITHUB_RECOVERY_STATE_KEY, type GitHubIngressStageResult } from "./chat-channels/github/webhook-ingress.js"; +import { createGitHubWebhookIngress, type GitHubIngressStageResult } from "./chat-channels/github/webhook-ingress.js"; import { createSlackSessionStop } from "./chat-channels/slack/session-stop.js"; import { createTelegramMaintenance } from "./chat-channels/telegram/maintenance.js"; -import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence, InboundRuntimeContext } from "./chat-channels/runtime-types.js"; +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence, InboundRuntimeContext, SlackCallbackSurface, InternalSetupState } from "./chat-channels/runtime-types.js"; import { createChatFailedRunRetry } from "./chat-channels/failed-run-retry.js"; import { createChatPublicationReplacement } from "./chat-channels/publication-replacement.js"; import { createChatIdentityLinks } from "./chat-channels/identity-links.js"; @@ -51,7 +67,7 @@ import { type GitHubLifecycleEvent, githubLifecycleEventFromRequest, } from "./chat-channels/github/messages.js"; -import { dueChatAction, nextChatActionAt, SLACK_COMMAND_POST_STALE_MS, SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS, PROVIDER_EFFECT_STALE_MS } from "./chat-action-work.js"; +import { dueChatAction, nextChatActionAt, SLACK_COMMAND_POST_STALE_MS, SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS } from "./chat-action-work.js"; import type { ScheduledChatAction } from "./chat-work-notifications.js"; import { notifyChatDeliveryWork, notifyChatPublicationWork, notifyChatEndpointWork, notifyChatActionWork, notifyChatVerificationWork } from "./chat-work-notifications.js"; import { syncSpekoVoiceTools } from "./voice/speko-agent-tools.js"; @@ -64,9 +80,7 @@ import type { ChatEndpointRuntime } from "./voice/voice-runtime.js"; import { parseMarkdown } from "chat"; import { githubPublicAttachmentsFromMessage } from "./chat-github-attachments.js"; import { - receiptReactionPayload, stageReceiptReactionRemovals, - type ReceiptReactionPayload, } from "./chat-receipt-reactions.js"; import { fastResponseService, enqueueFastResponse, fastResponseSourceCurrent, fastResponseTurnQueued } from "./fast-responses.js"; import { fastResponseRequests } from "@paperclipai/db"; @@ -89,7 +103,7 @@ import { registerSlackTaskAuthority, slackRunOrigin } from "./connectors/slack-a import { buildChatCommunicationGuidance } from "./chat-communication-guidance.js"; function githubPolicyRecord(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } -import { githubChatManagementService } from "./chat-github-management.js"; + import { githubReviewCheckService } from "./chat-github-checks.js"; import { githubAutomaticReviewEvent, @@ -112,12 +126,12 @@ import { githubAppJwt } from "./chat-github-client.js"; import { resumeExternalConversation } from "./slack-conversation-state.js"; import { settleExternalConversation } from "./slack-conversation-lifecycle.js"; import { runtimeCanonicalOrigin } from "./cloud-runtime-identity.js"; -import { takePhotonCompanion } from "./photon/attachments.js"; + import { writePhotonCheckpoint } from "./photon/receiver.js"; import { PhotonState } from "./photon/state.js"; -import { HEIF_CONTENT_TYPES, photonHeifPreview, validatePhotonImage } from "./photon/media.js"; -import { projectSafeChatPublicationText } from "./chat-publication-projection.js"; + + import { nativePhotonInteraction, publishPhotonPrompt, type PhotonPromptReceipt, type PhotonInteractionBinding } from "./photon/interactions.js"; @@ -139,7 +153,6 @@ import { desc, eq, gt, - gte, inArray, isNotNull, isNull, @@ -205,7 +218,6 @@ import type { ChatEndpointSetupState, ChatProvider, ChatPublicationBatchStatus, - ChatPublicationSummary, ConfigureChatEndpointInput, CreateChatEndpointInput, ExternalChannelBindingSummary, @@ -222,10 +234,8 @@ import { } from "@paperclipai/shared"; import { formatAttachmentSize, - isAllowedContentType, MAX_ATTACHMENT_BYTES, normalizeContentType, - normalizeUploadAttachmentContentType, } from "../attachment-types.js"; import { isUniqueViolation } from "../db-errors.js"; import { @@ -239,24 +249,17 @@ import { import { projectTeamsFilePublication } from "./chat-teams-file-publication.js"; import type { TeamsFileConsentEvent } from "./chat-teams-file-consent.js"; import { - chatFileTransferResolutionActions, projectChatFileTransfer, projectChatPublicationBatch, } from "./chat-publication-batches.js"; import { telegramAttachmentForUpload } from "./chat-telegram-photo.js"; import { - parseTelegramCallbackReceipt, readTelegramCallbackProvenance, telegramCallbackThreadId, TELEGRAM_PRIVATE_ACTION_UNAVAILABLE, - type TelegramCallbackReceipt, } from "./chat-telegram-ephemeral.js"; -import { - hasTelegramMediaProvenance, - identifyTelegramMedia, - telegramMediaNeedsIdentification, -} from "./chat-telegram-media-intake.js"; + import { normalizeTelegramRichMessage } from "./chat-telegram-rich-intake.js"; import { isTelegramDraftStopped, @@ -265,10 +268,7 @@ import { import { parseChatWebhookPublicBaseUrl } from "../chat-webhook-public-url.js"; -import { - parseGitHubReceiptIdentity, - type GitHubReceiptIdentity, -} from "./chat-github-receipt-reactions.js"; + import { badRequest, HttpError, @@ -281,14 +281,12 @@ import { logger } from "../middleware/logger.js"; import { redactSensitiveText } from "../redaction.js"; import type { StorageService } from "../storage/types.js"; import { - DiscordAdapterCompatibilityError, type ChatSdkCallbackEvent, type ChatSdkMessageCallbackEvent, type ChatSdkMessageUpdatedCallbackEvent, type ChatSdkProvider, type ChatSdkRuntime, type DiscordGatewayCallbackEvent, - type DiscordRootMentionAdmissionEvent, type ResolvedChatSdkProviderConfig, type SlackFileUploadAcceptedReceipt, } from "./chat-sdk-runtime.js"; @@ -322,10 +320,7 @@ import { requestGitHubAppWebhookRedelivery, } from "./chat-github-webhook-config.js"; import { - GITHUB_ATTACHMENT_BATCH_TIMEOUT_MS, - GitHubAttachmentUnavailableError, githubAttachmentLimitOmissions, - prepareGitHubPublicAttachment, restoreGitHubAttachmentLimitOmissions, } from "./chat-github-attachments.js"; import { getExternalChannelBindingSummary } from "./chat-channel-binding.js"; @@ -383,9 +378,7 @@ import { normalizeMicrosoftTeamsCredentialIds, } from "./chat-teams-credentials.js"; import { - isTeamsInlineImageContentType, prepareTeamsInlineImage, - TEAMS_INLINE_IMAGE_MAX_BYTES, } from "./chat-teams-inline-images.js"; import { shouldStreamSafePublicationText, @@ -419,44 +412,11 @@ import type { MessageDeletedEvent, ModalResponse, ModalSubmitEvent, - ReactionEvent, SlashCommandEvent, Thread, } from "chat"; import { Actions, Button, Card, CardText, LinkButton } from "chat"; -// Only these closed fields may reach Activity or a delivery-batch response. -// The resolution predicate yields one boolean, never the private envelope. -const fileTransferProjectionColumns = { - publicationId: chatTeamsFileTransfers.publicationId, - phase: chatTeamsFileTransfers.phase, - version: chatTeamsFileTransfers.version, - filename: chatTeamsFileTransfers.filename, - expiresAt: chatTeamsFileTransfers.expiresAt, - consentMessageId: chatTeamsFileTransfers.consentMessageId, - responseActivityId: chatTeamsFileTransfers.responseActivityId, - fileInfoMessageId: chatTeamsFileTransfers.fileInfoMessageId, - operatorConfirmed: sql` - coalesce(${chatTeamsFileTransfers.privateState}#>>'{resolution,schema}', '') = 'paperclip.teams.file-resolution.v1' - and coalesce(${chatTeamsFileTransfers.privateState}#>>'{resolution,action}', '') = 'mark_delivered' - and coalesce(${chatTeamsFileTransfers.privateState}#>>'{resolution,fromPhase}', '') = 'file_info_unknown' - `, -}; - -function publicationSummary( - row: typeof chatPublications.$inferSelect, -): ChatPublicationSummary { - return { - id: row.id, - state: row.state, - providerUrl: row.providerUrl, - attempts: row.attempts, - redactedError: row.redactedError, - nextAttemptAt: row.nextAttemptAt?.toISOString() ?? null, - publishedAt: row.publishedAt?.toISOString() ?? null, - }; -} - const PROVIDER_LABELS: Record = { speko: "Speko", "imessage-photon": "iMessage Photon", @@ -494,207 +454,6 @@ function publicationPrecedes( ); } -function slackRequestSignatureIsValid( - request: Request, - body: string, - signingSecret: string, -): boolean { - const timestamp = request.headers.get("x-slack-request-timestamp"); - const signature = request.headers.get("x-slack-signature"); - if (!timestamp || !signature) return false; - const timestampSeconds = Number(timestamp); - if ( - !Number.isFinite(timestampSeconds) || - Math.abs(Math.floor(Date.now() / 1000) - timestampSeconds) > 300 - ) - return false; - const expected = `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:${body}`).digest("hex")}`; - try { - return timingSafeEqual(Buffer.from(signature), Buffer.from(expected)); - } catch { - return false; - } -} - -function slackRequestWorkspaceId( - body: string, - contentType: string, -): string | null { - let payload: unknown; - try { - if (contentType.includes("application/json")) { - payload = JSON.parse(body); - } else { - const form = new URLSearchParams(body); - const interactivePayload = form.get("payload"); - payload = interactivePayload - ? JSON.parse(interactivePayload) - : Object.fromEntries(form.entries()); - } - } catch { - return null; - } - if (!payload || typeof payload !== "object") return null; - const record = payload as Record; - const nestedId = (value: unknown): string | null => { - if (!value || typeof value !== "object") return null; - const id = (value as Record).id; - return typeof id === "string" && id.length > 0 ? id : null; - }; - const stringField = (key: string): string | null => { - const value = record[key]; - return typeof value === "string" && value.length > 0 ? value : null; - }; - const authorization = Array.isArray(record.authorizations) - ? record.authorizations.find( - (value): value is Record => - Boolean(value) && typeof value === "object", - ) - : null; - const authorizationField = (key: string): string | null => { - const value = authorization?.[key]; - return typeof value === "string" && value.length > 0 ? value : null; - }; - - // A Grid event may carry both an enterprise id and the concrete workspace - // id. The bot identity is verified against auth.test's team_id, so prefer a - // team id and use the enterprise id only when Slack omitted team context. - return ( - stringField("team_id") ?? - nestedId(record.team) ?? - authorizationField("team_id") ?? - stringField("enterprise_id") ?? - nestedId(record.enterprise) ?? - authorizationField("enterprise_id") - ); -} - -function canonicalCallbackUrl(value: string): string | null { - try { - const url = new URL(value); - if (url.protocol !== "https:" && url.protocol !== "http:") return null; - url.hostname = url.hostname.replace(/\.$/, ""); - url.username = ""; - url.password = ""; - url.search = ""; - url.hash = ""; - return url.toString(); - } catch { - return null; - } -} - -// TLS terminates at a reverse proxy in many self-hosted deployments. The -// transport scheme is not evidence of URL drift; authority and path still are. -// This is diagnostic only: it does not trust forwarded headers or change auth. -function slackCallbackMatchesPublicUrl(observed: string, current: string): boolean { - const canonical = canonicalCallbackUrl(observed); - if (canonical === current) return true; - if (!canonical) return false; - const observedUrl = new URL(canonical); - const currentUrl = new URL(current); - return observedUrl.protocol === "http:" - && currentUrl.protocol === "https:" - && observedUrl.host === currentUrl.host - && observedUrl.pathname === currentUrl.pathname; -} - -type SlackCallbackInspection = { - surface: SlackCallbackSurface; - url: string; - isUrlVerification: boolean; -}; - -// Cloud records the public request host in dedicated headers so provider edges -// cannot replace it with their upstream host. Only use callback-health evidence on a -// claimed Cloud instance, after provider authentication succeeded. It must not -// change the Request passed to the adapter, signature verification, routing, -// board identity, or the configured URL used to generate callbacks. -function slackCallbackObservationUrl(request: Request): string | null { - const directUrl = canonicalCallbackUrl(request.url); - if (!directUrl || !runtimeCanonicalOrigin()) return directUrl; - const hasCloudEvidence = request.headers.has("x-paperclip-cloud-forwarded-host"); - const host = request.headers.get(hasCloudEvidence ? "x-paperclip-cloud-forwarded-host" : "x-forwarded-host")?.trim(); - const protocol = request.headers.get(hasCloudEvidence ? "x-paperclip-cloud-forwarded-proto" : "x-forwarded-proto")?.trim(); - if (!host || /[\s/@\\?#,]/.test(host) || (protocol !== "https" && protocol !== "http")) return directUrl; - try { - const origin = new URL(`${protocol}://${host}`); - // Accept one authority only, never credentials, paths, queries, or lists. - if (!origin.host || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) return directUrl; - const observed = new URL(directUrl); - observed.protocol = origin.protocol; - observed.host = origin.host; - observed.port = origin.port; - return canonicalCallbackUrl(observed.toString()); - } catch { - return directUrl; - } -} - -async function inspectSlackCallback( - request: Request, -): Promise { - const url = slackCallbackObservationUrl(request); - if (!url) return null; - const contentType = request.headers.get("content-type")?.toLowerCase() ?? ""; - let body: string; - try { - body = await request.text(); - } catch { - return null; - } - if (contentType.includes("application/json")) { - try { - const payload = JSON.parse(body) as { - type?: unknown; - challenge?: unknown; - }; - const isUrlVerification = - payload.type === "url_verification" && - typeof payload.challenge === "string" && - payload.challenge.length > 0; - if (isUrlVerification || payload.type === "event_callback") { - return { - surface: "events", - url, - isUrlVerification, - }; - } - } catch { - return null; - } - return null; - } - - const form = new URLSearchParams(body); - if (form.get("command")) { - return { surface: "slashCommands", url, isUrlVerification: false }; - } - const interactivePayload = form.get("payload"); - if (!interactivePayload) return null; - try { - const payload = JSON.parse(interactivePayload) as { type?: unknown }; - if ( - typeof payload.type === "string" && - [ - "block_actions", - "block_suggestion", - "dialog_submission", - "message_action", - "shortcut", - "view_closed", - "view_submission", - "workflow_step_edit", - ].includes(payload.type) - ) { - return { surface: "interactivity", url, isUrlVerification: false }; - } - } catch { - return null; - } - return null; -} - const CAPABILITIES: Record = { speko: { threads: false, directMessages: true, nativeStreaming: false, messageEdits: false, messageDeletes: false, reactions: false, files: false, cards: false, actions: false, modals: false, slashCommands: false, ephemeralMessages: false, proactiveDirectMessages: false }, "imessage-photon": { threads: false, directMessages: true, nativeStreaming: false, messageEdits: true, messageDeletes: false, reactions: false, files: true, cards: false, actions: true, modals: false, slashCommands: false, ephemeralMessages: false, proactiveDirectMessages: false }, @@ -841,13 +600,6 @@ const SUPPLIED_CREDENTIAL_KEYS: Record = { const MAX_ERROR_TEXT = 2_000; const DELIVERY_PROCESSING_STALE_MS = 60_000; -// Reactions can beat the transaction that persists a just-sent provider -// message link. Keep that narrow gap durable, but never retain an unbound -// reaction indefinitely or let it enter the ordinary inbound-message FIFO. -const REACTION_LINK_RETRY_DELAY_MS = 250; -const REACTION_LINK_MAX_DELAY_MS = 5_000; -const REACTION_LINK_MAX_ATTEMPTS = 20; -const REACTION_LINK_MAX_AGE_MS = 2 * 60_000; // A Discord root mention is durably staged before the adapter creates its // provider thread. Give the bounded provider retry loop ample time to finish; // if the process disappears, the delivery worker verifies that thread over @@ -887,17 +639,6 @@ const INGRESS_REORDER_WINDOW_MS: Partial> = { telegram: 750, }; type ResourceRow = typeof chatEndpointResources.$inferSelect; -type SlackCallbackSurface = keyof ChatEndpointCallbackSurfaces; -type SlackCallbackObservation = { url: string; observedAt: string }; -type InternalSetupState = ChatEndpointSetupState & { - photonIntakeAfter?: string; - runtimeGeneration?: number; - /** Binds pre-install URL verification to the signing secret later configured. */ - slackVerificationSigningFingerprint?: string; - slackCallbackSurfaces?: Partial< - Record - >; -}; type VerifiedProviderIdentity = { githubOwnerType?: "personal" | "organization"; githubOwnerLogin?: string; @@ -1054,130 +795,6 @@ type LiveInboundMessage = { thread: Thread; trigger: ChatSdkMessageCallbackEvent["trigger"]; }; -type ProviderEffectTarget = { - post: Thread["post"]; - postEphemeral?: Thread["postEphemeral"]; -}; -type ProviderEffectPayload = { - version: 1; - effect: "thread_message" | "ephemeral_message" | "telegram_callback_notice"; - telegramCallback?: TelegramCallbackReceipt; - telegramAuthorizationSha256?: string; - authorizationMode?: "principal" | "safe_notice"; - threadId: string; - userId?: string; - text: string; - fallbackText?: string; - settleDelivery: boolean; - completeConversationId?: string; - resourceId?: string; - runtimeGeneration: number; - credentialFingerprint: string; -}; - -function isSlackReceiptReactionAlreadyApplied(error: unknown): boolean { - const pending: Array<{ depth: number; value: unknown }> = [ - { depth: 0, value: error }, - ]; - const seen = new Set(); - while (pending.length > 0) { - const current = pending.shift(); - if ( - !current || - current.depth > 4 || - !current.value || - typeof current.value !== "object" || - seen.has(current.value) - ) { - continue; - } - seen.add(current.value); - const record = current.value as { - cause?: unknown; - code?: unknown; - data?: { error?: unknown }; - original?: unknown; - originalError?: unknown; - }; - if ( - record.code === "already_reacted" || - record.data?.error === "already_reacted" - ) { - return true; - } - for (const value of [record.cause, record.original, record.originalError]) { - pending.push({ depth: current.depth + 1, value }); - } - } - return false; -} - -function providerEffectPayload( - payload: Record, -): ProviderEffectPayload | null { - if ( - payload.version !== 1 || - (payload.effect !== "thread_message" && - payload.effect !== "ephemeral_message" && - payload.effect !== "telegram_callback_notice") || - typeof payload.threadId !== "string" || - !payload.threadId || - typeof payload.text !== "string" || - !payload.text || - typeof payload.settleDelivery !== "boolean" || - typeof payload.runtimeGeneration !== "number" || - !Number.isSafeInteger(payload.runtimeGeneration) || - typeof payload.credentialFingerprint !== "string" - ) { - return null; - } - if ( - payload.effect === "ephemeral_message" && - (typeof payload.userId !== "string" || !payload.userId) - ) { - return null; - } - for (const key of [ - "authorizationMode", - "userId", - "fallbackText", - "completeConversationId", - "resourceId", - ] as const) { - const value = payload[key]; - if (value !== undefined && (typeof value !== "string" || !value)) { - return null; - } - } - if ( - payload.authorizationMode !== undefined && - payload.authorizationMode !== "principal" && - payload.authorizationMode !== "safe_notice" - ) { - return null; - } - if (payload.effect === "telegram_callback_notice") { - const receipt = parseTelegramCallbackReceipt(payload.telegramCallback); - if ( - !receipt || - payload.authorizationMode !== "safe_notice" || - payload.text !== TELEGRAM_PRIVATE_ACTION_UNAVAILABLE || - payload.threadId !== telegramCallbackThreadId(receipt) || - payload.userId !== receipt.receiverUserId || - payload.settleDelivery !== false || - payload.fallbackText !== undefined || - payload.completeConversationId !== undefined || - typeof payload.telegramAuthorizationSha256 !== "string" || - !/^[a-f0-9]{64}$/.test(payload.telegramAuthorizationSha256) - ) - return null; - } else if ( - payload.telegramCallback !== undefined || - payload.telegramAuthorizationSha256 !== undefined - ) - return null; - return payload as ProviderEffectPayload; -} export interface ChatChannelServiceOptions { /** @@ -1489,15 +1106,6 @@ function safeTitle(text: string, fallback: string): string { return (line || fallback).slice(0, 160); } -function slackThreadChannelId(threadId: string): string | null { - if (!threadId.startsWith("slack:")) return null; - const channelAndThread = threadId.slice("slack:".length); - const separator = channelAndThread.indexOf(":"); - const channelId = - separator === -1 ? channelAndThread : channelAndThread.slice(0, separator); - return channelId || null; -} - function actionThreadMatchesConversation( provider: ChatProvider, actionThreadId: string, @@ -1514,24 +1122,6 @@ function actionThreadMatchesConversation( ); } -function slackTaskStartActivityStatus(action: { - status: string; - updatedAt: Date; -}): string { - const age = Date.now() - action.updatedAt.getTime(); - if (action.status === "validating" && age >= SLACK_COMMAND_POST_STALE_MS) { - return "queued"; - } - if ( - (action.status === "received" && age >= SLACK_COMMAND_POST_STALE_MS) || - (action.status === "resolving" && - age >= SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS) - ) { - return "delivery_unknown"; - } - return action.status; -} - function sanitizeFilename(value: string | undefined): string | null { if (!value) return null; const leaf = value.replaceAll("\\", "/").split("/").pop()?.trim(); @@ -1821,163 +1411,6 @@ export function createChatSdkStatePersistence(db: Db): ChatSdkStatePersistence { }; } -const OUTBOUND_ATTACHMENT_STORAGE_TIMEOUT_MS = 10_000; - -class OutboundAttachmentHydrationError extends Error { - readonly code = "CHAT_ATTACHMENT_HYDRATION_FAILED"; - - constructor(message: string, cause?: unknown) { - super(message, cause === undefined ? undefined : { cause }); - this.name = "OutboundAttachmentHydrationError"; - } -} - -class OutboundAttachmentValidationError extends Error { - readonly code = "CHAT_ATTACHMENT_VALIDATION_FAILED"; - - constructor(message: string) { - super(message); - this.name = "OutboundAttachmentValidationError"; - } -} - -function isOutboundAttachmentHydrationError( - error: unknown, -): error is OutboundAttachmentHydrationError { - return error instanceof OutboundAttachmentHydrationError; -} - -function isOutboundAttachmentValidationError( - error: unknown, -): error is OutboundAttachmentValidationError { - return error instanceof OutboundAttachmentValidationError; -} - -/** - * Read one already-authorized attachment before any provider transport starts. - * The database digest, rather than mutable object-store metadata, is the - * integrity boundary. Both acquisition and streaming are bounded so a stuck - * storage backend cannot hold the publication lane indefinitely. - */ -export async function hydrateOutboundAttachment(input: { - storage: StorageService; - companyId: string; - objectKey: string; - byteSize: number; - sha256: string; - filename: string; - mimeType: string; - timeoutMs?: number; -}): Promise { - const timeoutMs = - typeof input.timeoutMs === "number" && - Number.isFinite(input.timeoutMs) && - input.timeoutMs > 0 - ? input.timeoutMs - : OUTBOUND_ATTACHMENT_STORAGE_TIMEOUT_MS; - if ( - !Number.isSafeInteger(input.byteSize) || - input.byteSize <= 0 || - input.byteSize > MAX_ATTACHMENT_BYTES || - !/^[0-9a-f]{64}$/iu.test(input.sha256) - ) { - throw new OutboundAttachmentHydrationError( - "Chat publication attachment metadata is invalid", - ); - } - const acquisitionTimeoutError = new OutboundAttachmentHydrationError( - "Chat publication attachment storage read timed out", - ); - let acquisitionTimedOut = false; - let rejectAcquisition!: (error: Error) => void; - const acquisitionTimeout = new Promise((_resolve, reject) => { - rejectAcquisition = reject; - }); - const acquisitionTimer = setTimeout(() => { - acquisitionTimedOut = true; - rejectAcquisition(acquisitionTimeoutError); - }, timeoutMs); - acquisitionTimer.unref?.(); - const objectPromise = Promise.resolve() - .then(() => input.storage.getObject(input.companyId, input.objectKey)) - .then((object) => { - if (acquisitionTimedOut) object.stream.destroy(); - return object; - }) - .catch((error) => { - throw isOutboundAttachmentHydrationError(error) - ? error - : new OutboundAttachmentHydrationError( - "Chat publication attachment storage read failed", - error, - ); - }); - let object: Awaited>; - try { - object = await Promise.race([objectPromise, acquisitionTimeout]); - } finally { - clearTimeout(acquisitionTimer); - } - - if ( - object.contentLength !== undefined && - object.contentLength !== input.byteSize - ) { - object.stream.destroy(); - throw new OutboundAttachmentHydrationError( - "Chat publication attachment size changed after registration", - ); - } - - const streamTimeoutError = new OutboundAttachmentHydrationError( - "Chat publication attachment storage read timed out", - ); - const streamTimer = setTimeout(() => { - object.stream.destroy(streamTimeoutError); - }, timeoutMs); - streamTimer.unref?.(); - const chunks: Buffer[] = []; - const digest = createHash("sha256"); - let bytes = 0; - try { - for await (const chunk of object.stream) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - bytes += buffer.length; - if (bytes > input.byteSize || bytes > MAX_ATTACHMENT_BYTES) { - object.stream.destroy(); - throw new OutboundAttachmentHydrationError( - "Chat publication attachment exceeded its streaming byte limit", - ); - } - digest.update(buffer); - chunks.push(buffer); - } - } catch (error) { - throw isOutboundAttachmentHydrationError(error) - ? error - : new OutboundAttachmentHydrationError( - "Chat publication attachment storage read failed", - error, - ); - } finally { - clearTimeout(streamTimer); - } - - const actualDigest = digest.digest("hex"); - if (bytes !== input.byteSize || actualDigest !== input.sha256.toLowerCase()) { - if (!object.stream.destroyed) object.stream.destroy(); - throw new OutboundAttachmentHydrationError( - "Chat publication attachment integrity changed after registration", - ); - } - - return { - data: Buffer.concat(chunks, bytes), - filename: input.filename, - mimeType: input.mimeType, - }; -} - export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const runtime = options.runtime ?? createChatRuntime(); const githubManualMessages = new WeakMap(); @@ -2009,7 +1442,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { // An explicit webhook ingress remains separate from board/identity links. const getPublicBaseUrl = () => runtimeCanonicalOrigin() ?? configuredPublicBaseUrl; const { listPrincipals, createLinkIntent, previewIdentityLink, requestIdentityAccess, confirmIdentityLink, revokeLink } = createChatIdentityLinks(db, { - endpointRecord, getPublicBaseUrl, stageProviderEffect, scheduleProviderEffect, + endpointRecord, getPublicBaseUrl, stageProviderEffect: (...args) => stageProviderEffect(...args), scheduleProviderEffect, confirmationRuntimeFor: async (endpoint) => runtimeContexts.get((await runtimeFor(endpoint)) as object), }); // Task links must validate the original configured URL before normalization @@ -2459,67 +1892,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } - async function addReceiptReaction(input: { - deliveryId: string; - endpoint: EndpointRow; - message: Message; - runtimeContext?: LifecycleRuntimeFence; - thread: Thread; - }): Promise { - // GitHub acknowledges through one editable response comment. Continue to - // drain legacy eyes removals, but never create new eyes for these requests. - if (input.endpoint.provider === "github") return; - const record = await endpointRecord(input.endpoint.id); - if (!record) return; - // A live SDK object remains bound to the credentials that authenticated - // its callback. Preserve that original fence rather than sampling the - // endpoint again after a rotation and accidentally blessing an obsolete - // adapter with the new generation. - const context = input.runtimeContext ?? runtimeContextForRecord(record); - const [inserted] = await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "receipt_reaction"); - return tx - .insert(chatActions) - .values({ - companyId: input.endpoint.companyId, - endpointId: input.endpoint.id, - deliveryId: input.deliveryId, - kind: "receipt_reaction", - providerActionId: `receipt_reaction:${input.deliveryId}`, - payload: { - version: 1, - operation: "add", - threadId: input.thread.id, - messageId: input.message.id, - reaction: "eyes", - runtimeGeneration: context.generation, - credentialFingerprint: context.credentialFingerprint, - } satisfies ReceiptReactionPayload, - status: "received", - }) - .onConflictDoNothing() - .returning({ id: chatActions.id }); - }); - const actionId = - inserted?.id ?? - (await db - .select({ id: chatActions.id }) - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, input.endpoint.id), - eq( - chatActions.providerActionId, - `receipt_reaction:${input.deliveryId}`, - ), - ), - ) - .then((rows) => rows[0]?.id ?? null)); - if (!actionId) return; - await options.receiptReactionTransportBarrier?.(); - await processReceiptReaction(actionId); - } - function conversationDrainKey(endpointId: string, threadId: string) { return `${endpointId}:${createHash("sha256").update(durableExternalThreadIdentity(threadId)).digest("hex")}`; } @@ -2588,506 +1960,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .then((rows) => rows[0] ?? null); } - async function processReceiptReaction(actionId: string): Promise { - const initialAction = await db - .select({ endpointId: chatActions.endpointId }) - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "receipt_reaction"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!initialAction) return; - const initialEndpoint = await endpointRecord(initialAction.endpointId); - if (!initialEndpoint) return; - // Receipt reactions are GitHub repository mutations too. Serialize their - // final runtime-fence check and provider call with reconnect, secret - // rotation, pause, and removal so credentials cannot become obsolete - // between authorization and transport. - await withCredentialMutationLease(initialEndpoint.endpoint, (lease) => - processReceiptReactionWithCredentialLease(actionId, lease), - ); - } - - async function processReceiptReactionWithCredentialLease( - actionId: string, - credentialLease: CredentialMutationLeaseGuard, - ): Promise { - let action = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "receipt_reaction"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!action || ["processed", "cancelled"].includes(action.status)) return; - const payload = receiptReactionPayload(action.payload); - if (!payload) { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "receipt_reaction"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { - code: "receipt_reaction_payload_invalid", - retryable: false, - }, - updatedAt: new Date(), - }) - .where(eq(chatActions.id, action.id)); - }); - return; - } - if (payload.operation === "add" && action.deliveryId) { - const terminalRemoval = await db - .select({ id: chatActions.id }) - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, action.endpointId), - eq( - chatActions.providerActionId, - `receipt_reaction_remove:${action.deliveryId}`, - ), - ), - ) - .then((rows) => rows[0] ?? null); - if (terminalRemoval) { - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "receipt_reaction"); - await tx - .update(chatActions) - .set({ - status: "cancelled", - result: { - ...(typeof action!.result?.attempts === "number" - ? { attempts: action!.result.attempts } - : {}), - code: "receipt_reaction_superseded_by_terminal_publication", - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, action!.status), - ), - ); - await credentialLease.assertOwned(tx); - }); - return; - } - } - if (action.status === "processing") { - if (action.updatedAt > new Date(Date.now() - PROVIDER_EFFECT_STALE_MS)) - return; - // Adding or removing the same reaction is idempotent. A worker - // interruption can be retried safely without an ambiguous - // duplicate-message boundary. - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "receipt_reaction"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { - ...(action.result ?? {}), - retryable: true, - retryAt: new Date().toISOString(), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - eq(chatActions.updatedAt, action.updatedAt), - ), - ); - }); - action = await db - .select() - .from(chatActions) - .where(eq(chatActions.id, action.id)) - .then((rows) => rows[0] ?? null); - if (!action) return; - } - if (action.status === "failed") { - if (action.result?.retryable !== true) return; - const retryAt = - typeof action.result.retryAt === "string" - ? new Date(action.result.retryAt) - : null; - if (retryAt && retryAt > new Date()) return; - } else if (action.status !== "received") { - return; - } - const attempt = - (typeof action.result?.attempts === "number" ? action.result.attempts : 0) + - 1; - const retainedGitHubReceipt = parseGitHubReceiptIdentity( - action.result?.githubReceipt, - ); - const [claimed] = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "receipt_reaction"); - const rows = await tx - .update(chatActions) - .set({ - status: "processing", - result: { - attempts: attempt, - ...(retainedGitHubReceipt - ? { githubReceipt: retainedGitHubReceipt } - : {}), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, action!.status), - ), - ) - .returning(); - await credentialLease.assertOwned(tx); - return rows; - }); - if (!claimed) return; - - let reactionProvider: ChatProvider | null = null; - let githubReceipt: GitHubReceiptIdentity | null = retainedGitHubReceipt; - try { - const record = await endpointRecord(action.endpointId); - if ( - !record || - !["verifying", "active"].includes(record.endpoint.status) || - runtimeGeneration(record.endpoint.setup) !== payload.runtimeGeneration || - credentialFingerprint(record.credentialSecretRefs) !== - payload.credentialFingerprint - ) { - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "receipt_reaction"); - await tx - .update(chatActions) - .set({ - status: "cancelled", - result: { - attempts: attempt, - code: "receipt_reaction_runtime_superseded", - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ); - await credentialLease.assertOwned(tx); - }); - return; - } - reactionProvider = record.endpoint.provider; - // Resolve the adapter only after acquiring the credential lease and - // proving the action's durable generation/fingerprint is still current. - // Never reuse the live callback's adapter: it may hold an installation - // token minted by a superseded private key. - const endpointRuntime = await runtimeFor(record.endpoint); - const adapter = endpointRuntime.thread(payload.threadId).adapter; - await credentialLease.assertOwned(); - if (record.endpoint.provider === "slack") { - await endpointRuntime.applySlackReceiptReaction(payload, fetchImpl); - } else if (record.endpoint.provider === "github") { - const assertCurrent = async () => { - await credentialLease.assertOwned(); - const [current] = await db - .select({ endpoint: chatEndpoints, connection: toolConnections }) - .from(chatEndpoints) - .innerJoin( - toolConnections, - and( - eq(toolConnections.id, chatEndpoints.connectionId), - eq(toolConnections.companyId, chatEndpoints.companyId), - ), - ) - .where( - and( - eq(chatEndpoints.id, action!.endpointId), - eq(chatEndpoints.companyId, action!.companyId), - ), - ) - .limit(1); - if ( - !current || - !["active", "verifying"].includes(current.endpoint.status) || - !current.connection.enabled || - current.connection.status !== "active" || - runtimeGeneration(current.endpoint.setup) !== - payload.runtimeGeneration || - credentialFingerprint(current.connection.credentialSecretRefs) !== - payload.credentialFingerprint || - runtime.get(action!.endpointId) !== endpointRuntime - ) - throw new Error("GitHub receipt authority is no longer current"); - }; - const applied = await endpointRuntime.applyGitHubReceiptReaction( - { ...payload, ...(githubReceipt ? { githubReceipt } : {}) }, - assertCurrent, - fetchImpl, - ); - githubReceipt = parseGitHubReceiptIdentity(applied); - if ( - !githubReceipt || - (payload.operation === "add" && !githubReceipt.reactionId) - ) - throw new Error("GitHub receipt identity was not confirmed"); - } else if (payload.operation === "remove") { - await adapter.removeReaction( - payload.threadId, - payload.messageId, - payload.reaction, - ); - } else { - await adapter.addReaction( - payload.threadId, - payload.messageId, - payload.reaction, - ); - } - // The provider may already have accepted the idempotent reaction. Fence - // the local success receipt so a reclaimed lease cannot report success - // for an obsolete runtime. - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "receipt_reaction"); - const processed = await tx - .update(chatActions) - .set({ - status: "processed", - result: { - attempts: attempt, - ...(githubReceipt ? { githubReceipt } : {}), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!processed.length) { - throw new Error( - "Receipt-reaction ownership changed before success settlement", - ); - } - if (action.deliveryId) { - await tx - .update(chatDeliveries) - .set({ redactedError: null, updatedAt: new Date() }) - .where( - and( - eq(chatDeliveries.id, action.deliveryId), - eq(chatDeliveries.state, "processed"), - like(chatDeliveries.redactedError, "Receipt reaction %"), - ), - ); - } - await credentialLease.assertOwned(tx); - }); - } catch (error) { - if ( - payload.operation === "add" && - reactionProvider === "slack" && - isSlackReceiptReactionAlreadyApplied(error) - ) { - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "receipt_reaction"); - const processed = await tx - .update(chatActions) - .set({ - status: "processed", - result: { - attempts: attempt, - code: "receipt_reaction_already_applied", - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!processed.length) { - throw new Error( - "Receipt-reaction ownership changed before idempotent settlement", - ); - } - if (action.deliveryId) { - await tx - .update(chatDeliveries) - .set({ redactedError: null, updatedAt: new Date() }) - .where( - and( - eq(chatDeliveries.id, action.deliveryId), - eq(chatDeliveries.state, "processed"), - like(chatDeliveries.redactedError, "Receipt reaction %"), - ), - ); - } - await credentialLease.assertOwned(tx); - }); - return; - } - const disposition = classifyChatPublicationError(error, attempt); - const retryable = - disposition.kind === "retry" || - (disposition.kind === "delivery_unknown" && attempt < 3); - const retryAfterMs = - disposition.kind === "retry" - ? disposition.retryAfterMs - : 100 * 2 ** Math.max(0, attempt - 1); - const operationLabel = - payload.operation === "remove" ? " reaction removal" : " reaction"; - const diagnostic = - `Receipt${operationLabel} ${retryable ? "deferred" : "failed"} after ${attempt} attempt${attempt === 1 ? "" : "s"} (${disposition.kind}): ${redactError(error)}`.slice( - 0, - MAX_ERROR_TEXT, - ); - const retryAt = retryable - ? new Date(Date.now() + retryAfterMs).toISOString() - : null; - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - await notifyChatActionWork(tx, "receipt_reaction"); - const [failed] = await tx - .update(chatActions) - .set({ - status: "failed", - result: { - attempts: attempt, - ...(githubReceipt ? { githubReceipt } : {}), - code: - payload.operation === "remove" - ? `receipt_reaction_removal_${disposition.kind}` - : `receipt_reaction_${disposition.kind}`, - redactedError: diagnostic, - retryable, - retryAt, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (failed && action.deliveryId) { - await tx - .update(chatDeliveries) - .set({ redactedError: diagnostic, updatedAt: new Date() }) - .where( - and( - eq(chatDeliveries.id, action.deliveryId), - eq(chatDeliveries.state, "processed"), - ), - ); - } - await credentialLease.assertOwned(tx); - }); - logger.warn( - { - endpointId: action.endpointId, - deliveryId: action.deliveryId, - attempts: attempt, - disposition: disposition.kind, - operation: payload.operation, - retryAt, - }, - "external chat receipt reaction deferred", - ); - } - } - - async function processPendingReceiptReactions( - limit = 25, - onlyActionId?: string, - ) { - const actions = await db.select({ id: chatActions.id }).from(chatActions) - .where(and(dueChatAction("receipt_reaction"), onlyActionId ? eq(chatActions.id, onlyActionId) : undefined)) - .orderBy(asc(chatActions.createdAt)).limit(limit); - for (const action of actions) await processReceiptReaction(action.id); - return actions.length; - } - - async function stageProviderEffect( - database: DbOrTransaction, - input: { - endpoint: EndpointRow; - deliveryId?: string | null; - conversationId?: string | null; - principalId?: string | null; - providerActionId: string; - payload: Omit< - ProviderEffectPayload, - "runtimeGeneration" | "credentialFingerprint" - >; - runtimeContext: LifecycleRuntimeFence; - }, - ) { - await notifyChatActionWork(database, "provider_effect"); - const [inserted] = await database - .insert(chatActions) - .values({ - companyId: input.endpoint.companyId, - endpointId: input.endpoint.id, - deliveryId: input.deliveryId ?? null, - conversationId: input.conversationId ?? null, - principalId: input.principalId ?? null, - kind: "provider_effect", - providerActionId: input.providerActionId, - payload: { - ...input.payload, - runtimeGeneration: input.runtimeContext.generation, - credentialFingerprint: input.runtimeContext.credentialFingerprint, - }, - status: "received", - }) - .onConflictDoNothing() - .returning(); - if (inserted) return inserted; - return database - .select() - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, input.endpoint.id), - eq(chatActions.providerActionId, input.providerActionId), - eq(chatActions.kind, "provider_effect"), - ), - ) - .then((rows) => rows[0] ?? null); - } - async function stageAuthorizedTaskControlPublication( tx: DbOrTransaction, input: { @@ -3145,809 +2017,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return publication; } - function providerEffectDisposition( - error: unknown, - attempt: number, - providerEffectCompleted: boolean, - ) { - if (!providerEffectCompleted) - return classifyChatPublicationError(error, attempt); - return { - kind: "delivery_unknown" as const, - reason: - "Provider effect completed, but Paperclip could not confirm its durable result", - }; - } - - async function finalizeProviderEffectFailure(input: { - action: typeof chatActions.$inferSelect; - endpoint: EndpointRow; - payload: ProviderEffectPayload; - disposition: ReturnType; - attempt: number; - }): Promise { - const failure = redactSensitiveText(input.disposition.reason).slice( - 0, - MAX_ERROR_TEXT, - ); - const retryable = input.disposition.kind === "retry"; - const retryAt = - input.disposition.kind === "retry" - ? new Date(Date.now() + input.disposition.retryAfterMs) - : null; - let invalidateCurrentRuntime = false; - const finalized = await db.transaction(async (tx) => { - // Admission locks the endpoint before its delivery. Keep that order - // when failure settlement also needs to change endpoint/resource health. - const currentEndpoint = - input.disposition.kind === "endpoint_attention" || - input.disposition.kind === "resource_unavailable" - ? await runtimeCallbackEndpoint( - tx, - input.endpoint.id, - { - generation: input.payload.runtimeGeneration, - credentialFingerprint: input.payload.credentialFingerprint, - }, - ["verifying", "active"], - ) - : null; - await notifyChatActionWork(tx, "provider_effect"); - const [ownedAction] = await tx - .update(chatActions) - .set({ - status: - input.disposition.kind === "delivery_unknown" - ? "delivery_unknown" - : "failed", - result: { - code: `provider_effect_${input.disposition.kind}`, - attempts: input.attempt, - redactedError: failure, - retryable, - retryAt: retryAt?.toISOString() ?? null, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, input.action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${input.attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!ownedAction) return false; - if (input.payload.settleDelivery && input.action.deliveryId) { - await notifyChatDeliveryWork(tx); - await tx - .update(chatDeliveries) - .set({ - state: retryable ? "retry" : "failed", - nextAttemptAt: retryAt, - redactedError: - input.disposition.kind === "delivery_unknown" - ? `Provider response is unconfirmed: ${failure}`.slice( - 0, - MAX_ERROR_TEXT, - ) - : failure, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, input.action.deliveryId), - eq(chatDeliveries.state, "processing"), - ), - ); - } - if (input.disposition.kind === "endpoint_attention" && currentEndpoint) { - await tx - .update(chatEndpoints) - .set({ - status: "attention", - healthMessage: "Provider credentials or permissions need attention", - lastError: failure, - updatedAt: new Date(), - }) - .where(eq(chatEndpoints.id, currentEndpoint.id)); - await tx - .update(toolConnections) - .set({ - status: "disabled", - enabled: false, - healthStatus: "degraded", - healthMessage: "Provider credentials or permissions need attention", - lastError: failure, - healthCheckedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(toolConnections.id, currentEndpoint.connectionId)); - invalidateCurrentRuntime = true; - } else if ( - input.disposition.kind === "resource_unavailable" && - input.payload.resourceId && - currentEndpoint - ) { - await tx - .update(chatEndpointResources) - .set({ availability: "unavailable", updatedAt: new Date() }) - .where( - and( - eq(chatEndpointResources.id, input.payload.resourceId), - eq(chatEndpointResources.endpointId, currentEndpoint.id), - ), - ); - } - return true; - }); - if (finalized && invalidateCurrentRuntime) { - await invalidateRuntime(input.endpoint.id).catch(() => undefined); - } - return finalized; - } - - async function quarantineStaleProviderEffect( - action: typeof chatActions.$inferSelect, - ) { - const payload = providerEffectPayload(action.payload); - const failure = - "Provider delivery could not be confirmed after the worker stopped. The effect will not be replayed automatically."; - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "provider_effect"); - const [quarantined] = await tx - .update(chatActions) - .set({ - status: "delivery_unknown", - result: { - code: "provider_effect_delivery_unknown", - redactedError: failure, - retryable: false, - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - eq(chatActions.updatedAt, action.updatedAt), - sql`(${chatActions.result}->>'attempts')::int = ${ - typeof action.result?.attempts === "number" - ? action.result.attempts - : -1 - }`, - lte( - chatActions.updatedAt, - new Date(Date.now() - PROVIDER_EFFECT_STALE_MS), - ), - ), - ) - .returning({ id: chatActions.id }); - if (quarantined && payload?.settleDelivery && action.deliveryId) { - await notifyChatDeliveryWork(tx); - await tx - .update(chatDeliveries) - .set({ - state: "failed", - nextAttemptAt: null, - redactedError: failure, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, action.deliveryId), - eq(chatDeliveries.state, "processing"), - ), - ); - } - }); - } - - function providerEffectThreadResourceId( - provider: ChatProvider, - threadId: string, - ): string | null { - if (provider === "slack") { - return ( - slackThreadChannelId(threadId) ?? - (!threadId.includes(":") ? threadId : null) - ); - } - if (provider === "telegram") { - return /^telegram:([^:]+)/.exec(threadId)?.[1] ?? null; - } - if (provider === "github") { - return /^github:([^:]+)/.exec(threadId)?.[1]?.toLowerCase() ?? null; - } - if (provider === "discord") { - return /^discord:[^:]+:([^:]+)/.exec(threadId)?.[1] ?? null; - } - const conversationId = teamsConversationId(threadId); - return conversationId ? baseTeamsConversationId(conversationId) : null; - } - - // This is permission to return one fixed, neutral callback denial, not - // permission to execute the denied action or publish to the conversation. - // Retain the initial identity-policy snapshot so queued notices cannot cross - // a subsequent actor/link/reach change. No provider I/O occurs under locks. - async function lockTelegramCallbackNoticeAuthority( - tx: DbTransaction, - endpoint: EndpointRow, - receipt: TelegramCallbackReceipt, - principalId: string | null, - ): Promise { - if ( - endpoint.provider !== "telegram" || - endpoint.status !== "active" || - endpoint.companyId !== receipt.companyId || - endpoint.id !== receipt.endpointId || - endpoint.botExternalId !== receipt.botUserId || - !endpoint.providerAccountId || - !principalId || - Date.now() < receipt.receivedAtMs || - Date.now() >= receipt.deadlineAtMs - ) - return null; - const resource = - receipt.chatType === "private" - ? null - : await tx - .select() - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.companyId, endpoint.companyId), - eq(chatEndpointResources.endpointId, endpoint.id), - eq(chatEndpointResources.providerResourceId, receipt.chatId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if ( - receipt.chatType === "private" - ? !endpoint.allowDirectMessages - : !resource || !nonDirectDestinationAllowed(endpoint, resource) - ) - return null; - const principal = await tx - .select() - .from(chatExternalPrincipals) - .where( - and( - eq(chatExternalPrincipals.companyId, endpoint.companyId), - eq(chatExternalPrincipals.provider, endpoint.provider), - eq( - chatExternalPrincipals.providerAccountId, - endpoint.providerAccountId, - ), - eq(chatExternalPrincipals.id, principalId), - eq(chatExternalPrincipals.externalId, receipt.receiverUserId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if (!principal || principal.kind !== "user" || principal.isBot) return null; - const authorization = await lockCurrentPrincipalAuthorization( - tx, - endpoint, - principal.id, - ); - const link = await tx - .select({ - status: chatIdentityLinks.status, - userId: chatIdentityLinks.paperclipUserId, - }) - .from(chatIdentityLinks) - .where( - and( - eq(chatIdentityLinks.companyId, endpoint.companyId), - eq(chatIdentityLinks.endpointId, endpoint.id), - eq(chatIdentityLinks.principalId, principal.id), - ), - ) - .then((rows) => rows[0] ?? null); - return createHash("sha256") - .update( - JSON.stringify([ - endpoint.companyId, - endpoint.id, - receipt.botUserId, - resource?.id ?? null, - principal.id, - principal.externalId, - link, - authorization, - endpoint.allowUnlinkedPeople, - endpoint.sponsorUserId, - ]), - ) - .digest("hex"); - } - - async function lockCurrentProviderEffectAuthorization( - tx: DbTransaction, - action: typeof chatActions.$inferSelect, - endpoint: EndpointRow, - payload: ProviderEffectPayload, - ): Promise { - if (payload.effect === "telegram_callback_notice") { - const receipt = parseTelegramCallbackReceipt(payload.telegramCallback); - if (!receipt || !action.deliveryId) return false; - const delivery = await tx - .select() - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.id, action.deliveryId), - eq(chatDeliveries.companyId, action.companyId), - eq(chatDeliveries.endpointId, action.endpointId), - eq(chatDeliveries.state, "filtered"), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - const retained = - delivery && - parseTelegramCallbackReceipt(delivery.normalizedEvent.telegramCallback); - if ( - !delivery || - delivery.principalId !== action.principalId || - retained?.sourceSha256 !== receipt.sourceSha256 || - delivery.normalizedEvent.telegramAuthorizationSha256 !== - payload.telegramAuthorizationSha256 - ) - return false; - return ( - (await lockTelegramCallbackNoticeAuthority( - tx, - endpoint, - receipt, - action.principalId, - )) === payload.telegramAuthorizationSha256 - ); - } - const conversation = action.conversationId - ? await tx - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.companyId, endpoint.companyId), - eq(chatConversations.endpointId, endpoint.id), - eq(chatConversations.id, action.conversationId), - inArray(chatConversations.state, ["active", "waiting"]), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - if (action.conversationId && !conversation) return false; - const inferredResourceId = providerEffectThreadResourceId( - endpoint.provider, - payload.threadId, - ); - const resource = await tx - .select() - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.companyId, endpoint.companyId), - eq(chatEndpointResources.endpointId, endpoint.id), - payload.resourceId || conversation?.resourceId - ? eq( - chatEndpointResources.id, - payload.resourceId ?? conversation!.resourceId!, - ) - : inferredResourceId - ? eq(chatEndpointResources.providerResourceId, inferredResourceId) - : sql`false`, - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - const slackChannelId = - endpoint.provider === "slack" - ? slackThreadChannelId(payload.threadId) - : null; - const isDirectMessage = - conversation?.isDirectMessage === true || - resource?.type === "direct_message" || - Boolean(slackChannelId && /^D[A-Z0-9]+$/i.test(slackChannelId)); - const destinationAllowed = isDirectMessage - ? endpoint.allowDirectMessages - : nonDirectDestinationAllowed(endpoint, resource); - if (payload.authorizationMode === "safe_notice") return true; - if (!destinationAllowed) return false; - if (!action.principalId) return false; - return ( - await lockCurrentPrincipalAuthorization(tx, endpoint, action.principalId) - ).allowed; - } - - async function cancelUnauthorizedProviderEffect( - tx: DbTransaction, - action: typeof chatActions.$inferSelect, - attempt: number, - ): Promise { - await notifyChatActionWork(tx, "provider_effect"); - const [cancelled] = await tx - .update(chatActions) - .set({ - status: "cancelled", - result: { - attempts: attempt, - code: "provider_effect_no_longer_authorized", - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!cancelled) return false; - if (action.deliveryId) { - await notifyChatDeliveryWork(tx); - await tx - .update(chatDeliveries) - .set({ - state: "filtered", - processedAt: new Date(), - nextAttemptAt: null, - redactedError: - "Provider reply was suppressed because current chat authorization changed", - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, action.deliveryId), - eq(chatDeliveries.state, "processing"), - ), - ); - } - return true; - } - - async function processProviderEffect( - actionId: string, - liveTarget?: ProviderEffectTarget, - ): Promise<"processed" | "pending" | "failed" | "delivery_unknown"> { - const initial = await db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.id, actionId), - eq(chatActions.kind, "provider_effect"), - ), - ) - .then((rows) => rows[0] ?? null); - if (!initial) return "failed"; - const initialEndpoint = await endpointRecord(initial.endpointId); - if (!initialEndpoint) return "failed"; - - const processClaimedEffect = async () => { - let action = await db - .select() - .from(chatActions) - .where(eq(chatActions.id, actionId)) - .then((rows) => rows[0] ?? null); - if (!action) return "failed"; - const payload = providerEffectPayload(action.payload); - if (!payload) { - await db.transaction(async (tx) => { - await notifyChatActionWork(tx, "provider_effect"); - return tx - .update(chatActions) - .set({ - status: "failed", - result: { code: "provider_effect_payload_invalid" }, - updatedAt: new Date(), - }) - .where(eq(chatActions.id, action.id)); - }); - return "failed"; - } - if (action.status === "processed") return "processed"; - if (action.status === "delivery_unknown") return "delivery_unknown"; - if (action.status === "processing") { - if (action.updatedAt > new Date(Date.now() - PROVIDER_EFFECT_STALE_MS)) - return "pending"; - await quarantineStaleProviderEffect(action); - return "delivery_unknown"; - } - if (action.status === "failed") { - if (action.result?.retryable !== true) return "failed"; - const retryAt = - typeof action.result.retryAt === "string" - ? new Date(action.result.retryAt) - : null; - if (retryAt && retryAt > new Date()) return "pending"; - } else if (action.status !== "received") { - return "failed"; - } - - const attempt = - (typeof action.result?.attempts === "number" - ? action.result.attempts - : 0) + 1; - const record = await endpointRecord(action.endpointId); - let providerEffectCompleted = false; - try { - if (!record) { - throw Object.assign( - new Error("Chat endpoint is unavailable for this provider effect"), - { code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED" }, - ); - } - return await withCredentialMutationLease( - record.endpoint, - async (credentialLease) => { - const claim = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const currentAction = await tx - .select() - .from(chatActions) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.kind, "provider_effect"), - eq(chatActions.status, action!.status), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - if (!currentAction) return null; - const currentEndpoint = await runtimeCallbackEndpoint( - tx, - currentAction.endpointId, - { - generation: payload.runtimeGeneration, - credentialFingerprint: payload.credentialFingerprint, - }, - ["verifying", "active"], - ); - const authorized = currentEndpoint - ? await lockCurrentProviderEffectAuthorization( - tx, - currentAction, - currentEndpoint, - payload, - ) - : false; - await notifyChatActionWork(tx, "provider_effect"); - const [transportClaim] = await tx - .update(chatActions) - .set({ - status: "processing", - result: { attempts: attempt }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, currentAction.id), - eq(chatActions.status, currentAction.status), - ), - ) - .returning(); - if (!transportClaim) return null; - if (!currentEndpoint || !authorized) { - const cancelled = await cancelUnauthorizedProviderEffect( - tx, - transportClaim, - attempt, - ); - if (!cancelled) { - throw new Error( - "Provider effect ownership changed before authorization cancellation", - ); - } - return { outcome: "cancelled" as const }; - } - await credentialLease.assertOwned(tx); - return { - action: transportClaim, - endpoint: currentEndpoint, - outcome: "claimed" as const, - }; - }); - if (!claim) return "pending"; - if (claim.outcome === "cancelled") return "failed"; - action = claim.action; - - // `processing` is the durable, single-owner transport claim and - // authorization linearization point. Its transaction has committed - // before provider I/O begins. The renewable credential lease keeps - // runtime identity stable without holding database row locks. - let sent: { id: string; threadId: string } | null = null; - if (payload.effect === "telegram_callback_notice") { - sent = await ( - await runtimeFor(claim.endpoint) - ).sendTelegramCallbackNotice( - payload.telegramCallback!, - payload.text, - ); - } else { - const target = - liveTarget ?? - (await runtimeFor(claim.endpoint)).thread(payload.threadId); - if (payload.effect === "ephemeral_message") { - if (!target.postEphemeral) { - if (!payload.fallbackText) { - throw Object.assign( - new Error("Provider does not support ephemeral messages"), - { code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED" }, - ); - } - sent = await target.post(payload.fallbackText); - } else { - try { - sent = await target.postEphemeral( - payload.userId!, - payload.text, - { fallbackToDM: false }, - ); - } catch (error) { - if ( - !payload.fallbackText || - classifyChatPublicationError(error, attempt).kind !== - "failed" - ) { - throw error; - } - } - if (!sent && payload.fallbackText) { - sent = await target.post(payload.fallbackText); - } - } - } else if (claim.endpoint.provider === "imessage-photon") { - const adapter = (await runtimeFor(claim.endpoint)).getProviderAdapter(); - if (!(adapter instanceof PhotonChatAdapter)) throw new Error("Photon runtime unavailable"); - const receipt = await adapter.publish(payload.threadId, `effect:${action.id}`, projectSafeChatPublicationText(payload.text), { - assertCurrent: async () => { await credentialLease.assertOwned(); }, - }); - sent = { id: receipt.id, threadId: payload.threadId }; - } else { - sent = await target.post(payload.text); - } - } - if (!sent) { - throw Object.assign( - new Error("Provider did not accept the message"), - { code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED" }, - ); - } - providerEffectCompleted = true; - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - // A duplicate webhook holds the endpoint while waiting for the - // delivery's unique-index entry. Lock this endpoint first too, - // before settling the action/conversation/delivery, so neither - // transaction can wait on a row already held by the other. - const currentEndpoint = await runtimeCallbackEndpoint( - tx, - claim.endpoint.id, - { - generation: payload.runtimeGeneration, - credentialFingerprint: payload.credentialFingerprint, - }, - ["verifying", "active"], - ); - await notifyChatActionWork(tx, "provider_effect"); - const [completed] = await tx - .update(chatActions) - .set({ - status: "processed", - result: { - attempts: attempt, - providerMessageId: sent.id, - threadId: sent.threadId, - ...(payload.effect === "telegram_callback_notice" - ? { - code: - payload.telegramCallback?.chatType === "private" - ? "telegram_private_api_accepted" - : "telegram_ephemeral_api_accepted", - } - : {}), - }, - updatedAt: new Date(), - }) - .where( - and( - eq(chatActions.id, action!.id), - eq(chatActions.status, "processing"), - sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, - ), - ) - .returning({ id: chatActions.id }); - if (!completed) { - throw new Error( - "Provider effect ownership changed before commit", - ); - } - if (payload.completeConversationId) { - await tx - .update(chatConversations) - .set({ state: "completed", updatedAt: new Date() }) - .where( - and( - eq(chatConversations.id, payload.completeConversationId), - eq(chatConversations.endpointId, claim.endpoint.id), - inArray(chatConversations.state, ["active", "waiting"]), - ), - ); - } - if (payload.settleDelivery && action!.deliveryId) { - await notifyChatDeliveryWork(tx); - const [settled] = await tx - .update(chatDeliveries) - .set({ - state: "processed", - processedAt: new Date(), - nextAttemptAt: null, - redactedError: null, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, action!.deliveryId), - eq(chatDeliveries.state, "processing"), - ), - ) - .returning({ id: chatDeliveries.id }); - if (!settled) { - throw new Error( - "Provider effect delivery ownership changed before commit", - ); - } - } - if (currentEndpoint) { - await tx - .update(chatEndpoints) - .set({ lastEventAt: new Date(), updatedAt: new Date() }) - .where(eq(chatEndpoints.id, currentEndpoint.id)); - } - await credentialLease.assertOwned(tx); - }); - return "processed"; - }, - ); - } catch (error) { - const disposition = providerEffectDisposition( - error, - attempt, - providerEffectCompleted, - ); - try { - await finalizeProviderEffectFailure({ - action, - endpoint: record?.endpoint ?? initialEndpoint.endpoint, - payload, - disposition, - attempt, - }); - } catch (finalizationError) { - if (disposition.kind === "delivery_unknown") { - throw Object.assign( - new Error( - "Provider effect is ambiguous and its durable quarantine failed", - { cause: finalizationError }, - ), - { code: "CHAT_PROVIDER_EFFECT_AMBIGUOUS" }, - ); - } - throw finalizationError; - } - throw error; - } - }; - return processClaimedEffect(); - } - function scheduleProviderEffect( actionId: string, liveTarget?: ProviderEffectTarget, @@ -3957,36 +2026,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } - async function processPendingProviderEffects(limit = 25) { - const now = new Date(); - const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); - const actions = await db - .select() - .from(chatActions) - .where(dueChatAction("provider_effect", now.getTime())) - .orderBy(asc(chatActions.createdAt)) - .limit(limit); - for (const action of actions) { - if (action.status === "processing" && action.updatedAt <= staleBefore) { - await quarantineStaleProviderEffect(action); - continue; - } - try { - await processProviderEffect(action.id); - } catch (error) { - logger.warn( - { - endpointId: action.endpointId, - actionId: action.id, - error: redactError(error), - }, - "chat provider effect reconciliation failed", - ); - } - } - return actions.length; - } - async function runtimeCallbackEndpoint( tx: DbTransaction, endpointId: string, @@ -4102,212 +2141,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } - function discordGatewayFailureDetail( - event: DiscordGatewayCallbackEvent["event"], - ): string { - if (event.type === "guild_removed") { - return "Discord bot was removed from the configured server"; - } - if (event.type === "guild_unavailable") { - return "Discord server is temporarily unavailable; reconnecting"; - } - if (event.type === "disconnected") { - return `Discord Gateway disconnected${event.code === undefined ? "" : ` (code ${event.code})`}; reconnecting`; - } - if (event.type === "failure") { - const code = event.error.code ?? event.error.status; - return `Discord Gateway failed${code === undefined ? "" : ` (${String(code).slice(0, 64)})`}; ${event.fatal ? "credentials or privileged intents need attention" : "reconnecting"}`; - } - return "Discord Gateway connection needs attention"; - } - - function handleDiscordGatewayEvent( - callback: DiscordGatewayCallbackEvent, - context: RuntimeContext, - ): Promise { - const pending = (context.discordGatewayEventQueue ?? Promise.resolve()) - .catch(() => undefined) - .then(() => processDiscordGatewayEvent(callback, context)); - context.discordGatewayEventQueue = pending; - return pending; - } - - async function processDiscordGatewayEvent( - callback: DiscordGatewayCallbackEvent, - context: RuntimeContext, - ): Promise { - if ( - !(await ensureDiscordGatewayRuntimeIsCurrent( - callback.endpointId, - context, - )) - ) - return; - if (callback.sequence <= (context.discordGatewaySequence ?? 0)) return; - context.discordGatewaySequence = callback.sequence; - await options.discordGatewayEventBarrier?.(callback); - const record = await runtimeCallbackRecord(callback.endpointId, context, [ - "verifying", - "active", - "attention", - ]); - if (!record || record.endpoint.provider !== "discord") return; - const event = callback.event; - if (event.type === "channel_removed") { - await applyProviderLifecycleEffect( - record.endpoint, - { - kind: "resource", - provider: "discord", - providerEventId: `gateway:channel_removed:${event.channelId}`, - providerResourceId: event.channelId, - resourceType: "channel", - label: event.label?.trim() || event.channelId, - availability: "removed", - metadata: { source: "discord_gateway" }, - }, - context, - ); - return; - } - - await db.transaction(async (tx) => { - const current = await runtimeCallbackEndpoint( - tx, - callback.endpointId, - context, - ["verifying", "active", "attention"], - ); - if ( - !current || - current.provider !== "discord" || - !discordGatewayRuntimeIsCurrent(callback.endpointId, context) - ) { - return; - } - const now = new Date(); - if (event.type === "connecting") { - await tx - .update(chatEndpoints) - .set({ - healthMessage: "Discord Gateway connecting", - lastError: null, - lastEventAt: now, - updatedAt: now, - }) - .where(eq(chatEndpoints.id, current.id)); - await tx - .update(toolConnections) - .set({ - healthStatus: "unchecked", - healthMessage: "Discord Gateway connecting", - lastError: null, - healthCheckedAt: now, - updatedAt: now, - }) - .where(eq(toolConnections.id, current.connectionId)); - return; - } - if (event.type === "ready" || event.type === "guild_available") { - const gatewayOwnedAttention = - current.status === "attention" && - current.healthMessage?.startsWith("Discord ") === true; - if (current.status === "attention" && !gatewayOwnedAttention) return; - const status = - current.setup.step === "complete" ? "active" : "verifying"; - if (status !== current.status) await notifyChatEndpointWork(tx); - await tx - .update(chatEndpoints) - .set({ - status, - healthMessage: - status === "active" - ? "Connected" - : "Waiting for a test conversation", - lastError: null, - lastEventAt: now, - updatedAt: now, - }) - .where(eq(chatEndpoints.id, current.id)); - await tx - .update(toolConnections) - .set({ - status: "active", - enabled: true, - healthStatus: "healthy", - healthMessage: "Connected", - lastError: null, - healthCheckedAt: now, - updatedAt: now, - }) - .where(eq(toolConnections.id, current.connectionId)); - return; - } - - const detail = discordGatewayFailureDetail(event); - const transientGuildOutage = event.type === "guild_unavailable"; - const terminal = - event.type === "guild_removed" || - ((event.type === "failure" || event.type === "disconnected") && - event.fatal); - await tx - .update(chatEndpoints) - .set({ - status: transientGuildOutage - ? current.status - : event.type === "guild_removed" - ? "revoked" - : "attention", - healthMessage: detail, - lastError: detail, - lastEventAt: now, - ...(terminal - ? { - setup: { - ...current.setup, - runtimeGeneration: runtimeGeneration(current.setup) + 1, - } as InternalSetupState, - } - : {}), - updatedAt: now, - }) - .where(eq(chatEndpoints.id, current.id)); - await tx - .update(toolConnections) - .set({ - status: terminal ? "disabled" : "active", - enabled: !terminal, - healthStatus: event.type === "guild_removed" ? "failed" : "degraded", - healthMessage: detail, - lastError: detail, - healthCheckedAt: now, - updatedAt: now, - }) - .where(eq(toolConnections.id, current.connectionId)); - if (event.type === "guild_removed") { - await tx - .update(chatEndpointResources) - .set({ availability: "unavailable", updatedAt: now }) - .where( - and( - eq(chatEndpointResources.companyId, current.companyId), - eq(chatEndpointResources.endpointId, current.id), - ), - ); - await tx - .update(chatConversations) - .set({ state: "unavailable", updatedAt: now }) - .where( - and( - eq(chatConversations.companyId, current.companyId), - eq(chatConversations.endpointId, current.id), - inArray(chatConversations.state, ["active", "waiting"]), - ), - ); - } - }); - } - function serializeEndpoint( row: Awaited>, ): ChatEndpoint { @@ -5270,6 +3103,16 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const withCredentialMutationLease = chatCredentialMutationLease(db, options); + const { setupTestStatus, test } = createSetupProbes(db, { endpointRecord, withCredentialMutationLease, fetchImpl, findAuthorizedIdentityLink, runtimeGeneration, lockCurrentPrincipalAuthorization, get, setupTestActivationBarrier: () => options.setupTestActivationBarrier?.() ?? Promise.resolve() }); + const { listConversations, listActivity, listActivityPage } = createChatActivity(db, { endpointRecord, teamsTransferProtocol }); + const { lockTelegramCallbackNoticeAuthority } = createTelegramCallbackNoticeAuthority({ nonDirectDestinationAllowed, lockCurrentPrincipalAuthorization }); + const { stageProviderEffect, processProviderEffect, processPendingProviderEffects } = createProviderEffects(db, { endpointRecord, runtimeCallbackEndpoint, withCredentialMutationLease, runtimeFor, invalidateRuntime, lockCurrentPrincipalAuthorization, redactError, MAX_ERROR_TEXT, nonDirectDestinationAllowed, lockTelegramCallbackNoticeAuthority }); + const { publicationFiles } = createOutboundAttachments(db, { get storage() { return options.storage; } }); + const { ingestAttachments, attachmentOmissionDetail } = createInboundAttachments(db, { authorizeInboundWakeup, lifecycleRuntimeFence, sanitizeFilename, issuesSvc, isExternalActionAuthorizationChange, redactError, get storage() { return options.storage; } }); + const { handleReaction, processPendingReactionDeliveries } = createInboundReactions(db, { lockCurrentPrincipalAuthorization, runtimeCallbackRecord, runtimeCallbackEndpoint, ensurePrincipal, durableExternalThreadIdentity, recordCurrentMicrosoftTeamsRoute, lifecycleRuntimeFence, deliveryReady, redactError, nonDirectDestinationAllowed, reactionLinkPreflightBarrier: () => options.reactionLinkPreflightBarrier?.() ?? Promise.resolve(), reactionReplayEndpointLockBarrier: () => options.reactionReplayEndpointLockBarrier?.() ?? Promise.resolve(), reactionReplayConversationLockBarrier: () => options.reactionReplayConversationLockBarrier?.() ?? Promise.resolve() }); + const { addReceiptReaction, processReceiptReaction, processPendingReceiptReactions } = createReceiptReactions(db, { endpointRecord, runtimeContextForRecord, withCredentialMutationLease, runtimeFor, fetchImpl, runtime, runtimeGeneration, credentialFingerprint, redactError, MAX_ERROR_TEXT, receiptReactionTransportBarrier: () => options.receiptReactionTransportBarrier?.() ?? Promise.resolve() }); + const { admitDiscordRootMention, discordRootThreadPending, reconcileDiscordRootThread } = createDiscordRootThreads(db, { ensureDiscordGatewayRuntimeIsCurrent, withCredentialMutationLease, runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, processMessage, redactError, scheduleConversationDrain, lifecycleRuntimeFence, runtimeFor, DISCORD_ROOT_THREAD_CONFIRMATION_DELAY_MS, discordRootThreadTransportBarrier: () => options.discordRootThreadTransportBarrier?.() ?? Promise.resolve() }); + const { handleDiscordGatewayEvent } = createDiscordGatewayEvents(db, { ensureDiscordGatewayRuntimeIsCurrent, discordGatewayRuntimeIsCurrent, runtimeCallbackRecord, runtimeCallbackEndpoint, applyProviderLifecycleEffect, runtimeGeneration, discordGatewayEventBarrier: (event) => options.discordGatewayEventBarrier?.(event) ?? Promise.resolve() }); const { handlePhotonFailure, handlePhotonEvent } = createPhotonEvents(db, { runtimeCallbackRecord, runtimeCallbackEndpoint, invalidateRuntime, ensurePrincipal, requireCurrentExternalActionAuthorization, isExternalActionAuthorizationChange, scheduleMessageProcessing, processPendingPublications, questionResponses, redactError, renewDiscordGatewayOwnershipForMessageAdmission, handleSdkMessage }); const { handleTelegramGenerationStopped, reserveTelegramPublicationDraft } = createTelegramPublicationDrafts(db, { runtimeCallbackRecord, runtimeCallbackEndpoint, runtimeFor, runtime, runtimeContexts, credentialFingerprint, authorizeRetainedChatSourcePublication }); const { recordSlackFileUploadReceipt, stageSlackFilePublicationSuccess, processPendingSlackFileUploadReceipts, nextSlackReceiptAt } = createSlackFileReceipts(db, { endpointRecord, withCredentialMutationLease, runtimeCallbackEndpoint, nonDirectDestinationAllowed, runtimeFor, isExplicitOperatorPublication, recordExactOutboundPublicationLink }); @@ -7143,288 +4986,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { .limit(1).then((rows) => rows[0] ?? null); } - async function setupTestStatus(endpointId: string, userId: string) { - const record = await endpointRecord(endpointId); - if (!record) throw notFound("Chat endpoint not found"); - const startedAt = record.endpoint.setup.testStartedAt; - if (!startedAt) return { messageReceivedAt: null }; - const links = await db.select({ principalId: chatIdentityLinks.principalId }).from(chatIdentityLinks).where(and( - eq(chatIdentityLinks.companyId, record.endpoint.companyId), eq(chatIdentityLinks.endpointId, endpointId), - eq(chatIdentityLinks.paperclipUserId, userId), eq(chatIdentityLinks.status, "linked"), - )); - if (!links.length) return { messageReceivedAt: null }; - const principalIds = links.map((link) => link.principalId); - const [delivery, command] = await Promise.all([ - db.select({ at: chatDeliveries.createdAt }).from(chatDeliveries).where(and( - eq(chatDeliveries.companyId, record.endpoint.companyId), eq(chatDeliveries.endpointId, endpointId), - inArray(chatDeliveries.principalId, principalIds), gte(chatDeliveries.createdAt, new Date(startedAt)), - inArray(chatDeliveries.eventKind, ["message", "mention"]), - )).orderBy(asc(chatDeliveries.createdAt)).limit(1).then((rows) => rows[0]), - db.select({ at: chatActions.createdAt }).from(chatActions).where(and( - eq(chatActions.companyId, record.endpoint.companyId), eq(chatActions.endpointId, endpointId), - inArray(chatActions.principalId, principalIds), eq(chatActions.kind, "slash_task_start"), gte(chatActions.createdAt, new Date(startedAt)), - )).orderBy(asc(chatActions.createdAt)).limit(1).then((rows) => rows[0]), - ]); - const at = [delivery?.at, command?.at].filter((value): value is Date => Boolean(value)).sort((a, b) => a.getTime() - b.getTime())[0]; - return { messageReceivedAt: at?.toISOString() ?? null }; - } - - async function test(endpointId: string, finishOptions?: { optionalSlackTestForUser: string }) { - const initial = await endpointRecord(endpointId); - if (!initial) throw notFound("Chat endpoint not found"); - return withCredentialMutationLease( - initial.endpoint, - async (credentialLease) => { - // Reload only after owning the same lane as reconnect, pause, removal, - // and credential rotation. Qualification and activation must describe - // one exact provider runtime, never a setup snapshot that a concurrent - // reconnect has already superseded. - const record = await endpointRecord(endpointId); - if (!record) throw notFound("Chat endpoint not found"); - const endpoint = record.endpoint; - if (endpoint.status !== "verifying" || endpoint.setup.step !== "test") { - throw conflict("This connection is not waiting for a setup test", { - code: "chat_endpoint_not_testing", - }); - } - if (endpoint.provider === "github" && endpoint.setup.github) { - const verified = await githubChatManagementService(db, fetchImpl).verification(endpoint.id); - if (!verified.ready && !(finishOptions?.optionalSlackTestForUser && verified.connectionReady)) throw conflict("Finish verifying the App, repositories, and assigned agent's tools before activating this bot", { checks: verified.checks }); - } - const testStartedAtValue = endpoint.setup.testStartedAt; - const testStartedAt = testStartedAtValue - ? new Date(testStartedAtValue) - : null; - const optionalSlackTest = Boolean(finishOptions?.optionalSlackTestForUser); - if (optionalSlackTest) { - if (!["slack", "github"].includes(endpoint.provider) || !endpoint.setup.webhookVerifiedAt || !endpoint.providerAccountId) { - throw conflict("Verify the connection before finishing setup"); - } - const linked = await findAuthorizedIdentityLink(endpoint, finishOptions!.optionalSlackTestForUser); - if (!linked) throw forbidden("Connect your account before finishing setup"); - } - if (!optionalSlackTest) { - if ( - !endpoint.lastEventAt || - !testStartedAtValue || - !testStartedAt || - Number.isNaN(testStartedAt.getTime()) || - endpoint.lastEventAt < testStartedAt - ) { - throw conflict( - "Send the test message in the provider before completing setup", - { - code: "chat_test_message_missing", - }, - ); - } - const requiredTrigger = - ["telegram", "imessage-photon"].includes(endpoint.provider) - ? "direct_message" - : "subscribed_message"; - const qualifyingDelivery = await db - .select({ - id: chatDeliveries.id, - conversationId: chatDeliveries.conversationId, - processedAt: chatDeliveries.processedAt, - }) - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.companyId, endpoint.companyId), - eq(chatDeliveries.endpointId, endpoint.id), - eq(chatDeliveries.state, "processed"), - gte(chatDeliveries.processedAt, testStartedAt), - sql`${chatDeliveries.normalizedEvent}->>'trigger' = ${requiredTrigger}`, - ), - ) - .orderBy(desc(chatDeliveries.processedAt)) - .limit(1) - .then((rows) => rows[0] ?? null); - if ( - !qualifyingDelivery?.conversationId || - !qualifyingDelivery.processedAt - ) { - throw conflict( - ["telegram", "imessage-photon"].includes(endpoint.provider) - ? "Send the test direct message before completing setup" - : "Reply once without mentioning the agent before completing setup", - { code: "chat_test_follow_up_missing" }, - ); - } - const finalPublication = await db - .select({ - commentId: chatPublications.commentId, - payload: chatPublications.payload, - }) - .from(chatPublications) - .innerJoin( - issueComments, - and( - eq(issueComments.id, chatPublications.commentId), - eq(issueComments.companyId, chatPublications.companyId), - eq(issueComments.issueId, chatPublications.issueId), - eq(issueComments.authorType, "agent"), - eq(issueComments.authorAgentId, endpoint.assignedAgentId), - ), - ) - .innerJoin( - chatMessageLinks, - and( - eq(chatMessageLinks.companyId, chatPublications.companyId), - eq(chatMessageLinks.endpointId, chatPublications.endpointId), - eq( - chatMessageLinks.conversationId, - chatPublications.conversationId, - ), - eq(chatMessageLinks.deliveryId, qualifyingDelivery.id), - eq(chatMessageLinks.direction, "inbound"), - isNotNull(chatMessageLinks.commentId), - ), - ) - .innerJoin( - heartbeatRuns, - and( - eq(heartbeatRuns.id, issueComments.createdByRunId), - eq(heartbeatRuns.companyId, chatPublications.companyId), - eq(heartbeatRuns.agentId, endpoint.assignedAgentId), - eq(heartbeatRuns.status, "succeeded"), - eq( - sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`, - sql`${chatPublications.issueId}::text`, - ), - or( - sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`, - sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot} ->> 'commentId'`, - sql`coalesce(${heartbeatRuns.contextSnapshot} -> 'wakeCommentIds', '[]'::jsonb) ? ${chatMessageLinks.commentId}::text`, - ), - ), - ) - .where( - and( - eq(chatPublications.companyId, endpoint.companyId), - eq(chatPublications.endpointId, endpoint.id), - eq( - chatPublications.conversationId, - qualifyingDelivery.conversationId, - ), - eq(chatPublications.state, "published"), - gte(chatPublications.publishedAt, qualifyingDelivery.processedAt), - ), - ) - .orderBy(desc(chatPublications.publishedAt)) - .then((rows) => - rows.find( - (row) => - row.payload.interactionId === undefined && - row.payload.progressState === undefined && - row.commentId !== null, - ), - ); - let githubToolReply = false; - if (!finalPublication && endpoint.provider === "github") { - const setupRuns = await db.select({ id: heartbeatRuns.id, issueId: chatConversations.issueId }) - .from(heartbeatRuns) - .innerJoin(chatMessageLinks, and( - eq(chatMessageLinks.companyId, endpoint.companyId), - eq(chatMessageLinks.endpointId, endpoint.id), - eq(chatMessageLinks.deliveryId, qualifyingDelivery.id), - eq(chatMessageLinks.direction, "inbound"), - or( - sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot}->>'wakeCommentId'`, - sql`coalesce(${heartbeatRuns.contextSnapshot}->'wakeCommentIds', '[]'::jsonb) ? ${chatMessageLinks.commentId}::text`, - ), - )) - .innerJoin(chatConversations, and( - eq(chatConversations.id, qualifyingDelivery.conversationId), - eq(chatConversations.id, chatMessageLinks.conversationId), - eq(chatConversations.companyId, endpoint.companyId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${chatConversations.issueId}::text`, - )) - .where(and(eq(heartbeatRuns.companyId, endpoint.companyId), - eq(heartbeatRuns.agentId, endpoint.assignedAgentId), eq(heartbeatRuns.status, "succeeded"))) - .orderBy(desc(heartbeatRuns.createdAt)).limit(20); - for (const run of setupRuns) { - if (await githubRunReplyState(db, { companyId: endpoint.companyId, endpointId: endpoint.id, - issueId: run.issueId, runId: run.id }) === "confirmed") { - githubToolReply = true; - break; - } - } - } - if (!finalPublication && !githubToolReply) { - throw conflict( - "Wait for the Paperclip agent to reply to the setup turn before completing setup", - { - code: "chat_test_round_trip_incomplete", - }, - ); - } - } - await options.setupTestActivationBarrier?.(); - const expectedGeneration = runtimeGeneration(endpoint.setup); - await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - if (optionalSlackTest) { - const linked = await tx.select({ principalId: chatIdentityLinks.principalId }).from(chatIdentityLinks).where(and( - eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.paperclipUserId, finishOptions!.optionalSlackTestForUser), eq(chatIdentityLinks.status, "linked"), - )).limit(1).then((rows) => rows[0]); - if (!linked || !(await lockCurrentPrincipalAuthorization(tx, endpoint, linked.principalId)).allowed) throw forbidden("Your connected account is no longer authorized"); - } - await notifyChatEndpointWork(tx); - const [activated] = await tx - .update(chatEndpoints) - .set({ - status: "active", - setup: { - ...endpoint.setup, - step: "complete", - testStartedAt: null, - testSkipped: optionalSlackTest, - }, - healthMessage: optionalSlackTest ? "Connected; conversation test optional" : "Connected", - activatedAt: endpoint.activatedAt ?? new Date(), - updatedAt: new Date(), - }) - .where( - and( - eq(chatEndpoints.id, endpointId), - eq(chatEndpoints.status, "verifying"), - sql`${chatEndpoints.setup}->>'step' = 'test'`, - sql`coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::integer, 0) = ${expectedGeneration}`, - sql`${chatEndpoints.setup}->>'testStartedAt' = ${testStartedAtValue}`, - ), - ) - .returning({ id: chatEndpoints.id }); - if (!activated) { - throw conflict("This setup test was superseded by a reconnect", { - code: "chat_endpoint_test_superseded", - }); - } - await tx - .update(toolConnections) - .set({ - status: "active", - enabled: true, - healthStatus: "healthy", - healthMessage: optionalSlackTest ? "Connected; conversation test optional" : "Connected", - lastError: null, - healthCheckedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(toolConnections.id, endpoint.connectionId)); - if (optionalSlackTest) await logActivity(tx as unknown as Db, { - companyId: endpoint.companyId, actorType: "user", actorId: finishOptions!.optionalSlackTestForUser, - action: "chat.setup_completed", entityType: "chat_endpoint", entityId: endpointId, - details: { conversationTestOptional: true }, - }); - await credentialLease.assertOwned(tx); - }); - return get(endpointId); - }, - ); - } - async function ensurePrincipal( endpoint: EndpointRow, author: Author, @@ -7795,595 +5356,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return resource; } - async function ingestAttachments(input: { - endpoint: EndpointRow; - endpointRuntime: ChatEndpointRuntime; - deliveryId: string; - issueId: string; - issueCommentId: string; - attachments: Attachment[]; - attachmentLimitOmissions?: number; - unavailableReferenceCount?: number; - actorUserId: string | null; - }): Promise<{ - storedIds: string[]; - omissionReasons: Record; - }> { - const omissionReasons: Record = {}; - const omit = (reason: string, count = 1) => { - omissionReasons[reason] = (omissionReasons[reason] ?? 0) + count; - }; - const boundedAttachments = input.attachments.slice(0, 20); - const photonDerivatives = new Map(); - const originalPhotonAttachments = new Map(); - if ( - input.endpoint.provider === "microsoft-teams" && - input.unavailableReferenceCount - ) { - // A channel/group reference is not a download capability. Still tell - // the exact current-turn consumer that those bytes are unavailable, - // including when there are no ingestible attachments at all. - omit("download_unavailable", input.unavailableReferenceCount); - } - if ( - ["github", "imessage-photon"].includes(input.endpoint.provider) && - input.attachmentLimitOmissions - ) { - omit("attachment_limit", input.attachmentLimitOmissions); - } - if (input.attachments.length > boundedAttachments.length) { - omit( - "attachment_limit", - input.attachments.length - boundedAttachments.length, - ); - } - if (!options.storage) { - if (boundedAttachments.length) - omit("storage_unavailable", boundedAttachments.length); - return { storedIds: [], omissionReasons }; - } - const existingByFingerprint = new Map(); - const existingAttachments = await db - .select({ - id: issueAttachments.id, - byteSize: assets.byteSize, - contentType: assets.contentType, - originalFilename: assets.originalFilename, - sha256: assets.sha256, - }) - .from(issueAttachments) - .innerJoin( - assets, - and( - eq(assets.companyId, issueAttachments.companyId), - eq(assets.id, issueAttachments.assetId), - ), - ) - .where( - and( - eq(issueAttachments.companyId, input.endpoint.companyId), - eq(issueAttachments.issueId, input.issueId), - eq(issueAttachments.issueCommentId, input.issueCommentId), - ), - ); - for (const existing of existingAttachments) { - const fingerprint = JSON.stringify([ - existing.sha256, - existing.byteSize, - existing.contentType, - existing.originalFilename, - ]); - const ids = existingByFingerprint.get(fingerprint) ?? []; - ids.push(existing.id); - existingByFingerprint.set(fingerprint, ids); - } - const storedIds: string[] = []; - const withCurrentAttachmentAuthorization = async ( - work: (tx: DbTransaction) => Promise, - enforceUnchangedSource = false, - telegramMedia?: Attachment, - ): Promise => { - for (let attempt = 0; ; attempt++) { - try { - return await db.transaction(async (tx) => { - const action = await tx - .select() - .from(chatActions) - .where( - and( - eq(chatActions.companyId, input.endpoint.companyId), - eq(chatActions.endpointId, input.endpoint.id), - eq(chatActions.deliveryId, input.deliveryId), - eq(chatActions.kind, "inbound_wakeup"), - ), - ) - .limit(1) - .then((rows) => rows[0]); - if ( - !action || - !["preparing", "issued", "processing", "processed"].includes( - action.status, - ) || - action.payload.issueId !== input.issueId || - action.payload.commentId !== input.issueCommentId - ) - throw forbidden( - "The attachment's current chat admission is unavailable", - { code: "chat_action_authorization_changed" }, - ); - const current = await authorizeInboundWakeup(tx, action); - // Delivery settlement/replay can update these receipts without - // taking the issue lock. Lock in delivery -> action order only - // after policy authorization; NOWAIT breaks cross-worker lock - // inversions and the outer loop retries the whole transaction. - const lockedDelivery = await tx - .select() - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.companyId, input.endpoint.companyId), - eq(chatDeliveries.id, input.deliveryId), - ), - ) - .for("update", { noWait: true }) - .limit(1) - .then((rows) => rows[0]); - const lockedAction = await tx - .select() - .from(chatActions) - .where( - and( - eq(chatActions.companyId, input.endpoint.companyId), - eq(chatActions.id, action.id), - ), - ) - .for("update", { noWait: true }) - .limit(1) - .then((rows) => rows[0]); - const scopeKeys = [ - "version", - "issueId", - "commentId", - "agentId", - "sessionGeneration", - "requestedByActorType", - "requestedByActorId", - ] as const; - if ( - current.delivery.state !== "processing" || - !lockedDelivery || - !lockedAction || - lockedDelivery.state !== "processing" || - lockedDelivery.endpointId !== action.endpointId || - lockedDelivery.conversationId !== action.conversationId || - lockedDelivery.principalId !== action.principalId || - lockedDelivery.attempts !== current.delivery.attempts || - JSON.stringify(lifecycleRuntimeFence(lockedDelivery)) !== - JSON.stringify(lifecycleRuntimeFence(current.delivery)) || - lockedAction.endpointId !== action.endpointId || - lockedAction.deliveryId !== action.deliveryId || - lockedAction.conversationId !== action.conversationId || - lockedAction.principalId !== action.principalId || - lockedAction.kind !== "inbound_wakeup" || - !["preparing", "issued", "processing", "processed"].includes( - lockedAction.status, - ) || - scopeKeys.some( - (key) => lockedAction.payload[key] !== action.payload[key], - ) - ) - throw forbidden( - "The attachment's current chat admission is unavailable", - { code: "chat_action_authorization_changed" }, - ); - if (enforceUnchangedSource) { - if (telegramMedia) { - const normalized = lockedDelivery.normalizedEvent as { - principal?: { externalId?: string }; - message?: { - providerMessageId?: string; - attachments?: Array<{ recovery?: unknown }>; - }; - }; - const fence = lifecycleRuntimeFence(lockedDelivery); - const descriptor = - input.endpointRuntime.attachmentRecoveryDescriptor( - telegramMedia, - { - threadId: current.conversation.externalThreadId, - messageId: normalized.message?.providerMessageId ?? "", - principalExternalId: normalized.principal?.externalId, - runtimeGeneration: fence?.generation, - credentialFingerprint: fence?.credentialFingerprint, - }, - ); - if ( - descriptor?.locator.kind !== "telegram_media" || - !normalized.message?.attachments?.some((item) => { - const retained = item.recovery as { - version?: unknown; - provider?: unknown; - locator?: Record; - attachment?: Record; - } | null; - return ( - retained?.version === 1 && - retained.provider === "telegram" && - Object.entries(descriptor.locator).every( - ([key, value]) => retained.locator?.[key] === value, - ) && - Object.entries(descriptor.attachment).every( - ([key, value]) => retained.attachment?.[key] === value, - ) - ); - }) - ) - throw forbidden( - "The media's admitted source is unavailable", - { code: "chat_action_authorization_changed" }, - ); - } - const [source] = await tx - .select({ id: issueComments.id }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, input.endpoint.companyId), - eq(issueComments.issueId, input.issueId), - eq(issueComments.id, input.issueCommentId), - isNull(issueComments.deletedAt), - sql`${issueComments.updatedAt} = ${issueComments.createdAt}`, - ), - ) - .for("share", { noWait: true }); - // Lifecycle admission precedes its conversation drain. Pending - // source changes already invalidate this image download, even - // when the append-only original comment is still untouched. - const [lifecycle] = await tx - .select({ id: chatDeliveries.id }) - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.companyId, input.endpoint.companyId), - eq(chatDeliveries.endpointId, input.endpoint.id), - or( - isNull(chatDeliveries.conversationId), - eq( - chatDeliveries.conversationId, - current.conversation.id, - ), - ), - ne(chatDeliveries.state, "filtered"), - inArray(chatDeliveries.eventKind, [ - "message_updated", - "message_deleted", - "message_restored", - ]), - sql`${chatDeliveries.normalizedEvent}->'runtimeContext' = ${JSON.stringify(lifecycleRuntimeFence(lockedDelivery))}::jsonb`, - sql`${chatDeliveries.normalizedEvent}->'message'->>'targetProviderEventId' = ${lockedDelivery.providerEventId}`, - ), - ) - .limit(1); - if (!source || lifecycle) - throw forbidden( - "The image's admitted source changed before attachment registration", - { - code: "chat_action_authorization_changed", - }, - ); - } - return await work(tx); - }); - } catch (error) { - if (!isExternalChatWaitAuthorizationContention(error) || attempt >= 4) - throw error; - // Retry the complete short transaction; no network/storage work or - // backoff occurs while governance rows are locked. - await new Promise((resolve) => - setTimeout(resolve, 20 * (attempt + 1)), - ); - } - } - }; - // One shared deadline bounds the sequential batch; an expired batch never - // starts another GitHub request. Other providers retain their own policy. - const githubBatchSignal = - input.endpoint.provider === "github" - ? AbortSignal.timeout(GITHUB_ATTACHMENT_BATCH_TIMEOUT_MS) - : undefined; - // This new image-only lane shares one budget across token acquisition and - // downloads. DB/storage commit semantics remain explicit and unchanged. - const teamsInlineBatchSignal = - input.endpoint.provider === "microsoft-teams" - ? AbortSignal.timeout(10_000) - : undefined; - for (let attachment of boundedAttachments) { - const teamsInlineImage = - input.endpoint.provider === "microsoft-teams" && - input.endpointRuntime.attachmentRecoveryDescriptor(attachment)?.locator - .kind === "teams_inline_image"; - const telegramMedia = - input.endpoint.provider === "telegram" && - hasTelegramMediaProvenance(attachment); - const sourceBoundMedia = - teamsInlineImage || - telegramMedia || - input.endpoint.provider === "imessage-photon"; - const requireCurrentAttachmentAuthorization = - input.endpoint.provider === "github" || sourceBoundMedia; - try { - if (teamsInlineImage && teamsInlineBatchSignal?.aborted) { - omit("download_unavailable"); - continue; - } - // GitHub's anonymized upload URLs carry no trustworthy MIME metadata. - // Resolve public bytes only here, after durable comment admission, and - // then apply the same storage/type policy as every native attachment. - if (input.endpoint.provider === "github") { - await withCurrentAttachmentAuthorization(async () => {}); - attachment = await prepareGitHubPublicAttachment( - attachment, - githubBatchSignal, - async (request, signal) => { - await withCurrentAttachmentAuthorization(async () => {}); - return ( - (await input.endpointRuntime.resolveGitHubAttachmentComment?.( - request, - signal, - )) ?? null - ); - }, - ); - } - if ( - requireCurrentAttachmentAuthorization && - input.endpoint.provider !== "github" - ) - await withCurrentAttachmentAuthorization( - async () => {}, - sourceBoundMedia, - telegramMedia ? attachment : undefined, - ); - if ( - attachment.size !== undefined && - attachment.size > MAX_ATTACHMENT_BYTES - ) { - omit("declared_too_large"); - continue; - } - if (!attachment.fetchData) { - omit("download_unavailable"); - continue; - } - const originalFilename = sanitizeFilename(attachment.name); - let contentType = normalizeUploadAttachmentContentType({ - contentType: normalizeContentType( - attachment.mimeType ?? "application/octet-stream", - ), - originalFilename, - isAllowedContentType, - }); - // Reject a provider-declared type before allocating the downloaded - // payload. The actual byte length is checked again after the adapter's - // bounded fetch contract resolves. - // A filename may infer an Office MIME for ordinary documents, but it - // must not bypass byte identification for native media without MIME. - const identifyTelegram = - telegramMedia && - telegramMediaNeedsIdentification(attachment) && - normalizeContentType( - attachment.mimeType ?? "application/octet-stream", - ) === "application/octet-stream"; - if (!isAllowedContentType(contentType) && !identifyTelegram) { - omit("unsupported_type"); - continue; - } - const fetched = teamsInlineImage - ? await input.endpointRuntime.fetchTeamsInlineImage( - attachment, - teamsInlineBatchSignal!, - ) - : await attachment.fetchData(); - const body = Buffer.isBuffer(fetched) ? fetched : Buffer.from(fetched); - if (body.length === 0) { - omit("empty_download"); - continue; - } - if (body.length > MAX_ATTACHMENT_BYTES) { - omit("downloaded_too_large"); - continue; - } - if (identifyTelegram) { - contentType = - identifyTelegramMedia(attachment, body) ?? - "application/octet-stream"; - if (!isAllowedContentType(contentType)) { - omit("unsupported_type"); - continue; - } - } - const photonSourceHash = createHash("sha256") - .update(body) - .digest("hex"); - if (input.endpoint.provider === "imessage-photon") { - await validatePhotonImage(body, contentType); - const companion = takePhotonCompanion(body); - if (companion?.unavailable) omit("companion_unavailable"); - else if (companion) { - const video: Attachment = { type: "video", name: companion.fileName, mimeType: companion.mimeType, size: companion.data.length, fetchData: async () => companion.data }; - photonDerivatives.set(video, { source: attachment, sourceHash: photonSourceHash, kind: "live_photo_video" }); - boundedAttachments.push(video); - } - if ( - HEIF_CONTENT_TYPES.has(contentType) && - isAllowedContentType("image/jpeg") - ) { - try { - const preview = await photonHeifPreview(body); - const derivative: Attachment = { - type: "image", - name: `${originalFilename} (JPEG preview).jpg`, - mimeType: "image/jpeg", - size: preview.length, - fetchData: async () => preview, - }; - photonDerivatives.set(derivative, { source: attachment, sourceHash: photonSourceHash, kind: "heif_jpeg_preview" }); - boundedAttachments.push(derivative); - } catch { - omit("preview_unavailable"); - } - } - } - const fingerprint = JSON.stringify([ - photonSourceHash, - body.length, - contentType, - originalFilename, - ]); - const existingIds = existingByFingerprint.get(fingerprint); - const existingId = existingIds?.shift(); - if (requireCurrentAttachmentAuthorization) { - // Revoke during either API resolution or the anonymous byte download - // must prevent storage and attachment registration for that input. - await withCurrentAttachmentAuthorization( - async () => {}, - sourceBoundMedia, - telegramMedia ? attachment : undefined, - ); - } - const registerPhotonProvenance = async (attachmentId: string) => { - if (input.endpoint.provider !== "imessage-photon") return; - const relation = photonDerivatives.get(attachment); - if (!relation) { - originalPhotonAttachments.set(attachment, attachmentId); - return; - } - const originalAttachmentId = - originalPhotonAttachments.get(relation.source); - if (!originalAttachmentId) - throw new Error("Photon related attachment source is missing"); - await db - .insert(chatActions) - .values({ - companyId: input.endpoint.companyId, - endpointId: input.endpoint.id, - deliveryId: input.deliveryId, - kind: relation.kind === "heif_jpeg_preview" ? "attachment_derivative" : "attachment_companion", - providerActionId: `photon-${relation.kind}:${input.deliveryId}:${originalAttachmentId}`, - payload: { - originalAttachmentId, - derivativeAttachmentId: attachmentId, - originalSha256: relation.sourceHash, - derivativeSha256: photonSourceHash, - kind: relation.kind, - }, - status: "processed", - }) - .onConflictDoNothing(); - }; - if (existingId) { - storedIds.push(existingId); - await registerPhotonProvenance(existingId); - continue; - } - const stored = await options.storage.putFile({ - companyId: input.endpoint.companyId, - namespace: `issues/${input.issueId}`, - originalFilename, - contentType, - body, - }); - const registration = { - issueId: input.issueId, - issueCommentId: input.issueCommentId, - provider: stored.provider, - objectKey: stored.objectKey, - contentType: stored.contentType, - byteSize: stored.byteSize, - sha256: stored.sha256, - originalFilename: stored.originalFilename, - createdByUserId: input.actorUserId, - }; - let row; - try { - row = requireCurrentAttachmentAuthorization - ? await withCurrentAttachmentAuthorization( - (tx) => - issueService(tx as unknown as Db).createAttachment( - registration, - ), - sourceBoundMedia, - telegramMedia ? attachment : undefined, - ) - : await issuesSvc.createAttachment(registration); - } catch (error) { - // An explicit authorization denial happens before registration. Do - // not delete on ambiguous database errors which may have committed. - if ( - requireCurrentAttachmentAuthorization && - isExternalActionAuthorizationChange(error) - ) - await options.storage - .deleteObject(input.endpoint.companyId, stored.objectKey) - .catch(() => undefined); - throw error; - } - storedIds.push(row.id); - await registerPhotonProvenance(row.id); - } catch (error) { - if (isExternalActionAuthorizationChange(error)) throw error; - if ( - error instanceof PhotonError && - ["attachment_not_ready", "network", "quota", "credentials"].includes( - error.code, - ) - ) - throw error; - // Use the closed current-input omission vocabulary consumed by native - // prompts; provider-specific diagnostics remain redacted log codes. - omit( - error instanceof GitHubAttachmentUnavailableError - ? "download_unavailable" - : "processing_failed", - ); - // A malformed or unavailable provider attachment must not strand the - // durable text delivery. The rejected file is intentionally omitted; - // the delivery remains auditable through its normalized attachment - // metadata without persisting provider credentials or download URLs. - logger.warn( - { - endpointId: input.endpoint.id, - issueId: input.issueId, - ...(error instanceof GitHubAttachmentUnavailableError - ? { - deliveryId: input.deliveryId, - attachmentDiagnosticCode: error.code, - } - : {}), - attachmentName: sanitizeFilename(attachment.name), - error: redactError(error), - }, - "external chat attachment was rejected", - ); - } - } - return { storedIds, omissionReasons }; - } - - function attachmentOmissionDetail(result: { - omissionReasons: Record; - }): string | null { - const entries = Object.entries(result.omissionReasons).filter( - ([, count]) => count > 0, - ); - const omitted = entries.reduce((total, [, count]) => total + count, 0); - if (!omitted) return null; - const reasons = entries - .map(([reason, count]) => `${reason.replaceAll("_", " ")}: ${count}`) - .join(", "); - return `${omitted} external attachment${omitted === 1 ? " was" : "s were"} omitted (${reasons})`; - } - async function settleTelegramAttachmentOnlyFailure(input: { activeDelivery: DeliveryRow; attachmentResult: { @@ -8491,317 +5463,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return true; } - async function admitDiscordRootMention( - configuredEndpoint: EndpointRow, - event: DiscordRootMentionAdmissionEvent, - context: RuntimeContext, - ): Promise { - if ( - !(await ensureDiscordGatewayRuntimeIsCurrent( - configuredEndpoint.id, - context, - )) - ) { - return false; - } - return withCredentialMutationLease( - configuredEndpoint, - async (credentialLease) => { - const threadId = `discord:${event.guildId}:${event.channelId}:${event.messageId}`; - const providerEventId = `${threadId}:${event.messageId}`; - const allowed = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const endpoint = await runtimeCallbackEndpoint( - tx, - configuredEndpoint.id, - context, - ["verifying", "active"], - ); - const existing = await tx - .select({ state: chatDeliveries.state }) - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.endpointId, configuredEndpoint.id), - eq(chatDeliveries.providerEventId, providerEventId), - ), - ) - .then((rows) => rows[0] ?? null); - if ( - existing && - ["processed", "filtered", "failed"].includes(existing.state) - ) - return false; - - let reason: string | null = null; - if (!endpoint || endpoint.providerAccountId !== event.guildId) { - reason = "Connection is not active"; - } - const resource = endpoint - ? await tx - .select() - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.endpointId, endpoint.id), - eq(chatEndpointResources.type, "channel"), - eq( - chatEndpointResources.providerResourceId, - event.channelId, - ), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - const enabledCount = - endpoint?.status === "verifying" - ? await tx - .select({ count: sql`count(*)::int` }) - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.endpointId, endpoint.id), - eq(chatEndpointResources.type, "channel"), - eq(chatEndpointResources.enabled, true), - eq(chatEndpointResources.availability, "available"), - ), - ) - .then((rows) => rows[0]?.count ?? 0) - : 0; - const setupDestination = - endpoint?.status === "verifying" && - enabledCount === 0 && - resource?.availability === "available"; - if ( - !reason && - !( - endpoint?.allowGroupChats && - resource?.availability === "available" && - (resource.enabled || setupDestination) - ) - ) { - reason = "Destination is not enabled in Paperclip"; - } - - if (!reason && endpoint) { - const principal = await tx - .select({ id: chatExternalPrincipals.id }) - .from(chatExternalPrincipals) - .where( - and( - eq(chatExternalPrincipals.companyId, endpoint.companyId), - eq(chatExternalPrincipals.provider, "discord"), - eq(chatExternalPrincipals.providerAccountId, event.guildId), - eq(chatExternalPrincipals.externalId, event.userId), - ), - ) - .then((rows) => rows[0] ?? null); - let allowed: boolean; - let linkedDenied = false; - if (principal) { - const authorization = await lockCurrentPrincipalAuthorization( - tx, - endpoint, - principal.id, - ); - allowed = authorization.allowed; - linkedDenied = authorization.linkedDenied; - } else if (!endpoint.allowUnlinkedPeople) { - allowed = false; - } else if (!endpoint.sponsorUserId) { - allowed = true; - } else { - const sponsor = await tx - .select({ - status: companyMemberships.status, - membershipRole: companyMemberships.membershipRole, - }) - .from(companyMemberships) - .where( - and( - eq(companyMemberships.companyId, endpoint.companyId), - eq(companyMemberships.principalType, "user"), - eq(companyMemberships.principalId, endpoint.sponsorUserId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - allowed = - sponsor?.status === "active" && - sponsor.membershipRole !== "viewer"; - } - if (!allowed) { - reason = linkedDenied - ? "Linked Paperclip account is not currently permitted" - : endpoint.allowUnlinkedPeople - ? "Endpoint sponsor can no longer authorize external guests" - : "External identity must be linked to a Paperclip account"; - } - } - if (!reason) { - await credentialLease.assertOwned(tx); - return true; - } - - const filteredAt = new Date(); - const normalizedEvent = { - providerEventId, - kind: "mention", - trigger: "mention", - resource: { - type: "channel", - providerResourceId: event.channelId, - }, - conversation: { externalThreadId: threadId }, - message: { providerMessageId: event.messageId }, - filtering: { contentRetained: false, providerThreadCreated: false }, - }; - await notifyChatDeliveryWork(tx); - await tx - .insert(chatDeliveries) - .values({ - companyId: configuredEndpoint.companyId, - endpointId: configuredEndpoint.id, - providerEventId, - deduplicationKey: createHash("sha256") - .update(providerEventId) - .digest("hex"), - eventKind: "mention", - normalizedEvent, - state: "filtered", - redactedError: reason, - processedAt: filteredAt, - }) - .onConflictDoUpdate({ - target: [ - chatDeliveries.endpointId, - chatDeliveries.providerEventId, - ], - set: { - normalizedEvent, - principalId: null, - state: "filtered", - nextAttemptAt: null, - redactedError: reason, - processedAt: filteredAt, - updatedAt: filteredAt, - }, - }); - await tx - .update(chatEndpoints) - .set({ lastEventAt: filteredAt, updatedAt: filteredAt }) - .where(eq(chatEndpoints.id, configuredEndpoint.id)); - await credentialLease.assertOwned(tx); - return false; - }); - if (!allowed || !event.message || !context.endpointRuntime) { - return false; - } - const thread = context.endpointRuntime.thread(event.threadId); - await processMessage( - configuredEndpoint, - thread, - event.message, - "mention", - true, - null, - context, - undefined, - null, - true, - undefined, - true, - ); - const staged = await db - .select({ state: chatDeliveries.state }) - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.endpointId, configuredEndpoint.id), - eq(chatDeliveries.providerEventId, providerEventId), - ), - ) - .then((rows) => rows[0] ?? null); - if ( - !staged || - !["received", "retry", "processing"].includes(staged.state) - ) { - return false; - } - - await options.discordRootThreadTransportBarrier?.(); - await credentialLease.assertOwned(); - if ( - !(await ensureDiscordGatewayRuntimeIsCurrent( - configuredEndpoint.id, - context, - )) - ) { - return false; - } - try { - await context.endpointRuntime.ensureDiscordRootThread({ - channelId: event.channelId, - content: event.message.text, - messageId: event.messageId, - }); - } catch (error) { - // The durable provisional receipt owns recovery. Do not let a - // transient REST failure escape the Gateway event handler or ask the - // adapter to create the same thread outside Paperclip's lifecycle - // fence. - logger.warn( - { - endpointId: configuredEndpoint.id, - error: redactError(error), - providerEventId, - }, - "Discord root thread creation will be retried from the durable receipt", - ); - return false; - } - const promoted = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const current = await runtimeCallbackEndpoint( - tx, - configuredEndpoint.id, - context, - ["verifying", "active"], - ); - if (!current) return false; - await notifyChatDeliveryWork(tx); - const [row] = await tx - .update(chatDeliveries) - .set({ - state: "received", - normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, - nextAttemptAt: null, - redactedError: null, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.endpointId, configuredEndpoint.id), - eq(chatDeliveries.providerEventId, providerEventId), - eq(chatDeliveries.state, "retry"), - sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, - ), - ) - .returning({ id: chatDeliveries.id }); - await credentialLease.assertOwned(tx); - return Boolean(row); - }); - if (promoted) - scheduleConversationDrain(configuredEndpoint.id, threadId); - // Paperclip created the provider thread inside its credential/reach - // lease. Returning false prevents the adapter from creating it again; - // the durable drain owns the remaining task mutation. - return false; - }, - ); - } - async function stageInboundWakeup( tx: DbOrTransaction, input: { @@ -12161,197 +8822,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } - function discordRootThreadPending( - delivery: Pick, - ): boolean { - return ( - Boolean(delivery.normalizedEvent) && - typeof delivery.normalizedEvent === "object" && - !Array.isArray(delivery.normalizedEvent) && - (delivery.normalizedEvent as { providerThreadPending?: unknown }) - .providerThreadPending === true - ); - } - - async function reconcileDiscordRootThread( - endpoint: EndpointRow, - delivery: DeliveryRow, - threadId: string, - ): Promise { - try { - const normalized = delivery.normalizedEvent as { - message?: { providerMessageId?: unknown; text?: unknown }; - resource?: { providerResourceId?: unknown }; - }; - const discordThreadParts = threadId.split(":"); - const channelId = - discordThreadParts[0] === "discord" && discordThreadParts.length >= 4 - ? discordThreadParts[2] - : normalized.resource?.providerResourceId; - const messageId = normalized.message?.providerMessageId; - if (typeof channelId !== "string" || typeof messageId !== "string") { - throw new DiscordAdapterCompatibilityError( - "the provisional root receipt is missing its channel or message id", - ); - } - const runtimeFence = lifecycleRuntimeFence(delivery); - if (!runtimeFence) { - throw new DiscordAdapterCompatibilityError( - "the provisional root receipt is missing its runtime fence", - ); - } - return await withCredentialMutationLease( - endpoint, - async (credentialLease) => { - const currentEndpoint = await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const current = await runtimeCallbackEndpoint( - tx, - endpoint.id, - runtimeFence, - ["verifying", "active"], - ); - await credentialLease.assertOwned(tx); - return current; - }); - if (!currentEndpoint) { - throw conflict( - "Discord root thread credentials or connection state changed before recovery", - { code: "chat_endpoint_runtime_superseded" }, - ); - } - const endpointRuntime = await runtimeFor(currentEndpoint); - await credentialLease.assertOwned(); - await endpointRuntime.ensureDiscordRootThread({ - channelId, - messageId, - content: - typeof normalized.message?.text === "string" - ? normalized.message.text - : "Paperclip task", - }); - return await db.transaction(async (tx) => { - await credentialLease.assertOwned(tx); - const current = await runtimeCallbackEndpoint( - tx, - endpoint.id, - runtimeFence, - ["verifying", "active"], - ); - if (!current) return false; - await notifyChatDeliveryWork(tx); - const [promoted] = await tx - .update(chatDeliveries) - .set({ - state: "received", - normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, - nextAttemptAt: null, - redactedError: null, - updatedAt: new Date(), - }) - .where( - and( - eq(chatDeliveries.id, delivery.id), - eq(chatDeliveries.state, "retry"), - sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, - ), - ) - .returning({ id: chatDeliveries.id }); - await credentialLease.assertOwned(tx); - return Boolean(promoted); - }); - }, - ); - } catch (error) { - const disposition = classifyChatPublicationError( - error, - delivery.attempts + 1, - ); - const terminal = - disposition.kind === "resource_unavailable" || - disposition.kind === "failed"; - const retryAfterMs = - disposition.kind === "retry" - ? disposition.retryAfterMs - : DISCORD_ROOT_THREAD_CONFIRMATION_DELAY_MS; - const now = new Date(); - await db.transaction(async (tx) => { - if (disposition.kind === "resource_unavailable") { - const fence = lifecycleRuntimeFence(delivery); - const normalized = delivery.normalizedEvent as { - resource?: { providerResourceId?: unknown }; - }; - const providerResourceId = normalized.resource?.providerResourceId; - const current = fence - ? await runtimeCallbackEndpoint(tx, endpoint.id, fence, [ - "verifying", - "active", - "attention", - ]) - : null; - const resource = - current && typeof providerResourceId === "string" - ? await tx - .select({ id: chatEndpointResources.id }) - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.companyId, current.companyId), - eq(chatEndpointResources.endpointId, current.id), - eq( - chatEndpointResources.providerResourceId, - providerResourceId, - ), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - if (resource) { - await tx - .update(chatEndpointResources) - .set({ availability: "unavailable", updatedAt: now }) - .where(eq(chatEndpointResources.id, resource.id)); - await tx - .update(chatConversations) - .set({ state: "unavailable", updatedAt: now }) - .where( - and( - eq(chatConversations.companyId, current!.companyId), - eq(chatConversations.endpointId, current!.id), - eq(chatConversations.resourceId, resource.id), - inArray(chatConversations.state, ["active", "waiting"]), - ), - ); - } - } - await notifyChatDeliveryWork(tx); - await tx - .update(chatDeliveries) - .set({ - state: terminal ? "filtered" : "retry", - attempts: delivery.attempts + 1, - nextAttemptAt: terminal - ? null - : new Date(now.getTime() + retryAfterMs), - processedAt: terminal ? now : null, - redactedError: terminal - ? "Discord thread creation was not confirmed" - : redactError(error), - updatedAt: now, - }) - .where( - and( - eq(chatDeliveries.id, delivery.id), - eq(chatDeliveries.state, "retry"), - sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, - ), - ); - }); - return false; - } - } - function normalizedDeliveryThreadId(delivery: Pick): string | null { const normalized = delivery.normalizedEvent as { conversation?: { externalThreadId?: unknown }; @@ -14163,479 +10633,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { ); } - function reactionConversationThreadCondition( - provider: ChatProvider, - threadId: string, - ): SQL | undefined { - const exact = externalThreadIdentityCondition( - sql`${chatConversations.externalThreadId}`, - threadId, - ); - const slackDmChannel = - provider === "slack" ? slackThreadChannelId(threadId) : null; - return slackDmChannel && /^D[A-Z0-9-]*$/i.test(slackDmChannel) - ? or( - exact, - and( - eq(chatConversations.isDirectMessage, true), - eq(chatConversations.externalThreadId, `slack:${slackDmChannel}:`), - ), - ) - : exact; - } - - async function reactionConversationForMessage( - database: DbOrTransaction, - endpoint: EndpointRow, - threadId: string, - providerMessageId: string, - direction?: "outbound", - ): Promise { - return database - .select({ conversation: chatConversations }) - .from(chatMessageLinks) - .innerJoin( - chatConversations, - and( - eq(chatConversations.companyId, chatMessageLinks.companyId), - eq(chatConversations.endpointId, chatMessageLinks.endpointId), - eq(chatConversations.id, chatMessageLinks.conversationId), - ), - ) - .where( - and( - eq(chatMessageLinks.companyId, endpoint.companyId), - eq(chatMessageLinks.endpointId, endpoint.id), - eq(chatMessageLinks.providerMessageId, providerMessageId), - direction ? eq(chatMessageLinks.direction, direction) : undefined, - reactionConversationThreadCondition(endpoint.provider, threadId), - ), - ) - .orderBy(desc(chatConversations.sessionGeneration)) - .limit(1) - .then((rows) => rows[0]?.conversation ?? null); - } - - async function lockReactionConversationForMessage( - tx: DbTransaction, - endpoint: EndpointRow, - threadId: string, - providerMessageId: string, - direction?: "outbound", - ): Promise { - const candidate = await reactionConversationForMessage( - tx, - endpoint, - threadId, - providerMessageId, - direction, - ); - if (!candidate) return null; - const current = await tx - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.companyId, endpoint.companyId), - eq(chatConversations.endpointId, endpoint.id), - eq(chatConversations.id, candidate.id), - ), - ) - // Publication settlement inserts the outbound message link before it - // updates the endpoint. NO KEY UPDATE serializes state changes while - // remaining compatible with that link's conversation FK KEY SHARE. - .for("no key update") - .then((rows) => rows[0] ?? null); - if ( - !current || - !(current.isDirectMessage - ? ["active", "waiting", "completed"].includes(current.state) - : ["active", "waiting"].includes(current.state)) - ) { - return null; - } - const linkStillExists = await tx - .select({ id: chatMessageLinks.id }) - .from(chatMessageLinks) - .where( - and( - eq(chatMessageLinks.companyId, endpoint.companyId), - eq(chatMessageLinks.endpointId, endpoint.id), - eq(chatMessageLinks.conversationId, current.id), - eq(chatMessageLinks.providerMessageId, providerMessageId), - direction ? eq(chatMessageLinks.direction, direction) : undefined, - ), - ) - .limit(1) - .then((rows) => rows.length > 0); - return linkStillExists ? current : null; - } - - async function streamingReactionConversation( - database: DbOrTransaction, - endpoint: EndpointRow, - threadId: string, - ): Promise { - const candidates = await database - .selectDistinct({ - conversation: chatConversations, - sessionGeneration: chatConversations.sessionGeneration, - }) - .from(chatConversations) - .innerJoin( - chatPublications, - and( - eq(chatPublications.companyId, chatConversations.companyId), - eq(chatPublications.endpointId, chatConversations.endpointId), - eq(chatPublications.conversationId, chatConversations.id), - ), - ) - .where( - and( - eq(chatConversations.companyId, endpoint.companyId), - eq(chatConversations.endpointId, endpoint.id), - or( - inArray(chatConversations.state, ["active", "waiting"]), - and( - eq(chatConversations.isDirectMessage, true), - eq(chatConversations.state, "completed"), - ), - ), - reactionConversationThreadCondition(endpoint.provider, threadId), - eq(chatPublications.state, "streaming"), - ), - ) - .orderBy(desc(chatConversations.sessionGeneration)) - .limit(2); - if (candidates.length !== 1) return null; - return candidates[0]!.conversation; - } - - async function lockStreamingReactionConversation( - tx: DbTransaction, - endpoint: EndpointRow, - threadId: string, - ): Promise { - const candidate = await streamingReactionConversation( - tx, - endpoint, - threadId, - ); - if (!candidate) return null; - const current = await tx - .select() - .from(chatConversations) - .where( - and( - eq(chatConversations.companyId, endpoint.companyId), - eq(chatConversations.endpointId, endpoint.id), - eq(chatConversations.id, candidate.id), - or( - inArray(chatConversations.state, ["active", "waiting"]), - and( - eq(chatConversations.isDirectMessage, true), - eq(chatConversations.state, "completed"), - ), - ), - ), - ) - // Keep the same lock strength/order as the exact-link path above so an - // in-flight message-link insert cannot deadlock reaction admission. - .for("no key update") - .then((rows) => rows[0] ?? null); - if (!current) return null; - const stillStreaming = await tx - .select({ id: chatPublications.id }) - .from(chatPublications) - .where( - and( - eq(chatPublications.companyId, endpoint.companyId), - eq(chatPublications.endpointId, endpoint.id), - eq(chatPublications.conversationId, current.id), - eq(chatPublications.state, "streaming"), - ), - ) - .limit(1) - .then((rows) => rows.length > 0); - return stillStreaming ? current : null; - } - - async function authorizeReactionConversation( - tx: DbTransaction, - endpoint: EndpointRow, - conversation: ConversationRow, - principalId: string, - ): Promise { - const resource = - conversation.resourceId && !conversation.isDirectMessage - ? await tx - .select() - .from(chatEndpointResources) - .where( - and( - eq(chatEndpointResources.companyId, endpoint.companyId), - eq(chatEndpointResources.endpointId, endpoint.id), - eq(chatEndpointResources.id, conversation.resourceId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null) - : null; - const destinationAllowed = conversation.isDirectMessage - ? endpoint.allowDirectMessages - : nonDirectDestinationAllowed(endpoint, resource); - if (!destinationAllowed) return false; - return (await lockCurrentPrincipalAuthorization(tx, endpoint, principalId)) - .allowed; - } - - async function handleReaction( - event: ChatSdkCallbackEvent, - runtimeContext: RuntimeContext, - ) { - const record = await runtimeCallbackRecord( - event.endpointId, - runtimeContext, - ["active"], - ); - if ( - !record || - record.endpoint.status !== "active" || - record.endpoint.provider !== event.provider || - record.endpoint.capabilities.reactions !== true || - event.event.user.isMe || - event.event.user.isBot === true || - event.event.user.isSystem - ) { - return; - } - // A DM can have a newer task generation while this reaction still belongs - // to an older linked message. Resolve its exact lineage before choosing a - // generation; never attach it to the latest task merely sharing the chat. - // A just-sent provider message can be visible before its publication/link - // transaction commits. Only that exact thread's streaming publication is - // enough evidence to stage an otherwise-unbound reaction for replay. - let conversation = await reactionConversationForMessage( - db, - record.endpoint, - event.event.threadId, - event.event.messageId, - ); - await options.reactionLinkPreflightBarrier?.(); - const plausibleStreamingConversation = conversation - ? null - : await streamingReactionConversation( - db, - record.endpoint, - event.event.threadId, - ); - if (!conversation && !plausibleStreamingConversation) { - // Settlement can commit between the first exact-link lookup and the - // streaming-publication lookup. Recheck the durable link before treating - // the callback as an arbitrary unknown message. - conversation = await reactionConversationForMessage( - db, - record.endpoint, - event.event.threadId, - event.event.messageId, - ); - if (!conversation) return; - } - - const principal = await ensurePrincipal( - record.endpoint, - event.event.user, - event.event.raw, - ); - let rawFingerprint: string; - try { - rawFingerprint = createHash("sha256") - .update(JSON.stringify(event.event.raw ?? null)) - .digest("hex") - .slice(0, 24); - } catch { - rawFingerprint = "unavailable"; - } - const eventKind: ChatEventKind = event.event.added - ? "reaction_added" - : "reaction_removed"; - const providerEventId = [ - eventKind, - durableExternalThreadIdentity(event.event.threadId), - event.event.messageId, - stableExternalPrincipalId( - record.endpoint.provider, - event.event.user, - event.event.raw, - ), - event.event.rawEmoji, - rawFingerprint, - ].join(":"); - const normalizedEvent = { - providerEventId, - kind: eventKind, - conversation: { externalThreadId: event.event.threadId }, - message: { providerMessageId: event.event.messageId }, - reaction: { - emoji: event.event.emoji.name, - rawEmoji: event.event.rawEmoji, - added: event.event.added, - }, - runtimeContext: { - generation: runtimeContext.generation, - credentialFingerprint: runtimeContext.credentialFingerprint, - }, - }; - const admitted = await db.transaction(async (tx) => { - const currentEndpoint = await runtimeCallbackEndpoint( - tx, - event.endpointId, - runtimeContext, - ["active"], - ); - if (!currentEndpoint) return false; - const linkedConversation = await lockReactionConversationForMessage( - tx, - currentEndpoint, - event.event.threadId, - event.event.messageId, - ); - const currentConversation = - linkedConversation ?? - (await lockStreamingReactionConversation( - tx, - currentEndpoint, - event.event.threadId, - )); - if ( - !currentConversation || - !(await authorizeReactionConversation( - tx, - currentEndpoint, - currentConversation, - principal.principal.id, - )) - ) { - return false; - } - const now = new Date(); - if (linkedConversation) { - const expiryCutoff = new Date(now.getTime() - REACTION_LINK_MAX_AGE_MS); - const hasOutboundLink = await tx - .select({ id: chatMessageLinks.id }) - .from(chatMessageLinks) - .where( - and( - eq(chatMessageLinks.companyId, currentEndpoint.companyId), - eq(chatMessageLinks.endpointId, currentEndpoint.id), - eq(chatMessageLinks.conversationId, linkedConversation.id), - eq(chatMessageLinks.providerMessageId, event.event.messageId), - eq(chatMessageLinks.direction, "outbound"), - ), - ) - .limit(1) - .then((rows) => rows.length > 0); - await notifyChatDeliveryWork(tx); - const reconciled = hasOutboundLink - ? await tx - .update(chatDeliveries) - .set({ - conversationId: linkedConversation.id, - state: "processed", - attempts: sql`${chatDeliveries.attempts} + 1`, - nextAttemptAt: null, - processedAt: now, - redactedError: null, - updatedAt: now, - }) - .where( - and( - eq(chatDeliveries.endpointId, currentEndpoint.id), - eq(chatDeliveries.providerEventId, providerEventId), - inArray(chatDeliveries.state, ["received", "retry"]), - gt(chatDeliveries.receivedAt, expiryCutoff), - lt(chatDeliveries.attempts, REACTION_LINK_MAX_ATTEMPTS), - sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'generation' = ${String(runtimeContext.generation)}`, - sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'credentialFingerprint' = ${runtimeContext.credentialFingerprint}`, - ), - ) - .returning({ id: chatDeliveries.id }) - : []; - if (reconciled.length > 0) return true; - await notifyChatDeliveryWork(tx); - const expired = await tx - .update(chatDeliveries) - .set({ - state: "filtered", - nextAttemptAt: null, - processedAt: now, - redactedError: - "Reaction target did not become linked before replay expiry", - updatedAt: now, - }) - .where( - and( - eq(chatDeliveries.endpointId, currentEndpoint.id), - eq(chatDeliveries.providerEventId, providerEventId), - inArray(chatDeliveries.state, ["received", "retry"]), - or( - lte(chatDeliveries.receivedAt, expiryCutoff), - gte(chatDeliveries.attempts, REACTION_LINK_MAX_ATTEMPTS), - ), - sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'generation' = ${String(runtimeContext.generation)}`, - sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'credentialFingerprint' = ${runtimeContext.credentialFingerprint}`, - ), - ) - .returning({ id: chatDeliveries.id }); - if (expired.length > 0) return false; - } - await notifyChatDeliveryWork(tx); - const inserted = await tx - .insert(chatDeliveries) - .values({ - companyId: currentEndpoint.companyId, - endpointId: currentEndpoint.id, - conversationId: linkedConversation?.id ?? null, - principalId: principal.principal.id, - providerEventId, - deduplicationKey: createHash("sha256") - .update(providerEventId) - .digest("hex"), - eventKind, - normalizedEvent, - state: linkedConversation ? "processed" : "received", - attempts: linkedConversation ? 1 : 0, - nextAttemptAt: linkedConversation - ? null - : new Date(now.getTime() + REACTION_LINK_RETRY_DELAY_MS), - processedAt: linkedConversation ? now : null, - }) - .onConflictDoNothing() - .returning({ id: chatDeliveries.id }); - if (inserted.length > 0) return true; - return tx - .select({ id: chatDeliveries.id }) - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.endpointId, currentEndpoint.id), - eq(chatDeliveries.providerEventId, providerEventId), - inArray(chatDeliveries.state, ["received", "retry", "processed"]), - ), - ) - .limit(1) - .then((rows) => rows.length > 0); - }); - if (admitted) { - await recordCurrentMicrosoftTeamsRoute( - record.endpoint, - runtimeContext, - event.event.threadId, - event.event.raw, - ); - } - } - async function denyExternalAction( endpoint: EndpointRow, event: ChatSdkCallbackEvent, @@ -18629,56 +14626,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } } - async function verifySlackRequestUrl(endpoint: EndpointRow, request: Request): Promise { - const callback = await inspectSlackCallback(request.clone()); - if (!callback?.isUrlVerification) return null; - const body = await request.clone().text(); - let notifyAccount = false; - // URL verification needs only the signing secret. Do not initialize the - // SDK (which calls Slack auth.test), wait for OAuth, or acquire the creation - // lease: Slack can deliver this check while app setup still owns that lease. - const response = await db.transaction(async tx => { - const [current] = await tx.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpoint.id)).for("no key update"); - if (!current || ["archived", "paused", "revoked"].includes(current.status)) return new Response("ignored", { status: 200 }); - const [connection] = await tx.select({ status: toolConnections.status, refs: toolConnections.credentialSecretRefs }).from(toolConnections) - .where(and(eq(toolConnections.id, current.connectionId), eq(toolConnections.companyId, current.companyId))); - if (!connection || connection.status === "archived") return new Response("ignored", { status: 200 }); - let secret = (await resolveCredentialRefs(current, connection.refs.filter(ref => ref.configPath.replace(/^credentials\./, "") === "signingSecret"), tx)).signingSecret; - if (!secret && isProvisionedSlackSetup(current.setup.slackSetupMethod)) { - const [registration] = await tx.select().from(chatSlackRegistrations) - .where(and(eq(chatSlackRegistrations.endpointId, current.id), eq(chatSlackRegistrations.companyId, current.companyId))); - if (registration?.status !== "removed" && registration?.secretIds.signingSecret) { - secret = await secretService(tx as unknown as Db).resolveSecretValue(current.companyId, registration.secretIds.signingSecret, "latest", { - accessContext: { consumerType: "system", consumerId: `slack-registration:${current.id}`, configPath: "slack_registration.signingSecret", actorType: "system" }, - }); - } - } - if (!secret) return new Response("Slack signing secret is not ready yet", { status: 503 }); - if (!slackRequestSignatureIsValid(request, body, secret)) return new Response("Invalid signature", { status: 401 }); - const workspaceId = slackRequestWorkspaceId(body, request.headers.get("content-type") ?? ""); - if (workspaceId && current.providerAccountId && workspaceId !== current.providerAccountId) return new Response("ignored", { status: 200 }); - const observedAt = new Date(); - const setup = current.setup as InternalSetupState; - const matchesCurrentUrl = slackCallbackMatchesPublicUrl(callback.url, `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${current.publicId}/slack`); - await notifyChatVerificationWork(tx); - await tx.update(chatEndpoints).set({ setup: { - ...setup, - ...(matchesCurrentUrl ? { webhookVerifiedAt: setup.webhookVerifiedAt ?? observedAt.toISOString(), - slackVerificationSigningFingerprint: createHash("sha256").update(secret).digest("hex") } : {}), - slackCallbackSurfaces: { ...setup.slackCallbackSurfaces, events: { url: callback.url, observedAt: observedAt.toISOString() } }, - ...(matchesCurrentUrl && setup.slackAccount?.status === "linked" && !setup.slackAccount.verificationStatus - ? { slackAccount: { ...setup.slackAccount, verificationStatus: "pending" as const } } : {}), - } as InternalSetupState, ...(matchesCurrentUrl ? { healthMessage: "Slack Events Request URL verified" } : {}), updatedAt: observedAt }).where(eq(chatEndpoints.id, current.id)); - if (matchesCurrentUrl && !setup.webhookVerifiedAt) await logActivity(tx as unknown as Db, { companyId: current.companyId, actorType: "system", actorId: "slack-webhook", - action: "chat_endpoint.webhook_verified", entityType: "tool_connection", entityId: current.connectionId, - details: { endpointId: current.id, provider: "slack" } }); - notifyAccount = matchesCurrentUrl && setup.slackAccount?.status === "linked" && !setup.slackAccount.verificationStatus; - return Response.json({ challenge: (JSON.parse(body) as { challenge: string }).challenge }); - }); - if (notifyAccount) scheduleMessageProcessing(() => slackRegistration.notifyVerified(endpoint.id)); - return response; - } - async function handleWebhook( publicId: string, provider: ChatSdkProvider, @@ -19365,334 +15312,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return response; } - function normalizedPendingReaction(delivery: DeliveryRow): { - added: boolean; - emoji: string; - messageId: string; - rawEmoji: string; - runtimeContext: LifecycleRuntimeFence; - threadId: string; - } | null { - if ( - delivery.eventKind !== "reaction_added" && - delivery.eventKind !== "reaction_removed" - ) { - return null; - } - const normalized = delivery.normalizedEvent; - if ( - !normalized || - typeof normalized !== "object" || - Array.isArray(normalized) - ) { - return null; - } - const value = normalized as Record; - const conversation = value.conversation; - const message = value.message; - const reaction = value.reaction; - const runtimeContext = lifecycleRuntimeFence(delivery); - if ( - !conversation || - typeof conversation !== "object" || - Array.isArray(conversation) || - !message || - typeof message !== "object" || - Array.isArray(message) || - !reaction || - typeof reaction !== "object" || - Array.isArray(reaction) || - !runtimeContext - ) { - return null; - } - const threadId = (conversation as Record).externalThreadId; - const messageId = (message as Record).providerMessageId; - const reactionRecord = reaction as Record; - const added = reactionRecord.added; - const emoji = reactionRecord.emoji; - const rawEmoji = reactionRecord.rawEmoji; - return typeof threadId === "string" && - threadId.length > 0 && - typeof messageId === "string" && - messageId.length > 0 && - typeof added === "boolean" && - added === (delivery.eventKind === "reaction_added") && - typeof emoji === "string" && - emoji.length > 0 && - typeof rawEmoji === "string" && - rawEmoji.length > 0 - ? { added, emoji, messageId, rawEmoji, runtimeContext, threadId } - : null; - } - - async function processPendingReactionDelivery( - candidate: DeliveryRow, - ): Promise { - const reaction = normalizedPendingReaction(candidate); - if (!reaction || !candidate.principalId) { - const now = new Date(); - await db.transaction(async (tx) => { - await notifyChatDeliveryWork(tx); - return tx - .update(chatDeliveries) - .set({ - state: "failed", - nextAttemptAt: null, - processedAt: now, - redactedError: "Normalized reaction delivery is incomplete", - updatedAt: now, - }) - .where( - and( - eq(chatDeliveries.id, candidate.id), - inArray(chatDeliveries.state, ["received", "retry"]), - ), - ); - }); - return; - } - const principalId = candidate.principalId; - - await db.transaction(async (tx) => { - const currentEndpoint = await runtimeCallbackEndpoint( - tx, - candidate.endpointId, - reaction.runtimeContext, - ["active"], - ); - let conversation: ConversationRow | null = null; - let disposition: - | { kind: "processed"; conversationId: string } - | { kind: "retry" } - | { kind: "filtered"; reason: string }; - - if (!currentEndpoint) { - disposition = { - kind: "filtered", - reason: "Reaction callback belonged to an unavailable runtime", - }; - } else { - await options.reactionReplayEndpointLockBarrier?.(); - conversation = await lockReactionConversationForMessage( - tx, - currentEndpoint, - reaction.threadId, - reaction.messageId, - "outbound", - ); - if (conversation) { - await options.reactionReplayConversationLockBarrier?.(); - } - if (conversation) { - disposition = (await authorizeReactionConversation( - tx, - currentEndpoint, - conversation, - principalId, - )) - ? { kind: "processed", conversationId: conversation.id } - : { - kind: "filtered", - reason: - "Reaction destination or principal is no longer authorized", - }; - } else { - conversation = await lockStreamingReactionConversation( - tx, - currentEndpoint, - reaction.threadId, - ); - if (conversation) { - await options.reactionReplayConversationLockBarrier?.(); - } - if (!conversation) { - disposition = { - kind: "filtered", - reason: "Reaction target was not linked to a sent message", - }; - } else { - disposition = (await authorizeReactionConversation( - tx, - currentEndpoint, - conversation, - principalId, - )) - ? { kind: "retry" } - : { - kind: "filtered", - reason: - "Reaction destination or principal is no longer authorized", - }; - } - } - } - - // Endpoint, conversation, destination and identity locks are acquired - // before the delivery row so pause/revoke and replay use one lock order. - const locked = await tx - .select() - .from(chatDeliveries) - .where( - and( - eq(chatDeliveries.id, candidate.id), - inArray(chatDeliveries.state, ["received", "retry"]), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - // Authorization locks may wait behind a concurrent lifecycle mutation. - // Sample the expiry clock only after this delivery is exclusively held. - const decisionAt = new Date(); - if (!locked || !deliveryReady(locked, decisionAt)) return; - const lockedReaction = normalizedPendingReaction(locked); - if ( - !lockedReaction || - lockedReaction.runtimeContext.generation !== - reaction.runtimeContext.generation || - lockedReaction.runtimeContext.credentialFingerprint !== - reaction.runtimeContext.credentialFingerprint - ) { - disposition = { - kind: "filtered", - reason: "Reaction delivery identity changed before replay", - }; - } - - const nextAttempts = locked.attempts + 1; - const expired = - decisionAt.getTime() - locked.receivedAt.getTime() >= - REACTION_LINK_MAX_AGE_MS; - if (expired || locked.attempts >= REACTION_LINK_MAX_ATTEMPTS) { - disposition = { - kind: "filtered", - reason: "Reaction target did not become linked before replay expiry", - }; - } - if (disposition.kind === "retry" && !expired) { - if (nextAttempts < REACTION_LINK_MAX_ATTEMPTS) { - const delay = Math.min( - REACTION_LINK_RETRY_DELAY_MS * - 2 ** Math.min(Math.max(locked.attempts, 0), 5), - REACTION_LINK_MAX_DELAY_MS, - ); - await notifyChatDeliveryWork(tx); - await tx - .update(chatDeliveries) - .set({ - state: "retry", - attempts: nextAttempts, - nextAttemptAt: new Date(decisionAt.getTime() + delay), - redactedError: "Waiting for the sent message link", - updatedAt: decisionAt, - }) - .where( - and( - eq(chatDeliveries.id, locked.id), - eq(chatDeliveries.state, locked.state), - eq(chatDeliveries.attempts, locked.attempts), - ), - ); - return; - } - disposition = { - kind: "filtered", - reason: "Reaction target did not become linked before retry expiry", - }; - } else if (disposition.kind === "retry") { - disposition = { - kind: "filtered", - reason: "Reaction target did not become linked before age expiry", - }; - } - - const processedAt = new Date(); - await notifyChatDeliveryWork(tx); - await tx - .update(chatDeliveries) - .set( - disposition.kind === "processed" - ? { - conversationId: disposition.conversationId, - state: "processed", - attempts: nextAttempts, - nextAttemptAt: null, - processedAt, - redactedError: null, - updatedAt: processedAt, - } - : { - state: "filtered", - attempts: nextAttempts, - nextAttemptAt: null, - processedAt, - redactedError: disposition.reason, - updatedAt: processedAt, - }, - ) - .where( - and( - eq(chatDeliveries.id, locked.id), - eq(chatDeliveries.state, locked.state), - eq(chatDeliveries.attempts, locked.attempts), - ), - ); - }); - } - - async function processPendingReactionDeliveries( - limit: number, - onlyDeliveryId?: string, - ): Promise { - const now = new Date(); - const candidates = await db - .select() - .from(chatDeliveries) - .where( - and( - sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.provider = 'agentmail')`, - onlyDeliveryId ? eq(chatDeliveries.id, onlyDeliveryId) : undefined, - inArray(chatDeliveries.eventKind, [ - "reaction_added", - "reaction_removed", - ]), - or( - and( - eq(chatDeliveries.state, "received"), - or( - isNull(chatDeliveries.nextAttemptAt), - lte(chatDeliveries.nextAttemptAt, now), - ), - ), - and( - eq(chatDeliveries.state, "retry"), - or( - isNull(chatDeliveries.nextAttemptAt), - lte(chatDeliveries.nextAttemptAt, now), - ), - ), - ), - ), - ) - .orderBy(asc(chatDeliveries.receivedAt), asc(chatDeliveries.id)) - .limit(limit); - for (const candidate of candidates) { - try { - await processPendingReactionDelivery(candidate); - } catch (error) { - logger.warn( - { - endpointId: candidate.endpointId, - deliveryId: candidate.id, - error: redactError(error), - }, - "chat reaction-link reconciliation failed", - ); - } - } - return candidates.length; - } - /** * Reconcile a verified normalized delivery whose original request was * interrupted. Closed, credential-free provider descriptors reconstruct @@ -19926,96 +15545,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { return reactionCount + rows.length; } - async function listConversations(endpointId: string) { - const record = await endpointRecord(endpointId); - if (!record) throw notFound("Chat endpoint not found"); - const conversations = await db - .select() - .from(chatConversations) - .where(eq(chatConversations.endpointId, endpointId)) - .orderBy( - desc(chatConversations.lastActivityAt), - desc(chatConversations.createdAt), - ); - if (!conversations.length) return []; - const issueRows = await db - .select({ - id: issues.id, - identifier: issues.identifier, - title: issues.title, - status: issues.status, - updatedAt: issues.updatedAt, - }) - .from(issues) - .where( - inArray( - issues.id, - conversations.map((row) => row.issueId), - ), - ); - const publications = await db - .select() - .from(chatPublications) - .where(eq(chatPublications.endpointId, endpointId)) - .orderBy(desc(chatPublications.createdAt)); - const issueById = new Map(issueRows.map((row) => [row.id, row])); - const lastPublication = new Map< - string, - typeof chatPublications.$inferSelect - >(); - for (const publication of publications) - if (!lastPublication.has(publication.conversationId)) - lastPublication.set(publication.conversationId, publication); - return conversations.map((conversation) => { - const issue = issueById.get(conversation.issueId); - const externalUrl = - record.endpoint.provider === "telegram" && conversation.isDirectMessage - ? chatProviderConversationUrl({ - provider: "telegram", - botUsername: record.endpoint.botUsername, - threadId: conversation.externalThreadId, - providerMessageId: "", - raw: { chat: { type: "private" } }, - }) - : (conversation.providerUrl ?? - (record.endpoint.provider === "slack" - ? chatProviderConversationUrl({ - provider: "slack", - providerAccountId: record.endpoint.providerAccountId, - threadId: conversation.externalThreadId, - providerMessageId: "", - }) - : null)); - return { - id: conversation.id, - companyId: conversation.companyId, - endpointId: conversation.endpointId, - resourceId: conversation.resourceId, - externalLabel: conversation.externalLabel, - externalUrl, - externalConversationId: conversation.externalConversationId, - externalThreadId: conversation.externalThreadId, - sessionGeneration: conversation.sessionGeneration, - issueId: conversation.issueId, - issueIdentifier: issue?.identifier ?? null, - issueTitle: issue?.title ?? null, - isDirectMessage: conversation.isDirectMessage, - state: - record.endpoint.provider !== "imessage-photon" && - (issue?.status === "done" || issue?.status === "cancelled") - ? "completed" - : conversation.state, - lastActivityAt: conversation.lastActivityAt?.toISOString() ?? null, - createdAt: conversation.createdAt.toISOString(), - updatedAt: ( - conversation.lastActivityAt ?? conversation.updatedAt - ).toISOString(), - lastPublicationStatus: - lastPublication.get(conversation.id)?.state ?? null, - }; - }); - } - async function recordGitHubAutomaticSkip( endpoint: EndpointRow, deliveryId: string, event: string, reason: string | undefined, runtimeContext: LifecycleRuntimeFence, ) { @@ -20043,457 +15572,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } - async function listActivity(endpointId: string, page?: { limit: number; before?: { createdAt: string; id: string } }) { - const limit = page ? page.limit + 1 : 100; - // Match the millisecond precision of the public timestamp and use a UUID - // tie-breaker so equal timestamps never skip records between pages. - const before = (date: AnyPgColumn, id: AnyPgColumn) => page?.before - ? sql`(date_trunc('milliseconds', ${date}), ${id}) < (${page.before.createdAt}::timestamptz, ${page.before.id}::uuid)` - : undefined; - const recoveryIngress = alias(chatActions, "github_recovery_ingress"); - const [ - deliveries, - publications, - slashActions, - providerEffects, - githubIngressFailures, - slackSessionActions, - githubRecoveries, - githubRecoveryScans, - ] = await Promise.all([ - db - .select() - .from(chatDeliveries) - .where(and(eq(chatDeliveries.endpointId, endpointId), before(chatDeliveries.createdAt, chatDeliveries.id))) - .orderBy(desc(sql`date_trunc('milliseconds', ${chatDeliveries.createdAt})`), desc(chatDeliveries.id)) - .limit(limit), - db - .select() - .from(chatPublications) - .where(and(eq(chatPublications.endpointId, endpointId), before(chatPublications.createdAt, chatPublications.id))) - .orderBy(desc(sql`date_trunc('milliseconds', ${chatPublications.createdAt})`), desc(chatPublications.id)) - .limit(limit), - db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, endpointId), - before(chatActions.createdAt, chatActions.id), - eq(chatActions.kind, "slash_task_start"), - ), - ) - .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.createdAt})`), desc(chatActions.id)) - .limit(limit), - db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, endpointId), - before(chatActions.createdAt, chatActions.id), - eq(chatActions.kind, "provider_effect"), - eq(chatActions.status, "delivery_unknown"), - ), - ) - .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.createdAt})`), desc(chatActions.id)) - .limit(limit), - db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, endpointId), - before(chatActions.createdAt, chatActions.id), - eq(chatActions.kind, "github_webhook_ingress"), - eq(chatActions.status, "failed"), - sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'false'`, - ), - ) - .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.createdAt})`), desc(chatActions.id)) - .limit(limit), - db - .select() - .from(chatActions) - .where( - and( - eq(chatActions.endpointId, endpointId), - before(chatActions.updatedAt, chatActions.id), - inArray(chatActions.kind, [ - "slack_session_sync", - "slack_session_stop", - ]), - ), - ) - .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.updatedAt})`), desc(chatActions.id)) - .limit(limit), - db - .select({ - action: chatActions, - ingressStatus: recoveryIngress.status, - }) - .from(chatActions) - .leftJoin( - recoveryIngress, - and( - eq(recoveryIngress.companyId, chatActions.companyId), - eq(recoveryIngress.endpointId, chatActions.endpointId), - eq(recoveryIngress.kind, "github_webhook_ingress"), - sql`${recoveryIngress.providerActionId} = 'github_webhook_ingress:' || (${chatActions.payload}#>>'{original,guid}')`, - ), - ) - .where( - and( - eq(chatActions.endpointId, endpointId), - before(chatActions.updatedAt, chatActions.id), - eq(chatActions.kind, "github_webhook_recovery"), - ), - ) - .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.updatedAt})`), desc(chatActions.id)) - .limit(limit), - db - .select() - .from(chatSdkState) - .where( - and( - eq(chatSdkState.endpointId, endpointId), - before(chatSdkState.updatedAt, chatSdkState.id), - eq(chatSdkState.stateKey, GITHUB_RECOVERY_STATE_KEY), - ), - ) - .limit(1), - ]); - // A provider effect can be on another page. Replay safety must still - // consider every unresolved effect associated with these deliveries. - const ambiguousEffects = deliveries.length ? await db.select({ deliveryId: chatActions.deliveryId }) - .from(chatActions).where(and( - eq(chatActions.endpointId, endpointId), - eq(chatActions.kind, "provider_effect"), - eq(chatActions.status, "delivery_unknown"), - inArray(chatActions.deliveryId, deliveries.map((row) => row.id)), - )) : []; - const ambiguousProviderEffectDeliveryIds = new Set(ambiguousEffects.map((row) => row.deliveryId)); - const transferRows = publications.length - ? await db - .select(fileTransferProjectionColumns) - .from(chatTeamsFileTransfers) - .innerJoin( - chatPublications, - and( - eq(chatPublications.id, chatTeamsFileTransfers.publicationId), - eq(chatPublications.companyId, chatTeamsFileTransfers.companyId), - eq( - chatPublications.endpointId, - chatTeamsFileTransfers.endpointId, - ), - eq( - chatPublications.conversationId, - chatTeamsFileTransfers.conversationId, - ), - ), - ) - .where( - and( - eq(chatTeamsFileTransfers.companyId, publications[0]!.companyId), - eq(chatTeamsFileTransfers.endpointId, endpointId), - inArray( - chatTeamsFileTransfers.publicationId, - publications.map((row) => row.id), - ), - ), - ) - : []; - const transfers = new Map( - transferRows.map((row) => [row.publicationId, row]), - ); - const cancellableConflicts = new Map(); - for (const transfer of transferRows) { - if (transfer.phase !== "conflict") continue; - const publication = publications.find( - (row) => row.id === transfer.publicationId, - ); - if (!publication?.conversationId) continue; - if ( - await teamsTransferProtocol().canCancelConflict({ - companyId: publication.companyId, - endpointId, - conversationId: publication.conversationId, - publicationId: publication.id, - version: transfer.version, - }) - ) - cancellableConflicts.set(publication.id, transfer.version); - } - return [ - ...deliveries.map((row) => { - const normalized = - row.normalizedEvent && - typeof row.normalizedEvent === "object" && - !Array.isArray(row.normalizedEvent) - ? (row.normalizedEvent as Record) - : {}; - const deduplication = - normalized.deduplication && - typeof normalized.deduplication === "object" && - !Array.isArray(normalized.deduplication) - ? (normalized.deduplication as Record) - : null; - const duplicateCount = - typeof deduplication?.duplicateCount === "number" - ? deduplication.duplicateCount - : 0; - const label = row.eventKind.replaceAll("_", " "); - const outcome = - row.state === "filtered" - ? "ignored" - : row.state === "processed" - ? "processed" - : row.state; - return { - id: row.id, - kind: "delivery" as const, - status: row.state, - summary: `${label} ${outcome}${duplicateCount ? ` · ${duplicateCount} duplicate${duplicateCount === 1 ? "" : "s"} ignored` : ""}`, - detail: row.redactedError, - createdAt: row.createdAt.toISOString(), - replayable: - row.state === "failed" && - Boolean(row.conversationId) && - !ambiguousProviderEffectDeliveryIds.has(row.id), - resolutionActions: [], - }; - }), - ...publications.map((row) => { - const transfer = transfers.get(row.id); - const projected = projectChatFileTransfer( - publicationSummary(row), - transfer, - ); - const payload = row.payload as Partial; - const summary = payload.progressState - ? `${payload.progressState.replaceAll("_", " ")} update` - : payload.interactionId - ? "Interactive question" - : "Response to external conversation"; - return { - id: row.id, - kind: "publication" as const, - status: projected.state, - summary, - detail: projected.redactedError ?? null, - createdAt: row.createdAt.toISOString(), - replayable: !transfer && row.state === "failed", - ...(projected.fileTransfer - ? { fileTransfer: projected.fileTransfer } - : {}), - resolutionActions: transfer - ? chatFileTransferResolutionActions( - projected, - cancellableConflicts.has(row.id) - ? { - publicationId: row.id, - version: cancellableConflicts.get(row.id)!, - } - : undefined, - ) - : row.state === "delivery_unknown" - ? (["mark_delivered", "retry_anyway", "cancel"] as const) - : [], - }; - }), - ...slashActions.map((row) => { - const recovery = slackSlashTaskRecoveryPayload(row.payload); - // Keep this GET read-only while still surfacing a crashed attempt as - // actionable. The resolution POST persists the derived quarantine - // before it applies the operator's audited decision. - const status = slackTaskStartActivityStatus(row); - return { - id: row.id, - kind: "action" as const, - actionType: "slash_task_start" as const, - status, - summary: `Slack slash-command task start ${status.replaceAll("_", " ")}`, - detail: - status === "delivery_unknown" - ? recovery - ? "Slack may have accepted the task-start message, so Paperclip will not replay it automatically. Check Slack first; an explicit retry can create a duplicate starter message and task." - : "Slack may have accepted the task-start message. This older action lacks the context required for a safe explicit retry, so check Slack and cancel it here before submitting a new command." - : status === "provider_confirmed" - ? "Slack accepted the task-start message. Paperclip is completing durable task admission without sending another Slack message." - : status === "admitting" - ? "Slack accepted the task-start message. A Paperclip worker is admitting the task without replaying the Slack send." - : status === "failed" - ? "Slack rejected the task-start message. Submit the command again to retry." - : status === "cancelled" - ? "An operator cancelled this unconfirmed task start." - : null, - createdAt: row.createdAt.toISOString(), - replayable: false, - resolutionActions: - status === "delivery_unknown" - ? recovery - ? (["retry_anyway", "cancel"] as const) - : (["cancel"] as const) - : [], - }; - }), - ...providerEffects.map((row) => { - const payload = providerEffectPayload(row.payload); - return { - id: row.id, - kind: "action" as const, - actionType: "provider_effect" as const, - status: row.status, - summary: "Provider reply delivery unknown", - detail: payload?.completeConversationId - ? "The provider may have accepted this reply, but Paperclip could not confirm it or close the task conversation. Check the provider first. Marking it delivered closes the conversation; retrying can create a duplicate message." - : "The provider may have accepted this reply, but Paperclip could not confirm it. Check the provider first. Retrying can create a duplicate message.", - createdAt: row.createdAt.toISOString(), - replayable: false, - resolutionActions: [ - "mark_delivered", - "retry_anyway", - "cancel", - ] as const, - }; - }), - ...slackSessionActions.map((row) => { - const isSync = row.kind === "slack_session_sync"; - const retrying = - row.status === "received" && - row.result?.code === "slack_session_sync_retry"; - return { - id: row.id, - kind: "action" as const, - actionType: isSync - ? ("slack_session_sync" as const) - : ("slack_session_stop" as const), - status: retrying ? "retry" : row.status, - summary: isSync ? "Slack session status" : "Slack Stop request", - detail: isSync - ? row.result?.outcome === "unavailable" - ? "Slack has not enabled native session status for this destination. Message delivery continues." - : row.result?.code === "slack_session_sync_rejected" - ? "Slack rejected the session indicator update. Check app permissions and channel access; message delivery is tracked separately." - : retrying - ? "The Slack session indicator is waiting to sync. Paperclip will not resend the response." - : row.result?.sessionStatus === "processing" - ? "Working status is refreshed automatically while the run remains active." - : null - : row.status === "processed" - ? "Paperclip stopped the work authorized by this request." - : row.status === "cancelled" - ? "No work was stopped: this request was no longer authorized or its target was no longer current." - : row.status === "failed" - ? row.result?.retryable === true - ? "The Stop request could not finish yet. Paperclip will retry against the original work only." - : "The Stop request could not be completed. Check the task's current run before trying again." - : "Paperclip is processing this Stop request against its original task and run.", - createdAt: row.updatedAt.toISOString(), - replayable: false, - resolutionActions: [], - }; - }), - ...githubIngressFailures.map((row) => { - const deliveryId = - typeof row.payload.deliveryId === "string" - ? row.payload.deliveryId - : null; - return { - id: row.id, - kind: "action" as const, - actionType: "github_webhook_ingress" as const, - status: row.status, - summary: "GitHub webhook could not be processed", - detail: deliveryId - ? `GitHub delivery ${deliveryId} was authenticated, but Paperclip could not finish processing it. Fix the connection or destination, then redeliver this delivery from the GitHub App's Recent Deliveries page.` - : "An authenticated GitHub webhook could not be processed. Fix the connection or destination, then redeliver it from the GitHub App's Recent Deliveries page.", - createdAt: row.createdAt.toISOString(), - replayable: false, - resolutionActions: [], - }; - }), - ...githubRecoveries.map(({ action: row, ingressStatus }) => ({ - id: row.id, - kind: "repair" as const, - status: - row.status === "cancelled" - ? "cancelled" - : ingressStatus - ? "received" - : "pending", - summary: - row.status === "cancelled" - ? "GitHub webhook recovery skipped" - : ingressStatus - ? "GitHub webhook received after recovery request" - : "GitHub webhook redelivery requested", - detail: - row.status === "cancelled" - ? row.result?.code === "source_changed_or_unavailable" - ? "The original comment changed or is no longer available. Paperclip did not replay its old contents." - : "This callback was not an eligible current user comment. Paperclip did not replay it." - : ingressStatus - ? "Paperclip received this callback. Its normal access checks and processing still apply." - : "Paperclip asked GitHub to resend a recent missed message. This does not yet confirm receipt or a reply. Automatic requests are limited; if it remains unanswered, check the App's Recent Deliveries and send your request again in the current conversation.", - createdAt: row.updatedAt.toISOString(), - replayable: false, - resolutionActions: [], - })), - ...githubRecoveryScans.flatMap((row) => { - const value = row.value as { outcome?: string } | null; - const detail = - value?.outcome === "history_limit_reached" - ? "Recent webhook volume exceeded automatic recovery's bounded scan. No partial history was replayed. Check the App's Recent Deliveries for missed messages." - : value?.outcome === "callback_mismatch" - ? "The GitHub App's callback no longer matches this connection. Reconnect the App to repair it; repository access will not change." - : value?.outcome === "scan_failed" - ? "Paperclip could not check GitHub's failed deliveries. The check will retry with backoff; normal callbacks are still processed." - : null; - return detail - ? [ - { - id: row.id, - kind: "health" as const, - status: "attention", - summary: "GitHub webhook recovery needs attention", - detail, - createdAt: row.updatedAt.toISOString(), - replayable: false, - resolutionActions: [], - }, - ] - : []; - }), - ] - .sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id)) - .slice(0, limit); - } - - async function listActivityPage(endpointId: string, limit = 25, cursor?: string) { - if (!Number.isInteger(limit) || limit < 1 || limit > 100) throw badRequest("Activity limit must be between 1 and 100"); - let before: { createdAt: string; id: string } | undefined; - if (cursor !== undefined) { - try { - if (cursor.length > 256) throw new Error("Invalid cursor"); - const value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); - if (!Array.isArray(value) || value.length !== 2 - || typeof value[0] !== "string" || new Date(value[0]).toISOString() !== value[0] - || typeof value[1] !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value[1])) throw new Error("Invalid cursor"); - before = { createdAt: value[0], id: value[1] }; - } catch { throw badRequest("Invalid activity cursor"); } - } - const rows = await listActivity(endpointId, { limit, before }); - const items = rows.slice(0, limit); - const last = items.at(-1); - return { - items, - nextCursor: rows.length > limit && last - ? Buffer.from(JSON.stringify([last.createdAt, last.id])).toString("base64url") - : null, - }; - } - async function replayDelivery(endpointId: string, deliveryId: string) { const delivery = await db .select() @@ -22685,118 +17763,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }); } - async function publicationFiles( - publication: typeof chatPublications.$inferSelect, - payload: SafeChatPublicationPayload, - preparation: { teamsInlineImagesOnly?: boolean } = {}, - ): Promise { - if (!payload.attachmentIds?.length) return []; - try { - const rows = await db - .select({ - id: issueAttachments.id, - issueId: issueAttachments.issueId, - issueCommentId: issueAttachments.issueCommentId, - objectKey: assets.objectKey, - contentType: assets.contentType, - byteSize: assets.byteSize, - sha256: assets.sha256, - originalFilename: assets.originalFilename, - }) - .from(issueAttachments) - .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) - .where( - and( - eq(issueAttachments.companyId, publication.companyId), - eq(issueAttachments.issueId, publication.issueId), - inArray(issueAttachments.id, payload.attachmentIds), - ), - ); - const byId = new Map(rows.map((row) => [row.id, row])); - const orderedRows = payload.attachmentIds.map((attachmentId) => { - const row = byId.get(attachmentId); - if ( - !row || - (publication.commentId && - row.issueCommentId !== publication.commentId) || - row.byteSize <= 0 || - row.byteSize > MAX_ATTACHMENT_BYTES || - !row.objectKey.trim() || - !/^[0-9a-f]{64}$/iu.test(row.sha256) || - !isAllowedContentType(row.contentType) - ) - throw new OutboundAttachmentValidationError( - "Chat publication attachment is invalid or outside its authorized task comment", - ); - return row; - }); - // Picture messages have a different Teams contract from arbitrary files. - // Preserve the normal scope/metadata checks above, but never hydrate a - // document or oversized image just to discover it needs a task link. - if ( - preparation.teamsInlineImagesOnly && - orderedRows.some( - (row) => - row.byteSize > TEAMS_INLINE_IMAGE_MAX_BYTES || - !isTeamsInlineImageContentType(row.contentType), - ) - ) { - return []; - } - // Authorization and persisted metadata must fail definitively even when - // storage is unavailable; do not mask an invalid file as a transient outage. - if (!options.storage) - throw new OutboundAttachmentHydrationError( - "Attachment storage is unavailable for chat publication", - ); - const files: FileUpload[] = []; - for (const row of orderedRows) { - files.push( - await hydrateOutboundAttachment({ - storage: options.storage, - companyId: publication.companyId, - objectKey: row.objectKey, - byteSize: row.byteSize, - sha256: row.sha256, - filename: row.originalFilename ?? `attachment-${row.id}`, - mimeType: row.contentType, - }), - ); - } - return files; - } catch (error) { - if ( - isOutboundAttachmentHydrationError(error) || - isOutboundAttachmentValidationError(error) - ) { - throw error; - } - throw new OutboundAttachmentHydrationError( - "Chat publication attachment preparation failed", - error, - ); - } - } - - function discordMarkdownAttachment(text: string): FileUpload { - return { - data: Buffer.from(text, "utf8"), - filename: "paperclip-response.md", - mimeType: "text/markdown; charset=utf-8", - }; - } - - function telegramMarkdownAttachment(text: string): Attachment { - const data = Buffer.from(text, "utf8"); - return { - data, - mimeType: "text/markdown; charset=utf-8", - name: "paperclip-response.md", - size: data.byteLength, - type: "file", - }; - } - async function recordExactOutboundPublicationLink( tx: DbOrTransaction, input: { @@ -26176,6 +21142,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }, lease, actor.userId); }, }); + const { verifySlackRequestUrl } = createSlackCallbackVerification(db, { resolveCredentialRefs, getWebhookPublicBaseUrl, scheduleMessageProcessing, slackRegistration }); const sendSlackTestMessage = slackTestMessageService(db, { fetch: fetchImpl, diff --git a/server/src/services/chat-channels/activity.ts b/server/src/services/chat-channels/activity.ts new file mode 100644 index 00000000000..0a9824ae577 --- /dev/null +++ b/server/src/services/chat-channels/activity.ts @@ -0,0 +1,616 @@ +import { providerEffectPayload } from "./provider-effects.js"; + +import { slackSlashTaskRecoveryPayload } from "./slack/task-start.js"; +import { GITHUB_RECOVERY_STATE_KEY } from "./github/webhook-ingress.js"; + +import { + and, + desc, + eq, + inArray, + sql, +} from "drizzle-orm"; +import { alias, type AnyPgColumn } from "drizzle-orm/pg-core"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatConversations, + chatDeliveries, + chatPublications, + chatSdkState, + chatTeamsFileTransfers, + issues, +} from "@paperclipai/db"; +import type { + ChatPublicationSummary, + SafeChatPublicationPayload, +} from "@paperclipai/shared"; + +import { + teamsFileTransferService, +} from "../chat-teams-file-transfers.js"; + +import { + chatFileTransferResolutionActions, + projectChatFileTransfer, +} from "../chat-publication-batches.js"; + +import { + badRequest, + notFound, +} from "../../errors.js"; + +import { chatProviderConversationUrl } from "../chat-provider-links.js"; + +import type { EndpointRecord } from "./runtime-types.js"; +import { slackTaskStartActivityStatus } from "./slack/task-start.js"; +import { fileTransferProjectionColumns } from "./microsoft-teams/file-transfer-projection.js"; + +function publicationSummary( + row: typeof chatPublications.$inferSelect, +): ChatPublicationSummary { + return { + id: row.id, + state: row.state, + providerUrl: row.providerUrl, + attempts: row.attempts, + redactedError: row.redactedError, + nextAttemptAt: row.nextAttemptAt?.toISOString() ?? null, + publishedAt: row.publishedAt?.toISOString() ?? null, + }; +} +export { publicationSummary }; + +type ChatActivityOptions = { + endpointRecord: (endpointId: string) => Promise; + teamsTransferProtocol: () => Pick, "canCancelConflict">; +}; + +export function createChatActivity(db: Db, options: ChatActivityOptions) { + const { endpointRecord, teamsTransferProtocol } = options; + + + async function listConversations(endpointId: string) { + const record = await endpointRecord(endpointId); + if (!record) throw notFound("Chat endpoint not found"); + const conversations = await db + .select() + .from(chatConversations) + .where(eq(chatConversations.endpointId, endpointId)) + .orderBy( + desc(chatConversations.lastActivityAt), + desc(chatConversations.createdAt), + ); + if (!conversations.length) return []; + const issueRows = await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + updatedAt: issues.updatedAt, + }) + .from(issues) + .where( + inArray( + issues.id, + conversations.map((row) => row.issueId), + ), + ); + const publications = await db + .select() + .from(chatPublications) + .where(eq(chatPublications.endpointId, endpointId)) + .orderBy(desc(chatPublications.createdAt)); + const issueById = new Map(issueRows.map((row) => [row.id, row])); + const lastPublication = new Map< + string, + typeof chatPublications.$inferSelect + >(); + for (const publication of publications) + if (!lastPublication.has(publication.conversationId)) + lastPublication.set(publication.conversationId, publication); + return conversations.map((conversation) => { + const issue = issueById.get(conversation.issueId); + const externalUrl = + record.endpoint.provider === "telegram" && conversation.isDirectMessage + ? chatProviderConversationUrl({ + provider: "telegram", + botUsername: record.endpoint.botUsername, + threadId: conversation.externalThreadId, + providerMessageId: "", + raw: { chat: { type: "private" } }, + }) + : (conversation.providerUrl ?? + (record.endpoint.provider === "slack" + ? chatProviderConversationUrl({ + provider: "slack", + providerAccountId: record.endpoint.providerAccountId, + threadId: conversation.externalThreadId, + providerMessageId: "", + }) + : null)); + return { + id: conversation.id, + companyId: conversation.companyId, + endpointId: conversation.endpointId, + resourceId: conversation.resourceId, + externalLabel: conversation.externalLabel, + externalUrl, + externalConversationId: conversation.externalConversationId, + externalThreadId: conversation.externalThreadId, + sessionGeneration: conversation.sessionGeneration, + issueId: conversation.issueId, + issueIdentifier: issue?.identifier ?? null, + issueTitle: issue?.title ?? null, + isDirectMessage: conversation.isDirectMessage, + state: + record.endpoint.provider !== "imessage-photon" && + (issue?.status === "done" || issue?.status === "cancelled") + ? "completed" + : conversation.state, + lastActivityAt: conversation.lastActivityAt?.toISOString() ?? null, + createdAt: conversation.createdAt.toISOString(), + updatedAt: ( + conversation.lastActivityAt ?? conversation.updatedAt + ).toISOString(), + lastPublicationStatus: + lastPublication.get(conversation.id)?.state ?? null, + }; + }); + } + + async function listActivity(endpointId: string, page?: { limit: number; before?: { createdAt: string; id: string } }) { + const limit = page ? page.limit + 1 : 100; + // Match the millisecond precision of the public timestamp and use a UUID + // tie-breaker so equal timestamps never skip records between pages. + const before = (date: AnyPgColumn, id: AnyPgColumn) => page?.before + ? sql`(date_trunc('milliseconds', ${date}), ${id}) < (${page.before.createdAt}::timestamptz, ${page.before.id}::uuid)` + : undefined; + const recoveryIngress = alias(chatActions, "github_recovery_ingress"); + const [ + deliveries, + publications, + slashActions, + providerEffects, + githubIngressFailures, + slackSessionActions, + githubRecoveries, + githubRecoveryScans, + ] = await Promise.all([ + db + .select() + .from(chatDeliveries) + .where(and(eq(chatDeliveries.endpointId, endpointId), before(chatDeliveries.createdAt, chatDeliveries.id))) + .orderBy(desc(sql`date_trunc('milliseconds', ${chatDeliveries.createdAt})`), desc(chatDeliveries.id)) + .limit(limit), + db + .select() + .from(chatPublications) + .where(and(eq(chatPublications.endpointId, endpointId), before(chatPublications.createdAt, chatPublications.id))) + .orderBy(desc(sql`date_trunc('milliseconds', ${chatPublications.createdAt})`), desc(chatPublications.id)) + .limit(limit), + db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, endpointId), + before(chatActions.createdAt, chatActions.id), + eq(chatActions.kind, "slash_task_start"), + ), + ) + .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.createdAt})`), desc(chatActions.id)) + .limit(limit), + db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, endpointId), + before(chatActions.createdAt, chatActions.id), + eq(chatActions.kind, "provider_effect"), + eq(chatActions.status, "delivery_unknown"), + ), + ) + .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.createdAt})`), desc(chatActions.id)) + .limit(limit), + db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, endpointId), + before(chatActions.createdAt, chatActions.id), + eq(chatActions.kind, "github_webhook_ingress"), + eq(chatActions.status, "failed"), + sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'false'`, + ), + ) + .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.createdAt})`), desc(chatActions.id)) + .limit(limit), + db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, endpointId), + before(chatActions.updatedAt, chatActions.id), + inArray(chatActions.kind, [ + "slack_session_sync", + "slack_session_stop", + ]), + ), + ) + .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.updatedAt})`), desc(chatActions.id)) + .limit(limit), + db + .select({ + action: chatActions, + ingressStatus: recoveryIngress.status, + }) + .from(chatActions) + .leftJoin( + recoveryIngress, + and( + eq(recoveryIngress.companyId, chatActions.companyId), + eq(recoveryIngress.endpointId, chatActions.endpointId), + eq(recoveryIngress.kind, "github_webhook_ingress"), + sql`${recoveryIngress.providerActionId} = 'github_webhook_ingress:' || (${chatActions.payload}#>>'{original,guid}')`, + ), + ) + .where( + and( + eq(chatActions.endpointId, endpointId), + before(chatActions.updatedAt, chatActions.id), + eq(chatActions.kind, "github_webhook_recovery"), + ), + ) + .orderBy(desc(sql`date_trunc('milliseconds', ${chatActions.updatedAt})`), desc(chatActions.id)) + .limit(limit), + db + .select() + .from(chatSdkState) + .where( + and( + eq(chatSdkState.endpointId, endpointId), + before(chatSdkState.updatedAt, chatSdkState.id), + eq(chatSdkState.stateKey, GITHUB_RECOVERY_STATE_KEY), + ), + ) + .limit(1), + ]); + // A provider effect can be on another page. Replay safety must still + // consider every unresolved effect associated with these deliveries. + const ambiguousEffects = deliveries.length ? await db.select({ deliveryId: chatActions.deliveryId }) + .from(chatActions).where(and( + eq(chatActions.endpointId, endpointId), + eq(chatActions.kind, "provider_effect"), + eq(chatActions.status, "delivery_unknown"), + inArray(chatActions.deliveryId, deliveries.map((row) => row.id)), + )) : []; + const ambiguousProviderEffectDeliveryIds = new Set(ambiguousEffects.map((row) => row.deliveryId)); + const transferRows = publications.length + ? await db + .select(fileTransferProjectionColumns) + .from(chatTeamsFileTransfers) + .innerJoin( + chatPublications, + and( + eq(chatPublications.id, chatTeamsFileTransfers.publicationId), + eq(chatPublications.companyId, chatTeamsFileTransfers.companyId), + eq( + chatPublications.endpointId, + chatTeamsFileTransfers.endpointId, + ), + eq( + chatPublications.conversationId, + chatTeamsFileTransfers.conversationId, + ), + ), + ) + .where( + and( + eq(chatTeamsFileTransfers.companyId, publications[0]!.companyId), + eq(chatTeamsFileTransfers.endpointId, endpointId), + inArray( + chatTeamsFileTransfers.publicationId, + publications.map((row) => row.id), + ), + ), + ) + : []; + const transfers = new Map( + transferRows.map((row) => [row.publicationId, row]), + ); + const cancellableConflicts = new Map(); + for (const transfer of transferRows) { + if (transfer.phase !== "conflict") continue; + const publication = publications.find( + (row) => row.id === transfer.publicationId, + ); + if (!publication?.conversationId) continue; + if ( + await teamsTransferProtocol().canCancelConflict({ + companyId: publication.companyId, + endpointId, + conversationId: publication.conversationId, + publicationId: publication.id, + version: transfer.version, + }) + ) + cancellableConflicts.set(publication.id, transfer.version); + } + return [ + ...deliveries.map((row) => { + const normalized = + row.normalizedEvent && + typeof row.normalizedEvent === "object" && + !Array.isArray(row.normalizedEvent) + ? (row.normalizedEvent as Record) + : {}; + const deduplication = + normalized.deduplication && + typeof normalized.deduplication === "object" && + !Array.isArray(normalized.deduplication) + ? (normalized.deduplication as Record) + : null; + const duplicateCount = + typeof deduplication?.duplicateCount === "number" + ? deduplication.duplicateCount + : 0; + const label = row.eventKind.replaceAll("_", " "); + const outcome = + row.state === "filtered" + ? "ignored" + : row.state === "processed" + ? "processed" + : row.state; + return { + id: row.id, + kind: "delivery" as const, + status: row.state, + summary: `${label} ${outcome}${duplicateCount ? ` · ${duplicateCount} duplicate${duplicateCount === 1 ? "" : "s"} ignored` : ""}`, + detail: row.redactedError, + createdAt: row.createdAt.toISOString(), + replayable: + row.state === "failed" && + Boolean(row.conversationId) && + !ambiguousProviderEffectDeliveryIds.has(row.id), + resolutionActions: [], + }; + }), + ...publications.map((row) => { + const transfer = transfers.get(row.id); + const projected = projectChatFileTransfer( + publicationSummary(row), + transfer, + ); + const payload = row.payload as Partial; + const summary = payload.progressState + ? `${payload.progressState.replaceAll("_", " ")} update` + : payload.interactionId + ? "Interactive question" + : "Response to external conversation"; + return { + id: row.id, + kind: "publication" as const, + status: projected.state, + summary, + detail: projected.redactedError ?? null, + createdAt: row.createdAt.toISOString(), + replayable: !transfer && row.state === "failed", + ...(projected.fileTransfer + ? { fileTransfer: projected.fileTransfer } + : {}), + resolutionActions: transfer + ? chatFileTransferResolutionActions( + projected, + cancellableConflicts.has(row.id) + ? { + publicationId: row.id, + version: cancellableConflicts.get(row.id)!, + } + : undefined, + ) + : row.state === "delivery_unknown" + ? (["mark_delivered", "retry_anyway", "cancel"] as const) + : [], + }; + }), + ...slashActions.map((row) => { + const recovery = slackSlashTaskRecoveryPayload(row.payload); + // Keep this GET read-only while still surfacing a crashed attempt as + // actionable. The resolution POST persists the derived quarantine + // before it applies the operator's audited decision. + const status = slackTaskStartActivityStatus(row); + return { + id: row.id, + kind: "action" as const, + actionType: "slash_task_start" as const, + status, + summary: `Slack slash-command task start ${status.replaceAll("_", " ")}`, + detail: + status === "delivery_unknown" + ? recovery + ? "Slack may have accepted the task-start message, so Paperclip will not replay it automatically. Check Slack first; an explicit retry can create a duplicate starter message and task." + : "Slack may have accepted the task-start message. This older action lacks the context required for a safe explicit retry, so check Slack and cancel it here before submitting a new command." + : status === "provider_confirmed" + ? "Slack accepted the task-start message. Paperclip is completing durable task admission without sending another Slack message." + : status === "admitting" + ? "Slack accepted the task-start message. A Paperclip worker is admitting the task without replaying the Slack send." + : status === "failed" + ? "Slack rejected the task-start message. Submit the command again to retry." + : status === "cancelled" + ? "An operator cancelled this unconfirmed task start." + : null, + createdAt: row.createdAt.toISOString(), + replayable: false, + resolutionActions: + status === "delivery_unknown" + ? recovery + ? (["retry_anyway", "cancel"] as const) + : (["cancel"] as const) + : [], + }; + }), + ...providerEffects.map((row) => { + const payload = providerEffectPayload(row.payload); + return { + id: row.id, + kind: "action" as const, + actionType: "provider_effect" as const, + status: row.status, + summary: "Provider reply delivery unknown", + detail: payload?.completeConversationId + ? "The provider may have accepted this reply, but Paperclip could not confirm it or close the task conversation. Check the provider first. Marking it delivered closes the conversation; retrying can create a duplicate message." + : "The provider may have accepted this reply, but Paperclip could not confirm it. Check the provider first. Retrying can create a duplicate message.", + createdAt: row.createdAt.toISOString(), + replayable: false, + resolutionActions: [ + "mark_delivered", + "retry_anyway", + "cancel", + ] as const, + }; + }), + ...slackSessionActions.map((row) => { + const isSync = row.kind === "slack_session_sync"; + const retrying = + row.status === "received" && + row.result?.code === "slack_session_sync_retry"; + return { + id: row.id, + kind: "action" as const, + actionType: isSync + ? ("slack_session_sync" as const) + : ("slack_session_stop" as const), + status: retrying ? "retry" : row.status, + summary: isSync ? "Slack session status" : "Slack Stop request", + detail: isSync + ? row.result?.outcome === "unavailable" + ? "Slack has not enabled native session status for this destination. Message delivery continues." + : row.result?.code === "slack_session_sync_rejected" + ? "Slack rejected the session indicator update. Check app permissions and channel access; message delivery is tracked separately." + : retrying + ? "The Slack session indicator is waiting to sync. Paperclip will not resend the response." + : row.result?.sessionStatus === "processing" + ? "Working status is refreshed automatically while the run remains active." + : null + : row.status === "processed" + ? "Paperclip stopped the work authorized by this request." + : row.status === "cancelled" + ? "No work was stopped: this request was no longer authorized or its target was no longer current." + : row.status === "failed" + ? row.result?.retryable === true + ? "The Stop request could not finish yet. Paperclip will retry against the original work only." + : "The Stop request could not be completed. Check the task's current run before trying again." + : "Paperclip is processing this Stop request against its original task and run.", + createdAt: row.updatedAt.toISOString(), + replayable: false, + resolutionActions: [], + }; + }), + ...githubIngressFailures.map((row) => { + const deliveryId = + typeof row.payload.deliveryId === "string" + ? row.payload.deliveryId + : null; + return { + id: row.id, + kind: "action" as const, + actionType: "github_webhook_ingress" as const, + status: row.status, + summary: "GitHub webhook could not be processed", + detail: deliveryId + ? `GitHub delivery ${deliveryId} was authenticated, but Paperclip could not finish processing it. Fix the connection or destination, then redeliver this delivery from the GitHub App's Recent Deliveries page.` + : "An authenticated GitHub webhook could not be processed. Fix the connection or destination, then redeliver it from the GitHub App's Recent Deliveries page.", + createdAt: row.createdAt.toISOString(), + replayable: false, + resolutionActions: [], + }; + }), + ...githubRecoveries.map(({ action: row, ingressStatus }) => ({ + id: row.id, + kind: "repair" as const, + status: + row.status === "cancelled" + ? "cancelled" + : ingressStatus + ? "received" + : "pending", + summary: + row.status === "cancelled" + ? "GitHub webhook recovery skipped" + : ingressStatus + ? "GitHub webhook received after recovery request" + : "GitHub webhook redelivery requested", + detail: + row.status === "cancelled" + ? row.result?.code === "source_changed_or_unavailable" + ? "The original comment changed or is no longer available. Paperclip did not replay its old contents." + : "This callback was not an eligible current user comment. Paperclip did not replay it." + : ingressStatus + ? "Paperclip received this callback. Its normal access checks and processing still apply." + : "Paperclip asked GitHub to resend a recent missed message. This does not yet confirm receipt or a reply. Automatic requests are limited; if it remains unanswered, check the App's Recent Deliveries and send your request again in the current conversation.", + createdAt: row.updatedAt.toISOString(), + replayable: false, + resolutionActions: [], + })), + ...githubRecoveryScans.flatMap((row) => { + const value = row.value as { outcome?: string } | null; + const detail = + value?.outcome === "history_limit_reached" + ? "Recent webhook volume exceeded automatic recovery's bounded scan. No partial history was replayed. Check the App's Recent Deliveries for missed messages." + : value?.outcome === "callback_mismatch" + ? "The GitHub App's callback no longer matches this connection. Reconnect the App to repair it; repository access will not change." + : value?.outcome === "scan_failed" + ? "Paperclip could not check GitHub's failed deliveries. The check will retry with backoff; normal callbacks are still processed." + : null; + return detail + ? [ + { + id: row.id, + kind: "health" as const, + status: "attention", + summary: "GitHub webhook recovery needs attention", + detail, + createdAt: row.updatedAt.toISOString(), + replayable: false, + resolutionActions: [], + }, + ] + : []; + }), + ] + .sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id)) + .slice(0, limit); + } + + async function listActivityPage(endpointId: string, limit = 25, cursor?: string) { + if (!Number.isInteger(limit) || limit < 1 || limit > 100) throw badRequest("Activity limit must be between 1 and 100"); + let before: { createdAt: string; id: string } | undefined; + if (cursor !== undefined) { + try { + if (cursor.length > 256) throw new Error("Invalid cursor"); + const value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); + if (!Array.isArray(value) || value.length !== 2 + || typeof value[0] !== "string" || new Date(value[0]).toISOString() !== value[0] + || typeof value[1] !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value[1])) throw new Error("Invalid cursor"); + before = { createdAt: value[0], id: value[1] }; + } catch { throw badRequest("Invalid activity cursor"); } + } + const rows = await listActivity(endpointId, { limit, before }); + const items = rows.slice(0, limit); + const last = items.at(-1); + return { + items, + nextCursor: rows.length > limit && last + ? Buffer.from(JSON.stringify([last.createdAt, last.id])).toString("base64url") + : null, + }; + } + return { listConversations, listActivity, listActivityPage }; +} diff --git a/server/src/services/chat-channels/discord/gateway-events.ts b/server/src/services/chat-channels/discord/gateway-events.ts new file mode 100644 index 00000000000..e283fdc8630 --- /dev/null +++ b/server/src/services/chat-channels/discord/gateway-events.ts @@ -0,0 +1,253 @@ +import type { EndpointRow, DbTransaction, RuntimeContext, LifecycleRuntimeFence, InternalSetupState } from "../runtime-types.js"; + +import { notifyChatEndpointWork } from "../../chat-work-notifications.js"; + +import { + and, + eq, + inArray, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatConversations, + chatEndpointResources, + chatEndpoints, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatEndpointSetupState, +} from "@paperclipai/shared"; + +import { + type DiscordGatewayCallbackEvent, +} from "../../chat-sdk-runtime.js"; + +import { + type ChatProviderLifecycleEffect, +} from "../../chat-provider-lifecycle.js"; + +import type { EndpointRecord } from "../runtime-types.js"; + +type DiscordGatewayEventsOptions = { + ensureDiscordGatewayRuntimeIsCurrent: (endpointId: string, context: RuntimeContext) => Promise; + discordGatewayRuntimeIsCurrent: (endpointId: string, context: RuntimeContext) => boolean; + runtimeCallbackRecord: (endpointId: string, context: RuntimeContext, statuses: EndpointRow["status"][]) => Promise; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + applyProviderLifecycleEffect: (endpoint: EndpointRow, effect: ChatProviderLifecycleEffect, context?: RuntimeContext) => Promise; + runtimeGeneration: (setup: ChatEndpointSetupState) => number; + discordGatewayEventBarrier?: (event: DiscordGatewayCallbackEvent) => Promise; +}; + +export function createDiscordGatewayEvents(db: Db, options: DiscordGatewayEventsOptions) { + const { ensureDiscordGatewayRuntimeIsCurrent, discordGatewayRuntimeIsCurrent, runtimeCallbackRecord, runtimeCallbackEndpoint, applyProviderLifecycleEffect, runtimeGeneration } = options; + + + function discordGatewayFailureDetail( + event: DiscordGatewayCallbackEvent["event"], + ): string { + if (event.type === "guild_removed") { + return "Discord bot was removed from the configured server"; + } + if (event.type === "guild_unavailable") { + return "Discord server is temporarily unavailable; reconnecting"; + } + if (event.type === "disconnected") { + return `Discord Gateway disconnected${event.code === undefined ? "" : ` (code ${event.code})`}; reconnecting`; + } + if (event.type === "failure") { + const code = event.error.code ?? event.error.status; + return `Discord Gateway failed${code === undefined ? "" : ` (${String(code).slice(0, 64)})`}; ${event.fatal ? "credentials or privileged intents need attention" : "reconnecting"}`; + } + return "Discord Gateway connection needs attention"; + } + + function handleDiscordGatewayEvent( + callback: DiscordGatewayCallbackEvent, + context: RuntimeContext, + ): Promise { + const pending = (context.discordGatewayEventQueue ?? Promise.resolve()) + .catch(() => undefined) + .then(() => processDiscordGatewayEvent(callback, context)); + context.discordGatewayEventQueue = pending; + return pending; + } + + async function processDiscordGatewayEvent( + callback: DiscordGatewayCallbackEvent, + context: RuntimeContext, + ): Promise { + if ( + !(await ensureDiscordGatewayRuntimeIsCurrent( + callback.endpointId, + context, + )) + ) + return; + if (callback.sequence <= (context.discordGatewaySequence ?? 0)) return; + context.discordGatewaySequence = callback.sequence; + await options.discordGatewayEventBarrier?.(callback); + const record = await runtimeCallbackRecord(callback.endpointId, context, [ + "verifying", + "active", + "attention", + ]); + if (!record || record.endpoint.provider !== "discord") return; + const event = callback.event; + if (event.type === "channel_removed") { + await applyProviderLifecycleEffect( + record.endpoint, + { + kind: "resource", + provider: "discord", + providerEventId: `gateway:channel_removed:${event.channelId}`, + providerResourceId: event.channelId, + resourceType: "channel", + label: event.label?.trim() || event.channelId, + availability: "removed", + metadata: { source: "discord_gateway" }, + }, + context, + ); + return; + } + + await db.transaction(async (tx) => { + const current = await runtimeCallbackEndpoint( + tx, + callback.endpointId, + context, + ["verifying", "active", "attention"], + ); + if ( + !current || + current.provider !== "discord" || + !discordGatewayRuntimeIsCurrent(callback.endpointId, context) + ) { + return; + } + const now = new Date(); + if (event.type === "connecting") { + await tx + .update(chatEndpoints) + .set({ + healthMessage: "Discord Gateway connecting", + lastError: null, + lastEventAt: now, + updatedAt: now, + }) + .where(eq(chatEndpoints.id, current.id)); + await tx + .update(toolConnections) + .set({ + healthStatus: "unchecked", + healthMessage: "Discord Gateway connecting", + lastError: null, + healthCheckedAt: now, + updatedAt: now, + }) + .where(eq(toolConnections.id, current.connectionId)); + return; + } + if (event.type === "ready" || event.type === "guild_available") { + const gatewayOwnedAttention = + current.status === "attention" && + current.healthMessage?.startsWith("Discord ") === true; + if (current.status === "attention" && !gatewayOwnedAttention) return; + const status = + current.setup.step === "complete" ? "active" : "verifying"; + if (status !== current.status) await notifyChatEndpointWork(tx); + await tx + .update(chatEndpoints) + .set({ + status, + healthMessage: + status === "active" + ? "Connected" + : "Waiting for a test conversation", + lastError: null, + lastEventAt: now, + updatedAt: now, + }) + .where(eq(chatEndpoints.id, current.id)); + await tx + .update(toolConnections) + .set({ + status: "active", + enabled: true, + healthStatus: "healthy", + healthMessage: "Connected", + lastError: null, + healthCheckedAt: now, + updatedAt: now, + }) + .where(eq(toolConnections.id, current.connectionId)); + return; + } + + const detail = discordGatewayFailureDetail(event); + const transientGuildOutage = event.type === "guild_unavailable"; + const terminal = + event.type === "guild_removed" || + ((event.type === "failure" || event.type === "disconnected") && + event.fatal); + await tx + .update(chatEndpoints) + .set({ + status: transientGuildOutage + ? current.status + : event.type === "guild_removed" + ? "revoked" + : "attention", + healthMessage: detail, + lastError: detail, + lastEventAt: now, + ...(terminal + ? { + setup: { + ...current.setup, + runtimeGeneration: runtimeGeneration(current.setup) + 1, + } as InternalSetupState, + } + : {}), + updatedAt: now, + }) + .where(eq(chatEndpoints.id, current.id)); + await tx + .update(toolConnections) + .set({ + status: terminal ? "disabled" : "active", + enabled: !terminal, + healthStatus: event.type === "guild_removed" ? "failed" : "degraded", + healthMessage: detail, + lastError: detail, + healthCheckedAt: now, + updatedAt: now, + }) + .where(eq(toolConnections.id, current.connectionId)); + if (event.type === "guild_removed") { + await tx + .update(chatEndpointResources) + .set({ availability: "unavailable", updatedAt: now }) + .where( + and( + eq(chatEndpointResources.companyId, current.companyId), + eq(chatEndpointResources.endpointId, current.id), + ), + ); + await tx + .update(chatConversations) + .set({ state: "unavailable", updatedAt: now }) + .where( + and( + eq(chatConversations.companyId, current.companyId), + eq(chatConversations.endpointId, current.id), + inArray(chatConversations.state, ["active", "waiting"]), + ), + ); + } + }); + } + return { handleDiscordGatewayEvent }; +} diff --git a/server/src/services/chat-channels/discord/messages.ts b/server/src/services/chat-channels/discord/messages.ts index f61b2a2e8ef..61582abed26 100644 --- a/server/src/services/chat-channels/discord/messages.ts +++ b/server/src/services/chat-channels/discord/messages.ts @@ -1,3 +1,4 @@ +import type { FileUpload } from "chat"; import { createHash } from "node:crypto"; @@ -28,3 +29,11 @@ export function discordLifecycleFilesDigest(raw: unknown): string { : []; return createHash("sha256").update(JSON.stringify(projection)).digest("hex"); } + +export function discordMarkdownAttachment(text: string): FileUpload { + return { + data: Buffer.from(text, "utf8"), + filename: "paperclip-response.md", + mimeType: "text/markdown; charset=utf-8", + }; + } diff --git a/server/src/services/chat-channels/discord/root-threads.ts b/server/src/services/chat-channels/discord/root-threads.ts new file mode 100644 index 00000000000..1772ce375bb --- /dev/null +++ b/server/src/services/chat-channels/discord/root-threads.ts @@ -0,0 +1,570 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence, InboundRuntimeContext } from "../runtime-types.js"; + +import { notifyChatDeliveryWork } from "../../chat-work-notifications.js"; + +import type { ChatEndpointRuntime } from "../../voice/voice-runtime.js"; + +import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; + +import { + createHash, +} from "node:crypto"; +import { + and, + eq, + inArray, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatConversations, + chatDeliveries, + chatEndpointResources, + chatEndpoints, + chatExternalPrincipals, + companyMemberships, +} from "@paperclipai/db"; + +import { + conflict, +} from "../../../errors.js"; +import { logger } from "../../../middleware/logger.js"; + +import { + DiscordAdapterCompatibilityError, + type ChatSdkMessageCallbackEvent, + type DiscordRootMentionAdmissionEvent, +} from "../../chat-sdk-runtime.js"; + +import { classifyChatPublicationError } from "../../chat-publication-errors.js"; + +import type { + Message, + Thread, +} from "chat"; + +type DeliveryRow = typeof chatDeliveries.$inferSelect; + +type DiscordRootThreadsOptions = { + ensureDiscordGatewayRuntimeIsCurrent: (endpointId: string, context: RuntimeContext) => Promise; + withCredentialMutationLease: ReturnType; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; linkedDenied: boolean }>; + processMessage: (endpoint: EndpointRow, thread: Thread, message: Message, trigger: ChatSdkMessageCallbackEvent["trigger"], ingressOnly?: boolean, recoveredProviderUrl?: string | null, runtimeContext?: InboundRuntimeContext, providerUpdateId?: number, admittedDeliveryId?: string | null, receiptReactionSupported?: boolean, preauthorizedUserId?: string | null, deferDrainUntilFollowup?: boolean) => Promise; + redactError: (error: unknown) => string; + scheduleConversationDrain: (endpointId: string, threadId: string) => void; + lifecycleRuntimeFence: (delivery: DeliveryRow) => LifecycleRuntimeFence | null; + runtimeFor: (endpoint: EndpointRow) => Promise; + DISCORD_ROOT_THREAD_CONFIRMATION_DELAY_MS: number; + discordRootThreadTransportBarrier?: () => Promise; +}; + +export function createDiscordRootThreads(db: Db, options: DiscordRootThreadsOptions) { + const { ensureDiscordGatewayRuntimeIsCurrent, withCredentialMutationLease, runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, processMessage, redactError, scheduleConversationDrain, lifecycleRuntimeFence, runtimeFor, DISCORD_ROOT_THREAD_CONFIRMATION_DELAY_MS } = options; + + + async function admitDiscordRootMention( + configuredEndpoint: EndpointRow, + event: DiscordRootMentionAdmissionEvent, + context: RuntimeContext, + ): Promise { + if ( + !(await ensureDiscordGatewayRuntimeIsCurrent( + configuredEndpoint.id, + context, + )) + ) { + return false; + } + return withCredentialMutationLease( + configuredEndpoint, + async (credentialLease) => { + const threadId = `discord:${event.guildId}:${event.channelId}:${event.messageId}`; + const providerEventId = `${threadId}:${event.messageId}`; + const allowed = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const endpoint = await runtimeCallbackEndpoint( + tx, + configuredEndpoint.id, + context, + ["verifying", "active"], + ); + const existing = await tx + .select({ state: chatDeliveries.state }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.endpointId, configuredEndpoint.id), + eq(chatDeliveries.providerEventId, providerEventId), + ), + ) + .then((rows) => rows[0] ?? null); + if ( + existing && + ["processed", "filtered", "failed"].includes(existing.state) + ) + return false; + + let reason: string | null = null; + if (!endpoint || endpoint.providerAccountId !== event.guildId) { + reason = "Connection is not active"; + } + const resource = endpoint + ? await tx + .select() + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.endpointId, endpoint.id), + eq(chatEndpointResources.type, "channel"), + eq( + chatEndpointResources.providerResourceId, + event.channelId, + ), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const enabledCount = + endpoint?.status === "verifying" + ? await tx + .select({ count: sql`count(*)::int` }) + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.endpointId, endpoint.id), + eq(chatEndpointResources.type, "channel"), + eq(chatEndpointResources.enabled, true), + eq(chatEndpointResources.availability, "available"), + ), + ) + .then((rows) => rows[0]?.count ?? 0) + : 0; + const setupDestination = + endpoint?.status === "verifying" && + enabledCount === 0 && + resource?.availability === "available"; + if ( + !reason && + !( + endpoint?.allowGroupChats && + resource?.availability === "available" && + (resource.enabled || setupDestination) + ) + ) { + reason = "Destination is not enabled in Paperclip"; + } + + if (!reason && endpoint) { + const principal = await tx + .select({ id: chatExternalPrincipals.id }) + .from(chatExternalPrincipals) + .where( + and( + eq(chatExternalPrincipals.companyId, endpoint.companyId), + eq(chatExternalPrincipals.provider, "discord"), + eq(chatExternalPrincipals.providerAccountId, event.guildId), + eq(chatExternalPrincipals.externalId, event.userId), + ), + ) + .then((rows) => rows[0] ?? null); + let allowed: boolean; + let linkedDenied = false; + if (principal) { + const authorization = await lockCurrentPrincipalAuthorization( + tx, + endpoint, + principal.id, + ); + allowed = authorization.allowed; + linkedDenied = authorization.linkedDenied; + } else if (!endpoint.allowUnlinkedPeople) { + allowed = false; + } else if (!endpoint.sponsorUserId) { + allowed = true; + } else { + const sponsor = await tx + .select({ + status: companyMemberships.status, + membershipRole: companyMemberships.membershipRole, + }) + .from(companyMemberships) + .where( + and( + eq(companyMemberships.companyId, endpoint.companyId), + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, endpoint.sponsorUserId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + allowed = + sponsor?.status === "active" && + sponsor.membershipRole !== "viewer"; + } + if (!allowed) { + reason = linkedDenied + ? "Linked Paperclip account is not currently permitted" + : endpoint.allowUnlinkedPeople + ? "Endpoint sponsor can no longer authorize external guests" + : "External identity must be linked to a Paperclip account"; + } + } + if (!reason) { + await credentialLease.assertOwned(tx); + return true; + } + + const filteredAt = new Date(); + const normalizedEvent = { + providerEventId, + kind: "mention", + trigger: "mention", + resource: { + type: "channel", + providerResourceId: event.channelId, + }, + conversation: { externalThreadId: threadId }, + message: { providerMessageId: event.messageId }, + filtering: { contentRetained: false, providerThreadCreated: false }, + }; + await notifyChatDeliveryWork(tx); + await tx + .insert(chatDeliveries) + .values({ + companyId: configuredEndpoint.companyId, + endpointId: configuredEndpoint.id, + providerEventId, + deduplicationKey: createHash("sha256") + .update(providerEventId) + .digest("hex"), + eventKind: "mention", + normalizedEvent, + state: "filtered", + redactedError: reason, + processedAt: filteredAt, + }) + .onConflictDoUpdate({ + target: [ + chatDeliveries.endpointId, + chatDeliveries.providerEventId, + ], + set: { + normalizedEvent, + principalId: null, + state: "filtered", + nextAttemptAt: null, + redactedError: reason, + processedAt: filteredAt, + updatedAt: filteredAt, + }, + }); + await tx + .update(chatEndpoints) + .set({ lastEventAt: filteredAt, updatedAt: filteredAt }) + .where(eq(chatEndpoints.id, configuredEndpoint.id)); + await credentialLease.assertOwned(tx); + return false; + }); + if (!allowed || !event.message || !context.endpointRuntime) { + return false; + } + const thread = context.endpointRuntime.thread(event.threadId); + await processMessage( + configuredEndpoint, + thread, + event.message, + "mention", + true, + null, + context, + undefined, + null, + true, + undefined, + true, + ); + const staged = await db + .select({ state: chatDeliveries.state }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.endpointId, configuredEndpoint.id), + eq(chatDeliveries.providerEventId, providerEventId), + ), + ) + .then((rows) => rows[0] ?? null); + if ( + !staged || + !["received", "retry", "processing"].includes(staged.state) + ) { + return false; + } + + await options.discordRootThreadTransportBarrier?.(); + await credentialLease.assertOwned(); + if ( + !(await ensureDiscordGatewayRuntimeIsCurrent( + configuredEndpoint.id, + context, + )) + ) { + return false; + } + try { + await context.endpointRuntime.ensureDiscordRootThread({ + channelId: event.channelId, + content: event.message.text, + messageId: event.messageId, + }); + } catch (error) { + // The durable provisional receipt owns recovery. Do not let a + // transient REST failure escape the Gateway event handler or ask the + // adapter to create the same thread outside Paperclip's lifecycle + // fence. + logger.warn( + { + endpointId: configuredEndpoint.id, + error: redactError(error), + providerEventId, + }, + "Discord root thread creation will be retried from the durable receipt", + ); + return false; + } + const promoted = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const current = await runtimeCallbackEndpoint( + tx, + configuredEndpoint.id, + context, + ["verifying", "active"], + ); + if (!current) return false; + await notifyChatDeliveryWork(tx); + const [row] = await tx + .update(chatDeliveries) + .set({ + state: "received", + normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, + nextAttemptAt: null, + redactedError: null, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.endpointId, configuredEndpoint.id), + eq(chatDeliveries.providerEventId, providerEventId), + eq(chatDeliveries.state, "retry"), + sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, + ), + ) + .returning({ id: chatDeliveries.id }); + await credentialLease.assertOwned(tx); + return Boolean(row); + }); + if (promoted) + scheduleConversationDrain(configuredEndpoint.id, threadId); + // Paperclip created the provider thread inside its credential/reach + // lease. Returning false prevents the adapter from creating it again; + // the durable drain owns the remaining task mutation. + return false; + }, + ); + } + + function discordRootThreadPending( + delivery: Pick, + ): boolean { + return ( + Boolean(delivery.normalizedEvent) && + typeof delivery.normalizedEvent === "object" && + !Array.isArray(delivery.normalizedEvent) && + (delivery.normalizedEvent as { providerThreadPending?: unknown }) + .providerThreadPending === true + ); + } + + async function reconcileDiscordRootThread( + endpoint: EndpointRow, + delivery: DeliveryRow, + threadId: string, + ): Promise { + try { + const normalized = delivery.normalizedEvent as { + message?: { providerMessageId?: unknown; text?: unknown }; + resource?: { providerResourceId?: unknown }; + }; + const discordThreadParts = threadId.split(":"); + const channelId = + discordThreadParts[0] === "discord" && discordThreadParts.length >= 4 + ? discordThreadParts[2] + : normalized.resource?.providerResourceId; + const messageId = normalized.message?.providerMessageId; + if (typeof channelId !== "string" || typeof messageId !== "string") { + throw new DiscordAdapterCompatibilityError( + "the provisional root receipt is missing its channel or message id", + ); + } + const runtimeFence = lifecycleRuntimeFence(delivery); + if (!runtimeFence) { + throw new DiscordAdapterCompatibilityError( + "the provisional root receipt is missing its runtime fence", + ); + } + return await withCredentialMutationLease( + endpoint, + async (credentialLease) => { + const currentEndpoint = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const current = await runtimeCallbackEndpoint( + tx, + endpoint.id, + runtimeFence, + ["verifying", "active"], + ); + await credentialLease.assertOwned(tx); + return current; + }); + if (!currentEndpoint) { + throw conflict( + "Discord root thread credentials or connection state changed before recovery", + { code: "chat_endpoint_runtime_superseded" }, + ); + } + const endpointRuntime = await runtimeFor(currentEndpoint); + await credentialLease.assertOwned(); + await endpointRuntime.ensureDiscordRootThread({ + channelId, + messageId, + content: + typeof normalized.message?.text === "string" + ? normalized.message.text + : "Paperclip task", + }); + return await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const current = await runtimeCallbackEndpoint( + tx, + endpoint.id, + runtimeFence, + ["verifying", "active"], + ); + if (!current) return false; + await notifyChatDeliveryWork(tx); + const [promoted] = await tx + .update(chatDeliveries) + .set({ + state: "received", + normalizedEvent: sql`${chatDeliveries.normalizedEvent} - 'providerThreadPending'`, + nextAttemptAt: null, + redactedError: null, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, delivery.id), + eq(chatDeliveries.state, "retry"), + sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, + ), + ) + .returning({ id: chatDeliveries.id }); + await credentialLease.assertOwned(tx); + return Boolean(promoted); + }); + }, + ); + } catch (error) { + const disposition = classifyChatPublicationError( + error, + delivery.attempts + 1, + ); + const terminal = + disposition.kind === "resource_unavailable" || + disposition.kind === "failed"; + const retryAfterMs = + disposition.kind === "retry" + ? disposition.retryAfterMs + : DISCORD_ROOT_THREAD_CONFIRMATION_DELAY_MS; + const now = new Date(); + await db.transaction(async (tx) => { + if (disposition.kind === "resource_unavailable") { + const fence = lifecycleRuntimeFence(delivery); + const normalized = delivery.normalizedEvent as { + resource?: { providerResourceId?: unknown }; + }; + const providerResourceId = normalized.resource?.providerResourceId; + const current = fence + ? await runtimeCallbackEndpoint(tx, endpoint.id, fence, [ + "verifying", + "active", + "attention", + ]) + : null; + const resource = + current && typeof providerResourceId === "string" + ? await tx + .select({ id: chatEndpointResources.id }) + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.companyId, current.companyId), + eq(chatEndpointResources.endpointId, current.id), + eq( + chatEndpointResources.providerResourceId, + providerResourceId, + ), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + if (resource) { + await tx + .update(chatEndpointResources) + .set({ availability: "unavailable", updatedAt: now }) + .where(eq(chatEndpointResources.id, resource.id)); + await tx + .update(chatConversations) + .set({ state: "unavailable", updatedAt: now }) + .where( + and( + eq(chatConversations.companyId, current!.companyId), + eq(chatConversations.endpointId, current!.id), + eq(chatConversations.resourceId, resource.id), + inArray(chatConversations.state, ["active", "waiting"]), + ), + ); + } + } + await notifyChatDeliveryWork(tx); + await tx + .update(chatDeliveries) + .set({ + state: terminal ? "filtered" : "retry", + attempts: delivery.attempts + 1, + nextAttemptAt: terminal + ? null + : new Date(now.getTime() + retryAfterMs), + processedAt: terminal ? now : null, + redactedError: terminal + ? "Discord thread creation was not confirmed" + : redactError(error), + updatedAt: now, + }) + .where( + and( + eq(chatDeliveries.id, delivery.id), + eq(chatDeliveries.state, "retry"), + sql`${chatDeliveries.normalizedEvent}->>'providerThreadPending' = 'true'`, + ), + ); + }); + return false; + } + } + return { admitDiscordRootMention, discordRootThreadPending, reconcileDiscordRootThread }; +} diff --git a/server/src/services/chat-channels/inbound-attachments.ts b/server/src/services/chat-channels/inbound-attachments.ts new file mode 100644 index 00000000000..4be37251b61 --- /dev/null +++ b/server/src/services/chat-channels/inbound-attachments.ts @@ -0,0 +1,674 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction, LifecycleRuntimeFence } from "./runtime-types.js"; + +import type { ChatEndpointRuntime } from "../voice/voice-runtime.js"; + +import { takePhotonCompanion } from "../photon/attachments.js"; + +import { HEIF_CONTENT_TYPES, photonHeifPreview, validatePhotonImage } from "../photon/media.js"; + +import { PhotonError } from "../photon/cloud.js"; + +import { + createHash, +} from "node:crypto"; +import { + and, + eq, + inArray, + isNull, + ne, + or, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + assets, + chatActions, + chatConversations, + chatDeliveries, + issueComments, + issueAttachments, +} from "@paperclipai/db"; + +import { + isAllowedContentType, + MAX_ATTACHMENT_BYTES, + normalizeContentType, + normalizeUploadAttachmentContentType, +} from "../../attachment-types.js"; + +import { + hasTelegramMediaProvenance, + identifyTelegramMedia, + telegramMediaNeedsIdentification, +} from "../chat-telegram-media-intake.js"; + +import { + forbidden, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; + +import type { StorageService } from "../../storage/types.js"; + +import { issueService } from "../issues.js"; + +import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js"; + +import { + GITHUB_ATTACHMENT_BATCH_TIMEOUT_MS, + GitHubAttachmentUnavailableError, + prepareGitHubPublicAttachment, +} from "../chat-github-attachments.js"; + +import type { + Attachment, +} from "chat"; + +type DeliveryRow = typeof chatDeliveries.$inferSelect; + +type InboundAttachmentsOptions = { + storage?: StorageService; + authorizeInboundWakeup: (tx: DbOrTransaction, action: typeof chatActions.$inferSelect) => Promise<{ delivery: DeliveryRow; conversation: typeof chatConversations.$inferSelect }>; + lifecycleRuntimeFence: (delivery: DeliveryRow) => LifecycleRuntimeFence | null; + sanitizeFilename: (value: string | undefined) => string | null; + issuesSvc: Pick, "createAttachment">; + isExternalActionAuthorizationChange: (error: unknown) => boolean; + redactError: (error: unknown) => string; +}; + +export function createInboundAttachments(db: Db, options: InboundAttachmentsOptions) { + const { authorizeInboundWakeup, lifecycleRuntimeFence, sanitizeFilename, issuesSvc, isExternalActionAuthorizationChange, redactError } = options; + + + async function ingestAttachments(input: { + endpoint: EndpointRow; + endpointRuntime: ChatEndpointRuntime; + deliveryId: string; + issueId: string; + issueCommentId: string; + attachments: Attachment[]; + attachmentLimitOmissions?: number; + unavailableReferenceCount?: number; + actorUserId: string | null; + }): Promise<{ + storedIds: string[]; + omissionReasons: Record; + }> { + const omissionReasons: Record = {}; + const omit = (reason: string, count = 1) => { + omissionReasons[reason] = (omissionReasons[reason] ?? 0) + count; + }; + const boundedAttachments = input.attachments.slice(0, 20); + const photonDerivatives = new Map(); + const originalPhotonAttachments = new Map(); + if ( + input.endpoint.provider === "microsoft-teams" && + input.unavailableReferenceCount + ) { + // A channel/group reference is not a download capability. Still tell + // the exact current-turn consumer that those bytes are unavailable, + // including when there are no ingestible attachments at all. + omit("download_unavailable", input.unavailableReferenceCount); + } + if ( + ["github", "imessage-photon"].includes(input.endpoint.provider) && + input.attachmentLimitOmissions + ) { + omit("attachment_limit", input.attachmentLimitOmissions); + } + if (input.attachments.length > boundedAttachments.length) { + omit( + "attachment_limit", + input.attachments.length - boundedAttachments.length, + ); + } + if (!options.storage) { + if (boundedAttachments.length) + omit("storage_unavailable", boundedAttachments.length); + return { storedIds: [], omissionReasons }; + } + const existingByFingerprint = new Map(); + const existingAttachments = await db + .select({ + id: issueAttachments.id, + byteSize: assets.byteSize, + contentType: assets.contentType, + originalFilename: assets.originalFilename, + sha256: assets.sha256, + }) + .from(issueAttachments) + .innerJoin( + assets, + and( + eq(assets.companyId, issueAttachments.companyId), + eq(assets.id, issueAttachments.assetId), + ), + ) + .where( + and( + eq(issueAttachments.companyId, input.endpoint.companyId), + eq(issueAttachments.issueId, input.issueId), + eq(issueAttachments.issueCommentId, input.issueCommentId), + ), + ); + for (const existing of existingAttachments) { + const fingerprint = JSON.stringify([ + existing.sha256, + existing.byteSize, + existing.contentType, + existing.originalFilename, + ]); + const ids = existingByFingerprint.get(fingerprint) ?? []; + ids.push(existing.id); + existingByFingerprint.set(fingerprint, ids); + } + const storedIds: string[] = []; + const withCurrentAttachmentAuthorization = async ( + work: (tx: DbTransaction) => Promise, + enforceUnchangedSource = false, + telegramMedia?: Attachment, + ): Promise => { + for (let attempt = 0; ; attempt++) { + try { + return await db.transaction(async (tx) => { + const action = await tx + .select() + .from(chatActions) + .where( + and( + eq(chatActions.companyId, input.endpoint.companyId), + eq(chatActions.endpointId, input.endpoint.id), + eq(chatActions.deliveryId, input.deliveryId), + eq(chatActions.kind, "inbound_wakeup"), + ), + ) + .limit(1) + .then((rows) => rows[0]); + if ( + !action || + !["preparing", "issued", "processing", "processed"].includes( + action.status, + ) || + action.payload.issueId !== input.issueId || + action.payload.commentId !== input.issueCommentId + ) + throw forbidden( + "The attachment's current chat admission is unavailable", + { code: "chat_action_authorization_changed" }, + ); + const current = await authorizeInboundWakeup(tx, action); + // Delivery settlement/replay can update these receipts without + // taking the issue lock. Lock in delivery -> action order only + // after policy authorization; NOWAIT breaks cross-worker lock + // inversions and the outer loop retries the whole transaction. + const lockedDelivery = await tx + .select() + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.companyId, input.endpoint.companyId), + eq(chatDeliveries.id, input.deliveryId), + ), + ) + .for("update", { noWait: true }) + .limit(1) + .then((rows) => rows[0]); + const lockedAction = await tx + .select() + .from(chatActions) + .where( + and( + eq(chatActions.companyId, input.endpoint.companyId), + eq(chatActions.id, action.id), + ), + ) + .for("update", { noWait: true }) + .limit(1) + .then((rows) => rows[0]); + const scopeKeys = [ + "version", + "issueId", + "commentId", + "agentId", + "sessionGeneration", + "requestedByActorType", + "requestedByActorId", + ] as const; + if ( + current.delivery.state !== "processing" || + !lockedDelivery || + !lockedAction || + lockedDelivery.state !== "processing" || + lockedDelivery.endpointId !== action.endpointId || + lockedDelivery.conversationId !== action.conversationId || + lockedDelivery.principalId !== action.principalId || + lockedDelivery.attempts !== current.delivery.attempts || + JSON.stringify(lifecycleRuntimeFence(lockedDelivery)) !== + JSON.stringify(lifecycleRuntimeFence(current.delivery)) || + lockedAction.endpointId !== action.endpointId || + lockedAction.deliveryId !== action.deliveryId || + lockedAction.conversationId !== action.conversationId || + lockedAction.principalId !== action.principalId || + lockedAction.kind !== "inbound_wakeup" || + !["preparing", "issued", "processing", "processed"].includes( + lockedAction.status, + ) || + scopeKeys.some( + (key) => lockedAction.payload[key] !== action.payload[key], + ) + ) + throw forbidden( + "The attachment's current chat admission is unavailable", + { code: "chat_action_authorization_changed" }, + ); + if (enforceUnchangedSource) { + if (telegramMedia) { + const normalized = lockedDelivery.normalizedEvent as { + principal?: { externalId?: string }; + message?: { + providerMessageId?: string; + attachments?: Array<{ recovery?: unknown }>; + }; + }; + const fence = lifecycleRuntimeFence(lockedDelivery); + const descriptor = + input.endpointRuntime.attachmentRecoveryDescriptor( + telegramMedia, + { + threadId: current.conversation.externalThreadId, + messageId: normalized.message?.providerMessageId ?? "", + principalExternalId: normalized.principal?.externalId, + runtimeGeneration: fence?.generation, + credentialFingerprint: fence?.credentialFingerprint, + }, + ); + if ( + descriptor?.locator.kind !== "telegram_media" || + !normalized.message?.attachments?.some((item) => { + const retained = item.recovery as { + version?: unknown; + provider?: unknown; + locator?: Record; + attachment?: Record; + } | null; + return ( + retained?.version === 1 && + retained.provider === "telegram" && + Object.entries(descriptor.locator).every( + ([key, value]) => retained.locator?.[key] === value, + ) && + Object.entries(descriptor.attachment).every( + ([key, value]) => retained.attachment?.[key] === value, + ) + ); + }) + ) + throw forbidden( + "The media's admitted source is unavailable", + { code: "chat_action_authorization_changed" }, + ); + } + const [source] = await tx + .select({ id: issueComments.id }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, input.endpoint.companyId), + eq(issueComments.issueId, input.issueId), + eq(issueComments.id, input.issueCommentId), + isNull(issueComments.deletedAt), + sql`${issueComments.updatedAt} = ${issueComments.createdAt}`, + ), + ) + .for("share", { noWait: true }); + // Lifecycle admission precedes its conversation drain. Pending + // source changes already invalidate this image download, even + // when the append-only original comment is still untouched. + const [lifecycle] = await tx + .select({ id: chatDeliveries.id }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.companyId, input.endpoint.companyId), + eq(chatDeliveries.endpointId, input.endpoint.id), + or( + isNull(chatDeliveries.conversationId), + eq( + chatDeliveries.conversationId, + current.conversation.id, + ), + ), + ne(chatDeliveries.state, "filtered"), + inArray(chatDeliveries.eventKind, [ + "message_updated", + "message_deleted", + "message_restored", + ]), + sql`${chatDeliveries.normalizedEvent}->'runtimeContext' = ${JSON.stringify(lifecycleRuntimeFence(lockedDelivery))}::jsonb`, + sql`${chatDeliveries.normalizedEvent}->'message'->>'targetProviderEventId' = ${lockedDelivery.providerEventId}`, + ), + ) + .limit(1); + if (!source || lifecycle) + throw forbidden( + "The image's admitted source changed before attachment registration", + { + code: "chat_action_authorization_changed", + }, + ); + } + return await work(tx); + }); + } catch (error) { + if (!isExternalChatWaitAuthorizationContention(error) || attempt >= 4) + throw error; + // Retry the complete short transaction; no network/storage work or + // backoff occurs while governance rows are locked. + await new Promise((resolve) => + setTimeout(resolve, 20 * (attempt + 1)), + ); + } + } + }; + // One shared deadline bounds the sequential batch; an expired batch never + // starts another GitHub request. Other providers retain their own policy. + const githubBatchSignal = + input.endpoint.provider === "github" + ? AbortSignal.timeout(GITHUB_ATTACHMENT_BATCH_TIMEOUT_MS) + : undefined; + // This new image-only lane shares one budget across token acquisition and + // downloads. DB/storage commit semantics remain explicit and unchanged. + const teamsInlineBatchSignal = + input.endpoint.provider === "microsoft-teams" + ? AbortSignal.timeout(10_000) + : undefined; + for (let attachment of boundedAttachments) { + const teamsInlineImage = + input.endpoint.provider === "microsoft-teams" && + input.endpointRuntime.attachmentRecoveryDescriptor(attachment)?.locator + .kind === "teams_inline_image"; + const telegramMedia = + input.endpoint.provider === "telegram" && + hasTelegramMediaProvenance(attachment); + const sourceBoundMedia = + teamsInlineImage || + telegramMedia || + input.endpoint.provider === "imessage-photon"; + const requireCurrentAttachmentAuthorization = + input.endpoint.provider === "github" || sourceBoundMedia; + try { + if (teamsInlineImage && teamsInlineBatchSignal?.aborted) { + omit("download_unavailable"); + continue; + } + // GitHub's anonymized upload URLs carry no trustworthy MIME metadata. + // Resolve public bytes only here, after durable comment admission, and + // then apply the same storage/type policy as every native attachment. + if (input.endpoint.provider === "github") { + await withCurrentAttachmentAuthorization(async () => {}); + attachment = await prepareGitHubPublicAttachment( + attachment, + githubBatchSignal, + async (request, signal) => { + await withCurrentAttachmentAuthorization(async () => {}); + return ( + (await input.endpointRuntime.resolveGitHubAttachmentComment?.( + request, + signal, + )) ?? null + ); + }, + ); + } + if ( + requireCurrentAttachmentAuthorization && + input.endpoint.provider !== "github" + ) + await withCurrentAttachmentAuthorization( + async () => {}, + sourceBoundMedia, + telegramMedia ? attachment : undefined, + ); + if ( + attachment.size !== undefined && + attachment.size > MAX_ATTACHMENT_BYTES + ) { + omit("declared_too_large"); + continue; + } + if (!attachment.fetchData) { + omit("download_unavailable"); + continue; + } + const originalFilename = sanitizeFilename(attachment.name); + let contentType = normalizeUploadAttachmentContentType({ + contentType: normalizeContentType( + attachment.mimeType ?? "application/octet-stream", + ), + originalFilename, + isAllowedContentType, + }); + // Reject a provider-declared type before allocating the downloaded + // payload. The actual byte length is checked again after the adapter's + // bounded fetch contract resolves. + // A filename may infer an Office MIME for ordinary documents, but it + // must not bypass byte identification for native media without MIME. + const identifyTelegram = + telegramMedia && + telegramMediaNeedsIdentification(attachment) && + normalizeContentType( + attachment.mimeType ?? "application/octet-stream", + ) === "application/octet-stream"; + if (!isAllowedContentType(contentType) && !identifyTelegram) { + omit("unsupported_type"); + continue; + } + const fetched = teamsInlineImage + ? await input.endpointRuntime.fetchTeamsInlineImage( + attachment, + teamsInlineBatchSignal!, + ) + : await attachment.fetchData(); + const body = Buffer.isBuffer(fetched) ? fetched : Buffer.from(fetched); + if (body.length === 0) { + omit("empty_download"); + continue; + } + if (body.length > MAX_ATTACHMENT_BYTES) { + omit("downloaded_too_large"); + continue; + } + if (identifyTelegram) { + contentType = + identifyTelegramMedia(attachment, body) ?? + "application/octet-stream"; + if (!isAllowedContentType(contentType)) { + omit("unsupported_type"); + continue; + } + } + const photonSourceHash = createHash("sha256") + .update(body) + .digest("hex"); + if (input.endpoint.provider === "imessage-photon") { + await validatePhotonImage(body, contentType); + const companion = takePhotonCompanion(body); + if (companion?.unavailable) omit("companion_unavailable"); + else if (companion) { + const video: Attachment = { type: "video", name: companion.fileName, mimeType: companion.mimeType, size: companion.data.length, fetchData: async () => companion.data }; + photonDerivatives.set(video, { source: attachment, sourceHash: photonSourceHash, kind: "live_photo_video" }); + boundedAttachments.push(video); + } + if ( + HEIF_CONTENT_TYPES.has(contentType) && + isAllowedContentType("image/jpeg") + ) { + try { + const preview = await photonHeifPreview(body); + const derivative: Attachment = { + type: "image", + name: `${originalFilename} (JPEG preview).jpg`, + mimeType: "image/jpeg", + size: preview.length, + fetchData: async () => preview, + }; + photonDerivatives.set(derivative, { source: attachment, sourceHash: photonSourceHash, kind: "heif_jpeg_preview" }); + boundedAttachments.push(derivative); + } catch { + omit("preview_unavailable"); + } + } + } + const fingerprint = JSON.stringify([ + photonSourceHash, + body.length, + contentType, + originalFilename, + ]); + const existingIds = existingByFingerprint.get(fingerprint); + const existingId = existingIds?.shift(); + if (requireCurrentAttachmentAuthorization) { + // Revoke during either API resolution or the anonymous byte download + // must prevent storage and attachment registration for that input. + await withCurrentAttachmentAuthorization( + async () => {}, + sourceBoundMedia, + telegramMedia ? attachment : undefined, + ); + } + const registerPhotonProvenance = async (attachmentId: string) => { + if (input.endpoint.provider !== "imessage-photon") return; + const relation = photonDerivatives.get(attachment); + if (!relation) { + originalPhotonAttachments.set(attachment, attachmentId); + return; + } + const originalAttachmentId = + originalPhotonAttachments.get(relation.source); + if (!originalAttachmentId) + throw new Error("Photon related attachment source is missing"); + await db + .insert(chatActions) + .values({ + companyId: input.endpoint.companyId, + endpointId: input.endpoint.id, + deliveryId: input.deliveryId, + kind: relation.kind === "heif_jpeg_preview" ? "attachment_derivative" : "attachment_companion", + providerActionId: `photon-${relation.kind}:${input.deliveryId}:${originalAttachmentId}`, + payload: { + originalAttachmentId, + derivativeAttachmentId: attachmentId, + originalSha256: relation.sourceHash, + derivativeSha256: photonSourceHash, + kind: relation.kind, + }, + status: "processed", + }) + .onConflictDoNothing(); + }; + if (existingId) { + storedIds.push(existingId); + await registerPhotonProvenance(existingId); + continue; + } + const stored = await options.storage.putFile({ + companyId: input.endpoint.companyId, + namespace: `issues/${input.issueId}`, + originalFilename, + contentType, + body, + }); + const registration = { + issueId: input.issueId, + issueCommentId: input.issueCommentId, + provider: stored.provider, + objectKey: stored.objectKey, + contentType: stored.contentType, + byteSize: stored.byteSize, + sha256: stored.sha256, + originalFilename: stored.originalFilename, + createdByUserId: input.actorUserId, + }; + let row; + try { + row = requireCurrentAttachmentAuthorization + ? await withCurrentAttachmentAuthorization( + (tx) => + issueService(tx as unknown as Db).createAttachment( + registration, + ), + sourceBoundMedia, + telegramMedia ? attachment : undefined, + ) + : await issuesSvc.createAttachment(registration); + } catch (error) { + // An explicit authorization denial happens before registration. Do + // not delete on ambiguous database errors which may have committed. + if ( + requireCurrentAttachmentAuthorization && + isExternalActionAuthorizationChange(error) + ) + await options.storage + .deleteObject(input.endpoint.companyId, stored.objectKey) + .catch(() => undefined); + throw error; + } + storedIds.push(row.id); + await registerPhotonProvenance(row.id); + } catch (error) { + if (isExternalActionAuthorizationChange(error)) throw error; + if ( + error instanceof PhotonError && + ["attachment_not_ready", "network", "quota", "credentials"].includes( + error.code, + ) + ) + throw error; + // Use the closed current-input omission vocabulary consumed by native + // prompts; provider-specific diagnostics remain redacted log codes. + omit( + error instanceof GitHubAttachmentUnavailableError + ? "download_unavailable" + : "processing_failed", + ); + // A malformed or unavailable provider attachment must not strand the + // durable text delivery. The rejected file is intentionally omitted; + // the delivery remains auditable through its normalized attachment + // metadata without persisting provider credentials or download URLs. + logger.warn( + { + endpointId: input.endpoint.id, + issueId: input.issueId, + ...(error instanceof GitHubAttachmentUnavailableError + ? { + deliveryId: input.deliveryId, + attachmentDiagnosticCode: error.code, + } + : {}), + attachmentName: sanitizeFilename(attachment.name), + error: redactError(error), + }, + "external chat attachment was rejected", + ); + } + } + return { storedIds, omissionReasons }; + } + + function attachmentOmissionDetail(result: { + omissionReasons: Record; + }): string | null { + const entries = Object.entries(result.omissionReasons).filter( + ([, count]) => count > 0, + ); + const omitted = entries.reduce((total, [, count]) => total + count, 0); + if (!omitted) return null; + const reasons = entries + .map(([reason, count]) => `${reason.replaceAll("_", " ")}: ${count}`) + .join(", "); + return `${omitted} external attachment${omitted === 1 ? " was" : "s were"} omitted (${reasons})`; + } + return { ingestAttachments, attachmentOmissionDetail }; +} diff --git a/server/src/services/chat-channels/microsoft-teams/file-transfer-projection.ts b/server/src/services/chat-channels/microsoft-teams/file-transfer-projection.ts new file mode 100644 index 00000000000..34365a46184 --- /dev/null +++ b/server/src/services/chat-channels/microsoft-teams/file-transfer-projection.ts @@ -0,0 +1,18 @@ +import { sql } from "drizzle-orm"; +import { chatTeamsFileTransfers } from "@paperclipai/db"; + +export const fileTransferProjectionColumns = { + publicationId: chatTeamsFileTransfers.publicationId, + phase: chatTeamsFileTransfers.phase, + version: chatTeamsFileTransfers.version, + filename: chatTeamsFileTransfers.filename, + expiresAt: chatTeamsFileTransfers.expiresAt, + consentMessageId: chatTeamsFileTransfers.consentMessageId, + responseActivityId: chatTeamsFileTransfers.responseActivityId, + fileInfoMessageId: chatTeamsFileTransfers.fileInfoMessageId, + operatorConfirmed: sql` + coalesce(${chatTeamsFileTransfers.privateState}#>>'{resolution,schema}', '') = 'paperclip.teams.file-resolution.v1' + and coalesce(${chatTeamsFileTransfers.privateState}#>>'{resolution,action}', '') = 'mark_delivered' + and coalesce(${chatTeamsFileTransfers.privateState}#>>'{resolution,fromPhase}', '') = 'file_info_unknown' + `, +}; diff --git a/server/src/services/chat-channels/outbound-attachments.ts b/server/src/services/chat-channels/outbound-attachments.ts new file mode 100644 index 00000000000..0f2e3a8b02e --- /dev/null +++ b/server/src/services/chat-channels/outbound-attachments.ts @@ -0,0 +1,294 @@ +import { + createHash, +} from "node:crypto"; +import { + and, + eq, + inArray, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + assets, + chatPublications, + issueAttachments, +} from "@paperclipai/db"; +import type { + SafeChatPublicationPayload, +} from "@paperclipai/shared"; + +import { + isAllowedContentType, + MAX_ATTACHMENT_BYTES, +} from "../../attachment-types.js"; + +import type { StorageService } from "../../storage/types.js"; + +import { + isTeamsInlineImageContentType, + TEAMS_INLINE_IMAGE_MAX_BYTES, +} from "../chat-teams-inline-images.js"; + +import type { + FileUpload, +} from "chat"; + +const OUTBOUND_ATTACHMENT_STORAGE_TIMEOUT_MS = 10_000; + +class OutboundAttachmentHydrationError extends Error { + readonly code = "CHAT_ATTACHMENT_HYDRATION_FAILED"; + + constructor(message: string, cause?: unknown) { + super(message, cause === undefined ? undefined : { cause }); + this.name = "OutboundAttachmentHydrationError"; + } +} + +class OutboundAttachmentValidationError extends Error { + readonly code = "CHAT_ATTACHMENT_VALIDATION_FAILED"; + + constructor(message: string) { + super(message); + this.name = "OutboundAttachmentValidationError"; + } +} + +function isOutboundAttachmentHydrationError( + error: unknown, +): error is OutboundAttachmentHydrationError { + return error instanceof OutboundAttachmentHydrationError; +} + +function isOutboundAttachmentValidationError( + error: unknown, +): error is OutboundAttachmentValidationError { + return error instanceof OutboundAttachmentValidationError; +} + +/** + * Read one already-authorized attachment before any provider transport starts. + * The database digest, rather than mutable object-store metadata, is the + * integrity boundary. Both acquisition and streaming are bounded so a stuck + * storage backend cannot hold the publication lane indefinitely. + */ +export async function hydrateOutboundAttachment(input: { + storage: StorageService; + companyId: string; + objectKey: string; + byteSize: number; + sha256: string; + filename: string; + mimeType: string; + timeoutMs?: number; +}): Promise { + const timeoutMs = + typeof input.timeoutMs === "number" && + Number.isFinite(input.timeoutMs) && + input.timeoutMs > 0 + ? input.timeoutMs + : OUTBOUND_ATTACHMENT_STORAGE_TIMEOUT_MS; + if ( + !Number.isSafeInteger(input.byteSize) || + input.byteSize <= 0 || + input.byteSize > MAX_ATTACHMENT_BYTES || + !/^[0-9a-f]{64}$/iu.test(input.sha256) + ) { + throw new OutboundAttachmentHydrationError( + "Chat publication attachment metadata is invalid", + ); + } + const acquisitionTimeoutError = new OutboundAttachmentHydrationError( + "Chat publication attachment storage read timed out", + ); + let acquisitionTimedOut = false; + let rejectAcquisition!: (error: Error) => void; + const acquisitionTimeout = new Promise((_resolve, reject) => { + rejectAcquisition = reject; + }); + const acquisitionTimer = setTimeout(() => { + acquisitionTimedOut = true; + rejectAcquisition(acquisitionTimeoutError); + }, timeoutMs); + acquisitionTimer.unref?.(); + const objectPromise = Promise.resolve() + .then(() => input.storage.getObject(input.companyId, input.objectKey)) + .then((object) => { + if (acquisitionTimedOut) object.stream.destroy(); + return object; + }) + .catch((error) => { + throw isOutboundAttachmentHydrationError(error) + ? error + : new OutboundAttachmentHydrationError( + "Chat publication attachment storage read failed", + error, + ); + }); + let object: Awaited>; + try { + object = await Promise.race([objectPromise, acquisitionTimeout]); + } finally { + clearTimeout(acquisitionTimer); + } + + if ( + object.contentLength !== undefined && + object.contentLength !== input.byteSize + ) { + object.stream.destroy(); + throw new OutboundAttachmentHydrationError( + "Chat publication attachment size changed after registration", + ); + } + + const streamTimeoutError = new OutboundAttachmentHydrationError( + "Chat publication attachment storage read timed out", + ); + const streamTimer = setTimeout(() => { + object.stream.destroy(streamTimeoutError); + }, timeoutMs); + streamTimer.unref?.(); + const chunks: Buffer[] = []; + const digest = createHash("sha256"); + let bytes = 0; + try { + for await (const chunk of object.stream) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += buffer.length; + if (bytes > input.byteSize || bytes > MAX_ATTACHMENT_BYTES) { + object.stream.destroy(); + throw new OutboundAttachmentHydrationError( + "Chat publication attachment exceeded its streaming byte limit", + ); + } + digest.update(buffer); + chunks.push(buffer); + } + } catch (error) { + throw isOutboundAttachmentHydrationError(error) + ? error + : new OutboundAttachmentHydrationError( + "Chat publication attachment storage read failed", + error, + ); + } finally { + clearTimeout(streamTimer); + } + + const actualDigest = digest.digest("hex"); + if (bytes !== input.byteSize || actualDigest !== input.sha256.toLowerCase()) { + if (!object.stream.destroyed) object.stream.destroy(); + throw new OutboundAttachmentHydrationError( + "Chat publication attachment integrity changed after registration", + ); + } + + return { + data: Buffer.concat(chunks, bytes), + filename: input.filename, + mimeType: input.mimeType, + }; +} +export { isOutboundAttachmentHydrationError, isOutboundAttachmentValidationError }; + +type OutboundAttachmentsOptions = { storage?: StorageService; }; + +export function createOutboundAttachments(db: Db, options: OutboundAttachmentsOptions) { + + + + async function publicationFiles( + publication: typeof chatPublications.$inferSelect, + payload: SafeChatPublicationPayload, + preparation: { teamsInlineImagesOnly?: boolean } = {}, + ): Promise { + if (!payload.attachmentIds?.length) return []; + try { + const rows = await db + .select({ + id: issueAttachments.id, + issueId: issueAttachments.issueId, + issueCommentId: issueAttachments.issueCommentId, + objectKey: assets.objectKey, + contentType: assets.contentType, + byteSize: assets.byteSize, + sha256: assets.sha256, + originalFilename: assets.originalFilename, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where( + and( + eq(issueAttachments.companyId, publication.companyId), + eq(issueAttachments.issueId, publication.issueId), + inArray(issueAttachments.id, payload.attachmentIds), + ), + ); + const byId = new Map(rows.map((row) => [row.id, row])); + const orderedRows = payload.attachmentIds.map((attachmentId) => { + const row = byId.get(attachmentId); + if ( + !row || + (publication.commentId && + row.issueCommentId !== publication.commentId) || + row.byteSize <= 0 || + row.byteSize > MAX_ATTACHMENT_BYTES || + !row.objectKey.trim() || + !/^[0-9a-f]{64}$/iu.test(row.sha256) || + !isAllowedContentType(row.contentType) + ) + throw new OutboundAttachmentValidationError( + "Chat publication attachment is invalid or outside its authorized task comment", + ); + return row; + }); + // Picture messages have a different Teams contract from arbitrary files. + // Preserve the normal scope/metadata checks above, but never hydrate a + // document or oversized image just to discover it needs a task link. + if ( + preparation.teamsInlineImagesOnly && + orderedRows.some( + (row) => + row.byteSize > TEAMS_INLINE_IMAGE_MAX_BYTES || + !isTeamsInlineImageContentType(row.contentType), + ) + ) { + return []; + } + // Authorization and persisted metadata must fail definitively even when + // storage is unavailable; do not mask an invalid file as a transient outage. + if (!options.storage) + throw new OutboundAttachmentHydrationError( + "Attachment storage is unavailable for chat publication", + ); + const files: FileUpload[] = []; + for (const row of orderedRows) { + files.push( + await hydrateOutboundAttachment({ + storage: options.storage, + companyId: publication.companyId, + objectKey: row.objectKey, + byteSize: row.byteSize, + sha256: row.sha256, + filename: row.originalFilename ?? `attachment-${row.id}`, + mimeType: row.contentType, + }), + ); + } + return files; + } catch (error) { + if ( + isOutboundAttachmentHydrationError(error) || + isOutboundAttachmentValidationError(error) + ) { + throw error; + } + throw new OutboundAttachmentHydrationError( + "Chat publication attachment preparation failed", + error, + ); + } + } + return { publicationFiles }; +} diff --git a/server/src/services/chat-channels/provider-effects.ts b/server/src/services/chat-channels/provider-effects.ts new file mode 100644 index 00000000000..5ca3b4b1786 --- /dev/null +++ b/server/src/services/chat-channels/provider-effects.ts @@ -0,0 +1,957 @@ +import { createTelegramCallbackNoticeAuthority } from "./telegram/callback-notices.js"; + +import { slackThreadChannelId } from "./slack/messages.js"; + +import type { EndpointRow, DbTransaction, DbOrTransaction, LifecycleRuntimeFence } from "./runtime-types.js"; + +import { baseTeamsConversationId } from "./microsoft-teams/resources.js"; + +import { dueChatAction, PROVIDER_EFFECT_STALE_MS } from "../chat-action-work.js"; + +import { notifyChatDeliveryWork, notifyChatActionWork } from "../chat-work-notifications.js"; + +import type { ChatEndpointRuntime } from "../voice/voice-runtime.js"; + +import { chatCredentialMutationLease } from "../chat-credential-mutation-lease.js"; + +import { projectSafeChatPublicationText } from "../chat-publication-projection.js"; + +import { PhotonChatAdapter } from "../photon/adapter.js"; + +import { + and, + asc, + eq, + inArray, + lte, + sql, +} from "drizzle-orm"; + +import { + teamsConversationId, +} from "../chat-control-chronology.js"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatConversations, + chatDeliveries, + chatEndpointResources, + chatEndpoints, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatProvider, +} from "@paperclipai/shared"; + +import { + parseTelegramCallbackReceipt, + telegramCallbackThreadId, + TELEGRAM_PRIVATE_ACTION_UNAVAILABLE, + type TelegramCallbackReceipt, +} from "../chat-telegram-ephemeral.js"; + +import { logger } from "../../middleware/logger.js"; +import { redactSensitiveText } from "../../redaction.js"; + +import { classifyChatPublicationError } from "../chat-publication-errors.js"; + +import type { + Thread, +} from "chat"; + +import type { EndpointRecord } from "./runtime-types.js"; + +type ProviderEffectTarget = { + post: Thread["post"]; + postEphemeral?: Thread["postEphemeral"]; +}; +type ProviderEffectPayload = { + version: 1; + effect: "thread_message" | "ephemeral_message" | "telegram_callback_notice"; + telegramCallback?: TelegramCallbackReceipt; + telegramAuthorizationSha256?: string; + authorizationMode?: "principal" | "safe_notice"; + threadId: string; + userId?: string; + text: string; + fallbackText?: string; + settleDelivery: boolean; + completeConversationId?: string; + resourceId?: string; + runtimeGeneration: number; + credentialFingerprint: string; +}; + +function providerEffectPayload( + payload: Record, +): ProviderEffectPayload | null { + if ( + payload.version !== 1 || + (payload.effect !== "thread_message" && + payload.effect !== "ephemeral_message" && + payload.effect !== "telegram_callback_notice") || + typeof payload.threadId !== "string" || + !payload.threadId || + typeof payload.text !== "string" || + !payload.text || + typeof payload.settleDelivery !== "boolean" || + typeof payload.runtimeGeneration !== "number" || + !Number.isSafeInteger(payload.runtimeGeneration) || + typeof payload.credentialFingerprint !== "string" + ) { + return null; + } + if ( + payload.effect === "ephemeral_message" && + (typeof payload.userId !== "string" || !payload.userId) + ) { + return null; + } + for (const key of [ + "authorizationMode", + "userId", + "fallbackText", + "completeConversationId", + "resourceId", + ] as const) { + const value = payload[key]; + if (value !== undefined && (typeof value !== "string" || !value)) { + return null; + } + } + if ( + payload.authorizationMode !== undefined && + payload.authorizationMode !== "principal" && + payload.authorizationMode !== "safe_notice" + ) { + return null; + } + if (payload.effect === "telegram_callback_notice") { + const receipt = parseTelegramCallbackReceipt(payload.telegramCallback); + if ( + !receipt || + payload.authorizationMode !== "safe_notice" || + payload.text !== TELEGRAM_PRIVATE_ACTION_UNAVAILABLE || + payload.threadId !== telegramCallbackThreadId(receipt) || + payload.userId !== receipt.receiverUserId || + payload.settleDelivery !== false || + payload.fallbackText !== undefined || + payload.completeConversationId !== undefined || + typeof payload.telegramAuthorizationSha256 !== "string" || + !/^[a-f0-9]{64}$/.test(payload.telegramAuthorizationSha256) + ) + return null; + } else if ( + payload.telegramCallback !== undefined || + payload.telegramAuthorizationSha256 !== undefined + ) + return null; + return payload as ProviderEffectPayload; +} +export type { ProviderEffectTarget, ProviderEffectPayload }; +export { providerEffectPayload }; + +type ProviderEffectsOptions = { + endpointRecord: (endpointId: string) => Promise; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + withCredentialMutationLease: ReturnType; + runtimeFor: (endpoint: EndpointRow) => Promise; + invalidateRuntime: (endpointId: string) => Promise; + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean }>; + redactError: (error: unknown) => string; + MAX_ERROR_TEXT: number; + nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean; + lockTelegramCallbackNoticeAuthority: ReturnType["lockTelegramCallbackNoticeAuthority"]; +}; + +export function createProviderEffects(db: Db, options: ProviderEffectsOptions) { + const { endpointRecord, runtimeCallbackEndpoint, withCredentialMutationLease, runtimeFor, invalidateRuntime, lockCurrentPrincipalAuthorization, redactError, MAX_ERROR_TEXT, nonDirectDestinationAllowed, lockTelegramCallbackNoticeAuthority } = options; + + + async function stageProviderEffect( + database: DbOrTransaction, + input: { + endpoint: EndpointRow; + deliveryId?: string | null; + conversationId?: string | null; + principalId?: string | null; + providerActionId: string; + payload: Omit< + ProviderEffectPayload, + "runtimeGeneration" | "credentialFingerprint" + >; + runtimeContext: LifecycleRuntimeFence; + }, + ) { + await notifyChatActionWork(database, "provider_effect"); + const [inserted] = await database + .insert(chatActions) + .values({ + companyId: input.endpoint.companyId, + endpointId: input.endpoint.id, + deliveryId: input.deliveryId ?? null, + conversationId: input.conversationId ?? null, + principalId: input.principalId ?? null, + kind: "provider_effect", + providerActionId: input.providerActionId, + payload: { + ...input.payload, + runtimeGeneration: input.runtimeContext.generation, + credentialFingerprint: input.runtimeContext.credentialFingerprint, + }, + status: "received", + }) + .onConflictDoNothing() + .returning(); + if (inserted) return inserted; + return database + .select() + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, input.endpoint.id), + eq(chatActions.providerActionId, input.providerActionId), + eq(chatActions.kind, "provider_effect"), + ), + ) + .then((rows) => rows[0] ?? null); + } + + function providerEffectDisposition( + error: unknown, + attempt: number, + providerEffectCompleted: boolean, + ) { + if (!providerEffectCompleted) + return classifyChatPublicationError(error, attempt); + return { + kind: "delivery_unknown" as const, + reason: + "Provider effect completed, but Paperclip could not confirm its durable result", + }; + } + + async function finalizeProviderEffectFailure(input: { + action: typeof chatActions.$inferSelect; + endpoint: EndpointRow; + payload: ProviderEffectPayload; + disposition: ReturnType; + attempt: number; + }): Promise { + const failure = redactSensitiveText(input.disposition.reason).slice( + 0, + MAX_ERROR_TEXT, + ); + const retryable = input.disposition.kind === "retry"; + const retryAt = + input.disposition.kind === "retry" + ? new Date(Date.now() + input.disposition.retryAfterMs) + : null; + let invalidateCurrentRuntime = false; + const finalized = await db.transaction(async (tx) => { + // Admission locks the endpoint before its delivery. Keep that order + // when failure settlement also needs to change endpoint/resource health. + const currentEndpoint = + input.disposition.kind === "endpoint_attention" || + input.disposition.kind === "resource_unavailable" + ? await runtimeCallbackEndpoint( + tx, + input.endpoint.id, + { + generation: input.payload.runtimeGeneration, + credentialFingerprint: input.payload.credentialFingerprint, + }, + ["verifying", "active"], + ) + : null; + await notifyChatActionWork(tx, "provider_effect"); + const [ownedAction] = await tx + .update(chatActions) + .set({ + status: + input.disposition.kind === "delivery_unknown" + ? "delivery_unknown" + : "failed", + result: { + code: `provider_effect_${input.disposition.kind}`, + attempts: input.attempt, + redactedError: failure, + retryable, + retryAt: retryAt?.toISOString() ?? null, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, input.action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${input.attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!ownedAction) return false; + if (input.payload.settleDelivery && input.action.deliveryId) { + await notifyChatDeliveryWork(tx); + await tx + .update(chatDeliveries) + .set({ + state: retryable ? "retry" : "failed", + nextAttemptAt: retryAt, + redactedError: + input.disposition.kind === "delivery_unknown" + ? `Provider response is unconfirmed: ${failure}`.slice( + 0, + MAX_ERROR_TEXT, + ) + : failure, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, input.action.deliveryId), + eq(chatDeliveries.state, "processing"), + ), + ); + } + if (input.disposition.kind === "endpoint_attention" && currentEndpoint) { + await tx + .update(chatEndpoints) + .set({ + status: "attention", + healthMessage: "Provider credentials or permissions need attention", + lastError: failure, + updatedAt: new Date(), + }) + .where(eq(chatEndpoints.id, currentEndpoint.id)); + await tx + .update(toolConnections) + .set({ + status: "disabled", + enabled: false, + healthStatus: "degraded", + healthMessage: "Provider credentials or permissions need attention", + lastError: failure, + healthCheckedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(toolConnections.id, currentEndpoint.connectionId)); + invalidateCurrentRuntime = true; + } else if ( + input.disposition.kind === "resource_unavailable" && + input.payload.resourceId && + currentEndpoint + ) { + await tx + .update(chatEndpointResources) + .set({ availability: "unavailable", updatedAt: new Date() }) + .where( + and( + eq(chatEndpointResources.id, input.payload.resourceId), + eq(chatEndpointResources.endpointId, currentEndpoint.id), + ), + ); + } + return true; + }); + if (finalized && invalidateCurrentRuntime) { + await invalidateRuntime(input.endpoint.id).catch(() => undefined); + } + return finalized; + } + + async function quarantineStaleProviderEffect( + action: typeof chatActions.$inferSelect, + ) { + const payload = providerEffectPayload(action.payload); + const failure = + "Provider delivery could not be confirmed after the worker stopped. The effect will not be replayed automatically."; + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "provider_effect"); + const [quarantined] = await tx + .update(chatActions) + .set({ + status: "delivery_unknown", + result: { + code: "provider_effect_delivery_unknown", + redactedError: failure, + retryable: false, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + eq(chatActions.updatedAt, action.updatedAt), + sql`(${chatActions.result}->>'attempts')::int = ${ + typeof action.result?.attempts === "number" + ? action.result.attempts + : -1 + }`, + lte( + chatActions.updatedAt, + new Date(Date.now() - PROVIDER_EFFECT_STALE_MS), + ), + ), + ) + .returning({ id: chatActions.id }); + if (quarantined && payload?.settleDelivery && action.deliveryId) { + await notifyChatDeliveryWork(tx); + await tx + .update(chatDeliveries) + .set({ + state: "failed", + nextAttemptAt: null, + redactedError: failure, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, action.deliveryId), + eq(chatDeliveries.state, "processing"), + ), + ); + } + }); + } + + function providerEffectThreadResourceId( + provider: ChatProvider, + threadId: string, + ): string | null { + if (provider === "slack") { + return ( + slackThreadChannelId(threadId) ?? + (!threadId.includes(":") ? threadId : null) + ); + } + if (provider === "telegram") { + return /^telegram:([^:]+)/.exec(threadId)?.[1] ?? null; + } + if (provider === "github") { + return /^github:([^:]+)/.exec(threadId)?.[1]?.toLowerCase() ?? null; + } + if (provider === "discord") { + return /^discord:[^:]+:([^:]+)/.exec(threadId)?.[1] ?? null; + } + const conversationId = teamsConversationId(threadId); + return conversationId ? baseTeamsConversationId(conversationId) : null; + } + + async function lockCurrentProviderEffectAuthorization( + tx: DbTransaction, + action: typeof chatActions.$inferSelect, + endpoint: EndpointRow, + payload: ProviderEffectPayload, + ): Promise { + if (payload.effect === "telegram_callback_notice") { + const receipt = parseTelegramCallbackReceipt(payload.telegramCallback); + if (!receipt || !action.deliveryId) return false; + const delivery = await tx + .select() + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.id, action.deliveryId), + eq(chatDeliveries.companyId, action.companyId), + eq(chatDeliveries.endpointId, action.endpointId), + eq(chatDeliveries.state, "filtered"), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const retained = + delivery && + parseTelegramCallbackReceipt(delivery.normalizedEvent.telegramCallback); + if ( + !delivery || + delivery.principalId !== action.principalId || + retained?.sourceSha256 !== receipt.sourceSha256 || + delivery.normalizedEvent.telegramAuthorizationSha256 !== + payload.telegramAuthorizationSha256 + ) + return false; + return ( + (await lockTelegramCallbackNoticeAuthority( + tx, + endpoint, + receipt, + action.principalId, + )) === payload.telegramAuthorizationSha256 + ); + } + const conversation = action.conversationId + ? await tx + .select() + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, endpoint.companyId), + eq(chatConversations.endpointId, endpoint.id), + eq(chatConversations.id, action.conversationId), + inArray(chatConversations.state, ["active", "waiting"]), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + if (action.conversationId && !conversation) return false; + const inferredResourceId = providerEffectThreadResourceId( + endpoint.provider, + payload.threadId, + ); + const resource = await tx + .select() + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.companyId, endpoint.companyId), + eq(chatEndpointResources.endpointId, endpoint.id), + payload.resourceId || conversation?.resourceId + ? eq( + chatEndpointResources.id, + payload.resourceId ?? conversation!.resourceId!, + ) + : inferredResourceId + ? eq(chatEndpointResources.providerResourceId, inferredResourceId) + : sql`false`, + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const slackChannelId = + endpoint.provider === "slack" + ? slackThreadChannelId(payload.threadId) + : null; + const isDirectMessage = + conversation?.isDirectMessage === true || + resource?.type === "direct_message" || + Boolean(slackChannelId && /^D[A-Z0-9]+$/i.test(slackChannelId)); + const destinationAllowed = isDirectMessage + ? endpoint.allowDirectMessages + : nonDirectDestinationAllowed(endpoint, resource); + if (payload.authorizationMode === "safe_notice") return true; + if (!destinationAllowed) return false; + if (!action.principalId) return false; + return ( + await lockCurrentPrincipalAuthorization(tx, endpoint, action.principalId) + ).allowed; + } + + async function cancelUnauthorizedProviderEffect( + tx: DbTransaction, + action: typeof chatActions.$inferSelect, + attempt: number, + ): Promise { + await notifyChatActionWork(tx, "provider_effect"); + const [cancelled] = await tx + .update(chatActions) + .set({ + status: "cancelled", + result: { + attempts: attempt, + code: "provider_effect_no_longer_authorized", + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!cancelled) return false; + if (action.deliveryId) { + await notifyChatDeliveryWork(tx); + await tx + .update(chatDeliveries) + .set({ + state: "filtered", + processedAt: new Date(), + nextAttemptAt: null, + redactedError: + "Provider reply was suppressed because current chat authorization changed", + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, action.deliveryId), + eq(chatDeliveries.state, "processing"), + ), + ); + } + return true; + } + + async function processProviderEffect( + actionId: string, + liveTarget?: ProviderEffectTarget, + ): Promise<"processed" | "pending" | "failed" | "delivery_unknown"> { + const initial = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "provider_effect"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!initial) return "failed"; + const initialEndpoint = await endpointRecord(initial.endpointId); + if (!initialEndpoint) return "failed"; + + const processClaimedEffect = async () => { + let action = await db + .select() + .from(chatActions) + .where(eq(chatActions.id, actionId)) + .then((rows) => rows[0] ?? null); + if (!action) return "failed"; + const payload = providerEffectPayload(action.payload); + if (!payload) { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "provider_effect"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { code: "provider_effect_payload_invalid" }, + updatedAt: new Date(), + }) + .where(eq(chatActions.id, action.id)); + }); + return "failed"; + } + if (action.status === "processed") return "processed"; + if (action.status === "delivery_unknown") return "delivery_unknown"; + if (action.status === "processing") { + if (action.updatedAt > new Date(Date.now() - PROVIDER_EFFECT_STALE_MS)) + return "pending"; + await quarantineStaleProviderEffect(action); + return "delivery_unknown"; + } + if (action.status === "failed") { + if (action.result?.retryable !== true) return "failed"; + const retryAt = + typeof action.result.retryAt === "string" + ? new Date(action.result.retryAt) + : null; + if (retryAt && retryAt > new Date()) return "pending"; + } else if (action.status !== "received") { + return "failed"; + } + + const attempt = + (typeof action.result?.attempts === "number" + ? action.result.attempts + : 0) + 1; + const record = await endpointRecord(action.endpointId); + let providerEffectCompleted = false; + try { + if (!record) { + throw Object.assign( + new Error("Chat endpoint is unavailable for this provider effect"), + { code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED" }, + ); + } + return await withCredentialMutationLease( + record.endpoint, + async (credentialLease) => { + const claim = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + const currentAction = await tx + .select() + .from(chatActions) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.kind, "provider_effect"), + eq(chatActions.status, action!.status), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if (!currentAction) return null; + const currentEndpoint = await runtimeCallbackEndpoint( + tx, + currentAction.endpointId, + { + generation: payload.runtimeGeneration, + credentialFingerprint: payload.credentialFingerprint, + }, + ["verifying", "active"], + ); + const authorized = currentEndpoint + ? await lockCurrentProviderEffectAuthorization( + tx, + currentAction, + currentEndpoint, + payload, + ) + : false; + await notifyChatActionWork(tx, "provider_effect"); + const [transportClaim] = await tx + .update(chatActions) + .set({ + status: "processing", + result: { attempts: attempt }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, currentAction.id), + eq(chatActions.status, currentAction.status), + ), + ) + .returning(); + if (!transportClaim) return null; + if (!currentEndpoint || !authorized) { + const cancelled = await cancelUnauthorizedProviderEffect( + tx, + transportClaim, + attempt, + ); + if (!cancelled) { + throw new Error( + "Provider effect ownership changed before authorization cancellation", + ); + } + return { outcome: "cancelled" as const }; + } + await credentialLease.assertOwned(tx); + return { + action: transportClaim, + endpoint: currentEndpoint, + outcome: "claimed" as const, + }; + }); + if (!claim) return "pending"; + if (claim.outcome === "cancelled") return "failed"; + action = claim.action; + + // `processing` is the durable, single-owner transport claim and + // authorization linearization point. Its transaction has committed + // before provider I/O begins. The renewable credential lease keeps + // runtime identity stable without holding database row locks. + let sent: { id: string; threadId: string } | null = null; + if (payload.effect === "telegram_callback_notice") { + sent = await ( + await runtimeFor(claim.endpoint) + ).sendTelegramCallbackNotice( + payload.telegramCallback!, + payload.text, + ); + } else { + const target = + liveTarget ?? + (await runtimeFor(claim.endpoint)).thread(payload.threadId); + if (payload.effect === "ephemeral_message") { + if (!target.postEphemeral) { + if (!payload.fallbackText) { + throw Object.assign( + new Error("Provider does not support ephemeral messages"), + { code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED" }, + ); + } + sent = await target.post(payload.fallbackText); + } else { + try { + sent = await target.postEphemeral( + payload.userId!, + payload.text, + { fallbackToDM: false }, + ); + } catch (error) { + if ( + !payload.fallbackText || + classifyChatPublicationError(error, attempt).kind !== + "failed" + ) { + throw error; + } + } + if (!sent && payload.fallbackText) { + sent = await target.post(payload.fallbackText); + } + } + } else if (claim.endpoint.provider === "imessage-photon") { + const adapter = (await runtimeFor(claim.endpoint)).getProviderAdapter(); + if (!(adapter instanceof PhotonChatAdapter)) throw new Error("Photon runtime unavailable"); + const receipt = await adapter.publish(payload.threadId, `effect:${action.id}`, projectSafeChatPublicationText(payload.text), { + assertCurrent: async () => { await credentialLease.assertOwned(); }, + }); + sent = { id: receipt.id, threadId: payload.threadId }; + } else { + sent = await target.post(payload.text); + } + } + if (!sent) { + throw Object.assign( + new Error("Provider did not accept the message"), + { code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED" }, + ); + } + providerEffectCompleted = true; + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + // A duplicate webhook holds the endpoint while waiting for the + // delivery's unique-index entry. Lock this endpoint first too, + // before settling the action/conversation/delivery, so neither + // transaction can wait on a row already held by the other. + const currentEndpoint = await runtimeCallbackEndpoint( + tx, + claim.endpoint.id, + { + generation: payload.runtimeGeneration, + credentialFingerprint: payload.credentialFingerprint, + }, + ["verifying", "active"], + ); + await notifyChatActionWork(tx, "provider_effect"); + const [completed] = await tx + .update(chatActions) + .set({ + status: "processed", + result: { + attempts: attempt, + providerMessageId: sent.id, + threadId: sent.threadId, + ...(payload.effect === "telegram_callback_notice" + ? { + code: + payload.telegramCallback?.chatType === "private" + ? "telegram_private_api_accepted" + : "telegram_ephemeral_api_accepted", + } + : {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!completed) { + throw new Error( + "Provider effect ownership changed before commit", + ); + } + if (payload.completeConversationId) { + await tx + .update(chatConversations) + .set({ state: "completed", updatedAt: new Date() }) + .where( + and( + eq(chatConversations.id, payload.completeConversationId), + eq(chatConversations.endpointId, claim.endpoint.id), + inArray(chatConversations.state, ["active", "waiting"]), + ), + ); + } + if (payload.settleDelivery && action!.deliveryId) { + await notifyChatDeliveryWork(tx); + const [settled] = await tx + .update(chatDeliveries) + .set({ + state: "processed", + processedAt: new Date(), + nextAttemptAt: null, + redactedError: null, + updatedAt: new Date(), + }) + .where( + and( + eq(chatDeliveries.id, action!.deliveryId), + eq(chatDeliveries.state, "processing"), + ), + ) + .returning({ id: chatDeliveries.id }); + if (!settled) { + throw new Error( + "Provider effect delivery ownership changed before commit", + ); + } + } + if (currentEndpoint) { + await tx + .update(chatEndpoints) + .set({ lastEventAt: new Date(), updatedAt: new Date() }) + .where(eq(chatEndpoints.id, currentEndpoint.id)); + } + await credentialLease.assertOwned(tx); + }); + return "processed"; + }, + ); + } catch (error) { + const disposition = providerEffectDisposition( + error, + attempt, + providerEffectCompleted, + ); + try { + await finalizeProviderEffectFailure({ + action, + endpoint: record?.endpoint ?? initialEndpoint.endpoint, + payload, + disposition, + attempt, + }); + } catch (finalizationError) { + if (disposition.kind === "delivery_unknown") { + throw Object.assign( + new Error( + "Provider effect is ambiguous and its durable quarantine failed", + { cause: finalizationError }, + ), + { code: "CHAT_PROVIDER_EFFECT_AMBIGUOUS" }, + ); + } + throw finalizationError; + } + throw error; + } + }; + return processClaimedEffect(); + } + + async function processPendingProviderEffects(limit = 25) { + const now = new Date(); + const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS); + const actions = await db + .select() + .from(chatActions) + .where(dueChatAction("provider_effect", now.getTime())) + .orderBy(asc(chatActions.createdAt)) + .limit(limit); + for (const action of actions) { + if (action.status === "processing" && action.updatedAt <= staleBefore) { + await quarantineStaleProviderEffect(action); + continue; + } + try { + await processProviderEffect(action.id); + } catch (error) { + logger.warn( + { + endpointId: action.endpointId, + actionId: action.id, + error: redactError(error), + }, + "chat provider effect reconciliation failed", + ); + } + } + return actions.length; + } + return { stageProviderEffect, processProviderEffect, processPendingProviderEffects }; +} diff --git a/server/src/services/chat-channels/reactions.ts b/server/src/services/chat-channels/reactions.ts new file mode 100644 index 00000000000..83df24432e9 --- /dev/null +++ b/server/src/services/chat-channels/reactions.ts @@ -0,0 +1,891 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence, InboundRuntimeContext } from "./runtime-types.js"; + +import { + stableExternalPrincipalId, +} from "./message-lifecycle.js"; + +import { notifyChatDeliveryWork } from "../chat-work-notifications.js"; + +import { + createHash, +} from "node:crypto"; +import { + and, + asc, + desc, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + or, + sql, + type SQL, +} from "drizzle-orm"; + +import { + externalThreadIdentityCondition, +} from "../chat-control-chronology.js"; + +import type { Db } from "@paperclipai/db"; + +import { + chatConversations, + chatDeliveries, + chatEndpointResources, + chatMessageLinks, + chatPublications, +} from "@paperclipai/db"; +import type { + ChatEventKind, + ChatProvider, +} from "@paperclipai/shared"; + +import { logger } from "../../middleware/logger.js"; + +import { + type ChatSdkCallbackEvent, +} from "../chat-sdk-runtime.js"; + +import type { + Author, + ReactionEvent, +} from "chat"; + +import type { EndpointRecord } from "./runtime-types.js"; +import { slackThreadChannelId } from "./slack/messages.js"; + +// Reactions can beat the transaction that persists a just-sent provider +// message link. Keep that narrow gap durable, but never retain an unbound +// reaction indefinitely or let it enter the ordinary inbound-message FIFO. +const REACTION_LINK_RETRY_DELAY_MS = 250; +const REACTION_LINK_MAX_DELAY_MS = 5_000; +const REACTION_LINK_MAX_ATTEMPTS = 20; +const REACTION_LINK_MAX_AGE_MS = 2 * 60_000; +type DeliveryRow = typeof chatDeliveries.$inferSelect; +type ConversationRow = typeof chatConversations.$inferSelect; + +type InboundReactionsOptions = { + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean }>; + runtimeCallbackRecord: (endpointId: string, context: RuntimeContext, statuses: EndpointRow["status"][]) => Promise; + runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise; + ensurePrincipal: (endpoint: EndpointRow, author: Author, raw?: unknown) => Promise<{ principal: { id: string } }>; + durableExternalThreadIdentity: (threadId: string) => string; + recordCurrentMicrosoftTeamsRoute: (endpoint: EndpointRow, context: InboundRuntimeContext | undefined, threadId: string, raw: unknown) => Promise; + lifecycleRuntimeFence: (delivery: DeliveryRow) => LifecycleRuntimeFence | null; + deliveryReady: (delivery: DeliveryRow, now: Date) => boolean; + redactError: (error: unknown) => string; + nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean; + reactionLinkPreflightBarrier?: () => Promise; + reactionReplayEndpointLockBarrier?: () => Promise; + reactionReplayConversationLockBarrier?: () => Promise; +}; + +export function createInboundReactions(db: Db, options: InboundReactionsOptions) { + const { lockCurrentPrincipalAuthorization, runtimeCallbackRecord, runtimeCallbackEndpoint, ensurePrincipal, durableExternalThreadIdentity, recordCurrentMicrosoftTeamsRoute, lifecycleRuntimeFence, deliveryReady, redactError, nonDirectDestinationAllowed } = options; + + + function reactionConversationThreadCondition( + provider: ChatProvider, + threadId: string, + ): SQL | undefined { + const exact = externalThreadIdentityCondition( + sql`${chatConversations.externalThreadId}`, + threadId, + ); + const slackDmChannel = + provider === "slack" ? slackThreadChannelId(threadId) : null; + return slackDmChannel && /^D[A-Z0-9-]*$/i.test(slackDmChannel) + ? or( + exact, + and( + eq(chatConversations.isDirectMessage, true), + eq(chatConversations.externalThreadId, `slack:${slackDmChannel}:`), + ), + ) + : exact; + } + + async function reactionConversationForMessage( + database: DbOrTransaction, + endpoint: EndpointRow, + threadId: string, + providerMessageId: string, + direction?: "outbound", + ): Promise { + return database + .select({ conversation: chatConversations }) + .from(chatMessageLinks) + .innerJoin( + chatConversations, + and( + eq(chatConversations.companyId, chatMessageLinks.companyId), + eq(chatConversations.endpointId, chatMessageLinks.endpointId), + eq(chatConversations.id, chatMessageLinks.conversationId), + ), + ) + .where( + and( + eq(chatMessageLinks.companyId, endpoint.companyId), + eq(chatMessageLinks.endpointId, endpoint.id), + eq(chatMessageLinks.providerMessageId, providerMessageId), + direction ? eq(chatMessageLinks.direction, direction) : undefined, + reactionConversationThreadCondition(endpoint.provider, threadId), + ), + ) + .orderBy(desc(chatConversations.sessionGeneration)) + .limit(1) + .then((rows) => rows[0]?.conversation ?? null); + } + + async function lockReactionConversationForMessage( + tx: DbTransaction, + endpoint: EndpointRow, + threadId: string, + providerMessageId: string, + direction?: "outbound", + ): Promise { + const candidate = await reactionConversationForMessage( + tx, + endpoint, + threadId, + providerMessageId, + direction, + ); + if (!candidate) return null; + const current = await tx + .select() + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, endpoint.companyId), + eq(chatConversations.endpointId, endpoint.id), + eq(chatConversations.id, candidate.id), + ), + ) + // Publication settlement inserts the outbound message link before it + // updates the endpoint. NO KEY UPDATE serializes state changes while + // remaining compatible with that link's conversation FK KEY SHARE. + .for("no key update") + .then((rows) => rows[0] ?? null); + if ( + !current || + !(current.isDirectMessage + ? ["active", "waiting", "completed"].includes(current.state) + : ["active", "waiting"].includes(current.state)) + ) { + return null; + } + const linkStillExists = await tx + .select({ id: chatMessageLinks.id }) + .from(chatMessageLinks) + .where( + and( + eq(chatMessageLinks.companyId, endpoint.companyId), + eq(chatMessageLinks.endpointId, endpoint.id), + eq(chatMessageLinks.conversationId, current.id), + eq(chatMessageLinks.providerMessageId, providerMessageId), + direction ? eq(chatMessageLinks.direction, direction) : undefined, + ), + ) + .limit(1) + .then((rows) => rows.length > 0); + return linkStillExists ? current : null; + } + + async function streamingReactionConversation( + database: DbOrTransaction, + endpoint: EndpointRow, + threadId: string, + ): Promise { + const candidates = await database + .selectDistinct({ + conversation: chatConversations, + sessionGeneration: chatConversations.sessionGeneration, + }) + .from(chatConversations) + .innerJoin( + chatPublications, + and( + eq(chatPublications.companyId, chatConversations.companyId), + eq(chatPublications.endpointId, chatConversations.endpointId), + eq(chatPublications.conversationId, chatConversations.id), + ), + ) + .where( + and( + eq(chatConversations.companyId, endpoint.companyId), + eq(chatConversations.endpointId, endpoint.id), + or( + inArray(chatConversations.state, ["active", "waiting"]), + and( + eq(chatConversations.isDirectMessage, true), + eq(chatConversations.state, "completed"), + ), + ), + reactionConversationThreadCondition(endpoint.provider, threadId), + eq(chatPublications.state, "streaming"), + ), + ) + .orderBy(desc(chatConversations.sessionGeneration)) + .limit(2); + if (candidates.length !== 1) return null; + return candidates[0]!.conversation; + } + + async function lockStreamingReactionConversation( + tx: DbTransaction, + endpoint: EndpointRow, + threadId: string, + ): Promise { + const candidate = await streamingReactionConversation( + tx, + endpoint, + threadId, + ); + if (!candidate) return null; + const current = await tx + .select() + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, endpoint.companyId), + eq(chatConversations.endpointId, endpoint.id), + eq(chatConversations.id, candidate.id), + or( + inArray(chatConversations.state, ["active", "waiting"]), + and( + eq(chatConversations.isDirectMessage, true), + eq(chatConversations.state, "completed"), + ), + ), + ), + ) + // Keep the same lock strength/order as the exact-link path above so an + // in-flight message-link insert cannot deadlock reaction admission. + .for("no key update") + .then((rows) => rows[0] ?? null); + if (!current) return null; + const stillStreaming = await tx + .select({ id: chatPublications.id }) + .from(chatPublications) + .where( + and( + eq(chatPublications.companyId, endpoint.companyId), + eq(chatPublications.endpointId, endpoint.id), + eq(chatPublications.conversationId, current.id), + eq(chatPublications.state, "streaming"), + ), + ) + .limit(1) + .then((rows) => rows.length > 0); + return stillStreaming ? current : null; + } + + async function authorizeReactionConversation( + tx: DbTransaction, + endpoint: EndpointRow, + conversation: ConversationRow, + principalId: string, + ): Promise { + const resource = + conversation.resourceId && !conversation.isDirectMessage + ? await tx + .select() + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.companyId, endpoint.companyId), + eq(chatEndpointResources.endpointId, endpoint.id), + eq(chatEndpointResources.id, conversation.resourceId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const destinationAllowed = conversation.isDirectMessage + ? endpoint.allowDirectMessages + : nonDirectDestinationAllowed(endpoint, resource); + if (!destinationAllowed) return false; + return (await lockCurrentPrincipalAuthorization(tx, endpoint, principalId)) + .allowed; + } + + async function handleReaction( + event: ChatSdkCallbackEvent, + runtimeContext: RuntimeContext, + ) { + const record = await runtimeCallbackRecord( + event.endpointId, + runtimeContext, + ["active"], + ); + if ( + !record || + record.endpoint.status !== "active" || + record.endpoint.provider !== event.provider || + record.endpoint.capabilities.reactions !== true || + event.event.user.isMe || + event.event.user.isBot === true || + event.event.user.isSystem + ) { + return; + } + // A DM can have a newer task generation while this reaction still belongs + // to an older linked message. Resolve its exact lineage before choosing a + // generation; never attach it to the latest task merely sharing the chat. + // A just-sent provider message can be visible before its publication/link + // transaction commits. Only that exact thread's streaming publication is + // enough evidence to stage an otherwise-unbound reaction for replay. + let conversation = await reactionConversationForMessage( + db, + record.endpoint, + event.event.threadId, + event.event.messageId, + ); + await options.reactionLinkPreflightBarrier?.(); + const plausibleStreamingConversation = conversation + ? null + : await streamingReactionConversation( + db, + record.endpoint, + event.event.threadId, + ); + if (!conversation && !plausibleStreamingConversation) { + // Settlement can commit between the first exact-link lookup and the + // streaming-publication lookup. Recheck the durable link before treating + // the callback as an arbitrary unknown message. + conversation = await reactionConversationForMessage( + db, + record.endpoint, + event.event.threadId, + event.event.messageId, + ); + if (!conversation) return; + } + + const principal = await ensurePrincipal( + record.endpoint, + event.event.user, + event.event.raw, + ); + let rawFingerprint: string; + try { + rawFingerprint = createHash("sha256") + .update(JSON.stringify(event.event.raw ?? null)) + .digest("hex") + .slice(0, 24); + } catch { + rawFingerprint = "unavailable"; + } + const eventKind: ChatEventKind = event.event.added + ? "reaction_added" + : "reaction_removed"; + const providerEventId = [ + eventKind, + durableExternalThreadIdentity(event.event.threadId), + event.event.messageId, + stableExternalPrincipalId( + record.endpoint.provider, + event.event.user, + event.event.raw, + ), + event.event.rawEmoji, + rawFingerprint, + ].join(":"); + const normalizedEvent = { + providerEventId, + kind: eventKind, + conversation: { externalThreadId: event.event.threadId }, + message: { providerMessageId: event.event.messageId }, + reaction: { + emoji: event.event.emoji.name, + rawEmoji: event.event.rawEmoji, + added: event.event.added, + }, + runtimeContext: { + generation: runtimeContext.generation, + credentialFingerprint: runtimeContext.credentialFingerprint, + }, + }; + const admitted = await db.transaction(async (tx) => { + const currentEndpoint = await runtimeCallbackEndpoint( + tx, + event.endpointId, + runtimeContext, + ["active"], + ); + if (!currentEndpoint) return false; + const linkedConversation = await lockReactionConversationForMessage( + tx, + currentEndpoint, + event.event.threadId, + event.event.messageId, + ); + const currentConversation = + linkedConversation ?? + (await lockStreamingReactionConversation( + tx, + currentEndpoint, + event.event.threadId, + )); + if ( + !currentConversation || + !(await authorizeReactionConversation( + tx, + currentEndpoint, + currentConversation, + principal.principal.id, + )) + ) { + return false; + } + const now = new Date(); + if (linkedConversation) { + const expiryCutoff = new Date(now.getTime() - REACTION_LINK_MAX_AGE_MS); + const hasOutboundLink = await tx + .select({ id: chatMessageLinks.id }) + .from(chatMessageLinks) + .where( + and( + eq(chatMessageLinks.companyId, currentEndpoint.companyId), + eq(chatMessageLinks.endpointId, currentEndpoint.id), + eq(chatMessageLinks.conversationId, linkedConversation.id), + eq(chatMessageLinks.providerMessageId, event.event.messageId), + eq(chatMessageLinks.direction, "outbound"), + ), + ) + .limit(1) + .then((rows) => rows.length > 0); + await notifyChatDeliveryWork(tx); + const reconciled = hasOutboundLink + ? await tx + .update(chatDeliveries) + .set({ + conversationId: linkedConversation.id, + state: "processed", + attempts: sql`${chatDeliveries.attempts} + 1`, + nextAttemptAt: null, + processedAt: now, + redactedError: null, + updatedAt: now, + }) + .where( + and( + eq(chatDeliveries.endpointId, currentEndpoint.id), + eq(chatDeliveries.providerEventId, providerEventId), + inArray(chatDeliveries.state, ["received", "retry"]), + gt(chatDeliveries.receivedAt, expiryCutoff), + lt(chatDeliveries.attempts, REACTION_LINK_MAX_ATTEMPTS), + sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'generation' = ${String(runtimeContext.generation)}`, + sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'credentialFingerprint' = ${runtimeContext.credentialFingerprint}`, + ), + ) + .returning({ id: chatDeliveries.id }) + : []; + if (reconciled.length > 0) return true; + await notifyChatDeliveryWork(tx); + const expired = await tx + .update(chatDeliveries) + .set({ + state: "filtered", + nextAttemptAt: null, + processedAt: now, + redactedError: + "Reaction target did not become linked before replay expiry", + updatedAt: now, + }) + .where( + and( + eq(chatDeliveries.endpointId, currentEndpoint.id), + eq(chatDeliveries.providerEventId, providerEventId), + inArray(chatDeliveries.state, ["received", "retry"]), + or( + lte(chatDeliveries.receivedAt, expiryCutoff), + gte(chatDeliveries.attempts, REACTION_LINK_MAX_ATTEMPTS), + ), + sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'generation' = ${String(runtimeContext.generation)}`, + sql`${chatDeliveries.normalizedEvent}->'runtimeContext'->>'credentialFingerprint' = ${runtimeContext.credentialFingerprint}`, + ), + ) + .returning({ id: chatDeliveries.id }); + if (expired.length > 0) return false; + } + await notifyChatDeliveryWork(tx); + const inserted = await tx + .insert(chatDeliveries) + .values({ + companyId: currentEndpoint.companyId, + endpointId: currentEndpoint.id, + conversationId: linkedConversation?.id ?? null, + principalId: principal.principal.id, + providerEventId, + deduplicationKey: createHash("sha256") + .update(providerEventId) + .digest("hex"), + eventKind, + normalizedEvent, + state: linkedConversation ? "processed" : "received", + attempts: linkedConversation ? 1 : 0, + nextAttemptAt: linkedConversation + ? null + : new Date(now.getTime() + REACTION_LINK_RETRY_DELAY_MS), + processedAt: linkedConversation ? now : null, + }) + .onConflictDoNothing() + .returning({ id: chatDeliveries.id }); + if (inserted.length > 0) return true; + return tx + .select({ id: chatDeliveries.id }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.endpointId, currentEndpoint.id), + eq(chatDeliveries.providerEventId, providerEventId), + inArray(chatDeliveries.state, ["received", "retry", "processed"]), + ), + ) + .limit(1) + .then((rows) => rows.length > 0); + }); + if (admitted) { + await recordCurrentMicrosoftTeamsRoute( + record.endpoint, + runtimeContext, + event.event.threadId, + event.event.raw, + ); + } + } + + function normalizedPendingReaction(delivery: DeliveryRow): { + added: boolean; + emoji: string; + messageId: string; + rawEmoji: string; + runtimeContext: LifecycleRuntimeFence; + threadId: string; + } | null { + if ( + delivery.eventKind !== "reaction_added" && + delivery.eventKind !== "reaction_removed" + ) { + return null; + } + const normalized = delivery.normalizedEvent; + if ( + !normalized || + typeof normalized !== "object" || + Array.isArray(normalized) + ) { + return null; + } + const value = normalized as Record; + const conversation = value.conversation; + const message = value.message; + const reaction = value.reaction; + const runtimeContext = lifecycleRuntimeFence(delivery); + if ( + !conversation || + typeof conversation !== "object" || + Array.isArray(conversation) || + !message || + typeof message !== "object" || + Array.isArray(message) || + !reaction || + typeof reaction !== "object" || + Array.isArray(reaction) || + !runtimeContext + ) { + return null; + } + const threadId = (conversation as Record).externalThreadId; + const messageId = (message as Record).providerMessageId; + const reactionRecord = reaction as Record; + const added = reactionRecord.added; + const emoji = reactionRecord.emoji; + const rawEmoji = reactionRecord.rawEmoji; + return typeof threadId === "string" && + threadId.length > 0 && + typeof messageId === "string" && + messageId.length > 0 && + typeof added === "boolean" && + added === (delivery.eventKind === "reaction_added") && + typeof emoji === "string" && + emoji.length > 0 && + typeof rawEmoji === "string" && + rawEmoji.length > 0 + ? { added, emoji, messageId, rawEmoji, runtimeContext, threadId } + : null; + } + + async function processPendingReactionDelivery( + candidate: DeliveryRow, + ): Promise { + const reaction = normalizedPendingReaction(candidate); + if (!reaction || !candidate.principalId) { + const now = new Date(); + await db.transaction(async (tx) => { + await notifyChatDeliveryWork(tx); + return tx + .update(chatDeliveries) + .set({ + state: "failed", + nextAttemptAt: null, + processedAt: now, + redactedError: "Normalized reaction delivery is incomplete", + updatedAt: now, + }) + .where( + and( + eq(chatDeliveries.id, candidate.id), + inArray(chatDeliveries.state, ["received", "retry"]), + ), + ); + }); + return; + } + const principalId = candidate.principalId; + + await db.transaction(async (tx) => { + const currentEndpoint = await runtimeCallbackEndpoint( + tx, + candidate.endpointId, + reaction.runtimeContext, + ["active"], + ); + let conversation: ConversationRow | null = null; + let disposition: + | { kind: "processed"; conversationId: string } + | { kind: "retry" } + | { kind: "filtered"; reason: string }; + + if (!currentEndpoint) { + disposition = { + kind: "filtered", + reason: "Reaction callback belonged to an unavailable runtime", + }; + } else { + await options.reactionReplayEndpointLockBarrier?.(); + conversation = await lockReactionConversationForMessage( + tx, + currentEndpoint, + reaction.threadId, + reaction.messageId, + "outbound", + ); + if (conversation) { + await options.reactionReplayConversationLockBarrier?.(); + } + if (conversation) { + disposition = (await authorizeReactionConversation( + tx, + currentEndpoint, + conversation, + principalId, + )) + ? { kind: "processed", conversationId: conversation.id } + : { + kind: "filtered", + reason: + "Reaction destination or principal is no longer authorized", + }; + } else { + conversation = await lockStreamingReactionConversation( + tx, + currentEndpoint, + reaction.threadId, + ); + if (conversation) { + await options.reactionReplayConversationLockBarrier?.(); + } + if (!conversation) { + disposition = { + kind: "filtered", + reason: "Reaction target was not linked to a sent message", + }; + } else { + disposition = (await authorizeReactionConversation( + tx, + currentEndpoint, + conversation, + principalId, + )) + ? { kind: "retry" } + : { + kind: "filtered", + reason: + "Reaction destination or principal is no longer authorized", + }; + } + } + } + + // Endpoint, conversation, destination and identity locks are acquired + // before the delivery row so pause/revoke and replay use one lock order. + const locked = await tx + .select() + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.id, candidate.id), + inArray(chatDeliveries.state, ["received", "retry"]), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + // Authorization locks may wait behind a concurrent lifecycle mutation. + // Sample the expiry clock only after this delivery is exclusively held. + const decisionAt = new Date(); + if (!locked || !deliveryReady(locked, decisionAt)) return; + const lockedReaction = normalizedPendingReaction(locked); + if ( + !lockedReaction || + lockedReaction.runtimeContext.generation !== + reaction.runtimeContext.generation || + lockedReaction.runtimeContext.credentialFingerprint !== + reaction.runtimeContext.credentialFingerprint + ) { + disposition = { + kind: "filtered", + reason: "Reaction delivery identity changed before replay", + }; + } + + const nextAttempts = locked.attempts + 1; + const expired = + decisionAt.getTime() - locked.receivedAt.getTime() >= + REACTION_LINK_MAX_AGE_MS; + if (expired || locked.attempts >= REACTION_LINK_MAX_ATTEMPTS) { + disposition = { + kind: "filtered", + reason: "Reaction target did not become linked before replay expiry", + }; + } + if (disposition.kind === "retry" && !expired) { + if (nextAttempts < REACTION_LINK_MAX_ATTEMPTS) { + const delay = Math.min( + REACTION_LINK_RETRY_DELAY_MS * + 2 ** Math.min(Math.max(locked.attempts, 0), 5), + REACTION_LINK_MAX_DELAY_MS, + ); + await notifyChatDeliveryWork(tx); + await tx + .update(chatDeliveries) + .set({ + state: "retry", + attempts: nextAttempts, + nextAttemptAt: new Date(decisionAt.getTime() + delay), + redactedError: "Waiting for the sent message link", + updatedAt: decisionAt, + }) + .where( + and( + eq(chatDeliveries.id, locked.id), + eq(chatDeliveries.state, locked.state), + eq(chatDeliveries.attempts, locked.attempts), + ), + ); + return; + } + disposition = { + kind: "filtered", + reason: "Reaction target did not become linked before retry expiry", + }; + } else if (disposition.kind === "retry") { + disposition = { + kind: "filtered", + reason: "Reaction target did not become linked before age expiry", + }; + } + + const processedAt = new Date(); + await notifyChatDeliveryWork(tx); + await tx + .update(chatDeliveries) + .set( + disposition.kind === "processed" + ? { + conversationId: disposition.conversationId, + state: "processed", + attempts: nextAttempts, + nextAttemptAt: null, + processedAt, + redactedError: null, + updatedAt: processedAt, + } + : { + state: "filtered", + attempts: nextAttempts, + nextAttemptAt: null, + processedAt, + redactedError: disposition.reason, + updatedAt: processedAt, + }, + ) + .where( + and( + eq(chatDeliveries.id, locked.id), + eq(chatDeliveries.state, locked.state), + eq(chatDeliveries.attempts, locked.attempts), + ), + ); + }); + } + + async function processPendingReactionDeliveries( + limit: number, + onlyDeliveryId?: string, + ): Promise { + const now = new Date(); + const candidates = await db + .select() + .from(chatDeliveries) + .where( + and( + sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.provider = 'agentmail')`, + onlyDeliveryId ? eq(chatDeliveries.id, onlyDeliveryId) : undefined, + inArray(chatDeliveries.eventKind, [ + "reaction_added", + "reaction_removed", + ]), + or( + and( + eq(chatDeliveries.state, "received"), + or( + isNull(chatDeliveries.nextAttemptAt), + lte(chatDeliveries.nextAttemptAt, now), + ), + ), + and( + eq(chatDeliveries.state, "retry"), + or( + isNull(chatDeliveries.nextAttemptAt), + lte(chatDeliveries.nextAttemptAt, now), + ), + ), + ), + ), + ) + .orderBy(asc(chatDeliveries.receivedAt), asc(chatDeliveries.id)) + .limit(limit); + for (const candidate of candidates) { + try { + await processPendingReactionDelivery(candidate); + } catch (error) { + logger.warn( + { + endpointId: candidate.endpointId, + deliveryId: candidate.id, + error: redactError(error), + }, + "chat reaction-link reconciliation failed", + ); + } + } + return candidates.length; + } + return { handleReaction, processPendingReactionDeliveries }; +} diff --git a/server/src/services/chat-channels/receipt-reactions.ts b/server/src/services/chat-channels/receipt-reactions.ts new file mode 100644 index 00000000000..a733091a127 --- /dev/null +++ b/server/src/services/chat-channels/receipt-reactions.ts @@ -0,0 +1,586 @@ +import type { ChatRuntime } from "../chat-runtime.js"; +import type { EndpointRow, RuntimeContext, LifecycleRuntimeFence } from "./runtime-types.js"; + +import { dueChatAction, PROVIDER_EFFECT_STALE_MS } from "../chat-action-work.js"; + +import { notifyChatActionWork } from "../chat-work-notifications.js"; + +import type { ChatEndpointRuntime } from "../voice/voice-runtime.js"; + +import { + receiptReactionPayload, + type ReceiptReactionPayload, +} from "../chat-receipt-reactions.js"; + +import { chatCredentialMutationLease, type CredentialMutationLeaseGuard } from "../chat-credential-mutation-lease.js"; + +import { + and, + asc, + eq, + like, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatDeliveries, + chatEndpoints, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatEndpointSetupState, + ChatProvider, + ToolCredentialSecretRef, +} from "@paperclipai/shared"; + +import { + parseGitHubReceiptIdentity, + type GitHubReceiptIdentity, +} from "../chat-github-receipt-reactions.js"; + +import { logger } from "../../middleware/logger.js"; + +import { classifyChatPublicationError } from "../chat-publication-errors.js"; + +import type { + Message, + Thread, +} from "chat"; + +import type { EndpointRecord } from "./runtime-types.js"; +import { isSlackReceiptReactionAlreadyApplied } from "./slack/messages.js"; + +type ReceiptReactionsOptions = { + endpointRecord: (endpointId: string) => Promise; + runtimeContextForRecord: (record: EndpointRecord) => RuntimeContext; + withCredentialMutationLease: ReturnType; + runtimeFor: (endpoint: EndpointRow) => Promise; + fetchImpl: typeof fetch; + runtime: Pick; + runtimeGeneration: (setup: ChatEndpointSetupState) => number; + credentialFingerprint: (refs: ToolCredentialSecretRef[]) => string; + redactError: (error: unknown) => string; + MAX_ERROR_TEXT: number; + receiptReactionTransportBarrier?: () => Promise; +}; + +export function createReceiptReactions(db: Db, options: ReceiptReactionsOptions) { + const { endpointRecord, runtimeContextForRecord, withCredentialMutationLease, runtimeFor, fetchImpl, runtime, runtimeGeneration, credentialFingerprint, redactError, MAX_ERROR_TEXT } = options; + + + async function addReceiptReaction(input: { + deliveryId: string; + endpoint: EndpointRow; + message: Message; + runtimeContext?: LifecycleRuntimeFence; + thread: Thread; + }): Promise { + // GitHub acknowledges through one editable response comment. Continue to + // drain legacy eyes removals, but never create new eyes for these requests. + if (input.endpoint.provider === "github") return; + const record = await endpointRecord(input.endpoint.id); + if (!record) return; + // A live SDK object remains bound to the credentials that authenticated + // its callback. Preserve that original fence rather than sampling the + // endpoint again after a rotation and accidentally blessing an obsolete + // adapter with the new generation. + const context = input.runtimeContext ?? runtimeContextForRecord(record); + const [inserted] = await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "receipt_reaction"); + return tx + .insert(chatActions) + .values({ + companyId: input.endpoint.companyId, + endpointId: input.endpoint.id, + deliveryId: input.deliveryId, + kind: "receipt_reaction", + providerActionId: `receipt_reaction:${input.deliveryId}`, + payload: { + version: 1, + operation: "add", + threadId: input.thread.id, + messageId: input.message.id, + reaction: "eyes", + runtimeGeneration: context.generation, + credentialFingerprint: context.credentialFingerprint, + } satisfies ReceiptReactionPayload, + status: "received", + }) + .onConflictDoNothing() + .returning({ id: chatActions.id }); + }); + const actionId = + inserted?.id ?? + (await db + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, input.endpoint.id), + eq( + chatActions.providerActionId, + `receipt_reaction:${input.deliveryId}`, + ), + ), + ) + .then((rows) => rows[0]?.id ?? null)); + if (!actionId) return; + await options.receiptReactionTransportBarrier?.(); + await processReceiptReaction(actionId); + } + + async function processReceiptReaction(actionId: string): Promise { + const initialAction = await db + .select({ endpointId: chatActions.endpointId }) + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "receipt_reaction"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!initialAction) return; + const initialEndpoint = await endpointRecord(initialAction.endpointId); + if (!initialEndpoint) return; + // Receipt reactions are GitHub repository mutations too. Serialize their + // final runtime-fence check and provider call with reconnect, secret + // rotation, pause, and removal so credentials cannot become obsolete + // between authorization and transport. + await withCredentialMutationLease(initialEndpoint.endpoint, (lease) => + processReceiptReactionWithCredentialLease(actionId, lease), + ); + } + + async function processReceiptReactionWithCredentialLease( + actionId: string, + credentialLease: CredentialMutationLeaseGuard, + ): Promise { + let action = await db + .select() + .from(chatActions) + .where( + and( + eq(chatActions.id, actionId), + eq(chatActions.kind, "receipt_reaction"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!action || ["processed", "cancelled"].includes(action.status)) return; + const payload = receiptReactionPayload(action.payload); + if (!payload) { + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "receipt_reaction"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + code: "receipt_reaction_payload_invalid", + retryable: false, + }, + updatedAt: new Date(), + }) + .where(eq(chatActions.id, action.id)); + }); + return; + } + if (payload.operation === "add" && action.deliveryId) { + const terminalRemoval = await db + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.endpointId, action.endpointId), + eq( + chatActions.providerActionId, + `receipt_reaction_remove:${action.deliveryId}`, + ), + ), + ) + .then((rows) => rows[0] ?? null); + if (terminalRemoval) { + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "receipt_reaction"); + await tx + .update(chatActions) + .set({ + status: "cancelled", + result: { + ...(typeof action!.result?.attempts === "number" + ? { attempts: action!.result.attempts } + : {}), + code: "receipt_reaction_superseded_by_terminal_publication", + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, action!.status), + ), + ); + await credentialLease.assertOwned(tx); + }); + return; + } + } + if (action.status === "processing") { + if (action.updatedAt > new Date(Date.now() - PROVIDER_EFFECT_STALE_MS)) + return; + // Adding or removing the same reaction is idempotent. A worker + // interruption can be retried safely without an ambiguous + // duplicate-message boundary. + await db.transaction(async (tx) => { + await notifyChatActionWork(tx, "receipt_reaction"); + return tx + .update(chatActions) + .set({ + status: "failed", + result: { + ...(action.result ?? {}), + retryable: true, + retryAt: new Date().toISOString(), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + eq(chatActions.updatedAt, action.updatedAt), + ), + ); + }); + action = await db + .select() + .from(chatActions) + .where(eq(chatActions.id, action.id)) + .then((rows) => rows[0] ?? null); + if (!action) return; + } + if (action.status === "failed") { + if (action.result?.retryable !== true) return; + const retryAt = + typeof action.result.retryAt === "string" + ? new Date(action.result.retryAt) + : null; + if (retryAt && retryAt > new Date()) return; + } else if (action.status !== "received") { + return; + } + const attempt = + (typeof action.result?.attempts === "number" ? action.result.attempts : 0) + + 1; + const retainedGitHubReceipt = parseGitHubReceiptIdentity( + action.result?.githubReceipt, + ); + const [claimed] = await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "receipt_reaction"); + const rows = await tx + .update(chatActions) + .set({ + status: "processing", + result: { + attempts: attempt, + ...(retainedGitHubReceipt + ? { githubReceipt: retainedGitHubReceipt } + : {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, action!.status), + ), + ) + .returning(); + await credentialLease.assertOwned(tx); + return rows; + }); + if (!claimed) return; + + let reactionProvider: ChatProvider | null = null; + let githubReceipt: GitHubReceiptIdentity | null = retainedGitHubReceipt; + try { + const record = await endpointRecord(action.endpointId); + if ( + !record || + !["verifying", "active"].includes(record.endpoint.status) || + runtimeGeneration(record.endpoint.setup) !== payload.runtimeGeneration || + credentialFingerprint(record.credentialSecretRefs) !== + payload.credentialFingerprint + ) { + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "receipt_reaction"); + await tx + .update(chatActions) + .set({ + status: "cancelled", + result: { + attempts: attempt, + code: "receipt_reaction_runtime_superseded", + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ); + await credentialLease.assertOwned(tx); + }); + return; + } + reactionProvider = record.endpoint.provider; + // Resolve the adapter only after acquiring the credential lease and + // proving the action's durable generation/fingerprint is still current. + // Never reuse the live callback's adapter: it may hold an installation + // token minted by a superseded private key. + const endpointRuntime = await runtimeFor(record.endpoint); + const adapter = endpointRuntime.thread(payload.threadId).adapter; + await credentialLease.assertOwned(); + if (record.endpoint.provider === "slack") { + await endpointRuntime.applySlackReceiptReaction(payload, fetchImpl); + } else if (record.endpoint.provider === "github") { + const assertCurrent = async () => { + await credentialLease.assertOwned(); + const [current] = await db + .select({ endpoint: chatEndpoints, connection: toolConnections }) + .from(chatEndpoints) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, chatEndpoints.connectionId), + eq(toolConnections.companyId, chatEndpoints.companyId), + ), + ) + .where( + and( + eq(chatEndpoints.id, action!.endpointId), + eq(chatEndpoints.companyId, action!.companyId), + ), + ) + .limit(1); + if ( + !current || + !["active", "verifying"].includes(current.endpoint.status) || + !current.connection.enabled || + current.connection.status !== "active" || + runtimeGeneration(current.endpoint.setup) !== + payload.runtimeGeneration || + credentialFingerprint(current.connection.credentialSecretRefs) !== + payload.credentialFingerprint || + runtime.get(action!.endpointId) !== endpointRuntime + ) + throw new Error("GitHub receipt authority is no longer current"); + }; + const applied = await endpointRuntime.applyGitHubReceiptReaction( + { ...payload, ...(githubReceipt ? { githubReceipt } : {}) }, + assertCurrent, + fetchImpl, + ); + githubReceipt = parseGitHubReceiptIdentity(applied); + if ( + !githubReceipt || + (payload.operation === "add" && !githubReceipt.reactionId) + ) + throw new Error("GitHub receipt identity was not confirmed"); + } else if (payload.operation === "remove") { + await adapter.removeReaction( + payload.threadId, + payload.messageId, + payload.reaction, + ); + } else { + await adapter.addReaction( + payload.threadId, + payload.messageId, + payload.reaction, + ); + } + // The provider may already have accepted the idempotent reaction. Fence + // the local success receipt so a reclaimed lease cannot report success + // for an obsolete runtime. + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "receipt_reaction"); + const processed = await tx + .update(chatActions) + .set({ + status: "processed", + result: { + attempts: attempt, + ...(githubReceipt ? { githubReceipt } : {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!processed.length) { + throw new Error( + "Receipt-reaction ownership changed before success settlement", + ); + } + if (action.deliveryId) { + await tx + .update(chatDeliveries) + .set({ redactedError: null, updatedAt: new Date() }) + .where( + and( + eq(chatDeliveries.id, action.deliveryId), + eq(chatDeliveries.state, "processed"), + like(chatDeliveries.redactedError, "Receipt reaction %"), + ), + ); + } + await credentialLease.assertOwned(tx); + }); + } catch (error) { + if ( + payload.operation === "add" && + reactionProvider === "slack" && + isSlackReceiptReactionAlreadyApplied(error) + ) { + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "receipt_reaction"); + const processed = await tx + .update(chatActions) + .set({ + status: "processed", + result: { + attempts: attempt, + code: "receipt_reaction_already_applied", + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action!.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (!processed.length) { + throw new Error( + "Receipt-reaction ownership changed before idempotent settlement", + ); + } + if (action.deliveryId) { + await tx + .update(chatDeliveries) + .set({ redactedError: null, updatedAt: new Date() }) + .where( + and( + eq(chatDeliveries.id, action.deliveryId), + eq(chatDeliveries.state, "processed"), + like(chatDeliveries.redactedError, "Receipt reaction %"), + ), + ); + } + await credentialLease.assertOwned(tx); + }); + return; + } + const disposition = classifyChatPublicationError(error, attempt); + const retryable = + disposition.kind === "retry" || + (disposition.kind === "delivery_unknown" && attempt < 3); + const retryAfterMs = + disposition.kind === "retry" + ? disposition.retryAfterMs + : 100 * 2 ** Math.max(0, attempt - 1); + const operationLabel = + payload.operation === "remove" ? " reaction removal" : " reaction"; + const diagnostic = + `Receipt${operationLabel} ${retryable ? "deferred" : "failed"} after ${attempt} attempt${attempt === 1 ? "" : "s"} (${disposition.kind}): ${redactError(error)}`.slice( + 0, + MAX_ERROR_TEXT, + ); + const retryAt = retryable + ? new Date(Date.now() + retryAfterMs).toISOString() + : null; + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + await notifyChatActionWork(tx, "receipt_reaction"); + const [failed] = await tx + .update(chatActions) + .set({ + status: "failed", + result: { + attempts: attempt, + ...(githubReceipt ? { githubReceipt } : {}), + code: + payload.operation === "remove" + ? `receipt_reaction_removal_${disposition.kind}` + : `receipt_reaction_${disposition.kind}`, + redactedError: diagnostic, + retryable, + retryAt, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(chatActions.id, action.id), + eq(chatActions.status, "processing"), + sql`(${chatActions.result}->>'attempts')::int = ${attempt}`, + ), + ) + .returning({ id: chatActions.id }); + if (failed && action.deliveryId) { + await tx + .update(chatDeliveries) + .set({ redactedError: diagnostic, updatedAt: new Date() }) + .where( + and( + eq(chatDeliveries.id, action.deliveryId), + eq(chatDeliveries.state, "processed"), + ), + ); + } + await credentialLease.assertOwned(tx); + }); + logger.warn( + { + endpointId: action.endpointId, + deliveryId: action.deliveryId, + attempts: attempt, + disposition: disposition.kind, + operation: payload.operation, + retryAt, + }, + "external chat receipt reaction deferred", + ); + } + } + + async function processPendingReceiptReactions( + limit = 25, + onlyActionId?: string, + ) { + const actions = await db.select({ id: chatActions.id }).from(chatActions) + .where(and(dueChatAction("receipt_reaction"), onlyActionId ? eq(chatActions.id, onlyActionId) : undefined)) + .orderBy(asc(chatActions.createdAt)).limit(limit); + for (const action of actions) await processReceiptReaction(action.id); + return actions.length; + } + return { addReceiptReaction, processReceiptReaction, processPendingReceiptReactions }; +} diff --git a/server/src/services/chat-channels/runtime-types.ts b/server/src/services/chat-channels/runtime-types.ts index a764a0297eb..cd2f6881e9b 100644 --- a/server/src/services/chat-channels/runtime-types.ts +++ b/server/src/services/chat-channels/runtime-types.ts @@ -1,5 +1,5 @@ import type { Db, chatEndpoints } from "@paperclipai/db"; -import type { ToolCredentialSecretRef } from "@paperclipai/shared"; +import type { ToolCredentialSecretRef, ChatEndpointSetupState, ChatEndpointCallbackSurfaces } from "@paperclipai/shared"; import type { ChatEndpointRuntime } from "../voice/voice-runtime.js"; export type EndpointRow = typeof chatEndpoints.$inferSelect; @@ -36,3 +36,17 @@ export type EndpointRecord = { applicationId: string; credentialSecretRefs: ToolCredentialSecretRef[]; }; + +export type SlackCallbackSurface = keyof ChatEndpointCallbackSurfaces; + +export type SlackCallbackObservation = { url: string; observedAt: string }; + +export type InternalSetupState = ChatEndpointSetupState & { + photonIntakeAfter?: string; + runtimeGeneration?: number; + /** Binds pre-install URL verification to the signing secret later configured. */ + slackVerificationSigningFingerprint?: string; + slackCallbackSurfaces?: Partial< + Record + >; +}; diff --git a/server/src/services/chat-channels/setup-probes.ts b/server/src/services/chat-channels/setup-probes.ts new file mode 100644 index 00000000000..36d0f8f9f66 --- /dev/null +++ b/server/src/services/chat-channels/setup-probes.ts @@ -0,0 +1,353 @@ +import type { EndpointRow, DbOrTransaction } from "./runtime-types.js"; + +import { notifyChatEndpointWork } from "../chat-work-notifications.js"; + +import { chatCredentialMutationLease } from "../chat-credential-mutation-lease.js"; + +import { githubChatManagementService } from "../chat-github-management.js"; + +import { + and, + asc, + desc, + eq, + gte, + inArray, + isNotNull, + or, + sql, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatActions, + chatConversations, + chatDeliveries, + chatEndpoints, + chatIdentityLinks, + chatMessageLinks, + chatPublications, + heartbeatRuns, + issueComments, + toolConnections, +} from "@paperclipai/db"; +import type { + ChatEndpoint, + ChatEndpointSetupState, +} from "@paperclipai/shared"; + +import { + conflict, + forbidden, + notFound, +} from "../../errors.js"; + +import { + logActivity, +} from "../activity-log.js"; + +import { + githubRunReplyState, +} from "../chat-run-publications.js"; + +import type { EndpointRecord } from "./runtime-types.js"; + +type SetupProbesOptions = { + endpointRecord: (endpointId: string) => Promise; + withCredentialMutationLease: ReturnType; + fetchImpl: typeof fetch; + findAuthorizedIdentityLink: (endpoint: EndpointRow, userId: string) => Promise<{ id: string } | null>; + runtimeGeneration: (setup: ChatEndpointSetupState) => number; + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean }>; + get: (endpointId: string) => Promise; + setupTestActivationBarrier?: () => Promise; +}; + +export function createSetupProbes(db: Db, options: SetupProbesOptions) { + const { endpointRecord, withCredentialMutationLease, fetchImpl, findAuthorizedIdentityLink, runtimeGeneration, lockCurrentPrincipalAuthorization, get } = options; + + + async function setupTestStatus(endpointId: string, userId: string) { + const record = await endpointRecord(endpointId); + if (!record) throw notFound("Chat endpoint not found"); + const startedAt = record.endpoint.setup.testStartedAt; + if (!startedAt) return { messageReceivedAt: null }; + const links = await db.select({ principalId: chatIdentityLinks.principalId }).from(chatIdentityLinks).where(and( + eq(chatIdentityLinks.companyId, record.endpoint.companyId), eq(chatIdentityLinks.endpointId, endpointId), + eq(chatIdentityLinks.paperclipUserId, userId), eq(chatIdentityLinks.status, "linked"), + )); + if (!links.length) return { messageReceivedAt: null }; + const principalIds = links.map((link) => link.principalId); + const [delivery, command] = await Promise.all([ + db.select({ at: chatDeliveries.createdAt }).from(chatDeliveries).where(and( + eq(chatDeliveries.companyId, record.endpoint.companyId), eq(chatDeliveries.endpointId, endpointId), + inArray(chatDeliveries.principalId, principalIds), gte(chatDeliveries.createdAt, new Date(startedAt)), + inArray(chatDeliveries.eventKind, ["message", "mention"]), + )).orderBy(asc(chatDeliveries.createdAt)).limit(1).then((rows) => rows[0]), + db.select({ at: chatActions.createdAt }).from(chatActions).where(and( + eq(chatActions.companyId, record.endpoint.companyId), eq(chatActions.endpointId, endpointId), + inArray(chatActions.principalId, principalIds), eq(chatActions.kind, "slash_task_start"), gte(chatActions.createdAt, new Date(startedAt)), + )).orderBy(asc(chatActions.createdAt)).limit(1).then((rows) => rows[0]), + ]); + const at = [delivery?.at, command?.at].filter((value): value is Date => Boolean(value)).sort((a, b) => a.getTime() - b.getTime())[0]; + return { messageReceivedAt: at?.toISOString() ?? null }; + } + + async function test(endpointId: string, finishOptions?: { optionalSlackTestForUser: string }) { + const initial = await endpointRecord(endpointId); + if (!initial) throw notFound("Chat endpoint not found"); + return withCredentialMutationLease( + initial.endpoint, + async (credentialLease) => { + // Reload only after owning the same lane as reconnect, pause, removal, + // and credential rotation. Qualification and activation must describe + // one exact provider runtime, never a setup snapshot that a concurrent + // reconnect has already superseded. + const record = await endpointRecord(endpointId); + if (!record) throw notFound("Chat endpoint not found"); + const endpoint = record.endpoint; + if (endpoint.status !== "verifying" || endpoint.setup.step !== "test") { + throw conflict("This connection is not waiting for a setup test", { + code: "chat_endpoint_not_testing", + }); + } + if (endpoint.provider === "github" && endpoint.setup.github) { + const verified = await githubChatManagementService(db, fetchImpl).verification(endpoint.id); + if (!verified.ready && !(finishOptions?.optionalSlackTestForUser && verified.connectionReady)) throw conflict("Finish verifying the App, repositories, and assigned agent's tools before activating this bot", { checks: verified.checks }); + } + const testStartedAtValue = endpoint.setup.testStartedAt; + const testStartedAt = testStartedAtValue + ? new Date(testStartedAtValue) + : null; + const optionalSlackTest = Boolean(finishOptions?.optionalSlackTestForUser); + if (optionalSlackTest) { + if (!["slack", "github"].includes(endpoint.provider) || !endpoint.setup.webhookVerifiedAt || !endpoint.providerAccountId) { + throw conflict("Verify the connection before finishing setup"); + } + const linked = await findAuthorizedIdentityLink(endpoint, finishOptions!.optionalSlackTestForUser); + if (!linked) throw forbidden("Connect your account before finishing setup"); + } + if (!optionalSlackTest) { + if ( + !endpoint.lastEventAt || + !testStartedAtValue || + !testStartedAt || + Number.isNaN(testStartedAt.getTime()) || + endpoint.lastEventAt < testStartedAt + ) { + throw conflict( + "Send the test message in the provider before completing setup", + { + code: "chat_test_message_missing", + }, + ); + } + const requiredTrigger = + ["telegram", "imessage-photon"].includes(endpoint.provider) + ? "direct_message" + : "subscribed_message"; + const qualifyingDelivery = await db + .select({ + id: chatDeliveries.id, + conversationId: chatDeliveries.conversationId, + processedAt: chatDeliveries.processedAt, + }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.companyId, endpoint.companyId), + eq(chatDeliveries.endpointId, endpoint.id), + eq(chatDeliveries.state, "processed"), + gte(chatDeliveries.processedAt, testStartedAt), + sql`${chatDeliveries.normalizedEvent}->>'trigger' = ${requiredTrigger}`, + ), + ) + .orderBy(desc(chatDeliveries.processedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + if ( + !qualifyingDelivery?.conversationId || + !qualifyingDelivery.processedAt + ) { + throw conflict( + ["telegram", "imessage-photon"].includes(endpoint.provider) + ? "Send the test direct message before completing setup" + : "Reply once without mentioning the agent before completing setup", + { code: "chat_test_follow_up_missing" }, + ); + } + const finalPublication = await db + .select({ + commentId: chatPublications.commentId, + payload: chatPublications.payload, + }) + .from(chatPublications) + .innerJoin( + issueComments, + and( + eq(issueComments.id, chatPublications.commentId), + eq(issueComments.companyId, chatPublications.companyId), + eq(issueComments.issueId, chatPublications.issueId), + eq(issueComments.authorType, "agent"), + eq(issueComments.authorAgentId, endpoint.assignedAgentId), + ), + ) + .innerJoin( + chatMessageLinks, + and( + eq(chatMessageLinks.companyId, chatPublications.companyId), + eq(chatMessageLinks.endpointId, chatPublications.endpointId), + eq( + chatMessageLinks.conversationId, + chatPublications.conversationId, + ), + eq(chatMessageLinks.deliveryId, qualifyingDelivery.id), + eq(chatMessageLinks.direction, "inbound"), + isNotNull(chatMessageLinks.commentId), + ), + ) + .innerJoin( + heartbeatRuns, + and( + eq(heartbeatRuns.id, issueComments.createdByRunId), + eq(heartbeatRuns.companyId, chatPublications.companyId), + eq(heartbeatRuns.agentId, endpoint.assignedAgentId), + eq(heartbeatRuns.status, "succeeded"), + eq( + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`, + sql`${chatPublications.issueId}::text`, + ), + or( + sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`, + sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot} ->> 'commentId'`, + sql`coalesce(${heartbeatRuns.contextSnapshot} -> 'wakeCommentIds', '[]'::jsonb) ? ${chatMessageLinks.commentId}::text`, + ), + ), + ) + .where( + and( + eq(chatPublications.companyId, endpoint.companyId), + eq(chatPublications.endpointId, endpoint.id), + eq( + chatPublications.conversationId, + qualifyingDelivery.conversationId, + ), + eq(chatPublications.state, "published"), + gte(chatPublications.publishedAt, qualifyingDelivery.processedAt), + ), + ) + .orderBy(desc(chatPublications.publishedAt)) + .then((rows) => + rows.find( + (row) => + row.payload.interactionId === undefined && + row.payload.progressState === undefined && + row.commentId !== null, + ), + ); + let githubToolReply = false; + if (!finalPublication && endpoint.provider === "github") { + const setupRuns = await db.select({ id: heartbeatRuns.id, issueId: chatConversations.issueId }) + .from(heartbeatRuns) + .innerJoin(chatMessageLinks, and( + eq(chatMessageLinks.companyId, endpoint.companyId), + eq(chatMessageLinks.endpointId, endpoint.id), + eq(chatMessageLinks.deliveryId, qualifyingDelivery.id), + eq(chatMessageLinks.direction, "inbound"), + or( + sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot}->>'wakeCommentId'`, + sql`coalesce(${heartbeatRuns.contextSnapshot}->'wakeCommentIds', '[]'::jsonb) ? ${chatMessageLinks.commentId}::text`, + ), + )) + .innerJoin(chatConversations, and( + eq(chatConversations.id, qualifyingDelivery.conversationId), + eq(chatConversations.id, chatMessageLinks.conversationId), + eq(chatConversations.companyId, endpoint.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${chatConversations.issueId}::text`, + )) + .where(and(eq(heartbeatRuns.companyId, endpoint.companyId), + eq(heartbeatRuns.agentId, endpoint.assignedAgentId), eq(heartbeatRuns.status, "succeeded"))) + .orderBy(desc(heartbeatRuns.createdAt)).limit(20); + for (const run of setupRuns) { + if (await githubRunReplyState(db, { companyId: endpoint.companyId, endpointId: endpoint.id, + issueId: run.issueId, runId: run.id }) === "confirmed") { + githubToolReply = true; + break; + } + } + } + if (!finalPublication && !githubToolReply) { + throw conflict( + "Wait for the Paperclip agent to reply to the setup turn before completing setup", + { + code: "chat_test_round_trip_incomplete", + }, + ); + } + } + await options.setupTestActivationBarrier?.(); + const expectedGeneration = runtimeGeneration(endpoint.setup); + await db.transaction(async (tx) => { + await credentialLease.assertOwned(tx); + if (optionalSlackTest) { + const linked = await tx.select({ principalId: chatIdentityLinks.principalId }).from(chatIdentityLinks).where(and( + eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.paperclipUserId, finishOptions!.optionalSlackTestForUser), eq(chatIdentityLinks.status, "linked"), + )).limit(1).then((rows) => rows[0]); + if (!linked || !(await lockCurrentPrincipalAuthorization(tx, endpoint, linked.principalId)).allowed) throw forbidden("Your connected account is no longer authorized"); + } + await notifyChatEndpointWork(tx); + const [activated] = await tx + .update(chatEndpoints) + .set({ + status: "active", + setup: { + ...endpoint.setup, + step: "complete", + testStartedAt: null, + testSkipped: optionalSlackTest, + }, + healthMessage: optionalSlackTest ? "Connected; conversation test optional" : "Connected", + activatedAt: endpoint.activatedAt ?? new Date(), + updatedAt: new Date(), + }) + .where( + and( + eq(chatEndpoints.id, endpointId), + eq(chatEndpoints.status, "verifying"), + sql`${chatEndpoints.setup}->>'step' = 'test'`, + sql`coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::integer, 0) = ${expectedGeneration}`, + sql`${chatEndpoints.setup}->>'testStartedAt' = ${testStartedAtValue}`, + ), + ) + .returning({ id: chatEndpoints.id }); + if (!activated) { + throw conflict("This setup test was superseded by a reconnect", { + code: "chat_endpoint_test_superseded", + }); + } + await tx + .update(toolConnections) + .set({ + status: "active", + enabled: true, + healthStatus: "healthy", + healthMessage: optionalSlackTest ? "Connected; conversation test optional" : "Connected", + lastError: null, + healthCheckedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(toolConnections.id, endpoint.connectionId)); + if (optionalSlackTest) await logActivity(tx as unknown as Db, { + companyId: endpoint.companyId, actorType: "user", actorId: finishOptions!.optionalSlackTestForUser, + action: "chat.setup_completed", entityType: "chat_endpoint", entityId: endpointId, + details: { conversationTestOptional: true }, + }); + await credentialLease.assertOwned(tx); + }); + return get(endpointId); + }, + ); + } + return { setupTestStatus, test }; +} diff --git a/server/src/services/chat-channels/slack/callback-verification.ts b/server/src/services/chat-channels/slack/callback-verification.ts new file mode 100644 index 00000000000..ecddb2f665f --- /dev/null +++ b/server/src/services/chat-channels/slack/callback-verification.ts @@ -0,0 +1,300 @@ +import { isProvisionedSlackSetup } from "@paperclipai/shared"; +import type { EndpointRow, DbOrTransaction, SlackCallbackSurface, InternalSetupState } from "../runtime-types.js"; + +import { notifyChatVerificationWork } from "../../chat-work-notifications.js"; + +import { slackChatRegistrationService } from "../../chat-slack-registration.js"; +import { chatSlackRegistrations } from "@paperclipai/db"; + +import { runtimeCanonicalOrigin } from "../../cloud-runtime-identity.js"; + +import { + createHash, + createHmac, + timingSafeEqual, +} from "node:crypto"; +import { + and, + eq, +} from "drizzle-orm"; + +import type { Db } from "@paperclipai/db"; + +import { + chatEndpoints, + toolConnections, +} from "@paperclipai/db"; +import type { + ToolCredentialSecretRef, +} from "@paperclipai/shared"; + +import { + logActivity, +} from "../../activity-log.js"; + +import { secretService } from "../../secrets.js"; + +function slackRequestSignatureIsValid( + request: Request, + body: string, + signingSecret: string, +): boolean { + const timestamp = request.headers.get("x-slack-request-timestamp"); + const signature = request.headers.get("x-slack-signature"); + if (!timestamp || !signature) return false; + const timestampSeconds = Number(timestamp); + if ( + !Number.isFinite(timestampSeconds) || + Math.abs(Math.floor(Date.now() / 1000) - timestampSeconds) > 300 + ) + return false; + const expected = `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:${body}`).digest("hex")}`; + try { + return timingSafeEqual(Buffer.from(signature), Buffer.from(expected)); + } catch { + return false; + } +} + +function slackRequestWorkspaceId( + body: string, + contentType: string, +): string | null { + let payload: unknown; + try { + if (contentType.includes("application/json")) { + payload = JSON.parse(body); + } else { + const form = new URLSearchParams(body); + const interactivePayload = form.get("payload"); + payload = interactivePayload + ? JSON.parse(interactivePayload) + : Object.fromEntries(form.entries()); + } + } catch { + return null; + } + if (!payload || typeof payload !== "object") return null; + const record = payload as Record; + const nestedId = (value: unknown): string | null => { + if (!value || typeof value !== "object") return null; + const id = (value as Record).id; + return typeof id === "string" && id.length > 0 ? id : null; + }; + const stringField = (key: string): string | null => { + const value = record[key]; + return typeof value === "string" && value.length > 0 ? value : null; + }; + const authorization = Array.isArray(record.authorizations) + ? record.authorizations.find( + (value): value is Record => + Boolean(value) && typeof value === "object", + ) + : null; + const authorizationField = (key: string): string | null => { + const value = authorization?.[key]; + return typeof value === "string" && value.length > 0 ? value : null; + }; + + // A Grid event may carry both an enterprise id and the concrete workspace + // id. The bot identity is verified against auth.test's team_id, so prefer a + // team id and use the enterprise id only when Slack omitted team context. + return ( + stringField("team_id") ?? + nestedId(record.team) ?? + authorizationField("team_id") ?? + stringField("enterprise_id") ?? + nestedId(record.enterprise) ?? + authorizationField("enterprise_id") + ); +} + +function canonicalCallbackUrl(value: string): string | null { + try { + const url = new URL(value); + if (url.protocol !== "https:" && url.protocol !== "http:") return null; + url.hostname = url.hostname.replace(/\.$/, ""); + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + return url.toString(); + } catch { + return null; + } +} + +// TLS terminates at a reverse proxy in many self-hosted deployments. The +// transport scheme is not evidence of URL drift; authority and path still are. +// This is diagnostic only: it does not trust forwarded headers or change auth. +function slackCallbackMatchesPublicUrl(observed: string, current: string): boolean { + const canonical = canonicalCallbackUrl(observed); + if (canonical === current) return true; + if (!canonical) return false; + const observedUrl = new URL(canonical); + const currentUrl = new URL(current); + return observedUrl.protocol === "http:" + && currentUrl.protocol === "https:" + && observedUrl.host === currentUrl.host + && observedUrl.pathname === currentUrl.pathname; +} + +async function inspectSlackCallback( + request: Request, +): Promise { + const url = slackCallbackObservationUrl(request); + if (!url) return null; + const contentType = request.headers.get("content-type")?.toLowerCase() ?? ""; + let body: string; + try { + body = await request.text(); + } catch { + return null; + } + if (contentType.includes("application/json")) { + try { + const payload = JSON.parse(body) as { + type?: unknown; + challenge?: unknown; + }; + const isUrlVerification = + payload.type === "url_verification" && + typeof payload.challenge === "string" && + payload.challenge.length > 0; + if (isUrlVerification || payload.type === "event_callback") { + return { + surface: "events", + url, + isUrlVerification, + }; + } + } catch { + return null; + } + return null; + } + + const form = new URLSearchParams(body); + if (form.get("command")) { + return { surface: "slashCommands", url, isUrlVerification: false }; + } + const interactivePayload = form.get("payload"); + if (!interactivePayload) return null; + try { + const payload = JSON.parse(interactivePayload) as { type?: unknown }; + if ( + typeof payload.type === "string" && + [ + "block_actions", + "block_suggestion", + "dialog_submission", + "message_action", + "shortcut", + "view_closed", + "view_submission", + "workflow_step_edit", + ].includes(payload.type) + ) { + return { surface: "interactivity", url, isUrlVerification: false }; + } + } catch { + return null; + } + return null; +} + +type SlackCallbackInspection = { + surface: SlackCallbackSurface; + url: string; + isUrlVerification: boolean; +}; + +// Cloud records the public request host in dedicated headers so provider edges +// cannot replace it with their upstream host. Only use callback-health evidence on a +// claimed Cloud instance, after provider authentication succeeded. It must not +// change the Request passed to the adapter, signature verification, routing, +// board identity, or the configured URL used to generate callbacks. +function slackCallbackObservationUrl(request: Request): string | null { + const directUrl = canonicalCallbackUrl(request.url); + if (!directUrl || !runtimeCanonicalOrigin()) return directUrl; + const hasCloudEvidence = request.headers.has("x-paperclip-cloud-forwarded-host"); + const host = request.headers.get(hasCloudEvidence ? "x-paperclip-cloud-forwarded-host" : "x-forwarded-host")?.trim(); + const protocol = request.headers.get(hasCloudEvidence ? "x-paperclip-cloud-forwarded-proto" : "x-forwarded-proto")?.trim(); + if (!host || /[\s/@\\?#,]/.test(host) || (protocol !== "https" && protocol !== "http")) return directUrl; + try { + const origin = new URL(`${protocol}://${host}`); + // Accept one authority only, never credentials, paths, queries, or lists. + if (!origin.host || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) return directUrl; + const observed = new URL(directUrl); + observed.protocol = origin.protocol; + observed.host = origin.host; + observed.port = origin.port; + return canonicalCallbackUrl(observed.toString()); + } catch { + return directUrl; + } +} +export { slackRequestSignatureIsValid, slackRequestWorkspaceId, canonicalCallbackUrl, slackCallbackMatchesPublicUrl, inspectSlackCallback }; + +type SlackCallbackVerificationOptions = { + resolveCredentialRefs: (endpoint: EndpointRow, refs: ToolCredentialSecretRef[], database?: DbOrTransaction) => Promise>; + getWebhookPublicBaseUrl: () => string | null; + scheduleMessageProcessing: (task: () => Promise) => void; + slackRegistration: Pick, "notifyVerified">; +}; + +export function createSlackCallbackVerification(db: Db, options: SlackCallbackVerificationOptions) { + const { resolveCredentialRefs, getWebhookPublicBaseUrl, scheduleMessageProcessing, slackRegistration } = options; + + + async function verifySlackRequestUrl(endpoint: EndpointRow, request: Request): Promise { + const callback = await inspectSlackCallback(request.clone()); + if (!callback?.isUrlVerification) return null; + const body = await request.clone().text(); + let notifyAccount = false; + // URL verification needs only the signing secret. Do not initialize the + // SDK (which calls Slack auth.test), wait for OAuth, or acquire the creation + // lease: Slack can deliver this check while app setup still owns that lease. + const response = await db.transaction(async tx => { + const [current] = await tx.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpoint.id)).for("no key update"); + if (!current || ["archived", "paused", "revoked"].includes(current.status)) return new Response("ignored", { status: 200 }); + const [connection] = await tx.select({ status: toolConnections.status, refs: toolConnections.credentialSecretRefs }).from(toolConnections) + .where(and(eq(toolConnections.id, current.connectionId), eq(toolConnections.companyId, current.companyId))); + if (!connection || connection.status === "archived") return new Response("ignored", { status: 200 }); + let secret = (await resolveCredentialRefs(current, connection.refs.filter(ref => ref.configPath.replace(/^credentials\./, "") === "signingSecret"), tx)).signingSecret; + if (!secret && isProvisionedSlackSetup(current.setup.slackSetupMethod)) { + const [registration] = await tx.select().from(chatSlackRegistrations) + .where(and(eq(chatSlackRegistrations.endpointId, current.id), eq(chatSlackRegistrations.companyId, current.companyId))); + if (registration?.status !== "removed" && registration?.secretIds.signingSecret) { + secret = await secretService(tx as unknown as Db).resolveSecretValue(current.companyId, registration.secretIds.signingSecret, "latest", { + accessContext: { consumerType: "system", consumerId: `slack-registration:${current.id}`, configPath: "slack_registration.signingSecret", actorType: "system" }, + }); + } + } + if (!secret) return new Response("Slack signing secret is not ready yet", { status: 503 }); + if (!slackRequestSignatureIsValid(request, body, secret)) return new Response("Invalid signature", { status: 401 }); + const workspaceId = slackRequestWorkspaceId(body, request.headers.get("content-type") ?? ""); + if (workspaceId && current.providerAccountId && workspaceId !== current.providerAccountId) return new Response("ignored", { status: 200 }); + const observedAt = new Date(); + const setup = current.setup as InternalSetupState; + const matchesCurrentUrl = slackCallbackMatchesPublicUrl(callback.url, `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${current.publicId}/slack`); + await notifyChatVerificationWork(tx); + await tx.update(chatEndpoints).set({ setup: { + ...setup, + ...(matchesCurrentUrl ? { webhookVerifiedAt: setup.webhookVerifiedAt ?? observedAt.toISOString(), + slackVerificationSigningFingerprint: createHash("sha256").update(secret).digest("hex") } : {}), + slackCallbackSurfaces: { ...setup.slackCallbackSurfaces, events: { url: callback.url, observedAt: observedAt.toISOString() } }, + ...(matchesCurrentUrl && setup.slackAccount?.status === "linked" && !setup.slackAccount.verificationStatus + ? { slackAccount: { ...setup.slackAccount, verificationStatus: "pending" as const } } : {}), + } as InternalSetupState, ...(matchesCurrentUrl ? { healthMessage: "Slack Events Request URL verified" } : {}), updatedAt: observedAt }).where(eq(chatEndpoints.id, current.id)); + if (matchesCurrentUrl && !setup.webhookVerifiedAt) await logActivity(tx as unknown as Db, { companyId: current.companyId, actorType: "system", actorId: "slack-webhook", + action: "chat_endpoint.webhook_verified", entityType: "tool_connection", entityId: current.connectionId, + details: { endpointId: current.id, provider: "slack" } }); + notifyAccount = matchesCurrentUrl && setup.slackAccount?.status === "linked" && !setup.slackAccount.verificationStatus; + return Response.json({ challenge: (JSON.parse(body) as { challenge: string }).challenge }); + }); + if (notifyAccount) scheduleMessageProcessing(() => slackRegistration.notifyVerified(endpoint.id)); + return response; + } + return { verifySlackRequestUrl }; +} diff --git a/server/src/services/chat-channels/slack/messages.ts b/server/src/services/chat-channels/slack/messages.ts index eba1e833f2f..dece4360827 100644 --- a/server/src/services/chat-channels/slack/messages.ts +++ b/server/src/services/chat-channels/slack/messages.ts @@ -41,3 +41,49 @@ export function slackMessageSentAt(raw: unknown, messageId: string): Date | null const date = new Date(Number(ts) * 1_000); return Number.isFinite(date.getTime()) ? date : null; } + +export function isSlackReceiptReactionAlreadyApplied(error: unknown): boolean { + const pending: Array<{ depth: number; value: unknown }> = [ + { depth: 0, value: error }, + ]; + const seen = new Set(); + while (pending.length > 0) { + const current = pending.shift(); + if ( + !current || + current.depth > 4 || + !current.value || + typeof current.value !== "object" || + seen.has(current.value) + ) { + continue; + } + seen.add(current.value); + const record = current.value as { + cause?: unknown; + code?: unknown; + data?: { error?: unknown }; + original?: unknown; + originalError?: unknown; + }; + if ( + record.code === "already_reacted" || + record.data?.error === "already_reacted" + ) { + return true; + } + for (const value of [record.cause, record.original, record.originalError]) { + pending.push({ depth: current.depth + 1, value }); + } + } + return false; +} + +export function slackThreadChannelId(threadId: string): string | null { + if (!threadId.startsWith("slack:")) return null; + const channelAndThread = threadId.slice("slack:".length); + const separator = channelAndThread.indexOf(":"); + const channelId = + separator === -1 ? channelAndThread : channelAndThread.slice(0, separator); + return channelId || null; +} diff --git a/server/src/services/chat-channels/slack/task-start.ts b/server/src/services/chat-channels/slack/task-start.ts index 0ccb3793616..309953a69f9 100644 --- a/server/src/services/chat-channels/slack/task-start.ts +++ b/server/src/services/chat-channels/slack/task-start.ts @@ -1073,3 +1073,21 @@ export function createSlackTaskStart(db: Db, options: SlackTaskStartOptions) { } return { latestSlackDmControlThreadId, finalizeSlackTaskStartFailure, admitConfirmedSlackTaskStart, processSlackTaskStart, processPendingSlackTaskStarts }; } + +export function slackTaskStartActivityStatus(action: { + status: string; + updatedAt: Date; +}): string { + const age = Date.now() - action.updatedAt.getTime(); + if (action.status === "validating" && age >= SLACK_COMMAND_POST_STALE_MS) { + return "queued"; + } + if ( + (action.status === "received" && age >= SLACK_COMMAND_POST_STALE_MS) || + (action.status === "resolving" && + age >= SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS) + ) { + return "delivery_unknown"; + } + return action.status; +} diff --git a/server/src/services/chat-channels/telegram/callback-notices.ts b/server/src/services/chat-channels/telegram/callback-notices.ts new file mode 100644 index 00000000000..fe926e033b1 --- /dev/null +++ b/server/src/services/chat-channels/telegram/callback-notices.ts @@ -0,0 +1,128 @@ +import type { EndpointRow, DbTransaction, DbOrTransaction } from "../runtime-types.js"; + +import { + createHash, +} from "node:crypto"; +import { + and, + eq, +} from "drizzle-orm"; + +import { + chatEndpointResources, + chatExternalPrincipals, + chatIdentityLinks, +} from "@paperclipai/db"; + +import { + type TelegramCallbackReceipt, +} from "../../chat-telegram-ephemeral.js"; + +type TelegramCallbackNoticeAuthorityOptions = { + nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean; + lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; linkedDenied: boolean; userId: string | null; sponsorUserId?: string | null }>; +}; + +export function createTelegramCallbackNoticeAuthority(options: TelegramCallbackNoticeAuthorityOptions) { + const { nonDirectDestinationAllowed, lockCurrentPrincipalAuthorization } = options; + + + // This is permission to return one fixed, neutral callback denial, not + // permission to execute the denied action or publish to the conversation. + // Retain the initial identity-policy snapshot so queued notices cannot cross + // a subsequent actor/link/reach change. No provider I/O occurs under locks. + async function lockTelegramCallbackNoticeAuthority( + tx: DbTransaction, + endpoint: EndpointRow, + receipt: TelegramCallbackReceipt, + principalId: string | null, + ): Promise { + if ( + endpoint.provider !== "telegram" || + endpoint.status !== "active" || + endpoint.companyId !== receipt.companyId || + endpoint.id !== receipt.endpointId || + endpoint.botExternalId !== receipt.botUserId || + !endpoint.providerAccountId || + !principalId || + Date.now() < receipt.receivedAtMs || + Date.now() >= receipt.deadlineAtMs + ) + return null; + const resource = + receipt.chatType === "private" + ? null + : await tx + .select() + .from(chatEndpointResources) + .where( + and( + eq(chatEndpointResources.companyId, endpoint.companyId), + eq(chatEndpointResources.endpointId, endpoint.id), + eq(chatEndpointResources.providerResourceId, receipt.chatId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if ( + receipt.chatType === "private" + ? !endpoint.allowDirectMessages + : !resource || !nonDirectDestinationAllowed(endpoint, resource) + ) + return null; + const principal = await tx + .select() + .from(chatExternalPrincipals) + .where( + and( + eq(chatExternalPrincipals.companyId, endpoint.companyId), + eq(chatExternalPrincipals.provider, endpoint.provider), + eq( + chatExternalPrincipals.providerAccountId, + endpoint.providerAccountId, + ), + eq(chatExternalPrincipals.id, principalId), + eq(chatExternalPrincipals.externalId, receipt.receiverUserId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + if (!principal || principal.kind !== "user" || principal.isBot) return null; + const authorization = await lockCurrentPrincipalAuthorization( + tx, + endpoint, + principal.id, + ); + const link = await tx + .select({ + status: chatIdentityLinks.status, + userId: chatIdentityLinks.paperclipUserId, + }) + .from(chatIdentityLinks) + .where( + and( + eq(chatIdentityLinks.companyId, endpoint.companyId), + eq(chatIdentityLinks.endpointId, endpoint.id), + eq(chatIdentityLinks.principalId, principal.id), + ), + ) + .then((rows) => rows[0] ?? null); + return createHash("sha256") + .update( + JSON.stringify([ + endpoint.companyId, + endpoint.id, + receipt.botUserId, + resource?.id ?? null, + principal.id, + principal.externalId, + link, + authorization, + endpoint.allowUnlinkedPeople, + endpoint.sponsorUserId, + ]), + ) + .digest("hex"); + } + return { lockTelegramCallbackNoticeAuthority }; +} diff --git a/server/src/services/chat-channels/telegram/messages.ts b/server/src/services/chat-channels/telegram/messages.ts index 152b6478533..ac75997b145 100644 --- a/server/src/services/chat-channels/telegram/messages.ts +++ b/server/src/services/chat-channels/telegram/messages.ts @@ -1,3 +1,4 @@ +import type { Attachment } from "chat"; import { createHash } from "node:crypto"; import type { chatDeliveries } from "@paperclipai/db"; import type { ChatProvider } from "@paperclipai/shared"; @@ -221,3 +222,14 @@ export function telegramMessageSentAt(raw: unknown): Date | null { const sentAt = new Date(value * 1_000); return Number.isFinite(sentAt.getTime()) ? sentAt : null; } + +export function telegramMarkdownAttachment(text: string): Attachment { + const data = Buffer.from(text, "utf8"); + return { + data, + mimeType: "text/markdown; charset=utf-8", + name: "paperclip-response.md", + size: data.byteLength, + type: "file", + }; + }