diff --git a/doc/connections/slack-setup.md b/doc/connections/slack-setup.md index e167017e84f..2774531dd5f 100644 --- a/doc/connections/slack-setup.md +++ b/doc/connections/slack-setup.md @@ -179,3 +179,101 @@ An explicit managed reconnect reinstalls the saved app and verifies the replacem bot credentials. Its request ID is saved before dispatch, so a duplicate completed request does not reinstall again. Credential-configuration recovery still resumes the saved token after a downstream failure. No reconnect creates another Slack app. + +## Durable Cloud Events API delivery + +A broker that advertises `slackEvents.version=1` can accept a signed Events API +registration after ordinary configuration settles. The producer uses the saved +child app ID, current workspace, signing key, endpoint/connection and runtime +generation. It retains the existing public callback URL. Missing verified app +identity or broker support leaves the existing setup in place; it does not guess +an identity or substitute the Slack manager's signing key. + +Cloud stores the child signing key through its existing encrypted stack vault. +Registration metadata and receipts contain no plaintext credentials. An unchanged +credential/endpoint identity reuses its immutable vault ref across generations; +rotation and current registration authority remain separately fenced. Removal +retires exact queue membership and prevents an old generation from reenrolling. +The local configuration transaction first saves nonsecret enrollment or +retirement intent and signals the existing connection-events owner. Broker work +runs through that owner after commit; configuration APIs do not await an optional +Cloud request. Configure, pause and removal can complete locally while Cloud is unavailable. The owner +retries only retained due intent and reconstructs it after restart, including +removal for archived endpoints after credential cleanup. Confirmation replaces +only the same current intent and generation; a completed remote call with a +lost response remains pending. Current authorization errors retain visible +attention debt instead of claiming registration succeeded. Event draining and +acknowledgments run before enrollment reconciliation. The owner then admits one +due enrollment operation and schedules the earliest retained responsibility. +A busy credential lease or enrollment failure retains a bounded retry without +aborting that event drain. Rollback leaves no committed intent to run. + +Retirement sends the original responsible subject and the exact instance, +company, endpoint, public ID, connection and monotonic generation. Cloud can +publish a nonreceiving generation floor without a signing key, live descriptor +or previously returned binding ID. This fences delayed enrollment after local +removal. Normal current broker authority still applies; the subject alone does +not authorize an operation. An instance that never enrolled with Cloud creates +no new intent or background retry when Cloud is absent. Previously configured +retirement responsibility retains its original instance identity as visible +attention if that identity becomes unavailable. Unsupported capability or disabled +new enrollment retains bounded retry responsibility; a closed generation or +identity conflict requires attention. Current authorized confirmation is required +before clearing the exact pending intent. + +Ordinary redeploys retain the same stack identity. To move enrollment to another +stack, first confirm retirement under the old stack's authority, with no pending +old intent. Then explicitly reconfigure under the new authorized stack at a newer +runtime generation. The old stored identity remains until the new broker receipt +matches the current intent and generation. An environment change cannot transfer +live responsibility. Unavailable old authority remains visible attention. + +A missing or disabled signing credential also requires attention. Enrollment does +not retry it as a broker outage. Incoming receipts retain typed attention before +SDK admission; uncertain authentication does not acknowledge the receipt. + +Only authenticated `event_callback` messages use this durable lane. The sealed +queue retains original bytes, signature, timestamp and acceptance time. The app +checks current endpoint/generation authority before SDK handling. It verifies +original freshness at Cloud acceptance; ordinary external requests still use +current time. The credential snapshot lease releases before normal lifecycle +callbacks reacquire their own leases. + +Private replay provenance prevents internal replay URLs from replacing saved +external callback verification evidence. The normal transactional chat delivery +ledger remains the message admission authority. An ambiguous callback or ACK +failure retains its receipt; malformed receipts rejected before useful admission +can be explicitly ignored with bounded diagnostics. Credential-authentication +uncertainty requires attention and is not silently acknowledged. The broker's +existing retention horizon remains the disclosed recovery boundary. + +This does not qualify a complete Slack endpoint for sleep. Slash commands, +interactive actions, modal validation and challenges still need their full +synchronous response semantics. The endpoint blocker remains. Qualification +must cover normal signed enrollment, durable retry after removal, sealed drain, +the actual event worker and SDK, lost-ACK redelivery and one domain effect, +generation changes, and callback evidence. Synthetic protocol coverage does not +prove live Slack, complete configuration UI or a deployed stop/restart/wake journey. + +### Reproduce the paired Cloud acceptance + +Use the repository's supported Node version and installed dependencies. Compile +the companion Cloud checkout first. Set `PAPERCLIP_SLACK_TEST_DATABASE_URL` and +`SLACK_EVENT_TEST_DATABASE_URL` to two distinct, owned loopback PostgreSQL +`*_test` databases. Set `SLACK_TEST_CLOUD_DIST` to its absolute compiled `dist` +directory. This test applies actual migrations and uses only those two databases. +It does not create a PostgreSQL cluster or use a live Slack provider. + +From the app repository root, run: + +```sh +node server/scripts/test-slack-cloud-postgres.mjs +``` + +The launcher uses the current Node executable and one Vitest worker. It loads the +checked-in prewire guard in that actual worker before client construction. The +controlled process child explicitly loads the same guard before its script. +The guard rejects off-host and unowned socket targets before transport. Caught +unexpected attempts still fail the fixture. No private preload path or manual +`SLACK_GATE_GUARD_PID` setting is needed. The fixture remains opt-in during the +normal test suite; its default owner and guard regressions run without databases. diff --git a/server/scripts/test-slack-cloud-owner.mjs b/server/scripts/test-slack-cloud-owner.mjs new file mode 100644 index 00000000000..26b56d932c8 --- /dev/null +++ b/server/scripts/test-slack-cloud-owner.mjs @@ -0,0 +1,36 @@ +import { spawn } from "node:child_process"; +import { createRequire } from "node:module"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const patterns = { "--tool-removal-only": "public tool removal retains Slack retirement", "--fairness-only": "two due registrations", "--malformed-only": "malformed sealed Slack receipt", + "--wiring-only": "normal configure stages SQL intent", "--cas-only": "real SQL refuses an enrollment response" }; +const args = process.argv.slice(2); +if (!process.version.startsWith("v24.")) throw new Error("Slack owner fixture requires the current Node24 executable"); +if (args.length > 1 || args.length && !Object.hasOwn(patterns, args[0])) throw new Error("Unknown Slack owner fixture selector"); +const value = process.env.PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL; +if (!value) throw new Error("Set PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL to the exact owned external test database"); +const fixture = new URL(value), admin = new URL(process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ?? "invalid:"); +if (!["postgres:", "postgresql:"].includes(fixture.protocol) || fixture.hostname !== "127.0.0.1" || fixture.port !== "55448" + || !/^[a-zA-Z0-9_]+_test$/.test(fixture.pathname.slice(1)) || fixture.search || fixture.hash + || admin.protocol !== fixture.protocol || admin.hostname !== fixture.hostname || admin.port !== fixture.port + || admin.pathname !== "/postgres" || admin.search || admin.hash) throw new Error("Slack owner fixture requires its explicit owned PG17 URL/admin grant"); +const log = process.env.SLACK_GATE_PROCESS_LOG; +if (!log || !path.isAbsolute(log) || existsSync(log)) throw new Error("Slack owner fixture requires a fresh absolute guard log"); +const server = fileURLToPath(new URL("../", import.meta.url)); +const require = createRequire(new URL("../package.json", import.meta.url)); +const vitest = path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs"); +const guard = path.join(server, "src/__tests__/helpers/slack-cloud/prewire.cjs"); +const home = mkdtempSync(path.join(tmpdir(), "slack-owner-fixture-")); +const env = { PATH: process.env.PATH, HOME: process.env.HOME, CODEX_HOME: process.env.CODEX_HOME, LANG: process.env.LANG, + TMPDIR: home, PAPERCLIP_HOME: home, NODE_ENV: "test", NODE_OPTIONS: "", SLACK_GATE_EXPECT_PRELOAD: "1", + PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL: value, PAPERCLIP_SLACK_TEST_DATABASE_URL: value, + PAPERCLIP_TEST_POSTGRES_ADMIN_URL: process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL, SLACK_GATE_PROCESS_LOG: log }; +const child = spawn(process.execPath, ["--require", guard, vitest, "run", "src/__tests__/slack-cloud-owner-work.test.ts", + "--maxWorkers=1", "--no-file-parallelism", `--execArgv=--require=${guard}`, + ...(args.length ? [`--testNamePattern=${patterns[args[0]]}`] : [])], { cwd: server, env, stdio: "inherit" }); +for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => child.kill(signal)); +child.on("error", error => { console.error(error); process.exitCode = 1; }); +child.on("close", (code, signal) => { rmSync(home, { recursive: true, force: true }); process.exitCode = signal ? 1 : code ?? 1; }); diff --git a/server/scripts/test-slack-cloud-postgres.mjs b/server/scripts/test-slack-cloud-postgres.mjs new file mode 100644 index 00000000000..c26068b10a6 --- /dev/null +++ b/server/scripts/test-slack-cloud-postgres.mjs @@ -0,0 +1,47 @@ +import { spawn } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +if (process.argv.slice(2).length > 1 || process.argv.slice(2).some(arg => !['--instance-transition-only', '--normal-owner-only'].includes(arg))) throw new Error('Unknown Slack fixture selector'); +const transitionOnly = process.argv.includes('--instance-transition-only'); +const normalOwnerOnly = process.argv.includes('--normal-owner-only'); +const server = fileURLToPath(new URL('../', import.meta.url)); +const require = createRequire(new URL('../package.json', import.meta.url)); +const guard = path.join(server, 'src/__tests__/helpers/slack-cloud/prewire.cjs'); +const names = ['PAPERCLIP_SLACK_TEST_DATABASE_URL', 'SLACK_EVENT_TEST_DATABASE_URL']; +const urls = names.map(name => { + if (!process.env[name]) throw new Error(`Set ${name} to an owned dedicated PostgreSQL test database`); + let url; + try { url = new URL(process.env[name]); } catch { throw new Error(`${name} must name an explicit loopback *_test database`); } + if (!['postgres:', 'postgresql:'].includes(url.protocol) || !['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname) + || !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1))) + || ['options', 'database', 'dbname'].some(k => url.searchParams.has(k))) throw new Error(`${name} must name an explicit loopback *_test database`); + return url; +}); +if (urls[0].pathname === urls[1].pathname) throw new Error('Slack app and Cloud fixtures require distinct databases'); +if (!process.env.SLACK_TEST_CLOUD_DIST || !path.isAbsolute(process.env.SLACK_TEST_CLOUD_DIST) + || !existsSync(path.join(process.env.SLACK_TEST_CLOUD_DIST, 'connections/broker.js'))) throw new Error('SLACK_TEST_CLOUD_DIST must name the compiled companion Cloud dist directory'); +const vitest = path.join(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); +const ownedHome = mkdtempSync(path.join(tmpdir(), 'paperclip-slack-paired-')); +const env = { + PATH: process.env.PATH, HOME: process.env.HOME, CODEX_HOME: process.env.CODEX_HOME, + LANG: process.env.LANG, TMPDIR: ownedHome, PAPERCLIP_HOME: ownedHome, + NODE_ENV: 'test', NODE_OPTIONS: '', SLACK_GATE_EXPECT_PRELOAD: '1', + PAPERCLIP_SLACK_TEST_DATABASE_URL: process.env.PAPERCLIP_SLACK_TEST_DATABASE_URL, + SLACK_EVENT_TEST_DATABASE_URL: process.env.SLACK_EVENT_TEST_DATABASE_URL, + SLACK_TEST_CLOUD_DIST: process.env.SLACK_TEST_CLOUD_DIST, + SLACK_GATE_PROCESS_LOG: process.env.SLACK_GATE_PROCESS_LOG, +}; +const child = spawn(process.execPath, ['--require', guard, vitest, 'run', + ...(transitionOnly ? ['src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts'] : normalOwnerOnly + ? ['src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts'] : ['src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts', 'src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts']), '--maxWorkers=1', '--no-file-parallelism', `--execArgv=--require=${guard}`], +{ cwd: server, env, stdio: 'inherit' }); +for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); +child.on('error', error => { console.error(error); process.exitCode = 1; }); +child.on('close', (code, signal) => { + rmSync(ownedHome, { recursive: true, force: true }); + process.exitCode = signal ? 1 : code ?? 1; +}); diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index c60caf97dc6..07b9616c0a8 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -17,7 +17,7 @@ import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:f import os from "node:os"; import path from "node:path"; import { and, eq, sql } from "drizzle-orm"; -import { createDb, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; +import { createDb, type Db, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres"; import { fetchCompanyQuotaWindows } from "../services/quota-windows.js"; import { quotaCredentialRecoveryPath, quotaCredentialRecovery, quotaCredentialHash } from "../services/quota-credential-recovery.js"; @@ -2213,6 +2213,33 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello" const [agent] = await db.select().from(agents).where(eq(agents.id, agentId)); expect(agent.runtimeConfig.aiConnection).toBeUndefined(); }); + it("keeps account summaries ordered across sequential scans and repeat migration", async () => { + const orderedCompanyId = randomUUID(), userId = "stable-list-owner"; + await db.insert(companies).values({ id: orderedCompanyId, name: "Stable account list", issuePrefix: `O${orderedCompanyId.slice(0, 5)}` }); + await db.insert(companyMemberships).values({ companyId: orderedCompanyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" }); + const accounts = []; + for (const name of ["First account", "Second account", "Third account"]) { + accounts.push(await service.save(orderedCompanyId, userId, { provider: "anthropic", method: "api_key", ownership: "personal", name, apiKey: "fixture-stable-list", agentIds: [], allAgents: false }, "fixture-stable-list")); + } + const expected = accounts.map(({ connectionId: id, grantId }) => ({ id, grantId })) + .sort((a, b) => a.id.localeCompare(b.id) || a.grantId.localeCompare(b.grantId)); + await db.transaction(async tx => { + await tx.execute(sql`SET LOCAL enable_indexscan = off`); + await tx.execute(sql`SET LOCAL enable_bitmapscan = off`); + await tx.execute(sql`SET LOCAL enable_hashjoin = off`); + await tx.execute(sql`SET LOCAL enable_mergejoin = off`); + for (const account of [...expected].reverse()) { + await tx.update(toolConnections).set({ name: sql`${toolConnections.name}` }).where(eq(toolConnections.id, account.id)); + await tx.update(connectionGrants).set({ status: sql`${connectionGrants.status}` }).where(eq(connectionGrants.id, account.grantId)); + } + const transactionalService = aiConnectionService(tx as unknown as Db); + const before = await transactionalService.list(orderedCompanyId, userId); + expect(before.map(({ id, grantId }) => ({ id, grantId }))).toEqual(expected); + const migration = await readFile(new URL("../../../packages/db/src/migrations/0276_hard_mandroid.sql", import.meta.url), "utf8"); + for (const statement of migration.split("--> statement-breakpoint").filter(value => value.trim())) await tx.execute(sql.raw(statement)); + expect(await transactionalService.list(orderedCompanyId, userId)).toEqual(before); + }); + }); it("runs two Claude subscription executions for the same grant at the same time", async () => { // Claude writes no auth file back to the grant, so two runs share no // mutable state and must not wait for each other. diff --git a/server/src/__tests__/github-connection-enrollment-work.test.ts b/server/src/__tests__/github-connection-enrollment-work.test.ts index 80a014e7bb6..719da5e7157 100644 --- a/server/src/__tests__/github-connection-enrollment-work.test.ts +++ b/server/src/__tests__/github-connection-enrollment-work.test.ts @@ -90,6 +90,13 @@ const support = await getEmbeddedPostgresTestSupport(); grantId = grant!.id; const env = { PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: origin, PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test" }; const service = githubConnectionEventService(db, { env }); + const reserved: Awaited>[] = []; + try { + for (let slot = 0; slot < 4; slot++) reserved.push(await db.$client.reserve()); + await Promise.all(reserved.map(connection => connection`SELECT 1`)); + } finally { + for (const connection of reserved) connection.release(); + } vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: e => errors.push(e) }); const selects = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute"); diff --git a/server/src/__tests__/helpers/slack-cloud-connector-fixture.ts b/server/src/__tests__/helpers/slack-cloud-connector-fixture.ts new file mode 100644 index 00000000000..c20ad8eb752 --- /dev/null +++ b/server/src/__tests__/helpers/slack-cloud-connector-fixture.ts @@ -0,0 +1,97 @@ +import { createCipheriv, createHash, createHmac, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, verify } from "node:crypto"; +import { expect, vi } from "vitest"; +import { createPaperclipCloudConnector, type PaperclipCloudConnectorConfig } from "../../services/paperclip-cloud-connector.js"; + +export function createSlackCloudConnectorFixture(instanceId: string, companyId: string, subject: string) { + const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); + const config: PaperclipCloudConnectorConfig = { baseUrl: "http://127.0.0.1:1", instanceId, environment: "test", + signPrivateKey: signing.privateKey.export({ format: "jwk" }).d!, sealPrivateKey: sealing.privateKey.export({ format: "jwk" }).d! }; + const calls: { operation: string; enrollment?: Record }[] = []; + const acknowledged: string[] = []; + const events: Record[] = []; + let unavailable = false; + let beforeEnrollmentReceipt: (() => Promise) | undefined; + function seal(payload: unknown, profile: "github.code" | "slack.events.v1") { + const provider = profile === "github.code" ? "github" : "slack"; + const ephemeral = generateKeyPairSync("x25519"); + const epk = ephemeral.publicKey.export({ format: "jwk" }).x!; + const recipient = sealing.publicKey.export({ format: "jwk" }).x!; + const aad = Buffer.from([1, "X25519-HKDF-SHA256-A256GCM", "events", instanceId, "test", provider, profile, ""].join("\n")); + const key = Buffer.from(hkdfSync("sha256", diffieHellman({ privateKey: ephemeral.privateKey, publicKey: sealing.publicKey }), + Buffer.concat([Buffer.from(epk, "base64url"), Buffer.from(recipient, "base64url")]), aad, 32)); + const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad); + const ct = Buffer.concat([cipher.update(JSON.stringify(payload), "utf8"), cipher.final(), cipher.getAuthTag()]); + return { v: 1, alg: "X25519-HKDF-SHA256-A256GCM", purpose: "events", provider, profile, epk, + iv: iv.toString("base64url"), ct: ct.toString("base64url") }; + } + const requestErrors: unknown[] = []; + const request: typeof fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit): Promise => { + try { + const url = new URL(input instanceof Request ? input.url : String(input)); + expect(url.origin).toBe(config.baseUrl); + expect(init?.method).toBe("POST"); + const body = JSON.parse(String(init?.body)) as { request: string; enrollment?: string; acknowledgement?: string }; + const [header, payload, signature] = body.request.split("."); + expect(verify(null, Buffer.from(`${header}.${payload}`), signing.publicKey, Buffer.from(signature!, "base64url"))).toBe(true); + const claims = JSON.parse(Buffer.from(payload!, "base64url").toString("utf8")) as Record; + expect(claims).toMatchObject({ iss: instanceId, env: "test", aud: url.toString() }); + const operation = String(claims.op); + if (operation !== "status") { + const expectedSubject = operation === "slack-events" ? subject + : ["event-lease", "event-ack"].includes(operation) ? "github-bot-events" : undefined; + if (!expectedSubject) throw new Error(`Unexpected synthetic connector authority ${operation}`); + expect(claims).toMatchObject({ sub: expectedSubject, cid: companyId }); + } + if (unavailable) return Response.json({ error: "fixture_broker_unavailable" }, { status: 503 }); + if (operation === "status") return Response.json({ active: true, status: "active", profiles: [], slackEvents: { version: 1 } }); + if (operation === "slack-events") { + expect(claims.sh).toBe(createHash("sha256").update(body.enrollment!).digest("base64url")); + const enrollment = JSON.parse(body.enrollment!) as Record; + calls.push({ operation, enrollment: { ...enrollment, signingSecret: undefined } }); + await beforeEnrollmentReceipt?.(); + const { active, signingSecret: _secret, ...identity } = enrollment; + const bindingId = `slack-event:${createHash("sha256").update(JSON.stringify(identity)).digest("hex")}`; + return Response.json(active === false ? { retirement: { version: 1, instanceId, ...identity } } + : { enrollment: { version: 1, instanceId, ...identity, bindingId } }); + } + calls.push({ operation }); + if (operation === "event-lease") { + if (!events.length) return Response.json({ events: [], eventWakeVersion: 1 }); + const leased = events.splice(0); + return Response.json({ leaseId: "fixture-lease", eventWakeVersion: 1, + sealed: seal({ v: 1, instanceId, environment: "test", leaseId: "fixture-lease", events: leased }, "github.code") }); + } + if (operation === "event-ack") { + expect(claims.sh).toBe(createHash("sha256").update(body.acknowledgement!).digest("base64url")); + const value = JSON.parse(body.acknowledgement!) as { deliveryIds: string[] }; + acknowledged.push(...value.deliveryIds); return Response.json({ acknowledged: value.deliveryIds.length }); + } + throw new Error(`Unexpected synthetic connector operation ${operation}`); + } catch (error) { + requestErrors.push(error); + console.error("Synthetic Slack connector rejected its request", error); + throw error; + } + }); + function install() { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", config.baseUrl); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", config.signPrivateKey); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", config.sealPrivateKey); + } + function enqueue(identity: Record, signingSecret: string, eventId: string) { + const acceptedAt = new Date().toISOString(), timestamp = String(Math.floor(Date.now() / 1000)); + const body = Buffer.from(JSON.stringify({ type: "event_callback", team_id: identity.teamId, api_app_id: identity.appId, event_id: eventId, + event: { type: "app_mention", text: "<@UBOT> fixture callback", user: "UOPERATOR", channel: "CTEST", ts: `${Date.now() / 1000}`, event_ts: `${Date.now() / 1000}` } })); + const id = `slack_${String(identity.bindingId).slice("slack-event:".length)}_${createHash("sha256").update(eventId).digest("hex")}`; + const packet = { version: 1, instanceId, environment: "test", ...identity, eventId, acceptedAt, rawBody: body.toString("base64"), timestamp, + signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(body).digest("hex")}` }; + events.push({ id, provider: "slack", event: "event_callback", installationId: identity.bindingId, bindingIds: [identity.bindingId], createdAt: acceptedAt, + payload: { slackEvent: { bindingId: identity.bindingId, sealed: seal(packet, "slack.events.v1") } } }); + return { id, packet, body }; + } + return { config, request, requestErrors, calls, acknowledged, events, install, enqueue, connector: createPaperclipCloudConnector({ config, request }), + setUnavailable(value: boolean) { unavailable = value; }, + beforeReceipt(work: (() => Promise) | undefined) { beforeEnrollmentReceipt = work; } }; +} diff --git a/server/src/__tests__/helpers/slack-cloud/prewire.cjs b/server/src/__tests__/helpers/slack-cloud/prewire.cjs new file mode 100644 index 00000000000..04ff4b5fd43 --- /dev/null +++ b/server/src/__tests__/helpers/slack-cloud/prewire.cjs @@ -0,0 +1,94 @@ +const key = Symbol.for('paperclip.slack.fixture.prewire'); +if (!process[key]) { + const net = require('node:net'); + const tls = require('node:tls'); + const http = require('node:http'); + const https = require('node:https'); + const fs = require('node:fs'); + const loopback = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']); + const listeners = new Set(); + const pgTargets = new Set(); + const pgPorts = new Set([5432, 55448]); + for (const name of ['PAPERCLIP_SLACK_TEST_DATABASE_URL', 'SLACK_EVENT_TEST_DATABASE_URL']) { + if (!process.env[name]) continue; + let url; + try { url = new URL(process.env[name]); } catch { throw new Error('Invalid Slack fixture database target'); } + if (!['postgres:', 'postgresql:'].includes(url.protocol) || !loopback.has(url.hostname) + || !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1))) + || ['options', 'database', 'dbname'].some(k => url.searchParams.has(k))) throw new Error('Invalid Slack fixture database target'); + const port = Number(url.port || 5432); + pgPorts.add(port); + pgTargets.add(`${url.hostname}:${port}`); + } + const sockets = new Set(); + const observed = new WeakSet(); + const attempts = []; + let peakPgSockets = 0; + const record = value => { + if (process.env.SLACK_GATE_PROCESS_LOG) fs.appendFileSync(process.env.SLACK_GATE_PROCESS_LOG, JSON.stringify(value) + '\n'); + }; + const fail = (host, port) => { + attempts.push({ host: String(host), port: Number(port) || null }); + process.exitCode = 98; + throw new Error('Slack fixture blocked an unowned network target before transport'); + }; + const check = (host = 'localhost', port, protocol) => { + host = String(host).toLowerCase(); + if (!loopback.has(host)) fail(host, port); + if (protocol && !['http:', 'https:'].includes(protocol)) fail(host, port); + if (port && pgPorts.has(Number(port)) && !pgTargets.has(`${host}:${Number(port)}`)) fail(host, port); + if (port && !pgTargets.has(`${host}:${Number(port)}`) && !listeners.has(Number(port))) fail(host, port); + }; + const checkArgs = args => { + const first = args[0]; + if (first instanceof URL || typeof first === 'string' && /^https?:/.test(first)) { + const url = first instanceof URL ? first : new URL(first); + check(url.hostname, url.port || (url.protocol === 'https:' ? 443 : 80), url.protocol); + } else if (first && typeof first === 'object') { + if (first.path && !first.port && !first.host && !first.hostname) fail('unix-socket', 0); + check(first.hostname || first.host, first.port, first.protocol); + } else if (typeof first === 'number') check(typeof args[1] === 'string' ? args[1] : undefined, first); + else fail('unknown-target', 0); + }; + const listen = net.Server.prototype.listen; + net.Server.prototype.listen = function (...args) { + this.once('listening', () => { + const address = this.address(); + if (address && typeof address === 'object' && loopback.has(address.address)) { + listeners.add(address.port); + this.once('close', () => listeners.delete(address.port)); + } + }); + return Reflect.apply(listen, this, args); + }; + for (const module of [http, https]) for (const name of ['request', 'get']) { + const original = module[name]; + module[name] = function (...args) { checkArgs(args); return Reflect.apply(original, this, args); }; + } + const connect = net.Socket.prototype.connect; + net.Socket.prototype.connect = function (...args) { + const normalized = Array.isArray(args[0]) ? args[0] : args; + checkArgs(normalized); + const first = normalized[0]; + const port = typeof first === 'object' ? first.port : first; + if (pgPorts.has(Number(port)) && !observed.has(this)) { + observed.add(this); + this.once('connect', () => { sockets.add(this); peakPgSockets = Math.max(peakPgSockets, sockets.size); }); + this.once('close', () => sockets.delete(this)); + } + return Reflect.apply(connect, this, args); + }; + const tlsConnect = tls.connect; + tls.connect = function (...args) { checkArgs(args); return Reflect.apply(tlsConnect, this, args); }; + const fetch = globalThis.fetch; + if (fetch) globalThis.fetch = function (input, init) { + checkArgs([input instanceof Request ? input.url : input]); + return fetch(input, init); + }; + const stats = () => ({ pid: process.pid, peakPgSockets, remainingPgSockets: sockets.size, unexpectedAttempts: attempts.slice() }); + process[key] = Object.freeze({ pid: process.pid, stats }); + globalThis[key] = process[key]; + process.env.SLACK_GATE_GUARD_PID = String(process.pid); + record({ phase: 'repository-preload', pid: process.pid }); + process.on('exit', () => record({ phase: 'exit', ...stats(), exitCode: process.exitCode || 0 })); +} diff --git a/server/src/__tests__/helpers/slack-cloud/transport.ts b/server/src/__tests__/helpers/slack-cloud/transport.ts new file mode 100644 index 00000000000..3cf3d0cbd7f --- /dev/null +++ b/server/src/__tests__/helpers/slack-cloud/transport.ts @@ -0,0 +1,76 @@ +import http from "node:http"; +import https from "node:https"; +import { expect, vi } from "vitest"; + +export function guardSlackFixtureNetwork(options: { requirePreload?: boolean } = {}) { + const prewire = Reflect.get(process, Symbol.for("paperclip.slack.fixture.prewire")) as { + pid: number; + stats(): { pid: number; peakPgSockets: number; remainingPgSockets: number; unexpectedAttempts: unknown[] }; + } | undefined; + if (options.requirePreload) { + expect(prewire?.pid).toBe(process.pid); + expect(process.env.SLACK_GATE_GUARD_PID).toBe(String(process.pid)); + } + if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1") { + expect(process.env.SLACK_GATE_GUARD_PID).toBe(String(process.pid)); + console.info(JSON.stringify({ slackFixtureWorkerPid: process.pid, prewireGuardPid: process.env.SLACK_GATE_GUARD_PID })); + } + const blocked: string[] = []; + const check = (value: unknown) => { + const url = value instanceof URL ? value : typeof value === "string" ? new URL(value) : undefined; + const options = value as { hostname?: string; host?: string } | undefined; + const host = url?.hostname ?? options?.hostname ?? options?.host ?? "localhost"; + if (!["localhost", "127.0.0.1", "::1", "[::1]"].includes(host)) { + blocked.push(host); + throw new Error("Unexpected off-host Slack fixture request"); + } + }; + const realHttp = http.request, realHttps = https.request, realFetch = globalThis.fetch; + const httpSpy = vi.spyOn(http, "request").mockImplementation(((...args: unknown[]) => { + check(args[0]); return Reflect.apply(realHttp, http, args); + }) as typeof http.request); + const httpsSpy = vi.spyOn(https, "request").mockImplementation(((...args: unknown[]) => { + check(args[0]); return Reflect.apply(realHttps, https, args); + }) as typeof https.request); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { + check(input instanceof Request ? input.url : input); return realFetch(input, init); + }); + return { assertQuiet: () => { + expect(blocked).toEqual([]); + if (prewire) expect(prewire.stats().unexpectedAttempts).toEqual([]); + const stats = prewire?.stats; + if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1") { + expect(stats).toBeTypeOf("function"); + console.info(JSON.stringify({ slackFixtureWorkerPid: process.pid, networkStats: stats!() })); + } + }, restore: () => { + fetchSpy.mockRestore(); httpsSpy.mockRestore(); httpSpy.mockRestore(); + } }; +} + +export function stubSlackFixtureTransport(adapter: unknown) { + const client = (adapter as { _client: { + axios: { defaults: { adapter: unknown } }; + buildResult(response: unknown): Promise; + } })._client; + const failures: unknown[] = []; + const buildResult = client.buildResult.bind(client); + client.buildResult = async response => { + try { return await buildResult(response); } + catch (error) { failures.push(error); throw error; } + }; + const transport = vi.fn(async (config: { url: string; baseURL: string }) => { + const url = new URL(config.url, config.baseURL); + if (url.hostname !== "127.0.0.1") throw new Error("Slack fixture API origin changed"); + let data: unknown; + if (url.pathname.endsWith("/users.info")) data = { ok: true, user: { id: "UOPERATOR", name: "operator", is_bot: false, profile: { display_name: "Operator" } } }; + else if (url.pathname.endsWith("/conversations.info")) data = { ok: true, channel: { id: "CTEST", name: "fixture" } }; + else throw new Error(`Unexpected Slack fixture API method ${url.pathname}`); + return { data, status: 200, statusText: "OK", headers: {}, config, request: { path: url.pathname } }; + }); + client.axios.defaults.adapter = transport; + return Object.assign(transport, { assertNoErrors: async () => { + await Promise.all(transport.mock.results.map(result => result.value)); + expect(failures).toEqual([]); + } }); +} diff --git a/server/src/__tests__/slack-cloud-owner-work.test.ts b/server/src/__tests__/slack-cloud-owner-work.test.ts new file mode 100644 index 00000000000..fb66a4d278e --- /dev/null +++ b/server/src/__tests__/slack-cloud-owner-work.test.ts @@ -0,0 +1,433 @@ +import { createHmac, randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { SLACK_CHAT_BOT_SCOPES } from "@paperclipai/shared"; +import { activityLog, agentRuntimeState, agentTaskSessions, agentWakeupRequests, agents, applyPendingMigrations, authUsers, + chatDeliveries, chatEndpoints, chatExternalPrincipals, companySecretBindings, chatIdentityLinks, chatSlackRegistrations, companies, companyMemberships, + companySecrets, companySkills, costEvents, financeEvents, heartbeatRunEvents, heartbeatRuns, issueComments, issues, toolApplications, toolConnections, + closeCheckpointDb, closeFileStorageDb, closeLivenessDb, createDb } from "@paperclipai/db"; +import { databaseWorkPending, reconcileDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db/work-signals"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createSlackCloudConnectorFixture } from "./helpers/slack-cloud-connector-fixture.js"; +import { chatChannelService } from "../services/chat-channels.js"; +import { toolAccessService } from "../services/tool-access.js"; +import { secretService } from "../services/secrets.js"; +import { ChatRuntime } from "../services/chat-runtime.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { githubConnectionEventService } from "../services/github-connection-events.js"; +import { chatCredentialMutationLease } from "../services/chat-credential-mutation-lease.js"; +import { createSlackCloudEnrollment, registerSlackCloudEnrollmentOwner, reconcilePendingSlackCloudEnrollments } from "../services/chat-channels/slack/cloud-enrollment.js"; +import { conflict } from "../errors.js"; +import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "./helpers/slack-cloud/transport.js"; +import type { InternalSetupState } from "../services/chat-channels/runtime-types.js"; +import type { ChatSdkMessageCallbackEvent } from "../services/chat-sdk-runtime.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL"); +if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1" && !external) throw new Error("Guarded local Slack owner tests require their exact external test database"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("Slack Cloud default owner and SQL fences", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + let service: ReturnType; + let heartbeat: ReturnType; + let cloud: ReturnType; + let network: ReturnType; + let directory: string; + let companyId: string, actorId: string, agentId: string, applicationId: string; + let runtime: ChatRuntime; + let providerFetch: typeof fetch; + const callbacks: ChatSdkMessageCallbackEvent[] = []; + const transports: ReturnType[] = []; + const releases: (() => void)[] = []; + const heldLeases: Promise[] = []; + const deferred: (() => void | Promise)[] = []; + const secret = "0123456789abcdef0123456789abcdef"; + const row = async (id: string) => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, id)))[0]!; + const setup = async (id: string) => (await row(id)).setup as InternalSetupState; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("slack-cloud-owner-"); + const url = external ?? temporary!.connectionString; + await applyPendingMigrations(url); + db = createDb(url, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 60_000); + beforeEach(async () => { + callbacks.length = 0; transports.length = 0; releases.length = 0; heldLeases.length = 0; deferred.length = 0; + network = guardSlackFixtureNetwork({ requirePreload: process.env.SLACK_GATE_EXPECT_PRELOAD === "1" }); + directory = await mkdtemp(path.join(os.tmpdir(), "slack-owner-default-")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(directory, "master.key")); + companyId = randomUUID(); actorId = `slack-owner-${randomUUID()}`; agentId = randomUUID(); applicationId = randomUUID(); + cloud = createSlackCloudConnectorFixture(`instance-${randomUUID()}`, companyId, actorId); cloud.install(); + vi.spyOn(globalThis, "fetch").mockImplementation(cloud.request); + const now = new Date(); + await db.insert(companies).values({ id: companyId, name: "Slack default owner", issuePrefix: `S${companyId.slice(0, 7)}` }); + await db.insert(authUsers).values({ id: actorId, name: "Slack owner", email: `${actorId}@example.test`, emailVerified: true, createdAt: now, updatedAt: now }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, status: "active", membershipRole: "operator" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Slack fixture", adapterType: "process", status: "idle", + runtimeConfig: { heartbeat: { wakeOnDemand: false, intervalSec: 0 } } }); + await db.insert(companyMemberships).values({ companyId, principalType: "agent", principalId: agentId, status: "active", membershipRole: "member" }); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Slack", type: "chat" }); + runtime = makeRuntime(); + const slackFetch: typeof fetch = async (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.origin !== "https://slack.com") throw new Error(`Unexpected Slack fixture origin ${url.origin}`); + if (url.pathname === "/api/auth.test") { + expect(init?.method).toBe("POST"); + const token = new Headers(init?.headers).get("authorization") ?? ""; + const suffix = token.slice(token.lastIndexOf("-") + 1); + return Response.json({ ok: true, team_id: `T${suffix}`, team: "Fixture", user_id: `U${suffix}`, user: "fixture" }, + { headers: { "x-oauth-scopes": SLACK_CHAT_BOT_SCOPES.join(",") } }); + } + if (url.pathname === "/api/conversations.list") return Response.json({ ok: true, + channels: [{ id: "CTEST", name: "fixture", is_member: true }], response_metadata: { next_cursor: "" } }); + if (["/api/reactions.add", "/api/reactions.remove"].includes(url.pathname)) return Response.json({ ok: true }); + throw new Error(`Unexpected Slack fixture method ${url.pathname}`); + }; + providerFetch = slackFetch; + heartbeat = heartbeatService(db); + service = chatChannelService(db, { runtime, heartbeat, fetch: slackFetch, publicBaseUrl: "https://tenant.example", + deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); } }); + }); + afterEach(async () => { + const errors: unknown[] = []; + for (const release of releases) release(); + for (const work of heldLeases) { try { await work; } catch (error) { errors.push(error); } } + try { await service?.shutdown(); } catch (error) { errors.push(error); } + try { await heartbeat?.drainActiveRunExecutions(); } catch (error) { errors.push(error); } + try { + await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId)); + await db.delete(financeEvents).where(eq(financeEvents.companyId, companyId)); + await db.delete(costEvents).where(eq(costEvents.companyId, companyId)); + await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, companyId)); + await db.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, companyId)); + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, companyId)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + await db.delete(issueComments).where(eq(issueComments.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId)); + await db.delete(companySkills).where(eq(companySkills.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + await db.delete(authUsers).where(eq(authUsers.id, actorId)); + } catch (error) { errors.push(error); } + try { network.assertQuiet(); } catch (error) { errors.push(error); } + network.restore(); vi.restoreAllMocks(); vi.unstubAllEnvs(); + await rm(directory, { recursive: true, force: true }); + if (errors.length) throw new AggregateError(errors, "Slack owner fixture cleanup failed; original body failure is reported separately by Vitest"); + }); + afterAll(async () => { + const closed = await Promise.allSettled([closeCheckpointDb(db, 1), closeFileStorageDb(db, 1), closeLivenessDb(db, 1)]); + await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); + const errors = closed.flatMap(value => value.status === "rejected" ? [value.reason] : []); + if (errors.length) throw new AggregateError(errors, "Slack owner auxiliary shutdown failed"); + }); + function makeRuntime() { + const runtime = new ChatRuntime(); + const replace = runtime.replaceEndpoint.bind(runtime); + vi.spyOn(runtime, "replaceEndpoint").mockImplementation(async input => { + if (input.providerConfig.provider !== "slack") throw new Error("Unexpected fixture provider"); + const value = await replace({ ...input, providerConfig: { ...input.providerConfig, + credentials: { ...input.providerConfig.credentials, apiUrl: "http://127.0.0.1:1/api/" } }, + callbacks: { ...input.callbacks, onMessage: async event => { callbacks.push(event); await input.callbacks.onMessage(event); } } }); + transports.push(stubSlackFixtureTransport(value.getProviderAdapter())); return value; + }); + return runtime; + } + async function createEndpoint(suffix: string) { + const endpointId = randomUUID(), connectionId = randomUUID(), publicId = randomUUID().replaceAll("-", "").padEnd(43, "a"); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: `Slack ${suffix}`, uid: endpointId, + transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId, + provider: "slack", status: "draft", sponsorUserId: actorId, allowGroupChats: true, setup: { step: "provider_setup" } }); + await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", appId: `A${suffix}`, + manifest: {}, manifestHash: "fixture", callbackUri: `https://tenant.example/api/chat-webhooks/${publicId}/slack`, createdByUserId: actorId }); + await service.configure(endpointId, { action: "configure", credentials: { botToken: `xoxb-fixture-${suffix}`, signingSecret: secret } }, actorId); + expect((await setup(endpointId)).slackCloudIntent).toMatchObject({ companyId, endpointId, connectionId, + instanceId: cloud.config.instanceId, subject: actorId, active: true }); + return endpointId; + } + async function confirm(endpointId: string) { + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await owner.pollOnce({ forDelivery: true }); + expect((await setup(endpointId)).slackCloudIntent).toBeUndefined(); + expect((await setup(endpointId)).slackCloudEvents).toBeDefined(); + return owner; + } + async function activate(endpointId: string) { + const endpoint = await row(endpointId); + const accountId = endpoint.providerAccountId; + if (!accountId) throw new Error("Normal Slack configure did not confirm its workspace"); + const payload = JSON.stringify({ type: "url_verification", team_id: accountId, api_app_id: `A${accountId.slice(1)}`, challenge: "fixture-challenge" }); + const timestamp = String(Math.floor(Date.now() / 1000)); + const response = await service.handleWebhook(endpoint.publicId, "slack", new Request(`https://tenant.example/api/chat-webhooks/${endpoint.publicId}/slack`, { + method: "POST", headers: { "content-type": "application/json", "x-slack-request-timestamp": timestamp, + "x-slack-signature": `v0=${createHmac("sha256", secret).update(`v0:${timestamp}:${payload}`).digest("hex")}` }, body: payload })); + expect(response.status).toBe(200); await response.body?.cancel(); + await service.configure(endpointId, { action: "verify" }, actorId); + const [principal] = await db.insert(chatExternalPrincipals).values({ companyId, provider: "slack", providerAccountId: accountId, + externalId: "UOPERATOR", kind: "user", displayName: "Operator" }).returning(); + await db.insert(chatIdentityLinks).values({ companyId, endpointId, principalId: principal!.id, paperclipUserId: actorId, status: "linked", confirmedAt: new Date() }); + expect((await service.test(endpointId, { optionalSlackTestForUser: actorId })).status).toBe("active"); + } + async function holdCredentials(endpointId: string) { + let acquired!: () => void; + const ready = new Promise(resolve => { acquired = resolve; }); + const release = new Promise(resolve => { releases.push(resolve); }); + heldLeases.push(chatCredentialMutationLease(db)(await row(endpointId), async lease => { await lease.assertOwned(); acquired(); await release; })); + await ready; + } + + it("normal configure stages SQL intent, worker unseals into registered SDK and old generations cannot reenter", async () => { + const endpointId = await createEndpoint("BOT"); + await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const endpoint = await row(endpointId), saved = await setup(endpointId), enrolled = saved.slackCloudEvents!; + const callbackEvidence = structuredClone(saved.slackCallbackSurfaces); + const delivered = cloud.enqueue({ companyId, endpointId, publicId: endpoint.publicId, connectionId: endpoint.connectionId, + runtimeGeneration: saved.runtimeGeneration, teamId: enrolled.teamId, appId: enrolled.appId, bindingId: enrolled.bindingId }, secret, "EvDefaultOwner"); + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + expect((await owner.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(callbacks).toHaveLength(1); + expect(callbacks[0]!.message.author.userId).toBe("UOPERATOR"); + expect(cloud.acknowledged).toEqual([delivered.id]); + expect((await setup(endpointId)).slackCallbackSurfaces).toEqual(callbackEvidence); + expect(await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId))).toHaveLength(1); + for (const transport of transports) await transport.assertNoErrors(); + await service.configure(endpointId, { action: "pause" }, actorId); + const oldRequest = new Request("https://tenant.example/old-receipt", { method: "POST", body: new Uint8Array(delivered.body), + headers: { "content-type": "application/json", "x-slack-request-timestamp": delivered.packet.timestamp, "x-slack-signature": delivered.packet.signature } }); + const rejected = await service.handleWebhook(endpoint.publicId, "slack", oldRequest, { slackCloudGeneration: Number(saved.runtimeGeneration) }); + expect(await rejected.text()).toBe("superseded"); + expect(callbacks).toHaveLength(1); + await owner.pollOnce({ forDelivery: true }); + expect((await setup(endpointId)).slackCloudIntent).toBeUndefined(); + expect((await setup(endpointId)).slackCloudRetiredGeneration).toBeGreaterThan(Number(saved.runtimeGeneration)); + }, 60_000); + + it("two due registrations settle B while A still owns a busy credential lease", async () => { + const a = await createEndpoint("BUSY"); await confirm(a); await activate(a); await confirm(a); + const b = await createEndpoint("READY"); await confirm(b); await activate(b); await confirm(b); + await service.configure(a, { action: "pause" }, actorId); + const before = structuredClone((await setup(a)).slackCloudIntent!); + await service.configure(b, { action: "pause" }, actorId); + await holdCredentials(a); + const start = cloud.calls.length; + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await owner.pollOnce({ forDelivery: true }); + await owner.pollOnce({ forDelivery: true }); + expect((await setup(b)).slackCloudIntent).toBeUndefined(); + expect((await setup(b)).slackCloudRetiredGeneration).toBeDefined(); + const retained = (await setup(a)).slackCloudIntent!; + expect(retained).toMatchObject({ ...before, attempts: 1, failureCode: "credential_lease_busy", nextAttemptAt: expect.any(String) }); + expect(Date.parse(retained.nextAttemptAt!)).toBeGreaterThan(Date.now()); + expect(cloud.calls.slice(start).filter(call => call.enrollment?.endpointId === a)).toEqual([]); + expect(owner.nextPollAt()).not.toBeNull(); + releases.splice(0).forEach(release => release()); await Promise.all(heldLeases); + const delay = Math.max(0, Date.parse(retained.nextAttemptAt!) - Date.now()); + await new Promise(resolve => setTimeout(resolve, delay)); + await owner.pollOnce(); + expect((await setup(a)).slackCloudIntent).toBeUndefined(); + expect((await setup(a)).slackCloudRetiredGeneration).toBeDefined(); + }, 60_000); + + it("normal configuration rollback leaves no registration debt or retirement request", async () => { + const endpointId = await createEndpoint("ROLLBACK"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const before = await row(endpointId), requests = cloud.calls.length; + await service.shutdown(); runtime = makeRuntime(); + let rejectAfterIntent = false; + const stop = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { if (event === "intent") rejectAfterIntent = true; }); + service = chatChannelService(db, { runtime, heartbeat, fetch: providerFetch, publicBaseUrl: "https://tenant.example", + deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); }, renewCredentialMutationLease: async () => !rejectAfterIntent }); + try { + await expect(service.configure(endpointId, { action: "pause" }, actorId)).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); + expect((await row(endpointId)).status).toBe(before.status); + expect(await setup(endpointId)).toEqual(before.setup); + expect(cloud.calls).toHaveLength(requests); + } finally { stop(); } + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await owner.pollOnce({ forDelivery: true }); + expect(cloud.calls.filter(call => call.enrollment?.active === false)).toEqual([]); + }, 60_000); + + it.each(["intent", "generation"] as const)("real SQL refuses an enrollment response after newer %s custody wins", async change => { + const endpointId = await createEndpoint(`CAS${change}`); + const original = (await setup(endpointId)).slackCloudIntent!; + const successor = { ...original, id: randomUUID(), runtimeGeneration: original.runtimeGeneration + (change === "generation" ? 1 : 0) }; + cloud.beforeReceipt(async () => { + await db.update(chatEndpoints).set({ setup: sql`jsonb_set(jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(successor)}::jsonb), '{runtimeGeneration}', ${JSON.stringify(successor.runtimeGeneration)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, companyId))); + }); + const producer = createSlackCloudEnrollment(db, { resolveCredentials: async () => ({ signingSecret: secret }), + runtimeGeneration: value => Number(value.runtimeGeneration), connector: cloud.connector }); + await expect(chatCredentialMutationLease(db)(await row(endpointId), lease => producer(endpointId, undefined, lease))).rejects.toThrow("changed"); + expect((await setup(endpointId)).slackCloudIntent).toEqual(successor); + expect((await setup(endpointId)).slackCloudEvents).toBeUndefined(); + }, 60_000); + + it("malformed sealed Slack receipt is acknowledged once instead of retained as an owner outage", async () => { + const endpointId = await createEndpoint("INVALID"); await confirm(endpointId); + const endpoint = await row(endpointId), saved = await setup(endpointId), enrolled = saved.slackCloudEvents!; + const malformed = cloud.enqueue({ companyId, endpointId: "bad", publicId: endpoint.publicId, connectionId: endpoint.connectionId, + runtimeGeneration: saved.runtimeGeneration, teamId: enrolled.teamId, appId: enrolled.appId, bindingId: enrolled.bindingId }, secret, "EvMalformed"); + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + const result = await owner.pollOnce({ forDelivery: true }); + expect(result.ignored).toBe(1); expect(result.failed).toBe(0); + expect(cloud.acknowledged).toEqual([malformed.id]); + expect(callbacks).toEqual([]); + }, 60_000); + + it.each(["intent", "generation", "malformed-generation"] as const)( + "busy retry CAS preserves a concurrent %s winner and rethrows the exact owner error", async change => { + const endpointId = await createEndpoint(`BUSYCAS${change}`); + const before = await setup(endpointId), original = before.slackCloudIntent!; + const successor = change === "intent" ? { ...original, id: randomUUID() } : original; + const generation: unknown = change === "malformed-generation" ? { invalid: true } + : Number(before.runtimeGeneration) + (change === "generation" ? 1 : 0); + const busy = conflict("Synthetic owner credential lane is busy", { code: "chat_endpoint_credentials_busy" }); + const requests = cloud.calls.length; + const pendingBefore = databaseWorkPending(db, DELIVERY_QUEUES.connectionEvents), inheritedSignals: string[] = []; + const stopInherited = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { inheritedSignals.push(event); }); + try { + console.info(JSON.stringify({ slackRetrySignalProbe: "subscription", endpointId, pendingBefore, immediateSignals: [...inheritedSignals] })); + expect(inheritedSignals).toEqual(pendingBefore ? ["intent"] : []); + await reconcileDatabaseWork(db, DELIVERY_QUEUES.connectionEvents); + const pendingAfter = databaseWorkPending(db, DELIVERY_QUEUES.connectionEvents); + console.info(JSON.stringify({ slackRetrySignalProbe: "reconciled", endpointId, pendingBefore, pendingAfter, inheritedSignals: [...inheritedSignals] })); + expect(pendingAfter).toBe(false); + } finally { stopInherited(); } + const signals: string[] = []; + const stopSignals = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { signals.push(event); }); + const unregister = registerSlackCloudEnrollmentOwner(db, async id => { + expect(id).toBe(endpointId); + await db.update(chatEndpoints).set({ setup: sql`jsonb_set(jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(successor)}::jsonb), '{runtimeGeneration}', ${JSON.stringify(generation)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, companyId))); + throw busy; + }); + try { + await expect(reconcilePendingSlackCloudEnrollments(db, new Date())).rejects.toBe(busy); + const after = await setup(endpointId); + expect(after.slackCloudIntent).toEqual(successor); + expect(after.runtimeGeneration).toEqual(generation); + expect(after.slackCloudEvents).toBeUndefined(); + expect(cloud.calls).toHaveLength(requests); + expect(signals).toEqual([]); + } finally { unregister(); stopSignals(); } + }, 60_000); + + it.each(["unknown-error", "unrelated-conflict"] as const)( + "the %s owner failure propagates without a busy retry rewrite", async kind => { + const endpointId = await createEndpoint(`UNKNOWN${kind}`), before = await setup(endpointId); + const original = kind === "unknown-error" ? new Error("Synthetic unknown owner failure") + : conflict("Synthetic different authority failure", { code: "chat_endpoint_runtime_superseded" }); + const requests = cloud.calls.length; + const unregister = registerSlackCloudEnrollmentOwner(db, async id => { expect(id).toBe(endpointId); throw original; }); + try { + await expect(reconcilePendingSlackCloudEnrollments(db, new Date())).rejects.toBe(original); + expect(await setup(endpointId)).toEqual(before); + expect(cloud.calls).toHaveLength(requests); + } finally { unregister(); } + }, 60_000); + + it("an injected retry transaction failure preserves the original busy error and persistence cause", async () => { + const endpointId = await createEndpoint("RETRYWRITE"), before = await setup(endpointId); + const busy = conflict("Synthetic owner credential lane is busy", { code: "chat_endpoint_credentials_busy" }); + const writeError = new Error("Synthetic retry transaction unavailable"), requests = cloud.calls.length; + const unregister = registerSlackCloudEnrollmentOwner(db, async id => { expect(id).toBe(endpointId); throw busy; }); + const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(writeError); + try { + const error = await reconcilePendingSlackCloudEnrollments(db, new Date()).then(() => undefined, error => error); + expect(error).toBeInstanceOf(AggregateError); + if (!(error instanceof AggregateError)) throw new Error("Expected retry failure to retain both causes", { cause: error }); + expect(error.errors).toHaveLength(2); + expect(error.errors[0]).toBe(busy); + expect(error.errors[1]).toBe(writeError); + expect(transaction).toHaveBeenCalledOnce(); + expect(await setup(endpointId)).toEqual(before); + expect(cloud.calls).toHaveLength(requests); + } finally { transaction.mockRestore(); unregister(); } + }, 60_000); + + it("public tool removal retains Slack retirement custody through broker outage and ordinary owner recovery", async () => { + const endpointId = await createEndpoint("TOOLREMOVE"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const originalEndpoint = await row(endpointId), originalSetup = await setup(endpointId); + const [connection] = await db.select().from(toolConnections).where(eq(toolConnections.id, originalEndpoint.connectionId)); + const signingRef = connection!.credentialSecretRefs.find(ref => ref.configPath === "credentials.signingSecret"); + if (!signingRef) throw new Error("Normal Slack configure did not persist its signing credential binding"); + const credentialOwner = { consumerType: "tool_connection" as const, consumerId: connection!.id, + configPath: "credentials.signingSecret", actorType: "system" as const }; + await expect(secretService(db).resolveSecretValue(companyId, signingRef.secretId, "latest", credentialOwner)).resolves.toBe(secret); + cloud.setUnavailable(true); + const requests = vi.mocked(cloud.request).mock.calls.length; + const tools = toolAccessService(db); + const removed = await tools.archiveConnection(connection!.id, companyId, { actorType: "user", actorId }); + expect(removed.connection).toMatchObject({ id: connection!.id, status: "archived", enabled: false }); + const archived = await row(endpointId); + expect(archived.status).toBe("archived"); + const [revoked] = await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)); + expect(revoked!.credentialSecretRefs).toEqual([]); + expect(revoked!.credentialRefs).toEqual([]); + expect(await db.select().from(companySecretBindings).where(and(eq(companySecretBindings.companyId, companyId), + eq(companySecretBindings.targetType, "tool_connection"), eq(companySecretBindings.targetId, connection!.id)))).toEqual([]); + await expect(secretService(db).resolveSecretValue(companyId, signingRef.secretId, "latest", credentialOwner)).rejects.toThrow(); + expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests); + const intent = (await setup(endpointId)).slackCloudIntent; + expect(intent).toMatchObject({ version: 1, active: false, companyId, endpointId, connectionId: connection!.id, + publicId: originalEndpoint.publicId, runtimeGeneration: Number(originalSetup.runtimeGeneration), + subject: originalSetup.slackCloudSubject, instanceId: originalSetup.slackCloudInstanceId, attempts: 0 }); + expect(JSON.stringify(intent)).not.toContain(secret); + await tools.archiveConnection(connection!.id, companyId, { actorType: "user", actorId }); + expect((await setup(endpointId)).slackCloudIntent).toEqual(intent); + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await expect(owner.pollOnce({ forDelivery: true })).rejects.toMatchObject({ status: 503 }); + const retry = (await setup(endpointId)).slackCloudIntent!; + expect(retry).toMatchObject({ ...intent, attempts: 1, failureCode: "broker_unavailable", nextAttemptAt: expect.any(String) }); + const retryAt = Date.parse(retry.nextAttemptAt!); + expect(retryAt).toBeGreaterThan(Date.now()); + cloud.setUnavailable(false); cloud.install(); + await new Promise(resolve => setTimeout(resolve, Math.max(0, retryAt - Date.now()))); + await owner.pollOnce(); + expect((await setup(endpointId)).slackCloudIntent).toBeUndefined(); + expect((await setup(endpointId)).slackCloudRetiredGeneration).toBe(Number(originalSetup.runtimeGeneration)); + expect(cloud.calls.filter(call => call.operation === "slack-events" && call.enrollment?.active === false + && call.enrollment.endpointId === endpointId)).toHaveLength(1); + expect(callbacks).toEqual([]); + }, 60_000); + + it("public tool removal rolls back both archive writes without dropping current Slack custody or credentials", async () => { + const endpointId = await createEndpoint("TOOLROLLBACK"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const before = await row(endpointId); + const [connection] = await db.select().from(toolConnections).where(eq(toolConnections.id, before.connectionId)); + const failure = new Error("Synthetic removal transaction rollback"), requests = vi.mocked(cloud.request).mock.calls.length; + const realTransaction = db.transaction.bind(db); + const transaction = vi.spyOn(db, "transaction").mockImplementationOnce(async work => realTransaction(async tx => { + await work(tx); + const [archivedEndpoint] = await tx.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)); + const [archivedConnection] = await tx.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)); + expect(archivedEndpoint!.status).toBe("archived"); + expect(archivedConnection).toMatchObject({ status: "archived", enabled: false }); + throw failure; + })); + try { + await expect(toolAccessService(db).archiveConnection(connection!.id, companyId, { actorType: "user", actorId })).rejects.toBe(failure); + expect(await row(endpointId)).toMatchObject({ status: before.status, setup: before.setup }); + const [after] = await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)); + expect(after).toMatchObject({ status: connection!.status, enabled: connection!.enabled, + credentialSecretRefs: connection!.credentialSecretRefs, credentialRefs: connection!.credentialRefs }); + expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests); + } finally { transaction.mockRestore(); } + }, 60_000); + + it("public tool removal refuses a foreign company before Slack archival or retirement staging", async () => { + const endpointId = await createEndpoint("TOOLFOREIGN"); await confirm(endpointId); + const before = await row(endpointId), requests = vi.mocked(cloud.request).mock.calls.length; + await expect(toolAccessService(db).archiveConnection(before.connectionId, randomUUID(), { actorType: "user", actorId })).rejects.toMatchObject({ status: 404 }); + expect(await row(endpointId)).toEqual(before); + expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests); + }, 60_000); +}); diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 7a4943f1882..870eba73ea1 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -381,15 +381,27 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu // window. A company hint must not make that real deadline wait one second. await coordinator!.stop(); coordinator = undefined; const dueAt = Date.now() + 500; + const actualWakeup = wakeup.getMockImplementation()!; + let otherWakeEnteredAt: number | null = null; + wakeup.mockImplementation((id, options) => { + // Capture dispatch ENTRY before the original mock performs any DB I/O. + if (id === other.agentId) otherWakeEnteredAt ??= Date.now(); + return actualWakeup(id, options); + }); await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id)); await start(service); await flush(); await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" }); await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id, "More changes while another company is due", { userId: "board-user" }); await flush(); + const armedBeforeAdvance = coordinator!.nextWakeAt(); + // A null deadline can mean the due work is already running. Any still + // armed deadline must preserve the known earlier obligation. + if (armedBeforeAdvance !== null) expect(armedBeforeAdvance).toBeLessThanOrEqual(dueAt); await vi.advanceTimersByTimeAsync(500); await flush(); await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); - expect(Date.now()).toBeLessThan(dueAt + 500); + expect(otherWakeEnteredAt).not.toBeNull(); + expect(otherWakeEnteredAt!).toBeLessThan(dueAt + 500); expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent); const generated = (await statusCardService(db).getById(other.id))!; expect(generated.generatingIssueId).toBeTruthy(); diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts index a4081a4b4d7..16086ebaeef 100644 --- a/server/src/services/ai-connections.ts +++ b/server/src/services/ai-connections.ts @@ -174,7 +174,7 @@ export function aiConnectionService(db: Db) { : ("connected" as const), }, ]; - }); + }).sort((a, b) => a.id.localeCompare(b.id) || a.grantId.localeCompare(b.grantId)); } async function setDefault( companyId: string, diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index e6cdc79894a..00a3a307e8b 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -1,4 +1,6 @@ import { githubReviewCheckService } from "./chat-github-checks.js"; +import { createSlackCloudEventIngress } from "./chat-channels/slack/cloud-ingress.js"; +import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent, registerSlackCloudEnrollmentOwner } from "./chat-channels/slack/cloud-enrollment.js"; import { leasedChatProvider } from "./chat-channels/runtime/receiver-policy.js"; import { chatReceiverSchedule } from "./chat-channels/runtime/receiver-schedule.js"; import { registerChatReceiverWork } from "./chat-channels/runtime/receiver-work.js"; @@ -842,6 +844,10 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const withCredentialMutationLease = chatCredentialMutationLease(db, options); + const unregisterSlackCloudIngress = createSlackCloudEventIngress(db, { + withCredentialMutationLease, resolveCredentials, runtimeGeneration: endpoint => runtimeGeneration(endpoint.setup), + handleWebhook: (publicId, provider, request, generation) => handleWebhook(publicId, provider, request, { slackCloudGeneration: generation }), + }); const { list, get, create, update } = createChatEndpoints(db, { endpointRecord, CAPABILITIES, PROVIDER_LABELS, withCredentialMutationLease, getPublicBaseUrl, getWebhookPublicBaseUrl, @@ -2936,10 +2942,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { githubIngressActionId: string; githubIngressFence: LifecycleRuntimeFence; superseded?: boolean; - }, + } | { slackCloudGeneration: number }, ) { const serviceReceivedAtMs = Date.now(); - const githubScope = createGitHubWebhookScope(provider, internalContext, options.githubWebhookResponseBudgetMs, GITHUB_WEBHOOK_RESPONSE_BUDGET_MS); + const slackCloudGeneration = internalContext && "slackCloudGeneration" in internalContext ? internalContext.slackCloudGeneration : undefined; + const githubScope = createGitHubWebhookScope(provider, internalContext && "githubIngressActionId" in internalContext ? internalContext : undefined, options.githubWebhookResponseBudgetMs, GITHUB_WEBHOOK_RESPONSE_BUDGET_MS); const { replayingDurableGitHubIngress, ignoreSupersededIngress, matchesGitHubIngressFence, githubResponseDeadlineAt } = githubScope; const endpoint = await db .select() @@ -2962,6 +2969,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { throw notFound("Chat endpoint not found"); } recordChatWebhookStage("endpoint_resolved", endpoint.id); + if (provider === "slack" && slackCloudGeneration !== undefined && runtimeGeneration(endpoint.setup) !== slackCloudGeneration) + return new Response("superseded", { status: 200 }); // A pause is a durable ingress fence. Providers generally retry non-2xx // webhooks, so acknowledge late callbacks without recreating a runtime or // admitting any Paperclip mutation. @@ -2998,6 +3007,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_endpoint_runtime_superseded", }); } + if (provider === "slack" && slackCloudGeneration !== undefined && runtimeContext.generation !== slackCloudGeneration) + return new Response("superseded", { status: 200 }); // runtimeFor can return a newer instance than the leased preflight saw. // Keep the exact staged epoch; callbacks on a matching instance retain // their normal transactional fences if a pause wins after this check. @@ -3403,7 +3414,14 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null; } + const slackCloudEnrollment = createSlackCloudEnrollment(db, { resolveCredentials, runtimeGeneration }); + const unregisterSlackCloudEnrollment = registerSlackCloudEnrollmentOwner(db, async endpointId => { + const record = await endpointRecord(endpointId); + if (!record) throw new Error("Slack enrollment endpoint is absent"); + await withCredentialMutationLease(record.endpoint, lease => slackCloudEnrollment(endpointId, undefined, lease)); + }); const { configure, configureWithCredentialLease, runtimeConfig } = createChatConfiguration(db, { + stageConfigurationEffects: stageSlackCloudEnrollmentIntent, endpointRecord, get, withCredentialMutationLease, getWebhookPublicBaseUrl, invalidateRuntime, nativeBotIdentityMatches, assertNativeBotIdentityAvailable, isNativeBotIdentityUniqueViolation, nativeBotIdentityConflict, legacyGitHubLabelsMatchVerifiedCredentials, normalizedCredentials, verifyCredentials, @@ -3611,6 +3629,8 @@ return { unregisterSlackTaskAuthority(); voice.unregisterAgentTools(); unregisterGitHubCloudIngress(); + unregisterSlackCloudIngress(); + unregisterSlackCloudEnrollment(); unregisterCommittedResponseAuthority(); await Promise.allSettled([...publicationEndpointTasks.values()]); await Promise.allSettled([...backgroundMessageTasks]); diff --git a/server/src/services/chat-channels/configuration.ts b/server/src/services/chat-channels/configuration.ts index 8ded7cbb96e..8eb17d429bb 100644 --- a/server/src/services/chat-channels/configuration.ts +++ b/server/src/services/chat-channels/configuration.ts @@ -100,6 +100,7 @@ type ChatConfigurationOptions = Pick, " githubWizard: Pick["githubWizard"], "repairCloud">; nativeBotIdentityClaimBarrier?: () => Promise; allowLocalVoiceBoard?: boolean; + stageConfigurationEffects?: (transaction: Db, endpointId: string, actorUserId: string | null | undefined) => Promise; }; export function createChatConfiguration(db: Db, options: ChatConfigurationOptions) { const { endpointRecord, get, withCredentialMutationLease, getWebhookPublicBaseUrl, invalidateRuntime, nativeBotIdentityMatches, assertNativeBotIdentityAvailable, isNativeBotIdentityUniqueViolation, nativeBotIdentityConflict, legacyGitHubLabelsMatchVerifiedCredentials, normalizedCredentials, verifyCredentials, persistCredentials, clearCredentials, resolveCredentials, prepareProviderInventory, reconcileProviderResourceRows, reconcileDiscordCommands, runtimeFor, runtimeContextForRecord, stageTelegramMaintenance, processTelegramMaintenance, recordTelegramStopSubscription, fetchImpl, redactError, runtimeGeneration, leasedChatProvider, PROVIDER_LABELS, SUPPLIED_CREDENTIAL_KEYS, PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS, isSupportedTelegramWebhookBaseUrl } = options; @@ -176,6 +177,14 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption const record = await endpointRecord(endpointId); if (!record) throw notFound("Chat endpoint not found"); const endpoint = record.endpoint; + const mutate = (work: (tx: Parameters[0]>[0]) => Promise) => db.transaction(async tx => { + const result = await work(tx); + if (endpoint.provider === "slack" && options.stageConfigurationEffects) { + await options.stageConfigurationEffects(tx as unknown as Db, endpointId, actorUserId); + await credentialLease.assertOwned(tx); + } + return result; + }); if (input.action === "pause") { if (endpoint.status !== "active") { throw conflict("Only an active chat connection can be paused", { @@ -183,7 +192,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption }); } await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const pausedAt = new Date(); const current = await tx @@ -299,7 +308,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption let activatedEndpoint!: EndpointRow; try { await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const resumedAt = new Date(); const current = await tx @@ -384,7 +393,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption const failure = redactError(error); const failedAt = new Date(); await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); @@ -478,7 +487,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption }); } await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const current = await tx .select({ setup: chatEndpoints.setup, status: chatEndpoints.status }) @@ -724,7 +733,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption if (!endpoint.botExternalId) { try { await managementContext?.reauthorize(); - const claimed = await db.transaction(async (tx) => { + const claimed = await mutate(async (tx) => { await credentialLease.assertOwned(tx); await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); @@ -793,7 +802,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption (input.action === "configure" || input.action === "reconnect"); try { await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const current = await tx .select({ setup: chatEndpoints.setup }) @@ -892,7 +901,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption const failure = redactError(error); await invalidateRuntime(endpoint.id).catch(() => undefined); await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); @@ -922,7 +931,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption } await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); await logActivity(tx as unknown as Db, { companyId: endpoint.companyId, diff --git a/server/src/services/chat-channels/runtime-types.ts b/server/src/services/chat-channels/runtime-types.ts index c0c9dce4631..4b9e9e5f5ab 100644 --- a/server/src/services/chat-channels/runtime-types.ts +++ b/server/src/services/chat-channels/runtime-types.ts @@ -48,6 +48,34 @@ export type InternalSetupState = ChatEndpointSetupState & { runtimeGeneration?: number; /** Binds pre-install URL verification to the signing secret later configured. */ slackVerificationSigningFingerprint?: string; + + slackCloudSubject?: string; + slackCloudInstanceId?: string; + slackCloudRetiredGeneration?: number; + slackCloudIntent?: { + version: 1; + id: string; + subject: string; + instanceId: string; + companyId: string; + endpointId: string; + publicId: string; + connectionId: string; + runtimeGeneration: number; + active: boolean; + teamId?: string; + appId?: string; + attempts: number; + nextAttemptAt: string | null; + failureCode?: string; + }; + slackCloudEvents?: { + version: 1; + bindingId: string; + runtimeGeneration: number; + teamId: string; + appId: string; + }; slackCallbackSurfaces?: Partial< Record >; diff --git a/server/src/services/chat-channels/slack/cloud-drain.test.ts b/server/src/services/chat-channels/slack/cloud-drain.test.ts new file mode 100644 index 00000000000..375bfd8879a --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-drain.test.ts @@ -0,0 +1,75 @@ +import { generateKeyPairSync } from "node:crypto"; +import { registerSlackCloudEnrollmentOwner } from "./cloud-enrollment.js"; +import type { Db } from "@paperclipai/db"; +import { afterEach, expect, it, vi } from "vitest"; +import { githubConnectionEventService } from "../../github-connection-events.js"; +import type { PaperclipCloudConnector } from "../../paperclip-cloud-connector.js"; +import { registerSlackCloudEventIngress } from "./cloud-ingress.js"; +import { SlackCloudReceiptError, type SlackCloudEvent } from "./cloud-event.js"; + +afterEach(() => vi.unstubAllEnvs()); +function fixture(handler: (event: SlackCloudEvent) => Promise, pendingEnrollment = false) { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", ""); + let reads = 0; + const db = { select: () => { + const rows = [[], [], [{ companyId: "company" }], [{ id: "endpoint", setup: { slackCloudIntent: { nextAttemptAt: "2000-01-01T00:00:00Z" } } }]][reads++ % (pendingEnrollment ? 4 : 3)]!; + const query = { from: () => query, innerJoin: () => query, where: () => query, orderBy: () => query, limit: async () => rows, + then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(rows).then(resolve) }; + return query; + } } as unknown as Db; + const event: SlackCloudEvent = { id: "slack-receipt", provider: "slack", event: "event_callback", installationId: "binding", bindingIds: ["binding"], + createdAt: new Date().toISOString(), payload: {} }; + const connector = { leaseEvents: vi.fn(async () => ({ leaseId: "lease", events: [event] })), + acknowledgeEvents: vi.fn(async () => 1), eventWakeVersion: () => 1 as const }; + const unregister = registerSlackCloudEventIngress(db, handler); + let time = Date.now(); + const service = githubConnectionEventService(db, { connector: connector as unknown as PaperclipCloudConnector, now: () => new Date(time) }); + return { db, service, connector, unregister, event, time: () => time, advance: (ms: number) => { time += ms; } }; +} + +it("actual connection-event owner routes Slack and retries a lost ACK through the registered owner", async () => { + const handler = vi.fn(async () => {}), f = fixture(handler); + try { + f.connector.acknowledgeEvents.mockRejectedValueOnce(new Error("ack response lost")); + await expect(f.service.pollOnce({ forDelivery: true })).rejects.toThrow("ack response lost"); + expect((await f.service.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(handler).toHaveBeenCalledTimes(2); + expect(f.connector.leaseEvents).toHaveBeenCalledWith({ subject: "github-bot-events", companyId: "company", eventWakeVersion: 1 }); + expect(f.connector.acknowledgeEvents).toHaveBeenLastCalledWith(expect.objectContaining({ deliveryIds: [f.event.id] })); + } finally { f.unregister(); } +}); + +it.each(["retry", "attention", "invalid"] as const)("actual owner distinguishes %s receipt failure before ACK", async disposition => { + const f = fixture(async () => { throw new SlackCloudReceiptError("synthetic_code", disposition); }); + try { + const result = await f.service.pollOnce({ forDelivery: true }); + expect(result.failed).toBe(disposition === "invalid" ? 0 : 1); + expect(f.connector.acknowledgeEvents).toHaveBeenCalledTimes(disposition === "invalid" ? 1 : 0); + } finally { f.unregister(); } +}); + +it.each(["chat_endpoint_credentials_busy", "temporary enrollment owner failure"])("drains and ACKs before retaining %s on a bounded existing-worker retry", async message => { + const handler = vi.fn(async () => {}), f = fixture(handler, true); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "https://cloud.example.test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", generateKeyPairSync("ed25519").privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", generateKeyPairSync("x25519").privateKey.export({ format: "jwk" }).d!); + const owner = vi.fn(async () => { + expect(handler).toHaveBeenCalled(); + expect(f.connector.acknowledgeEvents).toHaveBeenCalled(); + throw new Error(message); + }); + const unregister = registerSlackCloudEnrollmentOwner(f.db, owner); + try { + expect((await f.service.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(owner).toHaveBeenCalledTimes(1); + expect(f.service.nextPollAt()).toBe(f.time() + 5_000); + await f.service.pollOnce(); + expect(owner).toHaveBeenCalledTimes(1); + expect(f.connector.leaseEvents).toHaveBeenCalledTimes(1); + f.advance(5_000); + expect((await f.service.pollOnce()).processed).toBe(1); + expect(owner).toHaveBeenCalledTimes(2); + } finally { unregister(); f.unregister(); } +}); diff --git a/server/src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts b/server/src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts new file mode 100644 index 00000000000..9121189db8f --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts @@ -0,0 +1,451 @@ +import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "../../../__tests__/helpers/slack-cloud/transport.js"; +import { createHmac, generateKeyPairSync, randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { and, eq, sql } from "drizzle-orm"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import type { ToolCredentialSecretRef } from "@paperclipai/shared"; +import { expect, it, vi } from "vitest"; +import { activityLog, agentRuntimeState, agentTaskSessions, agentWakeupRequests, agents, applyPendingMigrations, authUsers, chatConversations, chatDeliveries, chatExternalPrincipals, chatIdentityLinks, chatEndpointResources, chatEndpoints, chatMessageLinks, chatSlackRegistrations, heartbeatRunEvents, heartbeatRuns, companies, companyMemberships, companySecrets, companySkills, costEvents, financeEvents, closeCheckpointDb, closeFileStorageDb, closeLivenessDb, createDb, issueComments, issues, toolApplications, toolConnections } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "../../../lib/dedicated-test-database-url.js"; +import { subscribeDatabaseWork } from "@paperclipai/db/work-signals"; +import { DELIVERY_QUEUES } from "../../delivery-work-notifications.js"; +import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from "../../paperclip-cloud-connector.js"; +import { chatChannelService } from "../../chat-channels.js"; +import { ChatRuntime } from "../../chat-runtime.js"; +import { heartbeatService } from "../../heartbeat.js"; +import { githubConnectionEventService } from "../../github-connection-events.js"; +import { secretService } from "../../secrets.js"; +import type { ChatSdkMessageCallbackEvent } from "../../chat-sdk-runtime.js"; +import { createChatSdkEndpointRuntime } from "../../chat-sdk-runtime.js"; +import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent } from "./cloud-enrollment.js"; +import { createSlackCloudEventIngress, dispatchSlackCloudEvent } from "./cloud-ingress.js"; +import type { InternalSetupState } from "../runtime-types.js"; + +const appUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_TEST_DATABASE_URL"); +const cloudUrl = resolveExternalTestDatabaseUrl("SLACK_EVENT_TEST_DATABASE_URL"); +const cloudDist = process.env.SLACK_TEST_CLOUD_DIST; + +it.skipIf(!appUrl || !cloudUrl || !cloudDist)("signed Slack enrollment and actual event owner preserve one domain reply through lost ACK and removal retries", async () => { + const network = guardSlackFixtureNetwork({ requirePreload: true }); + const load = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, file)).href); + const [{ PostgresCloudHarnessRegistry }, { PostgresConnectorStore }, { ConnectorBroker }, { InMemoryProviderSecretStore }, + { providerRegistryFromEnvironment }, { CloudStackProvisioner }, { FakeCloudStackProvider }, { Pool }] = await Promise.all([ + load("registry/postgres.js"), load("connections/postgres.js"), load("connections/broker.js"), load("providers/neon.js"), + load("connections/providers.js"), load("provisioner/provisioner.js"), load("providers/fake.js"), + import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, "../node_modules/pg/lib/index.js")).href), + ]); + await applyPendingMigrations(appUrl!); + const db = createDb(appUrl!, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + const pool = new Pool({ connectionString: cloudUrl!, max: 2 }); + + const registry = Reflect.construct(PostgresCloudHarnessRegistry, [pool]) as { + applyMigrations(folder: string, timeout: number): Promise; + hydrate(): Promise; + close(): Promise; + }; + try { + await registry.applyMigrations(path.resolve(cloudDist!, "../migrations"), 30_000); + await registry.hydrate(); + } + catch (error) { await db.$client.end({ timeout: 1 }); await registry.close(); throw error; } + const store = new PostgresConnectorStore(pool), secrets = new InMemoryProviderSecretStore(); + const id = randomUUID(), companyId = randomUUID(), endpointId = randomUUID(), agentId = randomUUID(), connectionId = randomUUID(), applicationId = randomUUID(); + const actorId = `slack-owner-${id}`; + const instanceId = `instance-${id}`, stackId = `slack-${id}`, publicId = "a".repeat(43), signingSecret = "0123456789abcdef0123456789abcdef"; + const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"), now = new Date(); + let broker: InstanceType; + let enrollmentRequests = 0; + const server = createServer((req, res) => { + if (req.url === "/v1/connector/slack-events") enrollmentRequests++; + if (brokerUnavailable) { res.writeHead(503, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "fixture_broker_unavailable" })); return; } + if (loseEnrollmentResponse && req.url === "/v1/connector/slack-events") { + loseEnrollmentResponse = false; + res.end = (() => { res.destroy(); return res; }) as typeof res.end; + } + void broker.handle(req, res, new URL(req.url!, base)).catch((error: unknown) => { + if (!res.headersSent) res.writeHead(500); res.end(String(error)); + }); }); + let base = ""; + let brokerUnavailable = false, loseEnrollmentResponse = false; + let unregister: (() => void) | undefined; + let runtime: ReturnType | undefined; + const deliveryIds: string[] = []; + let normalService: ReturnType | undefined; + let secretDirectory: string | undefined; + let normalHeartbeat: ReturnType | undefined; + let releaseEnrollmentLease: (() => void) | undefined; + let heldEnrollmentLease: Promise | undefined; + try { + await new CloudStackProvisioner({ provider: new FakeCloudStackProvider(), registry }).provisionDevelopmentStack({ + idempotencyKey: id, accountGroup: { id: `account-${id}`, slug: stackId, displayName: "Slack fixture" }, + owner: { id: actorId, email: `${id}@example.test` }, + stack: { stackId, accountGroupId: `account-${id}`, stackSlug: stackId, displayName: "Slack fixture", kind: "demo", primaryHost: `${stackId}.example.test`, region: "us-east-1" }, + versionPlan: { appVersion: "fixture", schemaVersion: "fixture", rolloutTrack: "stable" }, + sleepPolicy: { desiredState: "awake" }, primaryCompany: { paperclipCompanyId: companyId, displayName: "Fixture", defaultBudgetCents: 0 }, + }); + await store.putCloudStackInstance({ id: instanceId, kind: "cloud_stack", companyStackId: stackId, environment: "test", + signPublicKey: signing.publicKey.export({ format: "jwk" }).x!, sealPublicKey: sealing.publicKey.export({ format: "jwk" }).x!, + origins: ["https://tenant.example"], activeOrigin: "https://tenant.example", label: "Fixture", status: "active", createdAt: now, updatedAt: now }); + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing fixture address"); + base = `http://127.0.0.1:${address.port}`; + const wake = vi.fn(async () => {}); + broker = new ConnectorBroker({ publicOrigin: base, environment: "test", store, + providers: providerRegistryFromEnvironment({}, "test"), secrets: { resolve: async () => undefined }, + authenticate: async () => ({ globalUserId: actorId }), canApproveInstance: async (principal: { globalUserId: string }) => principal.globalUserId === actorId, + slackEvents: { secrets, secretRefPrefix: "secret:///test/slack" }, + eventWakeEnrollmentEnabled: true, eventWakeDeliveryEnabled: true, wakeCloudStack: wake }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", signing.privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", sealing.privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); + await db.insert(companies).values({ id: companyId, name: "Slack", issuePrefix: `S${id.slice(0, 7)}` }); + await db.insert(authUsers).values({ id: actorId, name: "Slack owner", email: `${id}@owner.example.test`, emailVerified: true, createdAt: now, updatedAt: now }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, status: "active", membershipRole: "operator" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Slack", adapterType: "process", status: "idle", runtimeConfig: { heartbeat: { wakeOnDemand: true, intervalSec: 0 } } }); + await db.insert(companyMemberships).values({ companyId, principalType: "agent", principalId: agentId, status: "active", membershipRole: "member" }); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Slack", type: "chat" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: id, + transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, + publicId, provider: "slack", status: "active", providerAccountId: "TTEST", setup: { runtimeGeneration: 1 } as InternalSetupState }); + await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", + manifest: {}, manifestHash: "fixture", callbackUri: "https://tenant.example/callback", appId: "ATEST", createdByUserId: actorId }); + const withLease = chatCredentialMutationLease(db); + const resolveCredentials = vi.fn(async () => ({ signingSecret })); + const produce = createSlackCloudEnrollment(db, { resolveCredentials, + runtimeGeneration: setup => Number(setup.runtimeGeneration ?? 0) }); + const endpoint = async () => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)))[0]!; + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId)); + await withLease(await endpoint(), lease => produce(endpointId, actorId, lease)); + const saved = (await endpoint()).setup as InternalSetupState; + expect(saved.slackCloudEvents?.runtimeGeneration).toBe(1); + expect(JSON.stringify(saved)).not.toContain(signingSecret); + const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvPaired", team_id: "TTEST", api_app_id: "ATEST", + event: { type: "app_mention", text: "<@UBOT> hello", user: "UOPERATOR", channel: "CTEST", ts: "1788.500", event_ts: "1788.500" } })); + const timestamp = String(Math.floor(Date.now() / 1000)); + const request = { rawBody, timestamp, signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}` }; + const current = await broker.slackEvents.current(instanceId, publicId); + await expect(broker.slackEvents.receive(current, request)).rejects.toThrow("SLACK_DURABLE_WAKE_UNAVAILABLE"); + const connector = createPaperclipCloudConnector({ config: paperclipCloudConnectorConfigFromEnv()! }); + expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull(); + expect(await broker.slackEvents.receive(current, request)).toEqual({ status: 200, duplicate: false }); + await vi.waitFor(() => expect(wake).toHaveBeenCalledOnce()); + const leased = await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 }); + expect(leased?.failedEventIds).toBeUndefined(); expect(leased?.events).toHaveLength(1); + const event = leased!.events[0]!; deliveryIds.push(event.id); + if (event.provider !== "slack") throw new Error("Unexpected event owner"); + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready"); + const onMessage = vi.fn(async (_event: ChatSdkMessageCallbackEvent) => {}); + runtime = createChatSdkEndpointRuntime({ companyId, endpointId, logger: "silent", callbacks: { onMessage }, + persistence: { read: async () => null, compareAndSet: async () => true, deleteIfVersion: async () => true }, + providerConfig: { provider: "slack", userName: "paperclip-agent", credentials: { apiUrl: "http://127.0.0.1:1/api/", botToken: "xoxb-fixture", botUserId: "UBOT", signingSecret } } }); + await runtime.initialize(); + const transport = stubSlackFixtureTransport(runtime.getProviderAdapter()); + unregister = createSlackCloudEventIngress(db, { withCredentialMutationLease: withLease, resolveCredentials, + runtimeGeneration: value => Number((value.setup as InternalSetupState).runtimeGeneration ?? 0), + handleWebhook: async (_publicId, _provider, req, generation) => { + expect(generation).toBe(1); + + await withLease(await endpoint(), async lease => { await lease.assertOwned(); }); + return runtime!.handleWebhook(req); + } }); + await dispatchSlackCloudEvent(db, event); + expect(onMessage).toHaveBeenCalledOnce(); + expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ message: expect.objectContaining({ author: expect.objectContaining({ userId: "UOPERATOR", userName: "Operator" }) }) })); + expect(transport).toHaveBeenCalled(); + await transport.assertNoErrors(); + expect(await connector.acknowledgeEvents({ subject: actorId, companyId, leaseId: leased!.leaseId, deliveryIds: [event.id] })).toBe(1); + expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull(); + await db.update(chatEndpoints).set({ setup: { ...saved, runtimeGeneration: 2 } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId)); + await dispatchSlackCloudEvent(db, event); + expect(onMessage).toHaveBeenCalledOnce(); + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId)); + await withLease(await endpoint(), lease => produce(endpointId, actorId, lease)); + const replacement = ((await endpoint()).setup as InternalSetupState).slackCloudEvents!; + expect(replacement.runtimeGeneration).toBe(2); expect(replacement.bindingId).not.toBe(saved.slackCloudEvents!.bindingId); + const firstMessage = onMessage.mock.calls[0]![0]; + const [task] = await db.insert(issues).values({ companyId, title: "Existing Slack conversation", status: "in_progress", + assigneeAgentId: agentId, createdByUserId: actorId, identifier: `S${id.slice(0, 7)}-1`, issueNumber: 1 }).returning(); + const [resource] = await db.insert(chatEndpointResources).values({ companyId, endpointId, type: "channel", + providerResourceId: firstMessage.thread.channelId, label: "Slack fixture", availability: "available", enabled: true }).returning(); + await db.insert(chatConversations).values({ companyId, endpointId, issueId: task!.id, resourceId: resource!.id, + externalConversationId: firstMessage.thread.channelId, externalThreadId: firstMessage.thread.id, + externalLabel: "Slack fixture", isDirectMessage: firstMessage.thread.isDM }); + secretDirectory = await mkdtemp(path.join(os.tmpdir(), "slack-owner-secrets-")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(secretDirectory, "master.key")); + const markerPath = path.join(secretDirectory, "normal-process-marker.json"); + const markerText = `slack-domain-once-${id}`; + await db.update(agents).set({ adapterConfig: { command: process.execPath, cwd: secretDirectory, + args: ["--require", fileURLToPath(new URL("../../../__tests__/helpers/slack-cloud/prewire.cjs", import.meta.url)), "-e", `if(process.env.SLACK_GATE_GUARD_PID!==String(process.pid))throw Error("child prewire guard missing");require("node:fs").writeFileSync(${JSON.stringify(markerPath)},JSON.stringify({pid:process.pid,runId:process.env.PAPERCLIP_RUN_ID,marker:${JSON.stringify(markerText)}}),{flag:"ax"});console.log(${JSON.stringify(markerText)});`], + timeoutSec: 10, graceSec: 1 } }).where(eq(agents.id, agentId)); + const vault = secretService(db), refs: ToolCredentialSecretRef[] = []; + for (const [key, value] of Object.entries({ botToken: "xoxb-fixture", signingSecret })) { + const secret = await vault.create(companyId, { name: `slack-${key}-${id}`, provider: "local_encrypted", value }, { userId: actorId }); + await vault.createBinding({ companyId, secretId: secret.id, targetType: "tool_connection", targetId: connectionId, configPath: `credentials.${key}` }); + refs.push({ secretId: secret.id, versionSelector: "latest", configPath: `credentials.${key}`, required: true, projectionClass: "unclassified" }); + } + await db.update(toolConnections).set({ credentialSecretRefs: refs }).where(eq(toolConnections.id, connectionId)); + const [linkedPrincipal] = await db.insert(chatExternalPrincipals).values({ companyId, provider: "slack", providerAccountId: "TTEST", + externalId: "UOPERATOR", kind: "user", displayName: "Operator" }).returning(); + await db.insert(chatIdentityLinks).values({ companyId, endpointId, principalId: linkedPrincipal!.id, paperclipUserId: actorId, + status: "linked", confirmedAt: now }); + const canonicalCallback = "https://tenant.example/api/chat-webhooks/" + publicId + "/slack"; + await db.update(chatEndpoints).set({ botExternalId: "UBOT", sponsorUserId: actorId, allowGroupChats: true, + setup: { ...((await endpoint()).setup as InternalSetupState), step: "complete", webhookVerifiedAt: now.toISOString(), + slackCallbackSurfaces: { events: { url: canonicalCallback, observedAt: now.toISOString() } } } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId)); + const callbackEvidence = structuredClone(((await endpoint()).setup as InternalSetupState).slackCallbackSurfaces); + const actualRuntime = new ChatRuntime(), sdkCallbacks: ChatSdkMessageCallbackEvent[] = []; + const transports: ReturnType[] = []; + const replaceEndpoint = actualRuntime.replaceEndpoint.bind(actualRuntime); + vi.spyOn(actualRuntime, "replaceEndpoint").mockImplementation(async input => { + if (input.providerConfig.provider !== "slack") throw new Error("Unexpected fixture provider"); + const value = await replaceEndpoint({ ...input, providerConfig: { ...input.providerConfig, + credentials: { ...input.providerConfig.credentials, apiUrl: "http://127.0.0.1:1/api/" } }, + callbacks: { ...input.callbacks, onMessage: async message => { sdkCallbacks.push(message); await input.callbacks.onMessage(message); } } }); + transports.push(stubSlackFixtureTransport(value.getProviderAdapter())); + return value; + }); + normalHeartbeat = heartbeatService(db); + const receiptFetch: typeof fetch = async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (!["https://slack.com/api/reactions.add", "https://slack.com/api/reactions.remove"].includes(url)) throw new Error(`Unexpected normal Slack owner request ${url}`); + expect(init?.method).toBe("POST"); + expect(JSON.parse(String(init?.body))).toMatchObject({ channel: "CTEST", name: "eyes" }); + return Response.json({ ok: true }); + }; + const deferred: (() => void | Promise)[] = []; + normalService = chatChannelService(db, { runtime: actualRuntime, publicBaseUrl: "https://tenant.example", + deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); }, heartbeat: normalHeartbeat, fetch: receiptFetch }); + const ownerMessageTs = (Date.now() / 1000).toFixed(6); + const ownerRawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvActualOwner", team_id: "TTEST", api_app_id: "ATEST", + event: { type: "app_mention", text: "<@UBOT> one durable reply", user: "UOPERATOR", channel: "CTEST", ts: ownerMessageTs, event_ts: ownerMessageTs, thread_ts: "1788.500" } })); + const ownerTimestamp = String(Math.floor(Date.now() / 1000)); + const ownerRequest = { rawBody: ownerRawBody, timestamp: ownerTimestamp, + signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${ownerTimestamp}:`).update(ownerRawBody).digest("hex")}` }; + await broker.slackEvents.receive(await broker.slackEvents.current(instanceId, publicId), ownerRequest); + let lostAck = true; + const captured: NonNullable>>["events"] = []; + const ownerConnector = { ...connector, leaseEvents: async (input: Parameters[0]) => { + const batch = await connector.leaseEvents(input); if (batch) captured.push(...batch.events); return batch; + }, acknowledgeEvents: async (input: Parameters[0]) => { + if (lostAck) { lostAck = false; throw new Error("fixture ACK request lost before broker settlement"); } + return connector.acknowledgeEvents(input); + } }; + const busyEndpointId = randomUUID(), busyConnectionId = randomUUID(); + await db.insert(toolConnections).values({ id: busyConnectionId, companyId, applicationId, name: "Retirement waiting for credential mutation", uid: busyEndpointId, + transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" }); + await db.insert(chatEndpoints).values({ id: busyEndpointId, companyId, connectionId: busyConnectionId, assignedAgentId: agentId, + publicId: "f".repeat(43), provider: "slack", status: "archived", setup: { runtimeGeneration: 1 } as InternalSetupState }); + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, busyEndpointId, actorId)); + const [busyEndpoint] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + const originalBusyIntent = structuredClone((busyEndpoint!.setup as InternalSetupState).slackCloudIntent!); + let acquired!: () => void; + const acquiredPromise = new Promise(resolve => { acquired = resolve; }); + const releasePromise = new Promise(resolve => { releaseEnrollmentLease = resolve; }); + heldEnrollmentLease = withLease(busyEndpoint!, async lease => { await lease.assertOwned(); acquired(); await releasePromise; }); + await acquiredPromise; + const eventOwner = githubConnectionEventService(db, { connector: ownerConnector }); + await expect(eventOwner.pollOnce({ forDelivery: true })).rejects.toThrow("ACK request lost"); + expect(captured).toHaveLength(1); const actualReceiptId = captured[0]!.id; deliveryIds.push(actualReceiptId); + const beforeRedelivery = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId)); + expect(beforeRedelivery).toHaveLength(1); + const [retainedBusy] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + const busyIntent = (retainedBusy!.setup as InternalSetupState).slackCloudIntent!; + expect(busyIntent).toEqual({ ...originalBusyIntent, attempts: 1, failureCode: "credential_lease_busy", nextAttemptAt: expect.any(String) }); + const originalRetryAt = Date.parse(busyIntent.nextAttemptAt!); + expect(Number.isFinite(originalRetryAt)).toBe(true); + expect(originalRetryAt).toBeGreaterThan(Date.now()); + expect(originalRetryAt).toBeLessThanOrEqual(Date.now() + 5_000); + expect(eventOwner.nextPollAt()).not.toBeNull(); + expect(eventOwner.nextPollAt()!).toBeGreaterThan(Date.now()); + expect(eventOwner.nextPollAt()!).toBeLessThanOrEqual(Date.now() + 5_000); + releaseEnrollmentLease!(); await heldEnrollmentLease; + + const expired = await pool.query("UPDATE cloud_harness.connector_webhook_delivery_targets SET lease_expires_at=$3 WHERE connector_instance_id=$1 AND delivery_id=$2 AND status='leased' RETURNING delivery_id", + [instanceId, actualReceiptId, new Date(Date.now() - 1)]); + expect(expired.rowCount).toBe(1); + expect((await eventOwner.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(captured.map(value => value.id)).toEqual([actualReceiptId, actualReceiptId]); + const [beforeRetry] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + if (Date.now() < originalRetryAt) expect((beforeRetry!.setup as InternalSetupState).slackCloudIntent).toEqual(busyIntent); + await new Promise(resolve => setTimeout(resolve, Math.max(0, originalRetryAt - Date.now()))); + await eventOwner.pollOnce(); + const [settledBusy] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + expect((settledBusy!.setup as InternalSetupState).slackCloudIntent).toBeUndefined(); + expect((settledBusy!.setup as InternalSetupState).slackCloudRetiredGeneration).toBe(1); + expect((settledBusy!.setup as InternalSetupState).slackCloudRetiredGeneration).toBe(originalBusyIntent.runtimeGeneration); + expect(captured.map(value => value.id)).toEqual([actualReceiptId, actualReceiptId]); + + expect(captured[1]!.createdAt).toBe(captured[0]!.createdAt); expect(captured[1]!.payload).toEqual(captured[0]!.payload); + expect(sdkCallbacks).toHaveLength(2); expect(sdkCallbacks[0]!.message.id).toBe(sdkCallbacks[1]!.message.id); + expect(sdkCallbacks[0]!.thread.id).toBe(sdkCallbacks[1]!.thread.id); + const afterRedelivery = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId)); + expect(afterRedelivery).toHaveLength(1); expect(afterRedelivery[0]!.providerEventId).toBe(beforeRedelivery[0]!.providerEventId); + expect(deferred.length).toBeGreaterThan(0); + await deferred.shift()!(); + await normalService.processQueuedDeliveries(); + const links = await db.select().from(chatMessageLinks).where(and(eq(chatMessageLinks.endpointId, endpointId), eq(chatMessageLinks.direction, "inbound"))); + expect(links).toHaveLength(1); expect(links[0]!.deliveryId).toBe(afterRedelivery[0]!.id); + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, task!.id)); + expect(comments.filter(value => value.id === links[0]!.commentId)).toHaveLength(1); + expect(comments.find(value => value.id === links[0]!.commentId)?.body).toContain("one durable reply"); + const wakeReceipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId)); + expect(wakeReceipts).toHaveLength(1); + await normalHeartbeat.drainActiveRunExecutions(); + const completedRuns = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(completedRuns).toHaveLength(1); + expect(completedRuns[0]).toMatchObject({ status: "succeeded", exitCode: 0, wakeupRequestId: wakeReceipts[0]!.id }); + expect(completedRuns[0]!.startedAt).not.toBeNull(); + expect(completedRuns[0]!.finishedAt).not.toBeNull(); + expect(completedRuns[0]!.stdoutExcerpt).toContain(markerText); + const marker = JSON.parse(await readFile(markerPath, "utf8")) as { pid: number; runId: string; marker: string }; + expect(marker).toMatchObject({ runId: completedRuns[0]!.id, marker: markerText }); + expect(marker.pid).not.toBe(process.pid); + expect(((await endpoint()).setup as InternalSetupState).slackCallbackSurfaces).toEqual(callbackEvidence); + const signingRef = refs.find(ref => ref.configPath === "credentials.signingSecret")!; + await vault.update(signingRef.secretId, { status: "disabled" }); + try { + await expect(dispatchSlackCloudEvent(db, captured[1]!)).rejects.toMatchObject({ + code: "signing_credential_unavailable", disposition: "attention", + }); + expect(sdkCallbacks).toHaveLength(2); + } finally { await vault.update(signingRef.secretId, { status: "active" }); } + + expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull(); + for (const transport of transports) await transport.assertNoErrors(); + await normalService.shutdown(); normalService = undefined; + let rejectAfterIntent = false; + const beforeRollbackRequests = enrollmentRequests; + const stopSignals = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { if (event === "intent") rejectAfterIntent = true; }); + normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example", + scheduleDeferredWork: work => { deferred.push(work); }, renewCredentialMutationLease: async () => !rejectAfterIntent }); + try { + await expect(normalService.configure(endpointId, { action: "pause" }, actorId)).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); + expect((await endpoint()).status).toBe("active"); + expect(((await endpoint()).setup as InternalSetupState).runtimeGeneration).toBe(2); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toBeUndefined(); + expect(enrollmentRequests).toBe(beforeRollbackRequests); + } finally { stopSignals(); await normalService.shutdown(); normalService = undefined; } + normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example", + scheduleDeferredWork: work => { deferred.push(work); } }); + await db.transaction(async tx => { + await tx.update(chatEndpoints).set({ setup: { ...((await endpoint()).setup as InternalSetupState), runtimeGeneration: 3 } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId)); + await stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId); + }); + loseEnrollmentResponse = true; + await withLease(await endpoint(), lease => produce(endpointId, actorId, lease)); + expect((await broker.slackEvents.current(instanceId, publicId)).runtimeGeneration).toBe(3); + expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: true, runtimeGeneration: 3, failureCode: "broker_unavailable" }); + await withLease(await endpoint(), async lease => { + let transactionChecks = 0; + await produce(endpointId, actorId, { assertOwned: async transaction => { + await lease.assertOwned(transaction); + if (transaction && ++transactionChecks === 2) throw new Error("fixture final enrollment guard lost"); + } }); + }); + expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2); + const casProducer = createSlackCloudEnrollment(db, { runtimeGeneration: setup => Number(setup.runtimeGeneration), resolveCredentials: async () => { + await db.update(chatEndpoints).set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', '4'::jsonb)` }).where(eq(chatEndpoints.id, endpointId)); + return { signingSecret }; + } }); + await expect(withLease(await endpoint(), lease => casProducer(endpointId, actorId, lease))).rejects.toThrow("changed"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2); + brokerUnavailable = true; + expect((await normalService.configure(endpointId, { action: "pause" }, actorId)).status).toBe("paused"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 5, attempts: 0 }); + await expect(eventOwner.pollOnce({ forDelivery: true })).rejects.toThrow("status=503"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ failureCode: "broker_unavailable", attempts: 1 }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", ""); + expect((await normalService.configure(endpointId, { action: "remove" }, actorId)).status).toBe("archived"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 6, subject: actorId, attempts: 0 }); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 6, failureCode: "connector_configuration_unavailable", nextAttemptAt: null }); + expect((await db.select().from(toolConnections).where(eq(toolConnections.id, connectionId)))[0]!.credentialSecretRefs).toEqual([]); + expect((await db.select().from(chatSlackRegistrations).where(eq(chatSlackRegistrations.endpointId, endpointId)))[0]!.status).toBe("removed"); + await normalService.shutdown(); normalService = undefined; + brokerUnavailable = false; + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); + normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example", + scheduleDeferredWork: work => { deferred.push(work); } }); + const reconstructedOwner = githubConnectionEventService(db, { connector }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", `replacement-${instanceId}`); + await reconstructedOwner.pollOnce({ forDelivery: true }); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ instanceId, failureCode: "connector_configuration_unavailable", nextAttemptAt: null }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId); + await reconstructedOwner.pollOnce({ forDelivery: true }); + expect(reconstructedOwner.nextPollAt()).toBeNull(); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toBeUndefined(); + expect(((await endpoint()).setup as InternalSetupState).slackCloudRetiredGeneration).toBe(6); + expect(await broker.slackEvents.current(instanceId, publicId)).toBeUndefined(); + expect((await store.listWebhookBindings(instanceId)).find((value: { receivesEvents?: boolean }) => value.receivesEvents === false).providerMetadata.retirement.runtimeGeneration).toBe(6); + await expect(broker.slackEvents.enroll(await store.getInstance(instanceId), actorId, { companyId, endpointId, publicId, connectionId, + runtimeGeneration: 3, teamId: "TTEST", appId: "ATEST", signingSecret })).rejects.toThrow("GENERATION_CHANGED"); + + const pendingIds = [randomUUID(), randomUUID()]; + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", ""); + for (const [index, pendingId] of pendingIds.entries()) { + const pendingConnectionId = randomUUID(); + await db.insert(toolConnections).values({ id: pendingConnectionId, companyId, applicationId, name: "Retired Slack", uid: pendingId, + transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" }); + await db.insert(chatEndpoints).values({ id: pendingId, companyId, connectionId: pendingConnectionId, assignedAgentId: agentId, + publicId: (index === 0 ? "d" : "e").repeat(43), provider: "slack", status: "archived", providerAccountId: "TTEST", + setup: { runtimeGeneration: 1, slackCloudInstanceId: instanceId, slackCloudSubject: actorId } as InternalSetupState }); + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, pendingId, actorId)); + } + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); + await reconstructedOwner.pollOnce({ forDelivery: true }); + const remaining = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.companyId, companyId), + sql`${chatEndpoints.setup}->'slackCloudIntent' is not null`)); + expect(remaining).toHaveLength(1); + expect(pendingIds).toContain(remaining[0]!.id); + expect(reconstructedOwner.nextPollAt()).not.toBeNull(); + expect(reconstructedOwner.nextPollAt()!).toBeLessThanOrEqual(Date.now()); + await reconstructedOwner.pollOnce({ forDelivery: true }); + expect(await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.companyId, companyId), + sql`${chatEndpoints.setup}->'slackCloudIntent' is not null`))).toHaveLength(0); + expect(reconstructedOwner.nextPollAt()).toBeNull(); + + } catch (error) { + console.error("Slack paired fixture body failure", error); + throw error; + } finally { + releaseEnrollmentLease?.(); await heldEnrollmentLease; + await normalHeartbeat?.drainActiveRunExecutions(); + await normalService?.shutdown(); + unregister?.(); await runtime?.shutdown(); + if (server.listening) { server.close(); await once(server, "close"); } + vi.unstubAllEnvs(); + try { + await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId)); + await db.delete(financeEvents).where(eq(financeEvents.companyId, companyId)); + await db.delete(costEvents).where(eq(costEvents.companyId, companyId)); + await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, companyId)); + await db.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, companyId)); + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, companyId)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + await db.delete(issueComments).where(eq(issueComments.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId)); + await db.delete(companySkills).where(eq(companySkills.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + await db.delete(authUsers).where(eq(authUsers.id, actorId)); + await pool.query("DELETE FROM cloud_harness.connector_instances WHERE id=$1", [instanceId]); + await pool.query("DELETE FROM cloud_harness.connector_webhook_deliveries WHERE id=ANY($1::text[])", [deliveryIds]); + } finally { await closeCheckpointDb(db, 1); await closeFileStorageDb(db, 1); await closeLivenessDb(db, 1); + await db.$client.end({ timeout: 1 }); await registry.close(); + if (secretDirectory) await rm(secretDirectory, { recursive: true, force: true }); + try { network.assertQuiet(); } finally { network.restore(); } + } + } +}, 60_000); diff --git a/server/src/services/chat-channels/slack/cloud-enrollment.test.ts b/server/src/services/chat-channels/slack/cloud-enrollment.test.ts new file mode 100644 index 00000000000..1b5b47d0ad3 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-enrollment.test.ts @@ -0,0 +1,214 @@ +import { HttpError } from "../../../errors.js"; +import { generateKeyPairSync } from "node:crypto"; +import { chatEndpoints, chatSlackRegistrations, type Db } from "@paperclipai/db"; +import { afterEach, expect, it, vi } from "vitest"; +import { PaperclipCloudConnectorError } from "../../paperclip-cloud-connector.js"; +import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent } from "./cloud-enrollment.js"; +import type { InternalSetupState } from "../runtime-types.js"; +import type { CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js"; + +afterEach(() => vi.unstubAllEnvs()); +function configureCloud() { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "https://cloud.example.test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", generateKeyPairSync("ed25519").privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", generateKeyPairSync("x25519").privateKey.export({ format: "jwk" }).d!); +} +function jsonValues(value: unknown): unknown[] { + if (typeof value === "string") { try { return [JSON.parse(value)]; } catch { return []; } } + if (!value || typeof value !== "object") return []; + const chunks = Reflect.get(value, "queryChunks"); + return Array.isArray(chunks) ? chunks.flatMap(jsonValues) : []; +} +function fixture() { + configureCloud(); + const endpoint = { id: "00000000-0000-4000-8000-000000000001", companyId: "company", connectionId: "connection", publicId: "a".repeat(43), + provider: "slack", status: "active", providerAccountId: "TTEST", setup: { runtimeGeneration: 1 } as InternalSetupState }; + const connection = { status: "active", enabled: true }; + let rejectWrite = false; + const update = vi.fn((_table: object) => ({ set: (values: { setup: unknown }) => ({ where: () => { + const write = () => { + if (rejectWrite) return []; + const priorIntent = endpoint.setup.slackCloudIntent; + for (const value of jsonValues(values.setup)) { + if (value && typeof value === "object" && "active" in value && "id" in value) { + endpoint.setup.slackCloudIntent = value as NonNullable; + + } + else if (value && typeof value === "object" && "bindingId" in value) { delete endpoint.setup.slackCloudIntent; delete endpoint.setup.slackCloudRetiredGeneration; endpoint.setup.slackCloudEvents = value as NonNullable; endpoint.setup.slackCloudInstanceId = priorIntent!.instanceId; endpoint.setup.slackCloudSubject = priorIntent!.subject; } + else if (typeof value === "number") { delete endpoint.setup.slackCloudIntent; endpoint.setup.slackCloudRetiredGeneration = value; endpoint.setup.slackCloudInstanceId = priorIntent!.instanceId; endpoint.setup.slackCloudSubject = priorIntent!.subject; } + } + return [{ id: endpoint.id }]; + }; + return { then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(write()).then(resolve), returning: async () => write() }; + } }) })); + const db = { update, select: (fields?: Record) => { + let table: object, joined = false; + const rows = () => table === chatSlackRegistrations ? [{ appId: "ATEST" }] : joined + ? [{ endpoint: structuredClone(endpoint), connection: structuredClone(connection) }] + : fields?.setup ? [{ setup: structuredClone(endpoint.setup) }] : table === chatEndpoints ? [structuredClone(endpoint)] : []; + const query = { from: (value: object) => { table = value; return query; }, innerJoin: () => { joined = true; return query; }, + where: () => query, limit: async () => rows(), then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(rows()).then(resolve) }; + return query; + }, transaction: async (work: (tx: unknown) => Promise) => { + const previous = structuredClone(endpoint.setup); + try { return await work(db); } catch (error) { endpoint.setup = previous; throw error; } + } } as unknown as Db; + const resolveCredentials = vi.fn(async () => ({ signingSecret: "synthetic" })); + const connector = { slackEventsAvailable: vi.fn(async () => true), slackEventEnrollment: vi.fn(async (input: { enrollment: Record }): Promise => { + const { active: _active, signingSecret: _secret, ...identity } = input.enrollment; + return { version: 1, instanceId: "instance", ...identity, ...(input.enrollment.active ? { bindingId: `slack-event:${"b".repeat(64)}` } : {}) }; + }) }; + const lease = { assertOwned: vi.fn(async () => {}) } as unknown as CredentialMutationLeaseGuard; + const run = createSlackCloudEnrollment(db, { resolveCredentials, runtimeGeneration: setup => Number(setup.runtimeGeneration), connector }); + const stage = () => db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as Db, endpoint.id, "original-authorized-user")); + return { db, endpoint, connection, update, connector, resolveCredentials, lease, stage, run, + rejectWrites: () => { rejectWrite = true; } }; +} + +it("stages nonsecret responsibility before network and clears only an exactly confirmed current intent", async () => { + const f = fixture(); await f.stage(); + expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled(); + expect(JSON.stringify(f.endpoint.setup.slackCloudIntent)).not.toContain("synthetic"); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); + expect(f.endpoint.setup.slackCloudEvents?.runtimeGeneration).toBe(1); + expect(f.connector.slackEventEnrollment).toHaveBeenCalledWith(expect.objectContaining({ subject: "original-authorized-user" })); +}); + +it.each(["paused", "archived"])("retains credential-free %s removal responsibility after local cleanup", async status => { + const f = fixture(); f.endpoint.status = status; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); expect(f.endpoint.setup.slackCloudIntent?.active).toBe(false); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.resolveCredentials).not.toHaveBeenCalled(); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(2); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); +}); + +it("outage preserves a bounded pending retry; current broker denial preserves visible attention debt", async () => { + const f = fixture(); await f.stage(); + f.connector.slackEventsAvailable.mockRejectedValueOnce(new PaperclipCloudConnectorError("offline", "CONNECTOR_UNAVAILABLE")); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ attempts: 1, failureCode: "broker_unavailable" }); + expect(Date.parse(f.endpoint.setup.slackCloudIntent!.nextAttemptAt!)).toBeGreaterThan(Date.now()); + f.connector.slackEventEnrollment.mockRejectedValueOnce(new PaperclipCloudConnectorError("denied", "CONNECTOR_REQUEST_FAILED", 403)); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ failureCode: "authority_required", nextAttemptAt: null }); +}); + +it("receipt identity mismatch and failed transaction guard cannot publish a live descriptor", async () => { + const f = fixture(); await f.stage(); + f.connector.slackEventEnrollment.mockResolvedValueOnce({ version: 1, runtimeGeneration: 999 }); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); expect(f.endpoint.setup.slackCloudIntent).toBeDefined(); + f.rejectWrites(); await expect(f.run(f.endpoint.id, undefined, f.lease)).rejects.toThrow("changed"); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); expect(f.endpoint.setup.slackCloudIntent).toBeDefined(); +}); + +it("an absent Cloud configuration creates no new intent or background retry", async () => { + const f = fixture(); for (const key of ["BASE_URL", "INSTANCE_ID", "SIGN_PRIVATE_KEY", "SEAL_PRIVATE_KEY", "ENVIRONMENT"]) vi.stubEnv(`PAPERCLIP_CLOUD_CONNECTOR_${key}`, ""); + await f.stage(); expect(f.update).not.toHaveBeenCalled(); expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); +}); + +it.each(["SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_DISABLED"])("closed broker reason %s distinguishes permanent authority from rollout retry", async reason => { + const f = fixture(); await f.stage(); + f.connector.slackEventEnrollment.mockRejectedValueOnce(new PaperclipCloudConnectorError("rejected", "REAUTHORIZATION_REQUIRED", 409, undefined, reason)); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); + expect(f.endpoint.setup.slackCloudIntent!.nextAttemptAt === null).toBe(reason === "SLACK_ENROLLMENT_GENERATION_CHANGED"); + expect(f.endpoint.setup.slackCloudIntent!.failureCode).toBe(reason === "SLACK_ENROLLMENT_DISABLED" ? "enrollment_disabled" : "authority_required"); +}); + +it("losing Cloud identity retains configured retirement responsibility as attention without an empty retry", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + f.endpoint.status = "archived"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + for (const key of ["BASE_URL", "INSTANCE_ID", "SIGN_PRIVATE_KEY", "SEAL_PRIVATE_KEY", "ENVIRONMENT"]) vi.stubEnv(`PAPERCLIP_CLOUD_CONNECTOR_${key}`, ""); + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ active: false, instanceId: "instance", runtimeGeneration: 2, + failureCode: "connector_configuration_unavailable", nextAttemptAt: null }); + expect(f.connector.slackEventEnrollment).toHaveBeenCalledOnce(); +}); + +it.each([404, 422])("unavailable signing credential %s becomes retained attention instead of broker retry", async status => { + const f = fixture(); await f.stage(); + f.resolveCredentials.mockRejectedValueOnce(new HttpError(status, "synthetic credential failure")); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ failureCode: "signing_credential_unavailable", nextAttemptAt: null }); + expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled(); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); +}); + +it("temporary signing-credential store failure keeps the bounded retry", async () => { + const f = fixture(); await f.stage(); + f.resolveCredentials.mockRejectedValueOnce(new Error("temporary database failure")); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent?.failureCode).toBe("broker_unavailable"); + expect(Date.parse(f.endpoint.setup.slackCloudIntent!.nextAttemptAt!)).toBeGreaterThan(Date.now()); + expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled(); +}); + +it("only positive old retirement and an explicit new generation can stage a replacement instance", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + const oldDescriptor = structuredClone(f.endpoint.setup.slackCloudEvents); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "instance", subject: "original-authorized-user", + failureCode: "instance_replacement_requires_retirement", nextAttemptAt: null }); + expect(f.endpoint.setup.slackCloudEvents).toEqual(oldDescriptor); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance"); + f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 3; + await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(3); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 4; + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "replacement", runtimeGeneration: 4 }); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("instance"); + expect(f.endpoint.setup.slackCloudSubject).toBe("original-authorized-user"); + expect(f.endpoint.setup.slackCloudEvents).toEqual(oldDescriptor); + f.connector.slackEventEnrollment.mockImplementationOnce(async input => { + const { active: _active, signingSecret: _secret, ...identity } = input.enrollment; + return { version: 1, instanceId: "replacement", ...identity, bindingId: `slack-event:${"c".repeat(64)}` }; + }); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("replacement"); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBeUndefined(); + expect(f.endpoint.setup.slackCloudEvents?.runtimeGeneration).toBe(4); +}); + +it("a new config without an explicit actor cannot adopt even a retired old instance", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 3; + await stageSlackCloudEnrollmentIntent(f.db, f.endpoint.id, undefined); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "instance", subject: "original-authorized-user", + nextAttemptAt: null, failureCode: "instance_replacement_requires_retirement" }); +}); + +it("removal after an unconfirmed replacement targets its pending instance without new credentials", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 3; + await f.stage(); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("instance"); + f.endpoint.status = "archived"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 4; + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ active: false, instanceId: "replacement", runtimeGeneration: 4 }); + f.resolveCredentials.mockClear(); + f.connector.slackEventEnrollment.mockImplementationOnce(async input => { + const { active: _active, ...identity } = input.enrollment; + return { version: 1, instanceId: "replacement", ...identity }; + }); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.resolveCredentials).not.toHaveBeenCalled(); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("replacement"); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(4); +}); diff --git a/server/src/services/chat-channels/slack/cloud-enrollment.ts b/server/src/services/chat-channels/slack/cloud-enrollment.ts new file mode 100644 index 00000000000..be8e33394e3 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-enrollment.ts @@ -0,0 +1,218 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, or, sql } from "drizzle-orm"; +import { chatEndpoints, chatSlackRegistrations, toolConnections, type Db } from "@paperclipai/db"; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv, PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../../paperclip-cloud-connector.js"; +import type { CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js"; +import type { createChatCredentials } from "../credentials.js"; +import type { InternalSetupState } from "../runtime-types.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../delivery-work-notifications.js"; +import { logger } from "../../../middleware/logger.js"; +import { HttpError } from "../../../errors.js"; + +import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js"; +import { SlackCloudReceiptError } from "./cloud-event.js"; + +type Intent = NonNullable; +class SlackEnrollmentFailure extends Error { + constructor(readonly code: string) { super(code); } +} + +type Owner = (endpointId: string) => Promise; +const owners = new WeakMap(); + +export function registerSlackCloudEnrollmentOwner(db: Db, owner: Owner) { + owners.set(db, owner); + return () => { if (owners.get(db) === owner) owners.delete(db); }; +} + +function slackCloudEnrollmentIsActive(endpoint: { status: string }, connection: { status: string; enabled: boolean }) { + return ["active", "verifying", "attention"].includes(endpoint.status) && connection.status === "active" && connection.enabled; +} + +export async function stageSlackCloudEnrollmentIntent(db: Db, endpointId: string, actorUserId: string | null | undefined) { + let config: ReturnType; + try { config = paperclipCloudConnectorConfigFromEnv(); } catch { config = null; } + const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId))) + .where(eq(chatEndpoints.id, endpointId)).limit(1); + if (!current || current.endpoint.provider !== "slack") return; + const endpoint = current.endpoint, setup = endpoint.setup as InternalSetupState; + const generation = Number(setup.runtimeGeneration ?? 0); + const active = slackCloudEnrollmentIsActive(endpoint, current.connection); + const custodyInstance = setup.slackCloudIntent?.instanceId ?? setup.slackCloudInstanceId; + const custodySubject = setup.slackCloudIntent?.subject ?? setup.slackCloudSubject; + const oldRetired = !setup.slackCloudIntent && setup.slackCloudInstanceId + && Number.isSafeInteger(setup.slackCloudRetiredGeneration) && Number(setup.slackCloudRetiredGeneration) >= 1 + && Number(setup.slackCloudRetiredGeneration) >= (setup.slackCloudEvents?.runtimeGeneration ?? 1); + const changingInstance = !!custodyInstance && !!config && custodyInstance !== config.instanceId; + const mayReplace = active && changingInstance && oldRetired && !!actorUserId + && generation > Number(setup.slackCloudRetiredGeneration); + const blocked = changingInstance && !mayReplace; + const instanceId = mayReplace ? config!.instanceId : custodyInstance ?? config?.instanceId; + const subject = blocked || !active ? custodySubject ?? actorUserId : actorUserId ?? custodySubject; + if (!instanceId || !subject || !Number.isSafeInteger(generation) || generation < 1) return; + if (setup.slackCloudIntent?.runtimeGeneration === generation && setup.slackCloudIntent.active === active + && setup.slackCloudIntent.instanceId === instanceId && setup.slackCloudIntent.subject === subject) return; + if (!blocked && setup.slackCloudInstanceId === instanceId && setup.slackCloudSubject === subject + && (active ? setup.slackCloudEvents?.runtimeGeneration === generation : setup.slackCloudRetiredGeneration === generation)) return; + const [registration] = active ? await db.select({ appId: chatSlackRegistrations.appId }).from(chatSlackRegistrations) + .where(and(eq(chatSlackRegistrations.endpointId, endpointId), eq(chatSlackRegistrations.companyId, endpoint.companyId))).limit(1) : []; + const appId = registration?.appId ?? setup.slackCloudEvents?.appId; + if (active && (!appId || !endpoint.providerAccountId)) return; + const intent: Intent = { version: 1, id: randomUUID(), subject, instanceId, companyId: endpoint.companyId, endpointId, + publicId: endpoint.publicId, connectionId: endpoint.connectionId, runtimeGeneration: generation, active, + ...(active ? { appId: appId!, teamId: endpoint.providerAccountId! } : {}), attempts: 0, nextAttemptAt: config && !blocked ? new Date().toISOString() : null, + ...(blocked ? { failureCode: "instance_replacement_requires_retirement" } : !config ? { failureCode: "connector_configuration_unavailable" } : {}) }; + await notifyDeliveryWork(db, DELIVERY_QUEUES.connectionEvents); + await db.update(chatEndpoints).set({ + lastError: config && !blocked ? sql`${chatEndpoints.lastError}` : "Slack Cloud event registration requires attention", + setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(intent)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, endpoint.companyId))); +} + +export async function reconcilePendingSlackCloudEnrollments(db: Db, now: Date): Promise { + const config = paperclipCloudConnectorConfigFromEnv(); + if (!config) return null; + const recoverable = and( + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt' is null`, + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'failureCode' = 'connector_configuration_unavailable'`, + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'instanceId' = ${config.instanceId}`, + ); + const scheduled = and(eq(chatEndpoints.provider, "slack"), or( + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt' is not null`, recoverable)); + const order = sql`coalesce(${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt', '')`; + const pending = await db.select().from(chatEndpoints).where(scheduled).orderBy(order).limit(1); + const owner = owners.get(db); + const deadline = (intent: Intent | undefined) => { + if (!intent) return Number.POSITIVE_INFINITY; + if (intent.nextAttemptAt === null) return intent.failureCode === "connector_configuration_unavailable" && intent.instanceId === config.instanceId + ? now.getTime() : Number.POSITIVE_INFINITY; + const parsed = Date.parse(intent.nextAttemptAt); + return Number.isFinite(parsed) ? parsed : now.getTime(); + }; + let next = Number.POSITIVE_INFINITY; + for (const endpoint of pending) { + const intent = (endpoint.setup as InternalSetupState).slackCloudIntent!; + let due = deadline(intent); + if (!Number.isFinite(due)) continue; + if (due <= now.getTime()) { + if (owner) { + try { await owner(endpoint.id); } + catch (error) { + if (error instanceof HttpError && error.status === 409 && error.details && typeof error.details === "object" + && "code" in error.details && error.details.code === "chat_endpoint_credentials_busy") { + try { + if (intent.companyId === endpoint.companyId && intent.endpointId === endpoint.id + && intent.publicId === endpoint.publicId && intent.connectionId === endpoint.connectionId + && intent.runtimeGeneration === Number((endpoint.setup as InternalSetupState).runtimeGeneration ?? 0)) { + await db.transaction(async tx => { + const sameIntent = and(eq(chatEndpoints.id, endpoint.id), eq(chatEndpoints.companyId, intent.companyId), + eq(chatEndpoints.provider, "slack"), eq(chatEndpoints.publicId, intent.publicId), eq(chatEndpoints.connectionId, intent.connectionId), + sql`${chatEndpoints.setup}->'runtimeGeneration' = ${JSON.stringify(intent.runtimeGeneration)}::jsonb`, + sql`${chatEndpoints.setup}->'slackCloudIntent' = ${JSON.stringify(intent)}::jsonb`); + const [owned] = await tx.select({ id: chatEndpoints.id }).from(chatEndpoints).where(sameIntent).for("update").limit(1); + if (!owned) return; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + const retry = { ...intent, attempts: Math.min(intent.attempts + 1, 1_000_000), failureCode: "credential_lease_busy", + nextAttemptAt: new Date(Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(intent.attempts, 6))).toISOString() }; + const updated = await tx.update(chatEndpoints).set({ + setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(retry)}::jsonb)`, + }).where(sameIntent).returning({ id: chatEndpoints.id }); + if (updated.length !== 1) throw new Error("Slack enrollment busy retry custody changed"); + }); + } + } catch (retryError) { throw new AggregateError([error, retryError], "Slack enrollment busy retry persistence failed"); } + } + throw error; + } + } + else logger.warn({ endpointId: endpoint.id, reason: "slack_enrollment_owner_not_ready" }, "Slack enrollment responsibility retained"); + const [updated] = await db.select({ setup: chatEndpoints.setup }).from(chatEndpoints).where(eq(chatEndpoints.id, endpoint.id)).limit(1); + const retained = (updated?.setup as InternalSetupState | undefined)?.slackCloudIntent; + due = deadline(retained); + if (!owner) due = now.getTime() + 5_000; + } + next = Math.min(next, due); + } + if (pending.length > 0) { + const [earliest] = await db.select({ setup: chatEndpoints.setup }).from(chatEndpoints).where(scheduled).orderBy(order).limit(1); + next = Math.min(next, deadline((earliest?.setup as InternalSetupState | undefined)?.slackCloudIntent)); + } + return Number.isFinite(next) ? next : null; +} + +export function createSlackCloudEnrollment(db: Db, options: { + resolveCredentials: ReturnType["resolveCredentials"]; + runtimeGeneration(setup: InternalSetupState): number; + connector?: Pick; +}) { + return async (endpointId: string, _actorUserId: string | null | undefined, lease: CredentialMutationLeaseGuard) => { + const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId))) + .where(eq(chatEndpoints.id, endpointId)).limit(1); + if (!current || current.endpoint.provider !== "slack") return; + const endpoint = current.endpoint, setup = endpoint.setup as InternalSetupState, intent = setup.slackCloudIntent; + const config = paperclipCloudConnectorConfigFromEnv(); + if (!config || !intent) return; + const identity = { companyId: intent.companyId, endpointId: intent.endpointId, publicId: intent.publicId, + connectionId: intent.connectionId, runtimeGeneration: intent.runtimeGeneration }; + const active = slackCloudEnrollmentIsActive(endpoint, current.connection); + if (identity.companyId !== endpoint.companyId || identity.endpointId !== endpoint.id || identity.publicId !== endpoint.publicId + || identity.connectionId !== endpoint.connectionId || identity.runtimeGeneration !== options.runtimeGeneration(setup) || intent.active !== active) { + await db.transaction(async tx => { + await lease.assertOwned(tx); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + await tx.update(chatEndpoints).set({ + setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify({ ...intent, nextAttemptAt: null, failureCode: "intent_authority_changed" })}::jsonb)`, + lastError: "Slack Cloud event registration requires attention", + }).where(and(eq(chatEndpoints.id, endpoint.id), eq(chatEndpoints.companyId, endpoint.companyId), sql`${chatEndpoints.setup}->'slackCloudIntent'->>'id'=${intent.id}`)); + await lease.assertOwned(tx); + }); + logger.warn({ endpointId, reason: "intent_authority_changed" }, "Slack enrollment responsibility retained"); + return; + } + const connector = options.connector ?? createPaperclipCloudConnector({ config }); + const commit = async (change: { receipt?: Record; failure?: string; attention?: boolean; retryAt?: number }) => db.transaction(async tx => { + await lease.assertOwned(tx); + const next = change.failure ? { ...intent, attempts: Math.min(intent.attempts + 1, 1_000_000), failureCode: change.failure, + nextAttemptAt: change.attention ? null : new Date(Math.max(Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(intent.attempts, 6)), change.retryAt ?? 0)).toISOString() } : null; + const field = intent.active ? "slackCloudEvents" : "slackCloudRetiredGeneration"; + const enrolled = intent.active ? { version: 1, bindingId: change.receipt?.bindingId, runtimeGeneration: intent.runtimeGeneration, teamId: intent.teamId, appId: intent.appId } : intent.runtimeGeneration; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + const updated = await tx.update(chatEndpoints).set({ + lastError: change.attention ? "Slack Cloud event registration requires attention" : change.receipt + ? sql`case when ${chatEndpoints.lastError} = 'Slack Cloud event registration requires attention' then null else ${chatEndpoints.lastError} end` : sql`${chatEndpoints.lastError}`, + setup: next + ? sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(next)}::jsonb)` + : sql`jsonb_set(jsonb_set(jsonb_set(${chatEndpoints.setup} - 'slackCloudIntent' - 'slackCloudRetiredGeneration', ARRAY[${field}], ${JSON.stringify(enrolled)}::jsonb), '{slackCloudInstanceId}', ${JSON.stringify(intent.instanceId)}::jsonb), '{slackCloudSubject}', ${JSON.stringify(intent.subject)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, endpoint.companyId), + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'id'=${intent.id}`, + sql`coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::integer,0)=${intent.runtimeGeneration}`)).returning({ id: chatEndpoints.id }); + if (updated.length !== 1) throw new Error("Slack configuration changed before event enrollment committed"); + await lease.assertOwned(tx); + }); + try { + await lease.assertOwned(); + if (config.instanceId !== intent.instanceId) { await commit({ failure: "instance_authority_changed", attention: true }); return; } + if (!await connector.slackEventsAvailable()) { await commit({ failure: "unsupported_capability" }); return; } + const signingSecret = intent.active ? await resolveSlackCloudSigningSecret(options.resolveCredentials, endpoint, db, ["signingSecret"]) : undefined; + const receipt = await connector.slackEventEnrollment({ subject: intent.subject, companyId: intent.companyId, + enrollment: { active: intent.active, ...identity, ...(intent.active ? { teamId: intent.teamId, appId: intent.appId, signingSecret: signingSecret! } : {}) } }); + if (!receipt || typeof receipt !== "object" || Array.isArray(receipt)) throw new SlackEnrollmentFailure("receipt_unconfirmed"); + const value = receipt as Record; + if (value.version !== 1 || value.instanceId !== intent.instanceId || Object.entries(identity).some(([key, expected]) => value[key] !== expected) + || (intent.active && (value.teamId !== intent.teamId || value.appId !== intent.appId || typeof value.bindingId !== "string" || !/^slack-event:[a-f0-9]{64}$/.test(value.bindingId)))) + throw new SlackEnrollmentFailure("receipt_identity_changed"); + await commit({ receipt: value }); + } catch (error) { + const conflict = error instanceof PaperclipCloudConnectorError && error.status === 409 + && ["SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_IDENTITY_CHANGED", "SLACK_ENROLLMENT_CHANGED", "SLACK_ENROLLMENT_UNSETTLED"].includes(error.brokerReason ?? ""); + const credentialAttention = error instanceof SlackCloudReceiptError && error.code === "signing_credential_unavailable" && error.disposition === "attention"; + const attention = credentialAttention || error instanceof SlackEnrollmentFailure || conflict || (error instanceof PaperclipCloudConnectorError && [400, 401, 403].includes(error.status ?? 0)); + const code = credentialAttention ? "signing_credential_unavailable" : error instanceof SlackEnrollmentFailure ? error.code : attention ? "authority_required" + : error instanceof PaperclipCloudConnectorError && error.brokerReason === "SLACK_ENROLLMENT_DISABLED" ? "enrollment_disabled" : "broker_unavailable"; + await commit({ failure: code, attention, retryAt: error instanceof PaperclipCloudConnectorError ? error.retryAt : undefined }); + logger.warn({ endpointId, reason: code }, "Slack enrollment responsibility retained"); + } + }; +} diff --git a/server/src/services/chat-channels/slack/cloud-event.test.ts b/server/src/services/chat-channels/slack/cloud-event.test.ts new file mode 100644 index 00000000000..ea9b4a234c5 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-event.test.ts @@ -0,0 +1,110 @@ +import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "../../../__tests__/helpers/slack-cloud/transport.js"; +import { createHmac, createHash } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { verifySlackSignature } from "@chat-adapter/slack/webhook"; +import { createChatSdkEndpointRuntime } from "../../chat-sdk-runtime.js"; +import type { Db } from "@paperclipai/db"; +import type { EndpointRow } from "../runtime-types.js"; +import { createSlackWebhookCallbacks } from "./webhook-callbacks.js"; +import { receiveSlackCloudEvent, slackCloudAcceptedAt, type CurrentSlackCloudBinding, type SlackCloudEvent } from "./cloud-event.js"; + +const acceptedAt = Date.parse("2099-01-01T00:00:00Z"); +function fixture() { + const binding: CurrentSlackCloudBinding = { companyId: "company", endpointId: "endpoint", publicId: "a".repeat(43), + connectionId: "connection", runtimeGeneration: 1, teamId: "TTEST", appId: "ATEST", bindingId: `slack-event:${"b".repeat(64)}`, + signingSecret: "0123456789abcdef0123456789abcdef" }; + const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvFIRST", team_id: "TTEST", api_app_id: "ATEST", event: { type: "app_mention", text: "<@UBOT> hello", user: "UOPERATOR", channel: "CTEST", ts: "1788.500", event_ts: "1788.500" } })); + const timestamp = String(acceptedAt / 1000); + const signature = `v0=${createHmac("sha256", binding.signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}`; + const { signingSecret: _secret, ...identity } = binding; + const event: SlackCloudEvent = { id: `slack_${"b".repeat(64)}_${createHash("sha256").update("EvFIRST").digest("hex")}`, + provider: "slack", event: "event_callback", installationId: binding.bindingId, bindingIds: [binding.bindingId], + createdAt: new Date(acceptedAt).toISOString(), payload: { slackEvent: { version: 1, ...identity, + rawBody: rawBody.toString("base64"), timestamp, signature, eventId: "EvFIRST", acceptedAt: new Date(acceptedAt).toISOString() } } }; + return { binding, event, rawBody, timestamp, signature }; +} + +describe("authenticated Slack Cloud receipt consumer", () => { + it("crosses the real pinned Chat runtime after cold-start delay and preserves ordinary external freshness rejection", async () => { + const network = guardSlackFixtureNetwork(); + const f = fixture(), onMessage = vi.fn(async () => {}); + const runtime = createChatSdkEndpointRuntime({ companyId: f.binding.companyId, endpointId: f.binding.endpointId, + logger: "silent", callbacks: { onMessage }, persistence: { read: async () => null, + compareAndSet: async () => true, deleteIfVersion: async () => true }, + providerConfig: { provider: "slack", userName: "paperclip-agent", + credentials: { apiUrl: "http://127.0.0.1:1/api/", botToken: "xoxb-synthetic", botUserId: "UBOT", signingSecret: f.binding.signingSecret } } }); + try { + await runtime.initialize(); + const transport = stubSlackFixtureTransport(runtime.getProviderAdapter()); + await receiveSlackCloudEvent(f.event, f.binding, request => runtime.handleWebhook(request), acceptedAt + 600_000); + expect(onMessage).toHaveBeenCalledOnce(); + expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ message: expect.objectContaining({ author: expect.objectContaining({ userId: "UOPERATOR", userName: "Operator" }) }) })); + expect(transport).toHaveBeenCalled(); + await transport.assertNoErrors(); + expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ provider: "slack", endpointId: f.binding.endpointId })); + const response = await runtime.handleWebhook(new Request("https://external.invalid", { method: "POST", + headers: { "content-type": "application/json", "x-slack-request-timestamp": f.timestamp, "x-slack-signature": f.signature }, + body: new Uint8Array(f.rawBody) })); + expect(response.status).toBe(401); + expect(onMessage).toHaveBeenCalledOnce(); + } finally { + await runtime.shutdown(); + try { network.assertQuiet(); } finally { network.restore(); } + } + }); + it("replays original evidence after a cold-start delay without refreshing its timestamp", async () => { + const f = fixture(); + let delivered: Request | undefined; + await receiveSlackCloudEvent(f.event, f.binding, async request => { + delivered = request; + expect(slackCloudAcceptedAt(request)).toBe(acceptedAt); + expect(request.headers.get("x-slack-request-timestamp")).toBe(f.timestamp); + expect(request.headers.get("x-slack-signature")).toBe(f.signature); + const body = await request.text(); + expect(body).toBe(f.rawBody.toString()); + + await verifySlackSignature(body, request.headers, { signingSecret: f.binding.signingSecret, + now: () => slackCloudAcceptedAt(request) ?? acceptedAt + 600_000 }); + await expect(verifySlackSignature(body, request.headers, { signingSecret: f.binding.signingSecret, + now: () => acceptedAt + 600_000 })).rejects.toThrow("too old"); + return new Response("durably accepted"); + }, acceptedAt + 600_000); + expect(slackCloudAcceptedAt(delivered!)).toBeUndefined(); + expect(slackCloudAcceptedAt(new Request("https://external.invalid"))).toBeUndefined(); + }); + + it("ignores removed or superseded generations without invoking product admission", async () => { + const f = fixture(), handler = vi.fn(); + await receiveSlackCloudEvent(f.event, null, handler, acceptedAt); + await receiveSlackCloudEvent(f.event, { ...f.binding, runtimeGeneration: 2 }, handler, acceptedAt); + expect(handler).not.toHaveBeenCalled(); + }); + + it("retains current work on signature/body/receipt-time failure and on failed durable admission", async () => { + const f = fixture(), handler = vi.fn(async () => new Response(null, { status: 503 })); + await expect(receiveSlackCloudEvent(f.event, f.binding, handler, acceptedAt)).rejects.toThrow("callback_unaccepted"); + await expect(receiveSlackCloudEvent(f.event, { ...f.binding, signingSecret: "rotated" }, handler, acceptedAt)).rejects.toThrow("original_authentication_failed"); + await expect(receiveSlackCloudEvent({ ...f.event, createdAt: new Date(acceptedAt + 600_000).toISOString() }, f.binding, handler, acceptedAt)).rejects.toThrow("invalid_acceptance_time"); + await expect(receiveSlackCloudEvent(f.event, f.binding, handler, acceptedAt + 31 * 24 * 60 * 60_000)).rejects.toThrow("invalid_acceptance_time"); + expect(handler).toHaveBeenCalledTimes(1); + }); +}); + +it("private replay preserves canonical Slack callback evidence while ordinary external observation remains active", async () => { + const f = fixture(); + const transaction = vi.fn(async () => { throw new Error("external callback observation"); }); + const db = { transaction } as unknown as Db; + const callbacks = createSlackWebhookCallbacks(db, { resolveCredentials: async () => ({}), runtimeCallbackEndpoint: async () => null, + getWebhookPublicBaseUrl: () => "https://canonical.example", scheduleMessageProcessing: () => {}, slackRegistration: { notifyVerified: async () => {} } }); + const endpoint = { setup: { slackCallbackSurfaces: { events: { url: "https://canonical.example/slack", observedAt: "2099-01-01T00:00:00Z" } } } } as unknown as EndpointRow; + const before = structuredClone(endpoint.setup); + const context = { credentialFingerprint: "fixture", generation: 1, localEpoch: 1, version: "fixture" }; + await receiveSlackCloudEvent(f.event, f.binding, async request => { + await callbacks.recordSlackWebhookCallback(endpoint, request, request.clone(), context); + return new Response(null, { status: 200 }); + }, acceptedAt); + expect(endpoint.setup).toEqual(before); expect(transaction).not.toHaveBeenCalled(); + const external = new Request("https://canonical.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: new Uint8Array(f.rawBody) }); + await expect(callbacks.recordSlackWebhookCallback(endpoint, external, external.clone(), context)).rejects.toThrow("external callback observation"); + expect(transaction).toHaveBeenCalledOnce(); +}); diff --git a/server/src/services/chat-channels/slack/cloud-event.ts b/server/src/services/chat-channels/slack/cloud-event.ts new file mode 100644 index 00000000000..b589de638bc --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-event.ts @@ -0,0 +1,77 @@ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; + +const acceptedTimes = new WeakMap(); +export const slackCloudAcceptedAt = (request: Request): number | undefined => acceptedTimes.get(request); + +export class SlackCloudReceiptError extends Error { + constructor(readonly code: string, readonly disposition: "invalid" | "attention" | "retry") { super(code); } +} + +export interface SlackCloudEvent { + id: string; + provider: string; + event: string; + installationId: string | null; + bindingIds: string[]; + createdAt: string; + payload: Record; +} + +export interface CurrentSlackCloudBinding { + companyId: string; + endpointId: string; + publicId: string; + connectionId: string; + runtimeGeneration: number; + teamId: string; + appId: string; + bindingId: string; + signingSecret: string; +} + +export async function receiveSlackCloudEvent(event: SlackCloudEvent, current: CurrentSlackCloudBinding | null, + handleWebhook: (request: Request) => Promise, now = Date.now()): Promise { + if (!current) return; + const value = event.payload.slackEvent; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const packet = value as Record; + if (event.provider !== "slack" || event.event !== "event_callback" || packet.version !== 1) + throw new SlackCloudReceiptError("unsupported_receipt", "attention"); + if (packet.bindingId !== current.bindingId || packet.companyId !== current.companyId + || packet.endpointId !== current.endpointId || packet.publicId !== current.publicId + || packet.connectionId !== current.connectionId || packet.runtimeGeneration !== current.runtimeGeneration + || packet.teamId !== current.teamId || packet.appId !== current.appId) return; + if (event.installationId !== current.bindingId || !event.bindingIds.includes(current.bindingId) + || typeof packet.rawBody !== "string" || packet.rawBody.length > 1_398_104 + || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(packet.rawBody) + || typeof packet.timestamp !== "string" || !/^[0-9]{1,16}$/.test(packet.timestamp) + || typeof packet.signature !== "string" || !/^v0=[a-f0-9]{64}$/.test(packet.signature)) + throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const acceptedAt = Date.parse(event.createdAt); + + if (packet.acceptedAt !== event.createdAt || !Number.isFinite(acceptedAt) || acceptedAt > now + 300_000 || now - acceptedAt >= 30 * 24 * 60 * 60_000 + || Math.abs(Math.floor(acceptedAt / 1000) - Number(packet.timestamp)) > 300) + throw new SlackCloudReceiptError("invalid_acceptance_time", "attention"); + const body = Buffer.from(packet.rawBody, "base64"); + if (body.byteLength > 1_048_576) throw new SlackCloudReceiptError("receipt_too_large", "invalid"); + const signature = createHmac("sha256", current.signingSecret).update(`v0:${packet.timestamp}:`).update(body).digest(); + if (!timingSafeEqual(signature, Buffer.from(packet.signature.slice(3), "hex"))) + throw new SlackCloudReceiptError("original_authentication_failed", "attention"); + let payload: unknown; + try { payload = JSON.parse(body.toString("utf8")); } catch { throw new SlackCloudReceiptError("invalid_event", "invalid"); } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) throw new SlackCloudReceiptError("invalid_event", "invalid"); + const envelope = payload as Record; + if (envelope.type !== "event_callback" || envelope.team_id !== current.teamId || envelope.api_app_id !== current.appId + || typeof envelope.event_id !== "string" || !/^[A-Za-z0-9_-]{1,200}$/.test(envelope.event_id) || envelope.event_id !== packet.eventId + || event.id !== `slack_${current.bindingId.slice("slack-event:".length)}_${createHash("sha256").update(envelope.event_id).digest("hex")}`) + throw new SlackCloudReceiptError("event_identity_changed", "invalid"); + const request = new Request("https://paperclip.invalid/slack-cloud-ingress", { method: "POST", + headers: { "content-type": "application/json", "x-slack-request-timestamp": packet.timestamp, + "x-slack-signature": packet.signature }, body: new Uint8Array(body) }); + acceptedTimes.set(request, acceptedAt); + try { + const response = await handleWebhook(request); + await response.body?.cancel(); + if (!response.ok) throw new SlackCloudReceiptError("callback_unaccepted", "retry"); + } finally { acceptedTimes.delete(request); } +} diff --git a/server/src/services/chat-channels/slack/cloud-ingress.test.ts b/server/src/services/chat-channels/slack/cloud-ingress.test.ts new file mode 100644 index 00000000000..f0688ba5bc8 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-ingress.test.ts @@ -0,0 +1,47 @@ +import { HttpError } from "../../../errors.js"; +import type { Db } from "@paperclipai/db"; +import { expect, it, vi } from "vitest"; +import { createSlackCloudEventIngress, dispatchSlackCloudEvent, registerSlackCloudEventIngress } from "./cloud-ingress.js"; +import type { SlackCloudEvent } from "./cloud-event.js"; + +it("missing or failed runtime owner retains the leased event; retired owner cleanup cannot remove its replacement", async () => { + const db = {} as Db; + const event: SlackCloudEvent = { id: "receipt", provider: "slack", event: "event_callback", installationId: "binding", + bindingIds: ["binding"], createdAt: "2099-01-01T00:00:00Z", payload: {} }; + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready"); + const stopped = vi.fn(async () => { throw new Error("admission stopped"); }); + const unregisterOld = registerSlackCloudEventIngress(db, stopped); + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("admission stopped"); + const replacement = vi.fn(async () => {}); + const unregisterNew = registerSlackCloudEventIngress(db, replacement); + unregisterOld(); + await dispatchSlackCloudEvent(db, event); + expect(replacement).toHaveBeenCalledWith(event); + unregisterNew(); + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready"); +}); + +it.each([404, 422])("current ingress keeps credential failure %s as unacknowledged attention before callback admission", async status => { + const endpoint = { id: "00000000-0000-4000-8000-000000000001", companyId: "company", publicId: "public", connectionId: "connection", + provider: "slack", status: "active", providerAccountId: "TTEST", setup: { slackCloudEvents: { + version: 1, bindingId: "binding", runtimeGeneration: 1, teamId: "TTEST", appId: "ATEST" } } }; + const db = { select: () => { + let joined = false; + const query = { from: () => query, where: () => query, innerJoin: () => { joined = true; return query; }, + limit: async () => joined ? [{ endpoint, connection: { enabled: true, status: "active" } }] : [endpoint] }; + return query; + } } as unknown as Db; + const handleWebhook = vi.fn(async () => new Response()); + const withCredentialMutationLease: Parameters[1]["withCredentialMutationLease"] = + async (_endpoint, work) => work({ assertOwned: async () => {} } as Parameters[0]); + const unregister = createSlackCloudEventIngress(db, { withCredentialMutationLease, runtimeGeneration: () => 1, handleWebhook, + resolveCredentials: async () => { throw new HttpError(status, "synthetic credential failure"); } }); + try { + await expect(dispatchSlackCloudEvent(db, { id: "receipt", provider: "slack", event: "event_callback", installationId: "binding", + bindingIds: ["binding"], createdAt: new Date().toISOString(), payload: { slackEvent: { + endpointId: endpoint.id, companyId: endpoint.companyId, publicId: endpoint.publicId, + connectionId: endpoint.connectionId, runtimeGeneration: 1 } } })) + .rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" }); + expect(handleWebhook).not.toHaveBeenCalled(); + } finally { unregister(); } +}); diff --git a/server/src/services/chat-channels/slack/cloud-ingress.ts b/server/src/services/chat-channels/slack/cloud-ingress.ts new file mode 100644 index 00000000000..a4368552abc --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-ingress.ts @@ -0,0 +1,63 @@ +import type { Db } from "@paperclipai/db"; +import { chatEndpoints, toolConnections } from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import type { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; +import type { createChatCredentials } from "../credentials.js"; +import type { EndpointRow, InternalSetupState } from "../runtime-types.js"; +import { SlackCloudReceiptError, receiveSlackCloudEvent, type CurrentSlackCloudBinding, type SlackCloudEvent } from "./cloud-event.js"; + +import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js"; + +type Handler = (event: SlackCloudEvent) => Promise; +const handlers = new WeakMap(); + +export function registerSlackCloudEventIngress(db: Db, handler: Handler) { + handlers.set(db, handler); + return () => { if (handlers.get(db) === handler) handlers.delete(db); }; +} + +export async function dispatchSlackCloudEvent(db: Db, event: SlackCloudEvent): Promise { + const handler = handlers.get(db); + if (!handler) throw new Error("Slack Cloud ingress is not ready"); + await handler(event); +} + +export function createSlackCloudEventIngress(db: Db, options: { + withCredentialMutationLease: ReturnType; + resolveCredentials: ReturnType["resolveCredentials"]; + runtimeGeneration(endpoint: EndpointRow): number; + handleWebhook(publicId: string, provider: "slack", request: Request, expectedGeneration: number): Promise; +}) { + return registerSlackCloudEventIngress(db, async event => { + const value = event.payload.slackEvent; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const packet = value as Record; + if (typeof packet.endpointId !== "string" || !/^[a-f0-9-]{36}$/i.test(packet.endpointId)) + throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const [initial] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, packet.endpointId)).limit(1); + if (!initial || initial.provider !== "slack" || ["archived", "paused", "revoked"].includes(initial.status)) return; + const binding = await options.withCredentialMutationLease(initial, async (lease): Promise => { + const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId))) + .where(eq(chatEndpoints.id, initial.id)).limit(1); + if (!current || current.endpoint.provider !== "slack" || ["archived", "paused", "revoked"].includes(current.endpoint.status) + || current.connection.status !== "active" || !current.connection.enabled) return null; + const endpoint = current.endpoint; + if (packet.companyId !== endpoint.companyId || packet.publicId !== endpoint.publicId + || packet.connectionId !== endpoint.connectionId || packet.runtimeGeneration !== options.runtimeGeneration(endpoint)) return null; + const enrolled = (endpoint.setup as InternalSetupState).slackCloudEvents; + + if (!enrolled) throw new Error("Slack Cloud enrollment is not locally committed"); + if (enrolled.version !== 1 || enrolled.runtimeGeneration !== options.runtimeGeneration(endpoint) + || enrolled.teamId !== endpoint.providerAccountId) throw new Error("Slack Cloud enrollment is unsettled"); + const signingSecret = await resolveSlackCloudSigningSecret(options.resolveCredentials, endpoint, db, ["signingSecret"]); + await lease.assertOwned(); + return { companyId: endpoint.companyId, endpointId: endpoint.id, + publicId: endpoint.publicId, connectionId: endpoint.connectionId, runtimeGeneration: options.runtimeGeneration(endpoint), + bindingId: enrolled.bindingId, teamId: enrolled.teamId, appId: enrolled.appId, signingSecret }; + }); + if (!binding) return; + await receiveSlackCloudEvent(event, binding, + request => options.handleWebhook(binding.publicId, "slack", request, binding.runtimeGeneration)); + }); +} diff --git a/server/src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts b/server/src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts new file mode 100644 index 00000000000..9ff84fc43c5 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts @@ -0,0 +1,202 @@ +import { createHmac, generateKeyPairSync, randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { eq } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { SLACK_CHAT_BOT_SCOPES, type ToolCredentialSecretRef } from "@paperclipai/shared"; +import { activityLog, agents, applyPendingMigrations, authUsers, chatEndpoints, chatSlackRegistrations, closeCheckpointDb, closeFileStorageDb, closeLivenessDb, companies, companyMemberships, companySecrets, createDb, toolApplications, toolConnections } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "../../../lib/dedicated-test-database-url.js"; +import { heartbeatService } from "../../heartbeat.js"; +import { chatChannelService } from "../../chat-channels.js"; +import { createDeliveryWorkCoordinator } from "../../delivery-work-coordinator.js"; +import { githubConnectionEventService } from "../../github-connection-events.js"; +import { registerGitHubConnectionEventWork } from "../../github-connection-work.js"; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from "../../paperclip-cloud-connector.js"; +import { secretService } from "../../secrets.js"; +import type { InternalSetupState } from "../runtime-types.js"; +import { guardSlackFixtureNetwork } from "../../../__tests__/helpers/slack-cloud/transport.js"; + +const appUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_TEST_DATABASE_URL"); +const cloudUrl = resolveExternalTestDatabaseUrl("SLACK_EVENT_TEST_DATABASE_URL"); +const cloudDist = process.env.SLACK_TEST_CLOUD_DIST; + +it.skipIf(!appUrl || !cloudUrl || !cloudDist)("normal notifications enroll the first consumer and authorized reconfigure replaces only positively retired instance custody", async () => { + const network = guardSlackFixtureNetwork({ requirePreload: true }); + const load = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, file)).href); + const [{ PostgresCloudHarnessRegistry }, { PostgresConnectorStore }, { ConnectorBroker }, { InMemoryProviderSecretStore }, + { providerRegistryFromEnvironment }, { CloudStackProvisioner }, { FakeCloudStackProvider }, { Pool }] = await Promise.all([ + load("registry/postgres.js"), load("connections/postgres.js"), load("connections/broker.js"), load("providers/neon.js"), + load("connections/providers.js"), load("provisioner/provisioner.js"), load("providers/fake.js"), + import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, "../node_modules/pg/lib/index.js")).href), + ]); + await applyPendingMigrations(appUrl!); + const db = createDb(appUrl!, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + const pool = new Pool({ connectionString: cloudUrl!, max: 2 }); + const registry = Reflect.construct(PostgresCloudHarnessRegistry, [pool]) as { + applyMigrations(folder: string, timeout: number): Promise; hydrate(): Promise; close(): Promise; + }; + try { await registry.applyMigrations(path.resolve(cloudDist!, "../migrations"), 30_000); await registry.hydrate(); } + catch (error) { await db.$client.end({ timeout: 1 }); await registry.close(); throw error; } + const store = new PostgresConnectorStore(pool), secrets = new InMemoryProviderSecretStore(); + const id = randomUUID(), companyId = randomUUID(), endpointId = randomUUID(), connectionId = randomUUID(), applicationId = randomUUID(), agentId = randomUUID(); + const actor = `owner-${id}`, instanceA = `a-${id}`, instanceB = `b-${id}`, publicId = "h".repeat(43); + const stackIds = [instanceA, instanceB], signingSecret = "0123456789abcdef0123456789abcdef"; + const keys = stackIds.map(() => ({ sign: generateKeyPairSync("ed25519"), seal: generateKeyPairSync("x25519") })); + let broker: InstanceType, base = "", loseEnrollmentResponse = false; + const requests: { path: string; issuer?: string }[] = []; + const server = createServer((req, res) => { + requests.push({ path: req.url! }); + if (loseEnrollmentResponse && req.url === "/v1/connector/slack-events") { + loseEnrollmentResponse = false; + res.end = (() => { res.destroy(); return res; }) as typeof res.end; + } + void broker.handle(req, res, new URL(req.url!, base)).catch((error: unknown) => { + if (!res.headersSent) res.writeHead(500); res.end(String(error)); + }); + }); + const directory = await mkdtemp(path.join(os.tmpdir(), "slack-transition-")); + let service: ReturnType | undefined; + let heartbeat: ReturnType | undefined; + let coordinator: ReturnType | undefined; + let worker: ReturnType | undefined; + const errors: unknown[] = []; + const deliveryIds: string[] = []; + const endpoint = async () => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)))[0]!; + const setup = async () => (await endpoint()).setup as InternalSetupState; + const configureIdentity = (index: number) => { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", stackIds[index]!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", keys[index]!.sign.privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", keys[index]!.seal.privateKey.export({ format: "jwk" }).d!); + }; + try { + for (const [index, stackId] of stackIds.entries()) { + await new CloudStackProvisioner({ provider: new FakeCloudStackProvider(), registry }).provisionDevelopmentStack({ + idempotencyKey: stackId, accountGroup: { id: `account-${stackId}`, slug: stackId, displayName: "Transition" }, + owner: { id: actor, email: `${id}@example.test` }, + stack: { stackId, accountGroupId: `account-${stackId}`, stackSlug: stackId, displayName: "Transition", kind: "demo", primaryHost: `${stackId}.example.test`, region: "us-east-1" }, + versionPlan: { appVersion: "fixture", schemaVersion: "fixture", rolloutTrack: "stable" }, + sleepPolicy: { desiredState: "awake" }, primaryCompany: { paperclipCompanyId: index === 0 ? companyId : randomUUID(), displayName: "Transition", defaultBudgetCents: 0 }, + }); + await store.putCloudStackInstance({ id: stackId, kind: "cloud_stack", companyStackId: stackId, ownerUserId: actor, environment: "test", + signPublicKey: keys[index]!.sign.publicKey.export({ format: "jwk" }).x!, sealPublicKey: keys[index]!.seal.publicKey.export({ format: "jwk" }).x!, + origins: ["https://tenant.example"], activeOrigin: "https://tenant.example", label: "Transition", status: "active", createdAt: new Date(), updatedAt: new Date() }); + } + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing fixture address"); + base = `http://127.0.0.1:${address.port}`; + broker = new ConnectorBroker({ publicOrigin: base, environment: "test", store, + providers: providerRegistryFromEnvironment({}, "test"), secrets: { resolve: async () => undefined }, + authenticate: async () => ({ globalUserId: actor }), slackEvents: { secrets, secretRefPrefix: "secret:///test/transition" }, + eventWakeEnrollmentEnabled: true, eventWakeDeliveryEnabled: true, wakeCloudStack: async () => {} }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(directory, "master.key")); configureIdentity(0); + await db.insert(companies).values({ id: companyId, name: "Transition", issuePrefix: `T${id.slice(0, 7)}` }); + await db.insert(authUsers).values({ id: actor, name: "Owner", email: `${id}@owner.example.test`, emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actor, status: "active", membershipRole: "operator" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Transition", adapterType: "process", status: "idle" }); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Transition", type: "chat" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Transition", uid: id, + transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId, + provider: "slack", status: "paused", providerAccountId: "TTEST", botExternalId: "UBOT", setup: { runtimeGeneration: 0, step: "complete" } as InternalSetupState }); + await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", appId: "ATEST", + manifest: {}, manifestHash: "fixture", callbackUri: "https://tenant.example/callback", createdByUserId: actor }); + const vault = secretService(db), refs: ToolCredentialSecretRef[] = []; + for (const [key, value] of Object.entries({ botToken: "xoxb-fixture", signingSecret })) { + const secret = await vault.create(companyId, { name: `${key}-${id}`, provider: "local_encrypted", value }, { userId: actor }); + await vault.createBinding({ companyId, secretId: secret.id, targetType: "tool_connection", targetId: connectionId, configPath: `credentials.${key}` }); + refs.push({ secretId: secret.id, versionSelector: "latest", configPath: `credentials.${key}`, required: true, projectionClass: "unclassified" }); + } + await db.update(toolConnections).set({ credentialSecretRefs: refs }).where(eq(toolConnections.id, connectionId)); + const fetchImpl: typeof fetch = async input => { + const url = input instanceof Request ? new URL(input.url) : new URL(String(input)); + if (url.href === "https://slack.com/api/auth.test") return Response.json({ ok: true, team_id: "TTEST", user_id: "UBOT", user: "fixture" }, { headers: { "x-oauth-scopes": SLACK_CHAT_BOT_SCOPES.join(",") } }); + if (url.origin === "https://slack.com" && url.pathname === "/api/conversations.list") return Response.json({ ok: true, channels: [], response_metadata: {} }); + throw new Error(`Unexpected transition provider request ${url}`); + }; + heartbeat = heartbeatService(db); + service = chatChannelService(db, { heartbeat, fetch: fetchImpl, publicBaseUrl: "https://tenant.example" }); + const events = githubConnectionEventService(db); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + worker = registerGitHubConnectionEventWork(db, coordinator, events, () => true); await worker.ready; + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull()); + const leaseCount = () => requests.filter(value => value.path === "/v1/connector/events/lease").length; + expect(leaseCount()).toBe(0); + await service.configure(endpointId, { action: "resume" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudEvents?.runtimeGeneration).toBe(1), { timeout: 10_000 }); + await vi.waitFor(() => expect(leaseCount()).toBeGreaterThan(0), { timeout: 10_000 }); + expect((await store.getInstance(instanceA)).eventWakeVersion).toBe(1); + expect((await setup()).slackCloudInstanceId).toBe(instanceA); + await service.configure(endpointId, { action: "pause" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(2), { timeout: 10_000 }); + expect(await broker.slackEvents.current(instanceA, publicId)).toBeUndefined(); + expect((await store.listWebhookBindings(instanceA)).every((binding: { receivesEvents: boolean }) => !binding.receivesEvents)).toBe(true); + const retired = await setup(); configureIdentity(1); + const foreign = createPaperclipCloudConnector({ config: paperclipCloudConnectorConfigFromEnv()! }); + await expect(foreign.slackEventEnrollment({ subject: `foreign-${actor}`, companyId, enrollment: { + active: true, companyId, endpointId, publicId, connectionId, runtimeGeneration: 3, teamId: "TTEST", appId: "ATEST", signingSecret, + } })).rejects.toMatchObject({ status: 403 }); + expect(await broker.slackEvents.current(instanceB, publicId)).toBeUndefined(); + expect((await setup()).slackCloudIntent).toBeUndefined(); + expect((await setup()).slackCloudInstanceId).toBe(instanceA); + expect((await setup()).slackCloudSubject).toBe(actor); + await service.configure(endpointId, { action: "resume" }, actor); + const pending = await setup(); + expect(pending.slackCloudIntent).toMatchObject({ instanceId: instanceB, runtimeGeneration: 3, subject: actor }); + expect(pending.slackCloudInstanceId).toBe(instanceA); + expect(pending.slackCloudEvents).toEqual(retired.slackCloudEvents); + await vi.waitFor(async () => expect((await setup()).slackCloudInstanceId).toBe(instanceB), { timeout: 10_000 }); + expect((await setup()).slackCloudIntent).toBeUndefined(); + expect((await setup()).slackCloudRetiredGeneration).toBeUndefined(); + expect((await broker.slackEvents.current(instanceB, publicId)).runtimeGeneration).toBe(3); + expect(await broker.slackEvents.current(instanceA, publicId)).toBeUndefined(); + await service.configure(endpointId, { action: "pause" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(4), { timeout: 10_000 }); + loseEnrollmentResponse = true; + await service.configure(endpointId, { action: "resume" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudIntent).toMatchObject({ instanceId: instanceB, + runtimeGeneration: 5, failureCode: "broker_unavailable" }), { timeout: 10_000 }); + expect((await broker.slackEvents.current(instanceB, publicId)).runtimeGeneration).toBe(5); + expect((await setup()).slackCloudEvents?.runtimeGeneration).toBe(3); + const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: `EvLostReplacement-${id}`, team_id: "TTEST", api_app_id: "ATEST", + event: { type: "app_mention", text: "removed before admission", user: "UOPERATOR", channel: "CTEST", ts: "1788.600" } })); + const timestamp = String(Math.floor(Date.now() / 1000)); + await broker.slackEvents.receive(await broker.slackEvents.current(instanceB, publicId), { rawBody, timestamp, + signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}` }); + deliveryIds.push(...(await pool.query("SELECT delivery_id FROM cloud_harness.connector_webhook_delivery_targets WHERE connector_instance_id=$1", [instanceB])).rows.map((row: { delivery_id: string }) => row.delivery_id)); + await service.configure(endpointId, { action: "remove" }, actor); + expect((await db.select().from(toolConnections).where(eq(toolConnections.id, connectionId)))[0]!.credentialSecretRefs).toEqual([]); + await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(6), { timeout: 10_000 }); + expect((await setup()).slackCloudIntent).toBeUndefined(); + expect((await setup()).slackCloudInstanceId).toBe(instanceB); + expect(await broker.slackEvents.current(instanceB, publicId)).toBeUndefined(); + expect((await pool.query("SELECT delivery_id FROM cloud_harness.connector_webhook_delivery_targets WHERE connector_instance_id=$1", [instanceB])).rows).toEqual([]); + await expect(broker.slackEvents.enroll(await store.getInstance(instanceB), actor, { companyId, endpointId, publicId, connectionId, + runtimeGeneration: 5, teamId: "TTEST", appId: "ATEST", signingSecret })).rejects.toThrow("GENERATION_CHANGED"); + expect(errors).toEqual([]); + } catch (error) { console.error("Slack instance transition body failure", error); throw error; } + finally { + await worker?.stop(); await coordinator?.stop(); await service?.shutdown(); await heartbeat?.drainActiveRunExecutions(); + if (server.listening) { server.close(); await once(server, "close"); } + vi.unstubAllEnvs(); + try { + await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId)); + await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + await db.delete(authUsers).where(eq(authUsers.id, actor)); + await pool.query("DELETE FROM cloud_harness.connector_instances WHERE id=ANY($1::text[])", [stackIds]); + await pool.query("DELETE FROM cloud_harness.connector_webhook_deliveries WHERE id=ANY($1::text[])", [deliveryIds]); + } finally { + await closeCheckpointDb(db, 1); await closeFileStorageDb(db, 1); await closeLivenessDb(db, 1); + await db.$client.end({ timeout: 1 }); await registry.close(); await rm(directory, { recursive: true, force: true }); + try { network.assertQuiet(); } finally { network.restore(); } + } + } +}, 60_000); diff --git a/server/src/services/chat-channels/slack/cloud-signing-secret.test.ts b/server/src/services/chat-channels/slack/cloud-signing-secret.test.ts new file mode 100644 index 00000000000..efb331ba2e1 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-signing-secret.test.ts @@ -0,0 +1,19 @@ +import type { Db } from "@paperclipai/db"; +import { expect, it, vi } from "vitest"; +import { HttpError } from "../../../errors.js"; +import type { EndpointRow } from "../runtime-types.js"; +import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js"; + +it.each([400, 401, 403, 404, 422])("credential resolution %s cannot authenticate or acknowledge a receipt", async status => { + const resolve = vi.fn(async (): Promise> => { throw new HttpError(status, "synthetic"); }); + await expect(resolveSlackCloudSigningSecret(resolve, {} as EndpointRow, {} as Db, ["signingSecret"])) + .rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" }); +}); + +it("empty credential is attention, while transient failures retain their retry classification", async () => { + await expect(resolveSlackCloudSigningSecret(async () => ({}), {} as EndpointRow)) + .rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" }); + const transient = new Error("database temporarily unavailable"); + await expect(resolveSlackCloudSigningSecret(async () => { throw transient; }, {} as EndpointRow)).rejects.toBe(transient); + await expect(resolveSlackCloudSigningSecret(async () => ({ signingSecret: "synthetic" }), {} as EndpointRow)).resolves.toBe("synthetic"); +}); diff --git a/server/src/services/chat-channels/slack/cloud-signing-secret.ts b/server/src/services/chat-channels/slack/cloud-signing-secret.ts new file mode 100644 index 00000000000..96e0cdd27d0 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-signing-secret.ts @@ -0,0 +1,18 @@ +import { HttpError } from "../../../errors.js"; +import type { createChatCredentials } from "../credentials.js"; +import { SlackCloudReceiptError } from "./cloud-event.js"; + +type Resolver = ReturnType["resolveCredentials"]; + +export async function resolveSlackCloudSigningSecret(resolve: Resolver, ...args: Parameters): Promise { + let credentials: Record; + try { + credentials = await resolve(...args); + } catch (error) { + if (error instanceof HttpError && [400, 401, 403, 404, 422].includes(error.status)) + throw new SlackCloudReceiptError("signing_credential_unavailable", "attention"); + throw error; + } + if (!credentials.signingSecret) throw new SlackCloudReceiptError("signing_credential_unavailable", "attention"); + return credentials.signingSecret; +} diff --git a/server/src/services/chat-channels/slack/cloud-test-prewire.test.ts b/server/src/services/chat-channels/slack/cloud-test-prewire.test.ts new file mode 100644 index 00000000000..8e69ceacfd9 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-test-prewire.test.ts @@ -0,0 +1,57 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; + +const guard = fileURLToPath(new URL("../../../__tests__/helpers/slack-cloud/prewire.cjs", import.meta.url)); +const launcher = fileURLToPath(new URL("../../../../scripts/test-slack-cloud-postgres.mjs", import.meta.url)); +const cleanEnv = () => ({ PATH: process.env.PATH, HOME: process.env.HOME, NODE_OPTIONS: "" }); + +it.each([ + "require('node:http').get('http://example.invalid')", + "require('node:https').get('https://example.invalid')", + "require('node:net').connect(443,'example.invalid')", + "require('node:tls').connect(443,'example.invalid')", + "fetch('https://example.invalid')", + "require('node:net').connect(55448,'127.0.0.1')", + "require('node:http').get('http://127.0.0.1:1/')", +])("the checked-in guard fails caught unowned transport attempts: %s", operation => { + const child = spawnSync(process.execPath, ["--require", guard, "-e", + `let dnsCalls=0; require('node:dns').lookup=()=>{dnsCalls++;throw Error('DNS must not run');}; try { ${operation}; } catch {} console.log(JSON.stringify({...globalThis[Symbol.for('paperclip.slack.fixture.prewire')].stats(),dnsCalls}));`], + { env: cleanEnv(), encoding: "utf8", timeout: 5_000 }); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(98); + const stats = JSON.parse(child.stdout.trim()); + expect(stats.pid).toBe(child.pid); + expect(stats.unexpectedAttempts).toHaveLength(1); + expect(stats.peakPgSockets).toBe(0); + expect(stats.dnsCalls).toBe(0); +}); + +it("the same guard allows an actual owned loopback HTTP server and initializes before the child script", () => { + const child = spawnSync(process.execPath, ["--require", guard, "-e", ` + const http=require('node:http'); + if(process.env.SLACK_GATE_GUARD_PID!==String(process.pid))throw Error('preload missing'); + const server=http.createServer((req,res)=>res.end('owned response')); + server.listen(0,'127.0.0.1',()=>http.get('http://127.0.0.1:'+server.address().port,response=>{ + let body='';response.on('data',chunk=>body+=chunk);response.on('end',()=>server.close(()=>{ + if(body!=='owned response')throw Error('response missing'); + console.log(JSON.stringify(globalThis[Symbol.for('paperclip.slack.fixture.prewire')].stats())); + })); + }));`], { env: cleanEnv(), encoding: "utf8", timeout: 5_000 }); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(0); + expect(JSON.parse(child.stdout.trim())).toMatchObject({ pid: child.pid, peakPgSockets: 0, unexpectedAttempts: [] }); +}); + +it.each([ + {}, + { PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@127.0.0.1/live", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/cloud_test" }, + { PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@127.0.0.1/same_test", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/same_test" }, + { PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@example.invalid/app_test", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/cloud_test" }, +])("the repository launcher rejects unsafe or absent fixture settings before starting Vitest", settings => { + const child = spawnSync(process.execPath, [launcher], { env: { ...cleanEnv(), ...settings }, encoding: "utf8", timeout: 5_000 }); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(1); + expect(child.stderr).toMatch(/test database|loopback|distinct/); + expect(child.stdout).not.toContain("RUN"); +}); diff --git a/server/src/services/chat-channels/slack/webhook-callbacks.ts b/server/src/services/chat-channels/slack/webhook-callbacks.ts index f0fb377c371..e4235c1dee8 100644 --- a/server/src/services/chat-channels/slack/webhook-callbacks.ts +++ b/server/src/services/chat-channels/slack/webhook-callbacks.ts @@ -1,3 +1,4 @@ +import { slackCloudAcceptedAt } from "./cloud-event.js"; import { isProvisionedSlackSetup } from "@paperclipai/shared"; import { slackRequestSignatureIsValid, slackRequestWorkspaceId, slackCallbackMatchesPublicUrl, inspectSlackCallback } from "./callback-verification.js"; import type { InternalSetupState } from "../runtime-types.js"; @@ -53,6 +54,7 @@ export function createSlackWebhookCallbacks(db: Db, options: SlackWebhookOptions return null; } async function recordSlackWebhookCallback(endpoint: EndpointRow, request: Request, slackCallbackInspection: Request, runtimeContext: RuntimeContext): Promise { + if (slackCloudAcceptedAt(request) !== undefined) return; const eventBody = await slackCallbackInspection.clone().text(); const callback = await inspectSlackCallback(slackCallbackInspection); if (callback) { diff --git a/server/src/services/chat-sdk-runtime.ts b/server/src/services/chat-sdk-runtime.ts index 6afd8fb844c..602b77fff2d 100644 --- a/server/src/services/chat-sdk-runtime.ts +++ b/server/src/services/chat-sdk-runtime.ts @@ -17,6 +17,8 @@ import { } from "@chat-adapter/discord"; import type { SlackAdapterConfig } from "@chat-adapter/slack"; import { PaperclipSlackAdapter } from "./chat-channels/slack/adapter.js"; +import { verifySlackSignature } from "@chat-adapter/slack/webhook"; +import { slackCloudAcceptedAt } from "./chat-channels/slack/cloud-event.js"; import { createTeamsAdapter, type TeamsAdapter, @@ -1361,6 +1363,13 @@ function createProviderAdapter( case "slack": { const adapterConfig: SlackAdapterConfig = { ...config.credentials, + webhookVerifier: async (request, body) => { + await verifySlackSignature(body, request.headers, { + signingSecret: config.credentials.signingSecret, + now: () => slackCloudAcceptedAt(request) ?? Date.now(), + }); + return true; + }, logger: resolvedLogger, mode: "webhook", nativeStreaming: true, diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index e84092ed826..e92c27d9953 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -1,7 +1,10 @@ +import { SlackCloudReceiptError } from "./chat-channels/slack/cloud-event.js"; +import { reconcilePendingSlackCloudEnrollments } from "./chat-channels/slack/cloud-enrollment.js"; import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js"; +import { dispatchSlackCloudEvent } from "./chat-channels/slack/cloud-ingress.js"; import { chatEndpoints } from "@paperclipai/db"; import { randomUUID } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; @@ -224,6 +227,9 @@ export function githubConnectionEventService( ) { const now = options.now ?? (() => new Date()); let nextPollAt = 0; + let nextEnrollmentAt: number | null = null; + let nextEnrollmentDiagnosticAt = 0; + const slackDiagnostics = new Map(); let configuredConnector: { config: NonNullable>; connector: PaperclipCloudConnector; @@ -503,8 +509,27 @@ export function githubConnectionEventService( nextPollAt = Number.POSITIVE_INFINITY; return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; } + try { + return await drainConnectorEvents(connector); + } finally { + try { + nextEnrollmentAt = await reconcilePendingSlackCloudEnrollments(db, now()); + } catch { + const time = now().getTime(); + nextEnrollmentAt = time + 5_000; + if (time >= nextEnrollmentDiagnosticAt) { + nextEnrollmentDiagnosticAt = time + 300_000; + logger.warn({ reason: "enrollment_reconcile_unavailable" }, "Slack enrollment responsibility retained"); + } + } + } + } + + async function drainConnectorEvents(connector: PaperclipCloudConnector): Promise { const bindings = await activeBindings(); - const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + const githubBots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + const slackBots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "slack"), sql`${chatEndpoints.setup}->'slackCloudEvents' is not null`, sql`${chatEndpoints.status} in ('active','verifying','attention')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + const bots = [...githubBots, ...slackBots]; let identity: { subject: string; companyId: string } | null = bindings[0] ?? (bots[0] ? { subject: "github-bot-events", companyId: bots[0].companyId } : null); const hasConsumer = bindings.length > 0 || bots.length > 0; if (!identity) { @@ -517,7 +542,7 @@ export function githubConnectionEventService( if (grant && subject) identity = { subject, companyId: grant.companyId }; } if (!identity) { - const [history] = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`)).limit(1); + const [history] = await db.select().from(chatEndpoints).where(sql`(${chatEndpoints.provider} = 'github' AND ${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null) OR (${chatEndpoints.provider} = 'slack' AND ${chatEndpoints.setup}->'slackCloudEvents' is not null)`).limit(1); if (history) identity = { subject: "github-bot-events", companyId: history.companyId }; } if (!identity) { @@ -543,6 +568,23 @@ export function githubConnectionEventService( }; const acknowledge: string[] = []; for (const event of lease.events) { + if (event.provider === "slack") { + try { await dispatchSlackCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; } + catch (error) { + const code = error instanceof SlackCloudReceiptError ? error.code : "owner_or_admission_unavailable"; + const disposition = error instanceof SlackCloudReceiptError ? error.disposition : "retry"; + const time = now().getTime(); + if (time - (slackDiagnostics.get(event.id) ?? Number.NEGATIVE_INFINITY) >= 300_000) { + if (slackDiagnostics.size >= 1000) slackDiagnostics.delete(slackDiagnostics.keys().next().value!); + slackDiagnostics.set(event.id, time); + logger.warn({ deliveryId: event.id.slice(0, 200), reason: code, disposition, + recoveryExpiresAt: Number.isFinite(Date.parse(event.createdAt)) ? new Date(Date.parse(event.createdAt) + 30 * 24 * 60 * 60_000).toISOString() : null }, "Slack connector receipt requires attention"); + } + if (disposition === "invalid") { acknowledge.push(event.id); result.ignored += 1; } + else result.failed += 1; + } + continue; + } if (event.payload.githubApp !== undefined) { try { await dispatchGitHubBotCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; } catch { result.failed += 1; } @@ -583,14 +625,14 @@ export function githubConnectionEventService( } return { - nextPollAt: () => Number.isFinite(nextPollAt) ? nextPollAt : null, + nextPollAt: () => { const next = Math.min(nextPollAt, nextEnrollmentAt ?? Number.POSITIVE_INFINITY); return Number.isFinite(next) ? next : null; }, pollOnce(input: { forSetup?: boolean; forDelivery?: boolean } = {}): Promise { // A manager actively awaiting a dedicated App's signed ping must not // inherit a personal connection's old empty-queue backoff. This is a // bounded readiness probe, not an additional background polling loop. if (inFlight) return inFlight; const time = now().getTime(); - if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : nextPollAt)) { + if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : Math.min(nextPollAt, nextEnrollmentAt ?? Number.POSITIVE_INFINITY))) { return Promise.resolve({ leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }); } nextSetupPollAt = time + 5_000; diff --git a/server/src/services/paperclip-cloud-connector.test.ts b/server/src/services/paperclip-cloud-connector.test.ts index 07bd0bb119e..4d7c594153b 100644 --- a/server/src/services/paperclip-cloud-connector.test.ts +++ b/server/src/services/paperclip-cloud-connector.test.ts @@ -584,11 +584,13 @@ describe("Paperclip Cloud connector", () => { return Response.json({ active: true, status: "active", - profiles: ["gmail.read", "drive.write", "unknown.profile", "gmail.read"], + profiles: ["gmail.read", "drive.write", "unknown.profile", "slack.events.v1", "gmail.read"], + slackEvents: { version: 1 }, }); }); const connector = createPaperclipCloudConnector({ config: keys.config, request: request as typeof fetch }); await expect(connector.getCapabilities()).resolves.toEqual(["gmail.read", "drive.write"]); + await expect(connector.slackEventsAvailable()).resolves.toBe(true); }); it("fails capability discovery closed for inactive, legacy, malformed, or rejected status responses", async () => { @@ -699,10 +701,10 @@ function seal( recipientPublicKey: KeyObject, purpose: "initial" | "access" | "events", configValue: PaperclipCloudConnectorConfig, - profile: keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES | "slack.manager" | "github.bot" | "github.code", + profile: keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES | "slack.manager" | "slack.events.v1" | "github.bot" | "github.code", ) { - const provider = profile.startsWith("github.") ? "github" : profile === "slack.manager" ? "slack" : "google"; - const scopes = provider === "github" ? [] : profile === "slack.manager" ? SLACK_MANAGER_SCOPES : GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profile as keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES].scopes; + const provider = profile.startsWith("github.") ? "github" : profile.startsWith("slack.") ? "slack" : "google"; + const scopes = provider === "github" || profile === "slack.events.v1" ? [] : profile === "slack.manager" ? SLACK_MANAGER_SCOPES : GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profile as keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES].scopes; const ephemeral = generateKeyPairSync("x25519"); const ephemeralJwk = ephemeral.publicKey.export({ format: "jwk" }) as { x: string }; const recipientJwk = recipientPublicKey.export({ format: "jwk" }) as { x: string }; @@ -795,3 +797,66 @@ describe("dedicated App gateway delivery", () => { expect(JSON.stringify(result)).not.toContain('"ct"'); }); }); + + +describe("sealed Slack event connector transport", () => { + it("opens a Slack event through the normal outer lease without exposing ciphertext or changing callback bytes", async () => { + const keys = config(); + const createdAt = "2026-11-05T10:00:00.000Z"; + const bindingId = `slack-event:${"a".repeat(64)}`; + const packet = { version: 1, instanceId, environment: "staging", bindingId, acceptedAt: createdAt, + rawBody: Buffer.from('{ "event_id": "EvFixture", "text": "original bytes" }').toString("base64"), + timestamp: "1793872800", signature: `v0=${"b".repeat(64)}` }; + const inner = seal(packet, keys.sealPublicKey, "events", keys.config, "slack.events.v1"); + const outer = seal({ v: 1, instanceId, environment: "staging", leaseId: "lease-slack", events: [ + { id: "receipt-slack", provider: "slack", event: "event_callback", createdAt, installationId: bindingId, + bindingIds: [bindingId], payload: { slackEvent: { bindingId, sealed: inner } } }, + ] }, keys.sealPublicKey, "events", keys.config, "github.code"); + const connector = createPaperclipCloudConnector({ config: keys.config, + request: vi.fn(async () => Response.json({ leaseId: "lease-slack", sealed: outer })) as typeof fetch }); + const result = await connector.leaseEvents({ subject, companyId }); + expect(result?.failedEventIds).toBeUndefined(); + expect(result?.events).toHaveLength(1); + expect(result?.events[0]?.payload.slackEvent).toEqual(packet); + expect(JSON.stringify(inner)).not.toContain("original bytes"); + expect(JSON.stringify(result)).not.toContain('"ct"'); + }); + + it.each(["binding", "acceptedAt", "instance", "environment", "key", "profile"] as const)( + "isolates an invalid inner Slack %s while retaining a healthy sibling", async failure => { + const keys = config(); + const createdAt = "2026-11-05T10:00:00.000Z"; + const bindingId = `slack-event:${"a".repeat(64)}`; + const packet = { version: 1, instanceId, environment: "staging", bindingId, acceptedAt: createdAt, rawBody: "e30=" }; + const changed = { ...packet, ...(failure === "binding" ? { bindingId: "other" } : {}), + ...(failure === "acceptedAt" ? { acceptedAt: "2026-11-05T10:00:00.001Z" } : {}), + ...(failure === "instance" ? { instanceId: "foreign" } : {}), + ...(failure === "environment" ? { environment: "production" } : {}) }; + const event = (id: string, inner: unknown) => ({ id, provider: "slack", event: "event_callback", createdAt, + installationId: bindingId, bindingIds: [bindingId], payload: { slackEvent: { bindingId, sealed: inner } } }); + const bad = seal(changed, failure === "key" ? config().sealPublicKey : keys.sealPublicKey, "events", keys.config, + failure === "profile" ? "slack.manager" : "slack.events.v1"); + const outer = seal({ v: 1, instanceId, environment: "staging", leaseId: "lease-slack", events: [ + event("bad", bad), event("healthy", seal(packet, keys.sealPublicKey, "events", keys.config, "slack.events.v1")), + ] }, keys.sealPublicKey, "events", keys.config, "github.code"); + const connector = createPaperclipCloudConnector({ config: keys.config, + request: vi.fn(async () => Response.json({ leaseId: "lease-slack", sealed: outer })) as typeof fetch }); + await expect(connector.leaseEvents({ subject, companyId })).resolves.toMatchObject({ + leaseId: "lease-slack", failedEventIds: ["bad"], events: [{ id: "healthy", payload: { slackEvent: packet } }], + }); + }); + + it.each([true, false])("selects the matching signed Slack operation receipt (active=%s)", async active => { + const keys = config(); + const enrollment = { active, endpointId: "endpoint", runtimeGeneration: 7 }; + const enrolled = { version: 1, bindingId: "enrolled" }, retired = { version: 1, runtimeGeneration: 7 }; + const request = vi.fn(async () => Response.json({ enrollment: enrolled, retirement: retired })); + const connector = createPaperclipCloudConnector({ config: keys.config, request: request as typeof fetch }); + await expect(connector.slackEventEnrollment({ subject, companyId, enrollment })).resolves.toEqual(active ? enrolled : retired); + expect(request).toHaveBeenCalledOnce(); + const [url, init] = request.mock.calls[0] as unknown as [string | URL, RequestInit]; + expect(new URL(String(url)).pathname).toBe("/v1/connector/slack-events"); + expect(typeof init.body).toBe("string"); + expect(JSON.parse(String(init.body))).toMatchObject({ enrollment: JSON.stringify(enrollment) }); + }); +}); diff --git a/server/src/services/paperclip-cloud-connector.ts b/server/src/services/paperclip-cloud-connector.ts index 6159051c5d3..da98d8c793c 100644 --- a/server/src/services/paperclip-cloud-connector.ts +++ b/server/src/services/paperclip-cloud-connector.ts @@ -35,7 +35,7 @@ export const GMAIL_CONNECTOR_SCOPES = [ export { GOOGLE_WORKSPACE_CONNECTOR_PROFILES }; export type PaperclipCloudConnectorEnvironment = "development" | "test" | "staging" | "production"; -export type PaperclipCloudConnectorOperation = "status" | "session" | "claim" | "refresh" | "revoke" | "webhook-bind" | "event-lease" | "event-ack" | "github-app"; +export type PaperclipCloudConnectorOperation = "status" | "session" | "claim" | "refresh" | "revoke" | "webhook-bind" | "event-lease" | "event-ack" | "github-app" | "slack-events"; export type PaperclipCloudConnectorProfileId = GoogleWorkspaceConnectorProfileId | GitHubConnectorProfileId | AsanaConnectorProfileId | "slack.manager" | "github.bot"; export type PaperclipCloudConnectorProvider = "google" | "github" | "asana" | "slack"; @@ -75,7 +75,7 @@ export type SealedConnectorEvents = { leaseId: string; events: Array<{ id: string; - provider: "github"; + provider: "github" | "slack"; event: string; action: string | null; installationId: string | null; @@ -86,12 +86,14 @@ export type SealedConnectorEvents = { }>; }; +type SealedEnvelopeProfileId = PaperclipCloudConnectorProfileId | "slack.events.v1"; + type SealedEnvelope = { v: 1; alg: "X25519-HKDF-SHA256-A256GCM"; purpose: "initial" | "access" | "events"; provider: PaperclipCloudConnectorProvider; - profile: PaperclipCloudConnectorProfileId; + profile: SealedEnvelopeProfileId; epk: string; iv: string; ct: string; @@ -115,10 +117,14 @@ type ConnectorResponse = { events?: unknown; acknowledged?: unknown; githubApps?: { version: number }; + slackEvents?: { version: number }; + enrollment?: unknown; + retirement?: unknown; eventWakeVersion?: unknown; }; const ENDPOINTS: Record = { + "slack-events": "/v1/connector/slack-events", "github-app": "/v1/connector/github-apps", status: "/v1/connector/instance-status", session: "/v1/connector/sessions", @@ -147,6 +153,9 @@ const BROKER_REJECTION_REASONS = new Set([ "AUDIENCE_MISMATCH", "OPERATION_MISMATCH", "PAYLOAD_MISMATCH", "MALFORMED_REQUEST", "UNSUPPORTED_ALGORITHM", "RATE_LIMITED", "PROVIDER_OPERATION_FAILED", "PROVIDER_REAUTHORIZATION_REQUIRED", + "SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_IDENTITY_CHANGED", + "SLACK_ENROLLMENT_CHANGED", "SLACK_ENROLLMENT_UNSETTLED", "SLACK_ENROLLMENT_DISABLED", + "SLACK_COMPANY_MISMATCH", "SLACK_EVENTS_UNAVAILABLE", ]); const UNKNOWN_BROKER_REASON = "UNKNOWN_BROKER_ERROR"; @@ -192,6 +201,7 @@ export class PaperclipCloudConnectorError extends Error { readonly code: string, readonly status?: number, readonly retryAt?: number, + readonly brokerReason?: string, ) { super(message); this.name = "PaperclipCloudConnectorError"; @@ -280,7 +290,7 @@ export function createPaperclipCloudConnector(input: { async function call( operation: PaperclipCloudConnectorOperation, claims: { subject: string; companyId: string; profile?: PaperclipCloudConnectorProfileId; scopes?: string[]; returnUri?: string; returnState?: string; claimId?: string; redemptionId?: string; eventWakeVersion?: 0 | 1 }, - secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement"; value: string }, + secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement" | "enrollment"; value: string }, ): Promise { const endpoint = new URL(ENDPOINTS[operation], `${config.baseUrl}/`).toString(); const issuedAt = Math.floor(now() / 1000); @@ -333,6 +343,7 @@ export function createPaperclipCloudConnector(input: { response.status === 409 ? "REAUTHORIZATION_REQUIRED" : "CONNECTOR_REQUEST_FAILED", response.status, providerRetryAt(response.headers.get("retry-after"), now()), + reason, ); } try { @@ -381,6 +392,14 @@ export function createPaperclipCloudConnector(input: { return { eventWakeVersion: () => eventWakeVersion, + async slackEventsAvailable() { + const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" }); + return response.active === true && response.slackEvents?.version === 1; + }, + async slackEventEnrollment(values: { subject: string; companyId: string; enrollment: Record }) { + const response = await call("slack-events", values, { field: "enrollment", value: JSON.stringify(values.enrollment) }); + return values.enrollment.active === false ? response.retirement : response.enrollment; + }, async githubAppsAvailable() { const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" }); return response.active === true && response.githubApps?.version === 2; @@ -551,6 +570,16 @@ export function createPaperclipCloudConnector(input: { || delivery.registrationId !== packet.registrationId) throw badEnvelope(); event.payload = { githubApp: delivery }; } + if (event.provider === "slack") { + const packet = event.payload.slackEvent; + if (!isRecord(packet) || typeof packet.bindingId !== "string" || packet.sealed === undefined) throw badEnvelope(); + const sealed = parseEnvelope(packet.sealed, "events", "slack", "slack.events.v1"); + const delivery = decryptEnvelope(sealed, sealKey, config.instanceId, config.environment, "slack", "slack.events.v1", []); + if (!isRecord(delivery) || delivery.version !== 1 || delivery.instanceId !== config.instanceId + || delivery.environment !== config.environment || delivery.bindingId !== packet.bindingId + || delivery.acceptedAt !== event.createdAt) throw badEnvelope(); + event.payload = { slackEvent: delivery }; + } events.push(event); } catch { failedEventIds.push(event.id); @@ -684,7 +713,7 @@ function parseEnvelope( value: unknown, purpose: SealedEnvelope["purpose"], provider: PaperclipCloudConnectorProvider, - profile: PaperclipCloudConnectorProfileId, + profile: SealedEnvelopeProfileId, ): SealedEnvelope { if (!value || typeof value !== "object" || Array.isArray(value)) throw badEnvelope(); const candidate = value as Partial; @@ -737,7 +766,7 @@ function unsealEvents( if (parsed.v !== 1 || parsed.instanceId !== instanceId || parsed.environment !== environment || typeof parsed.leaseId !== "string" || !Array.isArray(parsed.events)) throw badEnvelope(); for (const event of parsed.events) { - if (!isRecord(event) || typeof event.id !== "string" || event.provider !== "github" + if (!isRecord(event) || typeof event.id !== "string" || !["github", "slack"].includes(event.provider) || typeof event.event !== "string" || !isRecord(event.payload) || !Array.isArray(event.bindingIds) || !event.bindingIds.every((id) => typeof id === "string")) throw badEnvelope(); } @@ -754,7 +783,7 @@ function decryptEnvelope( instanceId: string, environment: string, provider: PaperclipCloudConnectorProvider, - profile: PaperclipCloudConnectorProfileId, + profile: SealedEnvelopeProfileId, scopes: readonly string[], ): unknown { const ephemeralRaw = Buffer.from(envelope.epk, "base64url"); diff --git a/server/src/services/tool-access/connections/removal.ts b/server/src/services/tool-access/connections/removal.ts index be9bea0dd78..ae3016de93e 100644 --- a/server/src/services/tool-access/connections/removal.ts +++ b/server/src/services/tool-access/connections/removal.ts @@ -3,7 +3,8 @@ import { notifyVoiceChange } from "../../voice/voice-recovery-notifications.js"; import { notifyChatEndpointWork } from "../../chat-work-notifications.js"; import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js"; import { toConnection } from "./projection.js"; -import { actorBinding } from "../actor.js"; +import { actorBinding, connectionActorUserId } from "../actor.js"; +import { stageSlackCloudEnrollmentIntent } from "../../chat-channels/slack/cloud-enrollment.js"; import { vercelConnectResourcesFor } from "../vercel-credentials.js"; import type { ActorInfo } from "../types.js"; import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; @@ -241,6 +242,8 @@ async function removeConnection( await lease.assertOwned(tx); await tx.update(chatEndpoints).set({ status: "archived", updatedAt: new Date() }).where(eq(chatEndpoints.id, endpoint.id)); await tx.update(toolConnections).set({ status: "archived", enabled: false, updatedAt: new Date() }).where(eq(toolConnections.id, connection.id)); + await stageSlackCloudEnrollmentIntent(tx as unknown as Db, endpoint.id, connectionActorUserId(actor)); + await lease.assertOwned(tx); }); await removeSlackRegistration(db, endpoint.id, lease); return removeConnectionUnlocked(connectionId, companyId, actor);