From 4ef3fa427194d34e0086dbb02aa49fba23b1187b Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 6 Nov 2026 13:05:37 -0800 Subject: [PATCH 01/11] docs(release): prepare v2026.1106.0 stable notes before publish (#19607) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Each stable release publishes curated release notes as its GitHub Release body. > - The stable release workflow reads `releases/beta/v2026.1023.0-beta.0.md` from master when it starts. It publishes that exact revision. > - The notes on master show v2026.1103.0 and the date 2026-11-03. That version did not ship. > - The promotion of `2026.1023.0-beta.0` now starts on 2026-11-06, so the version is `2026.1106.0`. > - This pull request sets the header to the correct version and date before the stable release starts. > - The benefit is that the release page shows the version and date that actually ship. ## Linked Issues or Issue Description Refs #18842 (the previous pre-publish fix of these notes, for v2026.1103.0). Refs #17388 (the curated notes for the `2026.1023.0-beta.0` soak). **Problem** The stable notes for the promotion of `2026.1023.0-beta.0` show `# Paperclip v2026.1103.0` and `> Released: 2026-11-03`. The release for that version did not publish. `scripts/release.sh stable --print-version --date 2026-11-06` on master gives `2026.1106.0`. **Expected result** The notes show `# Paperclip v2026.1106.0` and `> Released: 2026-11-06`. The intro paragraph names v2026.1106.0. **Impact** The stable workflow publishes these notes as the GitHub Release body. If this pull request does not merge before the release starts, the release page shows the incorrect version and date. ## What Changed - Set the H1 to `# Paperclip v2026.1106.0`. - Set the release line to `> Released: 2026-11-06`. - Set the version in the first sentence of the intro paragraph to v2026.1106.0. - No other text changes. The diff is 3 lines in one file. ## Verification - `scripts/release.sh stable --print-version --date 2026-11-06` on master `d728be8fbffc95a903ffd58bc527c7e8f1a1c704` gives `2026.1106.0`. - The file has no other `1103` or `2026-11-03` strings after the change. - The audit of the content against `v2026.1023.0..d69631b01b9c658f80bf8bed389863dce82643e1` did not change since #18842. I ran it again: - `git rev-list --count` = 515. This matches the notes. - All 255 cited PR numbers are in the range. Five cited PRs (#14180, #16356, #16401, #16403, #16976) are squash commits without a `(#N)` suffix. - `git shortlog -sn` without `github-actions[bot]` gives 27 contributors. This matches the notes. The 21 credited handles do not change. - The migrations and the defaults in the Upgrade Guide were checked in #18842. The range did not change. - The file on this branch has blob `16f05f22d4d71de3a568b493de2ddcb8eadb7859`. This is identical to the locally checked edit. ## Risks - Low risk. This is a change to documentation only, in one file. - If the UTC date changes before the release starts, the release workflow keeps `stable_date=2026-11-06` to match this header. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use (shell, git, GitHub API). Used to check the version, run the audit again, and write the change. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass (documentation only; no tests apply) - [ ] I have added or updated tests where applicable (not applicable) - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- releases/beta/v2026.1023.0-beta.0.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/releases/beta/v2026.1023.0-beta.0.md b/releases/beta/v2026.1023.0-beta.0.md index ef11a77a4b2..16f05f22d4d 100644 --- a/releases/beta/v2026.1023.0-beta.0.md +++ b/releases/beta/v2026.1023.0-beta.0.md @@ -1,8 +1,8 @@ -# Paperclip v2026.1103.0 +# Paperclip v2026.1106.0 -> Released: 2026-11-03 +> Released: 2026-11-06 -Paperclip v2026.1103.0 carries 515 commits, promoted from `2026.1023.0-beta.0`. The headline is that the Paperclip Runner is now the default for new agents: new Claude, Codex, OpenCode, and Pi agents start on the native Runner on qualified macOS and Linux hosts, existing agents get a one-click "Upgrade runner" action, and GitHub Copilot joins as a native harness. Around that: project repositories that automatically become skill sources, model-backed prompt injection prevention, agent observability to Braintrust or any OTLP endpoint, a unified Google Workspace connection, bulk edits on the Agents roster, workspaces that compact and expire themselves with encrypted restore points, and Grok Bot and Muse as experimental Runner providers. Underneath, server restarts report ready sooner, the Docker image boots from a precompiled bundle, task pages load faster, and more background services stop polling the database. +Paperclip v2026.1106.0 carries 515 commits, promoted from `2026.1023.0-beta.0`. The headline is that the Paperclip Runner is now the default for new agents: new Claude, Codex, OpenCode, and Pi agents start on the native Runner on qualified macOS and Linux hosts, existing agents get a one-click "Upgrade runner" action, and GitHub Copilot joins as a native harness. Around that: project repositories that automatically become skill sources, model-backed prompt injection prevention, agent observability to Braintrust or any OTLP endpoint, a unified Google Workspace connection, bulk edits on the Agents roster, workspaces that compact and expire themselves with encrypted restore points, and Grok Bot and Muse as experimental Runner providers. Underneath, server restarts report ready sooner, the Docker image boots from a precompiled bundle, task pages load faster, and more background services stop polling the database. ## Breaking Changes From e28158b353fd274f0322be76d62cd223e0df599f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:08:43 +0000 Subject: [PATCH 02/11] chore(lockfile): refresh pnpm lockfiles (#19617) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip uses a dependency lock for reproducible builds. > - Dependency changes on master need matching lock entries. > - Paid eval reports also pin a reviewed offline correction. > - Reports must install before paid model work starts. > - This automated PR keeps the lock and report proof together. ## Linked Issues or Issue Description **What happened?** Dependency manifests on master changed after the last committed lock refresh. **Expected behavior** Frozen installs must match those manifests. Eval reports must retain matching offline lock proof. **Steps to reproduce** Run a frozen install before and after this lock refresh. ## What Changed - Refresh the dependency lock from the manifests on master. - Refresh the sandbox provider lockfiles from their manifests on master. - Include the matching offline report correction and hashes. ## Verification - Pinned pnpm resolution completed with lifecycle scripts disabled. - The report proof refresh accepted the reviewed committed baseline. - The workflow rejected changes outside the lock and report proof files. ## Risks The lockfile bot merges this PR without CI or review when every added package version is at least 3 days old on npm. Otherwise the PR waits for a maintainer, and the daily refresh tries again. The bot merges only the exact commit this workflow generated from manifests already reviewed on master. Unknown report lock baselines remain blocked. This update changes no permissions or paid enable gates. ## Model Used None — generated by pnpm and the deterministic report-lock updater. ## Checklist - [x] Thinking path and problem description included - [x] Generated lock and matching report proof included - [x] Release-age gate decides between bot merge and maintainer review Co-authored-by: lockfile-bot --- .../sandbox-providers/daytona/pnpm-lock.yaml | 4 +++- pnpm-lock.yaml | 4 +++- tests/runner-e2e/report-lock.json | 6 ++--- tests/runner-e2e/report-lock.patch | 23 ++++++++++--------- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml b/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml index c4483ecd951..38f45f94b93 100644 --- a/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml +++ b/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml @@ -483,7 +483,9 @@ packages: '@smithy/core@3.33.3': resolution: {integrity: sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==} engines: {node: '>=18.0.0'} - deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. + deprecated: |- + Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this + version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393. '@smithy/credential-provider-imds@4.5.2': resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a87f8e749f5..97cbf7b33af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5048,7 +5048,9 @@ packages: '@smithy/core@3.35.1': resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==} engines: {node: '>=18.0.0'} - deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. + deprecated: |- + Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this + version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393. '@smithy/credential-provider-imds@4.5.2': resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} diff --git a/tests/runner-e2e/report-lock.json b/tests/runner-e2e/report-lock.json index bd6c65c89b8..d6740c79df5 100644 --- a/tests/runner-e2e/report-lock.json +++ b/tests/runner-e2e/report-lock.json @@ -1,6 +1,6 @@ { - "inputSha256": "14049cb20d48f448b6f24e5445a75bf8d22ed3052fe9ff885f4c127775d958d3", - "outputSha256": "53f6c7c7f09a5cf0442ee2992a656eb2336ab6b13e665219a0cd1247475f13d9", - "patchSha256": "eb4adb2f3c8a2ebefe65d3100b581c5765318023d96aa8a80394c848d7299b9d", + "inputSha256": "53f6c7c7f09a5cf0442ee2992a656eb2336ab6b13e665219a0cd1247475f13d9", + "outputSha256": "5e2b012b50bb8b198384da4f5427c27acd948af405f89cde15bfa48bb10c0ad4", + "patchSha256": "2abdd6ec8e374bfb20b7e168ef010858493a50a044360175e1af5a2ca393f909", "basis": "Generated with the trusted master lockfile refresh. Review this offline correction and its hashes with the dependency changes." } diff --git a/tests/runner-e2e/report-lock.patch b/tests/runner-e2e/report-lock.patch index 6e5bb651368..090ee867256 100644 --- a/tests/runner-e2e/report-lock.patch +++ b/tests/runner-e2e/report-lock.patch @@ -1,14 +1,15 @@ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml -index 45b3aaf..a87f8e7 100644 +index a87f8e7..97cbf7b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml -@@ -1306,6 +1306,9 @@ importers: - lucide-react: - specifier: ^1.48.0 - version: 1.48.0(react@19.2.8) -+ mdast-util-from-markdown: -+ specifier: ^2.0.3 -+ version: 2.0.3 - mermaid: - specifier: ^12.1.0 - version: 12.1.0 +@@ -5048,7 +5048,9 @@ packages: + '@smithy/core@3.35.1': + resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==} + engines: {node: '>=18.0.0'} +- deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. ++ deprecated: |- ++ Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this ++ version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393. + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} From 586caa5f4935c0d79eed5c1a4456ce7fe171b1fc Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:10:44 -0600 Subject: [PATCH 03/11] fix(native): hand a shutdown-interrupted startup to the next controller (#19596) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The native Runner path runs an agent turn through a runnerd process that the control plane owns > - A restart of the control plane must keep the task going without a click, if that is safe > - A graceful shutdown closes the HTTP listener first and then drains the heartbeat runs > - A native run that was between provider dispatch and runnerd backend setup in that gap failed with `local_runtime_api_listener_unavailable`, so the drain did not see it as running and did not suspend it for the next boot > - This pull request makes that failure a restart detach: the run stays running with its claim, the drain suspends it, and the next controller adopts it > - The benefit is that a SIGTERM in this window no longer spends a failure and a same-run retry, and the task continues by itself ## Linked Issues or Issue Description Refs #19530 (the same restart window, before provider dispatch). **What happened?** In a restart chaos replay (ACPX Claude, local runner), a restart-recovered run got SIGTERM while `createRunnerdBackendWithinSessionClaim` built the runnerd backend. `resolveLocalRuntimeApiUrl()` threw `local_runtime_api_listener_unavailable` because shutdown had already closed the listener. The run settled `failed`. The drain then logged `restartSuspendedRunIds: []`. The next boot used the one bootstrap retry. A second SIGTERM hit the same window, and the task stopped at a manual Retry. **Expected behavior** A run that this process cannot continue because it is shutting down is handed to the next controller. It is not settled as failed. **Steps to reproduce** 1. Start a native Runner task and restart the server (K1) so that startup recovery resumes the run. 2. Send SIGTERM while the resumed run sets up its runnerd backend. 3. See the run fail with `local_runtime_api_listener_unavailable` and the drain report no suspended runs. ## What Changed - `server/src/runtime-api.ts`: `resolveLocalRuntimeApiUrl()` throws the new `LocalRuntimeApiListenerClosedError` when this process had a listener and closed it (the `listening` flag is false or the `close` event fired). The error message stays `local_runtime_api_listener_unavailable`, so the existing classifiers do not change. A listener that never started still throws the plain error. - `server/src/services/native-runtime/native-session-executor.ts`: for a local Runner, the startup failure handler treats `LocalRuntimeApiListenerClosedError` like a restart detach. It stops lease renewal, keeps the claim and throws `NativeControllerDetachedForRestartError`. Heartbeat already keeps such a run `running`. A remote Runner keeps the same-run bootstrap retry from #19462, because startup recovery has no process evidence to adopt a claimed remote startup (`native-runner-restart-preparation.integration.test.ts` still covers that path). - `doc/execution-semantics.md`: describe the new rule. - Tests: a heartbeat regression that closes the real runtime API listener before backend setup, then drains and restarts; a unit test for the closed versus never-started listener; two `vi.mock` factories keep the real module exports. ## Verification - Red before the fix: `pnpm exec vitest run src/__tests__/heartbeat-native-restart-suspension.test.ts -t listener` failed with `restartSuspendedRunIds: []` and the run `failed` (`local_runtime_api_listener_unavailable`, `bootstrap_retry`). - Green after the fix: the same test passes. The drain suspends the run, the run stays `running` with `awaiting_runner_reattach`, and the next controller claims it and reaches the backend once. - `pnpm exec vitest run src/__tests__/heartbeat-native-restart-suspension.test.ts src/http/server.test.ts src/__tests__/server-startup-feedback-export.test.ts src/services/native-runtime/native-session-executor.test.ts src/services/native-runtime/native-recovered-owner-followup.test.ts`: 1333 tests pass. One timing test in `native-session-executor.test.ts` failed once under host load average ~30 and passed alone. - `pnpm exec tsc --noEmit -p server/tsconfig.json`: pass. - Live, restart chaos harness, local ACPX Claude (`claude-sonnet-5`), `--rc06-masked`, this branch at a35ec8b845 (before the remote-scope commit 5dc63f24fb, which does not change the local path). The original replay schedule no longer reached the window, because the task now ends before the second fault. So I moved the faults earlier: K1 at 10 s, then K2 as soon as the recovered server is ready. - K1 + K2, 2 runs: both runs PASS with **R1 pass**. Each drain logged `restartSuspendedRunIds: []` (before: `[]`). The next boot classified the run `resume_dead_runner` and the same run completed. No `local_runtime_api_listener_unavailable` failure. - K1 + K2 + K2, 2 runs: each drain suspended the run. The fourth boot blocked the run with `native_restart_recovery_blocked`, because three restart claims used the three-attempt recovery budget. Retry then completed the task (R3 pass). That budget count is a separate open item. This pull request does not change it. ## Risks - Low. The new path applies only after this process closed its own listener, which happens only during shutdown. The run then waits for the drain or the next boot, as for the other restart detaches. - If shutdown does not reach the drain (for example SIGKILL after the listener closes), the run keeps its live claim until the claim expires. Then startup recovery handles it as a hard restart. This is the same as any run that is running at a hard kill. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, through Claude Code with tool use (shell, file edits, GitHub CLI). Extended reasoning on. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 7 +++ ...eartbeat-native-restart-suspension.test.ts | 51 +++++++++++++++++++ .../src/__tests__/helpers/native-restart.ts | 2 +- server/src/http/server.test.ts | 26 +++++++++- server/src/runtime-api.ts | 38 ++++++++++++-- .../native-recovered-owner-followup.test.ts | 2 +- .../native-session-executor.test.ts | 2 +- .../native-runtime/native-session-executor.ts | 11 +++- tests/runner-e2e/workflow-target.test.ts | 3 +- 9 files changed, 131 insertions(+), 11 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 986c65cbbf5..5c71fd55f82 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3317,6 +3317,13 @@ A graceful shutdown can also suspend a restart-recovery claim whose execution has not yet reached provider dispatch. That execution has lost its authority: it leaves the run running, its budget hold held and its workspace unmarked, and the next controller adopts the same run. It does not settle the run as failed. +Shutdown closes the runtime API listener before it drains runs. A local Runner +execution that then needs the listener (for example, to give a new runner its +callback origin) detaches the same way instead of failing. It keeps its claim, +the drain suspends it, and the next controller adopts the run. A remote Runner +execution has no process evidence on the controller host, so it keeps the +same-run bootstrap retry. A listener that never started is still a startup +failure. Remote Runner processes use their authenticated connection lease as the bound on disconnected lifetime. The controller's shorter connection-recovery budget limits its own attempt; it must not terminate the remote owner while another diff --git a/server/src/__tests__/heartbeat-native-restart-suspension.test.ts b/server/src/__tests__/heartbeat-native-restart-suspension.test.ts index 892cb063735..654f152bcf3 100644 --- a/server/src/__tests__/heartbeat-native-restart-suspension.test.ts +++ b/server/src/__tests__/heartbeat-native-restart-suspension.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { authUsers, budgetReservations, companyMemberships, completionContracts, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations } from "@paperclipai/db"; import { expect, it, vi } from "vitest"; +import { resolveLocalRuntimeApiUrl } from "../runtime-api.js"; import { heartbeatService } from "../services/heartbeat.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { currentNativeControllerIdentity } from "../services/native-runtime/native-restart-recovery.js"; @@ -77,6 +78,56 @@ describeEmbeddedPostgres("heartbeat native recovery interrupted by graceful shut 90_000, ); + it("hands a recovery to the next controller when shutdown closed the listener before backend setup", async () => { + const f = await seedRecoverableRun("LISTENER-CLOSED"); + const listener = fixture.current.server!; + const address = listener.address(); + if (!address || typeof address === "string") throw new Error("Expected a TCP runtime API listener"); + let listenerClosed: Promise | undefined; + // Graceful shutdown closes HTTP admission before it drains heartbeat runs. + // Backend setup then asks for the callback origin, as runnerd setup does. + const dying = vi.fn(() => ({ + descriptor: async () => { resolveLocalRuntimeApiUrl(); throw new Error("unreachable"); }, + openSession: async () => { resolveLocalRuntimeApiUrl(); throw new Error("unreachable"); }, + })); + const heartbeat = heartbeatService(f.db, { + nativeSessionBackendFactory: dying as never, + beforeNativeRuntimeSelection: async () => { + listener.closeIdleConnections(); + listenerClosed = new Promise(resolve => listener.close(() => resolve())); + }, + }); + let drained: unknown; + try { + try { + const { claims } = await heartbeat.recoverNativeRunsAfterRestart(new Date(), [f.runId]); + expect(claims).toEqual([expect.objectContaining({ runId: f.runId })]); + } finally { await heartbeat.drainActiveRunExecutions(); } + expect(dying).toHaveBeenCalled(); + drained = await heartbeat.drainRunningRunsForShutdown("SIGTERM", new Date(), [f.runId]); + await heartbeat.drainActiveRunExecutions(); + } finally { + await listenerClosed; + await new Promise(resolve => listener.listen(address.port, "127.0.0.1", resolve)); + } + expect(drained).toMatchObject({ restartSuspendedRunIds: [f.runId] }); + const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [owner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect({ status: run.status, errorCode: run.errorCode, finishedAt: run.finishedAt }).toEqual({ + status: "running", errorCode: null, finishedAt: null }); + expect(owner).toMatchObject({ phase: "observed", resultId: null, recoveryState: "awaiting_runner_reattach" }); + + await f.db.update(nativeRunFinalizations).set({ controllerPid: 2_000_000_001 }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const resumed = vi.fn(() => { throw new Error("next boot reached provider dispatch"); }); + const next = heartbeatService(f.db, { nativeSessionBackendFactory: resumed }); + try { + const { claims } = await next.recoverNativeRunsAfterRestart(new Date(), [f.runId]); + expect(claims).toEqual([expect.objectContaining({ runId: f.runId })]); + } finally { await next.drainActiveRunExecutions(); } + expect(resumed).toHaveBeenCalledOnce(); + }, 90_000); + it("still settles a recovery whose claim merely expired without a shutdown suspension", async () => { const f = await seedRecoverableRun("EXPIRED-NOT-SUSPENDED"); const dying = vi.fn(() => { throw new Error("An expired claim must not open a provider session"); }); diff --git a/server/src/__tests__/helpers/native-restart.ts b/server/src/__tests__/helpers/native-restart.ts index deccd592af1..8d642f4ca12 100644 --- a/server/src/__tests__/helpers/native-restart.ts +++ b/server/src/__tests__/helpers/native-restart.ts @@ -244,7 +244,7 @@ export function nativeRestartFixture() { }; } - return { seedRun, transportOptions, get current() { return { temporary, runtimeRoot, paperclipHome, apiUrl, companyId, agentId, successor }; } }; + return { seedRun, transportOptions, get current() { return { temporary, runtimeRoot, paperclipHome, apiUrl, companyId, agentId, successor, server }; } }; } export function remotePreparedWorkspace() { diff --git a/server/src/http/server.test.ts b/server/src/http/server.test.ts index b548858d23f..c4616ec6751 100644 --- a/server/src/http/server.test.ts +++ b/server/src/http/server.test.ts @@ -8,7 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import express from "express"; import { DurablePrpControlPlane } from "../vendor/paperclip-runner/index.js"; import { registerRunnerPrpAuthority, runnerPrpWebSocketInternals } from "../realtime/runner-prp-ws.js"; -import { resolveLocalRuntimeApiUrl } from "../runtime-api.js"; +import { LocalRuntimeApiListenerClosedError, resolveLocalRuntimeApiUrl } from "../runtime-api.js"; import { createPaperclipHttpServer } from "./server.js"; import { privateHostnameGuard } from "../middleware/private-hostname-guard.js"; @@ -206,6 +206,30 @@ describe("Paperclip HTTP server runner transport", () => { } }); + it("distinguishes a listener closed for shutdown from one that never served", async () => { + expect(() => resolveLocalRuntimeApiUrl()).toThrow("local_runtime_api_listener_unavailable"); + expect(() => resolveLocalRuntimeApiUrl()).not.toThrow(LocalRuntimeApiListenerClosedError); + const server = createPaperclipHttpServer((_req, res) => res.end("closing"), { apiUrl: "http://ignored.example.test" }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const client = await fetch(resolveLocalRuntimeApiUrl(), { keepalive: true }); + // `close()` stops admission at once; its close event waits for open requests. + const closed = new Promise(resolve => server.close(() => resolve())); + expect(() => resolveLocalRuntimeApiUrl()).toThrow(LocalRuntimeApiListenerClosedError); + await client.text(); + server.closeAllConnections(); + await closed; + expect(() => resolveLocalRuntimeApiUrl()).toThrow(LocalRuntimeApiListenerClosedError); + expect(() => resolveLocalRuntimeApiUrl()).toThrow("local_runtime_api_listener_unavailable"); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + try { + expect(resolveLocalRuntimeApiUrl()).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + } finally { + await new Promise(resolve => server.close(() => resolve())); + } + }); + it.each(["127.0.0.1", "::1", "::ffff:127.0.0.1"])("routes a guarded native read through the real %s listener", async host => { await guardedNativeRead(host); }); diff --git a/server/src/runtime-api.ts b/server/src/runtime-api.ts index 4c4f87868a6..b7d1a3d64f4 100644 --- a/server/src/runtime-api.ts +++ b/server/src/runtime-api.ts @@ -4,15 +4,37 @@ import { isIP } from "node:net"; const runtimeApiListeners = new Set(); const registeredRuntimeApiListeners = new WeakSet(); +let runtimeApiListenerClosed = false; + +/** + * The process had a runtime API listener and has since closed it, which only + * happens while it shuts down. Work that needs a callback origin must hand off + * to the next controller instead of failing as a provider or startup error. + */ +export class LocalRuntimeApiListenerClosedError extends Error { + constructor() { + // Keep the established failure code for classifiers and persisted rows. + super("local_runtime_api_listener_unavailable"); + this.name = "LocalRuntimeApiListenerClosedError"; + } +} /** Track listener ownership without treating configured or advertised URLs as authority. */ export function registerLocalRuntimeApiServer(server: Server): void { if (!registeredRuntimeApiListeners.has(server)) { registeredRuntimeApiListeners.add(server); - server.on("listening", () => runtimeApiListeners.add(server)); - server.on("close", () => runtimeApiListeners.delete(server)); + server.on("listening", () => { + runtimeApiListeners.add(server); + runtimeApiListenerClosed = false; + }); + server.on("close", () => { + if (runtimeApiListeners.delete(server)) runtimeApiListenerClosed = true; + }); + } + if (server.listening) { + runtimeApiListeners.add(server); + runtimeApiListenerClosed = false; } - if (server.listening) runtimeApiListeners.add(server); } function normalizeHost(value: string | null | undefined): string { @@ -94,6 +116,11 @@ export function chooseLocalRuntimeApiUrl(input: { export function resolveLocalRuntimeApiUrl(): string { const listeners = [...runtimeApiListeners].filter(server => server.listening); + // `server.close()` clears `listening` at once; its close event waits for + // admitted requests. Either state means this process stopped serving. + if (listeners.length === 0 && (runtimeApiListenerClosed || runtimeApiListeners.size > 0)) { + throw new LocalRuntimeApiListenerClosedError(); + } if (listeners.length !== 1) { throw new Error(listeners.length ? "local_runtime_api_listener_ambiguous" : "local_runtime_api_listener_unavailable"); } @@ -103,7 +130,10 @@ export function resolveLocalRuntimeApiUrl(): string { } export const runtimeApiInternals = { - resetForTests(): void { runtimeApiListeners.clear(); }, + resetForTests(): void { + runtimeApiListeners.clear(); + runtimeApiListenerClosed = false; + }, }; export function choosePrimaryRuntimeApiUrl(input: { diff --git a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts index f12aad1871f..8185661619d 100644 --- a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts +++ b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts @@ -25,7 +25,7 @@ import { import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; const toolDefinitions = vi.hoisted(() => vi.fn()); -vi.mock("../../runtime-api.js", () => ({ resolveLocalRuntimeApiUrl: () => "http://127.0.0.1:43219" })); +vi.mock("../../runtime-api.js", async importOriginal => ({ ...await importOriginal(), resolveLocalRuntimeApiUrl: () => "http://127.0.0.1:43219" })); vi.mock("./paperclip-runner-tool-authority.js", () => ({ PaperclipRunnerToolAuthority: class { readonly binding = {}; diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c568d5350f2..dd9be079322 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -275,7 +275,7 @@ const state = vi.hoisted(() => ({ release: null as null | (() => void), })); -vi.mock("../../runtime-api.js", () => ({ resolveLocalRuntimeApiUrl: state.localRuntimeApiUrl })); +vi.mock("../../runtime-api.js", async importOriginal => ({ ...await importOriginal(), resolveLocalRuntimeApiUrl: state.localRuntimeApiUrl })); const accountingEvents = vi.hoisted(() => ({ committed: undefined as undefined | ((event: PrpEvent) => Promise), duplicate: undefined as undefined | ((event: PrpEvent) => Promise) })); vi.mock("./paperclip-control-plane-port.js", async importOriginal => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c7fddc8388b..fd1216fe57c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5,7 +5,7 @@ import { logger } from "../../middleware/logger.js"; import { canRestoreMigratedNativeSession } from "./native-workspace-migration.js"; import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES, nativeHarnessBackupManifestSha256, retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js"; import { prepareCursorCloudRuntime } from "./cursor-cloud-runtime.js"; -import { resolveLocalRuntimeApiUrl } from "../../runtime-api.js"; +import { LocalRuntimeApiListenerClosedError, resolveLocalRuntimeApiUrl } from "../../runtime-api.js"; import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; import { readProviderFailure } from "@paperclipai/shared"; import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForNativeCancellation } from "./native-cancellation-delivery.js"; @@ -10676,7 +10676,14 @@ async function executePaperclipNativeSessionWithinScope( } else { error = identityRedactor.redact(error); } - if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) { + // A closed runtime API listener means this process is shutting down; it + // closes HTTP admission before it drains runs. For a local Runner, keep the + // claim for the drain to suspend: the next controller proves the Runner + // process identity and adopts the run without spending a failure or a + // same-run retry. A remote Runner has no process evidence on this host, so + // it keeps the bootstrap retry below. + if (nativeRunsDetachingForRestart.has(input.execution.binding.runId) + || (error instanceof LocalRuntimeApiListenerClosedError && input.runnerExecutionTarget?.kind !== "remote")) { await leaseRenewal.stop().catch(() => undefined); liveQuestions.close(); activeNativeSessions.delete(input.execution.binding.runId); diff --git a/tests/runner-e2e/workflow-target.test.ts b/tests/runner-e2e/workflow-target.test.ts index d02c01d647b..7f7aab50686 100644 --- a/tests/runner-e2e/workflow-target.test.ts +++ b/tests/runner-e2e/workflow-target.test.ts @@ -18,7 +18,8 @@ async function resolveTarget(event: string, cadence: string, branch = "master") encoding: "utf8", env: { ...process.env, EVENT_NAME: event, EVAL_CADENCE: cadence, IMAGE_ONLY: "false", TESTOPS_REQUESTED: "false", TARGET_BRANCH: branch, DEFAULT_BRANCH: "master", GITHUB_SHA: queuedSha, - MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip" }, + MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip", + IMAGE_ONLY: "false", TESTOPS_REQUESTED: "false" }, }); return { status: result.status, stderr: result.stderr, output: await readFile(output, "utf8").catch(() => "") }; From da9a4cb96c46945d693f0ddffdcb0e0743649a01 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:11:00 -0600 Subject: [PATCH 04/11] fix(native): retire a finished local Runner that stopped before suspending (#19597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The native Runner path runs each per-turn agent turn through a runnerd process, and the next turn must prove that the previous Runner is finished before it takes over the saved session root > - Under load, a local turn can have its result accepted while its `runner.suspend` command does not finish, so the Runner's own receipt stays unsuspended > - The retirement proof that handles the same case for sandbox Runners refused every local Runner with `target_not_sandbox_runner` > - So the next turn failed with `runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate` and its session root was quarantined > - This pull request extends the retirement to a local Runner when the prior Runner process is proven stopped by exact PID and start time > - The benefit is that a follow-up turn (for example, after a user answers a question) continues instead of failing ## Linked Issues or Issue Description Refs #19525 (keeps the accepted result in the same incident; this is the next run). **What happened?** In the runner E2E cell `local-session-integrity.runner-acpx-claude.local.structured-question-restart-resume`, the first turn asked a question and its result was accepted. The Runner then logged `runner did not prove durable suspension before checkpoint` (`runner.suspend` pending, Runner lifecycle `ready`). After a server restart and the answer, the next run failed with `runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate; retirement_refused=target_not_sandbox_runner`, and its root was quarantined. **Expected behavior** When the prior local Runner is provably stopped and every other retirement proof holds, the next run retires that owner as a continuity break and continues. If the stop cannot be proven, the run keeps the named rejection. **Steps to reproduce** 1. Run a local per-turn native turn whose `runner.suspend` does not finish after its result is accepted (host load average ~40). 2. Start the next turn for the same session. 3. See `retirement_refused=target_not_sandbox_runner`. ## What Changed - `server/src/services/native-runtime/native-restart-recovery.ts`: add `proveNativeLocalRunnerStopped`. It uses the existing restart-recovery checks: the recorded Runner PID is dead or now belongs to a process with a different start time; its recorded process group is not alive; each recorded provider process is dead or recycled. A missing PID, start time or process group, a live recorded process, or a start time that cannot be compared is not proof. When settlement already cleared the run's process fields, the saved `native.local_process_stopped` receipt is the Runner proof. Extract `nativeProviderProcessIdentities` (checkpoint plus run events) so restart recovery and this proof read provider PIDs the same way. - `server/src/services/native-runtime/native-session-executor.ts` (`retireFinishedPerTurnRunnerdOwner`, `readSettledRunnerdProjection`): a local target is now eligible. For a local Runner, the projection accepts the Runner's own unsuspended receipt for the same identity and unfinished Runner lifecycle commands (`runner.suspend`, `runner.drain`, `runner.shutdown`) only. All other proofs are unchanged: committed result, released lease, no recovery state, stopped controller, delivered terminal, other commands complete, tool inputs answered, quarantine copies. The process proof runs before retirement and again just before the archive. - `doc/execution-semantics.md`: describe the local case. - Tests: seven integration cases in `native-recovered-owner-followup.test.ts` and ten unit cases in `native-restart-recovery.test.ts`. ## Verification - Red before the fix: the four new integration cases failed with `retirement_refused=target_not_sandbox_runner`, which is the incident error. - Green after the fix: `pnpm exec vitest run src/services/native-runtime/native-recovered-owner-followup.test.ts` 35/35. The new cases prove that a stopped Runner is retired (root archived under `continuity-breaks`, no quarantine), and that a live recorded Runner (`runner_process_alive`), a missing identity (`runner_process_identity_missing`), and a pending non-lifecycle command (`journal_commands_unsettled(turn.start.pending)`) keep the rejection. All existing sandbox cases still pass, including "keeps an unsuspended Runner receipt decisive" for sandbox targets. - `native-restart-recovery.test.ts` unit cases: recycled PID, live process, unknown start time, live group, live provider, missing PID or start time. - `native-session-executor.test.ts` + `native-restart-recovery.test.ts`: 1190 pass. `native-runner-restart-recovery.integration.test.ts` + `native-restart-recovery-isolation.test.ts` (shared provider-PID extraction): 81 pass. `tsc --noEmit -p server/tsconfig.json`: pass. - Live: `pnpm test:e2e:runner -- --max-parallel 1 --max-automatic-retries 0 --id local-session-integrity.runner-acpx-claude.local.structured-question-restart-resume`, 3 runs at eec57dd5ed: 3/3 PASS. The host load average was ~15 (the incident had ~40), and `runner.suspend` finished in all three runs, so these runs show no regression but did not exercise the new retirement path. The integration cases cover that path. ## Risks - The new path lets a follow-up archive a local root whose Runner receipt is unsuspended. This is safe only if the Runner is stopped, so the proof uses exact PID and start-time identity and fails closed on any uncertainty. `ps` start times have one-second precision; a sub-second difference is treated as the same process (not proof). - A zombie-only process group on macOS can report as alive. That keeps the rejection, as before. - No quarantine behavior changes. A refused retirement still quarantines the root as before. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, through Claude Code with tool use (shell, file edits, GitHub CLI). Extended reasoning on. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 16 ++ .../native-recovered-owner-followup.test.ts | 104 +++++++++- .../native-restart-recovery.test.ts | 40 ++++ .../native-runtime/native-restart-recovery.ts | 183 +++++++++++------- .../native-runtime/native-session-executor.ts | 78 ++++++-- 5 files changed, 338 insertions(+), 83 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 5c71fd55f82..e5451eb7ace 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2379,6 +2379,22 @@ All of these must be true: - Each quarantined root for the scope passes the same journal and local Runner evidence checks for the same owner. +A local per-turn Runner can end the same way: the result was accepted, but its +`runner.suspend` command did not finish, so its own receipt is still +unsuspended. The next local run retires that owner with the same checks and +these differences: + +- The Runner's own receipt may be unsuspended. It may not be unreadable or + name another owner. +- Only the Runner's own lifecycle commands (`runner.suspend`, `runner.drain`, + `runner.shutdown`) may be unfinished. All other commands must be complete. +- The prior run's recorded Runner process is proven stopped by exact PID and + start time, as in restart recovery. A PID that now belongs to another + process counts as stopped. A live recorded process, a live process group, a + live recorded provider process, a missing identity, or a start time that + cannot be compared keeps the rejection, for example + `retirement_refused=runner_process_alive`. A later Retry checks again. + The provider session then follows the normal resume-or-replace policy. A resumed sandbox, a warm session, a plugin task, a restart claim, or any missing proof keeps the existing `prior_owner_terminal_state_indeterminate` diff --git a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts index 8185661619d..e969b20d0c2 100644 --- a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts +++ b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts @@ -185,7 +185,11 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered runner: { execute: vi.fn(async () => { throw new Error("admission must not reach the sandbox"); }) }, sandboxLeaseAcquisition: { outcome, providerLeaseId: `sandbox-${runId}` } } as never, }).then(() => "constructed", (error: Error) => error.message); - return { runId, logs, start, selectedRunId: selected?.id ?? null }; + // A local Runner keeps its controller root on this host between turns. + const startLocal = () => createRunnerdBackend({ + db, execution, runnerInstanceId: next.runnerInstanceId, onLog: async (_stream, chunk) => { logs.push(chunk); }, + }).then(() => "constructed", (error: Error) => error.message); + return { runId, logs, start, startLocal, selectedRunId: selected?.id ?? null }; } async function quarantined() { @@ -465,6 +469,104 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered expect(followUp.logs.join("")).not.toContain("finished without a saved runner checkpoint"); }, 60_000); + /** A spawned process that stands in for the prior local Runner. */ + async function runnerProcess() { + const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60_000)"], { stdio: "ignore" }); + await new Promise(resolve => child.once("spawn", resolve)); + const startedAt = await readProcessStartedAt(child.pid!); + expect(startedAt).toBeTruthy(); + return { child, pid: child.pid!, processStartedAt: new Date(startedAt!) }; + } + + async function stoppedRunnerProcess() { + const runner = await runnerProcess(); + const exited = new Promise(resolve => runner.child.once("exit", resolve)); + runner.child.kill("SIGKILL"); + await exited; + return runner; + } + + /** The state a local per-turn Runner leaves when its result was accepted but + * its suspend command never finished: the controller journal holds a pending + * runner.suspend and the Runner's own receipt is still `ready`. */ + async function seedUnsuspendedLocalOwner(runner: { pid: number; processStartedAt: Date }) { + const prior = await seedRecoveredPriorRun("pending-command"); + await db.update(heartbeatRuns).set({ processPid: runner.pid, processGroupId: runner.pid, processStartedAt: runner.processStartedAt }) + .where(eq(heartbeatRuns.id, prior.runId)); + const followUp = await admitFollowUp(prior); + // Learn the scope key from a held attempt, then keep its root in place. + expect(await followUp.start()).toMatch(/^runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate/); + const [copy] = await quarantined(); + const root = join(stateDirectory, copy!.split(".")[0]!); + await rename(join(stateDirectory, "quarantine", copy!), root); + await rm(join(stateDirectory, "quarantine"), { recursive: true, force: true }); + await mkdir(join(root, "runner"), { recursive: true }); + await writeFile(join(root, "runner", "runner-state.json"), JSON.stringify({ + schema: "paperclip.runner.durable.state.v1", lifecycle: "ready", ...prior.identity })); + return { prior, followUp, root }; + } + + it("retires a finished local owner whose Runner stopped before it proved suspension", async () => { + const { prior, followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + expect(await followUp.startLocal()).toBe(ADMITTED); + expect(await quarantined()).toEqual([]); + expect(existsSync(join(root, "runner"))).toBe(false); + expect(await archivedOwners(root)).toEqual([prior.runId]); + expect(followUp.logs.join("")).toContain(`Run ${prior.runId} finished without a saved runner checkpoint`); + }, 60_000); + + it("keeps a finished local owner held while its recorded Runner process is alive", async () => { + const runner = await runnerProcess(); + try { + const { followUp } = await seedUnsuspendedLocalOwner(runner); + expect(await followUp.startLocal()).toBe(stranded("runner_process_alive")); + expect(followUp.logs.join("")).not.toContain("finished without a saved runner checkpoint"); + } finally { + runner.child.kill("SIGKILL"); + } + }, 60_000); + + it.each(["no identity", "no process group"] as const)("keeps a finished local owner held with %s", async scenario => { + const { prior, followUp } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + await db.update(heartbeatRuns).set(scenario === "no identity" + ? { processPid: null, processGroupId: null, processStartedAt: null } : { processGroupId: null }) + .where(eq(heartbeatRuns.id, prior.runId)); + expect(await followUp.startLocal()).toBe(stranded("runner_process_identity_missing")); + }, 60_000); + + it("retires a finished local owner from its saved stop receipt after settlement cleared the process fields", async () => { + const runner = await stoppedRunnerProcess(); + const { prior, followUp, root } = await seedUnsuspendedLocalOwner(runner); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: prior.runId, seq: 900, eventType: "native.local_process_stopped", + stream: "system", payload: { processPid: runner.pid, processGroupId: runner.pid, processStartedAt: runner.processStartedAt.toISOString() } }); + await db.update(heartbeatRuns).set({ processPid: null, processGroupId: null, processStartedAt: null }) + .where(eq(heartbeatRuns.id, prior.runId)); + expect(await followUp.startLocal()).toBe(ADMITTED); + expect(await archivedOwners(root)).toEqual([prior.runId]); + }, 60_000); + + it("keeps a stopped local owner held while a provider process from its run events is alive", async () => { + const provider = await runnerProcess(); + try { + const { prior, followUp } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: prior.runId, seq: 901, eventType: "provider.event", + stream: "system", payload: { prpEvent: { payload: { processId: provider.pid, + processStartedAt: provider.processStartedAt.toISOString() } } } }); + expect(await followUp.startLocal()).toBe(stranded("provider_process_alive")); + } finally { + provider.child.kill("SIGKILL"); + } + }, 60_000); + + it("keeps a stopped local owner held by an unfinished non-lifecycle command", async () => { + const { followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + const path = join(root, "control-plane", "control-plane-state.json"); + const state = JSON.parse(await readFile(path, "utf8")); + state.commands.find((command: { type: string }) => command.type === "turn.start").status = "pending"; + await writeFile(path, JSON.stringify(state)); + expect(await followUp.startLocal()).toBe(stranded("journal_commands_unsettled(turn.start.pending)")); + }, 60_000); + it.each(["uncommitted result", "committed without result", "live controller lease", "unexpired controller lease", "recovery pending", "pending command", "unanswered tool input", "foreign quarantine"] as const)("keeps the prior owner held with %s", async scenario => { const prior = await seedRecoveredPriorRun(scenario === "pending command" ? "pending-command" diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index fd79245b6e5..52d8d15c9b7 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -5,8 +5,48 @@ import { evaluateNativeControllerTakeover, evaluateNativeProviderProcesses, nextNativeProviderAttempt, + proveNativeLocalRunnerStopped, } from "./native-restart-recovery.js"; +describe("local runner stop proof", () => { + const startedAt = new Date("2026-11-06T17:58:54.000Z"); + const base = { processPid: 72454, processGroupId: 72454, processStartedAt: startedAt }; + const probe = (alive: number[], starts: Record, groups: number[] = []) => ({ + isProcessAlive: (pid: number) => alive.includes(pid), + isProcessGroupAlive: (group: number) => groups.includes(group), + readProcessStartedAt: async (pid: number) => starts[pid] ?? null, + }); + + it.each([ + ["the runner and its group exited", base, probe([], {}), { stopped: true }], + ["the runner PID now belongs to another process", base, + probe([72454], { 72454: new Date(startedAt.getTime() + 60_000) }), { stopped: true }], + ["the recorded runner is alive", base, probe([72454], { 72454: startedAt }), + { stopped: false, reason: "runner_process_alive" }], + ["the live PID start time is unknown", base, probe([72454], {}), + { stopped: false, reason: "runner_process_identity_unverifiable" }], + ["the runner group is alive", base, probe([], {}, [72454]), + { stopped: false, reason: "runner_process_group_alive" }], + ["no runner PID was recorded", { ...base, processPid: null }, probe([], {}), + { stopped: false, reason: "runner_process_identity_missing" }], + ["no runner start time was recorded", { ...base, processStartedAt: null }, probe([], {}), + { stopped: false, reason: "runner_process_identity_missing" }], + ["no runner process group was recorded", { ...base, processGroupId: null }, probe([], {}), + { stopped: false, reason: "runner_process_identity_missing" }], + ["a stop receipt proved the runner absent", { ...base, processStartedAt: null, runnerStopRecorded: true }, + probe([], {}), { stopped: true }], + ] as const)("decides when %s", async (_case, identity, checks, expected) => { + await expect(proveNativeLocalRunnerStopped({ ...identity, ...checks })).resolves.toEqual(expected); + }); + + it("refuses while a recorded provider process is alive", async () => { + const providerStartedAt = new Date("2026-11-06T17:58:56.000Z"); + await expect(proveNativeLocalRunnerStopped({ ...base, + providerProcesses: [{ pid: 72460, processStartedAt: providerStartedAt }], + ...probe([72460], { 72460: providerStartedAt }) })).resolves.toEqual({ stopped: false, reason: "provider_process_alive" }); + }); +}); + describe("native restart recovery classification", () => { it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => { const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false, diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index d750a83a141..089d421450e 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -336,6 +336,118 @@ export async function evaluateNativeProviderProcesses(input: { }; } +/** Run event types that can carry a provider process identity. */ +export const NATIVE_PROVIDER_PROCESS_EVENT_TYPES = [ + "harness.ready", + "session.started", + "session.resumed", + "session.updated", + "turn.started", + "provider.event", + "provider.rpc_result", +] as const; + +function parsedProcessStart(value: unknown): Date | null { + const parsed = typeof value === "string" ? new Date(value) : null; + return parsed && !Number.isNaN(parsed.getTime()) ? parsed : null; +} + +/** Provider process identities from a saved checkpoint and recent run events. */ +export function nativeProviderProcessIdentities( + checkpoint: Record, + providerEvents: ReadonlyArray<{ payload: Record | null }>, +): NativeProviderProcessIdentity[] { + const identities: NativeProviderProcessIdentity[] = []; + const checkpointProcess = + checkpoint.process && typeof checkpoint.process === "object" && !Array.isArray(checkpoint.process) + ? (checkpoint.process as Record) + : {}; + for (const [field, startedAtField] of [ + ["providerPid", "providerProcessStartedAt"], + ["codexPid", "codexProcessStartedAt"], + ["sidecarPid", "sidecarProcessStartedAt"], + ["agentPid", "agentProcessStartedAt"], + ] as const) { + const value = checkpointProcess[field]; + if (typeof value === "number" && Number.isInteger(value) && value > 0) { + identities.push({ pid: value, processStartedAt: parsedProcessStart(checkpointProcess[startedAtField]) }); + } + } + for (const providerEvent of providerEvents) { + const prpEvent = providerEvent.payload?.prpEvent; + const event = prpEvent && typeof prpEvent === "object" && !Array.isArray(prpEvent) + ? (prpEvent as Record) : {}; + const eventPayload = event.payload && typeof event.payload === "object" && !Array.isArray(event.payload) + ? (event.payload as Record) : {}; + const processId = eventPayload.processId; + if (typeof processId === "number" && Number.isInteger(processId) && processId > 0) { + identities.push({ pid: processId, + processStartedAt: parsedProcessStart(eventPayload.processStartedAt ?? eventPayload.providerProcessStartedAt) }); + } + } + return identities; +} + +export type NativeLocalRunnerStopProof = + | { stopped: true } + | { + stopped: false; + reason: + | "runner_process_identity_missing" + | "runner_process_alive" + | "runner_process_identity_unverifiable" + | "runner_process_group_alive" + | "provider_process_alive" + | "provider_process_identity_unverifiable"; + }; + +/** + * Proves that a finished local runner and its recorded provider processes + * stopped, with the same exact PID and start-time checks as restart recovery. + * A PID that now belongs to a different process counts as stopped. A live + * process with the recorded identity, a live process group, or a live PID + * whose start time cannot be compared is not proof. + */ +export async function proveNativeLocalRunnerStopped(input: { + processPid: number | null; + processGroupId: number | null; + processStartedAt: Date | null; + providerProcesses?: readonly NativeProviderProcessIdentity[]; + /** A current, server-authored stop receipt already proved this exact Runner and group absent. */ + runnerStopRecorded?: boolean; + isProcessAlive?: (pid: number) => boolean; + isProcessGroupAlive?: (processGroupId: number) => boolean; + readProcessStartedAt?: (pid: number) => Promise; +}): Promise { + const refuse = (reason: Extract["reason"]) => + ({ stopped: false as const, reason }); + // A local Runner leads its own process group. Without both identities, a + // live child in that group could outlive the Runner unseen. + if (!input.processPid || !Number.isInteger(input.processPid) || input.processPid <= 1 || !input.processGroupId + || (!input.processStartedAt && !input.runnerStopRecorded)) { + return refuse("runner_process_identity_missing"); + } + if (!input.runnerStopRecorded) { + const runner = await evaluateNativeProviderProcesses({ + identities: [{ pid: input.processPid, processStartedAt: input.processStartedAt }], + isProcessAlive: input.isProcessAlive, + readProcessStartedAt: input.readProcessStartedAt, + }); + if (runner.livePids.length > 0) return refuse("runner_process_alive"); + if (runner.ambiguousLivePids.length > 0) return refuse("runner_process_identity_unverifiable"); + } + const groupAlive = input.isProcessGroupAlive ?? processGroupIsAlive; + if (groupAlive(input.processGroupId)) return refuse("runner_process_group_alive"); + const providers = await evaluateNativeProviderProcesses({ + identities: (input.providerProcesses ?? []).filter(identity => identity.pid !== input.processPid), + isProcessAlive: input.isProcessAlive, + readProcessStartedAt: input.readProcessStartedAt, + }); + if (providers.livePids.length > 0) return refuse("provider_process_alive"); + if (providers.ambiguousLivePids.length > 0) return refuse("provider_process_identity_unverifiable"); + return { stopped: true }; +} + export function classifyNativeRunnerRecoveryEvidence(input: { runnerPidAlive: boolean; runnerGroupAlive: boolean; @@ -898,79 +1010,12 @@ export async function claimNativeRestartRecoveries(input: { .where( and( eq(heartbeatRunEvents.runId, row.run.id), - inArray(heartbeatRunEvents.eventType, [ - "harness.ready", - "session.started", - "session.resumed", - "session.updated", - "turn.started", - "provider.event", - "provider.rpc_result", - ]), + inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_PROCESS_EVENT_TYPES]), ), ) .orderBy(desc(heartbeatRunEvents.id)) .limit(100); - const checkpointProcess = - checkpointRecord.process && - typeof checkpointRecord.process === "object" && - !Array.isArray(checkpointRecord.process) - ? (checkpointRecord.process as Record) - : {}; - const providerProcessIdentities: NativeProviderProcessIdentity[] = []; - const checkpointProcessFields = [ - ["providerPid", "providerProcessStartedAt"], - ["codexPid", "codexProcessStartedAt"], - ["sidecarPid", "sidecarProcessStartedAt"], - ["agentPid", "agentProcessStartedAt"], - ] as const; - for (const [field, startedAtField] of checkpointProcessFields) { - const value = checkpointProcess[field]; - if (typeof value === "number" && Number.isInteger(value) && value > 0) { - const rawStartedAt = checkpointProcess[startedAtField]; - const parsedStartedAt = - typeof rawStartedAt === "string" ? new Date(rawStartedAt) : null; - providerProcessIdentities.push({ - pid: value, - processStartedAt: - parsedStartedAt && !Number.isNaN(parsedStartedAt.getTime()) - ? parsedStartedAt - : null, - }); - } - } - for (const providerEvent of providerEvents) { - const prpEvent = providerEvent.payload?.prpEvent; - const event = - prpEvent && typeof prpEvent === "object" && !Array.isArray(prpEvent) - ? (prpEvent as Record) - : {}; - const eventPayload = - event.payload && - typeof event.payload === "object" && - !Array.isArray(event.payload) - ? (event.payload as Record) - : {}; - const processId = eventPayload.processId; - if ( - typeof processId === "number" && - Number.isInteger(processId) && - processId > 0 - ) { - const rawStartedAt = - eventPayload.processStartedAt ?? - eventPayload.providerProcessStartedAt; - const parsedStartedAt = - typeof rawStartedAt === "string" ? new Date(rawStartedAt) : null; - providerProcessIdentities.push({ - pid: processId, - processStartedAt: - parsedStartedAt && !Number.isNaN(parsedStartedAt.getTime()) - ? parsedStartedAt - : null, - }); - } - } + const providerProcessIdentities = nativeProviderProcessIdentities(checkpointRecord, providerEvents); const providerProcesses = await evaluateNativeProviderProcesses({ identities: (remoteProcess ? [] : providerProcessIdentities).filter( (identity) => identity.pid !== row.run.processPid, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index fd1216fe57c..98fee67a06c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -292,6 +292,10 @@ import { import { currentNativeControllerIdentity, evaluateNativeControllerTakeover, + proveNativeLocalRunnerStopped, + nativeProviderProcessIdentities, + NATIVE_PROVIDER_PROCESS_EVENT_TYPES, + type NativeLocalRunnerStopProof, isCurrentNativeControllerProcess, nextNativeProviderAttempt, type NativeControllerIdentity, @@ -5286,15 +5290,25 @@ function moveNativeContinuityState(root: string, archiveRoot: string): void { * Runner outbox can hold only post-terminal telemetry. Local lifecycle * evidence, even an unsuspended receipt, and unknown entries are owner * claims that this proof never overrides. + * + * `localRunner` names a local Runner that wrote its own lifecycle receipt in + * this root. Its unsuspended receipt and its own unfinished lifecycle commands + * (suspend, drain, shutdown) are claims of that one process. The caller must + * separately prove that process stopped before it relies on this projection. */ type SettledRunnerdProjection = { bytes: string; refused?: never } | { bytes?: never; refused: string }; -function readSettledRunnerdProjection(root: string, identity: Record): SettledRunnerdProjection { +const LOCAL_RUNNER_LIFECYCLE_COMMANDS = new Set(["runner.suspend", "runner.drain", "runner.shutdown"]); + +function readSettledRunnerdProjection(root: string, identity: Record, + options: { localRunner?: boolean } = {}): SettledRunnerdProjection { const refuse = (refused: string): SettledRunnerdProjection => ({ refused }); try { if (!isSafeNativeStateDirectory(root)) return refuse("root_unsafe"); const lifecycle = runnerdAuthorityLifecycle(root, identity as RunnerdDurableIdentity); - if (lifecycle !== "absent") return refuse(`runner_lifecycle_${lifecycle}`); + if (lifecycle !== "absent" && !(options.localRunner && lifecycle === "not_suspended")) { + return refuse(`runner_lifecycle_${lifecycle}`); + } // Every legitimate writer's entry is classified in native-session-root-entries.ts. // Held entries are known evidence that needs its own settlement; anything // else is writer drift. Name each kind separately. @@ -5306,6 +5320,11 @@ function readSettledRunnerdProjection(root: string, identity: Record !LOCAL_RUNNER_LIFECYCLE_COMMANDS.has(String(command.type)) || command.status === "completed") + : commands; + const settledControl = settledCommands === commands ? control : { ...control, commands: settledCommands }; const answered = new Set(commands .filter(command => command.type === "semantic_tool.result" && command.status === "completed") .map(command => record(command.payload).sourceEventId)); @@ -5315,11 +5334,11 @@ function readSettledRunnerdProjection(root: string, identity: Record record(control.identity)[key] !== value)) { return refuse("journal_identity_mismatch"); } - if (!isSettledRemoteWarmControlState(control, identity)) { + if (!isSettledRemoteWarmControlState(settledControl, identity)) { // Closed vocabulary only: known command types and statuses, never payloads. - const pending = commands.filter(command => command.status !== "completed") + const unsettled = settledCommands.filter(command => command.status !== "completed") .map(command => `${retirementReasonName("commandType", command.type)}.${retirementReasonName("commandStatus", command.status)}`); - return refuse(pending.length > 0 ? `journal_commands_unsettled(${[...new Set(pending)].slice(0, 4).join(",")})` + return refuse(unsettled.length > 0 ? `journal_commands_unsettled(${[...new Set(unsettled)].slice(0, 4).join(",")})` : "journal_terminal_unsettled"); } if ((control.committedEvents as unknown[]).map(record).some(event => @@ -5363,6 +5382,12 @@ function retirementReasonName(kind: keyof typeof RETIREMENT_REASON_NAMES, value: * that authority as a continuity break instead. This run then bootstraps its * own authority, and the provider session follows the ordinary * resume-or-replace policy. Any missing proof keeps the original rejection. + * + * A local per-turn Runner can end the same way: the result is accepted but its + * suspend command does not finish, so its own receipt stays unsuspended. That + * receipt is a claim of one recorded Runner process. Once that process and its + * recorded provider processes are proven stopped by exact PID and start time, + * the claim has no owner left and the same retirement applies. */ async function retireFinishedPerTurnRunnerdOwner(input: { db: Db; @@ -5377,11 +5402,14 @@ async function retireFinishedPerTurnRunnerdOwner(input: { // existing recovery path reads. Plugin tasks own a separate saved checkpoint. const refuse = (refused: string) => ({ retired: false as const, refused }); if (input.execution.session.lifecyclePolicy.mode !== "per_turn") return refuse("lifecycle_not_per_turn"); - if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) { - return refuse("target_not_sandbox_runner"); - } - if (!["created", "replacement"].includes(target.sandboxLeaseAcquisition?.outcome ?? "")) { - return refuse(`sandbox_${retirementReasonName("sandboxOutcome", target.sandboxLeaseAcquisition?.outcome)}`); + const localRunner = !target || target.kind === "local"; + if (!localRunner) { + if (target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) { + return refuse("target_not_sandbox_runner"); + } + if (!["created", "replacement"].includes(target.sandboxLeaseAcquisition?.outcome ?? "")) { + return refuse(`sandbox_${retirementReasonName("sandboxOutcome", target.sandboxLeaseAcquisition?.outcome)}`); + } } const identity: Record = {}; for (const key of ["runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId"]) { @@ -5389,7 +5417,7 @@ async function retireFinishedPerTurnRunnerdOwner(input: { if (typeof value !== "string" || !value) return refuse("durable_identity_incomplete"); identity[key] = value; } - const projection = readSettledRunnerdProjection(input.root, identity); + const projection = readSettledRunnerdProjection(input.root, identity, { localRunner }); if (projection.refused !== undefined) return refuse(projection.refused); const controlBytes = projection.bytes; const { binding } = input.execution; @@ -5421,6 +5449,29 @@ async function retireFinishedPerTurnRunnerdOwner(input: { remoteLeaseFenced: readNativeControllerRecovery(profile.nativeControllerRecovery, profile.sessionCheckpoint) !== null }); if (!stopped.allowed) return refuse(`controller_${retirementReasonName("controllerReason", stopped.reason)}`); const provenFinishedHere = finishedHere(); + const proveLocalRunnerStopped = async (): Promise => { + const run = prior!.run; + // Settlement may already have cleared the process fields after it saved a + // server-authored stop receipt; that receipt is the identity to check. + const released = run.processPid === null && run.processGroupId === null && run.processStartedAt === null; + const receipt = released ? await readNativeLocalProcessStop(input.db, run.companyId, run.id) : null; + if (released && !receipt) return { stopped: false, reason: "runner_process_identity_missing" }; + const providerEvents = await input.db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), + inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_PROCESS_EVENT_TYPES]))) + .orderBy(desc(heartbeatRunEvents.id)).limit(100); + return proveNativeLocalRunnerStopped({ + processPid: receipt?.processPid ?? run.processPid, + processGroupId: receipt?.processGroupId ?? run.processGroupId, + processStartedAt: receipt ? null : run.processStartedAt, + runnerStopRecorded: receipt !== null, + providerProcesses: nativeProviderProcessIdentities(record(profile.sessionCheckpoint), providerEvents), + }); + }; + if (localRunner) { + const runnerStopped = await proveLocalRunnerStopped(); + if (!runnerStopped.stopped) return refuse(runnerStopped.reason); + } // Earlier rejected admissions can quarantine the same projection. Each copy // must pass the same proof: a rebuilt root never carries the local Runner // evidence that held an earlier attempt, so that evidence stays decisive. @@ -5429,13 +5480,14 @@ async function retireFinishedPerTurnRunnerdOwner(input: { if (!isSafeNativeStateDirectory(quarantine)) return refuse("quarantine_unsafe"); for (const name of readdirSync(quarantine)) { if (!name.startsWith(`${basename(input.root)}.`)) continue; - const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity); + const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity, { localRunner }); if (copy.refused !== undefined) return refuse(`quarantine_${copy.refused}`); } } // Database reads can await. Retire only the exact evidence proven above. - if (readSettledRunnerdProjection(input.root, identity).bytes !== controlBytes) return refuse("journal_changed"); + if (readSettledRunnerdProjection(input.root, identity, { localRunner }).bytes !== controlBytes) return refuse("journal_changed"); if (provenFinishedHere && !finishedHere()) return refuse("controller_restarted_here"); + if (localRunner && !(await proveLocalRunnerStopped()).stopped) return refuse("runner_process_restarted"); const archivedAt = Date.now(); const archiveRoot = resolve(input.root, "continuity-breaks", `${archivedAt}-${randomUUID()}`); mkdirSync(archiveRoot, { recursive: true, mode: 0o700 }); From 876a2e4857b416cd64a7767135dbf75b4a365d4d Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:17:58 -0600 Subject: [PATCH 05/11] fix(recovery): recover a prior boot's unstarted run at lease expiry (#19578) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - After a server restart, a task must continue automatically and promptly when that is safe > - A user clicks Retry, and the server is killed before the new run starts a process > - Recovery leaves that run `running` until the orphan reap's five-minute stale-age threshold passes > - The user sees nothing happen for about five minutes > - This pull request recovers that run when its controller lease from the earlier boot expires > - The benefit is that the automatic retry starts about one lease period after the kill, not five minutes later ## Linked Issues or Issue Description **What happened?** A Retry admitted a legacy run. The server was killed before the run recorded a process. The run kept a controller lease from the killed boot and `executionStage: preparing`. After the restart, live recovery skipped the run until `updatedAt` was five minutes old (`server/src/services/heartbeat/recovery.ts`, `staleThresholdMs ?? 5 * 60_000`). Restart-chaos measured 5 m 01 s from run creation to `process_lost` and the automatic retry, twice (4 m 20 s after the server was ready). **Expected behavior** Recovery should claim the run as soon as the earlier boot's lease expires. The lease is the only liveness proof for a legacy controller, and no process was ever recorded. **Paperclip version or commit** `master` at `71313207eb`. **Steps to reproduce** Click Retry on an interrupted task, then SIGKILL the server within about one second. Observe the retried run stay `running` for about five minutes. ## What Changed - `recoverActiveExecutions` skips the stale-age gate for a legacy `running` run when all of these are true: its controller lease is expired (checked by the existing live-lease gate), the lease belongs to an earlier boot (`controllerBootId !== legacyControllerBootId`), the run is still `preparing` (no adapter dispatch), and the run has no recorded `processPid` or `processGroupId`. - The stage and process fields committed with the revocation are checked again. If the old controller dispatched before its lease expired, the run keeps the age guard. - The existing revocation (`revokeExpiredLegacyControllerWithState`) still requires the expired boot ID, so a concurrent renewal wins. The existing `restartInterruptedBeforeDispatch` rule still decides the single process-loss retry. - `doc/execution-semantics.md` describes the rule in section 10. - Two regressions in `server/src/__tests__/live-recovery-work.test.ts` (embedded Postgres, real residual-recovery workers). ## Verification - Red before the fix: `recovers a fresh run from a prior boot at lease expiry ...` failed with `expected 'running' to be 'failed'`. - Green after the fix: `pnpm exec vitest run src/__tests__/live-recovery-work.test.ts -t "stale-age|fresh foreign controller|suppressed live runs"` passes 4/4. The run fails `process_lost` with `executionRecovery.providerWorkStarted: false` within seconds of lease expiry, and exactly one retry runs to `succeeded`. - Race case: a DB trigger makes the row committed with the revocation say `dispatching` while the scan saw `preparing`. The run stays `running`, and no retry is created. - Guard cases: a run with an expired lease of the current boot, a run from an earlier boot with a recorded process, and a run from an earlier boot that was `dispatching` (for example a remote HTTP adapter) all stay `running` after four minutes. - Related suites: `heartbeat-process-recovery`, `heartbeat-observed-workspace-preservation`, `native-startup-cleanup-recovery`, `services/heartbeat/recovery.test.ts`, full `live-recovery-work`. - Live K8 probe (restart-chaos supervisor, local instance, ACPX Claude agent): the first run fails with a provider error, the probe clicks Retry, and the server is SIGKILLed 100 ms later, before the Retry run records a process. The probe then measures how long the Retry run stays `running`. | Build | Retry run created to `process_lost` | Server ready to `process_lost` | Automatic retry run | | --- | --- | --- | --- | | Base (`71313207eb`) | 300.1 s | 293.5 s | started at 5 m 00 s | | This PR | 60.1 s | 54.5 s | started 0.1 s after `process_lost` | The claim now happens when the killed boot's 60-second controller lease expires. In this probe the restart took about 6 s, so the claim came about 54 s after ready. In the restart-chaos waves, kill to ready took about 41 s, so the claim would come about 20 s after ready. - A chaos wave with the `retry` trigger (`w-20261106T195715-s2727`, branch #19513 merged locally) missed the trigger: no Retry was offered because recovery completed the task automatically (claude t01 PASS). The probe above makes the Retry-then-kill timing deterministic. ## Risks - Low risk. The change only removes an age wait. Ownership is still proved by the expired lease of another boot. A rolling deploy or hot restart whose earlier instance is still alive keeps renewing its lease, so the run is not touched. - Runs with a recorded process, and dispatched runs, keep the five-minute guard. A process on another host or a remote request is not inferred dead. - Native runs are not changed. Their ownership is in `native_run_finalizations` and startup native recovery. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code with tool use (shell, file edits, test execution). Extended reasoning enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 8 +++ .../src/__tests__/live-recovery-work.test.ts | 60 +++++++++++++++++++ server/src/services/heartbeat/recovery.ts | 26 +++++++- 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index e5451eb7ace..61af30f3920 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1647,6 +1647,14 @@ On startup and on the periodic recovery loop, Paperclip performs the following r 3. reconcile stranded assigned work 4. scan silent active runs only for source-aware terminal folding and legacy cleanup; API reads classify ordinary output silence for the board UI +The orphan reap normally waits until a `running` run has been unchanged for five +minutes. A legacy run whose controller lease belongs to an earlier server boot +and has expired, which was still preparing (before adapter dispatch), and which +has no recorded process is recovered when that lease expires instead. The +expired lease proves that the earlier owner is gone, and no provider work +started. A live lease, a lease of the current boot, a dispatched adapter, or a +recorded process keeps the five-minute guard. + The stranded-work pass closes the gap where issue state survives a crash but the wake/run path does not. The silent-run scan covers the separate case where a live process exists but has stopped producing observable output. Native finalization reconciliation (part of native recovery and the orphan diff --git a/server/src/__tests__/live-recovery-work.test.ts b/server/src/__tests__/live-recovery-work.test.ts index 1adb2d01bfb..3c774a05dda 100644 --- a/server/src/__tests__/live-recovery-work.test.ts +++ b/server/src/__tests__/live-recovery-work.test.ts @@ -301,6 +301,66 @@ describe("residual recovery production matrix", () => { }, { timeout: 45_000 }); await competing.stop(); extraStops.pop(); await quiet(); }); + it("recovers a fresh run from a prior boot at lease expiry instead of waiting for the stale-age threshold", async () => { + // RC-27: a Retry admitted just before a kill records a controller lease but + // no process. The prior boot's expired lease is the ownership proof. + const f = await fixture(), expiry = new Date(Date.now() + 3_000); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running", + runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expiry, + executionStage: "preparing", updatedAt: new Date(), startedAt: new Date() }).returning(); + await start(true); + expect((await getRun(run.id)).status).toBe("running"); + await vi.waitFor(async () => expect((await getRun(run.id)).status).toBe("failed"), { timeout: 15_000 }); + expect(Date.now() - expiry.getTime()).toBeLessThan(15_000); + expect(await getRun(run.id)).toMatchObject({ errorCode: "process_lost", + resultJson: expect.objectContaining({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }) }); + await vi.waitFor(async () => { + const retries = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id)); + expect(retries).toHaveLength(1); + await heartbeat.drainActiveRunExecutions(); + expect((await getRun(retries[0].id)).status).toBe("succeeded"); + }, { timeout: 45_000 }); + await quiet(); + }); + it("keeps the age guard when the prior boot dispatched between the scan and the revocation", async () => { + // Stage the race: the scanned row says `preparing`, and the row committed + // with the revocation says `dispatching` (the old controller's last renewal). + const f = await fixture(), expiry = new Date(Date.now() + 2_000); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running", + runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expiry, + executionStage: "preparing", updatedAt: new Date(), startedAt: new Date() }).returning(); + await db.execute(sql.raw(`create or replace function rc27_dispatch_on_revoke() returns trigger language plpgsql as $$ + begin if new.id = '${run.id}' and new.controller_boot_id is distinct from old.controller_boot_id + then new.execution_stage := 'dispatching'; end if; return new; end $$`)); + await db.execute(sql`create trigger rc27_dispatch_on_revoke before update on heartbeat_runs + for each row execute function rc27_dispatch_on_revoke()`); + extraStops.push(async () => { + await db.execute(sql`drop trigger if exists rc27_dispatch_on_revoke on heartbeat_runs`); + await db.execute(sql`drop function if exists rc27_dispatch_on_revoke()`); + }); + await start(true); + await vi.waitFor(async () => expect((await getRun(run.id)).controllerBootId).not.toBe(run.controllerBootId), { timeout: 15_000 }); + await realDelay(5_000); await Promise.allSettled([...passes]); + expect(await getRun(run.id)).toMatchObject({ status: "running", executionStage: "dispatching", errorCode: null }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id))).toEqual([]); + }); + it("keeps the stale-age guard for this boot, a recorded process, or a dispatched adapter", async () => { + const f = await fixture(), now = Date.now(), expired = new Date(now - 1_000); + const [sameBoot] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running", + runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: legacyControllerBootId, controllerLeaseExpiresAt: expired, + executionStage: "preparing", updatedAt: new Date(now), startedAt: new Date(now) }).returning(); + const [withProcess] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running", + runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expired, + executionStage: "dispatching", processPid: 2 ** 22 + 17, updatedAt: new Date(now), startedAt: new Date(now) }).returning(); + // A dispatched adapter may hold remote work without a local process. + const [dispatched] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running", + runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expired, + executionStage: "dispatching", updatedAt: new Date(now), startedAt: new Date(now) }).returning(); + await start(); await advance(4 * 60_000); + expect((await getRun(sameBoot.id)).status).toBe("running"); + expect((await getRun(withProcess.id)).status).toBe("running"); + expect((await getRun(dispatched.id)).status).toBe("running"); + }); it.each(["confirmed", "already_retired", "missing_terminal", "unproven_terminal", "ack_failure", "late_terminal"] as const)( "settles an API-only Stop after deferred controller retirement without inventing proof: %s", async mode => { const f = await fixture(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID(); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index aedc155f5fd..065ecafa922 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -2471,8 +2471,23 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD const legacyLease = await liveLegacyControllerExpiresAt(db, run); if (legacyLease) { scope.onDeferredDeadline?.(legacyLease.getTime()); continue; } + // A legacy controller proves it is alive only by renewing its lease, and + // the check above found no live lease (a missing expiry also lands here; + // revocation below then refuses the row). When the lease belongs to a + // previous boot, setup never reached adapter dispatch, and no process + // was recorded, the owner is gone and no provider work can be running: + // recover now instead of waiting for the age threshold. A dispatched + // adapter (for example a remote HTTP request) keeps the age guard. The + // stage is rechecked from the state committed with the revocation below. + const priorBootSetupExpired = + run.runtimeMode === "legacy" && + run.executionStage === "preparing" && + !!run.controllerBootId && + run.controllerBootId !== legacyControllerBootId && + !run.processPid && + !run.processGroupId; // Apply staleness threshold to avoid false positives - if (staleThresholdMs > 0) { + if (staleThresholdMs > 0 && !priorBootSetupExpired) { const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0; if (now.getTime() - refTime < staleThresholdMs) { scope.onDeferredDeadline?.(refTime + staleThresholdMs); @@ -2543,6 +2558,15 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD (!monitorNextCheckAt || monitorNextCheckAt.getTime() <= now.getTime()); const revokedState = await revokeExpiredLegacyControllerWithState(db, run); if (!revokedState) continue; + // The earlier snapshot may predate a final renewal that admitted the + // adapter. Revocation leaves a fresh foreign lease, so a later pass + // applies the age guard to that dispatched run. + if (priorBootSetupExpired && staleThresholdMs > 0 && (revokedState.executionStage !== "preparing" + || revokedState.processPid || revokedState.processGroupId)) { + const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0; + scope.onDeferredDeadline?.(Math.max(refTime + staleThresholdMs, Date.now() + 60_000)); + continue; + } // A legacy controller persists "dispatching" in the same lease renewal // that admits the adapter. An expired lease from another boot that is // still "preparing" proves a restart stopped setup before provider work. From 38068de5533793e5a7a3c9868a129eb2d3f4dff2 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:22:02 -0600 Subject: [PATCH 06/11] chore(evals): commit the Reliability TestOps plan and case identities (#19604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Runner E2E cases run from Allure TestOps plans through `runner-full-stack-e2e.yml` > - #19549 added the "Reliability — staging soak" plan. Its selection fails closed until real TestOps identities are committed > - Registration run https://github.com/paperclipai/paperclip/actions/runs/42289820474 on master (02214b1a3c) completed. It created plan 135 with cases 885–891 and assigned the plan to the standing GitHub job 166 > - This pull request commits those identities into `reliability-testops.mjs` > - The benefit is that a TestOps launch of plan 135 now selects the seven reliability cells ## Linked Issues or Issue Description Refs #19549 Refs #19601 **What happened?** The Reliability TestOps plan exists in TestOps, but the repository still had an empty `reliabilityRegistration`. A launch therefore failed closed with `product_regression_testops_reliability_unregistered`. **Expected behavior** A launch of plan 135 maps each TestOps case ID to its runner E2E cell and runs it. **Steps to reproduce** 1. Launch plan 135 from TestOps before this change. 2. The selection fails with `reliability_unregistered`. ## What Changed - `tests/runner-e2e/reliability-testops.mjs`: set `reliabilityRegistration` to plan 135 and the seven case IDs from the registration receipt. ## Verification - `node --test tests/runner-e2e/reliability-testops.test.mjs tests/runner-e2e/product-regression-testops.test.mjs`: 28/28 pass. - `registeredReliabilityCases()` with plan-2 IDs reserved resolves all seven cells: 885 interrupt-redirect, 886 reassign-midrun, 887 three-turn-revise, 888 long-silent-command, 889 tree-pause-release, 890 queue-steer (Codex), 891 queued-across-restart. - The registration receipt from run 42289820474 reports `complete: true` with assignment and parameters created. ## Risks - Low risk. The change only adds identities. A launch of plan 135 runs seven paid cells under the existing admission and budget caps. Nothing runs per PR or on a schedule. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`) in Claude Code with tool use (shell, GitHub CLI), standard context. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- tests/runner-e2e/reliability-testops.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/reliability-testops.mjs b/tests/runner-e2e/reliability-testops.mjs index 0262cb18fda..d687eb4cf30 100644 --- a/tests/runner-e2e/reliability-testops.mjs +++ b/tests/runner-e2e/reliability-testops.mjs @@ -37,8 +37,16 @@ export const reliabilityDefinitions = Object.freeze([ // Until every case and the plan have an identity, selecting this plan fails // closed with `product_regression_testops_reliability_unregistered`. export const reliabilityRegistration = Object.freeze({ - planId: null, - caseIds: Object.freeze({}), + planId: 135, + caseIds: Object.freeze({ + 'interrupt-redirect': 885, + 'reassign-midrun': 886, + 'three-turn-revise': 887, + 'long-silent-command': 888, + 'tree-pause-release': 889, + 'queue-steer': 890, + 'queued-across-restart': 891, + }), }); const fail = code => { throw new Error(`product_regression_testops_${code}`); }; From 2e35730637148b02c23aaeab413eef7be3e55652 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:22:51 -0600 Subject: [PATCH 07/11] feat(chat): receive sealed Slack events through the current SDK owner (#19019) ## Thinking Path > - Paperclip manages AI agents and their work. > - Slack connectors turn authenticated messages into company work. > - Direct callbacks need an awake tenant. > - Cloud can retain authenticated asynchronous events until the tenant wakes. > - Replay must preserve the original callback proof and current endpoint authority. > - Configuration changes must retain registration responsibility through failures. > - This PR enrolls verified Slack identities and drains sealed events through the normal SDK owner. ## Linked Issues or Issue Description **Subsystem affected** Server chat connectors and signed Cloud connector delivery. **Problem or motivation** Direct Slack Events API delivery depends on an awake tenant. Broker outages, lost responses and configuration changes must preserve delivery and registration responsibility. Enrollment failures must also leave other connector events able to drain. **Proposed solution** Stage nonsecret enrollment or retirement intent inside the local configuration transaction. Let the existing connection-event owner perform broker requests and retries. Replay sealed events through the current Slack SDK with their original authentication evidence. **Alternatives considered** An early acknowledgment for every callback changes command and modal semantics. A second delivery ledger duplicates existing ownership. Reusing a new stack identity to retire old registrations crosses an authorization boundary. A companion Cloud broker change is required. Related setup fixes: Refs #18837, Refs #18878. Kickoff searches found no competing async enrollment consumer. The ROADMAP's connector setup and access controls stay in place. ## What Changed - Persist enrollment and credential-free retirement intent before local configuration commits. Broker failure does not make a completed pause or removal look unsuccessful. - Bind company, endpoint, connection, public ID, generation and authorized actor. Preserve lost-response responsibility and recover only the supported same-instance configuration outage. - Permit an explicit move to a new authorized stack only after confirmed old retirement. Keep the old identity until the new broker receipt matches the current intent and generation. - Show permanent signing-credential failures as attention. Preserve transient retries and reject uncertain inbound authentication before SDK admission. - Drain events and acknowledgments before enrollment reconciliation. Persist a bounded retry on the exact busy intent so other due endpoints can enroll or retire. Preserve current custody under concurrent changes. - Stage retirement in the Apps removal transaction too. Archive the endpoint and connection, retain the original subject and instance, then revoke credentials. Retirement does not need the deleted Slack credential. - Acknowledge a malformed sealed Slack receipt as invalid once. Keep actual broker outages on the retry path. - Run the real configuration, SDK ingress, generation fences and public Apps removal paths in the default SQL suite. Keep Vitest transport helpers outside production services. - Preserve original callback bytes, signature, time and public URL evidence. Use the existing transactional chat-delivery ledger for redelivery. - Provide a checked-in prewire guard and launcher using the current Node executable. Add broker, worker, SDK, SQL and controlled-process acceptance. - Stabilize AI account summaries by ID and grant ID. This fixes a CI migration-repeat equality failure without changing SQL, authorization or reconnect behavior. - Open the four configured database connections before the enrollment test freezes its clock. Keep all 35 original assertions and all timeouts. - Measure status-card dispatch entry before its mock does database work. Keep the original strict 500 ms bound and check any still-armed earlier deadline. Bound that fixture's pool to four connections. ## Verification - Candidate: `54fa701d41b4ced62f4ff338d39b946015670ae3`. Normal merges preserve master `47027e7` and all owned changes. A read-only merge assessment against fetched master `cb5d7e44` is conflict-free. The only overlapping file has separate AI test additions. The branch has not merged that later master; fresh remote checks must validate the current PR test tree. - At this head, normal server production typechecking and declaration emission passed in 35.24s. The affected actual CI test shard passed in 6.27s. Other unchanged CI layers, 92 affected pure controls and 12 guard controls retain their source-matched results. - All **15 default real-PostgreSQL cases passed in 34.66s**, with no skips or filtering. They cover normal configure and SDK ingress, rollback, stale generations, malformed receipts, busy-lease fairness, concurrent intent custody, unknown errors, and public Apps removal. Removal tests check actual credential revocation, retained retirement through outage and repeat removal, one ordinary retry and confirmed retirement, transaction rollback and foreign-company refusal. - The public removal counterfactual failed before the fix: archival and credential revocation passed, but the inactive intent was absent. The same test source passed after the staging call. Separate counterfactuals also reproduced busy-endpoint starvation and malformed-receipt retries. Original failures remain retained. - The paired broker/SDK/process journey passed in 31.98s at ancestor `0324c011`. It preserves callback proof, lost-ACK redelivery, exactly one comment, one delivery and one successful controlled process. Its test, producer, SDK, guard and runtime bytes are unchanged. The new public Apps archive branch is not called by that journey; its evidence carries on that checked basis. - The new SQL cases need no companion Cloud checkout. Normal CI runs them with the repository fixture. For an existing dedicated PostgreSQL fixture, set `PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL` and `PAPERCLIP_TEST_POSTGRES_ADMIN_URL`, then run `node server/scripts/test-slack-cloud-owner.mjs` with Node24. The optional paired launcher still requires the normally built companion Cloud checkout. - Every pool was bounded to four or fewer connections. The final 15-case run sampled three runtime clients plus one observer. The independent final audit found zero clients and all five recorded processes absent. Earlier closing-socket observations are kept separate from final cleanup. - Earlier unrelated CI failures are documented: [enrollment timeout](https://github.com/paperclipai/paperclip/actions/runs/41944373937/job/126973483754) and [status-card observation bound](https://github.com/paperclipai/paperclip/actions/runs/41944373937/job/126973483843). The enrollment test retains all 35 assertions and its original timeouts. A controlled connection probe established the frozen-clock mechanism; the original remote access history remains unknown. The status-card fixture measures dispatch entry with the original strict 500ms bound. A real delayed-dispatch control fails that bound. - Full local workspace typecheck, build and test inventory was not run because its embedded-cluster and wider resource paths exceed this local lease. Exact-head remote CI now passes: 54 successful checks, three skips and Ponytail's neutral check, with aggregate verification green and the legacy security context independently confirmed successful. Both complete bot reviews are 5/5, with all 13 paginated review threads resolved. Seven execution jobs originally reported runner shutdown; one diagnostic rerun recovered them and the dependent aggregate check. Original logs remain retained, and the underlying shutdown cause is unknown. This is recovery evidence, not a product-fix claim. - Provider responses are controlled. This proves local SQL, broker, SDK and process behavior. It does not prove live Slack, agent tools, synchronous callbacks or deployed suspension. ## Risks The companion broker is required. Outage or conflicting authority can leave explicit pending or attention responsibility. A move between stacks requires confirmation from the old authorized owner first. Synchronous commands, actions, modal validation, options and challenges retain their deadlines and endpoint blocker. This slice grants no full Slack endpoint sleep exemption. No deployed suspension or savings claim is made. ## Model Used OpenAI `gpt-6.1-sol` workers through Codex, with reasoning, repository tools and code execution. The served backend revision and context limit are not exposed. A GPT-6-family coordinator performed technical review and integration. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip Co-authored-by: Codex --- doc/connections/slack-setup.md | 98 ++++ server/scripts/test-slack-cloud-owner.mjs | 36 ++ server/scripts/test-slack-cloud-postgres.mjs | 47 ++ server/src/__tests__/ai-connections.test.ts | 29 +- .../github-connection-enrollment-work.test.ts | 7 + .../helpers/slack-cloud-connector-fixture.ts | 97 ++++ .../__tests__/helpers/slack-cloud/prewire.cjs | 94 ++++ .../helpers/slack-cloud/transport.ts | 76 +++ .../__tests__/slack-cloud-owner-work.test.ts | 433 +++++++++++++++++ .../status-card-evaluation-work.test.ts | 14 +- server/src/services/ai-connections.ts | 2 +- server/src/services/chat-channels.ts | 24 +- .../services/chat-channels/configuration.ts | 25 +- .../services/chat-channels/runtime-types.ts | 28 ++ .../chat-channels/slack/cloud-drain.test.ts | 75 +++ .../slack/cloud-enrollment.postgres.test.ts | 451 ++++++++++++++++++ .../slack/cloud-enrollment.test.ts | 214 +++++++++ .../chat-channels/slack/cloud-enrollment.ts | 218 +++++++++ .../chat-channels/slack/cloud-event.test.ts | 110 +++++ .../chat-channels/slack/cloud-event.ts | 77 +++ .../chat-channels/slack/cloud-ingress.test.ts | 47 ++ .../chat-channels/slack/cloud-ingress.ts | 63 +++ ...cloud-instance-transition.postgres.test.ts | 202 ++++++++ .../slack/cloud-signing-secret.test.ts | 19 + .../slack/cloud-signing-secret.ts | 18 + .../slack/cloud-test-prewire.test.ts | 57 +++ .../chat-channels/slack/webhook-callbacks.ts | 2 + server/src/services/chat-sdk-runtime.ts | 9 + .../src/services/github-connection-events.ts | 50 +- .../paperclip-cloud-connector.test.ts | 73 ++- .../src/services/paperclip-cloud-connector.ts | 43 +- .../tool-access/connections/removal.ts | 5 +- 32 files changed, 2714 insertions(+), 29 deletions(-) create mode 100644 server/scripts/test-slack-cloud-owner.mjs create mode 100644 server/scripts/test-slack-cloud-postgres.mjs create mode 100644 server/src/__tests__/helpers/slack-cloud-connector-fixture.ts create mode 100644 server/src/__tests__/helpers/slack-cloud/prewire.cjs create mode 100644 server/src/__tests__/helpers/slack-cloud/transport.ts create mode 100644 server/src/__tests__/slack-cloud-owner-work.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-drain.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-enrollment.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-enrollment.ts create mode 100644 server/src/services/chat-channels/slack/cloud-event.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-event.ts create mode 100644 server/src/services/chat-channels/slack/cloud-ingress.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-ingress.ts create mode 100644 server/src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-signing-secret.test.ts create mode 100644 server/src/services/chat-channels/slack/cloud-signing-secret.ts create mode 100644 server/src/services/chat-channels/slack/cloud-test-prewire.test.ts diff --git a/doc/connections/slack-setup.md b/doc/connections/slack-setup.md index e167017e84f..2774531dd5f 100644 --- a/doc/connections/slack-setup.md +++ b/doc/connections/slack-setup.md @@ -179,3 +179,101 @@ An explicit managed reconnect reinstalls the saved app and verifies the replacem bot credentials. Its request ID is saved before dispatch, so a duplicate completed request does not reinstall again. Credential-configuration recovery still resumes the saved token after a downstream failure. No reconnect creates another Slack app. + +## Durable Cloud Events API delivery + +A broker that advertises `slackEvents.version=1` can accept a signed Events API +registration after ordinary configuration settles. The producer uses the saved +child app ID, current workspace, signing key, endpoint/connection and runtime +generation. It retains the existing public callback URL. Missing verified app +identity or broker support leaves the existing setup in place; it does not guess +an identity or substitute the Slack manager's signing key. + +Cloud stores the child signing key through its existing encrypted stack vault. +Registration metadata and receipts contain no plaintext credentials. An unchanged +credential/endpoint identity reuses its immutable vault ref across generations; +rotation and current registration authority remain separately fenced. Removal +retires exact queue membership and prevents an old generation from reenrolling. +The local configuration transaction first saves nonsecret enrollment or +retirement intent and signals the existing connection-events owner. Broker work +runs through that owner after commit; configuration APIs do not await an optional +Cloud request. Configure, pause and removal can complete locally while Cloud is unavailable. The owner +retries only retained due intent and reconstructs it after restart, including +removal for archived endpoints after credential cleanup. Confirmation replaces +only the same current intent and generation; a completed remote call with a +lost response remains pending. Current authorization errors retain visible +attention debt instead of claiming registration succeeded. Event draining and +acknowledgments run before enrollment reconciliation. The owner then admits one +due enrollment operation and schedules the earliest retained responsibility. +A busy credential lease or enrollment failure retains a bounded retry without +aborting that event drain. Rollback leaves no committed intent to run. + +Retirement sends the original responsible subject and the exact instance, +company, endpoint, public ID, connection and monotonic generation. Cloud can +publish a nonreceiving generation floor without a signing key, live descriptor +or previously returned binding ID. This fences delayed enrollment after local +removal. Normal current broker authority still applies; the subject alone does +not authorize an operation. An instance that never enrolled with Cloud creates +no new intent or background retry when Cloud is absent. Previously configured +retirement responsibility retains its original instance identity as visible +attention if that identity becomes unavailable. Unsupported capability or disabled +new enrollment retains bounded retry responsibility; a closed generation or +identity conflict requires attention. Current authorized confirmation is required +before clearing the exact pending intent. + +Ordinary redeploys retain the same stack identity. To move enrollment to another +stack, first confirm retirement under the old stack's authority, with no pending +old intent. Then explicitly reconfigure under the new authorized stack at a newer +runtime generation. The old stored identity remains until the new broker receipt +matches the current intent and generation. An environment change cannot transfer +live responsibility. Unavailable old authority remains visible attention. + +A missing or disabled signing credential also requires attention. Enrollment does +not retry it as a broker outage. Incoming receipts retain typed attention before +SDK admission; uncertain authentication does not acknowledge the receipt. + +Only authenticated `event_callback` messages use this durable lane. The sealed +queue retains original bytes, signature, timestamp and acceptance time. The app +checks current endpoint/generation authority before SDK handling. It verifies +original freshness at Cloud acceptance; ordinary external requests still use +current time. The credential snapshot lease releases before normal lifecycle +callbacks reacquire their own leases. + +Private replay provenance prevents internal replay URLs from replacing saved +external callback verification evidence. The normal transactional chat delivery +ledger remains the message admission authority. An ambiguous callback or ACK +failure retains its receipt; malformed receipts rejected before useful admission +can be explicitly ignored with bounded diagnostics. Credential-authentication +uncertainty requires attention and is not silently acknowledged. The broker's +existing retention horizon remains the disclosed recovery boundary. + +This does not qualify a complete Slack endpoint for sleep. Slash commands, +interactive actions, modal validation and challenges still need their full +synchronous response semantics. The endpoint blocker remains. Qualification +must cover normal signed enrollment, durable retry after removal, sealed drain, +the actual event worker and SDK, lost-ACK redelivery and one domain effect, +generation changes, and callback evidence. Synthetic protocol coverage does not +prove live Slack, complete configuration UI or a deployed stop/restart/wake journey. + +### Reproduce the paired Cloud acceptance + +Use the repository's supported Node version and installed dependencies. Compile +the companion Cloud checkout first. Set `PAPERCLIP_SLACK_TEST_DATABASE_URL` and +`SLACK_EVENT_TEST_DATABASE_URL` to two distinct, owned loopback PostgreSQL +`*_test` databases. Set `SLACK_TEST_CLOUD_DIST` to its absolute compiled `dist` +directory. This test applies actual migrations and uses only those two databases. +It does not create a PostgreSQL cluster or use a live Slack provider. + +From the app repository root, run: + +```sh +node server/scripts/test-slack-cloud-postgres.mjs +``` + +The launcher uses the current Node executable and one Vitest worker. It loads the +checked-in prewire guard in that actual worker before client construction. The +controlled process child explicitly loads the same guard before its script. +The guard rejects off-host and unowned socket targets before transport. Caught +unexpected attempts still fail the fixture. No private preload path or manual +`SLACK_GATE_GUARD_PID` setting is needed. The fixture remains opt-in during the +normal test suite; its default owner and guard regressions run without databases. diff --git a/server/scripts/test-slack-cloud-owner.mjs b/server/scripts/test-slack-cloud-owner.mjs new file mode 100644 index 00000000000..26b56d932c8 --- /dev/null +++ b/server/scripts/test-slack-cloud-owner.mjs @@ -0,0 +1,36 @@ +import { spawn } from "node:child_process"; +import { createRequire } from "node:module"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const patterns = { "--tool-removal-only": "public tool removal retains Slack retirement", "--fairness-only": "two due registrations", "--malformed-only": "malformed sealed Slack receipt", + "--wiring-only": "normal configure stages SQL intent", "--cas-only": "real SQL refuses an enrollment response" }; +const args = process.argv.slice(2); +if (!process.version.startsWith("v24.")) throw new Error("Slack owner fixture requires the current Node24 executable"); +if (args.length > 1 || args.length && !Object.hasOwn(patterns, args[0])) throw new Error("Unknown Slack owner fixture selector"); +const value = process.env.PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL; +if (!value) throw new Error("Set PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL to the exact owned external test database"); +const fixture = new URL(value), admin = new URL(process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ?? "invalid:"); +if (!["postgres:", "postgresql:"].includes(fixture.protocol) || fixture.hostname !== "127.0.0.1" || fixture.port !== "55448" + || !/^[a-zA-Z0-9_]+_test$/.test(fixture.pathname.slice(1)) || fixture.search || fixture.hash + || admin.protocol !== fixture.protocol || admin.hostname !== fixture.hostname || admin.port !== fixture.port + || admin.pathname !== "/postgres" || admin.search || admin.hash) throw new Error("Slack owner fixture requires its explicit owned PG17 URL/admin grant"); +const log = process.env.SLACK_GATE_PROCESS_LOG; +if (!log || !path.isAbsolute(log) || existsSync(log)) throw new Error("Slack owner fixture requires a fresh absolute guard log"); +const server = fileURLToPath(new URL("../", import.meta.url)); +const require = createRequire(new URL("../package.json", import.meta.url)); +const vitest = path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs"); +const guard = path.join(server, "src/__tests__/helpers/slack-cloud/prewire.cjs"); +const home = mkdtempSync(path.join(tmpdir(), "slack-owner-fixture-")); +const env = { PATH: process.env.PATH, HOME: process.env.HOME, CODEX_HOME: process.env.CODEX_HOME, LANG: process.env.LANG, + TMPDIR: home, PAPERCLIP_HOME: home, NODE_ENV: "test", NODE_OPTIONS: "", SLACK_GATE_EXPECT_PRELOAD: "1", + PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL: value, PAPERCLIP_SLACK_TEST_DATABASE_URL: value, + PAPERCLIP_TEST_POSTGRES_ADMIN_URL: process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL, SLACK_GATE_PROCESS_LOG: log }; +const child = spawn(process.execPath, ["--require", guard, vitest, "run", "src/__tests__/slack-cloud-owner-work.test.ts", + "--maxWorkers=1", "--no-file-parallelism", `--execArgv=--require=${guard}`, + ...(args.length ? [`--testNamePattern=${patterns[args[0]]}`] : [])], { cwd: server, env, stdio: "inherit" }); +for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => child.kill(signal)); +child.on("error", error => { console.error(error); process.exitCode = 1; }); +child.on("close", (code, signal) => { rmSync(home, { recursive: true, force: true }); process.exitCode = signal ? 1 : code ?? 1; }); diff --git a/server/scripts/test-slack-cloud-postgres.mjs b/server/scripts/test-slack-cloud-postgres.mjs new file mode 100644 index 00000000000..c26068b10a6 --- /dev/null +++ b/server/scripts/test-slack-cloud-postgres.mjs @@ -0,0 +1,47 @@ +import { spawn } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +if (process.argv.slice(2).length > 1 || process.argv.slice(2).some(arg => !['--instance-transition-only', '--normal-owner-only'].includes(arg))) throw new Error('Unknown Slack fixture selector'); +const transitionOnly = process.argv.includes('--instance-transition-only'); +const normalOwnerOnly = process.argv.includes('--normal-owner-only'); +const server = fileURLToPath(new URL('../', import.meta.url)); +const require = createRequire(new URL('../package.json', import.meta.url)); +const guard = path.join(server, 'src/__tests__/helpers/slack-cloud/prewire.cjs'); +const names = ['PAPERCLIP_SLACK_TEST_DATABASE_URL', 'SLACK_EVENT_TEST_DATABASE_URL']; +const urls = names.map(name => { + if (!process.env[name]) throw new Error(`Set ${name} to an owned dedicated PostgreSQL test database`); + let url; + try { url = new URL(process.env[name]); } catch { throw new Error(`${name} must name an explicit loopback *_test database`); } + if (!['postgres:', 'postgresql:'].includes(url.protocol) || !['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname) + || !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1))) + || ['options', 'database', 'dbname'].some(k => url.searchParams.has(k))) throw new Error(`${name} must name an explicit loopback *_test database`); + return url; +}); +if (urls[0].pathname === urls[1].pathname) throw new Error('Slack app and Cloud fixtures require distinct databases'); +if (!process.env.SLACK_TEST_CLOUD_DIST || !path.isAbsolute(process.env.SLACK_TEST_CLOUD_DIST) + || !existsSync(path.join(process.env.SLACK_TEST_CLOUD_DIST, 'connections/broker.js'))) throw new Error('SLACK_TEST_CLOUD_DIST must name the compiled companion Cloud dist directory'); +const vitest = path.join(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); +const ownedHome = mkdtempSync(path.join(tmpdir(), 'paperclip-slack-paired-')); +const env = { + PATH: process.env.PATH, HOME: process.env.HOME, CODEX_HOME: process.env.CODEX_HOME, + LANG: process.env.LANG, TMPDIR: ownedHome, PAPERCLIP_HOME: ownedHome, + NODE_ENV: 'test', NODE_OPTIONS: '', SLACK_GATE_EXPECT_PRELOAD: '1', + PAPERCLIP_SLACK_TEST_DATABASE_URL: process.env.PAPERCLIP_SLACK_TEST_DATABASE_URL, + SLACK_EVENT_TEST_DATABASE_URL: process.env.SLACK_EVENT_TEST_DATABASE_URL, + SLACK_TEST_CLOUD_DIST: process.env.SLACK_TEST_CLOUD_DIST, + SLACK_GATE_PROCESS_LOG: process.env.SLACK_GATE_PROCESS_LOG, +}; +const child = spawn(process.execPath, ['--require', guard, vitest, 'run', + ...(transitionOnly ? ['src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts'] : normalOwnerOnly + ? ['src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts'] : ['src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts', 'src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts']), '--maxWorkers=1', '--no-file-parallelism', `--execArgv=--require=${guard}`], +{ cwd: server, env, stdio: 'inherit' }); +for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); +child.on('error', error => { console.error(error); process.exitCode = 1; }); +child.on('close', (code, signal) => { + rmSync(ownedHome, { recursive: true, force: true }); + process.exitCode = signal ? 1 : code ?? 1; +}); diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index c60caf97dc6..07b9616c0a8 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -17,7 +17,7 @@ import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:f import os from "node:os"; import path from "node:path"; import { and, eq, sql } from "drizzle-orm"; -import { createDb, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; +import { createDb, type Db, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres"; import { fetchCompanyQuotaWindows } from "../services/quota-windows.js"; import { quotaCredentialRecoveryPath, quotaCredentialRecovery, quotaCredentialHash } from "../services/quota-credential-recovery.js"; @@ -2213,6 +2213,33 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello" const [agent] = await db.select().from(agents).where(eq(agents.id, agentId)); expect(agent.runtimeConfig.aiConnection).toBeUndefined(); }); + it("keeps account summaries ordered across sequential scans and repeat migration", async () => { + const orderedCompanyId = randomUUID(), userId = "stable-list-owner"; + await db.insert(companies).values({ id: orderedCompanyId, name: "Stable account list", issuePrefix: `O${orderedCompanyId.slice(0, 5)}` }); + await db.insert(companyMemberships).values({ companyId: orderedCompanyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" }); + const accounts = []; + for (const name of ["First account", "Second account", "Third account"]) { + accounts.push(await service.save(orderedCompanyId, userId, { provider: "anthropic", method: "api_key", ownership: "personal", name, apiKey: "fixture-stable-list", agentIds: [], allAgents: false }, "fixture-stable-list")); + } + const expected = accounts.map(({ connectionId: id, grantId }) => ({ id, grantId })) + .sort((a, b) => a.id.localeCompare(b.id) || a.grantId.localeCompare(b.grantId)); + await db.transaction(async tx => { + await tx.execute(sql`SET LOCAL enable_indexscan = off`); + await tx.execute(sql`SET LOCAL enable_bitmapscan = off`); + await tx.execute(sql`SET LOCAL enable_hashjoin = off`); + await tx.execute(sql`SET LOCAL enable_mergejoin = off`); + for (const account of [...expected].reverse()) { + await tx.update(toolConnections).set({ name: sql`${toolConnections.name}` }).where(eq(toolConnections.id, account.id)); + await tx.update(connectionGrants).set({ status: sql`${connectionGrants.status}` }).where(eq(connectionGrants.id, account.grantId)); + } + const transactionalService = aiConnectionService(tx as unknown as Db); + const before = await transactionalService.list(orderedCompanyId, userId); + expect(before.map(({ id, grantId }) => ({ id, grantId }))).toEqual(expected); + const migration = await readFile(new URL("../../../packages/db/src/migrations/0276_hard_mandroid.sql", import.meta.url), "utf8"); + for (const statement of migration.split("--> statement-breakpoint").filter(value => value.trim())) await tx.execute(sql.raw(statement)); + expect(await transactionalService.list(orderedCompanyId, userId)).toEqual(before); + }); + }); it("runs two Claude subscription executions for the same grant at the same time", async () => { // Claude writes no auth file back to the grant, so two runs share no // mutable state and must not wait for each other. diff --git a/server/src/__tests__/github-connection-enrollment-work.test.ts b/server/src/__tests__/github-connection-enrollment-work.test.ts index 80a014e7bb6..719da5e7157 100644 --- a/server/src/__tests__/github-connection-enrollment-work.test.ts +++ b/server/src/__tests__/github-connection-enrollment-work.test.ts @@ -90,6 +90,13 @@ const support = await getEmbeddedPostgresTestSupport(); grantId = grant!.id; const env = { PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: origin, PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test" }; const service = githubConnectionEventService(db, { env }); + const reserved: Awaited>[] = []; + try { + for (let slot = 0; slot < 4; slot++) reserved.push(await db.$client.reserve()); + await Promise.all(reserved.map(connection => connection`SELECT 1`)); + } finally { + for (const connection of reserved) connection.release(); + } vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: e => errors.push(e) }); const selects = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute"); diff --git a/server/src/__tests__/helpers/slack-cloud-connector-fixture.ts b/server/src/__tests__/helpers/slack-cloud-connector-fixture.ts new file mode 100644 index 00000000000..c20ad8eb752 --- /dev/null +++ b/server/src/__tests__/helpers/slack-cloud-connector-fixture.ts @@ -0,0 +1,97 @@ +import { createCipheriv, createHash, createHmac, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, verify } from "node:crypto"; +import { expect, vi } from "vitest"; +import { createPaperclipCloudConnector, type PaperclipCloudConnectorConfig } from "../../services/paperclip-cloud-connector.js"; + +export function createSlackCloudConnectorFixture(instanceId: string, companyId: string, subject: string) { + const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); + const config: PaperclipCloudConnectorConfig = { baseUrl: "http://127.0.0.1:1", instanceId, environment: "test", + signPrivateKey: signing.privateKey.export({ format: "jwk" }).d!, sealPrivateKey: sealing.privateKey.export({ format: "jwk" }).d! }; + const calls: { operation: string; enrollment?: Record }[] = []; + const acknowledged: string[] = []; + const events: Record[] = []; + let unavailable = false; + let beforeEnrollmentReceipt: (() => Promise) | undefined; + function seal(payload: unknown, profile: "github.code" | "slack.events.v1") { + const provider = profile === "github.code" ? "github" : "slack"; + const ephemeral = generateKeyPairSync("x25519"); + const epk = ephemeral.publicKey.export({ format: "jwk" }).x!; + const recipient = sealing.publicKey.export({ format: "jwk" }).x!; + const aad = Buffer.from([1, "X25519-HKDF-SHA256-A256GCM", "events", instanceId, "test", provider, profile, ""].join("\n")); + const key = Buffer.from(hkdfSync("sha256", diffieHellman({ privateKey: ephemeral.privateKey, publicKey: sealing.publicKey }), + Buffer.concat([Buffer.from(epk, "base64url"), Buffer.from(recipient, "base64url")]), aad, 32)); + const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad); + const ct = Buffer.concat([cipher.update(JSON.stringify(payload), "utf8"), cipher.final(), cipher.getAuthTag()]); + return { v: 1, alg: "X25519-HKDF-SHA256-A256GCM", purpose: "events", provider, profile, epk, + iv: iv.toString("base64url"), ct: ct.toString("base64url") }; + } + const requestErrors: unknown[] = []; + const request: typeof fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit): Promise => { + try { + const url = new URL(input instanceof Request ? input.url : String(input)); + expect(url.origin).toBe(config.baseUrl); + expect(init?.method).toBe("POST"); + const body = JSON.parse(String(init?.body)) as { request: string; enrollment?: string; acknowledgement?: string }; + const [header, payload, signature] = body.request.split("."); + expect(verify(null, Buffer.from(`${header}.${payload}`), signing.publicKey, Buffer.from(signature!, "base64url"))).toBe(true); + const claims = JSON.parse(Buffer.from(payload!, "base64url").toString("utf8")) as Record; + expect(claims).toMatchObject({ iss: instanceId, env: "test", aud: url.toString() }); + const operation = String(claims.op); + if (operation !== "status") { + const expectedSubject = operation === "slack-events" ? subject + : ["event-lease", "event-ack"].includes(operation) ? "github-bot-events" : undefined; + if (!expectedSubject) throw new Error(`Unexpected synthetic connector authority ${operation}`); + expect(claims).toMatchObject({ sub: expectedSubject, cid: companyId }); + } + if (unavailable) return Response.json({ error: "fixture_broker_unavailable" }, { status: 503 }); + if (operation === "status") return Response.json({ active: true, status: "active", profiles: [], slackEvents: { version: 1 } }); + if (operation === "slack-events") { + expect(claims.sh).toBe(createHash("sha256").update(body.enrollment!).digest("base64url")); + const enrollment = JSON.parse(body.enrollment!) as Record; + calls.push({ operation, enrollment: { ...enrollment, signingSecret: undefined } }); + await beforeEnrollmentReceipt?.(); + const { active, signingSecret: _secret, ...identity } = enrollment; + const bindingId = `slack-event:${createHash("sha256").update(JSON.stringify(identity)).digest("hex")}`; + return Response.json(active === false ? { retirement: { version: 1, instanceId, ...identity } } + : { enrollment: { version: 1, instanceId, ...identity, bindingId } }); + } + calls.push({ operation }); + if (operation === "event-lease") { + if (!events.length) return Response.json({ events: [], eventWakeVersion: 1 }); + const leased = events.splice(0); + return Response.json({ leaseId: "fixture-lease", eventWakeVersion: 1, + sealed: seal({ v: 1, instanceId, environment: "test", leaseId: "fixture-lease", events: leased }, "github.code") }); + } + if (operation === "event-ack") { + expect(claims.sh).toBe(createHash("sha256").update(body.acknowledgement!).digest("base64url")); + const value = JSON.parse(body.acknowledgement!) as { deliveryIds: string[] }; + acknowledged.push(...value.deliveryIds); return Response.json({ acknowledged: value.deliveryIds.length }); + } + throw new Error(`Unexpected synthetic connector operation ${operation}`); + } catch (error) { + requestErrors.push(error); + console.error("Synthetic Slack connector rejected its request", error); + throw error; + } + }); + function install() { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", config.baseUrl); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", config.signPrivateKey); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", config.sealPrivateKey); + } + function enqueue(identity: Record, signingSecret: string, eventId: string) { + const acceptedAt = new Date().toISOString(), timestamp = String(Math.floor(Date.now() / 1000)); + const body = Buffer.from(JSON.stringify({ type: "event_callback", team_id: identity.teamId, api_app_id: identity.appId, event_id: eventId, + event: { type: "app_mention", text: "<@UBOT> fixture callback", user: "UOPERATOR", channel: "CTEST", ts: `${Date.now() / 1000}`, event_ts: `${Date.now() / 1000}` } })); + const id = `slack_${String(identity.bindingId).slice("slack-event:".length)}_${createHash("sha256").update(eventId).digest("hex")}`; + const packet = { version: 1, instanceId, environment: "test", ...identity, eventId, acceptedAt, rawBody: body.toString("base64"), timestamp, + signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(body).digest("hex")}` }; + events.push({ id, provider: "slack", event: "event_callback", installationId: identity.bindingId, bindingIds: [identity.bindingId], createdAt: acceptedAt, + payload: { slackEvent: { bindingId: identity.bindingId, sealed: seal(packet, "slack.events.v1") } } }); + return { id, packet, body }; + } + return { config, request, requestErrors, calls, acknowledged, events, install, enqueue, connector: createPaperclipCloudConnector({ config, request }), + setUnavailable(value: boolean) { unavailable = value; }, + beforeReceipt(work: (() => Promise) | undefined) { beforeEnrollmentReceipt = work; } }; +} diff --git a/server/src/__tests__/helpers/slack-cloud/prewire.cjs b/server/src/__tests__/helpers/slack-cloud/prewire.cjs new file mode 100644 index 00000000000..04ff4b5fd43 --- /dev/null +++ b/server/src/__tests__/helpers/slack-cloud/prewire.cjs @@ -0,0 +1,94 @@ +const key = Symbol.for('paperclip.slack.fixture.prewire'); +if (!process[key]) { + const net = require('node:net'); + const tls = require('node:tls'); + const http = require('node:http'); + const https = require('node:https'); + const fs = require('node:fs'); + const loopback = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']); + const listeners = new Set(); + const pgTargets = new Set(); + const pgPorts = new Set([5432, 55448]); + for (const name of ['PAPERCLIP_SLACK_TEST_DATABASE_URL', 'SLACK_EVENT_TEST_DATABASE_URL']) { + if (!process.env[name]) continue; + let url; + try { url = new URL(process.env[name]); } catch { throw new Error('Invalid Slack fixture database target'); } + if (!['postgres:', 'postgresql:'].includes(url.protocol) || !loopback.has(url.hostname) + || !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1))) + || ['options', 'database', 'dbname'].some(k => url.searchParams.has(k))) throw new Error('Invalid Slack fixture database target'); + const port = Number(url.port || 5432); + pgPorts.add(port); + pgTargets.add(`${url.hostname}:${port}`); + } + const sockets = new Set(); + const observed = new WeakSet(); + const attempts = []; + let peakPgSockets = 0; + const record = value => { + if (process.env.SLACK_GATE_PROCESS_LOG) fs.appendFileSync(process.env.SLACK_GATE_PROCESS_LOG, JSON.stringify(value) + '\n'); + }; + const fail = (host, port) => { + attempts.push({ host: String(host), port: Number(port) || null }); + process.exitCode = 98; + throw new Error('Slack fixture blocked an unowned network target before transport'); + }; + const check = (host = 'localhost', port, protocol) => { + host = String(host).toLowerCase(); + if (!loopback.has(host)) fail(host, port); + if (protocol && !['http:', 'https:'].includes(protocol)) fail(host, port); + if (port && pgPorts.has(Number(port)) && !pgTargets.has(`${host}:${Number(port)}`)) fail(host, port); + if (port && !pgTargets.has(`${host}:${Number(port)}`) && !listeners.has(Number(port))) fail(host, port); + }; + const checkArgs = args => { + const first = args[0]; + if (first instanceof URL || typeof first === 'string' && /^https?:/.test(first)) { + const url = first instanceof URL ? first : new URL(first); + check(url.hostname, url.port || (url.protocol === 'https:' ? 443 : 80), url.protocol); + } else if (first && typeof first === 'object') { + if (first.path && !first.port && !first.host && !first.hostname) fail('unix-socket', 0); + check(first.hostname || first.host, first.port, first.protocol); + } else if (typeof first === 'number') check(typeof args[1] === 'string' ? args[1] : undefined, first); + else fail('unknown-target', 0); + }; + const listen = net.Server.prototype.listen; + net.Server.prototype.listen = function (...args) { + this.once('listening', () => { + const address = this.address(); + if (address && typeof address === 'object' && loopback.has(address.address)) { + listeners.add(address.port); + this.once('close', () => listeners.delete(address.port)); + } + }); + return Reflect.apply(listen, this, args); + }; + for (const module of [http, https]) for (const name of ['request', 'get']) { + const original = module[name]; + module[name] = function (...args) { checkArgs(args); return Reflect.apply(original, this, args); }; + } + const connect = net.Socket.prototype.connect; + net.Socket.prototype.connect = function (...args) { + const normalized = Array.isArray(args[0]) ? args[0] : args; + checkArgs(normalized); + const first = normalized[0]; + const port = typeof first === 'object' ? first.port : first; + if (pgPorts.has(Number(port)) && !observed.has(this)) { + observed.add(this); + this.once('connect', () => { sockets.add(this); peakPgSockets = Math.max(peakPgSockets, sockets.size); }); + this.once('close', () => sockets.delete(this)); + } + return Reflect.apply(connect, this, args); + }; + const tlsConnect = tls.connect; + tls.connect = function (...args) { checkArgs(args); return Reflect.apply(tlsConnect, this, args); }; + const fetch = globalThis.fetch; + if (fetch) globalThis.fetch = function (input, init) { + checkArgs([input instanceof Request ? input.url : input]); + return fetch(input, init); + }; + const stats = () => ({ pid: process.pid, peakPgSockets, remainingPgSockets: sockets.size, unexpectedAttempts: attempts.slice() }); + process[key] = Object.freeze({ pid: process.pid, stats }); + globalThis[key] = process[key]; + process.env.SLACK_GATE_GUARD_PID = String(process.pid); + record({ phase: 'repository-preload', pid: process.pid }); + process.on('exit', () => record({ phase: 'exit', ...stats(), exitCode: process.exitCode || 0 })); +} diff --git a/server/src/__tests__/helpers/slack-cloud/transport.ts b/server/src/__tests__/helpers/slack-cloud/transport.ts new file mode 100644 index 00000000000..3cf3d0cbd7f --- /dev/null +++ b/server/src/__tests__/helpers/slack-cloud/transport.ts @@ -0,0 +1,76 @@ +import http from "node:http"; +import https from "node:https"; +import { expect, vi } from "vitest"; + +export function guardSlackFixtureNetwork(options: { requirePreload?: boolean } = {}) { + const prewire = Reflect.get(process, Symbol.for("paperclip.slack.fixture.prewire")) as { + pid: number; + stats(): { pid: number; peakPgSockets: number; remainingPgSockets: number; unexpectedAttempts: unknown[] }; + } | undefined; + if (options.requirePreload) { + expect(prewire?.pid).toBe(process.pid); + expect(process.env.SLACK_GATE_GUARD_PID).toBe(String(process.pid)); + } + if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1") { + expect(process.env.SLACK_GATE_GUARD_PID).toBe(String(process.pid)); + console.info(JSON.stringify({ slackFixtureWorkerPid: process.pid, prewireGuardPid: process.env.SLACK_GATE_GUARD_PID })); + } + const blocked: string[] = []; + const check = (value: unknown) => { + const url = value instanceof URL ? value : typeof value === "string" ? new URL(value) : undefined; + const options = value as { hostname?: string; host?: string } | undefined; + const host = url?.hostname ?? options?.hostname ?? options?.host ?? "localhost"; + if (!["localhost", "127.0.0.1", "::1", "[::1]"].includes(host)) { + blocked.push(host); + throw new Error("Unexpected off-host Slack fixture request"); + } + }; + const realHttp = http.request, realHttps = https.request, realFetch = globalThis.fetch; + const httpSpy = vi.spyOn(http, "request").mockImplementation(((...args: unknown[]) => { + check(args[0]); return Reflect.apply(realHttp, http, args); + }) as typeof http.request); + const httpsSpy = vi.spyOn(https, "request").mockImplementation(((...args: unknown[]) => { + check(args[0]); return Reflect.apply(realHttps, https, args); + }) as typeof https.request); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { + check(input instanceof Request ? input.url : input); return realFetch(input, init); + }); + return { assertQuiet: () => { + expect(blocked).toEqual([]); + if (prewire) expect(prewire.stats().unexpectedAttempts).toEqual([]); + const stats = prewire?.stats; + if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1") { + expect(stats).toBeTypeOf("function"); + console.info(JSON.stringify({ slackFixtureWorkerPid: process.pid, networkStats: stats!() })); + } + }, restore: () => { + fetchSpy.mockRestore(); httpsSpy.mockRestore(); httpSpy.mockRestore(); + } }; +} + +export function stubSlackFixtureTransport(adapter: unknown) { + const client = (adapter as { _client: { + axios: { defaults: { adapter: unknown } }; + buildResult(response: unknown): Promise; + } })._client; + const failures: unknown[] = []; + const buildResult = client.buildResult.bind(client); + client.buildResult = async response => { + try { return await buildResult(response); } + catch (error) { failures.push(error); throw error; } + }; + const transport = vi.fn(async (config: { url: string; baseURL: string }) => { + const url = new URL(config.url, config.baseURL); + if (url.hostname !== "127.0.0.1") throw new Error("Slack fixture API origin changed"); + let data: unknown; + if (url.pathname.endsWith("/users.info")) data = { ok: true, user: { id: "UOPERATOR", name: "operator", is_bot: false, profile: { display_name: "Operator" } } }; + else if (url.pathname.endsWith("/conversations.info")) data = { ok: true, channel: { id: "CTEST", name: "fixture" } }; + else throw new Error(`Unexpected Slack fixture API method ${url.pathname}`); + return { data, status: 200, statusText: "OK", headers: {}, config, request: { path: url.pathname } }; + }); + client.axios.defaults.adapter = transport; + return Object.assign(transport, { assertNoErrors: async () => { + await Promise.all(transport.mock.results.map(result => result.value)); + expect(failures).toEqual([]); + } }); +} diff --git a/server/src/__tests__/slack-cloud-owner-work.test.ts b/server/src/__tests__/slack-cloud-owner-work.test.ts new file mode 100644 index 00000000000..fb66a4d278e --- /dev/null +++ b/server/src/__tests__/slack-cloud-owner-work.test.ts @@ -0,0 +1,433 @@ +import { createHmac, randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { SLACK_CHAT_BOT_SCOPES } from "@paperclipai/shared"; +import { activityLog, agentRuntimeState, agentTaskSessions, agentWakeupRequests, agents, applyPendingMigrations, authUsers, + chatDeliveries, chatEndpoints, chatExternalPrincipals, companySecretBindings, chatIdentityLinks, chatSlackRegistrations, companies, companyMemberships, + companySecrets, companySkills, costEvents, financeEvents, heartbeatRunEvents, heartbeatRuns, issueComments, issues, toolApplications, toolConnections, + closeCheckpointDb, closeFileStorageDb, closeLivenessDb, createDb } from "@paperclipai/db"; +import { databaseWorkPending, reconcileDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db/work-signals"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createSlackCloudConnectorFixture } from "./helpers/slack-cloud-connector-fixture.js"; +import { chatChannelService } from "../services/chat-channels.js"; +import { toolAccessService } from "../services/tool-access.js"; +import { secretService } from "../services/secrets.js"; +import { ChatRuntime } from "../services/chat-runtime.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { githubConnectionEventService } from "../services/github-connection-events.js"; +import { chatCredentialMutationLease } from "../services/chat-credential-mutation-lease.js"; +import { createSlackCloudEnrollment, registerSlackCloudEnrollmentOwner, reconcilePendingSlackCloudEnrollments } from "../services/chat-channels/slack/cloud-enrollment.js"; +import { conflict } from "../errors.js"; +import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "./helpers/slack-cloud/transport.js"; +import type { InternalSetupState } from "../services/chat-channels/runtime-types.js"; +import type { ChatSdkMessageCallbackEvent } from "../services/chat-sdk-runtime.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL"); +if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1" && !external) throw new Error("Guarded local Slack owner tests require their exact external test database"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("Slack Cloud default owner and SQL fences", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + let service: ReturnType; + let heartbeat: ReturnType; + let cloud: ReturnType; + let network: ReturnType; + let directory: string; + let companyId: string, actorId: string, agentId: string, applicationId: string; + let runtime: ChatRuntime; + let providerFetch: typeof fetch; + const callbacks: ChatSdkMessageCallbackEvent[] = []; + const transports: ReturnType[] = []; + const releases: (() => void)[] = []; + const heldLeases: Promise[] = []; + const deferred: (() => void | Promise)[] = []; + const secret = "0123456789abcdef0123456789abcdef"; + const row = async (id: string) => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, id)))[0]!; + const setup = async (id: string) => (await row(id)).setup as InternalSetupState; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("slack-cloud-owner-"); + const url = external ?? temporary!.connectionString; + await applyPendingMigrations(url); + db = createDb(url, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 60_000); + beforeEach(async () => { + callbacks.length = 0; transports.length = 0; releases.length = 0; heldLeases.length = 0; deferred.length = 0; + network = guardSlackFixtureNetwork({ requirePreload: process.env.SLACK_GATE_EXPECT_PRELOAD === "1" }); + directory = await mkdtemp(path.join(os.tmpdir(), "slack-owner-default-")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(directory, "master.key")); + companyId = randomUUID(); actorId = `slack-owner-${randomUUID()}`; agentId = randomUUID(); applicationId = randomUUID(); + cloud = createSlackCloudConnectorFixture(`instance-${randomUUID()}`, companyId, actorId); cloud.install(); + vi.spyOn(globalThis, "fetch").mockImplementation(cloud.request); + const now = new Date(); + await db.insert(companies).values({ id: companyId, name: "Slack default owner", issuePrefix: `S${companyId.slice(0, 7)}` }); + await db.insert(authUsers).values({ id: actorId, name: "Slack owner", email: `${actorId}@example.test`, emailVerified: true, createdAt: now, updatedAt: now }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, status: "active", membershipRole: "operator" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Slack fixture", adapterType: "process", status: "idle", + runtimeConfig: { heartbeat: { wakeOnDemand: false, intervalSec: 0 } } }); + await db.insert(companyMemberships).values({ companyId, principalType: "agent", principalId: agentId, status: "active", membershipRole: "member" }); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Slack", type: "chat" }); + runtime = makeRuntime(); + const slackFetch: typeof fetch = async (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.origin !== "https://slack.com") throw new Error(`Unexpected Slack fixture origin ${url.origin}`); + if (url.pathname === "/api/auth.test") { + expect(init?.method).toBe("POST"); + const token = new Headers(init?.headers).get("authorization") ?? ""; + const suffix = token.slice(token.lastIndexOf("-") + 1); + return Response.json({ ok: true, team_id: `T${suffix}`, team: "Fixture", user_id: `U${suffix}`, user: "fixture" }, + { headers: { "x-oauth-scopes": SLACK_CHAT_BOT_SCOPES.join(",") } }); + } + if (url.pathname === "/api/conversations.list") return Response.json({ ok: true, + channels: [{ id: "CTEST", name: "fixture", is_member: true }], response_metadata: { next_cursor: "" } }); + if (["/api/reactions.add", "/api/reactions.remove"].includes(url.pathname)) return Response.json({ ok: true }); + throw new Error(`Unexpected Slack fixture method ${url.pathname}`); + }; + providerFetch = slackFetch; + heartbeat = heartbeatService(db); + service = chatChannelService(db, { runtime, heartbeat, fetch: slackFetch, publicBaseUrl: "https://tenant.example", + deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); } }); + }); + afterEach(async () => { + const errors: unknown[] = []; + for (const release of releases) release(); + for (const work of heldLeases) { try { await work; } catch (error) { errors.push(error); } } + try { await service?.shutdown(); } catch (error) { errors.push(error); } + try { await heartbeat?.drainActiveRunExecutions(); } catch (error) { errors.push(error); } + try { + await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId)); + await db.delete(financeEvents).where(eq(financeEvents.companyId, companyId)); + await db.delete(costEvents).where(eq(costEvents.companyId, companyId)); + await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, companyId)); + await db.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, companyId)); + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, companyId)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + await db.delete(issueComments).where(eq(issueComments.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId)); + await db.delete(companySkills).where(eq(companySkills.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + await db.delete(authUsers).where(eq(authUsers.id, actorId)); + } catch (error) { errors.push(error); } + try { network.assertQuiet(); } catch (error) { errors.push(error); } + network.restore(); vi.restoreAllMocks(); vi.unstubAllEnvs(); + await rm(directory, { recursive: true, force: true }); + if (errors.length) throw new AggregateError(errors, "Slack owner fixture cleanup failed; original body failure is reported separately by Vitest"); + }); + afterAll(async () => { + const closed = await Promise.allSettled([closeCheckpointDb(db, 1), closeFileStorageDb(db, 1), closeLivenessDb(db, 1)]); + await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); + const errors = closed.flatMap(value => value.status === "rejected" ? [value.reason] : []); + if (errors.length) throw new AggregateError(errors, "Slack owner auxiliary shutdown failed"); + }); + function makeRuntime() { + const runtime = new ChatRuntime(); + const replace = runtime.replaceEndpoint.bind(runtime); + vi.spyOn(runtime, "replaceEndpoint").mockImplementation(async input => { + if (input.providerConfig.provider !== "slack") throw new Error("Unexpected fixture provider"); + const value = await replace({ ...input, providerConfig: { ...input.providerConfig, + credentials: { ...input.providerConfig.credentials, apiUrl: "http://127.0.0.1:1/api/" } }, + callbacks: { ...input.callbacks, onMessage: async event => { callbacks.push(event); await input.callbacks.onMessage(event); } } }); + transports.push(stubSlackFixtureTransport(value.getProviderAdapter())); return value; + }); + return runtime; + } + async function createEndpoint(suffix: string) { + const endpointId = randomUUID(), connectionId = randomUUID(), publicId = randomUUID().replaceAll("-", "").padEnd(43, "a"); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: `Slack ${suffix}`, uid: endpointId, + transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId, + provider: "slack", status: "draft", sponsorUserId: actorId, allowGroupChats: true, setup: { step: "provider_setup" } }); + await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", appId: `A${suffix}`, + manifest: {}, manifestHash: "fixture", callbackUri: `https://tenant.example/api/chat-webhooks/${publicId}/slack`, createdByUserId: actorId }); + await service.configure(endpointId, { action: "configure", credentials: { botToken: `xoxb-fixture-${suffix}`, signingSecret: secret } }, actorId); + expect((await setup(endpointId)).slackCloudIntent).toMatchObject({ companyId, endpointId, connectionId, + instanceId: cloud.config.instanceId, subject: actorId, active: true }); + return endpointId; + } + async function confirm(endpointId: string) { + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await owner.pollOnce({ forDelivery: true }); + expect((await setup(endpointId)).slackCloudIntent).toBeUndefined(); + expect((await setup(endpointId)).slackCloudEvents).toBeDefined(); + return owner; + } + async function activate(endpointId: string) { + const endpoint = await row(endpointId); + const accountId = endpoint.providerAccountId; + if (!accountId) throw new Error("Normal Slack configure did not confirm its workspace"); + const payload = JSON.stringify({ type: "url_verification", team_id: accountId, api_app_id: `A${accountId.slice(1)}`, challenge: "fixture-challenge" }); + const timestamp = String(Math.floor(Date.now() / 1000)); + const response = await service.handleWebhook(endpoint.publicId, "slack", new Request(`https://tenant.example/api/chat-webhooks/${endpoint.publicId}/slack`, { + method: "POST", headers: { "content-type": "application/json", "x-slack-request-timestamp": timestamp, + "x-slack-signature": `v0=${createHmac("sha256", secret).update(`v0:${timestamp}:${payload}`).digest("hex")}` }, body: payload })); + expect(response.status).toBe(200); await response.body?.cancel(); + await service.configure(endpointId, { action: "verify" }, actorId); + const [principal] = await db.insert(chatExternalPrincipals).values({ companyId, provider: "slack", providerAccountId: accountId, + externalId: "UOPERATOR", kind: "user", displayName: "Operator" }).returning(); + await db.insert(chatIdentityLinks).values({ companyId, endpointId, principalId: principal!.id, paperclipUserId: actorId, status: "linked", confirmedAt: new Date() }); + expect((await service.test(endpointId, { optionalSlackTestForUser: actorId })).status).toBe("active"); + } + async function holdCredentials(endpointId: string) { + let acquired!: () => void; + const ready = new Promise(resolve => { acquired = resolve; }); + const release = new Promise(resolve => { releases.push(resolve); }); + heldLeases.push(chatCredentialMutationLease(db)(await row(endpointId), async lease => { await lease.assertOwned(); acquired(); await release; })); + await ready; + } + + it("normal configure stages SQL intent, worker unseals into registered SDK and old generations cannot reenter", async () => { + const endpointId = await createEndpoint("BOT"); + await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const endpoint = await row(endpointId), saved = await setup(endpointId), enrolled = saved.slackCloudEvents!; + const callbackEvidence = structuredClone(saved.slackCallbackSurfaces); + const delivered = cloud.enqueue({ companyId, endpointId, publicId: endpoint.publicId, connectionId: endpoint.connectionId, + runtimeGeneration: saved.runtimeGeneration, teamId: enrolled.teamId, appId: enrolled.appId, bindingId: enrolled.bindingId }, secret, "EvDefaultOwner"); + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + expect((await owner.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(callbacks).toHaveLength(1); + expect(callbacks[0]!.message.author.userId).toBe("UOPERATOR"); + expect(cloud.acknowledged).toEqual([delivered.id]); + expect((await setup(endpointId)).slackCallbackSurfaces).toEqual(callbackEvidence); + expect(await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId))).toHaveLength(1); + for (const transport of transports) await transport.assertNoErrors(); + await service.configure(endpointId, { action: "pause" }, actorId); + const oldRequest = new Request("https://tenant.example/old-receipt", { method: "POST", body: new Uint8Array(delivered.body), + headers: { "content-type": "application/json", "x-slack-request-timestamp": delivered.packet.timestamp, "x-slack-signature": delivered.packet.signature } }); + const rejected = await service.handleWebhook(endpoint.publicId, "slack", oldRequest, { slackCloudGeneration: Number(saved.runtimeGeneration) }); + expect(await rejected.text()).toBe("superseded"); + expect(callbacks).toHaveLength(1); + await owner.pollOnce({ forDelivery: true }); + expect((await setup(endpointId)).slackCloudIntent).toBeUndefined(); + expect((await setup(endpointId)).slackCloudRetiredGeneration).toBeGreaterThan(Number(saved.runtimeGeneration)); + }, 60_000); + + it("two due registrations settle B while A still owns a busy credential lease", async () => { + const a = await createEndpoint("BUSY"); await confirm(a); await activate(a); await confirm(a); + const b = await createEndpoint("READY"); await confirm(b); await activate(b); await confirm(b); + await service.configure(a, { action: "pause" }, actorId); + const before = structuredClone((await setup(a)).slackCloudIntent!); + await service.configure(b, { action: "pause" }, actorId); + await holdCredentials(a); + const start = cloud.calls.length; + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await owner.pollOnce({ forDelivery: true }); + await owner.pollOnce({ forDelivery: true }); + expect((await setup(b)).slackCloudIntent).toBeUndefined(); + expect((await setup(b)).slackCloudRetiredGeneration).toBeDefined(); + const retained = (await setup(a)).slackCloudIntent!; + expect(retained).toMatchObject({ ...before, attempts: 1, failureCode: "credential_lease_busy", nextAttemptAt: expect.any(String) }); + expect(Date.parse(retained.nextAttemptAt!)).toBeGreaterThan(Date.now()); + expect(cloud.calls.slice(start).filter(call => call.enrollment?.endpointId === a)).toEqual([]); + expect(owner.nextPollAt()).not.toBeNull(); + releases.splice(0).forEach(release => release()); await Promise.all(heldLeases); + const delay = Math.max(0, Date.parse(retained.nextAttemptAt!) - Date.now()); + await new Promise(resolve => setTimeout(resolve, delay)); + await owner.pollOnce(); + expect((await setup(a)).slackCloudIntent).toBeUndefined(); + expect((await setup(a)).slackCloudRetiredGeneration).toBeDefined(); + }, 60_000); + + it("normal configuration rollback leaves no registration debt or retirement request", async () => { + const endpointId = await createEndpoint("ROLLBACK"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const before = await row(endpointId), requests = cloud.calls.length; + await service.shutdown(); runtime = makeRuntime(); + let rejectAfterIntent = false; + const stop = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { if (event === "intent") rejectAfterIntent = true; }); + service = chatChannelService(db, { runtime, heartbeat, fetch: providerFetch, publicBaseUrl: "https://tenant.example", + deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); }, renewCredentialMutationLease: async () => !rejectAfterIntent }); + try { + await expect(service.configure(endpointId, { action: "pause" }, actorId)).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); + expect((await row(endpointId)).status).toBe(before.status); + expect(await setup(endpointId)).toEqual(before.setup); + expect(cloud.calls).toHaveLength(requests); + } finally { stop(); } + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await owner.pollOnce({ forDelivery: true }); + expect(cloud.calls.filter(call => call.enrollment?.active === false)).toEqual([]); + }, 60_000); + + it.each(["intent", "generation"] as const)("real SQL refuses an enrollment response after newer %s custody wins", async change => { + const endpointId = await createEndpoint(`CAS${change}`); + const original = (await setup(endpointId)).slackCloudIntent!; + const successor = { ...original, id: randomUUID(), runtimeGeneration: original.runtimeGeneration + (change === "generation" ? 1 : 0) }; + cloud.beforeReceipt(async () => { + await db.update(chatEndpoints).set({ setup: sql`jsonb_set(jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(successor)}::jsonb), '{runtimeGeneration}', ${JSON.stringify(successor.runtimeGeneration)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, companyId))); + }); + const producer = createSlackCloudEnrollment(db, { resolveCredentials: async () => ({ signingSecret: secret }), + runtimeGeneration: value => Number(value.runtimeGeneration), connector: cloud.connector }); + await expect(chatCredentialMutationLease(db)(await row(endpointId), lease => producer(endpointId, undefined, lease))).rejects.toThrow("changed"); + expect((await setup(endpointId)).slackCloudIntent).toEqual(successor); + expect((await setup(endpointId)).slackCloudEvents).toBeUndefined(); + }, 60_000); + + it("malformed sealed Slack receipt is acknowledged once instead of retained as an owner outage", async () => { + const endpointId = await createEndpoint("INVALID"); await confirm(endpointId); + const endpoint = await row(endpointId), saved = await setup(endpointId), enrolled = saved.slackCloudEvents!; + const malformed = cloud.enqueue({ companyId, endpointId: "bad", publicId: endpoint.publicId, connectionId: endpoint.connectionId, + runtimeGeneration: saved.runtimeGeneration, teamId: enrolled.teamId, appId: enrolled.appId, bindingId: enrolled.bindingId }, secret, "EvMalformed"); + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + const result = await owner.pollOnce({ forDelivery: true }); + expect(result.ignored).toBe(1); expect(result.failed).toBe(0); + expect(cloud.acknowledged).toEqual([malformed.id]); + expect(callbacks).toEqual([]); + }, 60_000); + + it.each(["intent", "generation", "malformed-generation"] as const)( + "busy retry CAS preserves a concurrent %s winner and rethrows the exact owner error", async change => { + const endpointId = await createEndpoint(`BUSYCAS${change}`); + const before = await setup(endpointId), original = before.slackCloudIntent!; + const successor = change === "intent" ? { ...original, id: randomUUID() } : original; + const generation: unknown = change === "malformed-generation" ? { invalid: true } + : Number(before.runtimeGeneration) + (change === "generation" ? 1 : 0); + const busy = conflict("Synthetic owner credential lane is busy", { code: "chat_endpoint_credentials_busy" }); + const requests = cloud.calls.length; + const pendingBefore = databaseWorkPending(db, DELIVERY_QUEUES.connectionEvents), inheritedSignals: string[] = []; + const stopInherited = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { inheritedSignals.push(event); }); + try { + console.info(JSON.stringify({ slackRetrySignalProbe: "subscription", endpointId, pendingBefore, immediateSignals: [...inheritedSignals] })); + expect(inheritedSignals).toEqual(pendingBefore ? ["intent"] : []); + await reconcileDatabaseWork(db, DELIVERY_QUEUES.connectionEvents); + const pendingAfter = databaseWorkPending(db, DELIVERY_QUEUES.connectionEvents); + console.info(JSON.stringify({ slackRetrySignalProbe: "reconciled", endpointId, pendingBefore, pendingAfter, inheritedSignals: [...inheritedSignals] })); + expect(pendingAfter).toBe(false); + } finally { stopInherited(); } + const signals: string[] = []; + const stopSignals = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { signals.push(event); }); + const unregister = registerSlackCloudEnrollmentOwner(db, async id => { + expect(id).toBe(endpointId); + await db.update(chatEndpoints).set({ setup: sql`jsonb_set(jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(successor)}::jsonb), '{runtimeGeneration}', ${JSON.stringify(generation)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, companyId))); + throw busy; + }); + try { + await expect(reconcilePendingSlackCloudEnrollments(db, new Date())).rejects.toBe(busy); + const after = await setup(endpointId); + expect(after.slackCloudIntent).toEqual(successor); + expect(after.runtimeGeneration).toEqual(generation); + expect(after.slackCloudEvents).toBeUndefined(); + expect(cloud.calls).toHaveLength(requests); + expect(signals).toEqual([]); + } finally { unregister(); stopSignals(); } + }, 60_000); + + it.each(["unknown-error", "unrelated-conflict"] as const)( + "the %s owner failure propagates without a busy retry rewrite", async kind => { + const endpointId = await createEndpoint(`UNKNOWN${kind}`), before = await setup(endpointId); + const original = kind === "unknown-error" ? new Error("Synthetic unknown owner failure") + : conflict("Synthetic different authority failure", { code: "chat_endpoint_runtime_superseded" }); + const requests = cloud.calls.length; + const unregister = registerSlackCloudEnrollmentOwner(db, async id => { expect(id).toBe(endpointId); throw original; }); + try { + await expect(reconcilePendingSlackCloudEnrollments(db, new Date())).rejects.toBe(original); + expect(await setup(endpointId)).toEqual(before); + expect(cloud.calls).toHaveLength(requests); + } finally { unregister(); } + }, 60_000); + + it("an injected retry transaction failure preserves the original busy error and persistence cause", async () => { + const endpointId = await createEndpoint("RETRYWRITE"), before = await setup(endpointId); + const busy = conflict("Synthetic owner credential lane is busy", { code: "chat_endpoint_credentials_busy" }); + const writeError = new Error("Synthetic retry transaction unavailable"), requests = cloud.calls.length; + const unregister = registerSlackCloudEnrollmentOwner(db, async id => { expect(id).toBe(endpointId); throw busy; }); + const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(writeError); + try { + const error = await reconcilePendingSlackCloudEnrollments(db, new Date()).then(() => undefined, error => error); + expect(error).toBeInstanceOf(AggregateError); + if (!(error instanceof AggregateError)) throw new Error("Expected retry failure to retain both causes", { cause: error }); + expect(error.errors).toHaveLength(2); + expect(error.errors[0]).toBe(busy); + expect(error.errors[1]).toBe(writeError); + expect(transaction).toHaveBeenCalledOnce(); + expect(await setup(endpointId)).toEqual(before); + expect(cloud.calls).toHaveLength(requests); + } finally { transaction.mockRestore(); unregister(); } + }, 60_000); + + it("public tool removal retains Slack retirement custody through broker outage and ordinary owner recovery", async () => { + const endpointId = await createEndpoint("TOOLREMOVE"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const originalEndpoint = await row(endpointId), originalSetup = await setup(endpointId); + const [connection] = await db.select().from(toolConnections).where(eq(toolConnections.id, originalEndpoint.connectionId)); + const signingRef = connection!.credentialSecretRefs.find(ref => ref.configPath === "credentials.signingSecret"); + if (!signingRef) throw new Error("Normal Slack configure did not persist its signing credential binding"); + const credentialOwner = { consumerType: "tool_connection" as const, consumerId: connection!.id, + configPath: "credentials.signingSecret", actorType: "system" as const }; + await expect(secretService(db).resolveSecretValue(companyId, signingRef.secretId, "latest", credentialOwner)).resolves.toBe(secret); + cloud.setUnavailable(true); + const requests = vi.mocked(cloud.request).mock.calls.length; + const tools = toolAccessService(db); + const removed = await tools.archiveConnection(connection!.id, companyId, { actorType: "user", actorId }); + expect(removed.connection).toMatchObject({ id: connection!.id, status: "archived", enabled: false }); + const archived = await row(endpointId); + expect(archived.status).toBe("archived"); + const [revoked] = await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)); + expect(revoked!.credentialSecretRefs).toEqual([]); + expect(revoked!.credentialRefs).toEqual([]); + expect(await db.select().from(companySecretBindings).where(and(eq(companySecretBindings.companyId, companyId), + eq(companySecretBindings.targetType, "tool_connection"), eq(companySecretBindings.targetId, connection!.id)))).toEqual([]); + await expect(secretService(db).resolveSecretValue(companyId, signingRef.secretId, "latest", credentialOwner)).rejects.toThrow(); + expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests); + const intent = (await setup(endpointId)).slackCloudIntent; + expect(intent).toMatchObject({ version: 1, active: false, companyId, endpointId, connectionId: connection!.id, + publicId: originalEndpoint.publicId, runtimeGeneration: Number(originalSetup.runtimeGeneration), + subject: originalSetup.slackCloudSubject, instanceId: originalSetup.slackCloudInstanceId, attempts: 0 }); + expect(JSON.stringify(intent)).not.toContain(secret); + await tools.archiveConnection(connection!.id, companyId, { actorType: "user", actorId }); + expect((await setup(endpointId)).slackCloudIntent).toEqual(intent); + const owner = githubConnectionEventService(db, { connector: cloud.connector }); + await expect(owner.pollOnce({ forDelivery: true })).rejects.toMatchObject({ status: 503 }); + const retry = (await setup(endpointId)).slackCloudIntent!; + expect(retry).toMatchObject({ ...intent, attempts: 1, failureCode: "broker_unavailable", nextAttemptAt: expect.any(String) }); + const retryAt = Date.parse(retry.nextAttemptAt!); + expect(retryAt).toBeGreaterThan(Date.now()); + cloud.setUnavailable(false); cloud.install(); + await new Promise(resolve => setTimeout(resolve, Math.max(0, retryAt - Date.now()))); + await owner.pollOnce(); + expect((await setup(endpointId)).slackCloudIntent).toBeUndefined(); + expect((await setup(endpointId)).slackCloudRetiredGeneration).toBe(Number(originalSetup.runtimeGeneration)); + expect(cloud.calls.filter(call => call.operation === "slack-events" && call.enrollment?.active === false + && call.enrollment.endpointId === endpointId)).toHaveLength(1); + expect(callbacks).toEqual([]); + }, 60_000); + + it("public tool removal rolls back both archive writes without dropping current Slack custody or credentials", async () => { + const endpointId = await createEndpoint("TOOLROLLBACK"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId); + const before = await row(endpointId); + const [connection] = await db.select().from(toolConnections).where(eq(toolConnections.id, before.connectionId)); + const failure = new Error("Synthetic removal transaction rollback"), requests = vi.mocked(cloud.request).mock.calls.length; + const realTransaction = db.transaction.bind(db); + const transaction = vi.spyOn(db, "transaction").mockImplementationOnce(async work => realTransaction(async tx => { + await work(tx); + const [archivedEndpoint] = await tx.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)); + const [archivedConnection] = await tx.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)); + expect(archivedEndpoint!.status).toBe("archived"); + expect(archivedConnection).toMatchObject({ status: "archived", enabled: false }); + throw failure; + })); + try { + await expect(toolAccessService(db).archiveConnection(connection!.id, companyId, { actorType: "user", actorId })).rejects.toBe(failure); + expect(await row(endpointId)).toMatchObject({ status: before.status, setup: before.setup }); + const [after] = await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)); + expect(after).toMatchObject({ status: connection!.status, enabled: connection!.enabled, + credentialSecretRefs: connection!.credentialSecretRefs, credentialRefs: connection!.credentialRefs }); + expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests); + } finally { transaction.mockRestore(); } + }, 60_000); + + it("public tool removal refuses a foreign company before Slack archival or retirement staging", async () => { + const endpointId = await createEndpoint("TOOLFOREIGN"); await confirm(endpointId); + const before = await row(endpointId), requests = vi.mocked(cloud.request).mock.calls.length; + await expect(toolAccessService(db).archiveConnection(before.connectionId, randomUUID(), { actorType: "user", actorId })).rejects.toMatchObject({ status: 404 }); + expect(await row(endpointId)).toEqual(before); + expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests); + }, 60_000); +}); diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 7a4943f1882..870eba73ea1 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -381,15 +381,27 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu // window. A company hint must not make that real deadline wait one second. await coordinator!.stop(); coordinator = undefined; const dueAt = Date.now() + 500; + const actualWakeup = wakeup.getMockImplementation()!; + let otherWakeEnteredAt: number | null = null; + wakeup.mockImplementation((id, options) => { + // Capture dispatch ENTRY before the original mock performs any DB I/O. + if (id === other.agentId) otherWakeEnteredAt ??= Date.now(); + return actualWakeup(id, options); + }); await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id)); await start(service); await flush(); await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" }); await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id, "More changes while another company is due", { userId: "board-user" }); await flush(); + const armedBeforeAdvance = coordinator!.nextWakeAt(); + // A null deadline can mean the due work is already running. Any still + // armed deadline must preserve the known earlier obligation. + if (armedBeforeAdvance !== null) expect(armedBeforeAdvance).toBeLessThanOrEqual(dueAt); await vi.advanceTimersByTimeAsync(500); await flush(); await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); - expect(Date.now()).toBeLessThan(dueAt + 500); + expect(otherWakeEnteredAt).not.toBeNull(); + expect(otherWakeEnteredAt!).toBeLessThan(dueAt + 500); expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent); const generated = (await statusCardService(db).getById(other.id))!; expect(generated.generatingIssueId).toBeTruthy(); diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts index a4081a4b4d7..16086ebaeef 100644 --- a/server/src/services/ai-connections.ts +++ b/server/src/services/ai-connections.ts @@ -174,7 +174,7 @@ export function aiConnectionService(db: Db) { : ("connected" as const), }, ]; - }); + }).sort((a, b) => a.id.localeCompare(b.id) || a.grantId.localeCompare(b.grantId)); } async function setDefault( companyId: string, diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index e6cdc79894a..00a3a307e8b 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -1,4 +1,6 @@ import { githubReviewCheckService } from "./chat-github-checks.js"; +import { createSlackCloudEventIngress } from "./chat-channels/slack/cloud-ingress.js"; +import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent, registerSlackCloudEnrollmentOwner } from "./chat-channels/slack/cloud-enrollment.js"; import { leasedChatProvider } from "./chat-channels/runtime/receiver-policy.js"; import { chatReceiverSchedule } from "./chat-channels/runtime/receiver-schedule.js"; import { registerChatReceiverWork } from "./chat-channels/runtime/receiver-work.js"; @@ -842,6 +844,10 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { } const withCredentialMutationLease = chatCredentialMutationLease(db, options); + const unregisterSlackCloudIngress = createSlackCloudEventIngress(db, { + withCredentialMutationLease, resolveCredentials, runtimeGeneration: endpoint => runtimeGeneration(endpoint.setup), + handleWebhook: (publicId, provider, request, generation) => handleWebhook(publicId, provider, request, { slackCloudGeneration: generation }), + }); const { list, get, create, update } = createChatEndpoints(db, { endpointRecord, CAPABILITIES, PROVIDER_LABELS, withCredentialMutationLease, getPublicBaseUrl, getWebhookPublicBaseUrl, @@ -2936,10 +2942,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { githubIngressActionId: string; githubIngressFence: LifecycleRuntimeFence; superseded?: boolean; - }, + } | { slackCloudGeneration: number }, ) { const serviceReceivedAtMs = Date.now(); - const githubScope = createGitHubWebhookScope(provider, internalContext, options.githubWebhookResponseBudgetMs, GITHUB_WEBHOOK_RESPONSE_BUDGET_MS); + const slackCloudGeneration = internalContext && "slackCloudGeneration" in internalContext ? internalContext.slackCloudGeneration : undefined; + const githubScope = createGitHubWebhookScope(provider, internalContext && "githubIngressActionId" in internalContext ? internalContext : undefined, options.githubWebhookResponseBudgetMs, GITHUB_WEBHOOK_RESPONSE_BUDGET_MS); const { replayingDurableGitHubIngress, ignoreSupersededIngress, matchesGitHubIngressFence, githubResponseDeadlineAt } = githubScope; const endpoint = await db .select() @@ -2962,6 +2969,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { throw notFound("Chat endpoint not found"); } recordChatWebhookStage("endpoint_resolved", endpoint.id); + if (provider === "slack" && slackCloudGeneration !== undefined && runtimeGeneration(endpoint.setup) !== slackCloudGeneration) + return new Response("superseded", { status: 200 }); // A pause is a durable ingress fence. Providers generally retry non-2xx // webhooks, so acknowledge late callbacks without recreating a runtime or // admitting any Paperclip mutation. @@ -2998,6 +3007,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { code: "chat_endpoint_runtime_superseded", }); } + if (provider === "slack" && slackCloudGeneration !== undefined && runtimeContext.generation !== slackCloudGeneration) + return new Response("superseded", { status: 200 }); // runtimeFor can return a newer instance than the leased preflight saw. // Keep the exact staged epoch; callbacks on a matching instance retain // their normal transactional fences if a pause wins after this check. @@ -3403,7 +3414,14 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { : null; } + const slackCloudEnrollment = createSlackCloudEnrollment(db, { resolveCredentials, runtimeGeneration }); + const unregisterSlackCloudEnrollment = registerSlackCloudEnrollmentOwner(db, async endpointId => { + const record = await endpointRecord(endpointId); + if (!record) throw new Error("Slack enrollment endpoint is absent"); + await withCredentialMutationLease(record.endpoint, lease => slackCloudEnrollment(endpointId, undefined, lease)); + }); const { configure, configureWithCredentialLease, runtimeConfig } = createChatConfiguration(db, { + stageConfigurationEffects: stageSlackCloudEnrollmentIntent, endpointRecord, get, withCredentialMutationLease, getWebhookPublicBaseUrl, invalidateRuntime, nativeBotIdentityMatches, assertNativeBotIdentityAvailable, isNativeBotIdentityUniqueViolation, nativeBotIdentityConflict, legacyGitHubLabelsMatchVerifiedCredentials, normalizedCredentials, verifyCredentials, @@ -3611,6 +3629,8 @@ return { unregisterSlackTaskAuthority(); voice.unregisterAgentTools(); unregisterGitHubCloudIngress(); + unregisterSlackCloudIngress(); + unregisterSlackCloudEnrollment(); unregisterCommittedResponseAuthority(); await Promise.allSettled([...publicationEndpointTasks.values()]); await Promise.allSettled([...backgroundMessageTasks]); diff --git a/server/src/services/chat-channels/configuration.ts b/server/src/services/chat-channels/configuration.ts index 8ded7cbb96e..8eb17d429bb 100644 --- a/server/src/services/chat-channels/configuration.ts +++ b/server/src/services/chat-channels/configuration.ts @@ -100,6 +100,7 @@ type ChatConfigurationOptions = Pick, " githubWizard: Pick["githubWizard"], "repairCloud">; nativeBotIdentityClaimBarrier?: () => Promise; allowLocalVoiceBoard?: boolean; + stageConfigurationEffects?: (transaction: Db, endpointId: string, actorUserId: string | null | undefined) => Promise; }; export function createChatConfiguration(db: Db, options: ChatConfigurationOptions) { const { endpointRecord, get, withCredentialMutationLease, getWebhookPublicBaseUrl, invalidateRuntime, nativeBotIdentityMatches, assertNativeBotIdentityAvailable, isNativeBotIdentityUniqueViolation, nativeBotIdentityConflict, legacyGitHubLabelsMatchVerifiedCredentials, normalizedCredentials, verifyCredentials, persistCredentials, clearCredentials, resolveCredentials, prepareProviderInventory, reconcileProviderResourceRows, reconcileDiscordCommands, runtimeFor, runtimeContextForRecord, stageTelegramMaintenance, processTelegramMaintenance, recordTelegramStopSubscription, fetchImpl, redactError, runtimeGeneration, leasedChatProvider, PROVIDER_LABELS, SUPPLIED_CREDENTIAL_KEYS, PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS, isSupportedTelegramWebhookBaseUrl } = options; @@ -176,6 +177,14 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption const record = await endpointRecord(endpointId); if (!record) throw notFound("Chat endpoint not found"); const endpoint = record.endpoint; + const mutate = (work: (tx: Parameters[0]>[0]) => Promise) => db.transaction(async tx => { + const result = await work(tx); + if (endpoint.provider === "slack" && options.stageConfigurationEffects) { + await options.stageConfigurationEffects(tx as unknown as Db, endpointId, actorUserId); + await credentialLease.assertOwned(tx); + } + return result; + }); if (input.action === "pause") { if (endpoint.status !== "active") { throw conflict("Only an active chat connection can be paused", { @@ -183,7 +192,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption }); } await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const pausedAt = new Date(); const current = await tx @@ -299,7 +308,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption let activatedEndpoint!: EndpointRow; try { await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const resumedAt = new Date(); const current = await tx @@ -384,7 +393,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption const failure = redactError(error); const failedAt = new Date(); await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); @@ -478,7 +487,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption }); } await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const current = await tx .select({ setup: chatEndpoints.setup, status: chatEndpoints.status }) @@ -724,7 +733,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption if (!endpoint.botExternalId) { try { await managementContext?.reauthorize(); - const claimed = await db.transaction(async (tx) => { + const claimed = await mutate(async (tx) => { await credentialLease.assertOwned(tx); await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); @@ -793,7 +802,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption (input.action === "configure" || input.action === "reconnect"); try { await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); const current = await tx .select({ setup: chatEndpoints.setup }) @@ -892,7 +901,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption const failure = redactError(error); await invalidateRuntime(endpoint.id).catch(() => undefined); await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); @@ -922,7 +931,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption } await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await mutate(async (tx) => { await credentialLease.assertOwned(tx); await logActivity(tx as unknown as Db, { companyId: endpoint.companyId, diff --git a/server/src/services/chat-channels/runtime-types.ts b/server/src/services/chat-channels/runtime-types.ts index c0c9dce4631..4b9e9e5f5ab 100644 --- a/server/src/services/chat-channels/runtime-types.ts +++ b/server/src/services/chat-channels/runtime-types.ts @@ -48,6 +48,34 @@ export type InternalSetupState = ChatEndpointSetupState & { runtimeGeneration?: number; /** Binds pre-install URL verification to the signing secret later configured. */ slackVerificationSigningFingerprint?: string; + + slackCloudSubject?: string; + slackCloudInstanceId?: string; + slackCloudRetiredGeneration?: number; + slackCloudIntent?: { + version: 1; + id: string; + subject: string; + instanceId: string; + companyId: string; + endpointId: string; + publicId: string; + connectionId: string; + runtimeGeneration: number; + active: boolean; + teamId?: string; + appId?: string; + attempts: number; + nextAttemptAt: string | null; + failureCode?: string; + }; + slackCloudEvents?: { + version: 1; + bindingId: string; + runtimeGeneration: number; + teamId: string; + appId: string; + }; slackCallbackSurfaces?: Partial< Record >; diff --git a/server/src/services/chat-channels/slack/cloud-drain.test.ts b/server/src/services/chat-channels/slack/cloud-drain.test.ts new file mode 100644 index 00000000000..375bfd8879a --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-drain.test.ts @@ -0,0 +1,75 @@ +import { generateKeyPairSync } from "node:crypto"; +import { registerSlackCloudEnrollmentOwner } from "./cloud-enrollment.js"; +import type { Db } from "@paperclipai/db"; +import { afterEach, expect, it, vi } from "vitest"; +import { githubConnectionEventService } from "../../github-connection-events.js"; +import type { PaperclipCloudConnector } from "../../paperclip-cloud-connector.js"; +import { registerSlackCloudEventIngress } from "./cloud-ingress.js"; +import { SlackCloudReceiptError, type SlackCloudEvent } from "./cloud-event.js"; + +afterEach(() => vi.unstubAllEnvs()); +function fixture(handler: (event: SlackCloudEvent) => Promise, pendingEnrollment = false) { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", ""); + let reads = 0; + const db = { select: () => { + const rows = [[], [], [{ companyId: "company" }], [{ id: "endpoint", setup: { slackCloudIntent: { nextAttemptAt: "2000-01-01T00:00:00Z" } } }]][reads++ % (pendingEnrollment ? 4 : 3)]!; + const query = { from: () => query, innerJoin: () => query, where: () => query, orderBy: () => query, limit: async () => rows, + then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(rows).then(resolve) }; + return query; + } } as unknown as Db; + const event: SlackCloudEvent = { id: "slack-receipt", provider: "slack", event: "event_callback", installationId: "binding", bindingIds: ["binding"], + createdAt: new Date().toISOString(), payload: {} }; + const connector = { leaseEvents: vi.fn(async () => ({ leaseId: "lease", events: [event] })), + acknowledgeEvents: vi.fn(async () => 1), eventWakeVersion: () => 1 as const }; + const unregister = registerSlackCloudEventIngress(db, handler); + let time = Date.now(); + const service = githubConnectionEventService(db, { connector: connector as unknown as PaperclipCloudConnector, now: () => new Date(time) }); + return { db, service, connector, unregister, event, time: () => time, advance: (ms: number) => { time += ms; } }; +} + +it("actual connection-event owner routes Slack and retries a lost ACK through the registered owner", async () => { + const handler = vi.fn(async () => {}), f = fixture(handler); + try { + f.connector.acknowledgeEvents.mockRejectedValueOnce(new Error("ack response lost")); + await expect(f.service.pollOnce({ forDelivery: true })).rejects.toThrow("ack response lost"); + expect((await f.service.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(handler).toHaveBeenCalledTimes(2); + expect(f.connector.leaseEvents).toHaveBeenCalledWith({ subject: "github-bot-events", companyId: "company", eventWakeVersion: 1 }); + expect(f.connector.acknowledgeEvents).toHaveBeenLastCalledWith(expect.objectContaining({ deliveryIds: [f.event.id] })); + } finally { f.unregister(); } +}); + +it.each(["retry", "attention", "invalid"] as const)("actual owner distinguishes %s receipt failure before ACK", async disposition => { + const f = fixture(async () => { throw new SlackCloudReceiptError("synthetic_code", disposition); }); + try { + const result = await f.service.pollOnce({ forDelivery: true }); + expect(result.failed).toBe(disposition === "invalid" ? 0 : 1); + expect(f.connector.acknowledgeEvents).toHaveBeenCalledTimes(disposition === "invalid" ? 1 : 0); + } finally { f.unregister(); } +}); + +it.each(["chat_endpoint_credentials_busy", "temporary enrollment owner failure"])("drains and ACKs before retaining %s on a bounded existing-worker retry", async message => { + const handler = vi.fn(async () => {}), f = fixture(handler, true); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "https://cloud.example.test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", generateKeyPairSync("ed25519").privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", generateKeyPairSync("x25519").privateKey.export({ format: "jwk" }).d!); + const owner = vi.fn(async () => { + expect(handler).toHaveBeenCalled(); + expect(f.connector.acknowledgeEvents).toHaveBeenCalled(); + throw new Error(message); + }); + const unregister = registerSlackCloudEnrollmentOwner(f.db, owner); + try { + expect((await f.service.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(owner).toHaveBeenCalledTimes(1); + expect(f.service.nextPollAt()).toBe(f.time() + 5_000); + await f.service.pollOnce(); + expect(owner).toHaveBeenCalledTimes(1); + expect(f.connector.leaseEvents).toHaveBeenCalledTimes(1); + f.advance(5_000); + expect((await f.service.pollOnce()).processed).toBe(1); + expect(owner).toHaveBeenCalledTimes(2); + } finally { unregister(); f.unregister(); } +}); diff --git a/server/src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts b/server/src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts new file mode 100644 index 00000000000..9121189db8f --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts @@ -0,0 +1,451 @@ +import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "../../../__tests__/helpers/slack-cloud/transport.js"; +import { createHmac, generateKeyPairSync, randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { and, eq, sql } from "drizzle-orm"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import type { ToolCredentialSecretRef } from "@paperclipai/shared"; +import { expect, it, vi } from "vitest"; +import { activityLog, agentRuntimeState, agentTaskSessions, agentWakeupRequests, agents, applyPendingMigrations, authUsers, chatConversations, chatDeliveries, chatExternalPrincipals, chatIdentityLinks, chatEndpointResources, chatEndpoints, chatMessageLinks, chatSlackRegistrations, heartbeatRunEvents, heartbeatRuns, companies, companyMemberships, companySecrets, companySkills, costEvents, financeEvents, closeCheckpointDb, closeFileStorageDb, closeLivenessDb, createDb, issueComments, issues, toolApplications, toolConnections } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "../../../lib/dedicated-test-database-url.js"; +import { subscribeDatabaseWork } from "@paperclipai/db/work-signals"; +import { DELIVERY_QUEUES } from "../../delivery-work-notifications.js"; +import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from "../../paperclip-cloud-connector.js"; +import { chatChannelService } from "../../chat-channels.js"; +import { ChatRuntime } from "../../chat-runtime.js"; +import { heartbeatService } from "../../heartbeat.js"; +import { githubConnectionEventService } from "../../github-connection-events.js"; +import { secretService } from "../../secrets.js"; +import type { ChatSdkMessageCallbackEvent } from "../../chat-sdk-runtime.js"; +import { createChatSdkEndpointRuntime } from "../../chat-sdk-runtime.js"; +import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent } from "./cloud-enrollment.js"; +import { createSlackCloudEventIngress, dispatchSlackCloudEvent } from "./cloud-ingress.js"; +import type { InternalSetupState } from "../runtime-types.js"; + +const appUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_TEST_DATABASE_URL"); +const cloudUrl = resolveExternalTestDatabaseUrl("SLACK_EVENT_TEST_DATABASE_URL"); +const cloudDist = process.env.SLACK_TEST_CLOUD_DIST; + +it.skipIf(!appUrl || !cloudUrl || !cloudDist)("signed Slack enrollment and actual event owner preserve one domain reply through lost ACK and removal retries", async () => { + const network = guardSlackFixtureNetwork({ requirePreload: true }); + const load = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, file)).href); + const [{ PostgresCloudHarnessRegistry }, { PostgresConnectorStore }, { ConnectorBroker }, { InMemoryProviderSecretStore }, + { providerRegistryFromEnvironment }, { CloudStackProvisioner }, { FakeCloudStackProvider }, { Pool }] = await Promise.all([ + load("registry/postgres.js"), load("connections/postgres.js"), load("connections/broker.js"), load("providers/neon.js"), + load("connections/providers.js"), load("provisioner/provisioner.js"), load("providers/fake.js"), + import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, "../node_modules/pg/lib/index.js")).href), + ]); + await applyPendingMigrations(appUrl!); + const db = createDb(appUrl!, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + const pool = new Pool({ connectionString: cloudUrl!, max: 2 }); + + const registry = Reflect.construct(PostgresCloudHarnessRegistry, [pool]) as { + applyMigrations(folder: string, timeout: number): Promise; + hydrate(): Promise; + close(): Promise; + }; + try { + await registry.applyMigrations(path.resolve(cloudDist!, "../migrations"), 30_000); + await registry.hydrate(); + } + catch (error) { await db.$client.end({ timeout: 1 }); await registry.close(); throw error; } + const store = new PostgresConnectorStore(pool), secrets = new InMemoryProviderSecretStore(); + const id = randomUUID(), companyId = randomUUID(), endpointId = randomUUID(), agentId = randomUUID(), connectionId = randomUUID(), applicationId = randomUUID(); + const actorId = `slack-owner-${id}`; + const instanceId = `instance-${id}`, stackId = `slack-${id}`, publicId = "a".repeat(43), signingSecret = "0123456789abcdef0123456789abcdef"; + const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"), now = new Date(); + let broker: InstanceType; + let enrollmentRequests = 0; + const server = createServer((req, res) => { + if (req.url === "/v1/connector/slack-events") enrollmentRequests++; + if (brokerUnavailable) { res.writeHead(503, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "fixture_broker_unavailable" })); return; } + if (loseEnrollmentResponse && req.url === "/v1/connector/slack-events") { + loseEnrollmentResponse = false; + res.end = (() => { res.destroy(); return res; }) as typeof res.end; + } + void broker.handle(req, res, new URL(req.url!, base)).catch((error: unknown) => { + if (!res.headersSent) res.writeHead(500); res.end(String(error)); + }); }); + let base = ""; + let brokerUnavailable = false, loseEnrollmentResponse = false; + let unregister: (() => void) | undefined; + let runtime: ReturnType | undefined; + const deliveryIds: string[] = []; + let normalService: ReturnType | undefined; + let secretDirectory: string | undefined; + let normalHeartbeat: ReturnType | undefined; + let releaseEnrollmentLease: (() => void) | undefined; + let heldEnrollmentLease: Promise | undefined; + try { + await new CloudStackProvisioner({ provider: new FakeCloudStackProvider(), registry }).provisionDevelopmentStack({ + idempotencyKey: id, accountGroup: { id: `account-${id}`, slug: stackId, displayName: "Slack fixture" }, + owner: { id: actorId, email: `${id}@example.test` }, + stack: { stackId, accountGroupId: `account-${id}`, stackSlug: stackId, displayName: "Slack fixture", kind: "demo", primaryHost: `${stackId}.example.test`, region: "us-east-1" }, + versionPlan: { appVersion: "fixture", schemaVersion: "fixture", rolloutTrack: "stable" }, + sleepPolicy: { desiredState: "awake" }, primaryCompany: { paperclipCompanyId: companyId, displayName: "Fixture", defaultBudgetCents: 0 }, + }); + await store.putCloudStackInstance({ id: instanceId, kind: "cloud_stack", companyStackId: stackId, environment: "test", + signPublicKey: signing.publicKey.export({ format: "jwk" }).x!, sealPublicKey: sealing.publicKey.export({ format: "jwk" }).x!, + origins: ["https://tenant.example"], activeOrigin: "https://tenant.example", label: "Fixture", status: "active", createdAt: now, updatedAt: now }); + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing fixture address"); + base = `http://127.0.0.1:${address.port}`; + const wake = vi.fn(async () => {}); + broker = new ConnectorBroker({ publicOrigin: base, environment: "test", store, + providers: providerRegistryFromEnvironment({}, "test"), secrets: { resolve: async () => undefined }, + authenticate: async () => ({ globalUserId: actorId }), canApproveInstance: async (principal: { globalUserId: string }) => principal.globalUserId === actorId, + slackEvents: { secrets, secretRefPrefix: "secret:///test/slack" }, + eventWakeEnrollmentEnabled: true, eventWakeDeliveryEnabled: true, wakeCloudStack: wake }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", signing.privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", sealing.privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); + await db.insert(companies).values({ id: companyId, name: "Slack", issuePrefix: `S${id.slice(0, 7)}` }); + await db.insert(authUsers).values({ id: actorId, name: "Slack owner", email: `${id}@owner.example.test`, emailVerified: true, createdAt: now, updatedAt: now }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, status: "active", membershipRole: "operator" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Slack", adapterType: "process", status: "idle", runtimeConfig: { heartbeat: { wakeOnDemand: true, intervalSec: 0 } } }); + await db.insert(companyMemberships).values({ companyId, principalType: "agent", principalId: agentId, status: "active", membershipRole: "member" }); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Slack", type: "chat" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: id, + transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, + publicId, provider: "slack", status: "active", providerAccountId: "TTEST", setup: { runtimeGeneration: 1 } as InternalSetupState }); + await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", + manifest: {}, manifestHash: "fixture", callbackUri: "https://tenant.example/callback", appId: "ATEST", createdByUserId: actorId }); + const withLease = chatCredentialMutationLease(db); + const resolveCredentials = vi.fn(async () => ({ signingSecret })); + const produce = createSlackCloudEnrollment(db, { resolveCredentials, + runtimeGeneration: setup => Number(setup.runtimeGeneration ?? 0) }); + const endpoint = async () => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)))[0]!; + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId)); + await withLease(await endpoint(), lease => produce(endpointId, actorId, lease)); + const saved = (await endpoint()).setup as InternalSetupState; + expect(saved.slackCloudEvents?.runtimeGeneration).toBe(1); + expect(JSON.stringify(saved)).not.toContain(signingSecret); + const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvPaired", team_id: "TTEST", api_app_id: "ATEST", + event: { type: "app_mention", text: "<@UBOT> hello", user: "UOPERATOR", channel: "CTEST", ts: "1788.500", event_ts: "1788.500" } })); + const timestamp = String(Math.floor(Date.now() / 1000)); + const request = { rawBody, timestamp, signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}` }; + const current = await broker.slackEvents.current(instanceId, publicId); + await expect(broker.slackEvents.receive(current, request)).rejects.toThrow("SLACK_DURABLE_WAKE_UNAVAILABLE"); + const connector = createPaperclipCloudConnector({ config: paperclipCloudConnectorConfigFromEnv()! }); + expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull(); + expect(await broker.slackEvents.receive(current, request)).toEqual({ status: 200, duplicate: false }); + await vi.waitFor(() => expect(wake).toHaveBeenCalledOnce()); + const leased = await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 }); + expect(leased?.failedEventIds).toBeUndefined(); expect(leased?.events).toHaveLength(1); + const event = leased!.events[0]!; deliveryIds.push(event.id); + if (event.provider !== "slack") throw new Error("Unexpected event owner"); + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready"); + const onMessage = vi.fn(async (_event: ChatSdkMessageCallbackEvent) => {}); + runtime = createChatSdkEndpointRuntime({ companyId, endpointId, logger: "silent", callbacks: { onMessage }, + persistence: { read: async () => null, compareAndSet: async () => true, deleteIfVersion: async () => true }, + providerConfig: { provider: "slack", userName: "paperclip-agent", credentials: { apiUrl: "http://127.0.0.1:1/api/", botToken: "xoxb-fixture", botUserId: "UBOT", signingSecret } } }); + await runtime.initialize(); + const transport = stubSlackFixtureTransport(runtime.getProviderAdapter()); + unregister = createSlackCloudEventIngress(db, { withCredentialMutationLease: withLease, resolveCredentials, + runtimeGeneration: value => Number((value.setup as InternalSetupState).runtimeGeneration ?? 0), + handleWebhook: async (_publicId, _provider, req, generation) => { + expect(generation).toBe(1); + + await withLease(await endpoint(), async lease => { await lease.assertOwned(); }); + return runtime!.handleWebhook(req); + } }); + await dispatchSlackCloudEvent(db, event); + expect(onMessage).toHaveBeenCalledOnce(); + expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ message: expect.objectContaining({ author: expect.objectContaining({ userId: "UOPERATOR", userName: "Operator" }) }) })); + expect(transport).toHaveBeenCalled(); + await transport.assertNoErrors(); + expect(await connector.acknowledgeEvents({ subject: actorId, companyId, leaseId: leased!.leaseId, deliveryIds: [event.id] })).toBe(1); + expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull(); + await db.update(chatEndpoints).set({ setup: { ...saved, runtimeGeneration: 2 } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId)); + await dispatchSlackCloudEvent(db, event); + expect(onMessage).toHaveBeenCalledOnce(); + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId)); + await withLease(await endpoint(), lease => produce(endpointId, actorId, lease)); + const replacement = ((await endpoint()).setup as InternalSetupState).slackCloudEvents!; + expect(replacement.runtimeGeneration).toBe(2); expect(replacement.bindingId).not.toBe(saved.slackCloudEvents!.bindingId); + const firstMessage = onMessage.mock.calls[0]![0]; + const [task] = await db.insert(issues).values({ companyId, title: "Existing Slack conversation", status: "in_progress", + assigneeAgentId: agentId, createdByUserId: actorId, identifier: `S${id.slice(0, 7)}-1`, issueNumber: 1 }).returning(); + const [resource] = await db.insert(chatEndpointResources).values({ companyId, endpointId, type: "channel", + providerResourceId: firstMessage.thread.channelId, label: "Slack fixture", availability: "available", enabled: true }).returning(); + await db.insert(chatConversations).values({ companyId, endpointId, issueId: task!.id, resourceId: resource!.id, + externalConversationId: firstMessage.thread.channelId, externalThreadId: firstMessage.thread.id, + externalLabel: "Slack fixture", isDirectMessage: firstMessage.thread.isDM }); + secretDirectory = await mkdtemp(path.join(os.tmpdir(), "slack-owner-secrets-")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(secretDirectory, "master.key")); + const markerPath = path.join(secretDirectory, "normal-process-marker.json"); + const markerText = `slack-domain-once-${id}`; + await db.update(agents).set({ adapterConfig: { command: process.execPath, cwd: secretDirectory, + args: ["--require", fileURLToPath(new URL("../../../__tests__/helpers/slack-cloud/prewire.cjs", import.meta.url)), "-e", `if(process.env.SLACK_GATE_GUARD_PID!==String(process.pid))throw Error("child prewire guard missing");require("node:fs").writeFileSync(${JSON.stringify(markerPath)},JSON.stringify({pid:process.pid,runId:process.env.PAPERCLIP_RUN_ID,marker:${JSON.stringify(markerText)}}),{flag:"ax"});console.log(${JSON.stringify(markerText)});`], + timeoutSec: 10, graceSec: 1 } }).where(eq(agents.id, agentId)); + const vault = secretService(db), refs: ToolCredentialSecretRef[] = []; + for (const [key, value] of Object.entries({ botToken: "xoxb-fixture", signingSecret })) { + const secret = await vault.create(companyId, { name: `slack-${key}-${id}`, provider: "local_encrypted", value }, { userId: actorId }); + await vault.createBinding({ companyId, secretId: secret.id, targetType: "tool_connection", targetId: connectionId, configPath: `credentials.${key}` }); + refs.push({ secretId: secret.id, versionSelector: "latest", configPath: `credentials.${key}`, required: true, projectionClass: "unclassified" }); + } + await db.update(toolConnections).set({ credentialSecretRefs: refs }).where(eq(toolConnections.id, connectionId)); + const [linkedPrincipal] = await db.insert(chatExternalPrincipals).values({ companyId, provider: "slack", providerAccountId: "TTEST", + externalId: "UOPERATOR", kind: "user", displayName: "Operator" }).returning(); + await db.insert(chatIdentityLinks).values({ companyId, endpointId, principalId: linkedPrincipal!.id, paperclipUserId: actorId, + status: "linked", confirmedAt: now }); + const canonicalCallback = "https://tenant.example/api/chat-webhooks/" + publicId + "/slack"; + await db.update(chatEndpoints).set({ botExternalId: "UBOT", sponsorUserId: actorId, allowGroupChats: true, + setup: { ...((await endpoint()).setup as InternalSetupState), step: "complete", webhookVerifiedAt: now.toISOString(), + slackCallbackSurfaces: { events: { url: canonicalCallback, observedAt: now.toISOString() } } } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId)); + const callbackEvidence = structuredClone(((await endpoint()).setup as InternalSetupState).slackCallbackSurfaces); + const actualRuntime = new ChatRuntime(), sdkCallbacks: ChatSdkMessageCallbackEvent[] = []; + const transports: ReturnType[] = []; + const replaceEndpoint = actualRuntime.replaceEndpoint.bind(actualRuntime); + vi.spyOn(actualRuntime, "replaceEndpoint").mockImplementation(async input => { + if (input.providerConfig.provider !== "slack") throw new Error("Unexpected fixture provider"); + const value = await replaceEndpoint({ ...input, providerConfig: { ...input.providerConfig, + credentials: { ...input.providerConfig.credentials, apiUrl: "http://127.0.0.1:1/api/" } }, + callbacks: { ...input.callbacks, onMessage: async message => { sdkCallbacks.push(message); await input.callbacks.onMessage(message); } } }); + transports.push(stubSlackFixtureTransport(value.getProviderAdapter())); + return value; + }); + normalHeartbeat = heartbeatService(db); + const receiptFetch: typeof fetch = async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (!["https://slack.com/api/reactions.add", "https://slack.com/api/reactions.remove"].includes(url)) throw new Error(`Unexpected normal Slack owner request ${url}`); + expect(init?.method).toBe("POST"); + expect(JSON.parse(String(init?.body))).toMatchObject({ channel: "CTEST", name: "eyes" }); + return Response.json({ ok: true }); + }; + const deferred: (() => void | Promise)[] = []; + normalService = chatChannelService(db, { runtime: actualRuntime, publicBaseUrl: "https://tenant.example", + deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); }, heartbeat: normalHeartbeat, fetch: receiptFetch }); + const ownerMessageTs = (Date.now() / 1000).toFixed(6); + const ownerRawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvActualOwner", team_id: "TTEST", api_app_id: "ATEST", + event: { type: "app_mention", text: "<@UBOT> one durable reply", user: "UOPERATOR", channel: "CTEST", ts: ownerMessageTs, event_ts: ownerMessageTs, thread_ts: "1788.500" } })); + const ownerTimestamp = String(Math.floor(Date.now() / 1000)); + const ownerRequest = { rawBody: ownerRawBody, timestamp: ownerTimestamp, + signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${ownerTimestamp}:`).update(ownerRawBody).digest("hex")}` }; + await broker.slackEvents.receive(await broker.slackEvents.current(instanceId, publicId), ownerRequest); + let lostAck = true; + const captured: NonNullable>>["events"] = []; + const ownerConnector = { ...connector, leaseEvents: async (input: Parameters[0]) => { + const batch = await connector.leaseEvents(input); if (batch) captured.push(...batch.events); return batch; + }, acknowledgeEvents: async (input: Parameters[0]) => { + if (lostAck) { lostAck = false; throw new Error("fixture ACK request lost before broker settlement"); } + return connector.acknowledgeEvents(input); + } }; + const busyEndpointId = randomUUID(), busyConnectionId = randomUUID(); + await db.insert(toolConnections).values({ id: busyConnectionId, companyId, applicationId, name: "Retirement waiting for credential mutation", uid: busyEndpointId, + transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" }); + await db.insert(chatEndpoints).values({ id: busyEndpointId, companyId, connectionId: busyConnectionId, assignedAgentId: agentId, + publicId: "f".repeat(43), provider: "slack", status: "archived", setup: { runtimeGeneration: 1 } as InternalSetupState }); + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, busyEndpointId, actorId)); + const [busyEndpoint] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + const originalBusyIntent = structuredClone((busyEndpoint!.setup as InternalSetupState).slackCloudIntent!); + let acquired!: () => void; + const acquiredPromise = new Promise(resolve => { acquired = resolve; }); + const releasePromise = new Promise(resolve => { releaseEnrollmentLease = resolve; }); + heldEnrollmentLease = withLease(busyEndpoint!, async lease => { await lease.assertOwned(); acquired(); await releasePromise; }); + await acquiredPromise; + const eventOwner = githubConnectionEventService(db, { connector: ownerConnector }); + await expect(eventOwner.pollOnce({ forDelivery: true })).rejects.toThrow("ACK request lost"); + expect(captured).toHaveLength(1); const actualReceiptId = captured[0]!.id; deliveryIds.push(actualReceiptId); + const beforeRedelivery = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId)); + expect(beforeRedelivery).toHaveLength(1); + const [retainedBusy] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + const busyIntent = (retainedBusy!.setup as InternalSetupState).slackCloudIntent!; + expect(busyIntent).toEqual({ ...originalBusyIntent, attempts: 1, failureCode: "credential_lease_busy", nextAttemptAt: expect.any(String) }); + const originalRetryAt = Date.parse(busyIntent.nextAttemptAt!); + expect(Number.isFinite(originalRetryAt)).toBe(true); + expect(originalRetryAt).toBeGreaterThan(Date.now()); + expect(originalRetryAt).toBeLessThanOrEqual(Date.now() + 5_000); + expect(eventOwner.nextPollAt()).not.toBeNull(); + expect(eventOwner.nextPollAt()!).toBeGreaterThan(Date.now()); + expect(eventOwner.nextPollAt()!).toBeLessThanOrEqual(Date.now() + 5_000); + releaseEnrollmentLease!(); await heldEnrollmentLease; + + const expired = await pool.query("UPDATE cloud_harness.connector_webhook_delivery_targets SET lease_expires_at=$3 WHERE connector_instance_id=$1 AND delivery_id=$2 AND status='leased' RETURNING delivery_id", + [instanceId, actualReceiptId, new Date(Date.now() - 1)]); + expect(expired.rowCount).toBe(1); + expect((await eventOwner.pollOnce({ forDelivery: true })).processed).toBe(1); + expect(captured.map(value => value.id)).toEqual([actualReceiptId, actualReceiptId]); + const [beforeRetry] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + if (Date.now() < originalRetryAt) expect((beforeRetry!.setup as InternalSetupState).slackCloudIntent).toEqual(busyIntent); + await new Promise(resolve => setTimeout(resolve, Math.max(0, originalRetryAt - Date.now()))); + await eventOwner.pollOnce(); + const [settledBusy] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId)); + expect((settledBusy!.setup as InternalSetupState).slackCloudIntent).toBeUndefined(); + expect((settledBusy!.setup as InternalSetupState).slackCloudRetiredGeneration).toBe(1); + expect((settledBusy!.setup as InternalSetupState).slackCloudRetiredGeneration).toBe(originalBusyIntent.runtimeGeneration); + expect(captured.map(value => value.id)).toEqual([actualReceiptId, actualReceiptId]); + + expect(captured[1]!.createdAt).toBe(captured[0]!.createdAt); expect(captured[1]!.payload).toEqual(captured[0]!.payload); + expect(sdkCallbacks).toHaveLength(2); expect(sdkCallbacks[0]!.message.id).toBe(sdkCallbacks[1]!.message.id); + expect(sdkCallbacks[0]!.thread.id).toBe(sdkCallbacks[1]!.thread.id); + const afterRedelivery = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId)); + expect(afterRedelivery).toHaveLength(1); expect(afterRedelivery[0]!.providerEventId).toBe(beforeRedelivery[0]!.providerEventId); + expect(deferred.length).toBeGreaterThan(0); + await deferred.shift()!(); + await normalService.processQueuedDeliveries(); + const links = await db.select().from(chatMessageLinks).where(and(eq(chatMessageLinks.endpointId, endpointId), eq(chatMessageLinks.direction, "inbound"))); + expect(links).toHaveLength(1); expect(links[0]!.deliveryId).toBe(afterRedelivery[0]!.id); + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, task!.id)); + expect(comments.filter(value => value.id === links[0]!.commentId)).toHaveLength(1); + expect(comments.find(value => value.id === links[0]!.commentId)?.body).toContain("one durable reply"); + const wakeReceipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId)); + expect(wakeReceipts).toHaveLength(1); + await normalHeartbeat.drainActiveRunExecutions(); + const completedRuns = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(completedRuns).toHaveLength(1); + expect(completedRuns[0]).toMatchObject({ status: "succeeded", exitCode: 0, wakeupRequestId: wakeReceipts[0]!.id }); + expect(completedRuns[0]!.startedAt).not.toBeNull(); + expect(completedRuns[0]!.finishedAt).not.toBeNull(); + expect(completedRuns[0]!.stdoutExcerpt).toContain(markerText); + const marker = JSON.parse(await readFile(markerPath, "utf8")) as { pid: number; runId: string; marker: string }; + expect(marker).toMatchObject({ runId: completedRuns[0]!.id, marker: markerText }); + expect(marker.pid).not.toBe(process.pid); + expect(((await endpoint()).setup as InternalSetupState).slackCallbackSurfaces).toEqual(callbackEvidence); + const signingRef = refs.find(ref => ref.configPath === "credentials.signingSecret")!; + await vault.update(signingRef.secretId, { status: "disabled" }); + try { + await expect(dispatchSlackCloudEvent(db, captured[1]!)).rejects.toMatchObject({ + code: "signing_credential_unavailable", disposition: "attention", + }); + expect(sdkCallbacks).toHaveLength(2); + } finally { await vault.update(signingRef.secretId, { status: "active" }); } + + expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull(); + for (const transport of transports) await transport.assertNoErrors(); + await normalService.shutdown(); normalService = undefined; + let rejectAfterIntent = false; + const beforeRollbackRequests = enrollmentRequests; + const stopSignals = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { if (event === "intent") rejectAfterIntent = true; }); + normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example", + scheduleDeferredWork: work => { deferred.push(work); }, renewCredentialMutationLease: async () => !rejectAfterIntent }); + try { + await expect(normalService.configure(endpointId, { action: "pause" }, actorId)).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); + expect((await endpoint()).status).toBe("active"); + expect(((await endpoint()).setup as InternalSetupState).runtimeGeneration).toBe(2); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toBeUndefined(); + expect(enrollmentRequests).toBe(beforeRollbackRequests); + } finally { stopSignals(); await normalService.shutdown(); normalService = undefined; } + normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example", + scheduleDeferredWork: work => { deferred.push(work); } }); + await db.transaction(async tx => { + await tx.update(chatEndpoints).set({ setup: { ...((await endpoint()).setup as InternalSetupState), runtimeGeneration: 3 } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId)); + await stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId); + }); + loseEnrollmentResponse = true; + await withLease(await endpoint(), lease => produce(endpointId, actorId, lease)); + expect((await broker.slackEvents.current(instanceId, publicId)).runtimeGeneration).toBe(3); + expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: true, runtimeGeneration: 3, failureCode: "broker_unavailable" }); + await withLease(await endpoint(), async lease => { + let transactionChecks = 0; + await produce(endpointId, actorId, { assertOwned: async transaction => { + await lease.assertOwned(transaction); + if (transaction && ++transactionChecks === 2) throw new Error("fixture final enrollment guard lost"); + } }); + }); + expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2); + const casProducer = createSlackCloudEnrollment(db, { runtimeGeneration: setup => Number(setup.runtimeGeneration), resolveCredentials: async () => { + await db.update(chatEndpoints).set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', '4'::jsonb)` }).where(eq(chatEndpoints.id, endpointId)); + return { signingSecret }; + } }); + await expect(withLease(await endpoint(), lease => casProducer(endpointId, actorId, lease))).rejects.toThrow("changed"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2); + brokerUnavailable = true; + expect((await normalService.configure(endpointId, { action: "pause" }, actorId)).status).toBe("paused"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 5, attempts: 0 }); + await expect(eventOwner.pollOnce({ forDelivery: true })).rejects.toThrow("status=503"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ failureCode: "broker_unavailable", attempts: 1 }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", ""); + expect((await normalService.configure(endpointId, { action: "remove" }, actorId)).status).toBe("archived"); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 6, subject: actorId, attempts: 0 }); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 6, failureCode: "connector_configuration_unavailable", nextAttemptAt: null }); + expect((await db.select().from(toolConnections).where(eq(toolConnections.id, connectionId)))[0]!.credentialSecretRefs).toEqual([]); + expect((await db.select().from(chatSlackRegistrations).where(eq(chatSlackRegistrations.endpointId, endpointId)))[0]!.status).toBe("removed"); + await normalService.shutdown(); normalService = undefined; + brokerUnavailable = false; + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); + normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example", + scheduleDeferredWork: work => { deferred.push(work); } }); + const reconstructedOwner = githubConnectionEventService(db, { connector }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", `replacement-${instanceId}`); + await reconstructedOwner.pollOnce({ forDelivery: true }); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ instanceId, failureCode: "connector_configuration_unavailable", nextAttemptAt: null }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId); + await reconstructedOwner.pollOnce({ forDelivery: true }); + expect(reconstructedOwner.nextPollAt()).toBeNull(); + expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toBeUndefined(); + expect(((await endpoint()).setup as InternalSetupState).slackCloudRetiredGeneration).toBe(6); + expect(await broker.slackEvents.current(instanceId, publicId)).toBeUndefined(); + expect((await store.listWebhookBindings(instanceId)).find((value: { receivesEvents?: boolean }) => value.receivesEvents === false).providerMetadata.retirement.runtimeGeneration).toBe(6); + await expect(broker.slackEvents.enroll(await store.getInstance(instanceId), actorId, { companyId, endpointId, publicId, connectionId, + runtimeGeneration: 3, teamId: "TTEST", appId: "ATEST", signingSecret })).rejects.toThrow("GENERATION_CHANGED"); + + const pendingIds = [randomUUID(), randomUUID()]; + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", ""); + for (const [index, pendingId] of pendingIds.entries()) { + const pendingConnectionId = randomUUID(); + await db.insert(toolConnections).values({ id: pendingConnectionId, companyId, applicationId, name: "Retired Slack", uid: pendingId, + transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" }); + await db.insert(chatEndpoints).values({ id: pendingId, companyId, connectionId: pendingConnectionId, assignedAgentId: agentId, + publicId: (index === 0 ? "d" : "e").repeat(43), provider: "slack", status: "archived", providerAccountId: "TTEST", + setup: { runtimeGeneration: 1, slackCloudInstanceId: instanceId, slackCloudSubject: actorId } as InternalSetupState }); + await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, pendingId, actorId)); + } + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); + await reconstructedOwner.pollOnce({ forDelivery: true }); + const remaining = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.companyId, companyId), + sql`${chatEndpoints.setup}->'slackCloudIntent' is not null`)); + expect(remaining).toHaveLength(1); + expect(pendingIds).toContain(remaining[0]!.id); + expect(reconstructedOwner.nextPollAt()).not.toBeNull(); + expect(reconstructedOwner.nextPollAt()!).toBeLessThanOrEqual(Date.now()); + await reconstructedOwner.pollOnce({ forDelivery: true }); + expect(await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.companyId, companyId), + sql`${chatEndpoints.setup}->'slackCloudIntent' is not null`))).toHaveLength(0); + expect(reconstructedOwner.nextPollAt()).toBeNull(); + + } catch (error) { + console.error("Slack paired fixture body failure", error); + throw error; + } finally { + releaseEnrollmentLease?.(); await heldEnrollmentLease; + await normalHeartbeat?.drainActiveRunExecutions(); + await normalService?.shutdown(); + unregister?.(); await runtime?.shutdown(); + if (server.listening) { server.close(); await once(server, "close"); } + vi.unstubAllEnvs(); + try { + await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId)); + await db.delete(financeEvents).where(eq(financeEvents.companyId, companyId)); + await db.delete(costEvents).where(eq(costEvents.companyId, companyId)); + await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, companyId)); + await db.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, companyId)); + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, companyId)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + await db.delete(issueComments).where(eq(issueComments.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId)); + await db.delete(companySkills).where(eq(companySkills.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + await db.delete(authUsers).where(eq(authUsers.id, actorId)); + await pool.query("DELETE FROM cloud_harness.connector_instances WHERE id=$1", [instanceId]); + await pool.query("DELETE FROM cloud_harness.connector_webhook_deliveries WHERE id=ANY($1::text[])", [deliveryIds]); + } finally { await closeCheckpointDb(db, 1); await closeFileStorageDb(db, 1); await closeLivenessDb(db, 1); + await db.$client.end({ timeout: 1 }); await registry.close(); + if (secretDirectory) await rm(secretDirectory, { recursive: true, force: true }); + try { network.assertQuiet(); } finally { network.restore(); } + } + } +}, 60_000); diff --git a/server/src/services/chat-channels/slack/cloud-enrollment.test.ts b/server/src/services/chat-channels/slack/cloud-enrollment.test.ts new file mode 100644 index 00000000000..1b5b47d0ad3 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-enrollment.test.ts @@ -0,0 +1,214 @@ +import { HttpError } from "../../../errors.js"; +import { generateKeyPairSync } from "node:crypto"; +import { chatEndpoints, chatSlackRegistrations, type Db } from "@paperclipai/db"; +import { afterEach, expect, it, vi } from "vitest"; +import { PaperclipCloudConnectorError } from "../../paperclip-cloud-connector.js"; +import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent } from "./cloud-enrollment.js"; +import type { InternalSetupState } from "../runtime-types.js"; +import type { CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js"; + +afterEach(() => vi.unstubAllEnvs()); +function configureCloud() { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "https://cloud.example.test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", generateKeyPairSync("ed25519").privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", generateKeyPairSync("x25519").privateKey.export({ format: "jwk" }).d!); +} +function jsonValues(value: unknown): unknown[] { + if (typeof value === "string") { try { return [JSON.parse(value)]; } catch { return []; } } + if (!value || typeof value !== "object") return []; + const chunks = Reflect.get(value, "queryChunks"); + return Array.isArray(chunks) ? chunks.flatMap(jsonValues) : []; +} +function fixture() { + configureCloud(); + const endpoint = { id: "00000000-0000-4000-8000-000000000001", companyId: "company", connectionId: "connection", publicId: "a".repeat(43), + provider: "slack", status: "active", providerAccountId: "TTEST", setup: { runtimeGeneration: 1 } as InternalSetupState }; + const connection = { status: "active", enabled: true }; + let rejectWrite = false; + const update = vi.fn((_table: object) => ({ set: (values: { setup: unknown }) => ({ where: () => { + const write = () => { + if (rejectWrite) return []; + const priorIntent = endpoint.setup.slackCloudIntent; + for (const value of jsonValues(values.setup)) { + if (value && typeof value === "object" && "active" in value && "id" in value) { + endpoint.setup.slackCloudIntent = value as NonNullable; + + } + else if (value && typeof value === "object" && "bindingId" in value) { delete endpoint.setup.slackCloudIntent; delete endpoint.setup.slackCloudRetiredGeneration; endpoint.setup.slackCloudEvents = value as NonNullable; endpoint.setup.slackCloudInstanceId = priorIntent!.instanceId; endpoint.setup.slackCloudSubject = priorIntent!.subject; } + else if (typeof value === "number") { delete endpoint.setup.slackCloudIntent; endpoint.setup.slackCloudRetiredGeneration = value; endpoint.setup.slackCloudInstanceId = priorIntent!.instanceId; endpoint.setup.slackCloudSubject = priorIntent!.subject; } + } + return [{ id: endpoint.id }]; + }; + return { then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(write()).then(resolve), returning: async () => write() }; + } }) })); + const db = { update, select: (fields?: Record) => { + let table: object, joined = false; + const rows = () => table === chatSlackRegistrations ? [{ appId: "ATEST" }] : joined + ? [{ endpoint: structuredClone(endpoint), connection: structuredClone(connection) }] + : fields?.setup ? [{ setup: structuredClone(endpoint.setup) }] : table === chatEndpoints ? [structuredClone(endpoint)] : []; + const query = { from: (value: object) => { table = value; return query; }, innerJoin: () => { joined = true; return query; }, + where: () => query, limit: async () => rows(), then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(rows()).then(resolve) }; + return query; + }, transaction: async (work: (tx: unknown) => Promise) => { + const previous = structuredClone(endpoint.setup); + try { return await work(db); } catch (error) { endpoint.setup = previous; throw error; } + } } as unknown as Db; + const resolveCredentials = vi.fn(async () => ({ signingSecret: "synthetic" })); + const connector = { slackEventsAvailable: vi.fn(async () => true), slackEventEnrollment: vi.fn(async (input: { enrollment: Record }): Promise => { + const { active: _active, signingSecret: _secret, ...identity } = input.enrollment; + return { version: 1, instanceId: "instance", ...identity, ...(input.enrollment.active ? { bindingId: `slack-event:${"b".repeat(64)}` } : {}) }; + }) }; + const lease = { assertOwned: vi.fn(async () => {}) } as unknown as CredentialMutationLeaseGuard; + const run = createSlackCloudEnrollment(db, { resolveCredentials, runtimeGeneration: setup => Number(setup.runtimeGeneration), connector }); + const stage = () => db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as Db, endpoint.id, "original-authorized-user")); + return { db, endpoint, connection, update, connector, resolveCredentials, lease, stage, run, + rejectWrites: () => { rejectWrite = true; } }; +} + +it("stages nonsecret responsibility before network and clears only an exactly confirmed current intent", async () => { + const f = fixture(); await f.stage(); + expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled(); + expect(JSON.stringify(f.endpoint.setup.slackCloudIntent)).not.toContain("synthetic"); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); + expect(f.endpoint.setup.slackCloudEvents?.runtimeGeneration).toBe(1); + expect(f.connector.slackEventEnrollment).toHaveBeenCalledWith(expect.objectContaining({ subject: "original-authorized-user" })); +}); + +it.each(["paused", "archived"])("retains credential-free %s removal responsibility after local cleanup", async status => { + const f = fixture(); f.endpoint.status = status; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); expect(f.endpoint.setup.slackCloudIntent?.active).toBe(false); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.resolveCredentials).not.toHaveBeenCalled(); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(2); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); +}); + +it("outage preserves a bounded pending retry; current broker denial preserves visible attention debt", async () => { + const f = fixture(); await f.stage(); + f.connector.slackEventsAvailable.mockRejectedValueOnce(new PaperclipCloudConnectorError("offline", "CONNECTOR_UNAVAILABLE")); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ attempts: 1, failureCode: "broker_unavailable" }); + expect(Date.parse(f.endpoint.setup.slackCloudIntent!.nextAttemptAt!)).toBeGreaterThan(Date.now()); + f.connector.slackEventEnrollment.mockRejectedValueOnce(new PaperclipCloudConnectorError("denied", "CONNECTOR_REQUEST_FAILED", 403)); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ failureCode: "authority_required", nextAttemptAt: null }); +}); + +it("receipt identity mismatch and failed transaction guard cannot publish a live descriptor", async () => { + const f = fixture(); await f.stage(); + f.connector.slackEventEnrollment.mockResolvedValueOnce({ version: 1, runtimeGeneration: 999 }); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); expect(f.endpoint.setup.slackCloudIntent).toBeDefined(); + f.rejectWrites(); await expect(f.run(f.endpoint.id, undefined, f.lease)).rejects.toThrow("changed"); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); expect(f.endpoint.setup.slackCloudIntent).toBeDefined(); +}); + +it("an absent Cloud configuration creates no new intent or background retry", async () => { + const f = fixture(); for (const key of ["BASE_URL", "INSTANCE_ID", "SIGN_PRIVATE_KEY", "SEAL_PRIVATE_KEY", "ENVIRONMENT"]) vi.stubEnv(`PAPERCLIP_CLOUD_CONNECTOR_${key}`, ""); + await f.stage(); expect(f.update).not.toHaveBeenCalled(); expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); +}); + +it.each(["SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_DISABLED"])("closed broker reason %s distinguishes permanent authority from rollout retry", async reason => { + const f = fixture(); await f.stage(); + f.connector.slackEventEnrollment.mockRejectedValueOnce(new PaperclipCloudConnectorError("rejected", "REAUTHORIZATION_REQUIRED", 409, undefined, reason)); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); + expect(f.endpoint.setup.slackCloudIntent!.nextAttemptAt === null).toBe(reason === "SLACK_ENROLLMENT_GENERATION_CHANGED"); + expect(f.endpoint.setup.slackCloudIntent!.failureCode).toBe(reason === "SLACK_ENROLLMENT_DISABLED" ? "enrollment_disabled" : "authority_required"); +}); + +it("losing Cloud identity retains configured retirement responsibility as attention without an empty retry", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + f.endpoint.status = "archived"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + for (const key of ["BASE_URL", "INSTANCE_ID", "SIGN_PRIVATE_KEY", "SEAL_PRIVATE_KEY", "ENVIRONMENT"]) vi.stubEnv(`PAPERCLIP_CLOUD_CONNECTOR_${key}`, ""); + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ active: false, instanceId: "instance", runtimeGeneration: 2, + failureCode: "connector_configuration_unavailable", nextAttemptAt: null }); + expect(f.connector.slackEventEnrollment).toHaveBeenCalledOnce(); +}); + +it.each([404, 422])("unavailable signing credential %s becomes retained attention instead of broker retry", async status => { + const f = fixture(); await f.stage(); + f.resolveCredentials.mockRejectedValueOnce(new HttpError(status, "synthetic credential failure")); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ failureCode: "signing_credential_unavailable", nextAttemptAt: null }); + expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled(); + expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); +}); + +it("temporary signing-credential store failure keeps the bounded retry", async () => { + const f = fixture(); await f.stage(); + f.resolveCredentials.mockRejectedValueOnce(new Error("temporary database failure")); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudIntent?.failureCode).toBe("broker_unavailable"); + expect(Date.parse(f.endpoint.setup.slackCloudIntent!.nextAttemptAt!)).toBeGreaterThan(Date.now()); + expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled(); +}); + +it("only positive old retirement and an explicit new generation can stage a replacement instance", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + const oldDescriptor = structuredClone(f.endpoint.setup.slackCloudEvents); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "instance", subject: "original-authorized-user", + failureCode: "instance_replacement_requires_retirement", nextAttemptAt: null }); + expect(f.endpoint.setup.slackCloudEvents).toEqual(oldDescriptor); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance"); + f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 3; + await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(3); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 4; + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "replacement", runtimeGeneration: 4 }); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("instance"); + expect(f.endpoint.setup.slackCloudSubject).toBe("original-authorized-user"); + expect(f.endpoint.setup.slackCloudEvents).toEqual(oldDescriptor); + f.connector.slackEventEnrollment.mockImplementationOnce(async input => { + const { active: _active, signingSecret: _secret, ...identity } = input.enrollment; + return { version: 1, instanceId: "replacement", ...identity, bindingId: `slack-event:${"c".repeat(64)}` }; + }); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("replacement"); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBeUndefined(); + expect(f.endpoint.setup.slackCloudEvents?.runtimeGeneration).toBe(4); +}); + +it("a new config without an explicit actor cannot adopt even a retired old instance", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 3; + await stageSlackCloudEnrollmentIntent(f.db, f.endpoint.id, undefined); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "instance", subject: "original-authorized-user", + nextAttemptAt: null, failureCode: "instance_replacement_requires_retirement" }); +}); + +it("removal after an unconfirmed replacement targets its pending instance without new credentials", async () => { + const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2; + await f.stage(); await f.run(f.endpoint.id, undefined, f.lease); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement"); + f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 3; + await f.stage(); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("instance"); + f.endpoint.status = "archived"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 4; + await f.stage(); + expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ active: false, instanceId: "replacement", runtimeGeneration: 4 }); + f.resolveCredentials.mockClear(); + f.connector.slackEventEnrollment.mockImplementationOnce(async input => { + const { active: _active, ...identity } = input.enrollment; + return { version: 1, instanceId: "replacement", ...identity }; + }); + await f.run(f.endpoint.id, undefined, f.lease); + expect(f.resolveCredentials).not.toHaveBeenCalled(); + expect(f.endpoint.setup.slackCloudIntent).toBeUndefined(); + expect(f.endpoint.setup.slackCloudInstanceId).toBe("replacement"); + expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(4); +}); diff --git a/server/src/services/chat-channels/slack/cloud-enrollment.ts b/server/src/services/chat-channels/slack/cloud-enrollment.ts new file mode 100644 index 00000000000..be8e33394e3 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-enrollment.ts @@ -0,0 +1,218 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, or, sql } from "drizzle-orm"; +import { chatEndpoints, chatSlackRegistrations, toolConnections, type Db } from "@paperclipai/db"; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv, PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../../paperclip-cloud-connector.js"; +import type { CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js"; +import type { createChatCredentials } from "../credentials.js"; +import type { InternalSetupState } from "../runtime-types.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../delivery-work-notifications.js"; +import { logger } from "../../../middleware/logger.js"; +import { HttpError } from "../../../errors.js"; + +import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js"; +import { SlackCloudReceiptError } from "./cloud-event.js"; + +type Intent = NonNullable; +class SlackEnrollmentFailure extends Error { + constructor(readonly code: string) { super(code); } +} + +type Owner = (endpointId: string) => Promise; +const owners = new WeakMap(); + +export function registerSlackCloudEnrollmentOwner(db: Db, owner: Owner) { + owners.set(db, owner); + return () => { if (owners.get(db) === owner) owners.delete(db); }; +} + +function slackCloudEnrollmentIsActive(endpoint: { status: string }, connection: { status: string; enabled: boolean }) { + return ["active", "verifying", "attention"].includes(endpoint.status) && connection.status === "active" && connection.enabled; +} + +export async function stageSlackCloudEnrollmentIntent(db: Db, endpointId: string, actorUserId: string | null | undefined) { + let config: ReturnType; + try { config = paperclipCloudConnectorConfigFromEnv(); } catch { config = null; } + const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId))) + .where(eq(chatEndpoints.id, endpointId)).limit(1); + if (!current || current.endpoint.provider !== "slack") return; + const endpoint = current.endpoint, setup = endpoint.setup as InternalSetupState; + const generation = Number(setup.runtimeGeneration ?? 0); + const active = slackCloudEnrollmentIsActive(endpoint, current.connection); + const custodyInstance = setup.slackCloudIntent?.instanceId ?? setup.slackCloudInstanceId; + const custodySubject = setup.slackCloudIntent?.subject ?? setup.slackCloudSubject; + const oldRetired = !setup.slackCloudIntent && setup.slackCloudInstanceId + && Number.isSafeInteger(setup.slackCloudRetiredGeneration) && Number(setup.slackCloudRetiredGeneration) >= 1 + && Number(setup.slackCloudRetiredGeneration) >= (setup.slackCloudEvents?.runtimeGeneration ?? 1); + const changingInstance = !!custodyInstance && !!config && custodyInstance !== config.instanceId; + const mayReplace = active && changingInstance && oldRetired && !!actorUserId + && generation > Number(setup.slackCloudRetiredGeneration); + const blocked = changingInstance && !mayReplace; + const instanceId = mayReplace ? config!.instanceId : custodyInstance ?? config?.instanceId; + const subject = blocked || !active ? custodySubject ?? actorUserId : actorUserId ?? custodySubject; + if (!instanceId || !subject || !Number.isSafeInteger(generation) || generation < 1) return; + if (setup.slackCloudIntent?.runtimeGeneration === generation && setup.slackCloudIntent.active === active + && setup.slackCloudIntent.instanceId === instanceId && setup.slackCloudIntent.subject === subject) return; + if (!blocked && setup.slackCloudInstanceId === instanceId && setup.slackCloudSubject === subject + && (active ? setup.slackCloudEvents?.runtimeGeneration === generation : setup.slackCloudRetiredGeneration === generation)) return; + const [registration] = active ? await db.select({ appId: chatSlackRegistrations.appId }).from(chatSlackRegistrations) + .where(and(eq(chatSlackRegistrations.endpointId, endpointId), eq(chatSlackRegistrations.companyId, endpoint.companyId))).limit(1) : []; + const appId = registration?.appId ?? setup.slackCloudEvents?.appId; + if (active && (!appId || !endpoint.providerAccountId)) return; + const intent: Intent = { version: 1, id: randomUUID(), subject, instanceId, companyId: endpoint.companyId, endpointId, + publicId: endpoint.publicId, connectionId: endpoint.connectionId, runtimeGeneration: generation, active, + ...(active ? { appId: appId!, teamId: endpoint.providerAccountId! } : {}), attempts: 0, nextAttemptAt: config && !blocked ? new Date().toISOString() : null, + ...(blocked ? { failureCode: "instance_replacement_requires_retirement" } : !config ? { failureCode: "connector_configuration_unavailable" } : {}) }; + await notifyDeliveryWork(db, DELIVERY_QUEUES.connectionEvents); + await db.update(chatEndpoints).set({ + lastError: config && !blocked ? sql`${chatEndpoints.lastError}` : "Slack Cloud event registration requires attention", + setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(intent)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, endpoint.companyId))); +} + +export async function reconcilePendingSlackCloudEnrollments(db: Db, now: Date): Promise { + const config = paperclipCloudConnectorConfigFromEnv(); + if (!config) return null; + const recoverable = and( + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt' is null`, + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'failureCode' = 'connector_configuration_unavailable'`, + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'instanceId' = ${config.instanceId}`, + ); + const scheduled = and(eq(chatEndpoints.provider, "slack"), or( + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt' is not null`, recoverable)); + const order = sql`coalesce(${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt', '')`; + const pending = await db.select().from(chatEndpoints).where(scheduled).orderBy(order).limit(1); + const owner = owners.get(db); + const deadline = (intent: Intent | undefined) => { + if (!intent) return Number.POSITIVE_INFINITY; + if (intent.nextAttemptAt === null) return intent.failureCode === "connector_configuration_unavailable" && intent.instanceId === config.instanceId + ? now.getTime() : Number.POSITIVE_INFINITY; + const parsed = Date.parse(intent.nextAttemptAt); + return Number.isFinite(parsed) ? parsed : now.getTime(); + }; + let next = Number.POSITIVE_INFINITY; + for (const endpoint of pending) { + const intent = (endpoint.setup as InternalSetupState).slackCloudIntent!; + let due = deadline(intent); + if (!Number.isFinite(due)) continue; + if (due <= now.getTime()) { + if (owner) { + try { await owner(endpoint.id); } + catch (error) { + if (error instanceof HttpError && error.status === 409 && error.details && typeof error.details === "object" + && "code" in error.details && error.details.code === "chat_endpoint_credentials_busy") { + try { + if (intent.companyId === endpoint.companyId && intent.endpointId === endpoint.id + && intent.publicId === endpoint.publicId && intent.connectionId === endpoint.connectionId + && intent.runtimeGeneration === Number((endpoint.setup as InternalSetupState).runtimeGeneration ?? 0)) { + await db.transaction(async tx => { + const sameIntent = and(eq(chatEndpoints.id, endpoint.id), eq(chatEndpoints.companyId, intent.companyId), + eq(chatEndpoints.provider, "slack"), eq(chatEndpoints.publicId, intent.publicId), eq(chatEndpoints.connectionId, intent.connectionId), + sql`${chatEndpoints.setup}->'runtimeGeneration' = ${JSON.stringify(intent.runtimeGeneration)}::jsonb`, + sql`${chatEndpoints.setup}->'slackCloudIntent' = ${JSON.stringify(intent)}::jsonb`); + const [owned] = await tx.select({ id: chatEndpoints.id }).from(chatEndpoints).where(sameIntent).for("update").limit(1); + if (!owned) return; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + const retry = { ...intent, attempts: Math.min(intent.attempts + 1, 1_000_000), failureCode: "credential_lease_busy", + nextAttemptAt: new Date(Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(intent.attempts, 6))).toISOString() }; + const updated = await tx.update(chatEndpoints).set({ + setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(retry)}::jsonb)`, + }).where(sameIntent).returning({ id: chatEndpoints.id }); + if (updated.length !== 1) throw new Error("Slack enrollment busy retry custody changed"); + }); + } + } catch (retryError) { throw new AggregateError([error, retryError], "Slack enrollment busy retry persistence failed"); } + } + throw error; + } + } + else logger.warn({ endpointId: endpoint.id, reason: "slack_enrollment_owner_not_ready" }, "Slack enrollment responsibility retained"); + const [updated] = await db.select({ setup: chatEndpoints.setup }).from(chatEndpoints).where(eq(chatEndpoints.id, endpoint.id)).limit(1); + const retained = (updated?.setup as InternalSetupState | undefined)?.slackCloudIntent; + due = deadline(retained); + if (!owner) due = now.getTime() + 5_000; + } + next = Math.min(next, due); + } + if (pending.length > 0) { + const [earliest] = await db.select({ setup: chatEndpoints.setup }).from(chatEndpoints).where(scheduled).orderBy(order).limit(1); + next = Math.min(next, deadline((earliest?.setup as InternalSetupState | undefined)?.slackCloudIntent)); + } + return Number.isFinite(next) ? next : null; +} + +export function createSlackCloudEnrollment(db: Db, options: { + resolveCredentials: ReturnType["resolveCredentials"]; + runtimeGeneration(setup: InternalSetupState): number; + connector?: Pick; +}) { + return async (endpointId: string, _actorUserId: string | null | undefined, lease: CredentialMutationLeaseGuard) => { + const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId))) + .where(eq(chatEndpoints.id, endpointId)).limit(1); + if (!current || current.endpoint.provider !== "slack") return; + const endpoint = current.endpoint, setup = endpoint.setup as InternalSetupState, intent = setup.slackCloudIntent; + const config = paperclipCloudConnectorConfigFromEnv(); + if (!config || !intent) return; + const identity = { companyId: intent.companyId, endpointId: intent.endpointId, publicId: intent.publicId, + connectionId: intent.connectionId, runtimeGeneration: intent.runtimeGeneration }; + const active = slackCloudEnrollmentIsActive(endpoint, current.connection); + if (identity.companyId !== endpoint.companyId || identity.endpointId !== endpoint.id || identity.publicId !== endpoint.publicId + || identity.connectionId !== endpoint.connectionId || identity.runtimeGeneration !== options.runtimeGeneration(setup) || intent.active !== active) { + await db.transaction(async tx => { + await lease.assertOwned(tx); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + await tx.update(chatEndpoints).set({ + setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify({ ...intent, nextAttemptAt: null, failureCode: "intent_authority_changed" })}::jsonb)`, + lastError: "Slack Cloud event registration requires attention", + }).where(and(eq(chatEndpoints.id, endpoint.id), eq(chatEndpoints.companyId, endpoint.companyId), sql`${chatEndpoints.setup}->'slackCloudIntent'->>'id'=${intent.id}`)); + await lease.assertOwned(tx); + }); + logger.warn({ endpointId, reason: "intent_authority_changed" }, "Slack enrollment responsibility retained"); + return; + } + const connector = options.connector ?? createPaperclipCloudConnector({ config }); + const commit = async (change: { receipt?: Record; failure?: string; attention?: boolean; retryAt?: number }) => db.transaction(async tx => { + await lease.assertOwned(tx); + const next = change.failure ? { ...intent, attempts: Math.min(intent.attempts + 1, 1_000_000), failureCode: change.failure, + nextAttemptAt: change.attention ? null : new Date(Math.max(Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(intent.attempts, 6)), change.retryAt ?? 0)).toISOString() } : null; + const field = intent.active ? "slackCloudEvents" : "slackCloudRetiredGeneration"; + const enrolled = intent.active ? { version: 1, bindingId: change.receipt?.bindingId, runtimeGeneration: intent.runtimeGeneration, teamId: intent.teamId, appId: intent.appId } : intent.runtimeGeneration; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + const updated = await tx.update(chatEndpoints).set({ + lastError: change.attention ? "Slack Cloud event registration requires attention" : change.receipt + ? sql`case when ${chatEndpoints.lastError} = 'Slack Cloud event registration requires attention' then null else ${chatEndpoints.lastError} end` : sql`${chatEndpoints.lastError}`, + setup: next + ? sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(next)}::jsonb)` + : sql`jsonb_set(jsonb_set(jsonb_set(${chatEndpoints.setup} - 'slackCloudIntent' - 'slackCloudRetiredGeneration', ARRAY[${field}], ${JSON.stringify(enrolled)}::jsonb), '{slackCloudInstanceId}', ${JSON.stringify(intent.instanceId)}::jsonb), '{slackCloudSubject}', ${JSON.stringify(intent.subject)}::jsonb)` }) + .where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, endpoint.companyId), + sql`${chatEndpoints.setup}->'slackCloudIntent'->>'id'=${intent.id}`, + sql`coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::integer,0)=${intent.runtimeGeneration}`)).returning({ id: chatEndpoints.id }); + if (updated.length !== 1) throw new Error("Slack configuration changed before event enrollment committed"); + await lease.assertOwned(tx); + }); + try { + await lease.assertOwned(); + if (config.instanceId !== intent.instanceId) { await commit({ failure: "instance_authority_changed", attention: true }); return; } + if (!await connector.slackEventsAvailable()) { await commit({ failure: "unsupported_capability" }); return; } + const signingSecret = intent.active ? await resolveSlackCloudSigningSecret(options.resolveCredentials, endpoint, db, ["signingSecret"]) : undefined; + const receipt = await connector.slackEventEnrollment({ subject: intent.subject, companyId: intent.companyId, + enrollment: { active: intent.active, ...identity, ...(intent.active ? { teamId: intent.teamId, appId: intent.appId, signingSecret: signingSecret! } : {}) } }); + if (!receipt || typeof receipt !== "object" || Array.isArray(receipt)) throw new SlackEnrollmentFailure("receipt_unconfirmed"); + const value = receipt as Record; + if (value.version !== 1 || value.instanceId !== intent.instanceId || Object.entries(identity).some(([key, expected]) => value[key] !== expected) + || (intent.active && (value.teamId !== intent.teamId || value.appId !== intent.appId || typeof value.bindingId !== "string" || !/^slack-event:[a-f0-9]{64}$/.test(value.bindingId)))) + throw new SlackEnrollmentFailure("receipt_identity_changed"); + await commit({ receipt: value }); + } catch (error) { + const conflict = error instanceof PaperclipCloudConnectorError && error.status === 409 + && ["SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_IDENTITY_CHANGED", "SLACK_ENROLLMENT_CHANGED", "SLACK_ENROLLMENT_UNSETTLED"].includes(error.brokerReason ?? ""); + const credentialAttention = error instanceof SlackCloudReceiptError && error.code === "signing_credential_unavailable" && error.disposition === "attention"; + const attention = credentialAttention || error instanceof SlackEnrollmentFailure || conflict || (error instanceof PaperclipCloudConnectorError && [400, 401, 403].includes(error.status ?? 0)); + const code = credentialAttention ? "signing_credential_unavailable" : error instanceof SlackEnrollmentFailure ? error.code : attention ? "authority_required" + : error instanceof PaperclipCloudConnectorError && error.brokerReason === "SLACK_ENROLLMENT_DISABLED" ? "enrollment_disabled" : "broker_unavailable"; + await commit({ failure: code, attention, retryAt: error instanceof PaperclipCloudConnectorError ? error.retryAt : undefined }); + logger.warn({ endpointId, reason: code }, "Slack enrollment responsibility retained"); + } + }; +} diff --git a/server/src/services/chat-channels/slack/cloud-event.test.ts b/server/src/services/chat-channels/slack/cloud-event.test.ts new file mode 100644 index 00000000000..ea9b4a234c5 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-event.test.ts @@ -0,0 +1,110 @@ +import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "../../../__tests__/helpers/slack-cloud/transport.js"; +import { createHmac, createHash } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { verifySlackSignature } from "@chat-adapter/slack/webhook"; +import { createChatSdkEndpointRuntime } from "../../chat-sdk-runtime.js"; +import type { Db } from "@paperclipai/db"; +import type { EndpointRow } from "../runtime-types.js"; +import { createSlackWebhookCallbacks } from "./webhook-callbacks.js"; +import { receiveSlackCloudEvent, slackCloudAcceptedAt, type CurrentSlackCloudBinding, type SlackCloudEvent } from "./cloud-event.js"; + +const acceptedAt = Date.parse("2099-01-01T00:00:00Z"); +function fixture() { + const binding: CurrentSlackCloudBinding = { companyId: "company", endpointId: "endpoint", publicId: "a".repeat(43), + connectionId: "connection", runtimeGeneration: 1, teamId: "TTEST", appId: "ATEST", bindingId: `slack-event:${"b".repeat(64)}`, + signingSecret: "0123456789abcdef0123456789abcdef" }; + const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvFIRST", team_id: "TTEST", api_app_id: "ATEST", event: { type: "app_mention", text: "<@UBOT> hello", user: "UOPERATOR", channel: "CTEST", ts: "1788.500", event_ts: "1788.500" } })); + const timestamp = String(acceptedAt / 1000); + const signature = `v0=${createHmac("sha256", binding.signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}`; + const { signingSecret: _secret, ...identity } = binding; + const event: SlackCloudEvent = { id: `slack_${"b".repeat(64)}_${createHash("sha256").update("EvFIRST").digest("hex")}`, + provider: "slack", event: "event_callback", installationId: binding.bindingId, bindingIds: [binding.bindingId], + createdAt: new Date(acceptedAt).toISOString(), payload: { slackEvent: { version: 1, ...identity, + rawBody: rawBody.toString("base64"), timestamp, signature, eventId: "EvFIRST", acceptedAt: new Date(acceptedAt).toISOString() } } }; + return { binding, event, rawBody, timestamp, signature }; +} + +describe("authenticated Slack Cloud receipt consumer", () => { + it("crosses the real pinned Chat runtime after cold-start delay and preserves ordinary external freshness rejection", async () => { + const network = guardSlackFixtureNetwork(); + const f = fixture(), onMessage = vi.fn(async () => {}); + const runtime = createChatSdkEndpointRuntime({ companyId: f.binding.companyId, endpointId: f.binding.endpointId, + logger: "silent", callbacks: { onMessage }, persistence: { read: async () => null, + compareAndSet: async () => true, deleteIfVersion: async () => true }, + providerConfig: { provider: "slack", userName: "paperclip-agent", + credentials: { apiUrl: "http://127.0.0.1:1/api/", botToken: "xoxb-synthetic", botUserId: "UBOT", signingSecret: f.binding.signingSecret } } }); + try { + await runtime.initialize(); + const transport = stubSlackFixtureTransport(runtime.getProviderAdapter()); + await receiveSlackCloudEvent(f.event, f.binding, request => runtime.handleWebhook(request), acceptedAt + 600_000); + expect(onMessage).toHaveBeenCalledOnce(); + expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ message: expect.objectContaining({ author: expect.objectContaining({ userId: "UOPERATOR", userName: "Operator" }) }) })); + expect(transport).toHaveBeenCalled(); + await transport.assertNoErrors(); + expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ provider: "slack", endpointId: f.binding.endpointId })); + const response = await runtime.handleWebhook(new Request("https://external.invalid", { method: "POST", + headers: { "content-type": "application/json", "x-slack-request-timestamp": f.timestamp, "x-slack-signature": f.signature }, + body: new Uint8Array(f.rawBody) })); + expect(response.status).toBe(401); + expect(onMessage).toHaveBeenCalledOnce(); + } finally { + await runtime.shutdown(); + try { network.assertQuiet(); } finally { network.restore(); } + } + }); + it("replays original evidence after a cold-start delay without refreshing its timestamp", async () => { + const f = fixture(); + let delivered: Request | undefined; + await receiveSlackCloudEvent(f.event, f.binding, async request => { + delivered = request; + expect(slackCloudAcceptedAt(request)).toBe(acceptedAt); + expect(request.headers.get("x-slack-request-timestamp")).toBe(f.timestamp); + expect(request.headers.get("x-slack-signature")).toBe(f.signature); + const body = await request.text(); + expect(body).toBe(f.rawBody.toString()); + + await verifySlackSignature(body, request.headers, { signingSecret: f.binding.signingSecret, + now: () => slackCloudAcceptedAt(request) ?? acceptedAt + 600_000 }); + await expect(verifySlackSignature(body, request.headers, { signingSecret: f.binding.signingSecret, + now: () => acceptedAt + 600_000 })).rejects.toThrow("too old"); + return new Response("durably accepted"); + }, acceptedAt + 600_000); + expect(slackCloudAcceptedAt(delivered!)).toBeUndefined(); + expect(slackCloudAcceptedAt(new Request("https://external.invalid"))).toBeUndefined(); + }); + + it("ignores removed or superseded generations without invoking product admission", async () => { + const f = fixture(), handler = vi.fn(); + await receiveSlackCloudEvent(f.event, null, handler, acceptedAt); + await receiveSlackCloudEvent(f.event, { ...f.binding, runtimeGeneration: 2 }, handler, acceptedAt); + expect(handler).not.toHaveBeenCalled(); + }); + + it("retains current work on signature/body/receipt-time failure and on failed durable admission", async () => { + const f = fixture(), handler = vi.fn(async () => new Response(null, { status: 503 })); + await expect(receiveSlackCloudEvent(f.event, f.binding, handler, acceptedAt)).rejects.toThrow("callback_unaccepted"); + await expect(receiveSlackCloudEvent(f.event, { ...f.binding, signingSecret: "rotated" }, handler, acceptedAt)).rejects.toThrow("original_authentication_failed"); + await expect(receiveSlackCloudEvent({ ...f.event, createdAt: new Date(acceptedAt + 600_000).toISOString() }, f.binding, handler, acceptedAt)).rejects.toThrow("invalid_acceptance_time"); + await expect(receiveSlackCloudEvent(f.event, f.binding, handler, acceptedAt + 31 * 24 * 60 * 60_000)).rejects.toThrow("invalid_acceptance_time"); + expect(handler).toHaveBeenCalledTimes(1); + }); +}); + +it("private replay preserves canonical Slack callback evidence while ordinary external observation remains active", async () => { + const f = fixture(); + const transaction = vi.fn(async () => { throw new Error("external callback observation"); }); + const db = { transaction } as unknown as Db; + const callbacks = createSlackWebhookCallbacks(db, { resolveCredentials: async () => ({}), runtimeCallbackEndpoint: async () => null, + getWebhookPublicBaseUrl: () => "https://canonical.example", scheduleMessageProcessing: () => {}, slackRegistration: { notifyVerified: async () => {} } }); + const endpoint = { setup: { slackCallbackSurfaces: { events: { url: "https://canonical.example/slack", observedAt: "2099-01-01T00:00:00Z" } } } } as unknown as EndpointRow; + const before = structuredClone(endpoint.setup); + const context = { credentialFingerprint: "fixture", generation: 1, localEpoch: 1, version: "fixture" }; + await receiveSlackCloudEvent(f.event, f.binding, async request => { + await callbacks.recordSlackWebhookCallback(endpoint, request, request.clone(), context); + return new Response(null, { status: 200 }); + }, acceptedAt); + expect(endpoint.setup).toEqual(before); expect(transaction).not.toHaveBeenCalled(); + const external = new Request("https://canonical.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: new Uint8Array(f.rawBody) }); + await expect(callbacks.recordSlackWebhookCallback(endpoint, external, external.clone(), context)).rejects.toThrow("external callback observation"); + expect(transaction).toHaveBeenCalledOnce(); +}); diff --git a/server/src/services/chat-channels/slack/cloud-event.ts b/server/src/services/chat-channels/slack/cloud-event.ts new file mode 100644 index 00000000000..b589de638bc --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-event.ts @@ -0,0 +1,77 @@ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; + +const acceptedTimes = new WeakMap(); +export const slackCloudAcceptedAt = (request: Request): number | undefined => acceptedTimes.get(request); + +export class SlackCloudReceiptError extends Error { + constructor(readonly code: string, readonly disposition: "invalid" | "attention" | "retry") { super(code); } +} + +export interface SlackCloudEvent { + id: string; + provider: string; + event: string; + installationId: string | null; + bindingIds: string[]; + createdAt: string; + payload: Record; +} + +export interface CurrentSlackCloudBinding { + companyId: string; + endpointId: string; + publicId: string; + connectionId: string; + runtimeGeneration: number; + teamId: string; + appId: string; + bindingId: string; + signingSecret: string; +} + +export async function receiveSlackCloudEvent(event: SlackCloudEvent, current: CurrentSlackCloudBinding | null, + handleWebhook: (request: Request) => Promise, now = Date.now()): Promise { + if (!current) return; + const value = event.payload.slackEvent; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const packet = value as Record; + if (event.provider !== "slack" || event.event !== "event_callback" || packet.version !== 1) + throw new SlackCloudReceiptError("unsupported_receipt", "attention"); + if (packet.bindingId !== current.bindingId || packet.companyId !== current.companyId + || packet.endpointId !== current.endpointId || packet.publicId !== current.publicId + || packet.connectionId !== current.connectionId || packet.runtimeGeneration !== current.runtimeGeneration + || packet.teamId !== current.teamId || packet.appId !== current.appId) return; + if (event.installationId !== current.bindingId || !event.bindingIds.includes(current.bindingId) + || typeof packet.rawBody !== "string" || packet.rawBody.length > 1_398_104 + || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(packet.rawBody) + || typeof packet.timestamp !== "string" || !/^[0-9]{1,16}$/.test(packet.timestamp) + || typeof packet.signature !== "string" || !/^v0=[a-f0-9]{64}$/.test(packet.signature)) + throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const acceptedAt = Date.parse(event.createdAt); + + if (packet.acceptedAt !== event.createdAt || !Number.isFinite(acceptedAt) || acceptedAt > now + 300_000 || now - acceptedAt >= 30 * 24 * 60 * 60_000 + || Math.abs(Math.floor(acceptedAt / 1000) - Number(packet.timestamp)) > 300) + throw new SlackCloudReceiptError("invalid_acceptance_time", "attention"); + const body = Buffer.from(packet.rawBody, "base64"); + if (body.byteLength > 1_048_576) throw new SlackCloudReceiptError("receipt_too_large", "invalid"); + const signature = createHmac("sha256", current.signingSecret).update(`v0:${packet.timestamp}:`).update(body).digest(); + if (!timingSafeEqual(signature, Buffer.from(packet.signature.slice(3), "hex"))) + throw new SlackCloudReceiptError("original_authentication_failed", "attention"); + let payload: unknown; + try { payload = JSON.parse(body.toString("utf8")); } catch { throw new SlackCloudReceiptError("invalid_event", "invalid"); } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) throw new SlackCloudReceiptError("invalid_event", "invalid"); + const envelope = payload as Record; + if (envelope.type !== "event_callback" || envelope.team_id !== current.teamId || envelope.api_app_id !== current.appId + || typeof envelope.event_id !== "string" || !/^[A-Za-z0-9_-]{1,200}$/.test(envelope.event_id) || envelope.event_id !== packet.eventId + || event.id !== `slack_${current.bindingId.slice("slack-event:".length)}_${createHash("sha256").update(envelope.event_id).digest("hex")}`) + throw new SlackCloudReceiptError("event_identity_changed", "invalid"); + const request = new Request("https://paperclip.invalid/slack-cloud-ingress", { method: "POST", + headers: { "content-type": "application/json", "x-slack-request-timestamp": packet.timestamp, + "x-slack-signature": packet.signature }, body: new Uint8Array(body) }); + acceptedTimes.set(request, acceptedAt); + try { + const response = await handleWebhook(request); + await response.body?.cancel(); + if (!response.ok) throw new SlackCloudReceiptError("callback_unaccepted", "retry"); + } finally { acceptedTimes.delete(request); } +} diff --git a/server/src/services/chat-channels/slack/cloud-ingress.test.ts b/server/src/services/chat-channels/slack/cloud-ingress.test.ts new file mode 100644 index 00000000000..f0688ba5bc8 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-ingress.test.ts @@ -0,0 +1,47 @@ +import { HttpError } from "../../../errors.js"; +import type { Db } from "@paperclipai/db"; +import { expect, it, vi } from "vitest"; +import { createSlackCloudEventIngress, dispatchSlackCloudEvent, registerSlackCloudEventIngress } from "./cloud-ingress.js"; +import type { SlackCloudEvent } from "./cloud-event.js"; + +it("missing or failed runtime owner retains the leased event; retired owner cleanup cannot remove its replacement", async () => { + const db = {} as Db; + const event: SlackCloudEvent = { id: "receipt", provider: "slack", event: "event_callback", installationId: "binding", + bindingIds: ["binding"], createdAt: "2099-01-01T00:00:00Z", payload: {} }; + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready"); + const stopped = vi.fn(async () => { throw new Error("admission stopped"); }); + const unregisterOld = registerSlackCloudEventIngress(db, stopped); + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("admission stopped"); + const replacement = vi.fn(async () => {}); + const unregisterNew = registerSlackCloudEventIngress(db, replacement); + unregisterOld(); + await dispatchSlackCloudEvent(db, event); + expect(replacement).toHaveBeenCalledWith(event); + unregisterNew(); + await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready"); +}); + +it.each([404, 422])("current ingress keeps credential failure %s as unacknowledged attention before callback admission", async status => { + const endpoint = { id: "00000000-0000-4000-8000-000000000001", companyId: "company", publicId: "public", connectionId: "connection", + provider: "slack", status: "active", providerAccountId: "TTEST", setup: { slackCloudEvents: { + version: 1, bindingId: "binding", runtimeGeneration: 1, teamId: "TTEST", appId: "ATEST" } } }; + const db = { select: () => { + let joined = false; + const query = { from: () => query, where: () => query, innerJoin: () => { joined = true; return query; }, + limit: async () => joined ? [{ endpoint, connection: { enabled: true, status: "active" } }] : [endpoint] }; + return query; + } } as unknown as Db; + const handleWebhook = vi.fn(async () => new Response()); + const withCredentialMutationLease: Parameters[1]["withCredentialMutationLease"] = + async (_endpoint, work) => work({ assertOwned: async () => {} } as Parameters[0]); + const unregister = createSlackCloudEventIngress(db, { withCredentialMutationLease, runtimeGeneration: () => 1, handleWebhook, + resolveCredentials: async () => { throw new HttpError(status, "synthetic credential failure"); } }); + try { + await expect(dispatchSlackCloudEvent(db, { id: "receipt", provider: "slack", event: "event_callback", installationId: "binding", + bindingIds: ["binding"], createdAt: new Date().toISOString(), payload: { slackEvent: { + endpointId: endpoint.id, companyId: endpoint.companyId, publicId: endpoint.publicId, + connectionId: endpoint.connectionId, runtimeGeneration: 1 } } })) + .rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" }); + expect(handleWebhook).not.toHaveBeenCalled(); + } finally { unregister(); } +}); diff --git a/server/src/services/chat-channels/slack/cloud-ingress.ts b/server/src/services/chat-channels/slack/cloud-ingress.ts new file mode 100644 index 00000000000..a4368552abc --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-ingress.ts @@ -0,0 +1,63 @@ +import type { Db } from "@paperclipai/db"; +import { chatEndpoints, toolConnections } from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import type { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; +import type { createChatCredentials } from "../credentials.js"; +import type { EndpointRow, InternalSetupState } from "../runtime-types.js"; +import { SlackCloudReceiptError, receiveSlackCloudEvent, type CurrentSlackCloudBinding, type SlackCloudEvent } from "./cloud-event.js"; + +import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js"; + +type Handler = (event: SlackCloudEvent) => Promise; +const handlers = new WeakMap(); + +export function registerSlackCloudEventIngress(db: Db, handler: Handler) { + handlers.set(db, handler); + return () => { if (handlers.get(db) === handler) handlers.delete(db); }; +} + +export async function dispatchSlackCloudEvent(db: Db, event: SlackCloudEvent): Promise { + const handler = handlers.get(db); + if (!handler) throw new Error("Slack Cloud ingress is not ready"); + await handler(event); +} + +export function createSlackCloudEventIngress(db: Db, options: { + withCredentialMutationLease: ReturnType; + resolveCredentials: ReturnType["resolveCredentials"]; + runtimeGeneration(endpoint: EndpointRow): number; + handleWebhook(publicId: string, provider: "slack", request: Request, expectedGeneration: number): Promise; +}) { + return registerSlackCloudEventIngress(db, async event => { + const value = event.payload.slackEvent; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const packet = value as Record; + if (typeof packet.endpointId !== "string" || !/^[a-f0-9-]{36}$/i.test(packet.endpointId)) + throw new SlackCloudReceiptError("invalid_receipt", "invalid"); + const [initial] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, packet.endpointId)).limit(1); + if (!initial || initial.provider !== "slack" || ["archived", "paused", "revoked"].includes(initial.status)) return; + const binding = await options.withCredentialMutationLease(initial, async (lease): Promise => { + const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId))) + .where(eq(chatEndpoints.id, initial.id)).limit(1); + if (!current || current.endpoint.provider !== "slack" || ["archived", "paused", "revoked"].includes(current.endpoint.status) + || current.connection.status !== "active" || !current.connection.enabled) return null; + const endpoint = current.endpoint; + if (packet.companyId !== endpoint.companyId || packet.publicId !== endpoint.publicId + || packet.connectionId !== endpoint.connectionId || packet.runtimeGeneration !== options.runtimeGeneration(endpoint)) return null; + const enrolled = (endpoint.setup as InternalSetupState).slackCloudEvents; + + if (!enrolled) throw new Error("Slack Cloud enrollment is not locally committed"); + if (enrolled.version !== 1 || enrolled.runtimeGeneration !== options.runtimeGeneration(endpoint) + || enrolled.teamId !== endpoint.providerAccountId) throw new Error("Slack Cloud enrollment is unsettled"); + const signingSecret = await resolveSlackCloudSigningSecret(options.resolveCredentials, endpoint, db, ["signingSecret"]); + await lease.assertOwned(); + return { companyId: endpoint.companyId, endpointId: endpoint.id, + publicId: endpoint.publicId, connectionId: endpoint.connectionId, runtimeGeneration: options.runtimeGeneration(endpoint), + bindingId: enrolled.bindingId, teamId: enrolled.teamId, appId: enrolled.appId, signingSecret }; + }); + if (!binding) return; + await receiveSlackCloudEvent(event, binding, + request => options.handleWebhook(binding.publicId, "slack", request, binding.runtimeGeneration)); + }); +} diff --git a/server/src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts b/server/src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts new file mode 100644 index 00000000000..9ff84fc43c5 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts @@ -0,0 +1,202 @@ +import { createHmac, generateKeyPairSync, randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { eq } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { SLACK_CHAT_BOT_SCOPES, type ToolCredentialSecretRef } from "@paperclipai/shared"; +import { activityLog, agents, applyPendingMigrations, authUsers, chatEndpoints, chatSlackRegistrations, closeCheckpointDb, closeFileStorageDb, closeLivenessDb, companies, companyMemberships, companySecrets, createDb, toolApplications, toolConnections } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "../../../lib/dedicated-test-database-url.js"; +import { heartbeatService } from "../../heartbeat.js"; +import { chatChannelService } from "../../chat-channels.js"; +import { createDeliveryWorkCoordinator } from "../../delivery-work-coordinator.js"; +import { githubConnectionEventService } from "../../github-connection-events.js"; +import { registerGitHubConnectionEventWork } from "../../github-connection-work.js"; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from "../../paperclip-cloud-connector.js"; +import { secretService } from "../../secrets.js"; +import type { InternalSetupState } from "../runtime-types.js"; +import { guardSlackFixtureNetwork } from "../../../__tests__/helpers/slack-cloud/transport.js"; + +const appUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_TEST_DATABASE_URL"); +const cloudUrl = resolveExternalTestDatabaseUrl("SLACK_EVENT_TEST_DATABASE_URL"); +const cloudDist = process.env.SLACK_TEST_CLOUD_DIST; + +it.skipIf(!appUrl || !cloudUrl || !cloudDist)("normal notifications enroll the first consumer and authorized reconfigure replaces only positively retired instance custody", async () => { + const network = guardSlackFixtureNetwork({ requirePreload: true }); + const load = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, file)).href); + const [{ PostgresCloudHarnessRegistry }, { PostgresConnectorStore }, { ConnectorBroker }, { InMemoryProviderSecretStore }, + { providerRegistryFromEnvironment }, { CloudStackProvisioner }, { FakeCloudStackProvider }, { Pool }] = await Promise.all([ + load("registry/postgres.js"), load("connections/postgres.js"), load("connections/broker.js"), load("providers/neon.js"), + load("connections/providers.js"), load("provisioner/provisioner.js"), load("providers/fake.js"), + import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, "../node_modules/pg/lib/index.js")).href), + ]); + await applyPendingMigrations(appUrl!); + const db = createDb(appUrl!, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + const pool = new Pool({ connectionString: cloudUrl!, max: 2 }); + const registry = Reflect.construct(PostgresCloudHarnessRegistry, [pool]) as { + applyMigrations(folder: string, timeout: number): Promise; hydrate(): Promise; close(): Promise; + }; + try { await registry.applyMigrations(path.resolve(cloudDist!, "../migrations"), 30_000); await registry.hydrate(); } + catch (error) { await db.$client.end({ timeout: 1 }); await registry.close(); throw error; } + const store = new PostgresConnectorStore(pool), secrets = new InMemoryProviderSecretStore(); + const id = randomUUID(), companyId = randomUUID(), endpointId = randomUUID(), connectionId = randomUUID(), applicationId = randomUUID(), agentId = randomUUID(); + const actor = `owner-${id}`, instanceA = `a-${id}`, instanceB = `b-${id}`, publicId = "h".repeat(43); + const stackIds = [instanceA, instanceB], signingSecret = "0123456789abcdef0123456789abcdef"; + const keys = stackIds.map(() => ({ sign: generateKeyPairSync("ed25519"), seal: generateKeyPairSync("x25519") })); + let broker: InstanceType, base = "", loseEnrollmentResponse = false; + const requests: { path: string; issuer?: string }[] = []; + const server = createServer((req, res) => { + requests.push({ path: req.url! }); + if (loseEnrollmentResponse && req.url === "/v1/connector/slack-events") { + loseEnrollmentResponse = false; + res.end = (() => { res.destroy(); return res; }) as typeof res.end; + } + void broker.handle(req, res, new URL(req.url!, base)).catch((error: unknown) => { + if (!res.headersSent) res.writeHead(500); res.end(String(error)); + }); + }); + const directory = await mkdtemp(path.join(os.tmpdir(), "slack-transition-")); + let service: ReturnType | undefined; + let heartbeat: ReturnType | undefined; + let coordinator: ReturnType | undefined; + let worker: ReturnType | undefined; + const errors: unknown[] = []; + const deliveryIds: string[] = []; + const endpoint = async () => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)))[0]!; + const setup = async () => (await endpoint()).setup as InternalSetupState; + const configureIdentity = (index: number) => { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", stackIds[index]!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", keys[index]!.sign.privateKey.export({ format: "jwk" }).d!); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", keys[index]!.seal.privateKey.export({ format: "jwk" }).d!); + }; + try { + for (const [index, stackId] of stackIds.entries()) { + await new CloudStackProvisioner({ provider: new FakeCloudStackProvider(), registry }).provisionDevelopmentStack({ + idempotencyKey: stackId, accountGroup: { id: `account-${stackId}`, slug: stackId, displayName: "Transition" }, + owner: { id: actor, email: `${id}@example.test` }, + stack: { stackId, accountGroupId: `account-${stackId}`, stackSlug: stackId, displayName: "Transition", kind: "demo", primaryHost: `${stackId}.example.test`, region: "us-east-1" }, + versionPlan: { appVersion: "fixture", schemaVersion: "fixture", rolloutTrack: "stable" }, + sleepPolicy: { desiredState: "awake" }, primaryCompany: { paperclipCompanyId: index === 0 ? companyId : randomUUID(), displayName: "Transition", defaultBudgetCents: 0 }, + }); + await store.putCloudStackInstance({ id: stackId, kind: "cloud_stack", companyStackId: stackId, ownerUserId: actor, environment: "test", + signPublicKey: keys[index]!.sign.publicKey.export({ format: "jwk" }).x!, sealPublicKey: keys[index]!.seal.publicKey.export({ format: "jwk" }).x!, + origins: ["https://tenant.example"], activeOrigin: "https://tenant.example", label: "Transition", status: "active", createdAt: new Date(), updatedAt: new Date() }); + } + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing fixture address"); + base = `http://127.0.0.1:${address.port}`; + broker = new ConnectorBroker({ publicOrigin: base, environment: "test", store, + providers: providerRegistryFromEnvironment({}, "test"), secrets: { resolve: async () => undefined }, + authenticate: async () => ({ globalUserId: actor }), slackEvents: { secrets, secretRefPrefix: "secret:///test/transition" }, + eventWakeEnrollmentEnabled: true, eventWakeDeliveryEnabled: true, wakeCloudStack: async () => {} }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(directory, "master.key")); configureIdentity(0); + await db.insert(companies).values({ id: companyId, name: "Transition", issuePrefix: `T${id.slice(0, 7)}` }); + await db.insert(authUsers).values({ id: actor, name: "Owner", email: `${id}@owner.example.test`, emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actor, status: "active", membershipRole: "operator" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Transition", adapterType: "process", status: "idle" }); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Transition", type: "chat" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Transition", uid: id, + transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId, + provider: "slack", status: "paused", providerAccountId: "TTEST", botExternalId: "UBOT", setup: { runtimeGeneration: 0, step: "complete" } as InternalSetupState }); + await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", appId: "ATEST", + manifest: {}, manifestHash: "fixture", callbackUri: "https://tenant.example/callback", createdByUserId: actor }); + const vault = secretService(db), refs: ToolCredentialSecretRef[] = []; + for (const [key, value] of Object.entries({ botToken: "xoxb-fixture", signingSecret })) { + const secret = await vault.create(companyId, { name: `${key}-${id}`, provider: "local_encrypted", value }, { userId: actor }); + await vault.createBinding({ companyId, secretId: secret.id, targetType: "tool_connection", targetId: connectionId, configPath: `credentials.${key}` }); + refs.push({ secretId: secret.id, versionSelector: "latest", configPath: `credentials.${key}`, required: true, projectionClass: "unclassified" }); + } + await db.update(toolConnections).set({ credentialSecretRefs: refs }).where(eq(toolConnections.id, connectionId)); + const fetchImpl: typeof fetch = async input => { + const url = input instanceof Request ? new URL(input.url) : new URL(String(input)); + if (url.href === "https://slack.com/api/auth.test") return Response.json({ ok: true, team_id: "TTEST", user_id: "UBOT", user: "fixture" }, { headers: { "x-oauth-scopes": SLACK_CHAT_BOT_SCOPES.join(",") } }); + if (url.origin === "https://slack.com" && url.pathname === "/api/conversations.list") return Response.json({ ok: true, channels: [], response_metadata: {} }); + throw new Error(`Unexpected transition provider request ${url}`); + }; + heartbeat = heartbeatService(db); + service = chatChannelService(db, { heartbeat, fetch: fetchImpl, publicBaseUrl: "https://tenant.example" }); + const events = githubConnectionEventService(db); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + worker = registerGitHubConnectionEventWork(db, coordinator, events, () => true); await worker.ready; + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull()); + const leaseCount = () => requests.filter(value => value.path === "/v1/connector/events/lease").length; + expect(leaseCount()).toBe(0); + await service.configure(endpointId, { action: "resume" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudEvents?.runtimeGeneration).toBe(1), { timeout: 10_000 }); + await vi.waitFor(() => expect(leaseCount()).toBeGreaterThan(0), { timeout: 10_000 }); + expect((await store.getInstance(instanceA)).eventWakeVersion).toBe(1); + expect((await setup()).slackCloudInstanceId).toBe(instanceA); + await service.configure(endpointId, { action: "pause" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(2), { timeout: 10_000 }); + expect(await broker.slackEvents.current(instanceA, publicId)).toBeUndefined(); + expect((await store.listWebhookBindings(instanceA)).every((binding: { receivesEvents: boolean }) => !binding.receivesEvents)).toBe(true); + const retired = await setup(); configureIdentity(1); + const foreign = createPaperclipCloudConnector({ config: paperclipCloudConnectorConfigFromEnv()! }); + await expect(foreign.slackEventEnrollment({ subject: `foreign-${actor}`, companyId, enrollment: { + active: true, companyId, endpointId, publicId, connectionId, runtimeGeneration: 3, teamId: "TTEST", appId: "ATEST", signingSecret, + } })).rejects.toMatchObject({ status: 403 }); + expect(await broker.slackEvents.current(instanceB, publicId)).toBeUndefined(); + expect((await setup()).slackCloudIntent).toBeUndefined(); + expect((await setup()).slackCloudInstanceId).toBe(instanceA); + expect((await setup()).slackCloudSubject).toBe(actor); + await service.configure(endpointId, { action: "resume" }, actor); + const pending = await setup(); + expect(pending.slackCloudIntent).toMatchObject({ instanceId: instanceB, runtimeGeneration: 3, subject: actor }); + expect(pending.slackCloudInstanceId).toBe(instanceA); + expect(pending.slackCloudEvents).toEqual(retired.slackCloudEvents); + await vi.waitFor(async () => expect((await setup()).slackCloudInstanceId).toBe(instanceB), { timeout: 10_000 }); + expect((await setup()).slackCloudIntent).toBeUndefined(); + expect((await setup()).slackCloudRetiredGeneration).toBeUndefined(); + expect((await broker.slackEvents.current(instanceB, publicId)).runtimeGeneration).toBe(3); + expect(await broker.slackEvents.current(instanceA, publicId)).toBeUndefined(); + await service.configure(endpointId, { action: "pause" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(4), { timeout: 10_000 }); + loseEnrollmentResponse = true; + await service.configure(endpointId, { action: "resume" }, actor); + await vi.waitFor(async () => expect((await setup()).slackCloudIntent).toMatchObject({ instanceId: instanceB, + runtimeGeneration: 5, failureCode: "broker_unavailable" }), { timeout: 10_000 }); + expect((await broker.slackEvents.current(instanceB, publicId)).runtimeGeneration).toBe(5); + expect((await setup()).slackCloudEvents?.runtimeGeneration).toBe(3); + const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: `EvLostReplacement-${id}`, team_id: "TTEST", api_app_id: "ATEST", + event: { type: "app_mention", text: "removed before admission", user: "UOPERATOR", channel: "CTEST", ts: "1788.600" } })); + const timestamp = String(Math.floor(Date.now() / 1000)); + await broker.slackEvents.receive(await broker.slackEvents.current(instanceB, publicId), { rawBody, timestamp, + signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}` }); + deliveryIds.push(...(await pool.query("SELECT delivery_id FROM cloud_harness.connector_webhook_delivery_targets WHERE connector_instance_id=$1", [instanceB])).rows.map((row: { delivery_id: string }) => row.delivery_id)); + await service.configure(endpointId, { action: "remove" }, actor); + expect((await db.select().from(toolConnections).where(eq(toolConnections.id, connectionId)))[0]!.credentialSecretRefs).toEqual([]); + await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(6), { timeout: 10_000 }); + expect((await setup()).slackCloudIntent).toBeUndefined(); + expect((await setup()).slackCloudInstanceId).toBe(instanceB); + expect(await broker.slackEvents.current(instanceB, publicId)).toBeUndefined(); + expect((await pool.query("SELECT delivery_id FROM cloud_harness.connector_webhook_delivery_targets WHERE connector_instance_id=$1", [instanceB])).rows).toEqual([]); + await expect(broker.slackEvents.enroll(await store.getInstance(instanceB), actor, { companyId, endpointId, publicId, connectionId, + runtimeGeneration: 5, teamId: "TTEST", appId: "ATEST", signingSecret })).rejects.toThrow("GENERATION_CHANGED"); + expect(errors).toEqual([]); + } catch (error) { console.error("Slack instance transition body failure", error); throw error; } + finally { + await worker?.stop(); await coordinator?.stop(); await service?.shutdown(); await heartbeat?.drainActiveRunExecutions(); + if (server.listening) { server.close(); await once(server, "close"); } + vi.unstubAllEnvs(); + try { + await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId)); + await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + await db.delete(authUsers).where(eq(authUsers.id, actor)); + await pool.query("DELETE FROM cloud_harness.connector_instances WHERE id=ANY($1::text[])", [stackIds]); + await pool.query("DELETE FROM cloud_harness.connector_webhook_deliveries WHERE id=ANY($1::text[])", [deliveryIds]); + } finally { + await closeCheckpointDb(db, 1); await closeFileStorageDb(db, 1); await closeLivenessDb(db, 1); + await db.$client.end({ timeout: 1 }); await registry.close(); await rm(directory, { recursive: true, force: true }); + try { network.assertQuiet(); } finally { network.restore(); } + } + } +}, 60_000); diff --git a/server/src/services/chat-channels/slack/cloud-signing-secret.test.ts b/server/src/services/chat-channels/slack/cloud-signing-secret.test.ts new file mode 100644 index 00000000000..efb331ba2e1 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-signing-secret.test.ts @@ -0,0 +1,19 @@ +import type { Db } from "@paperclipai/db"; +import { expect, it, vi } from "vitest"; +import { HttpError } from "../../../errors.js"; +import type { EndpointRow } from "../runtime-types.js"; +import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js"; + +it.each([400, 401, 403, 404, 422])("credential resolution %s cannot authenticate or acknowledge a receipt", async status => { + const resolve = vi.fn(async (): Promise> => { throw new HttpError(status, "synthetic"); }); + await expect(resolveSlackCloudSigningSecret(resolve, {} as EndpointRow, {} as Db, ["signingSecret"])) + .rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" }); +}); + +it("empty credential is attention, while transient failures retain their retry classification", async () => { + await expect(resolveSlackCloudSigningSecret(async () => ({}), {} as EndpointRow)) + .rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" }); + const transient = new Error("database temporarily unavailable"); + await expect(resolveSlackCloudSigningSecret(async () => { throw transient; }, {} as EndpointRow)).rejects.toBe(transient); + await expect(resolveSlackCloudSigningSecret(async () => ({ signingSecret: "synthetic" }), {} as EndpointRow)).resolves.toBe("synthetic"); +}); diff --git a/server/src/services/chat-channels/slack/cloud-signing-secret.ts b/server/src/services/chat-channels/slack/cloud-signing-secret.ts new file mode 100644 index 00000000000..96e0cdd27d0 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-signing-secret.ts @@ -0,0 +1,18 @@ +import { HttpError } from "../../../errors.js"; +import type { createChatCredentials } from "../credentials.js"; +import { SlackCloudReceiptError } from "./cloud-event.js"; + +type Resolver = ReturnType["resolveCredentials"]; + +export async function resolveSlackCloudSigningSecret(resolve: Resolver, ...args: Parameters): Promise { + let credentials: Record; + try { + credentials = await resolve(...args); + } catch (error) { + if (error instanceof HttpError && [400, 401, 403, 404, 422].includes(error.status)) + throw new SlackCloudReceiptError("signing_credential_unavailable", "attention"); + throw error; + } + if (!credentials.signingSecret) throw new SlackCloudReceiptError("signing_credential_unavailable", "attention"); + return credentials.signingSecret; +} diff --git a/server/src/services/chat-channels/slack/cloud-test-prewire.test.ts b/server/src/services/chat-channels/slack/cloud-test-prewire.test.ts new file mode 100644 index 00000000000..8e69ceacfd9 --- /dev/null +++ b/server/src/services/chat-channels/slack/cloud-test-prewire.test.ts @@ -0,0 +1,57 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; + +const guard = fileURLToPath(new URL("../../../__tests__/helpers/slack-cloud/prewire.cjs", import.meta.url)); +const launcher = fileURLToPath(new URL("../../../../scripts/test-slack-cloud-postgres.mjs", import.meta.url)); +const cleanEnv = () => ({ PATH: process.env.PATH, HOME: process.env.HOME, NODE_OPTIONS: "" }); + +it.each([ + "require('node:http').get('http://example.invalid')", + "require('node:https').get('https://example.invalid')", + "require('node:net').connect(443,'example.invalid')", + "require('node:tls').connect(443,'example.invalid')", + "fetch('https://example.invalid')", + "require('node:net').connect(55448,'127.0.0.1')", + "require('node:http').get('http://127.0.0.1:1/')", +])("the checked-in guard fails caught unowned transport attempts: %s", operation => { + const child = spawnSync(process.execPath, ["--require", guard, "-e", + `let dnsCalls=0; require('node:dns').lookup=()=>{dnsCalls++;throw Error('DNS must not run');}; try { ${operation}; } catch {} console.log(JSON.stringify({...globalThis[Symbol.for('paperclip.slack.fixture.prewire')].stats(),dnsCalls}));`], + { env: cleanEnv(), encoding: "utf8", timeout: 5_000 }); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(98); + const stats = JSON.parse(child.stdout.trim()); + expect(stats.pid).toBe(child.pid); + expect(stats.unexpectedAttempts).toHaveLength(1); + expect(stats.peakPgSockets).toBe(0); + expect(stats.dnsCalls).toBe(0); +}); + +it("the same guard allows an actual owned loopback HTTP server and initializes before the child script", () => { + const child = spawnSync(process.execPath, ["--require", guard, "-e", ` + const http=require('node:http'); + if(process.env.SLACK_GATE_GUARD_PID!==String(process.pid))throw Error('preload missing'); + const server=http.createServer((req,res)=>res.end('owned response')); + server.listen(0,'127.0.0.1',()=>http.get('http://127.0.0.1:'+server.address().port,response=>{ + let body='';response.on('data',chunk=>body+=chunk);response.on('end',()=>server.close(()=>{ + if(body!=='owned response')throw Error('response missing'); + console.log(JSON.stringify(globalThis[Symbol.for('paperclip.slack.fixture.prewire')].stats())); + })); + }));`], { env: cleanEnv(), encoding: "utf8", timeout: 5_000 }); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(0); + expect(JSON.parse(child.stdout.trim())).toMatchObject({ pid: child.pid, peakPgSockets: 0, unexpectedAttempts: [] }); +}); + +it.each([ + {}, + { PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@127.0.0.1/live", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/cloud_test" }, + { PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@127.0.0.1/same_test", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/same_test" }, + { PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@example.invalid/app_test", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/cloud_test" }, +])("the repository launcher rejects unsafe or absent fixture settings before starting Vitest", settings => { + const child = spawnSync(process.execPath, [launcher], { env: { ...cleanEnv(), ...settings }, encoding: "utf8", timeout: 5_000 }); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(1); + expect(child.stderr).toMatch(/test database|loopback|distinct/); + expect(child.stdout).not.toContain("RUN"); +}); diff --git a/server/src/services/chat-channels/slack/webhook-callbacks.ts b/server/src/services/chat-channels/slack/webhook-callbacks.ts index f0fb377c371..e4235c1dee8 100644 --- a/server/src/services/chat-channels/slack/webhook-callbacks.ts +++ b/server/src/services/chat-channels/slack/webhook-callbacks.ts @@ -1,3 +1,4 @@ +import { slackCloudAcceptedAt } from "./cloud-event.js"; import { isProvisionedSlackSetup } from "@paperclipai/shared"; import { slackRequestSignatureIsValid, slackRequestWorkspaceId, slackCallbackMatchesPublicUrl, inspectSlackCallback } from "./callback-verification.js"; import type { InternalSetupState } from "../runtime-types.js"; @@ -53,6 +54,7 @@ export function createSlackWebhookCallbacks(db: Db, options: SlackWebhookOptions return null; } async function recordSlackWebhookCallback(endpoint: EndpointRow, request: Request, slackCallbackInspection: Request, runtimeContext: RuntimeContext): Promise { + if (slackCloudAcceptedAt(request) !== undefined) return; const eventBody = await slackCallbackInspection.clone().text(); const callback = await inspectSlackCallback(slackCallbackInspection); if (callback) { diff --git a/server/src/services/chat-sdk-runtime.ts b/server/src/services/chat-sdk-runtime.ts index 6afd8fb844c..602b77fff2d 100644 --- a/server/src/services/chat-sdk-runtime.ts +++ b/server/src/services/chat-sdk-runtime.ts @@ -17,6 +17,8 @@ import { } from "@chat-adapter/discord"; import type { SlackAdapterConfig } from "@chat-adapter/slack"; import { PaperclipSlackAdapter } from "./chat-channels/slack/adapter.js"; +import { verifySlackSignature } from "@chat-adapter/slack/webhook"; +import { slackCloudAcceptedAt } from "./chat-channels/slack/cloud-event.js"; import { createTeamsAdapter, type TeamsAdapter, @@ -1361,6 +1363,13 @@ function createProviderAdapter( case "slack": { const adapterConfig: SlackAdapterConfig = { ...config.credentials, + webhookVerifier: async (request, body) => { + await verifySlackSignature(body, request.headers, { + signingSecret: config.credentials.signingSecret, + now: () => slackCloudAcceptedAt(request) ?? Date.now(), + }); + return true; + }, logger: resolvedLogger, mode: "webhook", nativeStreaming: true, diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index e84092ed826..e92c27d9953 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -1,7 +1,10 @@ +import { SlackCloudReceiptError } from "./chat-channels/slack/cloud-event.js"; +import { reconcilePendingSlackCloudEnrollments } from "./chat-channels/slack/cloud-enrollment.js"; import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js"; +import { dispatchSlackCloudEvent } from "./chat-channels/slack/cloud-ingress.js"; import { chatEndpoints } from "@paperclipai/db"; import { randomUUID } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; @@ -224,6 +227,9 @@ export function githubConnectionEventService( ) { const now = options.now ?? (() => new Date()); let nextPollAt = 0; + let nextEnrollmentAt: number | null = null; + let nextEnrollmentDiagnosticAt = 0; + const slackDiagnostics = new Map(); let configuredConnector: { config: NonNullable>; connector: PaperclipCloudConnector; @@ -503,8 +509,27 @@ export function githubConnectionEventService( nextPollAt = Number.POSITIVE_INFINITY; return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; } + try { + return await drainConnectorEvents(connector); + } finally { + try { + nextEnrollmentAt = await reconcilePendingSlackCloudEnrollments(db, now()); + } catch { + const time = now().getTime(); + nextEnrollmentAt = time + 5_000; + if (time >= nextEnrollmentDiagnosticAt) { + nextEnrollmentDiagnosticAt = time + 300_000; + logger.warn({ reason: "enrollment_reconcile_unavailable" }, "Slack enrollment responsibility retained"); + } + } + } + } + + async function drainConnectorEvents(connector: PaperclipCloudConnector): Promise { const bindings = await activeBindings(); - const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + const githubBots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + const slackBots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "slack"), sql`${chatEndpoints.setup}->'slackCloudEvents' is not null`, sql`${chatEndpoints.status} in ('active','verifying','attention')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + const bots = [...githubBots, ...slackBots]; let identity: { subject: string; companyId: string } | null = bindings[0] ?? (bots[0] ? { subject: "github-bot-events", companyId: bots[0].companyId } : null); const hasConsumer = bindings.length > 0 || bots.length > 0; if (!identity) { @@ -517,7 +542,7 @@ export function githubConnectionEventService( if (grant && subject) identity = { subject, companyId: grant.companyId }; } if (!identity) { - const [history] = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`)).limit(1); + const [history] = await db.select().from(chatEndpoints).where(sql`(${chatEndpoints.provider} = 'github' AND ${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null) OR (${chatEndpoints.provider} = 'slack' AND ${chatEndpoints.setup}->'slackCloudEvents' is not null)`).limit(1); if (history) identity = { subject: "github-bot-events", companyId: history.companyId }; } if (!identity) { @@ -543,6 +568,23 @@ export function githubConnectionEventService( }; const acknowledge: string[] = []; for (const event of lease.events) { + if (event.provider === "slack") { + try { await dispatchSlackCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; } + catch (error) { + const code = error instanceof SlackCloudReceiptError ? error.code : "owner_or_admission_unavailable"; + const disposition = error instanceof SlackCloudReceiptError ? error.disposition : "retry"; + const time = now().getTime(); + if (time - (slackDiagnostics.get(event.id) ?? Number.NEGATIVE_INFINITY) >= 300_000) { + if (slackDiagnostics.size >= 1000) slackDiagnostics.delete(slackDiagnostics.keys().next().value!); + slackDiagnostics.set(event.id, time); + logger.warn({ deliveryId: event.id.slice(0, 200), reason: code, disposition, + recoveryExpiresAt: Number.isFinite(Date.parse(event.createdAt)) ? new Date(Date.parse(event.createdAt) + 30 * 24 * 60 * 60_000).toISOString() : null }, "Slack connector receipt requires attention"); + } + if (disposition === "invalid") { acknowledge.push(event.id); result.ignored += 1; } + else result.failed += 1; + } + continue; + } if (event.payload.githubApp !== undefined) { try { await dispatchGitHubBotCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; } catch { result.failed += 1; } @@ -583,14 +625,14 @@ export function githubConnectionEventService( } return { - nextPollAt: () => Number.isFinite(nextPollAt) ? nextPollAt : null, + nextPollAt: () => { const next = Math.min(nextPollAt, nextEnrollmentAt ?? Number.POSITIVE_INFINITY); return Number.isFinite(next) ? next : null; }, pollOnce(input: { forSetup?: boolean; forDelivery?: boolean } = {}): Promise { // A manager actively awaiting a dedicated App's signed ping must not // inherit a personal connection's old empty-queue backoff. This is a // bounded readiness probe, not an additional background polling loop. if (inFlight) return inFlight; const time = now().getTime(); - if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : nextPollAt)) { + if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : Math.min(nextPollAt, nextEnrollmentAt ?? Number.POSITIVE_INFINITY))) { return Promise.resolve({ leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }); } nextSetupPollAt = time + 5_000; diff --git a/server/src/services/paperclip-cloud-connector.test.ts b/server/src/services/paperclip-cloud-connector.test.ts index 07bd0bb119e..4d7c594153b 100644 --- a/server/src/services/paperclip-cloud-connector.test.ts +++ b/server/src/services/paperclip-cloud-connector.test.ts @@ -584,11 +584,13 @@ describe("Paperclip Cloud connector", () => { return Response.json({ active: true, status: "active", - profiles: ["gmail.read", "drive.write", "unknown.profile", "gmail.read"], + profiles: ["gmail.read", "drive.write", "unknown.profile", "slack.events.v1", "gmail.read"], + slackEvents: { version: 1 }, }); }); const connector = createPaperclipCloudConnector({ config: keys.config, request: request as typeof fetch }); await expect(connector.getCapabilities()).resolves.toEqual(["gmail.read", "drive.write"]); + await expect(connector.slackEventsAvailable()).resolves.toBe(true); }); it("fails capability discovery closed for inactive, legacy, malformed, or rejected status responses", async () => { @@ -699,10 +701,10 @@ function seal( recipientPublicKey: KeyObject, purpose: "initial" | "access" | "events", configValue: PaperclipCloudConnectorConfig, - profile: keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES | "slack.manager" | "github.bot" | "github.code", + profile: keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES | "slack.manager" | "slack.events.v1" | "github.bot" | "github.code", ) { - const provider = profile.startsWith("github.") ? "github" : profile === "slack.manager" ? "slack" : "google"; - const scopes = provider === "github" ? [] : profile === "slack.manager" ? SLACK_MANAGER_SCOPES : GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profile as keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES].scopes; + const provider = profile.startsWith("github.") ? "github" : profile.startsWith("slack.") ? "slack" : "google"; + const scopes = provider === "github" || profile === "slack.events.v1" ? [] : profile === "slack.manager" ? SLACK_MANAGER_SCOPES : GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profile as keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES].scopes; const ephemeral = generateKeyPairSync("x25519"); const ephemeralJwk = ephemeral.publicKey.export({ format: "jwk" }) as { x: string }; const recipientJwk = recipientPublicKey.export({ format: "jwk" }) as { x: string }; @@ -795,3 +797,66 @@ describe("dedicated App gateway delivery", () => { expect(JSON.stringify(result)).not.toContain('"ct"'); }); }); + + +describe("sealed Slack event connector transport", () => { + it("opens a Slack event through the normal outer lease without exposing ciphertext or changing callback bytes", async () => { + const keys = config(); + const createdAt = "2026-11-05T10:00:00.000Z"; + const bindingId = `slack-event:${"a".repeat(64)}`; + const packet = { version: 1, instanceId, environment: "staging", bindingId, acceptedAt: createdAt, + rawBody: Buffer.from('{ "event_id": "EvFixture", "text": "original bytes" }').toString("base64"), + timestamp: "1793872800", signature: `v0=${"b".repeat(64)}` }; + const inner = seal(packet, keys.sealPublicKey, "events", keys.config, "slack.events.v1"); + const outer = seal({ v: 1, instanceId, environment: "staging", leaseId: "lease-slack", events: [ + { id: "receipt-slack", provider: "slack", event: "event_callback", createdAt, installationId: bindingId, + bindingIds: [bindingId], payload: { slackEvent: { bindingId, sealed: inner } } }, + ] }, keys.sealPublicKey, "events", keys.config, "github.code"); + const connector = createPaperclipCloudConnector({ config: keys.config, + request: vi.fn(async () => Response.json({ leaseId: "lease-slack", sealed: outer })) as typeof fetch }); + const result = await connector.leaseEvents({ subject, companyId }); + expect(result?.failedEventIds).toBeUndefined(); + expect(result?.events).toHaveLength(1); + expect(result?.events[0]?.payload.slackEvent).toEqual(packet); + expect(JSON.stringify(inner)).not.toContain("original bytes"); + expect(JSON.stringify(result)).not.toContain('"ct"'); + }); + + it.each(["binding", "acceptedAt", "instance", "environment", "key", "profile"] as const)( + "isolates an invalid inner Slack %s while retaining a healthy sibling", async failure => { + const keys = config(); + const createdAt = "2026-11-05T10:00:00.000Z"; + const bindingId = `slack-event:${"a".repeat(64)}`; + const packet = { version: 1, instanceId, environment: "staging", bindingId, acceptedAt: createdAt, rawBody: "e30=" }; + const changed = { ...packet, ...(failure === "binding" ? { bindingId: "other" } : {}), + ...(failure === "acceptedAt" ? { acceptedAt: "2026-11-05T10:00:00.001Z" } : {}), + ...(failure === "instance" ? { instanceId: "foreign" } : {}), + ...(failure === "environment" ? { environment: "production" } : {}) }; + const event = (id: string, inner: unknown) => ({ id, provider: "slack", event: "event_callback", createdAt, + installationId: bindingId, bindingIds: [bindingId], payload: { slackEvent: { bindingId, sealed: inner } } }); + const bad = seal(changed, failure === "key" ? config().sealPublicKey : keys.sealPublicKey, "events", keys.config, + failure === "profile" ? "slack.manager" : "slack.events.v1"); + const outer = seal({ v: 1, instanceId, environment: "staging", leaseId: "lease-slack", events: [ + event("bad", bad), event("healthy", seal(packet, keys.sealPublicKey, "events", keys.config, "slack.events.v1")), + ] }, keys.sealPublicKey, "events", keys.config, "github.code"); + const connector = createPaperclipCloudConnector({ config: keys.config, + request: vi.fn(async () => Response.json({ leaseId: "lease-slack", sealed: outer })) as typeof fetch }); + await expect(connector.leaseEvents({ subject, companyId })).resolves.toMatchObject({ + leaseId: "lease-slack", failedEventIds: ["bad"], events: [{ id: "healthy", payload: { slackEvent: packet } }], + }); + }); + + it.each([true, false])("selects the matching signed Slack operation receipt (active=%s)", async active => { + const keys = config(); + const enrollment = { active, endpointId: "endpoint", runtimeGeneration: 7 }; + const enrolled = { version: 1, bindingId: "enrolled" }, retired = { version: 1, runtimeGeneration: 7 }; + const request = vi.fn(async () => Response.json({ enrollment: enrolled, retirement: retired })); + const connector = createPaperclipCloudConnector({ config: keys.config, request: request as typeof fetch }); + await expect(connector.slackEventEnrollment({ subject, companyId, enrollment })).resolves.toEqual(active ? enrolled : retired); + expect(request).toHaveBeenCalledOnce(); + const [url, init] = request.mock.calls[0] as unknown as [string | URL, RequestInit]; + expect(new URL(String(url)).pathname).toBe("/v1/connector/slack-events"); + expect(typeof init.body).toBe("string"); + expect(JSON.parse(String(init.body))).toMatchObject({ enrollment: JSON.stringify(enrollment) }); + }); +}); diff --git a/server/src/services/paperclip-cloud-connector.ts b/server/src/services/paperclip-cloud-connector.ts index 6159051c5d3..da98d8c793c 100644 --- a/server/src/services/paperclip-cloud-connector.ts +++ b/server/src/services/paperclip-cloud-connector.ts @@ -35,7 +35,7 @@ export const GMAIL_CONNECTOR_SCOPES = [ export { GOOGLE_WORKSPACE_CONNECTOR_PROFILES }; export type PaperclipCloudConnectorEnvironment = "development" | "test" | "staging" | "production"; -export type PaperclipCloudConnectorOperation = "status" | "session" | "claim" | "refresh" | "revoke" | "webhook-bind" | "event-lease" | "event-ack" | "github-app"; +export type PaperclipCloudConnectorOperation = "status" | "session" | "claim" | "refresh" | "revoke" | "webhook-bind" | "event-lease" | "event-ack" | "github-app" | "slack-events"; export type PaperclipCloudConnectorProfileId = GoogleWorkspaceConnectorProfileId | GitHubConnectorProfileId | AsanaConnectorProfileId | "slack.manager" | "github.bot"; export type PaperclipCloudConnectorProvider = "google" | "github" | "asana" | "slack"; @@ -75,7 +75,7 @@ export type SealedConnectorEvents = { leaseId: string; events: Array<{ id: string; - provider: "github"; + provider: "github" | "slack"; event: string; action: string | null; installationId: string | null; @@ -86,12 +86,14 @@ export type SealedConnectorEvents = { }>; }; +type SealedEnvelopeProfileId = PaperclipCloudConnectorProfileId | "slack.events.v1"; + type SealedEnvelope = { v: 1; alg: "X25519-HKDF-SHA256-A256GCM"; purpose: "initial" | "access" | "events"; provider: PaperclipCloudConnectorProvider; - profile: PaperclipCloudConnectorProfileId; + profile: SealedEnvelopeProfileId; epk: string; iv: string; ct: string; @@ -115,10 +117,14 @@ type ConnectorResponse = { events?: unknown; acknowledged?: unknown; githubApps?: { version: number }; + slackEvents?: { version: number }; + enrollment?: unknown; + retirement?: unknown; eventWakeVersion?: unknown; }; const ENDPOINTS: Record = { + "slack-events": "/v1/connector/slack-events", "github-app": "/v1/connector/github-apps", status: "/v1/connector/instance-status", session: "/v1/connector/sessions", @@ -147,6 +153,9 @@ const BROKER_REJECTION_REASONS = new Set([ "AUDIENCE_MISMATCH", "OPERATION_MISMATCH", "PAYLOAD_MISMATCH", "MALFORMED_REQUEST", "UNSUPPORTED_ALGORITHM", "RATE_LIMITED", "PROVIDER_OPERATION_FAILED", "PROVIDER_REAUTHORIZATION_REQUIRED", + "SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_IDENTITY_CHANGED", + "SLACK_ENROLLMENT_CHANGED", "SLACK_ENROLLMENT_UNSETTLED", "SLACK_ENROLLMENT_DISABLED", + "SLACK_COMPANY_MISMATCH", "SLACK_EVENTS_UNAVAILABLE", ]); const UNKNOWN_BROKER_REASON = "UNKNOWN_BROKER_ERROR"; @@ -192,6 +201,7 @@ export class PaperclipCloudConnectorError extends Error { readonly code: string, readonly status?: number, readonly retryAt?: number, + readonly brokerReason?: string, ) { super(message); this.name = "PaperclipCloudConnectorError"; @@ -280,7 +290,7 @@ export function createPaperclipCloudConnector(input: { async function call( operation: PaperclipCloudConnectorOperation, claims: { subject: string; companyId: string; profile?: PaperclipCloudConnectorProfileId; scopes?: string[]; returnUri?: string; returnState?: string; claimId?: string; redemptionId?: string; eventWakeVersion?: 0 | 1 }, - secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement"; value: string }, + secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement" | "enrollment"; value: string }, ): Promise { const endpoint = new URL(ENDPOINTS[operation], `${config.baseUrl}/`).toString(); const issuedAt = Math.floor(now() / 1000); @@ -333,6 +343,7 @@ export function createPaperclipCloudConnector(input: { response.status === 409 ? "REAUTHORIZATION_REQUIRED" : "CONNECTOR_REQUEST_FAILED", response.status, providerRetryAt(response.headers.get("retry-after"), now()), + reason, ); } try { @@ -381,6 +392,14 @@ export function createPaperclipCloudConnector(input: { return { eventWakeVersion: () => eventWakeVersion, + async slackEventsAvailable() { + const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" }); + return response.active === true && response.slackEvents?.version === 1; + }, + async slackEventEnrollment(values: { subject: string; companyId: string; enrollment: Record }) { + const response = await call("slack-events", values, { field: "enrollment", value: JSON.stringify(values.enrollment) }); + return values.enrollment.active === false ? response.retirement : response.enrollment; + }, async githubAppsAvailable() { const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" }); return response.active === true && response.githubApps?.version === 2; @@ -551,6 +570,16 @@ export function createPaperclipCloudConnector(input: { || delivery.registrationId !== packet.registrationId) throw badEnvelope(); event.payload = { githubApp: delivery }; } + if (event.provider === "slack") { + const packet = event.payload.slackEvent; + if (!isRecord(packet) || typeof packet.bindingId !== "string" || packet.sealed === undefined) throw badEnvelope(); + const sealed = parseEnvelope(packet.sealed, "events", "slack", "slack.events.v1"); + const delivery = decryptEnvelope(sealed, sealKey, config.instanceId, config.environment, "slack", "slack.events.v1", []); + if (!isRecord(delivery) || delivery.version !== 1 || delivery.instanceId !== config.instanceId + || delivery.environment !== config.environment || delivery.bindingId !== packet.bindingId + || delivery.acceptedAt !== event.createdAt) throw badEnvelope(); + event.payload = { slackEvent: delivery }; + } events.push(event); } catch { failedEventIds.push(event.id); @@ -684,7 +713,7 @@ function parseEnvelope( value: unknown, purpose: SealedEnvelope["purpose"], provider: PaperclipCloudConnectorProvider, - profile: PaperclipCloudConnectorProfileId, + profile: SealedEnvelopeProfileId, ): SealedEnvelope { if (!value || typeof value !== "object" || Array.isArray(value)) throw badEnvelope(); const candidate = value as Partial; @@ -737,7 +766,7 @@ function unsealEvents( if (parsed.v !== 1 || parsed.instanceId !== instanceId || parsed.environment !== environment || typeof parsed.leaseId !== "string" || !Array.isArray(parsed.events)) throw badEnvelope(); for (const event of parsed.events) { - if (!isRecord(event) || typeof event.id !== "string" || event.provider !== "github" + if (!isRecord(event) || typeof event.id !== "string" || !["github", "slack"].includes(event.provider) || typeof event.event !== "string" || !isRecord(event.payload) || !Array.isArray(event.bindingIds) || !event.bindingIds.every((id) => typeof id === "string")) throw badEnvelope(); } @@ -754,7 +783,7 @@ function decryptEnvelope( instanceId: string, environment: string, provider: PaperclipCloudConnectorProvider, - profile: PaperclipCloudConnectorProfileId, + profile: SealedEnvelopeProfileId, scopes: readonly string[], ): unknown { const ephemeralRaw = Buffer.from(envelope.epk, "base64url"); diff --git a/server/src/services/tool-access/connections/removal.ts b/server/src/services/tool-access/connections/removal.ts index be9bea0dd78..ae3016de93e 100644 --- a/server/src/services/tool-access/connections/removal.ts +++ b/server/src/services/tool-access/connections/removal.ts @@ -3,7 +3,8 @@ import { notifyVoiceChange } from "../../voice/voice-recovery-notifications.js"; import { notifyChatEndpointWork } from "../../chat-work-notifications.js"; import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js"; import { toConnection } from "./projection.js"; -import { actorBinding } from "../actor.js"; +import { actorBinding, connectionActorUserId } from "../actor.js"; +import { stageSlackCloudEnrollmentIntent } from "../../chat-channels/slack/cloud-enrollment.js"; import { vercelConnectResourcesFor } from "../vercel-credentials.js"; import type { ActorInfo } from "../types.js"; import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js"; @@ -241,6 +242,8 @@ async function removeConnection( await lease.assertOwned(tx); await tx.update(chatEndpoints).set({ status: "archived", updatedAt: new Date() }).where(eq(chatEndpoints.id, endpoint.id)); await tx.update(toolConnections).set({ status: "archived", enabled: false, updatedAt: new Date() }).where(eq(toolConnections.id, connection.id)); + await stageSlackCloudEnrollmentIntent(tx as unknown as Db, endpoint.id, connectionActorUserId(actor)); + await lease.assertOwned(tx); }); await removeSlackRegistration(db, endpoint.id, lease); return removeConnectionUnlocked(connectionId, companyId, actor); From b6e9615aed0f189eea9690f43c04857a3e422be4 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:25:57 -0600 Subject: [PATCH 08/11] feat(ci): request advisory merge review after PR verification (#19480) ## Thinking Path > - Paperclip helps people manage AI agents and their work. > - Pull requests already run CI and automated code review. > - A completed check does not start a final advisory merge review. > - A private routine webhook can connect these two steps. > - The webhook credential must stay outside PR code and public logs. > - This pull request adds an optional notification from trusted branch code. > - The routine records a recommendation. A person keeps merge control. ## Linked Issues or Issue Description **What existing behavior does this improve?** It connects successful PR verification to an advisory Paperclip routine. **Subsystem affected** GitHub Actions and the existing routine webhook API. **Current behavior** A PR can finish verification without asking a routine for a final merge recommendation. **Proposed behavior** A trusted workflow reads the current PR and its complete check list. It sends the PR identity to a private webhook when required checks pass and other reported checks finish successfully, as skipped, or as neutral. The receiving routine runs the configured merge review skill and records its decision on its Paperclip task. **Reason and benefit** This gives maintainers an advisory decision after verification. It does not grant merge control to the routine. **Breaking changes** None. Missing secrets or delivery errors leave verification results unchanged. Searched open and closed PRs and issues for related work. No duplicate notification workflow was found. Related API work: #18058 and #18581. This change uses the routine API and does not alter chat credential setup. ## What Changed - Add a default-branch workflow for PR workflow completions, external checks, and commit statuses. - Use read permissions and a GitHub environment restricted to master for the private URL and bearer token. - Require successful `ci / verify`, `ci / e2e`, and additional branch-rule checks. Read every status page and refresh the PR before delivery. - Use a stable idempotency key for each PR, head, and base. Reject stale events and changed candidates. - Verify the two core check runs belong to the same real GitHub Actions suite for the unchanged PR caller and master-owned reusable CI. Reject forged names, renamed callers, and incomplete file listings. - Keep HTTP redirects disabled, bound request time, and print fixed delivery states only. - Add regression checks and document private setup, pause, rotation, and acceptance steps. - Grant commit-status read access. Add a separate PR API probe with the same read permissions and the real Actions token. The probe has no private environment or webhook secrets and sends no notification. ## Verification - Pass: `node --test .github/scripts/tests/notify-pr-merge-review.test.mjs` (21 tests after the security and token-permission fixes). - Pass: actionlint v1.7.12 on `.github/workflows/pr-merge-review.yml`. - Pass: live GitHub query of a current PR and its complete status list. This found a GraphQL field mismatch during development. The corrected query passed with 62 checks. - Pass: the configured bearer webhook returned `test_received` with `routineStarted=false` in setup mode. Readback confirmed an active routine and environment secrets restricted to the master branch. - Not run: `pnpm verify --explain` could not start because this checkout has no installed workspace dependencies. Required CI will supply broad coverage. - The first quality-gate run rejected a CI-only title because this change adds a script. The feature title passed the later gate. - The security review found that PR-controlled check names could forge eligibility. A focused regression failed before the fix and passed after suite, run, caller, and reusable-workflow checks were added. The corrected provenance query also passed against a completed historical CI run without sending a webhook. - Earlier CI at ec2c2dc failed in `workspace-recovery.test.ts` with a generated company-prefix collision, in `idle-domain-deadlines-postgres.test.ts` with a deadline snapshot mismatch and timeout, and in `CoordinatorTasks.test.tsx` with a route query mismatch. The browser shard also recorded a CSS preload test failure before cancellation. These failures remain recorded in [run 42245040952](https://github.com/paperclipai/paperclip/actions/runs/42245040952). This PR does not edit those production paths. Current master is merged and fresh CI is required; the earlier failures are not claimed resolved. - The repeated browser, general-server, and UI lanes passed at 715cee88ce. The earlier failures remain historical; their causes are not all established. The Runner package lane was still running before the next fix. - A reviewer found missing `statuses: read`, which the broader local token did not expose. The permission regression failed before the fix and passed after it. The new read-only API probe will test the query with the actual Actions token. - Pass at c3c84394b7: the real Actions-token API probe, typecheck, build, browser aggregate, security scan, and Greptile 5/5. No unresolved review threads remain. - The only remaining CI failure is a 15 second timeout in the existing retained-native-session-cleanup-work backlog test (server shard 6/12). This file and its production implementation are unchanged from the previous candidate whose lane passed. The one scoped rerun passed at the same head, and the required verify aggregate now passes. The timeout remains in [run 42250393645, attempt 1](https://github.com/paperclipai/paperclip/actions/runs/42250393645/attempts/1); its cause is not established. This is recovery evidence, not a claimed root-cause fix. - A nonblocking review suggestion to defer branch-rule lookup until a PR is matched remains a minor efficiency follow-up. It is not a correctness or security finding. - Pending: a real notification and resulting routine task after this workflow reaches master. The setup test did not start a PR review. ## Risks - This workflow uses secrets. Keep the environment restricted to the master branch. It runs only trusted branch code and does not read PR artifacts, caches, or executable files. - GitHub event timing and check visibility can race. Delivery is best effort. A missed event can omit a review; it cannot change CI results or merge a PR. - Optional skipped and neutral checks permit an assessment. They do not prove that a check ran. Required checks must succeed. The routine must assess unavailable scanner coverage and other evidence gaps before its recommendation. - Add any new PR verification workflow name to the completion trigger list. - PRs that change the primary CI caller, `.github/workflows/pr.yml`, need a manual advisory review until their caller change reaches master. This prevents a PR from replacing verification with a forged result. - This PR changes CI configuration. CODEOWNERS review and normal merge rules still apply. ## Model Used OpenAI GPT-6 family through Codex. The exact deployment model ID and context window are not exposed to this run. Used reasoning, tool use, and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with available version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .github/scripts/notify-pr-merge-review.mjs | 177 ++++++++++++ .../tests/notify-pr-merge-review.test.mjs | 263 ++++++++++++++++++ .github/workflows/pr-merge-review.yml | 57 ++++ .github/workflows/pr-review-api-access.yml | 43 +++ doc/PR-MERGE-REVIEW-WEBHOOK.md | 85 ++++++ 5 files changed, 625 insertions(+) create mode 100644 .github/scripts/notify-pr-merge-review.mjs create mode 100644 .github/scripts/tests/notify-pr-merge-review.test.mjs create mode 100644 .github/workflows/pr-merge-review.yml create mode 100644 .github/workflows/pr-review-api-access.yml create mode 100644 doc/PR-MERGE-REVIEW-WEBHOOK.md diff --git a/.github/scripts/notify-pr-merge-review.mjs b/.github/scripts/notify-pr-merge-review.mjs new file mode 100644 index 00000000000..d8151919ba0 --- /dev/null +++ b/.github/scripts/notify-pr-merge-review.mjs @@ -0,0 +1,177 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const repository = "paperclipai/paperclip"; +const shaPattern = /^[a-f0-9]{40}$/; +const minimumChecks = ["ci / verify", "ci / e2e"]; +const gh = (args) => JSON.parse(execFileSync("gh", args, { + encoding: "utf8", maxBuffer: 16 * 1024 * 1024, timeout: 30_000, + // Never allow command diagnostics or a response to reach the Actions log. + stdio: ["ignore", "pipe", "pipe"], +})); +const api = (endpoint) => gh(["api", endpoint]); +const pages = (endpoint) => gh(["api", "--paginate", "--slurp", endpoint]).flat(); +const query = `query($number:Int!, $endCursor:String) { + repository(owner:"paperclipai", name:"paperclip") { + pullRequest(number:$number) { + number url state isDraft headRefOid baseRefOid baseRefName changedFiles + statusCheckRollup { contexts(first:100, after:$endCursor) { + totalCount pageInfo { hasNextPage endCursor } + nodes { + __typename + ... on CheckRun { name status conclusion checkSuite { databaseId app { databaseId } + workflowRun { databaseId } } } + ... on StatusContext { context state } + } + } } + } + } +}`; + +export function snapshot(number, request = gh) { + let candidate; + let cursor; + const checks = []; + do { + const args = ["api", "graphql", "-f", `query=${query}`, "-F", `number=${number}`]; + if (cursor) args.push("-f", `endCursor=${cursor}`); + const result = request(args); + if (result.errors?.length) throw new Error("Incomplete check listing"); + const pr = result.data.repository.pullRequest; + if (candidate && (pr.headRefOid !== candidate.headRefOid || pr.baseRefOid !== candidate.baseRefOid)) { + throw new Error("Candidate changed during check pagination"); + } + candidate = pr; + const contexts = pr.statusCheckRollup.contexts; + checks.push(...contexts.nodes); + if (!contexts.pageInfo.hasNextPage) { + if (checks.length !== contexts.totalCount) throw new Error("Incomplete check listing"); + break; + } + const next = contexts.pageInfo.endCursor; + if (!next || next === cursor || checks.length >= contexts.totalCount) throw new Error("Incomplete check listing"); + cursor = next; + } while (checks.length < 10_000); + if (candidate.statusCheckRollup.contexts.pageInfo.hasNextPage) throw new Error("Check listing limit exceeded"); + return { ...candidate, statusCheckRollup: checks }; +} + +export function green(pr, requiredNames = minimumChecks) { + if (pr.state !== "OPEN" || pr.isDraft || pr.baseRefName !== "master" || + !shaPattern.test(pr.headRefOid) || !shaPattern.test(pr.baseRefOid)) return false; + const checks = pr.statusCheckRollup; + if (!Array.isArray(checks) || checks.length === 0) return false; + const relevant = checks; + const passed = (check) => check.__typename === "StatusContext" + ? check.state === "SUCCESS" + : check.__typename === "CheckRun" && check.status === "COMPLETED" && check.conclusion === "SUCCESS"; + // Optional skipped/neutral checks can mean no work or unavailable coverage. + // They permit assessment, not approval. Required checks must really succeed. + return relevant.every((check) => passed(check) || (check.__typename === "CheckRun" && + check.status === "COMPLETED" && ["SKIPPED", "NEUTRAL"].includes(check.conclusion))) && + [...new Set([...minimumChecks, ...requiredNames])].every((name) => + relevant.some((check) => (check.name ?? check.context) === name && passed(check))); +} + +export function trustedVerification(pr, { read = api, list = pages } = {}) { + // Names alone are forgeable in pull_request YAML. Require the real Actions + // suite for the unmodified PR caller and its master-owned reusable workflow. + const actionsChecks = pr.statusCheckRollup.filter((check) => + check.__typename === "CheckRun" && check.status === "COMPLETED" && check.conclusion === "SUCCESS" && + check.checkSuite?.app?.databaseId === 15368 && + Number.isSafeInteger(check.checkSuite?.workflowRun?.databaseId)); + const runIds = new Set(actionsChecks.filter((check) => minimumChecks.includes(check.name)) + .map((check) => check.checkSuite.workflowRun.databaseId)); + let trusted = false; + for (const id of runIds) { + const run = read(`/repos/${repository}/actions/runs/${id}`); + if (run.repository?.full_name !== repository || run.path !== ".github/workflows/pr.yml" || + run.event !== "pull_request" || run.head_sha !== pr.headRefOid || + run.status !== "completed" || run.conclusion !== "success") continue; + if (!minimumChecks.every((name) => actionsChecks.some((check) => check.name === name && + check.checkSuite.workflowRun.databaseId === id && check.checkSuite.databaseId === run.check_suite_id))) continue; + const reusable = run.referenced_workflows?.find((entry) => + entry.path === `${repository}/.github/workflows/pr-trusted.yml@master` && + entry.ref === "refs/heads/master" && shaPattern.test(entry.sha)); + if (reusable) { trusted = true; break; } + } + if (!trusted || !Number.isSafeInteger(pr.changedFiles) || pr.changedFiles < 0) return false; + const files = list(`/repos/${repository}/pulls/${pr.number}/files?per_page=100`); + // GitHub caps this API at 3,000 files. A partial listing cannot prove that + // the caller is unmodified. Do not execute or download any of these files. + return files.length === pr.changedFiles && files.every((file) => + typeof file.filename === "string" && ![file.filename, file.previous_filename].includes(".github/workflows/pr.yml")); +} + +export async function deliver(pr, { url, token, fetcher = fetch } = {}) { + if (!url || !token) return "unconfigured"; + try { + const target = new URL(url); + if (target.protocol !== "https:" || target.username || target.password || target.search || target.hash || + !/^\/api\/routine-triggers\/public\/[a-f0-9]+\/fire$/.test(target.pathname)) return "invalid_configuration"; + const response = await fetcher(target, { + method: "POST", redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { + "Content-Type": "application/json", Authorization: `Bearer ${token}`, + "Idempotency-Key": `pr-review:${repository}:${pr.number}:${pr.headRefOid}:${pr.baseRefOid}`, + }, + // No PR title, body, comments, or executable instructions cross this boundary. + body: JSON.stringify({ variables: { + pr_url: `https://github.com/${repository}/pull/${pr.number}`, + pr_number: String(pr.number), head_sha: pr.headRefOid, base_sha: pr.baseRefOid, + } }), + }); + // Do not read the response body: a proxy can echo the URL or credentials. + await response.body?.cancel(); + return response.ok ? "delivered" : "not_delivered"; + } catch { + // Fetch errors can contain the private URL. Keep the result fixed and quiet. + return "not_delivered"; + } +} + +export async function notify(event, { read = api, list = pages, view = snapshot, send = deliver } = {}) { + if (event.repository?.full_name !== repository) return []; + const run = event.workflow_run; + if (run && (run.status !== "completed" || !["pull_request", "pull_request_target"].includes(run.event))) return []; + const sha = run?.head_sha ?? event.check_run?.head_sha ?? event.sha; + if (!shaPattern.test(sha)) return []; + // Resolve forks through the commit API too: workflow_run.pull_requests can be empty. + const associated = run?.pull_requests?.length ? run.pull_requests + : list(`/repos/${repository}/commits/${sha}/pulls?per_page=100`); + const required = read(`/repos/${repository}/rules/branches/master`) + .filter((rule) => rule.type === "required_status_checks") + .flatMap((rule) => rule.parameters.required_status_checks.map((check) => check.context)); + const outcomes = []; + for (const number of new Set(associated.map((pr) => pr.number))) { + if (!Number.isSafeInteger(number) || number <= 0) continue; + const pr = view(number); + // Target-context workflows use the base SHA. All other events must still + // describe this PR's current head; an old successful run cannot trigger it. + if ((run?.event !== "pull_request_target" && pr.headRefOid !== sha) || !green(pr, required)) continue; + const fresh = view(number); + if (fresh.headRefOid !== pr.headRefOid || fresh.baseRefOid !== pr.baseRefOid || + !green(fresh, required) || !trustedVerification(fresh, { read, list })) continue; + outcomes.push(await send(fresh)); + } + return outcomes; +} + +async function main() { + try { + const url = process.env.PAPERCLIP_PR_REVIEW_WEBHOOK_URL; + const token = process.env.PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN; + if (!url || !token) { + console.log("PR review notification is not configured; skipped."); + return; + } + const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); + const outcomes = await notify(event, { send: (pr) => deliver(pr, { url, token }) }); + console.log(outcomes.length ? `PR review notification: ${outcomes.join(", ")}.` : "No current green PR to notify."); + } catch { + console.log("PR review notification was unavailable; verification results are unchanged."); + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) await main(); diff --git a/.github/scripts/tests/notify-pr-merge-review.test.mjs b/.github/scripts/tests/notify-pr-merge-review.test.mjs new file mode 100644 index 00000000000..dbb8dca94da --- /dev/null +++ b/.github/scripts/tests/notify-pr-merge-review.test.mjs @@ -0,0 +1,263 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { green, deliver, notify, snapshot, trustedVerification } from "../notify-pr-merge-review.mjs"; + +const head = "a".repeat(40); +const base = "b".repeat(40); +const check = (name, conclusion = "SUCCESS", extra = {}) => ({ + __typename: "CheckRun", name, status: "COMPLETED", conclusion, workflowName: "PR", + checkSuite: { databaseId: 8, app: { databaseId: 15368 }, workflowRun: { databaseId: 7 } }, + ...extra, +}); +const candidate = (extra = {}) => ({ + number: 42, state: "OPEN", isDraft: false, headRefOid: head, baseRefOid: base, + baseRefName: "master", changedFiles: 1, statusCheckRollup: [check("ci / verify"), check("ci / e2e")], ...extra, +}); +const event = (extra = {}) => ({ repository: { full_name: "paperclipai/paperclip" }, sha: head, ...extra }); +const trustedRun = (extra = {}) => ({ + id: 7, repository: { full_name: "paperclipai/paperclip" }, path: ".github/workflows/pr.yml", + event: "pull_request", head_sha: head, status: "completed", conclusion: "success", check_suite_id: 8, + referenced_workflows: [{ path: "paperclipai/paperclip/.github/workflows/pr-trusted.yml@master", + ref: "refs/heads/master", sha: base }], ...extra, +}); +const files = [{ filename: "README.md" }]; +const read = (endpoint) => endpoint.includes("/rules/") ? [] : trustedRun(); +const list = (endpoint) => endpoint.includes("/files?") ? files : [{ number: 42 }]; +const deps = (extra = {}) => ({ + read, list, view: () => candidate(), + send: async () => "delivered", ...extra, +}); + +test("reads every check page and fails closed on incomplete or changing listings", () => { + const page = (nodes, hasNextPage, extra = {}) => ({ data: { repository: { pullRequest: candidate({ + statusCheckRollup: { contexts: { totalCount: 2, nodes, + pageInfo: { hasNextPage, endCursor: "next" } } }, ...extra, + }) } } }); + let calls = 0; + const pr = snapshot(42, (args) => { + if (++calls === 1) return page([check("ci / verify")], true); + assert.ok(args.includes("endCursor=next")); + return page([check("ci / e2e")], false); + }); + assert.equal(green(pr), true); + const own = check("notify", "", { status: "IN_PROGRESS", workflowName: undefined, + checkSuite: { workflowRun: { workflow: { name: "PR merge review notification" } } } }); + const withOwn = snapshot(42, () => ({ data: { repository: { pullRequest: candidate({ + statusCheckRollup: { contexts: { totalCount: 3, nodes: [...candidate().statusCheckRollup, own], + pageInfo: { hasNextPage: false } } }, + }) } } })); + assert.equal(green(withOwn), false); + assert.throws(() => snapshot(42, () => page([check("ci / verify")], false)), /Incomplete/); + calls = 0; + assert.throws(() => snapshot(42, () => ++calls === 1 ? page([check("ci / verify")], true) + : page([check("ci / e2e")], false, { headRefOid: "d".repeat(40) })), /Candidate changed/); + assert.throws(() => snapshot(42, () => ({ errors: [{ message: "unavailable" }] })), /Incomplete/); +}); + +test("requires real success from both aggregate checks and additional repository rules", () => { + assert.equal(green(candidate()), true); + for (const state of ["FAILURE", "CANCELLED", "TIMED_OUT", "ACTION_REQUIRED", "NEUTRAL", "SKIPPED", ""]) { + assert.equal(green(candidate({ statusCheckRollup: [check("ci / verify", state), check("ci / e2e")] })), false, state); + } + assert.equal(green(candidate({ statusCheckRollup: [check("ci / verify")] })), false); + assert.equal(green(candidate(), ["extra required"]), false); + assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, check("extra required")] }), ["extra required"]), true); + assert.equal(green(candidate({ statusCheckRollup: [] })), false); +}); + +test("waits for pending checks and statuses, permits skipped optional routing lanes", () => { + for (const extra of [check("tests", "", { status: "IN_PROGRESS" }), check("tests", "FAILURE"), + { __typename: "StatusContext", context: "security", state: "PENDING" }, + { __typename: "StatusContext", context: "security", state: "ERROR" }]) { + assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, extra] })), false); + } + assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, + check("not selected", "SKIPPED"), { __typename: "StatusContext", context: "security", state: "SUCCESS" }] })), true); + assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, + check("optional scanner coverage", "NEUTRAL")] })), true); +}); + +test("workflow names cannot hide pending or failing checks", () => { + const extras = ["PR merge review notification", "CI profile"].map((workflowName) => + check("notify", "", { status: "IN_PROGRESS", workflowName })); + assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, ...extras] })), false); + assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, + check("another", "", { status: "QUEUED", workflowName: "Other verification" })] })), false); +}); + +test("rejects closed, draft, wrong-base, and malformed candidates", () => { + for (const extra of [{ state: "MERGED" }, { isDraft: true }, { baseRefName: "release" }, + { headRefOid: "invalid" }, { baseRefOid: "invalid" }]) assert.equal(green(candidate(extra)), false); +}); + +test("resolves fork PRs from the head commit when workflow payload has no PRs", async () => { + const seen = []; + const outcomes = await notify(event({ workflow_run: { + status: "completed", event: "pull_request", head_sha: head, pull_requests: [], + } }), deps({ list: (endpoint) => { seen.push(endpoint); return list(endpoint); } })); + assert.deepEqual(outcomes, ["delivered"]); + assert.match(seen[0], new RegExp(`/commits/${head}/pulls\\?per_page=100$`)); +}); + +test("reads additional required rules and holds until their check succeeds", async () => { + const outcomes = await notify(event(), deps({ read: () => [{ type: "required_status_checks", + parameters: { required_status_checks: [{ context: "new gate" }] } }] })); + assert.deepEqual(outcomes, []); +}); + +test("forged successful check names from a PR-controlled workflow cannot notify", async () => { + const input = deps({ read: (endpoint) => endpoint.includes("/rules/") ? [] : { + id: 7, path: ".github/workflows/attacker.yml", event: "pull_request", + head_sha: head, status: "completed", conclusion: "success", + } }); + assert.deepEqual(await notify(event(), input), []); +}); + +test("requires genuine suite, caller, revision, and master reusable workflow provenance", () => { + assert.equal(trustedVerification(candidate(), deps()), true); + for (const changes of [ + { path: ".github/workflows/attacker.yml" }, { event: "pull_request_target" }, + { head_sha: "c".repeat(40) }, { status: "in_progress" }, { conclusion: "failure" }, + { repository: { full_name: "attacker/repo" } }, { check_suite_id: 99 }, + { referenced_workflows: [] }, + { referenced_workflows: [{ path: "paperclipai/paperclip/.github/workflows/pr-trusted.yml@master", + ref: "refs/pull/42/merge", sha: base }] }, + ]) { + assert.equal(trustedVerification(candidate(), deps({ read: () => trustedRun(changes) })), false); + } + const wrongSuite = check("ci / verify", "SUCCESS", { + checkSuite: { databaseId: 99, app: { databaseId: 1 }, workflowRun: { databaseId: 7 } }, + }); + assert.equal(trustedVerification(candidate({ + statusCheckRollup: [wrongSuite, check("ci / e2e")], + }), deps()), false); +}); + +test("modified or renamed caller and truncated file listings cannot notify", async () => { + for (const fileList of [[], [{ filename: ".github/workflows/pr.yml" }], + [{ filename: "moved.yml", previous_filename: ".github/workflows/pr.yml" }]]) { + const dependencies = deps({ list: (endpoint) => endpoint.includes("/files?") + ? fileList : [{ number: 42 }] }); + assert.deepEqual(await notify(event(), dependencies), []); + } + assert.equal(trustedVerification(candidate({ changedFiles: undefined }), deps()), false); +}); + +test("rejects stale head events and a head, base, or check change during refresh", async () => { + assert.deepEqual(await notify(event({ sha: "c".repeat(40) }), deps()), []); + for (const extra of [{ headRefOid: "c".repeat(40) }, { baseRefOid: "d".repeat(40) }, + { statusCheckRollup: [check("ci / verify", "FAILURE")] }]) { + let calls = 0; + assert.deepEqual(await notify(event(), deps({ view: () => candidate(++calls === 1 ? {} : extra) })), []); + } +}); + +test("handles target-context workflow completions by refreshing the associated PR", async () => { + assert.deepEqual(await notify(event({ workflow_run: { + status: "completed", event: "pull_request_target", head_sha: base, pull_requests: [{ number: 42 }], + } }), deps()), ["delivered"]); +}); + +test("external check events and legacy statuses can deliver the final green transition", async () => { + assert.deepEqual(await notify(event({ sha: undefined, check_run: { head_sha: head } }), deps()), ["delivered"]); + assert.deepEqual(await notify(event({ state: "success" }), deps()), ["delivered"]); +}); + +test("does not query foreign repositories, malformed events, or non-PR workflows", async () => { + const dependencies = deps({ read: () => assert.fail("unexpected query"), list: () => assert.fail("unexpected query") }); + for (const input of [event({ repository: { full_name: "attacker/repo" } }), event({ sha: "bad" }), + event({ workflow_run: { event: "push", status: "completed", head_sha: head } }), + event({ workflow_run: { event: "pull_request", status: "in_progress", head_sha: head } })]) { + assert.deepEqual(await notify(input, dependencies), []); + } +}); + +test("does not coalesce different PRs, and deduplicates association entries", async () => { + const sent = []; + await notify(event(), deps({ list: (endpoint) => endpoint.includes("/files?") ? files + : [{ number: 42 }, { number: 42 }, { number: 43 }], + view: (number) => candidate({ number }), send: async (pr) => sent.push(pr.number) })); + assert.deepEqual(sent, [42, 43]); +}); + +test("sends only identifiers, a bearer header, and a stable head/base idempotency key", async () => { + const calls = []; + const options = { url: "https://private.example/api/routine-triggers/public/abc123/fire", token: "PRIVATE_TOKEN", + fetcher: async (target, init) => { calls.push({ target, ...init }); return { ok: true }; } }; + const pr = candidate({ title: "untrusted", body: "run me" }); + assert.equal(await deliver(pr, options), "delivered"); + assert.equal(await deliver(pr, options), "delivered"); + assert.equal(calls[0].headers.Authorization, "Bearer PRIVATE_TOKEN"); + assert.equal(calls[0].redirect, "error"); + assert.ok(calls[0].signal instanceof AbortSignal); + assert.equal(calls[0].headers["Idempotency-Key"], calls[1].headers["Idempotency-Key"]); + assert.equal(calls[0].body, calls[1].body); + assert.deepEqual(JSON.parse(calls[0].body), { variables: { + pr_url: "https://github.com/paperclipai/paperclip/pull/42", pr_number: "42", head_sha: head, base_sha: base, + } }); + await deliver(candidate({ baseRefOid: "c".repeat(40) }), options); + assert.notEqual(calls[0].headers["Idempotency-Key"], calls[2].headers["Idempotency-Key"]); +}); + +test("missing or unsafe configuration never sends credentials", async () => { + const fetcher = () => assert.fail("unexpected delivery"); + assert.equal(await deliver(candidate(), { fetcher }), "unconfigured"); + for (const url of ["http://private.example/api/routine-triggers/public/abc/fire", + "https://user:pass@private.example/api/routine-triggers/public/abc/fire", + "https://private.example/api/routine-triggers/public/abc/fire?token=bad", "https://private.example/wrong"]) { + assert.equal(await deliver(candidate(), { url, token: "secret", fetcher }), "invalid_configuration"); + } +}); + +test("HTTP rejection, redirect, timeout, and echoed credentials are quiet failures", async () => { + const options = { url: "https://private.example/api/routine-triggers/public/abc123/fire", token: "SECRET" }; + assert.equal(await deliver(candidate(), { ...options, fetcher: async () => ({ ok: false, + text: () => assert.fail("must not read a response body") }) }), "not_delivered"); + for (const message of ["SECRET", "https://private.example", "redirect", "timeout"]) { + assert.equal(await deliver(candidate(), { ...options, fetcher: async () => { throw new Error(message); } }), "not_delivered"); + } +}); + +test("command failures cannot print a private URL, token, or event payload", () => { + const directory = mkdtempSync(path.join(tmpdir(), "pr-notifier-test-")); + try { + const file = path.join(directory, "event.json"); + writeFileSync(file, "PRIVATE_EVENT_INVALID_JSON"); + const run = spawnSync(process.execPath, [".github/scripts/notify-pr-merge-review.mjs"], { + encoding: "utf8", env: { ...process.env, GITHUB_EVENT_PATH: file, + PAPERCLIP_PR_REVIEW_WEBHOOK_URL: "PRIVATE_URL_SENTINEL", PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN: "PRIVATE_TOKEN_SENTINEL" }, + }); + assert.equal(run.status, 0); + assert.doesNotMatch(run.stdout + run.stderr, /PRIVATE_/); + assert.match(run.stdout, /verification results are unchanged/); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); + +test("workflow uses read permissions, a protected environment, trusted checkout, and optional delivery", () => { + const workflow = readFileSync(".github/workflows/pr-merge-review.yml", "utf8"); + assert.match(workflow, /environment: pr-merge-review/); + assert.match(workflow, /ref: \$\{\{ github.sha \}\}/); + assert.match(workflow, /persist-credentials: false/); + assert.match(workflow, /continue-on-error: true/); + for (const permission of ["actions", "checks", "contents", "pull-requests", "statuses"]) { + assert.ok(workflow.includes(` ${permission}: read`), `Missing ${permission}: read`); + } + assert.doesNotMatch(workflow, /: write|download-artifact|restore@|pull_request_target:/); + for (const name of ["PAPERCLIP_PR_REVIEW_WEBHOOK_URL", "PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN"]) { + assert.ok(workflow.includes(`secrets.${name}`)); + assert.ok(!workflow.includes(`vars.${name}`)); + } +}); + +test("API probe uses the same read permissions without private webhook access", () => { + const probe = readFileSync(".github/workflows/pr-review-api-access.yml", "utf8"); + for (const permission of ["actions", "checks", "contents", "pull-requests", "statuses"]) { + assert.ok(probe.includes(` ${permission}: read`)); + } + assert.match(probe, /snapshot\(Number\(process.env.PR_NUMBER\)\)/); + assert.doesNotMatch(probe, /\$\{\{ secrets\.|environment:|: write|deliver\(/); +}); diff --git a/.github/workflows/pr-merge-review.yml b/.github/workflows/pr-merge-review.yml new file mode 100644 index 00000000000..87c08abbc44 --- /dev/null +++ b/.github/workflows/pr-merge-review.yml @@ -0,0 +1,57 @@ +name: PR merge review notification + +on: + workflow_run: + workflows: + - PR + - Runner Windows + - Sentry SDK contract + - Docker Runner check + - commitperclip PR Review + - Storybook Visual + - Storybook Deploy + - CI work benchmark + - Incremental CI qualification + - Backup checkpoint qualification + - PR review API access + types: [completed] + # External check providers and legacy commit statuses can finish after CI. + check_run: + types: [completed] + status: + +permissions: + actions: read + checks: read + contents: read + pull-requests: read + statuses: read + +jobs: + notify: + if: >- + github.repository == 'paperclipai/paperclip' && + (github.event_name != 'workflow_run' || + github.event.workflow_run.event == 'pull_request' || + github.event.workflow_run.event == 'pull_request_target') + runs-on: ubuntu-latest + timeout-minutes: 5 + # Restrict this environment to master. Repository secrets would also be + # available to workflows changed by an in-repository PR. + environment: pr-merge-review + continue-on-error: true + steps: + # Only the trusted default-branch commit. Never use the PR SHA, caches, + # artifacts, dependencies, or executable files from a pull request. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: .github/scripts + - name: Notify the advisory review routine + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PAPERCLIP_PR_REVIEW_WEBHOOK_URL: ${{ secrets.PAPERCLIP_PR_REVIEW_WEBHOOK_URL }} + PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN: ${{ secrets.PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN }} + run: node .github/scripts/notify-pr-merge-review.mjs diff --git a/.github/workflows/pr-review-api-access.yml b/.github/workflows/pr-review-api-access.yml new file mode 100644 index 00000000000..cdf916fa29e --- /dev/null +++ b/.github/workflows/pr-review-api-access.yml @@ -0,0 +1,43 @@ +name: PR review API access + +on: + pull_request: + paths: + - .github/scripts/notify-pr-merge-review.mjs + - .github/scripts/tests/notify-pr-merge-review.test.mjs + - .github/workflows/pr-merge-review.yml + - .github/workflows/pr-review-api-access.yml + +permissions: + actions: read + checks: read + contents: read + pull-requests: read + statuses: read + +jobs: + probe: + if: github.repository == 'paperclipai/paperclip' + runs-on: ubuntu-latest + timeout-minutes: 5 + # This job has no environment or private webhook secrets. It proves that + # the notifier's status query works with the actual read-only Actions token. + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + sparse-checkout: .github/scripts + - name: Read the PR status rollup with the Actions token + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + node --input-type=module <<'JS' + import assert from 'node:assert/strict'; + import { snapshot } from './.github/scripts/notify-pr-merge-review.mjs'; + const pr = snapshot(Number(process.env.PR_NUMBER)); + assert.equal(pr.headRefOid, process.env.PR_HEAD_SHA); + assert.ok(pr.statusCheckRollup.length > 0); + console.log('Actions token can read the complete PR status rollup.'); + JS diff --git a/doc/PR-MERGE-REVIEW-WEBHOOK.md b/doc/PR-MERGE-REVIEW-WEBHOOK.md new file mode 100644 index 00000000000..6526aca0999 --- /dev/null +++ b/doc/PR-MERGE-REVIEW-WEBHOOK.md @@ -0,0 +1,85 @@ +# Advisory PR review notifications + +The optional `PR merge review notification` workflow sends a private webhook +after a current, open, non-draft PR targeting `master` passes verification. +It does not approve or merge a PR. Webhook failures do not change PR CI results. + +## Private configuration + +Create a GitHub Actions environment named `pr-merge-review`. Restrict its +deployment branch policy to the `master` branch only, with no tag policy. +Keep these two values in **environment secrets**, not repository variables, +repository secrets, committed files, task comments, or workflow outputs: + +- `PAPERCLIP_PR_REVIEW_WEBHOOK_URL`: a Paperclip routine webhook URL. +- `PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN`: its separate bearer signing secret. + +Use a webhook trigger with `signingMode: bearer`. The routine should use +`always_enqueue` concurrency so separate PRs cannot be coalesced into one task. +Define required text variables `pr_url`, `pr_number`, `head_sha`, and `base_sha`. +The notifier sends only those identifiers. The receiving agent must treat them +as untrusted data and verify the repository, PR identity, and current revision. + +Give the routine instructions to run `pr-merge-review` and publish its MERGE, +HOLD, or DO NOT MERGE recommendation on the routine execution's Paperclip task. +Require a fresh check of the head, base, reviews, required checks, and task gates. +Keep GitHub approval, merge, auto-merge, and public-comment actions disabled in +the instructions for this advisory phase. Configure the routine in Paperclip; +do not put instance-specific URLs or credentials in this repository. + +## Delivery and verification + +The notification workflow runs trusted default-branch code. It never checks out +PR code or downloads PR artifacts or caches. It reacts to the repository's PR +workflow completions, external check completions, and legacy commit statuses. +When adding another PR verification workflow, add its name to `workflow_run`. + +The separate `PR review API access` workflow checks notifier changes using +the real read-only Actions token. It has no private environment or webhook +credentials and sends no webhook. Both workflows explicitly grant +`actions`, `checks`, `contents`, `pull-requests`, and `statuses` read access. +The GraphQL status rollup needs both check and commit-status permissions. + +The notifier reads the current PR's status rollup and branch rules. Both +`ci / verify` and `ci / e2e`, plus any additional required checks, must succeed. +Other checks must succeed or finish as skipped or neutral. Optional skipped or +neutral checks permit an assessment; they do not prove successful verification. +For example, a neutral scanner can mean coverage is unavailable. The advisory +routine evaluates these evidence gaps and reviews before recommending merge. +Workflow names do not exclude checks. The notifier and CI profile exporter run +on the default branch, so their own checks do not belong to the PR revision. + +Check names alone are not trusted. Both core aggregates must belong to the same +GitHub Actions suite and a successful run of the repository's actual +`.github/workflows/pr.yml`. Its run metadata must reference +`pr-trusted.yml@master`. The complete PR file list must prove that the caller +`pr.yml` was not changed or renamed. A missing or truncated list stops delivery. +PRs that change this caller need a manual advisory review until their caller +change reaches the trusted branch. Other workflow changes can still be reviewed. + +A final refresh rejects head or base changes during evaluation. Paperclip's +idempotency receipt deduplicates events for the same PR, head, and base. A new +base can produce a new review. The routine must refresh again before its final +recommendation. Event delivery and host check visibility can race; notifications +are best effort, with no polling or delivery guarantee. + +The HTTP request uses TLS, a bearer header, no redirects, and a 15 second timeout. +Logs contain fixed delivery states only. Request and response bodies, URLs, +tokens, and raw exceptions are never logged. Missing secrets skip delivery. + +Run the focused regression checks with: + +```sh +node --test .github/scripts/tests/notify-pr-merge-review.test.mjs +``` + +For setup verification, use Paperclip's webhook `setupPending` test mode before +activation. This verifies authentication and ingress without starting a review. +After the workflow is merged to `master`, inspect a real successful workflow +delivery and its linked routine task to prove end-to-end operation. A saved +configuration or a setup test alone does not prove that journey. + +To pause reviews, pause the routine in Paperclip or disable its trigger. To stop +notifications, disable this workflow or remove the two environment secrets. +Rotate the trigger signing secret in Paperclip and replace the token environment +secret together. Keep the URL secret even though it has no embedded bearer token. From 6148fbd31f587941e4d83933c9596f0d032ded52 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:26:10 -0600 Subject: [PATCH 09/11] feat(ui): distinguish GitHub resources in inline Markdown links (#19476) ## Thinking Path > - Paperclip helps people manage AI agents and their work. > - Task conversations and documents use the shared Markdown renderer. > - GitHub repository, pull request, and issue links currently use the same icon. > - Long pasted URLs make these references hard to scan in a conversation. > - This change identifies each resource from its URL and shows a compact label with a distinct icon. > - The exact destination and custom link text stay intact. Connected status data keeps its existing behavior. ## Linked Issues or Issue Description **What existing behavior does this improve?** Inline GitHub links in rendered Markdown. **Current behavior** Repositories, pull requests, and issues all use the GitHub mark. Pasted resource links show the full URL. **Proposed behavior** Pull requests use the pull request icon. Issues use the issue icon. Repositories and other GitHub pages keep the brand mark. Pasted resource URLs show `owner/repository#number` or `owner/repository`. Custom labels keep their text and formatting. **Reason and benefit** Readers can identify GitHub resource types and scan references in a narrow conversation. Full URLs, query strings, and comment anchors stay available through the link destination. Link rendering makes no remote metadata requests. **Breaking changes** The visible text of a bare GitHub resource URL becomes compact. The destination does not change. No API, database, or connection changes are required. Searches found no duplicate work for GitHub inline resource presentation. Related work: [same-origin Markdown routing](https://github.com/paperclipai/paperclip/pull/17864). This change preserves that route handling. ## What Changed - Add a pure GitHub URL classifier with exact host checks and resource labels. - Move ordinary URL presentation into `MarkdownUrlLink`. Keep internal references, workspace links, custom resolvers, and connected metadata in their existing order. - Add separate icons and accessible resource names. Preserve authored Markdown titles and formatted labels. - Add Playground, Narrow, and Streaming Storybook examples that mount `MarkdownBody`. - Add parser and renderer tests, eight Chromium browser cases, and the browser command in the development guide. - Consider Open Graph loading. Use the existing connected metadata path for remote status and titles. Add no arbitrary page fetches. ## Verification - Focused Vitest coverage: 129 passing tests across nine files for GitHub parsing, Markdown rendering, connected metadata, document rendering, and workspace-file rendering. - `pnpm check:token-gates`. - `pnpm --filter @paperclipai/ui typecheck`. - `pnpm --filter @paperclipai/ui build`. - `pnpm build-storybook`. - Storybook catalog generation/check and 30 catalog/map tests. - Site-route regression: Stars, Education, and enterprise pages keep general GitHub branding and their full URL text. Three parser cases failed before the fix; parser and renderer cases now pass. - Streaming regression: three resource cases failed before the fix and pass after it. The fix preserves the generated word span and custom formatted labels. - Chromium: `STORYBOOK_TEST_BASE_URL=http://localhost:6106 pnpm exec playwright test inline-links.spec.ts --config=tests/storybook-visual/playwright.config.ts --project=stories`. - Browser scope: eight passing cases for ordinary/streaming modes, dark/light at desktop and 390px mobile, completed play assertions, exact comment destinations, safe external attributes, keyboard focus, wrapping, and no GitHub requests. Screenshots are saved as task artifacts; no review images are committed. - The first browser attempt in the resumed environment could not launch Chromium because its executable was missing. The browser was installed, and all eight cases then passed. - The browser cases use synthetic text and one connected metadata fixture. They do not prove provider authentication or server persistence. No visual baseline comparison is claimed; the repository baseline manifest is unpublished. - Initial typecheck found two unsupported story assertion options and an unbuilt SDK dependency in the fresh checkout. The options were removed and the SDK was built before the successful retry. - `pnpm verify` returned 2 because the new browser spec requires broad CI coverage. Focused tests and the browser suite supply local evidence. Required CI supplies broad coverage. ## Risks - Compact labels hide URL detail from the visible line. The exact href and default tooltip preserve that detail. - GitHub site routes can change. Unknown paths retain the general GitHub treatment. Lookalike hosts and credential-bearing URLs do not get GitHub resource treatment. - Connected external-object status and title rendering keeps priority over URL-derived presentation. ## Model Used OpenAI Codex, GPT-6 (`gpt-6` family), with reasoning, tool use, browser checks, and code execution. The exact deployment ID and context window are not exposed in this run. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/DEVELOPING.md | 10 ++ tests/storybook-visual/inline-links.spec.ts | 44 +++++++++ ui/src/components/MarkdownBody.test.tsx | 92 +++++++++++++++--- ui/src/components/MarkdownBody.tsx | 96 +------------------ ui/src/components/MarkdownUrlLink.tsx | 66 +++++++++++++ ui/src/lib/github-link.test.ts | 38 ++++++++ ui/src/lib/github-link.ts | 52 ++++++++++ .../stories/tasks/inline-links.stories.tsx | 91 ++++++++++++++++++ 8 files changed, 385 insertions(+), 104 deletions(-) create mode 100644 tests/storybook-visual/inline-links.spec.ts create mode 100644 ui/src/components/MarkdownUrlLink.tsx create mode 100644 ui/src/lib/github-link.test.ts create mode 100644 ui/src/lib/github-link.ts create mode 100644 ui/storybook/stories/tasks/inline-links.stories.tsx diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index fd93ba96369..53215d9dd47 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -522,6 +522,16 @@ pnpm build-storybook These run the `@paperclipai/ui` Storybook on port `6006` and build the static output to `ui/storybook-static/`. +**Inline links** mounts the shared Markdown renderer with GitHub repository, +pull request, and issue references, custom labels, connected status metadata, +generic URL fallbacks, and streaming playback. Pasted GitHub resource URLs use compact labels; +the exact destination, query, and fragment stay intact. The presentation makes +no remote metadata requests. Run its light/dark desktop/mobile checks with: + +```sh +STORYBOOK_TEST_BASE_URL=http://localhost:6006 pnpm exec playwright test inline-links.spec.ts --config=tests/storybook-visual/playwright.config.ts --project=stories +``` + **Task chat → Failure notices** covers compact run errors and adjacent recovery notices in both presentation modes, with expanded details, fixture retry callbacks (including the legacy run retry when recovery retry is unavailable), diff --git a/tests/storybook-visual/inline-links.spec.ts b/tests/storybook-visual/inline-links.spec.ts new file mode 100644 index 00000000000..3a32d13e539 --- /dev/null +++ b/tests/storybook-visual/inline-links.spec.ts @@ -0,0 +1,44 @@ +import { expect, test } from "@playwright/test"; + +for (const theme of ["dark", "light"]) { + for (const mobile of [false, true]) { + for (const streaming of [false, true]) { + test(`GitHub inline links [${theme}, ${mobile ? "mobile" : "desktop"}, ${streaming ? "streaming" : "ordinary"}]`, async ({ page }, testInfo) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors: string[] = []; + const providerRequests: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + page.on("request", (request) => { + if (new URL(request.url()).hostname === "github.com") providerRequests.push(request.url()); + }); + const id = `tasks-inline-links--${streaming ? "streaming" : mobile ? "narrow" : "playground"}`; + await page.goto(`/iframe.html?id=${id}&viewMode=story&globals=theme:${theme}`); + await expect.poll(() => page.evaluate(() => { + const channel = (window as unknown as { + __STORYBOOK_ADDONS_CHANNEL__?: { data: Record> }; + }).__STORYBOOK_ADDONS_CHANNEL__; + return channel?.data.storyFinished?.[0]?.status; + })).toBe("success"); + const main = page.getByRole("main"); + const pr = main.getByRole("link", { name: "GitHub pull request: acme/web#241", exact: true }); + await expect(pr).toHaveAttribute("rel", "noopener noreferrer"); + if (streaming) { + await expect(pr.locator(".conversation-stream-word")).toHaveText("acme/web#241"); + } + const comment = main.getByRole("link", { name: "GitHub pull request: Jump to the review comment" }); + await expect(comment).toHaveAttribute("href", "https://github.com/acme/web/pull/241/files?diff=split#discussion_r123"); + await expect(comment).toHaveAttribute("title", "Review comment on the exact diff"); + await pr.focus(); + await expect(pr).toBeFocused(); + await pr.blur(); + const settings = main.getByRole("link", { name: "company settings" }); + await expect(settings).not.toHaveAttribute("target"); + await expect(main.locator("a[data-github-link] .lucide-external-link")).toHaveCount(0); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + expect(errors).toEqual([]); + expect(providerRequests).toEqual([]); + await page.screenshot({ path: testInfo.outputPath(`inline-links-${theme}-${mobile ? "mobile" : "desktop"}-${streaming ? "streaming" : "ordinary"}.png`), fullPage: true }); + }); + } + } +} diff --git a/ui/src/components/MarkdownBody.test.tsx b/ui/src/components/MarkdownBody.test.tsx index 827943ecac0..dd5d6163149 100644 --- a/ui/src/components/MarkdownBody.test.tsx +++ b/ui/src/components/MarkdownBody.test.tsx @@ -15,6 +15,7 @@ import { } from "@paperclipai/shared"; import { ThemeProvider } from "../context/ThemeContext"; import { MarkdownBody } from "./MarkdownBody"; +import { MarkdownWordStreaming } from "./MarkdownWordStreaming"; import { queryKeys } from "../lib/queryKeys"; import type { WorkspaceFileAvailabilityTarget } from "../lib/workspace-file-availability"; @@ -66,6 +67,7 @@ function renderMarkdown( children: string, seededIssues: Array<{ identifier: string; status: string; title?: string; cacheKey?: string }> = [], props: Partial> = {}, + streaming = false, ) { const queryClient = new QueryClient({ defaultOptions: { @@ -87,7 +89,9 @@ function renderMarkdown( return renderToStaticMarkup( - {children} + + {children} + , ); @@ -607,23 +611,85 @@ describe("MarkdownBody", () => { expect(html).toContain('rel="noreferrer"'); }); - it("prefixes GitHub markdown links with the GitHub icon glued to the first character", () => { - const html = renderMarkdown("[https://github.com/paperclipai/paperclip/pull/4099](https://github.com/paperclipai/paperclip/pull/4099)"); + it("gives pull request URLs a compact label and a distinct icon", () => { + const href = "https://github.com/paperclipai/paperclip/pull/4099"; + const html = renderMarkdown(`[${href}](${href})`); - expect(html).toContain('.*lucide-github.*?<\/svg>h<\/span>/); - expect(html).toContain("ttps://github.com/paperclipai/paperclip/pull/4099"); + expect(html).toContain(`.*lucide-git-pull-request.*?<\/svg>p<\/span>/); + expect(html.replace(/<[^>]*>/g, "")).toContain("paperclipai/paperclip#4099"); expect(html).not.toContain("lucide-external-link"); }); - it("prefixes GitHub autolinks with the GitHub icon", () => { - const html = renderMarkdown("See https://github.com/paperclipai/paperclip/issues/1778"); + it("gives issue autolinks their own icon and keeps the exact destination", () => { + const html = renderMarkdown("See https://github.com/paperclipai/paperclip/issues/1778?source=chat#issuecomment-123"); - expect(html).toContain(']*>/g, "")).toContain("paperclipai/paperclip#1778"); + }); + + it.each([ + ["https://github.com/acme/web/pull/241", "acme/web#241"], + ["https://github.com/acme/web/issues/42", "acme/web#42"], + ["https://github.com/acme/web", "acme/web"], + ])("compacts streaming URL text while preserving its word wrapper: %s", (href, label) => { + const html = renderMarkdown(href, [], {}, true); + expect(html).toContain(`class="conversation-stream-word">${label}`); + expect(html).toContain(`href="${href}"`); + expect(html.replace(/<[^>]*>/g, "")).not.toContain(href); + }); + + it("preserves custom streaming labels and formatted URL labels", () => { + const href = "https://github.com/acme/web/pull/241"; + const html = renderMarkdown(`[Review **the PR**](${href}) and [**${href}**](${href})`, [], {}, true); + expect(html).toContain(""); + expect(html).toContain('class="conversation-stream-word"'); + const text = html.replace(/<[^>]*>/g, ""); + expect(text).toContain("Review the PR"); + expect(text).toContain(href); + expect(text).not.toContain("acme/web#241"); + }); + + it("keeps custom labels, formatted text, and authored titles", () => { + const html = renderMarkdown('[**Review** the PR](https://github.com/acme/web/pull/241 "Ready for review")'); + expect(html).toContain("Review the PR"); + expect(html).toContain('title="Ready for review"'); + expect(html).toContain('data-github-link="pull_request"'); + expect(html).toContain("lucide-git-pull-request"); + expect(html).not.toContain("acme/web#241"); + }); + + it("keeps GitHub branding for repositories and other GitHub pages", () => { + const html = renderMarkdown("https://github.com/acme/web and [GitHub Actions](https://github.com/features/actions)"); + expect(html).toContain('data-github-link="repository"'); + expect(html.replace(/<[^>]*>/g, "")).toContain("acme/web"); + expect(html).toContain('data-github-link="github"'); + expect(html).toContain("lucide-github"); + }); + + it.each([ + "https://github.com/stars/torvalds", + "https://github.com/education/teachers", + "https://github.com/enterprises/acme", + ])("keeps GitHub site pages as branded full URL links: %s", (href) => { + const html = renderMarkdown(href); + expect(html).toContain('data-github-link="github"'); + expect(html).not.toContain('data-github-link="repository"'); + expect(html.replace(/<[^>]*>/g, "")).toContain(href); + expect(html).toContain(`href="${href}"`); + }); + + it("keeps full URL labels on lookalike hosts", () => { + const html = renderMarkdown("https://github.com.example.com/acme/web/pull/241"); + expect(html).not.toContain("data-github-link"); + expect(html).toContain("lucide-external-link"); + expect(html.replace(/<[^>]*>/g, "")).toContain("https://github.com.example.com/acme/web/pull/241"); }); ; diff --git a/ui/src/components/MarkdownBody.tsx b/ui/src/components/MarkdownBody.tsx index 2063d7485af..1d9d7cca954 100644 --- a/ui/src/components/MarkdownBody.tsx +++ b/ui/src/components/MarkdownBody.tsx @@ -1,7 +1,7 @@ import { MarkdownWordStreaming, rehypeStreamingWords } from "./MarkdownWordStreaming"; import { isValidElement, memo, useContext, useCallback, useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { Check, Copy, ExternalLink, MessageCircle, WrapText } from "lucide-react"; +import { Check, Copy, MessageCircle, WrapText } from "lucide-react"; import { useCoordinatorPresentation } from "./coordinator/CoordinatorPresentation"; import { useCoordinatorChatReference } from "./coordinator/CoordinatorChatReferences"; import { SidebarDocumentLink } from "./SidebarDocumentLink"; @@ -9,7 +9,7 @@ import { parseDocumentAnnotationHash } from "@/lib/document-annotation-hash"; import Markdown, { defaultUrlTransform, type Components, type Options } from "react-markdown"; import remarkGfm from "remark-gfm"; import { cn } from "../lib/utils"; -import { GithubIcon } from "@/components/icons/github-icon"; +import { MarkdownUrlLink } from "./MarkdownUrlLink"; import { Link, useCaseHref } from "@/lib/router"; import { useTheme } from "../context/ThemeContext"; import { useOptionalCompany } from "../context/CompanyContext"; @@ -572,78 +572,6 @@ function createRemarkWikiLinks(options: { }; } -function isGitHubUrl(href: string | null | undefined): boolean { - if (!href) return false; - try { - const url = new URL(href); - return url.protocol === "https:" && (url.hostname === "github.com" || url.hostname === "www.github.com"); - } catch { - return false; - } -} - -function isExternalHttpUrl(href: string | null | undefined): boolean { - if (!href) return false; - try { - const url = new URL(href); - if (url.protocol !== "http:" && url.protocol !== "https:") return false; - if (typeof window === "undefined") return true; - return url.origin !== window.location.origin; - } catch { - return false; - } -} - -function renderLinkBody( - children: ReactNode, - leadingIcon: ReactNode, - trailingIcon: ReactNode, -): ReactNode { - if (!leadingIcon && !trailingIcon) return children; - - // React-markdown can pass arrays/elements for styled link text; the nowrap - // splitting below is intentionally limited to plain text links. - if (typeof children === "string" && children.length > 0) { - if (children.length === 1) { - return ( - - {leadingIcon} - {children} - {trailingIcon} - - ); - } - const first = children[0]; - const last = children[children.length - 1]; - const middle = children.slice(1, -1); - return ( - <> - {leadingIcon ? ( - - {leadingIcon} - {first} - - ) : first} - {middle} - {trailingIcon ? ( - - {last} - {trailingIcon} - - ) : last} - - ); - } - - return ( - <> - {leadingIcon} - {children} - {trailingIcon} - - ); -} - function CodeBlock({ children, preProps, @@ -1009,24 +937,10 @@ function MarkdownBodyImpl({ ) : <>{linkChildren}; } - const isGitHubLink = isGitHubUrl(href); - const isExternal = isExternalHttpUrl(href); - const leadingIcon = isGitHubLink ? ( - - {renderLinkBody(linkChildren, leadingIcon, trailingIcon)} - + + {linkChildren} + ); }, }; diff --git a/ui/src/components/MarkdownUrlLink.tsx b/ui/src/components/MarkdownUrlLink.tsx new file mode 100644 index 00000000000..79995ecff2f --- /dev/null +++ b/ui/src/components/MarkdownUrlLink.tsx @@ -0,0 +1,66 @@ +import { cloneElement, isValidElement, type CSSProperties, type ReactNode } from "react"; +import { CircleDot, ExternalLink, GitPullRequest } from "lucide-react"; +import { GithubIcon } from "./icons/github-icon"; +import { parseGitHubLink } from "@/lib/github-link"; + +function isExternalHttpUrl(href: string | undefined): boolean { + if (!href) return false; + try { + const url = new URL(href); + if (url.protocol !== "http:" && url.protocol !== "https:") return false; + return typeof window === "undefined" || url.origin !== window.location.origin; + } catch { + return false; + } +} + +function renderLinkBody(children: ReactNode, leadingIcon: ReactNode, trailingIcon: ReactNode): ReactNode { + if (!leadingIcon && !trailingIcon) return children; + if (typeof children !== "string" || !children.length) return <>{leadingIcon}{children}{trailingIcon}; + if (children.length === 1) { + return {leadingIcon}{children}{trailingIcon}; + } + return <> + {leadingIcon ? {leadingIcon}{children[0]} : children[0]} + {children.slice(1, -1)} + {trailingIcon ? {children.at(-1)}{trailingIcon} : children.at(-1)} + ; +} + +function compactUrlLabel(children: ReactNode, href: string | undefined, label: string): ReactNode { + if (children === href) return label; + const child = Array.isArray(children) && children.length === 1 ? children[0] : children; + if ( + isValidElement<{ className?: string; children?: ReactNode }>(child) + && child.type === "span" + && child.props.className?.split(/\s+/).includes("conversation-stream-word") + && child.props.children === href + ) { + return cloneElement(child, undefined, label); + } + return children; +} + +export function MarkdownUrlLink({ href, children, title, style }: { + href?: string; + children: ReactNode; + title?: string; + style?: CSSProperties; +}) { + const github = parseGitHubLink(href); + const external = isExternalHttpUrl(href); + const Icon = github?.kind === "pull_request" ? GitPullRequest : github?.kind === "issue" ? CircleDot : GithubIcon; + const label = github?.label ? compactUrlLabel(children, href, github.label) : children; + const leadingIcon = github ?