From eb12cfc96b18950d4b582ba38a47adddf3458b12 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:38:43 -0500 Subject: [PATCH] perf(github): activate connection work from events and deadlines (#18395) Reviewed and verified at f77620b9a4b76cc05508612ee894a20b751651bb. Co-Authored-By: Paperclip --- .../2026-10-30-github-connection-sleep.md | 141 ++++ packages/db/src/schema/tool_access.ts | 2 + packages/shared/src/types/tool-access.ts | 3 + packages/shared/src/validators/tool-access.ts | 2 + .../cloud-control-task-drain.test.ts | 22 +- .../__tests__/github-catalog-timing.test.ts | 14 +- .../github-connection-enrollment-work.test.ts | 139 ++++ .../github-connection-events.test.ts | 71 +- .../__tests__/github-connection-work.test.ts | 726 ++++++++++++++++++ server/src/__tests__/openapi-routes.test.ts | 4 + .../server-startup-feedback-export.test.ts | 48 +- server/src/app.ts | 2 + server/src/index.ts | 46 +- server/src/middleware/cloud-control.ts | 1 + .../modules/wake-queue/adapters/postgres.ts | 2 +- server/src/routes/connection-event-wake.ts | 13 + server/src/services/agent-configuration.ts | 2 +- server/src/services/agent-deletion.ts | 2 +- server/src/services/agent-lifecycle.ts | 2 +- server/src/services/budgets.ts | 8 +- .../configuration-work-notifications.ts | 6 +- .../services/chat-channels/configuration.ts | 8 + server/src/services/chat-github-wizard.ts | 5 +- server/src/services/cloud-runtime-identity.ts | 1 + server/src/services/companies.ts | 5 +- server/src/services/company-deletion.ts | 2 +- server/src/services/connection-event-wake.ts | 12 + .../services/delivery-work-notifications.ts | 2 + .../execution-control-notifications.ts | 14 +- .../services/github-connection-events.test.ts | 46 +- .../src/services/github-connection-events.ts | 170 ++-- .../github-connection-notifications.ts | 26 + server/src/services/github-connection-work.ts | 39 + server/src/services/heartbeat/queue.ts | 4 +- server/src/services/instance-settings.ts | 2 +- .../issue-review-state-notifications.ts | 8 +- .../src/services/issue-thread-interactions.ts | 16 +- server/src/services/issue-title.ts | 4 +- server/src/services/issue-tree-control.ts | 2 +- server/src/services/issues.ts | 24 +- .../native-review-participant.ts | 4 +- .../native-workspace-reconstruction.ts | 6 +- ...ative-status-reassessment-notifications.ts | 4 +- .../src/services/paperclip-cloud-connector.ts | 12 +- .../src/services/provider-retry-after.test.ts | 26 + server/src/services/provider-retry-after.ts | 6 + .../services/slack-conversation-lifecycle.ts | 2 +- .../src/services/slack-conversation-state.ts | 2 +- server/src/services/tool-access.ts | 13 +- .../tool-access/connections/grants.ts | 2 + .../tool-access/connections/removal.ts | 2 + .../tool-access/github-access-discovery.ts | 2 + .../tool-access/github-grant-continuity.ts | 286 ++++--- .../tool-access/oauth/access-finalization.ts | 2 + .../tool-access/oauth/cloud-callback.ts | 61 +- .../tool-access/oauth/grant-refresh.ts | 35 +- .../tool-access/oauth/refresh-leases.ts | 21 +- 57 files changed, 1776 insertions(+), 356 deletions(-) create mode 100644 doc/plans/2026-10-30-github-connection-sleep.md create mode 100644 server/src/__tests__/github-connection-enrollment-work.test.ts create mode 100644 server/src/__tests__/github-connection-work.test.ts create mode 100644 server/src/routes/connection-event-wake.ts create mode 100644 server/src/services/connection-event-wake.ts create mode 100644 server/src/services/github-connection-notifications.ts create mode 100644 server/src/services/github-connection-work.ts create mode 100644 server/src/services/provider-retry-after.test.ts create mode 100644 server/src/services/provider-retry-after.ts diff --git a/doc/plans/2026-10-30-github-connection-sleep.md b/doc/plans/2026-10-30-github-connection-sleep.md new file mode 100644 index 00000000000..287b479b07c --- /dev/null +++ b/doc/plans/2026-10-30-github-connection-sleep.md @@ -0,0 +1,141 @@ +# GitHub connection delivery and credential continuity + +GitHub event delivery and credential continuity have separate obligations. +The event worker uses the shared delivery coordinator. It performs a startup +reconciliation and responds to committed configuration changes or authenticated +Cloud hints. Credential work registers the actual next grant expiry, provider +continuity check, or saved retry deadline. The aggregate heartbeat no longer +calls either service, including when the heartbeat scheduler is disabled. + +## Event delivery contract + +The Cloud webhook target queue remains durable truth. The tenant leases sealed +events, validates their bindings, persists company-scoped receipts, applies the +existing installation or pull-request behavior, and acknowledges completed work. +A receipt row lock prevents overlapping receivers from applying an installation +delta twice. Completed receipts remain passive history. The existing normalized +merged-pull-request callback keeps its normalized key. Snapshot and confirmation +reads/writes share the receipt transaction, including at pool size one; existing +confirmation publications and direct wakes flush only after the outer commit. +The pre-existing non-chat direct-wake crash gap remains: chat-bound continuations +have their existing durable outbox, while an unbound direct wake is best effort. + +A new tenant sends signed event-lease claim `ewv=1`. A compatible Cloud broker +records delivery version 1 on the existing instance registration before returning +`eventWakeVersion=1`. Only then does the tenant stop configured-empty polling. +Cloud reconstructs unacknowledged targets after restart and retries the exact +registered tenant when its current queue lease permits another delivery. A +successful HTTP hint does not acknowledge the target. Missing, stopped or drained +tenants therefore retain their work. The hint carries no provider payload or +client-selected destination. The tenant checks a signed, stack-bound, single-use +`connector-events:wake` control assertion on the exact POST endpoint. + +Old Cloud and self-hosted registrations retain scoped reconciliation with the +existing backoff. No connector configuration or no registration history settles +to no recurring owner SQL or provider traffic. Disabled consumers can acknowledge +queued events as ignored using their trusted local registration history; they +cannot apply events to an inactive connection. Removing polling does not remove +the host sleep safety blockers. Worker quiet is not proof of host suspension. + +## Durable credential debt + +A grant stores a unique `bindingSync` revision and removed installation IDs before +Cloud registration calls. Successful delivery clears only the locked current +revision. Failure retains the saved intent and an exponential retry deadline, +including provider Retry-After advice. A restart loads that deadline from the +grant. A retry write compares the attempted credential/installation revision under +the grant lock and cannot delay work committed by a newer revision. Restoration +creates a new binding revision and removes active installations from the pending +retirement list; binding commands also reject legacy conflicting retirements. +Binding-only retries use the existing credential refresh lease and current +secret resolver without repeating account/repository discovery. The initial OAuth +callback uses the same saved debt and settlement path after connection activation. + +Cloud binding calls acquire the company share lock before locking the grant and +its connection, matching company deletion's parent-before-child order. A deletion +that wins admission leaves no registration to apply; a handoff already admitted +keeps its fence through completion. The grant, connection, company status and +binding revision are re-read under those locks before calling Cloud. +That admission fence prevents a concurrent disable, revoke, or archive from +finishing before an already-admitted registration call. The API has no remote +revision fence that could safely replace those locks with two short transactions. +Each HTTP request and response body has a physical 15-second abort deadline. +Four calls run in an awaited batch; a failed batch leaves the saved debt and +releases the transaction after all four outcomes settle. Other companies can +continue through separate pool connections. Slow successful multi-batch passes +can delay writes to the same grant/connection and company status changes; the +schema permits up to 100 active and 100 removed installation IDs. This is not an +instance-wide lock or an idle hold. + +Credential expiry remains enforced by the existing use path. Scheduled continuity +checks run one hour before recorded access expiry or after the existing 30-day +continuity window. A successful renewal that leaves the credential short-lived +or unchanged retains a one-hour retry obligation instead of causing rapid calls. +Changed credential metadata invalidates the old retry revision, so an earlier new +expiry can preempt it. Disabled connections and archived companies do not perform +credential work. Reactivation uses the normal committed configuration signal. + +GitHub owners receive only relevant credential/configuration signals, not generic +health or receiver notifications. Continuity reconstruction is company-scoped +after the startup scan. Event draining retains instance-wide binding matching +because Cloud leases an instance queue. Event hints coalesce with a five-second +minimum pass spacing; an unrelated connection PATCH cannot trigger that drain. +OAuth rotation merges only its owned OAuth fields, lease acquisition compares +current metadata, and cleanup atomically removes only the matching lease path, +preserving concurrent GitHub installation changes and binding-removal debt. + +Supported local producers are the owning connection, OAuth, grant and company +services in one tenant process. Their notifications belong to the outer database +transaction. A savepoint does not commit a wake, and rollback cancels the write. +Arbitrary SQL writers must use these notification contracts or restart the owner; +process-local signals do not promise independent-writer or replica notification. +External event delivery remains durable in Cloud across processes and restarts. + +## Paired rollout and rollback + +1. Apply the companion Cloud registration-column migration and deploy its delivery + worker with the control signer present. Keep new enrollment disabled initially. +2. Deploy this tenant version. It keeps fallback polling against older Cloud. +3. Enable Cloud enrollment. Version 1 activation is durable and idempotent. Turning + the new-enrollment flag off must still service all existing version 1 records. +4. Before removing the Cloud delivery worker, restart affected tenants with + `PAPERCLIP_GITHUB_EVENT_POLLING=1`. They arm fallback before sending signed + `ewv=0`. Confirm the Cloud registration is cleared. A failed handoff leaves + fallback active and Cloud retry intact. Alternatively, restore the old polling + tenant image first while retaining the compatible Cloud worker. +5. Do not deploy a Cloud downgrade that removes responsibility for still-enrolled + tenants. There is no capability expiry, renewal poll, or periodic tenant hint. + +## Evidence + +Focused tests use isolated PostgreSQL plus controlled HTTP provider simulations. +They cover first service write after quiet, saved results, terminal receipts, +concurrent replay, signed callbacks, outer commit/rollback, drain/resume, startup +wiring in both scheduler branches, credential backoff and changed deadlines. +Cloud acceptance uses its actual broker/store/notifier boundaries and a controlled +HTTP tenant. These tests do not establish a deployed fleet rollout or cost savings. + +## Enrollment after worker startup + +The event owner resolves the current connector only when a delivery or setup pass +is admitted. A service created before self-host enrollment can therefore use a +later completed enrollment without being recreated. The full normalized identity +and target configuration keys the connector cache, preserving negotiated event +wake support while it is unchanged. An absent identity clears the cache; a +replacement identity or changed key creates a new connector. Explicitly injected +connectors keep their original lifetime. + +Enrollment does not add a timer, database scan or network probe. An ordinary +credential/configuration producer, authenticated control wake or active setup +request admits the next pass. Unconfigured passes return before owner SQL or +HTTP. A pass already in flight keeps its connector for that lease and its matching +acknowledgment; identity discovery never changes authority halfway through a pass. + +The real PostgreSQL/loopback regression starts the registered owner unenrolled, +completes the canonical signed enrollment flow, then applies and acknowledges an +encrypted installation event. Removing the identity parks the same owner, and a +second enrollment proves the replacement signing/sealing identity is used. Each +applied installation update has one bounded empty follow-up from its credential +hint; the test joins that saved deadline once before checking 25-minute SQL/HTTP +quiet windows. This is process-local enrollment recovery, not a deployed Cloud +rollout or a new cancellation policy. diff --git a/packages/db/src/schema/tool_access.ts b/packages/db/src/schema/tool_access.ts index b8b317ddb91..bbe6c30e584 100644 --- a/packages/db/src/schema/tool_access.ts +++ b/packages/db/src/schema/tool_access.ts @@ -231,6 +231,8 @@ export const connectionGrants = pgTable( appSlug?: string; accessRevision?: string; lastAccessRefreshAt?: string; + bindingSync?: { id: string; removeInstallationIds: string[] }; + continuityRetry?: { at: string; attempts: number; revision: string }; lastWebhookAt?: string; webhookHealth?: "pending" | "healthy" | "unhealthy"; }; diff --git a/packages/shared/src/types/tool-access.ts b/packages/shared/src/types/tool-access.ts index 0fbd422fdad..5627ca6a174 100644 --- a/packages/shared/src/types/tool-access.ts +++ b/packages/shared/src/types/tool-access.ts @@ -266,6 +266,9 @@ export interface ConnectionGrant { appSlug?: string; accessRevision?: string; lastAccessRefreshAt?: string; + /** Unsettled Cloud binding changes; retained until the exact revision is delivered. */ + bindingSync?: { id: string; removeInstallationIds: string[] }; + continuityRetry?: { at: string; attempts: number; revision: string }; lastWebhookAt?: string; webhookHealth?: "pending" | "healthy" | "unhealthy"; }; diff --git a/packages/shared/src/validators/tool-access.ts b/packages/shared/src/validators/tool-access.ts index bafe22e80e1..914af74e7ab 100644 --- a/packages/shared/src/validators/tool-access.ts +++ b/packages/shared/src/validators/tool-access.ts @@ -240,6 +240,8 @@ export const connectionGrantSchema = z.object({ managementUrl: z.string().url().max(2000).optional(), appSlug: z.string().regex(/^[a-z0-9-]{1,100}$/).optional(), lastAccessRefreshAt: z.string().datetime().optional(), + bindingSync: z.object({ id: z.string().uuid(), removeInstallationIds: z.array(z.string().regex(/^[1-9][0-9]{0,30}$/)).max(100) }).optional(), + continuityRetry: z.object({ at: z.string().datetime(), attempts: z.number().int().min(1).max(16), revision: z.string().max(2000) }).optional(), lastWebhookAt: z.string().datetime().optional(), webhookHealth: z.enum(["pending", "healthy", "unhealthy"]).optional(), }).optional(), diff --git a/server/src/__tests__/cloud-control-task-drain.test.ts b/server/src/__tests__/cloud-control-task-drain.test.ts index 3f0ab8e1951..46c2962edc5 100644 --- a/server/src/__tests__/cloud-control-task-drain.test.ts +++ b/server/src/__tests__/cloud-control-task-drain.test.ts @@ -1,7 +1,9 @@ +import { connectionEventWakeRoutes } from "../routes/connection-event-wake.js"; +import { registerConnectionEventWake } from "../services/connection-event-wake.js"; import { generateKeyPairSync, sign } from "node:crypto"; import express from "express"; import request from "supertest"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, @@ -184,6 +186,7 @@ describe("verifyCloudControlAssertion", () => { }); describe("cloudControlMiddleware", () => { + const eventOwner = {}; const savedEnv: Record = {}; beforeEach(() => { @@ -208,6 +211,7 @@ describe("cloudControlMiddleware", () => { next(); }); app.use(cloudControlMiddleware()); + app.use(connectionEventWakeRoutes(eventOwner)); app.all("/api/instance/task-drain", (req, res) => { res.json({ actor: req.actor }); }); @@ -241,6 +245,22 @@ describe("cloudControlMiddleware", () => { return `${header}.${payload}.${signature}`; } + it("accepts only signed exact-action event hints and leaves unavailable receivers retryable", async () => { + const app = createApp(); const path = "/api/instance/connector-events/wake"; + expect((await request(app).post(path)).status).toBe(403); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("lifecycle:read"))).status).toBe(401); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("connector-events:wake"))).status).toBe(503); + const wake = vi.fn(); const stop = registerConnectionEventWake(eventOwner, wake); + try { + const token = freshAssertion("connector-events:wake"); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, token)).status).toBe(202); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, token)).status).toBe(401); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("connector-events:wake"))).status).toBe(202); + expect(wake).toHaveBeenCalledTimes(2); + } finally { stop(); } + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("connector-events:wake"))).status).toBe(503); + }); + it("installs a synthetic instance-admin board actor for a valid assertion, per method", async () => { const app = createApp(); for (const [method, action] of [ diff --git a/server/src/__tests__/github-catalog-timing.test.ts b/server/src/__tests__/github-catalog-timing.test.ts index 4ee15c6db85..5d0c4835d84 100644 --- a/server/src/__tests__/github-catalog-timing.test.ts +++ b/server/src/__tests__/github-catalog-timing.test.ts @@ -1,9 +1,10 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { context, trace } from "@opentelemetry/api"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { - authUsers, companies, companyMemberships, connectionGrants, createDb, + applyPendingMigrations, authUsers, companies, companySecrets, companyMemberships, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections, } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; @@ -15,15 +16,18 @@ import type { PaperclipCloudConnector } from "../services/paperclip-cloud-connec import { toolAccessService } from "../services/tool-access.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -const support = await getEmbeddedPostgresTestSupport(); +const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_CATALOG_TEST_DATABASE_URL"); +const support = databaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("GitHub repository catalog execution timing", () => { let temporary: Awaited>; let db: ReturnType; + const ownedCompanies: string[] = []; let exporter: InMemorySpanExporter, provider: TracerProvider, manager: AsyncLocalStorageContextManager; beforeAll(async () => { - temporary = await startEmbeddedPostgresTestDatabase("paperclip-github-catalog-timing-"); - db = createDb(temporary.connectionString, { maxConnections: 3 }); + if (databaseUrl) await applyPendingMigrations(databaseUrl); + else temporary = await startEmbeddedPostgresTestDatabase("paperclip-github-catalog-timing-"); + db = createDb(databaseUrl ?? temporary.connectionString, { maxConnections: 3 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); beforeEach(() => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); @@ -38,6 +42,7 @@ const support = await getEmbeddedPostgresTestSupport(); vi.restoreAllMocks(); vi.unstubAllEnvs(); trace.disable(); context.disable(); manager.disable(); await provider.shutdown(); + for (const id of ownedCompanies.splice(0)) { await db.delete(companyMemberships).where(eq(companyMemberships.companyId, id)); await db.delete(companySecrets).where(eq(companySecrets.companyId, id)); await db.delete(companies).where(eq(companies.id, id)); } }); afterAll(async () => { await db?.$client.end({ timeout: 0 }); @@ -47,6 +52,7 @@ const support = await getEmbeddedPostgresTestSupport(); async function fixture() { const userId = `catalog-user-${randomUUID()}`; const [company] = await db.insert(companies).values({ name: "Catalog timing", issuePrefix: `GH${randomUUID().slice(0, 6)}` }).returning(); + ownedCompanies.push(company.id); await db.insert(authUsers).values({ id: userId, name: "Catalog owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); await db.insert(companyMemberships).values({ companyId: company.id, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "GitHub catalog", type: "mcp_http" }).returning(); diff --git a/server/src/__tests__/github-connection-enrollment-work.test.ts b/server/src/__tests__/github-connection-enrollment-work.test.ts new file mode 100644 index 00000000000..c888565b3d9 --- /dev/null +++ b/server/src/__tests__/github-connection-enrollment-work.test.ts @@ -0,0 +1,139 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createServer, type Server } from "node:http"; +import { once } from "node:events"; +import { createCipheriv, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID, verify } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { companies, connectionEventDeliveries, connectionGrants, createDb, toolApplications, toolConnections } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { githubConnectionEventService } from "../services/github-connection-events.js"; +import { registerGitHubConnectionEventWork } from "../services/github-connection-work.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { wakeConnectionEvents } from "../services/connection-event-wake.js"; +import { completePaperclipCloudConnectorEnrollment, loadPaperclipCloudConnectorIdentity, startPaperclipCloudConnectorEnrollment } from "../services/paperclip-cloud-connector-enrollment.js"; +import { paperclipCloudConnectorIdentityPath } from "../services/paperclip-cloud-connector-enrollment.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("GitHub enrollment on a live event worker (real PostgreSQL and signed loopback Cloud)", () => { + let temporary: Awaited>; + let db: ReturnType; + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("github-live-enrollment-"); + db = createDb(temporary.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 90_000); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + it("enrolls without recreation, drains an authenticated event, forgets removed identity and re-enrolls from quiet", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "github-live-enrollment-")); + const previous = { home: process.env.PAPERCLIP_HOME, instance: process.env.PAPERCLIP_INSTANCE_ID }; + process.env.PAPERCLIP_HOME = root; process.env.PAPERCLIP_INSTANCE_ID = "enrollment-worker"; + let server: Server | undefined; + let worker: ReturnType | undefined; + let coordinator: ReturnType | undefined; + const errors: unknown[] = [], httpErrors: unknown[] = []; + let requests = 0, acknowledged = 0, leases = 0, enrollment: Record | undefined; + let origin = "", eventId = randomUUID(), grantId = "", companyId = ""; + const identities: string[] = []; + const delivered = new Set(); + try { + server = createServer(async (req, res) => { + try { + requests++; + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const body = JSON.parse(Buffer.concat(chunks).toString()); + if (req.url === "/v1/connector/enrollments") { + enrollment = body; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ enrollmentId: "enrollment", verificationUrl: `${origin}/connections/enroll?id=enrollment`, expiresAt: new Date(Date.now() + 60_000).toISOString() })); return; + } + expect(enrollment).toBeDefined(); + const [header, payload, signature] = body.request.split("."); + expect(verify(null, Buffer.from(`${header}.${payload}`), createPublicKey({ key: { kty: "OKP", crv: "Ed25519", x: enrollment!.signPublicKey }, format: "jwk" }), Buffer.from(signature, "base64url"))).toBe(true); + const claims = JSON.parse(Buffer.from(payload, "base64url").toString()); + expect(claims.iss).toBe(enrollment!.instanceId); expect(claims.env).toBe("test"); expect(claims.aud).toBe(`${origin}${req.url}`); + res.setHeader("content-type", "application/json"); + if (req.url === "/v1/connector/enrollment-claims") { + res.end(JSON.stringify({ id: enrollment!.instanceId, environment: "test", origins: [enrollment!.origin] })); return; + } + identities.push(claims.iss); + if (req.url === "/v1/connector/events/ack") { acknowledged++; delivered.add(eventId); res.end(JSON.stringify({ acknowledged: 1 })); return; } + expect(req.url).toBe("/v1/connector/events/lease"); leases++; + if (delivered.has(eventId)) { res.end(JSON.stringify({ events: [], eventWakeVersion: 1 })); return; } + const ephemeral = generateKeyPairSync("x25519"), epk = ephemeral.publicKey.export({ format: "jwk" }).x!; + const recipient = createPublicKey({ key: { kty: "OKP", crv: "X25519", x: enrollment!.sealPublicKey }, format: "jwk" }); + const aad = Buffer.from([1, "X25519-HKDF-SHA256-A256GCM", "events", claims.iss, "test", "github", "github.code", ""].join("\n")); + const key = Buffer.from(hkdfSync("sha256", diffieHellman({ privateKey: ephemeral.privateKey, publicKey: recipient }), Buffer.concat([Buffer.from(epk, "base64url"), Buffer.from(enrollment!.sealPublicKey, "base64url")]), aad, 32)); + const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad); + const packet = { v: 1, instanceId: claims.iss, environment: "test", leaseId: "lease", events: [{ id: eventId, provider: "github", event: "installation", action: "unsuspend", installationId: "101", repositoryId: null, createdAt: new Date().toISOString(), bindingIds: [`${grantId}_101`], payload: { installationId: "101" } }] }; + const ct = Buffer.concat([cipher.update(JSON.stringify(packet)), cipher.final(), cipher.getAuthTag()]); + res.end(JSON.stringify({ leaseId: "lease", eventWakeVersion: 1, sealed: { v: 1, alg: "X25519-HKDF-SHA256-A256GCM", purpose: "events", provider: "github", profile: "github.code", epk, iv: iv.toString("base64url"), ct: ct.toString("base64url") } })); + } catch (error) { httpErrors.push(error); res.writeHead(500).end(); } + }); + const listening = once(server, "listening"); server.listen(0, "127.0.0.1"); await listening; + const address = server.address(); if (!address || typeof address === "string") throw new Error("No loopback listener"); + origin = `http://127.0.0.1:${address.port}`; + companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Live enrollment", issuePrefix: "GLE" }); + const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: "GitHub", type: "mcp_http", status: "active" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: app!.id, uid: randomUUID(), name: "GitHub", status: "active", enabled: true, transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_user", config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } }, transportConfig: {} }).returning(); + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection!.id, kind: "user", subjectUserId: "owner", status: "active", providerTenant: { github: { userId: "42", login: "octocat", installationIds: ["101"], installationCount: 1, repositoryCount: 0, repositorySelection: "all", installationOwnerLogins: [], webhookHealth: "unhealthy" } } }).returning(); + grantId = grant!.id; + const env = { PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: origin, PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test" }; + const service = githubConnectionEventService(db, { env }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: e => errors.push(e) }); + const selects = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute"); + worker = registerGitHubConnectionEventWork(db, coordinator, service, () => true); + await worker.ready; + async function quiet(stage: string) { + await vi.waitFor(() => { expect(idleWorkSnapshot().active, stage).toBe(0); expect(coordinator!.nextWakeAt(), stage).toBeNull(); }); + selects.mockClear(); transactions.mockClear(); executes.mockClear(); const before = requests; + await vi.advanceTimersByTimeAsync(25 * 60_000); + expect(selects).not.toHaveBeenCalled(); expect(transactions).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); expect(requests).toBe(before); + } + await quiet("unenrolled startup"); expect(requests).toBe(0); + async function enroll() { + await startPaperclipCloudConnectorEnrollment({ origin: "https://self-hosted.example.test", env, companyId }); + const pending = loadPaperclipCloudConnectorIdentity()!; + await completePaperclipCloudConnectorEnrollment({ enrollmentId: pending.pending!.enrollmentId, state: pending.pending!.returnState, approvalCode: "approved", env }); + return loadPaperclipCloudConnectorIdentity()!.instanceId; + } + const firstIdentity = await enroll(); + expect(wakeConnectionEvents(db)).toBe(true); + await vi.advanceTimersByTimeAsync(6_000); + await vi.waitFor(() => expect(acknowledged).toBe(1)); + expect(await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, eventId))).toMatchObject([{ status: "processed" }]); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, grantId)))[0]!.providerTenant!.github!.webhookHealth).toBe("healthy"); + // The applied installation update publishes one credential-change hint. + // Join that existing minimum-spacing pass once; the broker now has no debt. + const followup = coordinator.nextWakeAt(); expect(followup).not.toBeNull(); + expect(followup! - Date.now()).toBeLessThanOrEqual(5_000); + await vi.advanceTimersByTimeAsync(Math.max(0, followup! - Date.now()) + 1); + await vi.waitFor(() => expect(leases).toBe(2)); + await quiet("first event plus its one empty follow-up"); + await rm(paperclipCloudConnectorIdentityPath()); + const oldLeases = leases; expect(wakeConnectionEvents(db)).toBe(true); await vi.advanceTimersByTimeAsync(6_000); + await quiet("removed identity"); expect(leases).toBe(oldLeases); + eventId = randomUUID(); const secondIdentity = await enroll(); expect(secondIdentity).not.toBe(firstIdentity); + expect(wakeConnectionEvents(db)).toBe(true); await vi.advanceTimersByTimeAsync(6_000); + await vi.waitFor(() => expect(acknowledged).toBe(2)); + expect(identities).toContain(firstIdentity); expect(identities).toContain(secondIdentity); + expect(identities.slice(identities.indexOf(secondIdentity))).toEqual(expect.arrayContaining([secondIdentity, secondIdentity])); + expect(identities.slice(identities.indexOf(secondIdentity)).every(id => id === secondIdentity)).toBe(true); + const replacementFollowup = coordinator.nextWakeAt(); expect(replacementFollowup).not.toBeNull(); + expect(replacementFollowup! - Date.now()).toBeLessThanOrEqual(5_000); + await vi.advanceTimersByTimeAsync(Math.max(0, replacementFollowup! - Date.now()) + 1); + await vi.waitFor(() => expect(leases).toBe(oldLeases + 2)); + await quiet("replacement event plus its one empty follow-up"); expect(errors).toEqual([]); expect(httpErrors).toEqual([]); + } finally { + await worker?.stop(); await coordinator?.stop(); vi.restoreAllMocks(); vi.useRealTimers(); + if (server) { const closed = once(server, "close"); server.close(); server.closeAllConnections(); await closed; } + if (previous.home === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previous.home; + if (previous.instance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = previous.instance; + await rm(root, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/server/src/__tests__/github-connection-events.test.ts b/server/src/__tests__/github-connection-events.test.ts index 0417a434718..bef7020d59c 100644 --- a/server/src/__tests__/github-connection-events.test.ts +++ b/server/src/__tests__/github-connection-events.test.ts @@ -11,6 +11,8 @@ import { connectionGrants, createDb, externalObjects, + issues, + issueThreadInteractions, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections, @@ -36,12 +38,14 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { beforeAll(async () => { if (!externalTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-github-events-"); if (externalTestDatabaseUrl) await applyPendingMigrations(externalTestDatabaseUrl); - db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); + db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { await db.delete(activityLog); await db.delete(connectionEventDeliveries); + await db.delete(issueThreadInteractions); + await db.delete(issues); await db.delete(externalObjects); await db.delete(connectionGrants); await db.delete(toolConnections); @@ -106,6 +110,10 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { }, }, }); + const [issue] = await db.insert(issues).values({ companyId, title: "Merge release", status: "in_progress" }).returning(); + const [confirmation] = await db.insert(issueThreadInteractions).values({ companyId, issueId: issue!.id, + kind: "request_confirmation", title: "Confirm pull request merged", continuationPolicy: "none", + payload: { version: 1, prompt: "Confirm https://github.com/paperclipai/paperclip/pull/123 is merged" } }).returning(); const externalObjectId = randomUUID(); await db.insert(externalObjects).values({ id: externalObjectId, @@ -186,13 +194,17 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { await started; const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined); - await expect(service.pollOnce({ forSetup: activeSetup })).resolves.toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 }); + const outcome = await service.pollOnce({ forSetup: activeSetup }); + const [processingReceipt] = await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id)); + expect(processingReceipt?.lastError).toBeNull(); + expect(outcome).toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 }); expect(warning.mock.calls).toEqual([[{ deliveryId: "undecryptable-delivery", reason: "inner_event_authentication_failed", }, "GitHub connector event retained for retry"]]); warning.mockRestore(); releaseRefresh(); await expect(oldRefresh).resolves.toMatchObject({ refreshed: false, reason: "refresh_superseded" }); + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, confirmation!.id)))[0]!.status).toBe("accepted"); const [snapshot] = await db.select().from(externalObjects).where(eq(externalObjects.id, externalObjectId)); expect(snapshot).toMatchObject({ statusKey: "merged", @@ -263,7 +275,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { expect(connector.acknowledgeEvents).toHaveBeenCalledTimes(2); }); - it.each([false, true])("applies installation events once without discarding newer verified access (refreshed: %s)", async (refreshed) => { + it.each([false, true, "rollback-race"])("applies installation events once without discarding newer verified access (mode: %s)", async (mode) => { + const refreshed = mode === true; const companyId = randomUUID(); const applicationId = randomUUID(); const connectionId = randomUUID(); @@ -361,7 +374,55 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { throw new Error("fixture live subscriber failed"); }); try { - await expect(service.pollOnce()).resolves.toMatchObject({ processed: 1, duplicate: 0, failed: 0 }); + const competing = githubConnectionEventService(db, { connector, now: () => currentTime }); + if (mode === "rollback-race") { + let enter!: () => void, release!: () => void, releaseFailure!: () => void; + const entered = new Promise(resolve => { enter = resolve; }); + const mayRollback = new Promise(resolve => { release = resolve; }); + const mayMarkFailure = new Promise(resolve => { releaseFailure = resolve; }); + const transaction = db.transaction.bind(db), update = db.update.bind(db); + let first = true; + vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: unknown) => Promise) => { + const rejectAfterApply = first; first = false; + return transaction(async tx => { + const value = await callback(tx); + if (rejectAfterApply) { enter(); await mayRollback; throw new Error("fixture rollback before receipt commit"); } + return value; + }); + }) as typeof db.transaction); + vi.spyOn(db, "update").mockImplementation(((table: unknown) => { + const builder = update(table as typeof connectionEventDeliveries); + if (table === connectionEventDeliveries) { + const set = builder.set.bind(builder); + builder.set = ((values: { status?: string }) => { + const statement = set(values); + if (values.status === "failed") { + const where = statement.where.bind(statement); + statement.where = (async (...args: Parameters) => { await mayMarkFailure; return where(...args); }) as typeof statement.where; + } + return statement; + }) as typeof builder.set; + } + return builder; + }) as typeof db.update); + const rejected = service.pollOnce(); + const firstResult = Promise.allSettled([rejected]); + try { + await entered; + const succeeding = competing.pollOnce(); + release(); + expect(await succeeding).toMatchObject({ processed: 1, failed: 0 }); + expect((await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id)))[0]!.status).toBe("processed"); + releaseFailure(); + expect(await rejected).toMatchObject({ failed: 1 }); + expect((await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id)))[0]!.status).toBe("processed"); + } finally { release(); releaseFailure(); await firstResult; vi.restoreAllMocks(); } + } else { + const outcomes = await Promise.all([service.pollOnce(), competing.pollOnce()]); + expect(outcomes.reduce((sum, result) => sum + result.processed, 0)).toBe(1); + expect(outcomes.reduce((sum, result) => sum + result.duplicate, 0)).toBe(1); + expect(outcomes.every(result => result.failed === 0)).toBe(true); + } } finally { unsubscribe(); } await health.expectDeadline(currentTime.getTime() + 900_000 - (refreshed ? 60_000 : 0)); await health.expectQuiet(); @@ -382,6 +443,6 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { connectionEventDeliveries.providerDeliveryId, leasedEvent.id, )); - expect(receipt).toMatchObject({ status: "processed", attempts: 1 }); + expect(receipt).toMatchObject({ status: "processed", attempts: mode === "rollback-race" ? 2 : 1 }); }); }); diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts new file mode 100644 index 00000000000..5ee7db0df5e --- /dev/null +++ b/server/src/__tests__/github-connection-work.test.ts @@ -0,0 +1,726 @@ +import { notifyChatEndpointConfiguration } from "../services/chat-channels/configuration-work-notifications.js"; +import { deleteCompany } from "../services/company-deletion.js"; +import { toolAccessService } from "../services/tool-access.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { randomUUID, generateKeyPairSync, verify, createHash, diffieHellman, hkdfSync, randomBytes, createCipheriv } from "node:crypto"; +import { createServer, type Server } from "node:http"; +import { once } from "node:events"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { applyPendingMigrations, companies, activityLog, companySecrets, connectionEventDeliveries, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createGitHubGrantContinuity, githubGrantContinuityDeadline, githubContinuityRevision } from "../services/tool-access/github-grant-continuity.js"; +import { createPaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; +import { githubConnectionEventService } from "../services/github-connection-events.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerGitHubConnectionContinuityWork, registerGitHubConnectionEventWork } from "../services/github-connection-work.js"; +import { notifyGitHubCredentialChange } from "../services/github-connection-notifications.js"; +import { createOAuthCredentialStore } from "../services/tool-access/oauth/credential-store.js"; +import { createOAuthRefreshLeases } from "../services/tool-access/oauth/refresh-leases.js"; +import { createOAuthGrantRefresh } from "../services/tool-access/oauth/grant-refresh.js"; +import { secretService } from "../services/secrets.js"; +import { writeConnectionCredential } from "../services/connection-credentials.js"; +import { GITHUB_CONNECTOR_PROFILES } from "@paperclipai/shared"; +import { notifyToolConnectionHealth } from "../services/tool-connection-health-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_WORK_TEST_DATABASE_URL"); +const embeddedSupport = databaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describeDatabase = embeddedSupport.supported ? describe : describe.skip; + +describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP (simulation)", () => { + let db: ReturnType; + let temporaryDatabase: Awaited> | undefined; + let server: Server, origin: string; + let leaseTimes: number[] = []; + let bindings: Array>, leases: number, failBindings: boolean, githubRequests: number; + const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); + let releaseRefresh: (() => void) | undefined, refreshArrived: (() => void) | undefined; + let refreshGate: Promise | undefined; + let discoveryFailureGate: Promise | undefined, discoveryArrived: (() => void) | undefined; + let heldBindingCompany: string | undefined, bindingGate: Promise | undefined; + let bindingArrived: (() => void) | undefined, bindingClosed: (() => void) | undefined; + const realFetch = globalThis.fetch; + let companyIds: string[] = []; + beforeAll(async () => { + if (databaseUrl) await applyPendingMigrations(databaseUrl); + else temporaryDatabase = await startEmbeddedPostgresTestDatabase("paperclip-github-work-"); + // Only scheduler time is virtual here. Driver housekeeping must not expire + // a real socket connection while that fake clock advances across idle windows. + db = createDb(databaseUrl ?? temporaryDatabase!.connectionString, { + maxConnections: 4, idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0, + }); + // This explicitly assigned external test database is exclusive to this suite. + await db.transaction(async tx => { await tx.execute(sql`SET LOCAL client_min_messages = warning`); await tx.execute(sql`TRUNCATE companies CASCADE`); }); + server = createServer(async (req, res) => { + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const raw = Buffer.concat(chunks).toString(); + if (req.url!.startsWith("/v1/connector/")) { + const body = JSON.parse(raw), [header, payload, signature] = body.request.split("."); + expect(verify(null, Buffer.from(`${header}.${payload}`), signing.publicKey, Buffer.from(signature, "base64url"))).toBe(true); + const claims = JSON.parse(Buffer.from(payload, "base64url").toString()); + expect(claims.iss).toBe("instance"); expect(claims.env).toBe("test"); + expect(claims.aud).toBe(`${origin}${req.url}`); + if (req.url!.endsWith("/refresh")) { + refreshArrived?.(); await refreshGate; + const scopes = [...GITHUB_CONNECTOR_PROFILES["github.code"].scopes]; + const algorithm = "X25519-HKDF-SHA256-A256GCM"; + const purpose = "access"; + const aad = Buffer.from([1, algorithm, purpose, "instance", "test", "github", "github.code", scopes.sort().join(" ")].join("\n")); + const ephemeral = generateKeyPairSync("x25519"); + const epk = Buffer.from(ephemeral.publicKey.export({ format: "jwk" }).x!, "base64url"); + const recipient = Buffer.from(sealing.publicKey.export({ format: "jwk" }).x!, "base64url"); + const key = Buffer.from(hkdfSync("sha256", diffieHellman({ privateKey: ephemeral.privateKey, publicKey: sealing.publicKey }), Buffer.concat([epk, recipient]), aad, 32)); + const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad); + const credentials = { v: 1, accessToken: "rotated-access", refreshToken: "rotated-refresh", tokenType: "Bearer", scopes, + accessTokenExpiresAt: new Date(Date.now() + 8 * 3600_000).toISOString(), refreshTokenExpiresAt: null, + subject: claims.sub, companyId: claims.cid, instanceId: "instance", environment: "test", provider: "github", profile: "github.code" }; + const ct = Buffer.concat([cipher.update(JSON.stringify(credentials)), cipher.final(), cipher.getAuthTag()]); + res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ sealed: { v: 1, alg: algorithm, purpose, provider: "github", profile: "github.code", epk: epk.toString("base64url"), iv: iv.toString("base64url"), ct: ct.toString("base64url") } })); return; + } + if (req.url!.endsWith("/events/lease")) { + leases++; leaseTimes.push(Date.now()); res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ events: [], ...(claims.ewv === 1 ? { eventWakeVersion: 1 } : {}) })); return; + } + expect(claims.sh).toBe(createHash("sha256").update(body.binding).digest("base64url")); + bindings.push(JSON.parse(body.binding)); + if (claims.cid === heldBindingCompany) { + res.writeHead(200, { "content-type": "application/json" }); res.write("{"); + res.once("close", () => bindingClosed?.()); bindingArrived?.(); await bindingGate; + res.end("}"); return; + } + res.writeHead(failBindings ? 503 : 200, { "content-type": "application/json" }).end("{}"); return; + } + if (req.url === "/oauth/token") { + expect(new URLSearchParams(raw).get("grant_type")).toBe("refresh_token"); + expect(new URLSearchParams(raw).get("refresh_token")).toBe("saved-refresh"); + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ access_token: "renewed-access", refresh_token: "renewed-refresh", token_type: "Bearer", expires_in: 3600 })); return; + } + githubRequests++; + if (req.url === "/user" && discoveryFailureGate) { + discoveryArrived?.(); await discoveryFailureGate; + res.writeHead(503, { "content-type": "application/json", "retry-after": "86400" }).end("{}"); return; + } + const body = req.url === "/user" ? { id: 42, login: "octocat" } + : req.url!.startsWith("/user/installations?") ? { installations: [{ id: 101, repository_selection: "all", account: { login: "octocat" } }] } + : { total_count: 1, repositories: [{ id: 999, full_name: "octocat/repo" }] }; + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(body)); + }); + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); if (!address || typeof address === "string") throw new Error("HTTP fixture did not start"); + origin = `http://127.0.0.1:${address.port}`; + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterEach(async () => { + vi.restoreAllMocks(); vi.useRealTimers(); + for (const id of companyIds) { await db.delete(activityLog).where(eq(activityLog.companyId, id)); await db.delete(companySecrets).where(eq(companySecrets.companyId, id)); await db.delete(companies).where(eq(companies.id, id)); } companyIds = []; + }); + afterAll(async () => { server.close(); await once(server, "close"); await db.$client.end({ timeout: 1 }); await temporaryDatabase?.cleanup(); }); + + async function fixture(ids: string[] = []) { + bindings = []; leases = 0; leaseTimes = []; failBindings = false; githubRequests = 0; + const companyId = randomUUID(); companyIds.push(companyId); + const [company] = await db.insert(companies).values({ id: companyId, name: "GitHub work test", issuePrefix: `GW${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}` }).returning(); + const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: "GitHub", type: "mcp_server", status: "active" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: app!.id, uid: randomUUID(), name: "GitHub", status: "active", enabled: true, + transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_user", config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } }, transportConfig: {} }).returning(); + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection!.id, kind: "user", subjectUserId: "owner", status: "active", + credentialSecretRefs: [{ configPath: "oauth.access_token", secretId: randomUUID(), label: "Access", provider: "local_encrypted" }], + providerTenant: { oauth: { accessTokenExpiresAt: new Date(Date.now() + 8 * 3600_000).toISOString(), refreshedAt: new Date().toISOString() }, github: { + userId: "42", login: "octocat", installationCount: ids.length, repositoryCount: 1, repositorySelection: "all", installationIds: ids, + installationOwnerLogins: ["octocat"], accessRevision: randomUUID(), lastAccessRefreshAt: new Date().toISOString(), webhookHealth: "healthy", + } } }).returning(); + const connector = createPaperclipCloudConnector({ config: { baseUrl: origin, instanceId: "instance", environment: "test", + signPrivateKey: signing.privateKey.export({ format: "jwk" }).d!, sealPrivateKey: sealing.privateKey.export({ format: "jwk" }).d! } }); + type Dependencies = Parameters[1]; + const dependencies: Dependencies = { + now: () => new Date(), currentCloudConnector: () => connector, + refreshOAuthGrantCredentials: async input => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, input.grantId)))[0]!, + resolveOAuthGrantSecret: (async () => ({ value: "controlled-token" })) as Dependencies["resolveOAuthGrantSecret"], + }; + const service = createGitHubGrantContinuity(db, dependencies); + return { company: company!, connection: connection!, grant: grant!, connector, service, dependencies }; + } + const fakeClock = () => vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + async function settled(coordinator: ReturnType) { + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + expect(coordinator.nextWakeAt()).not.toBeNull(); + } + + it("ordinary grant refresh activates events from quiet, commits durable result, then performs no owner SQL/HTTP across old cadence windows", async () => { + const f = await fixture(); fakeClock(); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const events = githubConnectionEventService(db, { connector: f.connector }); + const ew = registerGitHubConnectionEventWork(db, coordinator, events, () => true); + const cw = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); + try { + await Promise.all([ew.ready, cw.ready]); expect(leases).toBe(1); + const startupLeases = leases; + await f.service.refreshManagedGitHubGrantAccess(f.connection, f.grant, { actorType: "system", actorId: "test" }); + await vi.advanceTimersByTimeAsync(6_000); + await vi.waitFor(() => expect(leases).toBeGreaterThan(startupLeases)); + await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.installationIds).toEqual(["101"]); + expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); + expect(bindings).toContainEqual(expect.objectContaining({ active: true, installationId: "101", id: `${f.grant.id}_101` })); + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"), execute = vi.spyOn(db, "execute"); + const before = { leases, githubRequests, bindings: bindings.length }; + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); + expect({ leases, githubRequests, bindings: bindings.length }).toEqual(before); expect(errors).toEqual([]); + } finally { await ew.stop(); await cw.stop(); await coordinator.stop(); } + }); + + it("failed binding removal persists retry/debt through restart, uses current lease deadline and settles without refetching GitHub metadata", async () => { + const f = await fixture(["101"]); fakeClock(); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, + bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } } }).where(eq(connectionGrants.id, f.grant.id)); + failBindings = true; + expect(await f.service.sweepGitHubConnectionContinuity()).toMatchObject({ due: 1, failed: 1 }); + const [failed] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(failed!.providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["202"]); + const retryAt = Date.parse(failed!.providerTenant!.github!.continuityRetry!.at); + const attemptedRemovals = bindings.filter(binding => binding.installationId === "202").length; + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBe(retryAt); + await vi.advanceTimersByTimeAsync(29_000); + expect(await restarted.sweepGitHubConnectionContinuity()).toMatchObject({ due: 0 }); + failBindings = false; await vi.advanceTimersByTimeAsync(2_000); + await Promise.all([restarted.sweepGitHubConnectionContinuity(), restarted.sweepGitHubConnectionContinuity()]); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); expect(saved!.providerTenant!.github!.continuityRetry).toBeUndefined(); + expect(bindings.filter(binding => binding.installationId === "202")).toHaveLength(attemptedRemovals + 1); + expect(githubRequests).toBe(0); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBeGreaterThan(Date.now()); + await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, f.connection.id)); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); + await db.update(toolConnections).set({ enabled: true }).where(eq(toolConnections.id, f.connection.id)); + await db.update(companies).set({ status: "archived" }).where(eq(companies.id, f.company.id)); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); + }); + + it.each(["credential", "installation"])("does not attach a withheld provider failure to newer %s work", async change => { + const f = await fixture(["101"]); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + let release!: () => void; + const arrived = new Promise(resolve => { discoveryArrived = resolve; }); + discoveryFailureGate = new Promise(resolve => { release = resolve; }); + const pending = f.service.sweepGitHubConnectionContinuity(f.company.id); + const finished = Promise.allSettled([pending]); + try { + await Promise.race([arrived, pending.then(() => { throw new Error("Expected held provider request"); })]); + const revision = randomUUID(); + const expiry = new Date(Date.now() + 10 * 60_000).toISOString(); + await db.transaction(async tx => { + await notifyGitHubCredentialChange(tx, f.connection); + await tx.update(connectionGrants).set({ providerTenant: change === "credential" + ? sql`jsonb_set(${connectionGrants.providerTenant}, '{oauth,accessTokenExpiresAt}', ${JSON.stringify(expiry)}::jsonb)` + : sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify({ id: revision, removeInstallationIds: ["202"] })}::jsonb)`, + updatedAt: new Date() }).where(eq(connectionGrants.id, f.grant.id)); + }); + release(); + expect(await pending).toMatchObject({ due: 1, failed: 1 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.continuityRetry).toBeUndefined(); + if (change === "credential") expect(saved!.providerTenant!.oauth!.accessTokenExpiresAt).toBe(expiry); + else expect(saved!.providerTenant!.github!.bindingSync!.id).toBe(revision); + expect(githubGrantContinuityDeadline(f.connection, saved!)).toBeLessThanOrEqual(Date.now()); + discoveryFailureGate = undefined; + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, refreshed: 1, failed: 0 }); + expect(bindings).toContainEqual(expect.objectContaining({ installationId: "101", active: true })); + if (change === "installation") expect(bindings).toContainEqual(expect.objectContaining({ installationId: "202", active: false })); + const [settled] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(settled!.providerTenant!.github!.bindingSync).toBeUndefined(); + const next = await restarted.nextGitHubConnectionContinuityAt(f.company.id); + expect(next).toBeLessThan(Date.now() + 24 * 60 * 60_000); + } finally { release(); await finished; discoveryFailureGate = undefined; discoveryArrived = undefined; } + }); + + it("retains provider backoff for the actual attempted revision after its own credential rotation", async () => { + const f = await fixture(["101"]); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + const owner = createGitHubGrantContinuity(db, { ...f.dependencies, refreshOAuthGrantCredentials: async () => { + const [rotated] = await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{oauth}', ${JSON.stringify({ + accessTokenExpiresAt: new Date(Date.now() + 10 * 60_000).toISOString(), refreshedAt: new Date().toISOString(), + })}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)).returning(); + return rotated!; + } }); + discoveryFailureGate = Promise.resolve(); + try { + expect(await owner.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, failed: 1 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.continuityRetry!.revision).toBe(githubContinuityRevision(saved!)); + expect(await owner.nextGitHubConnectionContinuityAt(f.company.id)).toBeGreaterThan(Date.now() + 23 * 60 * 60_000); + const calls = githubRequests; + expect(await createGitHubGrantContinuity(db, f.dependencies).sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 0 }); + expect(githubRequests).toBe(calls); + } finally { discoveryFailureGate = undefined; } + }); + + it.each([["suspend", "unsuspend"], ["unsuspend", "suspend"]])("reconciles same-batch installation order %s then %s and duplicate replay", async (first, last) => { + const f = await fixture(["101"]); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } } }).where(eq(connectionGrants.id, f.grant.id)); + const events = [first, last].map(action => ({ id: randomUUID(), provider: "github" as const, event: "installation", action, + installationId: "101", repositoryId: null, createdAt: new Date(Date.now() + 1000).toISOString(), bindingIds: [`${f.grant.id}_101`], payload: { installationId: "101" } })); + const acknowledge = vi.fn(async () => events.length); + const ingress = githubConnectionEventService(db, { connector: { ...f.connector, leaseEvents: async () => ({ leaseId: "installation-batch", events }), acknowledgeEvents: acknowledge } }); + expect(await ingress.pollOnce({ forDelivery: true })).toMatchObject({ processed: 2, failed: 0 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + const active = last === "unsuspend"; + expect(saved!.providerTenant!.github!.installationIds).toEqual(active ? ["101"] : []); + expect(saved!.providerTenant!.github!.bindingSync!.removeInstallationIds.sort()).toEqual(active ? ["202"] : ["101", "202"]); + expect(githubGrantContinuityDeadline(f.connection, saved!)).toBe(0); + await f.service.syncGitHubGrantBindings(f.connection, saved!); + expect(bindings.filter(binding => binding.installationId === "101")).toEqual([expect.objectContaining({ active })]); + expect(bindings.filter(binding => binding.installationId === "202")).toEqual([expect.objectContaining({ active: false })]); + const [settled] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(settled!.providerTenant!.github!.bindingSync).toBeUndefined(); + // Batch duplicates must retain terminal receipts and emit no registration changes. + expect(await ingress.pollOnce({ forDelivery: true })).toMatchObject({ processed: 0, duplicate: active ? 2 : 0, ignored: active ? 0 : 2, failed: 0 }); + await f.service.syncGitHubGrantBindings(f.connection, saved!); + expect(bindings).toHaveLength(2); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)))[0]!.providerTenant).toEqual(settled!.providerTenant); + expect((await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.companyId, f.company.id))).map(receipt => receipt.status)).toEqual(["processed", "processed"]); + expect(acknowledge).toHaveBeenCalledTimes(2); + }); + + it("does not retire active installations from a legacy conflicting binding intent", async () => { + const f = await fixture(["101"]); + const [saved] = await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101", "202"] } } } }).where(eq(connectionGrants.id, f.grant.id)).returning(); + await f.service.syncGitHubGrantBindings(f.connection, saved!); + expect(bindings.filter(binding => binding.installationId === "101")).toEqual([expect.objectContaining({ active: true })]); + expect(bindings.filter(binding => binding.installationId === "202")).toEqual([expect.objectContaining({ active: false })]); + }); + + it("preserves webhook removal debt across an actual signed, held Cloud token rotation and lease cleanup", async () => { + const f = await fixture(["101"]); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); + const refs = []; + for (const path of ["oauth.access_token", "oauth.refresh_token"]) { + const saved = await writeConnectionCredential(db, { companyId: f.company.id, connectionName: "GitHub", configPath: path, label: path, value: path, ownerUserId: "owner" }); + refs.push({ configPath: path, secretId: saved.secret.id, versionSelector: "latest" as const }); + } + const [connection] = await db.update(toolConnections).set({ credentialSource: "paperclip_vault", config: { ...f.connection.config, + oauth: { strategy: "paperclip_cloud_connector", connectorProfile: "github.code" } } }).where(eq(toolConnections.id, f.connection.id)).returning(); + const [grant] = await db.update(connectionGrants).set({ credentialSecretRefs: refs }).where(eq(connectionGrants.id, f.grant.id)).returning(); + const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)))[0]!; + const store = createOAuthCredentialStore(db, secretService(db)); + await store.syncCredentialBindings(connection!, refs); + const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, store.getOAuthGrantRow); + const rotation = createOAuthGrantRefresh(db, { now: () => new Date(), store, leases, getConnectionRow: getConnection, + currentCloudConnector: () => f.connector, exchangeOAuthToken: vi.fn() }); + const arrived = new Promise(resolve => { refreshArrived = resolve; }); + refreshGate = new Promise(resolve => { releaseRefresh = resolve; }); + const pending = rotation({ companyId: f.company.id, connectionId: connection!.id, grantId: grant!.id, forceRefresh: true }); + const result = Promise.allSettled([pending]); + try { + await Promise.race([arrived, pending.then(() => { throw new Error("Rotation returned before the Cloud HTTP boundary"); })]); + const connector = { ...f.connector, acknowledgeEvents: async () => 1, leaseEvents: async () => ({ leaseId: "removal", events: [{ id: randomUUID(), provider: "github" as const, + event: "installation", action: "deleted", installationId: "101", repositoryId: null, createdAt: new Date(Date.now() + 1000).toISOString(), bindingIds: [`${grant!.id}_101`], payload: { installationId: "101" } }] }) }; + expect(await githubConnectionEventService(db, { connector }).pollOnce()).toMatchObject({ processed: 1 }); + const [before] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + expect(before!.providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["101"]); + releaseRefresh!(); + const [completion] = await result; if (completion.status === "rejected") throw completion.reason; + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + expect(saved!.providerTenant!.github).toEqual(before!.providerTenant!.github); + expect(saved!.providerTenant!.oauth).toMatchObject({ tokenType: "Bearer" }); + expect(saved!.providerTenant!.oauth!.refreshLease).toBeUndefined(); + await f.service.syncGitHubGrantBindings(connection!, saved!); + expect(bindings).toContainEqual(expect.objectContaining({ installationId: "101", active: false })); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)))[0]!.providerTenant!.github!.bindingSync).toBeUndefined(); + } finally { releaseRefresh?.(); await result; refreshGate = undefined; refreshArrived = undefined; vi.unstubAllEnvs(); } + }); + + it("bounds a held Cloud body without releasing revocation admission, while another company progresses and restart retains removal debt", async () => { + const f = await fixture([]), other = await fixture([]); + for (const row of [f, other]) await db.update(connectionGrants).set({ providerTenant: { + ...row.grant.providerTenant, github: { ...row.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, + } }).where(eq(connectionGrants.id, row.grant.id)); + const latest = async (id: string) => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, id)))[0]!; + heldBindingCompany = f.company.id; + let release!: () => void; + bindingGate = new Promise(resolve => { release = resolve; }); + const arrived = new Promise(resolve => { bindingArrived = resolve; }); + const closed = new Promise(resolve => { bindingClosed = resolve; }); + const timeout = AbortSignal.timeout.bind(AbortSignal); + const deadlines = vi.spyOn(AbortSignal, "timeout").mockImplementation(ms => timeout(ms === 15_000 ? 3_000 : ms)); + let completed = false; + const attempt = f.service.syncGitHubGrantBindings(f.connection, await latest(f.grant.id)); + const outcome = attempt.then(value => ({ value, error: null }), error => ({ value: null, error })).finally(() => { completed = true; }); + try { + await arrived; + const otherResult = await other.service.syncGitHubGrantBindings(other.connection, await latest(other.grant.id)); + expect(otherResult.providerTenant!.github!.bindingSync).toBeUndefined(); + await db.update(companies).set({ name: "Unrelated company changed while body held" }).where(eq(companies.id, other.company.id)); + expect(completed).toBe(false); + expect((await outcome).error).toMatchObject({ code: "CONNECTOR_BAD_RESPONSE" }); + await closed; + expect(deadlines).toHaveBeenCalledWith(15_000); + expect((await latest(f.grant.id)).providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["101"]); + heldBindingCompany = undefined; release(); + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, refreshed: 1, failed: 0 }); + expect((await latest(f.grant.id)).providerTenant!.github!.bindingSync).toBeUndefined(); + } finally { heldBindingCompany = undefined; release(); await outcome; bindingGate = undefined; bindingArrived = undefined; bindingClosed = undefined; } + }); + + it("serializes binding handoff behind canonical company deletion without a child lock cycle", async () => { + const f = await fixture([]); + const [pending] = await db.update(connectionGrants).set({ providerTenant: { + ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, + bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, + } }).where(eq(connectionGrants.id, f.grant.id)).returning(); + let release!: () => void, entered!: (pid: number) => void; + const gate = new Promise(resolve => { release = resolve; }); + const companyLocked = new Promise(resolve => { entered = resolve; }); + const transaction = db.transaction.bind(db); + // Pause after the canonical deletion's actual company lock, before its child writes. + const spy = vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: unknown) => Promise) => + transaction(async tx => { + const select = tx.select.bind(tx); + tx.select = ((...args: Parameters) => { + const query = select(...args), from = query.from.bind(query); + query.from = ((table: unknown) => { + const builder = from(table as typeof companies); + if (table === companies) { + const lock = builder.for.bind(builder); + builder.for = ((strength: Parameters[0]) => { + const locked = lock(strength); + if (strength === "no key update") { + const then = locked.then.bind(locked); + locked.then = ((resolve: (rows: unknown) => unknown, reject: (error: unknown) => unknown) => then(async rows => { + const [owner] = await tx.execute(sql`select pg_backend_pid() as pid`); + entered(Number(owner!.pid)); await gate; return resolve(rows); + }, reject)) as typeof locked.then; + } + return locked; + }) as typeof builder.for; + } + return builder; + }) as typeof query.from; + return query; + }) as typeof tx.select; + return callback(tx); + })) as typeof db.transaction); + const deletion = deleteCompany(db, f.company.id); + const deleted = Promise.allSettled([deletion]); + let handoff: Promise | undefined; + try { + const deletionPid = await companyLocked; + handoff = f.service.syncGitHubGrantBindings(f.connection, pending!); + const completed = Promise.allSettled([deletion, handoff]); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity + where datname = current_database() and query like '%"companies"%' and query like '%for share%' + and ${deletionPid} = any(pg_blocking_pids(pid))`); + expect(waiting).toHaveLength(1); + }, { timeout: 5_000, interval: 20 }); + release(); + const results = await completed; + expect(results.map(row => row.status), JSON.stringify(results)).toEqual(["fulfilled", "fulfilled"]); + expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(0); + expect(await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id))).toHaveLength(0); + expect(bindings).toHaveLength(0); + } finally { release(); await deleted; await handoff?.catch(() => {}); spy.mockRestore(); } + }); + + it("lets an admitted binding finish before canonical company deletion", async () => { + const f = await fixture([]); + const [pending] = await db.update(connectionGrants).set({ providerTenant: { + ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, + bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, + } }).where(eq(connectionGrants.id, f.grant.id)).returning(); + heldBindingCompany = f.company.id; + let release!: () => void; + bindingGate = new Promise(resolve => { release = resolve; }); + const arrived = new Promise(resolve => { bindingArrived = resolve; }); + const handoff = f.service.syncGitHubGrantBindings(f.connection, pending!); + const handedOff = Promise.allSettled([handoff]); + let deletion: Promise | undefined; + try { + await arrived; + deletion = deleteCompany(db, f.company.id); + const deleted = Promise.allSettled([deletion]); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity + where datname = current_database() and query like '%"companies"%' and query like '%for no key update%' + and cardinality(pg_blocking_pids(pid)) > 0`); + expect(waiting).toHaveLength(1); + }, { timeout: 5_000, interval: 20 }); + release(); + expect((await handedOff)[0]!.status).toBe("fulfilled"); + expect((await deleted)[0]!.status).toBe("fulfilled"); + expect(bindings).toEqual([expect.objectContaining({ grantId: f.grant.id, connectionId: f.connection.id, installationId: "101", active: false })]); + expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(0); + expect(await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id))).toHaveLength(0); + } finally { release(); await handedOff; await deletion?.catch(() => {}); heldBindingCompany = undefined; + bindingGate = undefined; bindingArrived = undefined; } + }); + + it.each(["revocation", "revision", "rollback"] as const)("revalidates a late %s after waiting for company admission", async change => { + const f = await fixture([]), revision = randomUUID(); + const [pending] = await db.update(connectionGrants).set({ providerTenant: { + ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, + bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, + } }).where(eq(connectionGrants.id, f.grant.id)).returning(); + let entered!: (pid: number) => void, release!: () => void; + const locked = new Promise(resolve => { entered = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const mutation = db.transaction(async tx => { + await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update"); + const [owner] = await tx.execute(sql`select pg_backend_pid() as pid`); + entered(Number(owner!.pid)); await gate; + if (change === "revision") await tx.update(connectionGrants).set({ providerTenant: { + ...pending!.providerTenant, github: { ...pending!.providerTenant!.github!, + bindingSync: { id: revision, removeInstallationIds: ["202"] } }, + } }).where(eq(connectionGrants.id, f.grant.id)); + else await tx.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id)); + if (change === "rollback") throw new Error("rollback late revocation"); + }); + const mutated = Promise.allSettled([mutation]); + let handoff: Promise | undefined; + try { + const ownerPid = await locked; + handoff = f.service.syncGitHubGrantBindings(f.connection, pending!); + const handedOff = Promise.allSettled([handoff]); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select pid from pg_stat_activity + where datname = current_database() and query like '%"companies"%' and query like '%for share%' + and ${ownerPid} = any(pg_blocking_pids(pid))`); + expect(waiting).toHaveLength(1); + }, { timeout: 5_000, interval: 20 }); + release(); + expect((await mutated)[0]!.status).toBe(change === "rollback" ? "rejected" : "fulfilled"); + expect((await handedOff)[0]!.status).toBe("fulfilled"); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + if (change === "rollback") { + expect(bindings).toEqual([expect.objectContaining({ grantId: f.grant.id, installationId: "101", active: false })]); + expect(saved!.status).toBe("active"); expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); + } else { + expect(bindings).toHaveLength(0); + if (change === "revocation") expect(saved!.status).toBe("revoked"); + else expect(saved!.providerTenant!.github!.bindingSync).toEqual({ id: revision, removeInstallationIds: ["202"] }); + } + } finally { release(); await mutated; await handoff?.catch(() => {}); } + }); + + it("renews an installation created without metadata using the saved connection expiry", async () => { + const f = await fixture(); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); + try { + const refs = []; + for (const path of ["oauth.access_token", "oauth.refresh_token"]) { + const saved = await writeConnectionCredential(db, { companyId: f.company.id, connectionName: "OAuth", configPath: path, label: path, + value: path === "oauth.refresh_token" ? "saved-refresh" : "expired-access" }); + refs.push({ configPath: path, secretId: saved.secret.id, versionSelector: "latest" as const }); + } + await db.update(toolConnections).set({ credentialSource: "paperclip_vault", credentialPolicy: "shared", config: { oauth: { + provider: "fixture", tokenUrl: `${origin}/oauth/token`, clientId: "fixture-client", clientRegistrationSource: "dcr", clientTokenEndpointAuthMethod: "none", + expiresAt: new Date(Date.now() - 60_000).toISOString(), + } } }).where(eq(toolConnections.id, f.connection.id)); + const api = toolAccessService(db); + const installation = await api.addConnectionInstallation(f.connection.id, { credentialSecretRefs: refs }); + const [before] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)); + expect(before!.providerTenant).toBeNull(); + expect((await db.execute(sql`select provider_tenant is null as absent from connection_grants where id = ${installation.id}`))[0]!.absent).toBe(true); + const renewed = await api.refreshOAuthGrantCredentials({ companyId: f.company.id, connectionId: f.connection.id, grantId: installation.id }); + expect(renewed.providerTenant!.oauth).toMatchObject({ tokenType: "Bearer", accessTokenExpiresAt: expect.any(String) }); + expect(Date.parse(renewed.providerTenant!.oauth!.accessTokenExpiresAt!)).toBeGreaterThan(Date.now() + 3500_000); + expect(renewed.providerTenant!.oauth!.refreshLease).toBeUndefined(); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)))[0]!.providerTenant).toEqual(renewed.providerTenant); + } finally { vi.unstubAllEnvs(); } + }, 40_000); + + it.each(["sql-null", "json-null", "object"] as const)("claims current %s metadata, fences stale snapshots and rolls back lease writes", async mode => { + const f = await fixture(); + const api = toolAccessService(db); + const installation = await api.addConnectionInstallation(f.connection.id, {}); + const metadata = mode === "sql-null" ? sql`null` : mode === "json-null" ? sql`'null'::jsonb` : sql`'{"name":"current"}'::jsonb`; + await db.update(connectionGrants).set({ providerTenant: metadata }).where(eq(connectionGrants.id, installation.id)); + const getGrant = async () => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)))[0]!; + const initial = await getGrant(); + const leases = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, getGrant); + const claimed = await leases.acquireOAuthGrantRefreshLease(f.connection, initial, true); + expect(claimed.leaseId).toBeTruthy(); + expect(claimed.grant.providerTenant!.oauth!.refreshLease!.id).toBe(claimed.leaseId); + if (mode === "object") expect(claimed.grant.providerTenant!.name).toBe("current"); + await leases.clearOAuthGrantRefreshLease(claimed.grant, claimed.leaseId!); + await db.update(connectionGrants).set({ providerTenant: metadata }).where(eq(connectionGrants.id, installation.id)); + const newer = { name: "concurrent", github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } }; + let raced = false; + const staleRead = async () => { + const snapshot = await getGrant(); + if (!raced) { raced = true; await db.update(connectionGrants).set({ providerTenant: newer }).where(eq(connectionGrants.id, installation.id)); } + return snapshot; + }; + const racingLeases = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, staleRead); + const retained = await racingLeases.acquireOAuthGrantRefreshLease(f.connection, initial, true); + expect(retained.grant.providerTenant).toMatchObject(newer); + expect(retained.leaseId).toBeTruthy(); + await racingLeases.clearOAuthGrantRefreshLease(retained.grant, retained.leaseId!); + await db.update(connectionGrants).set({ providerTenant: metadata }).where(eq(connectionGrants.id, installation.id)); + await expect(db.transaction(async tx => { + const read = async () => (await tx.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)))[0]!; + const transactional = createOAuthRefreshLeases(tx as typeof db, () => new Date(), async () => f.connection, read); + const lease = await transactional.acquireOAuthGrantRefreshLease(f.connection, await read(), true); + expect(lease.leaseId).toBeTruthy(); + throw new Error("rollback lease"); + })).rejects.toThrow("rollback lease"); + expect((await getGrant()).providerTenant).toEqual(initial.providerTenant); + const retried = await leases.acquireOAuthGrantRefreshLease(f.connection, initial, true); + expect(retried.leaseId).toBeTruthy(); + await leases.clearOAuthGrantRefreshLease(retried.grant, retried.leaseId!); + }, 40_000); + + it("lease acquire and cleanup cannot replace a concurrent webhook metadata revision", async () => { + const f = await fixture(["101"]); + let interleave = true; + const removal = { id: randomUUID(), removeInstallationIds: ["202"] }; + const getGrant = async () => { + const [snapshot] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + if (interleave) { + interleave = false; + await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify(removal)}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)); + } + return snapshot!; + }; + const leaseOwner = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, getGrant); + const lease = await leaseOwner.acquireOAuthGrantRefreshLease(f.connection, f.grant, true); + expect(lease.leaseId).toBeTruthy(); + expect(lease.grant.providerTenant!.github!.bindingSync).toEqual(removal); + // Cleanup deliberately reads a stale snapshot immediately before a new delta. + const second = { ...removal, id: randomUUID(), removeInstallationIds: ["202", "303"] }; + const cleanup = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, async () => { + const snapshot = await getGrant(); + await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify(second)}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)); + return snapshot; + }); + const cleared = await cleanup.clearOAuthGrantRefreshLease(lease.grant, lease.leaseId!); + expect(cleared.providerTenant!.github!.bindingSync).toEqual(second); + expect(cleared.providerTenant!.oauth!.refreshLease).toBeUndefined(); + }); + + it("ignores unrelated health write bursts and scopes credential reconstruction to its owning company", async () => { + const f = await fixture(["101"]), other = await fixture(["202"]); fakeClock(); + // A large unrelated company set must not enter a targeted credential reload. + for (let index = 0; index < 50; index++) { + const id = randomUUID(); companyIds.push(id); + await db.insert(companies).values({ id, name: "Unrelated credential owner", issuePrefix: `U${index}${id.slice(0, 6)}` }); + } + const [unrelatedConnection] = await db.update(toolConnections).set({ config: {}, transportConfig: {}, transport: "rest_api", authKind: "none", credentialRefs: [], credentialSecretRefs: [] }).where(eq(toolConnections.id, other.connection.id)).returning(); + await db.update(connectionGrants).set({ credentialSecretRefs: [] }).where(eq(connectionGrants.id, other.grant.id)); + const api = toolAccessService(db); + const loadTimes: number[] = []; + const schedule = f.service.githubConnectionContinuitySchedule.bind(f.service); + const load = vi.spyOn(f.service, "githubConnectionContinuitySchedule").mockImplementation(companyId => { loadTimes.push(Date.now()); return schedule(companyId); }); + const sweep = vi.spyOn(f.service, "sweepGitHubConnectionContinuity"); + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const continuity = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); + const events = registerGitHubConnectionEventWork(db, coordinator, githubConnectionEventService(db, { connector: f.connector }), () => true); + try { + await Promise.all([continuity.ready, events.ready]); + await vi.waitFor(() => expect(leases).toBe(1)); + await vi.waitFor(() => expect(load).toHaveBeenCalledTimes(1)); + load.mockClear(); loadTimes.length = 0; sweep.mockClear(); + for (let i = 0; i < 30; i++) { + await db.transaction(async tx => { await notifyToolConnectionHealth(tx, other.connection); await notifyChatEndpointConfiguration(tx, { id: randomUUID(), provider: "discord" }); }); + await api.updateConnection(unrelatedConnection!.id, { name: `Unrelated patch ${i}` }); + } + await vi.advanceTimersByTimeAsync(10_000); + expect(load).not.toHaveBeenCalled(); expect(sweep).not.toHaveBeenCalled(); expect(leases).toBe(1); + expect(errors).toEqual([]); + for (let index = 0; index < 30; index++) await db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); }); + expect(leases).toBeLessThanOrEqual(2); + await vi.advanceTimersByTimeAsync(6_000); + await vi.waitFor(() => expect(load).toHaveBeenCalledWith(f.company.id)); + expect(load.mock.calls.every(([companyId]) => companyId === f.company.id)).toBe(true); + expect(sweep).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(leases).toBeGreaterThanOrEqual(2)); + // The first hint can run immediately; changes during it get one spaced pass. + expect(leases).toBeLessThanOrEqual(3); expect(load.mock.calls.length).toBeLessThanOrEqual(2); + if (loadTimes.length === 2) expect(loadTimes[1]! - loadTimes[0]!).toBeGreaterThanOrEqual(5_000); + if (leaseTimes.length === 3) expect(leaseTimes[2]! - leaseTimes[1]!).toBeGreaterThanOrEqual(5_000); + // A hint arriving during the first SQL pass retains one spaced generation. + // Start the empty-window assertion after that concrete work has settled. + await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); + expect(leases).toBeLessThanOrEqual(3); expect(load.mock.calls.length).toBeLessThanOrEqual(2); + if (loadTimes.length === 2) expect(loadTimes[1]! - loadTimes[0]!).toBeGreaterThanOrEqual(5_000); + const quiet = { leases, loads: load.mock.calls.length }; + await vi.advanceTimersByTimeAsync(60_000); + expect({ leases, loads: load.mock.calls.length }).toEqual(quiet); + expect(errors).toEqual([]); + } finally { await events.stop(); await continuity.stop(); await coordinator.stop(); } + }); + + it("settles only the company and grant that own saved binding debt", async () => { + const f = await fixture(["101"]), other = await fixture(["202"]); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, + bindingSync: { id: randomUUID(), removeInstallationIds: [] } } } }).where(eq(connectionGrants.id, f.grant.id)); + await f.service.sweepGitHubConnectionContinuity(); + expect(bindings).toHaveLength(1); expect(bindings[0]).toMatchObject({ grantId: f.grant.id, installationId: "101" }); + const [untouched] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, other.grant.id)); + expect(untouched!.providerTenant).toEqual(other.grant.providerTenant); + }); + + it("retains a real deadline when successful renewal leaves a short-lived credential unchanged", async () => { + const f = await fixture(["101"]); fakeClock(); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + expect(await f.service.sweepGitHubConnectionContinuity()).toMatchObject({ due: 1, refreshed: 1 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + const retryAt = Date.parse(saved!.providerTenant!.github!.continuityRetry!.at); + expect(retryAt).toBeGreaterThanOrEqual(Date.now() + 59 * 60_000); + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBe(retryAt); + const requests = githubRequests; await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(await restarted.sweepGitHubConnectionContinuity()).toMatchObject({ due: 0 }); expect(githubRequests).toBe(requests); + }); + + it("outer commit, rollback and first producer during drain retain the deadline and resume obligation without cross-company work", async () => { + const f = await fixture(["101"]); fakeClock(); + let enabled = false; + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => enabled, canReconcile: () => true, onError: error => errors.push(error) }); + const worker = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => enabled); + try { + await worker.ready; + const dueAt = Date.now() + 90 * 60_000; + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + const transaction = db.transaction(async tx => { + await tx.transaction(async nested => { + await notifyGitHubCredentialChange(nested, f.connection); + await nested.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(dueAt + 3600_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + }); + await held; + }); + await new Promise(resolve => setImmediate(resolve)); + expect(bindings).toHaveLength(0); release(); await transaction; + await vi.advanceTimersByTimeAsync(31_000); + expect(idleWorkSnapshot().active).toBe(0); expect(bindings).toHaveLength(0); + enabled = true; await vi.advanceTimersByTimeAsync(31_000); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(dueAt)); + await expect(db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); + await tx.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id)); throw new Error("rollback"); })).rejects.toThrow("rollback"); + await vi.advanceTimersByTimeAsync(31_000); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(dueAt)); expect(errors).toEqual([]); + } finally { await worker.stop(); await coordinator.stop(); } + }); +}); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 43a5887f870..45381053729 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -48,6 +48,7 @@ const apiPrefixes: Record = { "company-skills.ts": "/api", "company-skill-policy.ts": "/api", "connection-intents.ts": "/api", + "connection-event-wake.ts": "/", "costs.ts": "/api", "dashboard.ts": "/api", "dot-runner.ts": "/api", @@ -114,6 +115,8 @@ const HTTP_METHODS = new Set([ const explicitOpenApiCoverageExclusions = new Set(); const explicitOpenApiOperationCoverageExclusions = new Set([ + // Exact Cloud-control signed action; ordinary board/agent credentials cannot invoke it. + "POST /api/instance/connector-events/wake", // Muse uses its closed, versioned receiver protocol and dedicated credentials, // documented in doc/muse-personal-agent.md and the shared protocol contract. // Board connection/qualification management remains in this OpenAPI document. @@ -181,6 +184,7 @@ function normalizeExpressPath(routePath: string) { } function resolveMountedPath(file: string, prefix: string, routePath: string) { + if (file === "connection-event-wake.ts") return routePath; if (file === "public-mcp.ts" && (routePath.startsWith("/mcp/oauth/") || routePath.startsWith("/.well-known/"))) return routePath; if ( (file === "chat-channels.ts" || file === "email.ts") && diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 63b607e0019..c351f7e0ff6 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,6 +1,6 @@ -import { routineExecutionStatusService } from "../services/routine-run-status.js"; -import { attentionRetentionService } from "../services/attention-retention.js"; import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js"; +import { attentionRetentionService } from "../services/attention-retention.js"; +import { routineExecutionStatusService } from "../services/routine-run-status.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -14,7 +14,7 @@ import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; import * as idleLocalWork from "../services/idle-local-work.js"; import { idleBackupWakeMarker, markIdleBackupWakeRequired } from "../services/idle-database-backup.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; -import { decisionService, decisionRetentionService, toolAccessService, reconcileBuiltInAgentsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup } from "../services/index.js"; +import { decisionService, decisionRetentionService, toolAccessService, githubConnectionEventService, reconcileBuiltInAgentsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup } from "../services/index.js"; import { companies, runDatabaseBackup, type Db } from "@paperclipai/db"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -39,6 +39,13 @@ vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudCon vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify })); vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) })); +// Producer transaction/notification behavior is covered by the real PostgreSQL +// GitHub worker suite; this startup seam supplies a lightweight database mock. +vi.mock("../services/delivery-work-notifications.js", async () => ({ + ...await vi.importActual("../services/delivery-work-notifications.js"), + subscribeDeliveryWork: vi.fn(() => () => {}), +})); + const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON; @@ -451,6 +458,7 @@ vi.mock("../services/index.js", () => ({ externalObjectService: externalObjectsServiceFactoryMock, heartbeatService: heartbeatServiceFactoryMock, githubConnectionEventService: vi.fn(() => ({ + nextPollAt: vi.fn(() => null), pollOnce: vi.fn(async () => ({ leased: 0, processed: 0, @@ -502,6 +510,8 @@ vi.mock("../services/index.js", () => ({ needsAttention: 0, failed: 0, })), + nextGitHubConnectionContinuityAt: vi.fn(async () => null), + githubConnectionContinuitySchedule: vi.fn(async () => []), sweepGitHubConnectionContinuity: vi.fn(async () => ({ checked: 0, due: 0, @@ -1357,6 +1367,32 @@ describe("startServer feedback export wiring", () => { expect(completionSweepMock).not.toHaveBeenCalled(); }); + it.each([true, false])("registers GitHub delivery/deadline workers once without aggregate polling (scheduler=%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30_000 })); + const ticks: Array<() => void> = []; + const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, ms: number) => { + if (ms === 30_000) ticks.push(callback); + return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + for (const queue of [DELIVERY_QUEUES.connectionEvents, DELIVERY_QUEUES.connectionContinuity]) { + expect(deliveryWorkRegisterMock.mock.calls.filter(([registered]) => registered === queue)).toHaveLength(1); + } + const polls = vi.mocked(githubConnectionEventService).mock.results.map(result => result.value.pollOnce); + const sweeps = vi.mocked(toolAccessService).mock.results.map(result => result.value.sweepGitHubConnectionContinuity); + const schedules = vi.mocked(toolAccessService).mock.results.map(result => result.value.githubConnectionContinuitySchedule); + expect(sweeps.reduce((sum, call) => sum + call.mock.calls.length, 0)).toBe(0); + const count = () => [...polls, ...sweeps, ...schedules].reduce((sum, call) => sum + call.mock.calls.length, 0); + const startup = count(); + expect(startup).toBe(2); + for (let i = 0; i < 3; i++) for (const tick of ticks) tick(); + await new Promise(resolve => setImmediate(resolve)); + expect(count()).toBe(startup); + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.connectionEvents || error.queue === DELIVERY_QUEUES.connectionContinuity)).toEqual([]); + } finally { intervals.mockRestore(); } + }); + it("does not rescan idle tool health from the aggregate maintenance tick", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, heartbeatSchedulerIntervalMs: 30_000 })); let tick: (() => void) | undefined; @@ -1498,12 +1534,12 @@ describe("startServer feedback export wiring", () => { await Promise.resolve(); expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); - expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); - expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); - expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalledTimes(1); expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled(); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); const routineWorkerIndex = deliveryWorkRegisterMock.mock.calls.findIndex(([queue]) => queue === DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/app.ts b/server/src/app.ts index 344bd0dd061..8a985cbc2de 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,3 +1,4 @@ +import { connectionEventWakeRoutes } from "./routes/connection-event-wake.js"; import { registerGitHubReviewProjectionWork } from "./services/chat-channels/github/review-projection-work.js"; import { finishServiceShutdown } from "./shutdown.js"; import { promptInjectionPreventionRoutes } from "./routes/prompt-injection-prevention.js"; @@ -582,6 +583,7 @@ export async function createApp( // REPLACES whatever actor the request otherwise resolved to, and only on // the one endpoint it authorizes (see the middleware for the contract). app.use(cloudControlMiddleware()); + app.use(connectionEventWakeRoutes(db)); app.use(skillAccessRequestContext); app.use("/api/auth", authRoutes(db)); if (opts.betterAuthHandler) { diff --git a/server/src/index.ts b/server/src/index.ts index 39a2cc95960..63dcb5c0b76 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,8 +1,9 @@ -import { registerNativeHarnessBackupExpirationWork } from "./services/native-runtime/native-harness-backup-expiration-work.js"; -import { registerMergeConfirmationWork } from "./services/merged-pr-confirmation-work.js"; +import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js"; import { registerRoutineExecutionStatusWork } from "./services/routine-execution-status-work.js"; import { routineExecutionStatusService } from "./services/routine-run-status.js"; -import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js"; +import { registerMergeConfirmationWork } from "./services/merged-pr-confirmation-work.js"; +import { registerNativeHarnessBackupExpirationWork } from "./services/native-runtime/native-harness-backup-expiration-work.js"; +import { registerGitHubConnectionEventWork, registerGitHubConnectionContinuityWork } from "./services/github-connection-work.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -1459,35 +1460,13 @@ async function startServerWithDatabaseTeardown( ?? process.env.PAPERCLIP_TOOL_RUNTIME_TRUSTED_HOST ?? null, }); - const scheduleGitHubConnectionEventPoll = (forSetup = false) => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - const work = githubConnectionEvents.pollOnce({ forSetup }) - .then((result) => { - if (result.leased > 0 || result.failed > 0) { - logger.info(result, "GitHub connection event poll completed"); - } - }) - .catch((err) => { - logger.error({ err }, "GitHub connection event poll failed"); - }); - trackHeartbeatSchedulerWork(work); - return work; - }; + const githubCanRun = () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped; + const githubEventWorker = registerGitHubConnectionEventWork(db, deliveryWork, githubConnectionEvents, githubCanRun); + const githubContinuityWorker = registerGitHubConnectionContinuityWork(db, deliveryWork, tools, githubCanRun); + server.on("close", () => { void githubEventWorker.stop(); void githubContinuityWorker.stop(); }); server.on("close", registerGitHubBotCloudSetupPoller(db, async () => { - await scheduleGitHubConnectionEventPoll(true); + if (githubCanRun()) await githubConnectionEvents.pollOnce({ forSetup: true }); })); - const scheduleGitHubConnectionContinuitySweep = () => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(tools.sweepGitHubConnectionContinuity() - .then((result) => { - if (result.due > 0 || result.failed > 0) { - logger.info(result, "GitHub connection continuity sweep completed"); - } - }) - .catch((err) => { - logger.error({ err }, "GitHub connection continuity sweep failed"); - })); - }; await timeStartupRecovery("tool_reviews", async () => { if (config.heartbeatSchedulerEnabled) { @@ -1518,8 +1497,7 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready : app.locals.toolGateway.cleanupExpiredSessions()).catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed")); await timeStartupRecovery("delivery_workers", () => Promise.all([budgetEnforcementWorker, accountingWorker, fastResponseWorker, ...deliveryWorkers].map(worker => worker.ready))); - scheduleGitHubConnectionEventPoll(); - scheduleGitHubConnectionContinuitySweep(); + await Promise.all([githubEventWorker.ready, githubContinuityWorker.ready]); if (heartbeat) { const secretProposals = createSecretProposalsService(db as any); @@ -1911,8 +1889,6 @@ async function startServerWithDatabaseTeardown( if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleGitHubConnectionEventPoll(); - scheduleGitHubConnectionContinuitySweep(); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; @@ -1974,8 +1950,6 @@ async function startServerWithDatabaseTeardown( // as the enabled path. await startEnvironmentCleanup(); startHeartbeatSchedulerInterval(() => { - scheduleGitHubConnectionEventPoll(); - scheduleGitHubConnectionContinuitySweep(); }); } diff --git a/server/src/middleware/cloud-control.ts b/server/src/middleware/cloud-control.ts index f0e44503e8b..b726aae3092 100644 --- a/server/src/middleware/cloud-control.ts +++ b/server/src/middleware/cloud-control.ts @@ -8,6 +8,7 @@ import { /** Endpoint → method → the one action a control assertion must name to take it. */ const ACTIONS_BY_ENDPOINT: Record> = { + "/api/instance/connector-events/wake": { POST: "connector-events:wake" }, "/api/instance/task-drain": { GET: "task-drain:read", POST: "task-drain:start", diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 2309741de26..9780b880bb9 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,6 +1,6 @@ -import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js"; import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; diff --git a/server/src/routes/connection-event-wake.ts b/server/src/routes/connection-event-wake.ts new file mode 100644 index 00000000000..c4a71e87717 --- /dev/null +++ b/server/src/routes/connection-event-wake.ts @@ -0,0 +1,13 @@ +import { Router } from "express"; +import { wakeConnectionEvents } from "../services/connection-event-wake.js"; + +export function connectionEventWakeRoutes(owner: object) { + const router = Router(); + router.post("/api/instance/connector-events/wake", (req, res) => { + if (req.actor.source !== "cloud_control") { res.status(403).json({ error: "cloud_control_required" }); return; } + res.set("Cache-Control", "no-store"); + if (!wakeConnectionEvents(owner)) { res.status(503).json({ error: "connection_event_receiver_unavailable" }); return; } + res.status(202).json({ accepted: true }); + }); + return router; +} diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 745dd9ff1ab..5354c0a5b5a 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -1,6 +1,6 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 8c3defcbd86..ca18e8f02ec 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,5 +1,5 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index b7d42462ba0..c5ae14996ad 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,6 +1,6 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { isAgentStatusInvokable } from "@paperclipai/shared"; import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index effbb8577e0..3d5c2c00352 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,7 +1,7 @@ -import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; -import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; @@ -327,8 +327,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await notifyRunDispatch(db, { companyId: policy.companyId }); await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); - await notifyExternalObjectRefresh(db, policy.companyId); await notifyStatusCardIssueChange(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -393,8 +393,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await notifyRunDispatch(db, { companyId: policy.companyId }); await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); - await notifyExternalObjectRefresh(db, policy.companyId); await notifyStatusCardIssueChange(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/chat-channels/configuration-work-notifications.ts b/server/src/services/chat-channels/configuration-work-notifications.ts index 9d3240aee12..8a301b7cd0f 100644 --- a/server/src/services/chat-channels/configuration-work-notifications.ts +++ b/server/src/services/chat-channels/configuration-work-notifications.ts @@ -1,3 +1,4 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyGitHubWebhookRecovery } from "./github/webhook-recovery-notifications.js"; import { notifyVoiceChange } from "../voice/voice-recovery-notifications.js"; import { notifyChatReceiver, notifyChatConnectionReceiver, notifyChatCompanyReceivers } from "./runtime/receiver-work-notifications.js"; @@ -7,7 +8,10 @@ import { notifyDiscordCommandRegistration } from "./discord/command-work-notific export async function notifyChatEndpointConfiguration(transaction: object, endpoint: { id: string; provider: ChatProvider }) { if (endpoint.provider === "speko") await notifyVoiceChange(transaction, { kind: "endpoint", id: endpoint.id }); await notifyChatReceiver(transaction, endpoint); - if (endpoint.provider === "github") await notifyGitHubWebhookRecovery(transaction, { kind: "endpoint", id: endpoint.id }); + if (endpoint.provider === "github") { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents); + await notifyGitHubWebhookRecovery(transaction, { kind: "endpoint", id: endpoint.id }); + } if (endpoint.provider === "discord") { await notifyDiscordCommandRegistration(transaction, { kind: "endpoint", id: endpoint.id }); } diff --git a/server/src/services/chat-channels/configuration.ts b/server/src/services/chat-channels/configuration.ts index fa1b1d5b1ce..8ded7cbb96e 100644 --- a/server/src/services/chat-channels/configuration.ts +++ b/server/src/services/chat-channels/configuration.ts @@ -1,3 +1,4 @@ +import { notifyChatEndpointConfiguration } from "./configuration-work-notifications.js"; import { notifyChatConnectionHealth } from "./connection-health-notifications.js"; import type { ConnectionManagementContext } from "../connection-management.js"; import { CHAT_HUMAN_CREDENTIAL_FIELDS } from "./credentials.js"; @@ -196,6 +197,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption code: "chat_endpoint_not_active", }); } + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -334,6 +336,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption inArray(chatDeliveries.state, ["received", "retry"]), ), ); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); [activatedEndpoint] = await tx .update(chatEndpoints) @@ -383,6 +386,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -487,6 +491,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption code: "chat_endpoint_already_removed", }); } + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -721,6 +726,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); const claimed = await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); const rows = await tx .update(chatEndpoints) @@ -800,6 +806,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await installSpekoVoiceTools(tx, endpoint, actorUserId); } const preserveSlackVerification = preservesProvisionedSlackVerification(current.setup, credentials.signingSecret ?? "", `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/slack`); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -887,6 +894,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) diff --git a/server/src/services/chat-github-wizard.ts b/server/src/services/chat-github-wizard.ts index cd5f38a1c75..abde5141afd 100644 --- a/server/src/services/chat-github-wizard.ts +++ b/server/src/services/chat-github-wizard.ts @@ -1,3 +1,4 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyGitHubWebhookRecovery } from "./chat-channels/github/webhook-recovery-notifications.js"; import type { AuthorizationActor } from "./authorization.js"; import { connectionManagementService, type ConnectionManagementContext } from "./connection-management.js"; @@ -173,8 +174,10 @@ export function githubChatWizardService( values: Partial>, ) { await db.transaction(async tx => { - if (values.cloudRegistrationId !== undefined) + if (values.cloudRegistrationId !== undefined) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents, bot.companyId); await notifyGitHubWebhookRecovery(tx, { kind: "endpoint", id: bot.id }); + } await tx .update(chatEndpoints) .set({ diff --git a/server/src/services/cloud-runtime-identity.ts b/server/src/services/cloud-runtime-identity.ts index 435c7f7562b..80b3333b36f 100644 --- a/server/src/services/cloud-runtime-identity.ts +++ b/server/src/services/cloud-runtime-identity.ts @@ -451,6 +451,7 @@ export const CLOUD_CONTROL_ACTIONS = [ "task-drain:stop", "lifecycle:read", "lifecycle:unarchive-primary", + "connector-events:wake", ] as const; export type CloudControlAction = (typeof CLOUD_CONTROL_ACTIONS)[number]; diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 04552f28fe4..8fbe9aa6f33 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,6 +1,7 @@ -import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyGitHubCompanyChange } from "./github-connection-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; @@ -337,6 +338,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { await notifyRunDispatch(tx, { companyId: id }); + await notifyGitHubCompanyChange(tx, id); await notifyDeferredExecutionEligibility(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); @@ -497,6 +499,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (!wasAlreadyArchived) { await notifyRunDispatch(tx, { companyId: id }); + await notifyGitHubCompanyChange(tx, id); await notifyDeferredExecutionEligibility(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 37894b17113..66e81df7673 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,5 +1,5 @@ -import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; diff --git a/server/src/services/connection-event-wake.ts b/server/src/services/connection-event-wake.ts new file mode 100644 index 00000000000..87840eb0979 --- /dev/null +++ b/server/src/services/connection-event-wake.ts @@ -0,0 +1,12 @@ +/** Process-local hint only; Cloud retains the durable target until event acknowledgment. */ +const receivers = new WeakMap void>(); +export function registerConnectionEventWake(owner: object, wake: () => void) { + receivers.set(owner, wake); + return () => { if (receivers.get(owner) === wake) receivers.delete(owner); }; +} +export function wakeConnectionEvents(owner: object): boolean { + const wake = receivers.get(owner); + if (!wake) return false; + wake(); + return true; +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 99a1a270145..15bbd0ba5eb 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -122,6 +122,8 @@ export const DELIVERY_QUEUES = { feedback: "feedback-exports", chatCompletion: "chat-completions", connection: "connection-continuations", + connectionEvents: "connection-events", + connectionContinuity: "connection-continuity", gitRestorePoints: "git-restore-points", remoteWorkspaceSnapshots: "remote-workspace-snapshots", nativeWorkspaceSyncRetention: "native-workspace-sync-retention", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 7d3746159dd..894fb7d835b 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,20 +1,20 @@ -import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; -import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { - if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); if (scope) { await notifyAttentionRetentionWork(transaction, scope); - await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyRunDispatch(transaction, scope); + await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyDeferredExecutionEligibility(transaction, scope); + await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); } for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors, @@ -30,10 +30,10 @@ export function writeExecutionControl( scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); - if (scope) await notifyAttentionRetentionWork(transaction, scope); - if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyRunDispatch(transaction, scope); + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); + if (scope) await notifyAttentionRetentionWork(transaction, scope); + if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); if (scope) await notifyDeferredExecutionEligibility(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); diff --git a/server/src/services/github-connection-events.test.ts b/server/src/services/github-connection-events.test.ts index 93031ecae62..35ebfd07d5a 100644 --- a/server/src/services/github-connection-events.test.ts +++ b/server/src/services/github-connection-events.test.ts @@ -3,7 +3,7 @@ import { chatEndpoints, type Db } from "@paperclipai/db"; import { githubConnectionEventService } from "./github-connection-events.js"; import type { PaperclipCloudConnector } from "./paperclip-cloud-connector.js"; -function fixture() { +function fixture(push = false, forcePolling = false) { let time = Date.parse("2026-09-04T12:00:00Z"); const bindings = [{ grant: { id: "grant", companyId: "company", kind: "user", subjectUserId: "owner", status: "active", @@ -13,13 +13,14 @@ function fixture() { const bots: Array<{ companyId: string }> = []; // Only SQL is stubbed. Backoff, lease admission and concurrent execution use // the real service; relational persistence has its own Postgres suite. + const result = (rows: T[]) => Object.assign(Promise.resolve(rows), { limit: async () => rows.slice(0, 1) }); const select = vi.fn(() => ({ from: (table: unknown) => table === chatEndpoints - ? { where: async () => bots } - : { innerJoin: () => ({ where: async () => bindings }) } })); + ? { where: () => result(bots) } + : { innerJoin: () => ({ where: () => result(bindings) }) } })); const leaseEvents = vi.fn().mockResolvedValue(null); - const connector = { leaseEvents } as unknown as PaperclipCloudConnector; - const service = githubConnectionEventService({ select } as unknown as Db, { connector, now: () => new Date(time) }); - return { service, leaseEvents, select, bots, advance: (ms: number) => { time += ms; } }; + const connector = { leaseEvents, eventWakeVersion: () => push ? 1 : null } as unknown as PaperclipCloudConnector; + const service = githubConnectionEventService({ select } as unknown as Db, { connector, env: forcePolling ? { PAPERCLIP_GITHUB_EVENT_POLLING: "1" } : {}, now: () => new Date(time) }); + return { service, leaseEvents, select, bots, bindings, advance: (ms: number) => { time += ms; } }; } describe("GitHub event polling during active App setup", () => { @@ -72,3 +73,36 @@ describe("GitHub event polling during active App setup", () => { expect(f.leaseEvents).toHaveBeenCalledTimes(2); }); }); + +describe("GitHub external delivery activation", () => { + it("does no SQL/network with no connector configuration after startup or cadence windows", async () => { + const select = vi.fn(() => { throw new Error("unconfigured worker touched SQL"); }); + const service = githubConnectionEventService({ select } as unknown as Db, { env: {} }); + for (let i = 0; i < 4; i++) await service.pollOnce({ forDelivery: true }); + expect(service.nextPollAt()).toBeNull(); expect(select).not.toHaveBeenCalled(); + }); + it("settles configured-idle after enrollment and wakes only on explicit delivery hints", async () => { + const f = fixture(true); + await f.service.pollOnce(); expect(f.service.nextPollAt()).toBeNull(); + const reads = f.select.mock.calls.length; + for (let i = 0; i < 4; i++) { f.advance(300_000); await f.service.pollOnce(); } + expect(f.select).toHaveBeenCalledTimes(reads); expect(f.leaseEvents).toHaveBeenCalledTimes(1); + await f.service.pollOnce({ forDelivery: true }); expect(f.leaseEvents).toHaveBeenCalledTimes(2); + expect(f.service.nextPollAt()).toBeNull(); + }); + it("arms forced fallback before a failed Cloud enrollment revocation", async () => { + const f = fixture(true, true); + f.leaseEvents.mockRejectedValueOnce(new Error("handoff unavailable")); + await expect(f.service.pollOnce()).rejects.toThrow("handoff unavailable"); + expect(f.service.nextPollAt()).not.toBeNull(); + expect(f.leaseEvents.mock.calls[0]![0].eventWakeVersion).toBe(0); + f.advance(5_000); await f.service.pollOnce(); + expect(f.leaseEvents).toHaveBeenCalledTimes(2); expect(f.service.nextPollAt()).not.toBeNull(); + }); + it("retains old-Cloud fallback and settles once all bindings are disabled", async () => { + const f = fixture(); await f.service.pollOnce(); expect(f.service.nextPollAt()).not.toBeNull(); + f.bindings.length = 0; f.advance(10_000); await f.service.pollOnce(); + expect(f.service.nextPollAt()).toBeNull(); const reads = f.select.mock.calls.length; + f.advance(3600_000); await f.service.pollOnce(); expect(f.select).toHaveBeenCalledTimes(reads); + }); +}); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 643bc557b93..af936353bdd 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -1,8 +1,10 @@ import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js"; import { chatEndpoints } from "@paperclipai/db"; import { randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; import { connectionEventDeliveries, connectionGrants, @@ -10,7 +12,7 @@ import { toolConnections, type Db, } from "@paperclipai/db"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, ne, sql } from "drizzle-orm"; import { logActivity, publishActivity, @@ -222,6 +224,22 @@ export function githubConnectionEventService( ) { const now = options.now ?? (() => new Date()); let nextPollAt = 0; + let configuredConnector: { + config: NonNullable>; + connector: PaperclipCloudConnector; + } | undefined; + function currentConnector() { + if (options.connector) return options.connector; + // Enrollment and identity replacement can happen after this owner starts. + // Read only on an admitted pass; preserve negotiated delivery while unchanged. + const config = paperclipCloudConnectorConfigFromEnv(options.env); + if (!config) { configuredConnector = undefined; return null; } + if (!configuredConnector || !isDeepStrictEqual(configuredConnector.config, config)) { + configuredConnector = { config, connector: createPaperclipCloudConnector({ config }) }; + } + return configuredConnector.connector; + } + const forcePolling = (options.env ?? process.env).PAPERCLIP_GITHUB_EVENT_POLLING === "1"; let emptyPolls = 0; let nextSetupPollAt = 0; let inFlight: Promise | null = null; @@ -234,50 +252,51 @@ export function githubConnectionEventService( eq(toolConnections.companyId, connectionGrants.companyId), )) .where(and( + sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, + sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`, eq(connectionGrants.status, "active"), eq(toolConnections.status, "active"), eq(toolConnections.enabled, true), + sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`, )); return bindingRows(rows); } - async function applyPullRequestEvent(companyId: string, event: LeasedEvent) { + async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise>) { const snapshot = githubSnapshotUpdate(event.payload); if (!snapshot) return; const appliedAt = now(); - await db.transaction(async tx => { - await notifyExternalObjectRefresh(tx, companyId, { providerKey: "github", objectType: "pull_request", - externalId: snapshot.externalId, ignoreExternalIdCase: true }); - await tx.update(externalObjects).set({ - statusKey: snapshot.statusKey, - statusLabel: snapshot.statusLabel, - statusCategory: snapshot.statusCategory, - statusTone: snapshot.statusTone, - statusIconKey: snapshot.statusIconKey, - isTerminal: snapshot.merged || snapshot.state === "closed", - data: sql`${externalObjects.data} || ${JSON.stringify(snapshot.data)}::jsonb`, - remoteVersion: snapshot.remoteVersion, - lastResolvedAt: appliedAt, - lastChangedAt: appliedAt, - nextRefreshAt: appliedAt, - refreshStartedAt: null, - refreshToken: null, - updatedAt: appliedAt, - }).where(and( - eq(externalObjects.companyId, companyId), - eq(externalObjects.providerKey, "github"), - eq(externalObjects.objectType, "pull_request"), - sql`lower(${externalObjects.externalId}) = lower(${snapshot.externalId})`, - )); - }); + await notifyExternalObjectRefresh(database, companyId, { providerKey: "github", objectType: "pull_request", + externalId: snapshot.externalId, ignoreExternalIdCase: true }); + await database.update(externalObjects).set({ + statusKey: snapshot.statusKey, + statusLabel: snapshot.statusLabel, + statusCategory: snapshot.statusCategory, + statusTone: snapshot.statusTone, + statusIconKey: snapshot.statusIconKey, + isTerminal: snapshot.merged || snapshot.state === "closed", + data: sql`${externalObjects.data} || ${JSON.stringify(snapshot.data)}::jsonb`, + remoteVersion: snapshot.remoteVersion, + lastResolvedAt: appliedAt, + lastChangedAt: appliedAt, + nextRefreshAt: appliedAt, + refreshStartedAt: null, + refreshToken: null, + updatedAt: appliedAt, + }).where(and( + eq(externalObjects.companyId, companyId), + eq(externalObjects.providerKey, "github"), + eq(externalObjects.objectType, "pull_request"), + sql`lower(${externalObjects.externalId}) = lower(${snapshot.externalId})`, + )); if (snapshot.merged && event.action === "closed") { - await issueThreadInteractionService(db, { wakeup: options.wakeup }) + await issueThreadInteractionService(database, { wakeup: options.wakeup }) .sweepMergedPullRequestConfirmations([{ companyId, owner: snapshot.owner, repo: snapshot.repo, number: snapshot.number, - }]); + }], { companyId }, undefined, { deferConfirmationCommitEffects: effect => effects.push(effect) }); } } @@ -325,6 +344,11 @@ export function githubConnectionEventService( // Lifecycle webhooks carry IDs, not the user token’s complete repository view. // Discard the snapshot until Refresh access verifies it again. accessRevision: randomUUID(), + ...(unavailable || github.bindingSync || !github.installationIds.includes(binding.installationId) ? { bindingSync: { + id: randomUUID(), removeInstallationIds: [...new Set([ + ...(github.bindingSync?.removeInstallationIds ?? []), ...(unavailable ? [binding.installationId] : []), + ])].filter(id => !installationIds.includes(id)), + } } : {}), repositories: undefined, installationIds, installationCount: installationIds.length, @@ -334,6 +358,7 @@ export function githubConnectionEventService( webhookHealth: unavailable ? "unhealthy" as const : "healthy" as const, }, }; + await notifyGitHubCredentialChange(database, { companyId: binding.companyId, config: { sourceTemplateKey: "github" } }); await database.update(connectionGrants).set({ providerTenant, updatedAt: now() }) .where(and(eq(connectionGrants.id, binding.grantId), eq(connectionGrants.companyId, binding.companyId))); await notifyToolConnectionHealth(database, { companyId: binding.companyId, id: binding.connectionId }); @@ -373,17 +398,22 @@ export function githubConnectionEventService( eq(connectionEventDeliveries.providerDeliveryId, event.id), )).limit(1); if (existing?.status === "processed") return "duplicate" as const; - await db.update(connectionEventDeliveries).set({ - status: "received", - attempts: sql`${connectionEventDeliveries.attempts} + 1`, - lastError: null, - updatedAt: receiptAt, - }).where(eq(connectionEventDeliveries.id, existing!.id)); + } const postCommitPublications: ActivityPublication[] = []; + const postCommitEffects: Array<(committedDb: Db) => Promise> = []; + let applied = true; try { const applyAndFinalize = async (database: Db) => { - if (event.event === "pull_request") await applyPullRequestEvent(companyId, normalizedEvent); + const [locked] = await database.select().from(connectionEventDeliveries).where(and( + eq(connectionEventDeliveries.companyId, companyId), eq(connectionEventDeliveries.provider, event.provider), + eq(connectionEventDeliveries.providerDeliveryId, event.id), + )).for("update").limit(1); + if (locked!.status === "processed") { applied = false; return; } + if (!receipt) await database.update(connectionEventDeliveries).set({ + status: "received", attempts: sql`${connectionEventDeliveries.attempts} + 1`, lastError: null, updatedAt: receiptAt, + }).where(eq(connectionEventDeliveries.id, locked!.id)); + if (event.event === "pull_request") await applyPullRequestEvent(database, companyId, normalizedEvent, postCommitEffects); if (event.event === "installation" || event.event === "installation_repositories") { for (const binding of bindings) await applyInstallationEvent(database, binding, normalizedEvent); } else { @@ -435,11 +465,7 @@ export function githubConnectionEventService( }, }, postCommitPublications); }; - if (event.event === "installation" || event.event === "installation_repositories") { - await db.transaction(async (tx) => applyAndFinalize(tx as unknown as Db)); - } else { - await applyAndFinalize(db); - } + await db.transaction(async (tx) => applyAndFinalize(tx as unknown as Db)); } catch (error) { await db.update(connectionEventDeliveries).set({ status: "failed", @@ -449,6 +475,7 @@ export function githubConnectionEventService( eq(connectionEventDeliveries.companyId, companyId), eq(connectionEventDeliveries.provider, event.provider), eq(connectionEventDeliveries.providerDeliveryId, event.id), + ne(connectionEventDeliveries.status, "processed"), )); throw error; } @@ -466,31 +493,47 @@ export function githubConnectionEventService( }, "GitHub webhook activity publication failed after commit"); } } - return "processed" as const; + for (const effect of postCommitEffects) await effect(db); + return applied ? "processed" as const : "duplicate" as const; } async function runPoll(): Promise { - const bindings = await activeBindings(); - const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} <> 'archived'`)); - if (bindings.length === 0 && bots.length === 0) { - nextPollAt = now().getTime() + 5_000; - return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; - } - const config = options.connector ? null : paperclipCloudConnectorConfigFromEnv(options.env); - const connector = options.connector ?? (config ? createPaperclipCloudConnector({ config }) : null); + const connector = currentConnector(); if (!connector) { - nextPollAt = now().getTime() + 5 * 60_000; + nextPollAt = Number.POSITIVE_INFINITY; return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; } - const first = bindings[0] ?? { subject: "github-bot-events", companyId: bots[0]!.companyId }; - const lease = await connector.leaseEvents({ subject: first.subject, companyId: first.companyId }); + const bindings = await activeBindings(); + const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + let identity: { subject: string; companyId: string } | null = bindings[0] ?? (bots[0] ? { subject: "github-bot-events", companyId: bots[0].companyId } : null); + const hasConsumer = bindings.length > 0 || bots.length > 0; + if (!identity) { + const [history] = await db.select({ grant: connectionGrants, connection: toolConnections }).from(connectionGrants) + .innerJoin(toolConnections, and(eq(toolConnections.id, connectionGrants.connectionId), eq(toolConnections.companyId, connectionGrants.companyId))) + .where(and(sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`)).limit(1); + const grant = history?.grant; + const subject = grant?.kind === "agent" && grant.subjectAgentId ? `agent:${grant.subjectAgentId}` + : grant?.kind === "user" ? grant.subjectUserId : null; + if (grant && subject) identity = { subject, companyId: grant.companyId }; + } + if (!identity) { + const [history] = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`)).limit(1); + if (history) identity = { subject: "github-bot-events", companyId: history.companyId }; + } + if (!identity) { + nextPollAt = Number.POSITIVE_INFINITY; + return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; + } + const first = identity; + const lease = await connector.leaseEvents({ subject: first.subject, companyId: first.companyId, eventWakeVersion: forcePolling ? 0 : 1 }); if (!lease) { emptyPolls += 1; - nextPollAt = now().getTime() + Math.min(bots.length ? 15_000 : 5 * 60_000, 5_000 * (2 ** Math.min(emptyPolls, 6))); + nextPollAt = !hasConsumer || (!forcePolling && connector.eventWakeVersion?.() === 1) ? Number.POSITIVE_INFINITY + : now().getTime() + Math.min(bots.length ? 15_000 : 5 * 60_000, 5_000 * (2 ** Math.min(emptyPolls, 6))); return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; } emptyPolls = 0; - nextPollAt = now().getTime() + 5_000; + nextPollAt = !hasConsumer || (!forcePolling && connector.eventWakeVersion?.() === 1) ? Number.POSITIVE_INFINITY : now().getTime() + 5_000; const result: GitHubConnectionEventPollResult = { leased: lease.events.length + (lease.failedEventIds?.length ?? 0), processed: 0, @@ -519,17 +562,7 @@ export function githubConnectionEventService( result[status] += 1; } acknowledge.push(event.id); - if (event.event === "installation" && (event.action === "deleted" || event.action === "suspend")) { - await Promise.all(matched.map((binding) => connector.setWebhookBinding({ - subject: binding.subject, - companyId: binding.companyId, - id: binding.id, - installationId: binding.installationId, - connectionId: binding.connectionId, - grantId: binding.grantId, - active: false, - }))); - } + } catch { result.failed += 1; } @@ -550,13 +583,14 @@ export function githubConnectionEventService( } return { - pollOnce(input: { forSetup?: boolean } = {}): Promise { + nextPollAt: () => Number.isFinite(nextPollAt) ? nextPollAt : null, + pollOnce(input: { forSetup?: boolean; forDelivery?: boolean } = {}): Promise { // A manager actively awaiting a dedicated App's signed ping must not // inherit a personal connection's old empty-queue backoff. This is a // bounded readiness probe, not an additional background polling loop. if (inFlight) return inFlight; const time = now().getTime(); - if (time < (input.forSetup ? nextSetupPollAt : nextPollAt)) { + if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : nextPollAt)) { return Promise.resolve({ leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }); } nextSetupPollAt = time + 5_000; diff --git a/server/src/services/github-connection-notifications.ts b/server/src/services/github-connection-notifications.ts new file mode 100644 index 00000000000..55811a3a64d --- /dev/null +++ b/server/src/services/github-connection-notifications.ts @@ -0,0 +1,26 @@ +import { toolConnections, chatEndpoints, type Db } from "@paperclipai/db"; +import { and, eq, sql } from "drizzle-orm"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +export async function notifyGitHubCredentialChange(transaction: object, connection: Pick) { + if (connection.config.sourceTemplateKey !== "github") return; + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionContinuity, connection.companyId); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents, connection.companyId); +} + +/** Company lifecycle changes only activate owners with a supported registration. */ +export async function notifyGitHubCompanyChange(transaction: object, companyId: string) { + const db = transaction as Db; + const [connection] = await db.select({ id: toolConnections.id }).from(toolConnections).where(and( + eq(toolConnections.companyId, companyId), sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, + )).limit(1); + if (connection) { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionContinuity, companyId); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents, companyId); + return; + } + const [bot] = await db.select({ id: chatEndpoints.id }).from(chatEndpoints).where(and( + eq(chatEndpoints.companyId, companyId), eq(chatEndpoints.provider, "github"), + sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, + )).limit(1); + if (bot) await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents, companyId); +} diff --git a/server/src/services/github-connection-work.ts b/server/src/services/github-connection-work.ts new file mode 100644 index 00000000000..45d2d0ee6e2 --- /dev/null +++ b/server/src/services/github-connection-work.ts @@ -0,0 +1,39 @@ +import { logger } from "../middleware/logger.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerConnectionEventWake } from "./connection-event-wake.js"; +import type { githubConnectionEventService } from "./github-connection-events.js"; +import type { createGitHubGrantContinuity } from "./tool-access/github-grant-continuity.js"; + +type Coordinator = ReturnType; +export function registerGitHubConnectionEventWork(db: Db, coordinator: Coordinator, + service: ReturnType, canRun: () => boolean) { + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.connectionEvents, { + retryMs: 5_000, minimumSpacingMs: 5_000, notificationCoalesceMs: 1_000, canRun, + run: () => service.pollOnce({ forDelivery: true }), + next: async () => service.nextPollAt(), + }); + const stops = [registerConnectionEventWake(db, worker.wake)]; + return { ...worker, stop: async () => { for (const stop of stops) stop(); await worker.stop(); } }; +} +export function registerGitHubConnectionContinuityWork(db: Db, coordinator: Coordinator, + service: Pick, "sweepGitHubConnectionContinuity" | "nextGitHubConnectionContinuityAt" | "githubConnectionContinuitySchedule">, + canRun: () => boolean) { + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.connectionContinuity, + minimumPassIntervalMs: 5_000, + decodeScope: hint => hint === "*" ? undefined : hint, + matches: (entry, companyId?: string) => !companyId || entry.companyId === companyId, + load: companyId => service.githubConnectionContinuitySchedule(companyId), + check: async entry => { + const result = await service.sweepGitHubConnectionContinuity(entry.companyId); + if (result.failed) logger.warn(result, "GitHub credential continuity retained retry obligations"); + const dueAt = await service.nextGitHubConnectionContinuityAt(entry.companyId); + return dueAt === null ? null : { ...entry, dueAt }; + }, + }, canRun); + return worker; +} diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index d5df95eaf81..f99953c1037 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,9 +1,9 @@ -import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; -import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index ea6069844b2..0c4687e8493 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1,6 +1,6 @@ -import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index d1dff39e1ee..e9ced05ed80 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,15 +1,15 @@ -import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; -import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { - await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId); - await notifyStatusCardIssueChange(tx, scope.companyId); await notifyRunDispatch(tx, scope); + await notifyStatusCardIssueChange(tx, scope.companyId); + await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 194cddbfee9..9000768a668 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,5 +1,5 @@ -import { notifyMergeConfirmationWork, type MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyMergeConfirmationWork, type MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js"; @@ -3040,9 +3040,13 @@ export function issueThreadInteractionService( }> = [], scope?: MergeConfirmationScope, signal?: AbortSignal, + mutationOptions: InteractionResolutionMutationOptions = {}, ) => { - const candidates = await mergedPullRequestConfirmationCandidates(scope ?? - (mergedHints.length === 1 ? { companyId: mergedHints[0]!.companyId } : undefined)); + const scopes = scope ? [scope] : mergedHints.length + ? [...new Set(mergedHints.map(hint => hint.companyId))].map(companyId => ({ companyId })) + : [undefined]; + const candidates: Awaited> = []; + for (const candidateScope of scopes) candidates.push(...await mergedPullRequestConfirmationCandidates(candidateScope)); const eligible = mergedHints.length ? candidates.filter(candidate => mergedHints.some(hint => hint.companyId === candidate.issue.companyId && candidate.references.some(reference => hint.owner.toLowerCase() === reference.owner.toLowerCase() && @@ -3110,6 +3114,7 @@ export function issueThreadInteractionService( current: candidate.interaction, signal, input: {}, + mutationOptions, actor: { systemId: "system:pr-merged", resolutionDetails: { @@ -3150,7 +3155,7 @@ export function issueThreadInteractionService( ) { continue; } - await opts.wakeup(wakeIssue.assigneeAgentId, { + const wake = async () => { await opts.wakeup!(wakeIssue.assigneeAgentId!, { source: "automation", triggerDetail: "system", reason: "issue_commented", @@ -3181,6 +3186,9 @@ export function issueThreadInteractionService( source: "merged_pull_request_sweep", }, }); + }; + if (mutationOptions.deferConfirmationCommitEffects) mutationOptions.deferConfirmationCommitEffects(wake); + else await wake(); woken += 1; } diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index 4894a7b070e..35e30bbf32c 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,5 +1,5 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared"; @@ -78,8 +78,8 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } - await notifyAttentionRetentionWork(tx, { companyId }); await notifyStatusCardIssueChange(tx, companyId); + await notifyAttentionRetentionWork(tx, { companyId }); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); const { publication } = await persistActivity(tx, { diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 3f2cf140237..5da95e6ff97 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,5 +1,5 @@ -import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 2455b28ef99..ad0e9e9ccff 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,12 +1,12 @@ -import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; -import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; -import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; -import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; -import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; -import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -10656,8 +10656,8 @@ export function issueService(db: Db) { await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); values.id ??= randomUUID(); - await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId }); await notifyStatusCardIssueChange(tx, companyId); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId }); if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); @@ -11541,19 +11541,19 @@ export function issueService(db: Db) { (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; } + await notifyStatusCardIssueChange(tx, receiptExisting.companyId); if (patch.status !== undefined || patch.completedAt !== undefined || patch.cancelledAt !== undefined) { await notifyNativeHarnessBackupExpiration(tx, { companyId: receiptExisting.companyId, issueId: id }); } - await notifyStatusCardIssueChange(tx, receiptExisting.companyId); if ((patch.status !== undefined && patch.status !== receiptExisting.status) || patch.completedAt !== undefined || patch.cancelledAt !== undefined || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } + await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId }); if (patch.status !== undefined || patch.hiddenAt !== undefined) { await notifyMergeConfirmationWork(tx, { companyId: receiptExisting.companyId, issueId: id }); } - await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId }); if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined @@ -12171,8 +12171,8 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { - await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyStatusCardIssueChange(tx, issueCompany.companyId); + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId }); @@ -12308,8 +12308,8 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { - await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyStatusCardIssueChange(tx, issueCompany.companyId); + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId }); @@ -12624,8 +12624,8 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } - await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyStatusCardIssueChange(tx, existing.companyId); + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 16787a140b4..b51627d1352 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,5 +1,5 @@ -import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { agents, heartbeatRuns, @@ -242,8 +242,8 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; - await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyStatusCardIssueChange(tx, input.companyId); + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); const claimed = await tx.update(issues).set({ executionRunId: input.runId, executionAgentNameKey: input.agentNameKey, diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index bc34cc3bb3c..2daf462972f 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,6 +1,6 @@ -import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; -import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -191,8 +191,8 @@ export async function reconstructNativeWorkspace(input: { }).where(eq(issueRecoveryActions.id, source.action.id)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); - await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId); await notifyStatusCardIssueChange(tx, input.companyId); + await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId); await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id)); const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId, diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index c49f600ce5f..3fb378c7605 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,13 +1,13 @@ -import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { - await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); await notifyStatusCardIssueChange(tx, scope.companyId); + await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); diff --git a/server/src/services/paperclip-cloud-connector.ts b/server/src/services/paperclip-cloud-connector.ts index b1461d3e14d..6159051c5d3 100644 --- a/server/src/services/paperclip-cloud-connector.ts +++ b/server/src/services/paperclip-cloud-connector.ts @@ -1,3 +1,4 @@ +import { providerRetryAt } from "./provider-retry-after.js"; import { SLACK_MANAGER_SCOPES, ASANA_CONNECTOR_SCOPES, isAsanaConnectorProfileId, type AsanaConnectorProfileId } from "@paperclipai/shared"; import { createDecipheriv, @@ -114,6 +115,7 @@ type ConnectorResponse = { events?: unknown; acknowledged?: unknown; githubApps?: { version: number }; + eventWakeVersion?: unknown; }; const ENDPOINTS: Record = { @@ -189,6 +191,7 @@ export class PaperclipCloudConnectorError extends Error { message: string, readonly code: string, readonly status?: number, + readonly retryAt?: number, ) { super(message); this.name = "PaperclipCloudConnectorError"; @@ -272,10 +275,11 @@ export function createPaperclipCloudConnector(input: { const now = input.now ?? Date.now; const signingKey = privateKey(config.signPrivateKey, "ed25519"); const sealKey = privateKey(config.sealPrivateKey, "x25519"); + let eventWakeVersion: 1 | null = null; async function call( operation: PaperclipCloudConnectorOperation, - claims: { subject: string; companyId: string; profile?: PaperclipCloudConnectorProfileId; scopes?: string[]; returnUri?: string; returnState?: string; claimId?: string; redemptionId?: string }, + claims: { subject: string; companyId: string; profile?: PaperclipCloudConnectorProfileId; scopes?: string[]; returnUri?: string; returnState?: string; claimId?: string; redemptionId?: string; eventWakeVersion?: 0 | 1 }, secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement"; value: string }, ): Promise { const endpoint = new URL(ENDPOINTS[operation], `${config.baseUrl}/`).toString(); @@ -291,6 +295,7 @@ export function createPaperclipCloudConnector(input: { exp: issuedAt + 60, jti: randomUUID(), }; + if (claims.eventWakeVersion !== undefined) payload.ewv = claims.eventWakeVersion; if (claims.returnUri !== undefined) payload.ruri = claims.returnUri; if (claims.returnState !== undefined) payload.rst = claims.returnState; if (claims.claimId !== undefined) payload.cl = claims.claimId; @@ -327,6 +332,7 @@ export function createPaperclipCloudConnector(input: { `Paperclip Cloud connector rejected the request (operation=${operation}, status=${response.status}, reason=${reason})`, response.status === 409 ? "REAUTHORIZATION_REQUIRED" : "CONNECTOR_REQUEST_FAILED", response.status, + providerRetryAt(response.headers.get("retry-after"), now()), ); } try { @@ -374,6 +380,7 @@ export function createPaperclipCloudConnector(input: { } return { + eventWakeVersion: () => eventWakeVersion, async githubAppsAvailable() { const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" }); return response.active === true && response.githubApps?.version === 2; @@ -520,8 +527,9 @@ export function createPaperclipCloudConnector(input: { }); await call("webhook-bind", { ...values, profile: "github.code" }, { field: "binding", value: binding }); }, - async leaseEvents(values: { subject: string; companyId: string }): Promise<{ leaseId: string; events: SealedConnectorEvents["events"]; failedEventIds?: string[] } | null> { + async leaseEvents(values: { subject: string; companyId: string; eventWakeVersion?: 0 | 1 }): Promise<{ leaseId: string; events: SealedConnectorEvents["events"]; failedEventIds?: string[] } | null> { const response = await call("event-lease", values); + eventWakeVersion = response.eventWakeVersion === 1 && values.eventWakeVersion !== 0 ? 1 : null; if (Array.isArray(response.events) && response.events.length === 0) return null; if (typeof response.leaseId !== "string") { throw new PaperclipCloudConnectorError("Paperclip Cloud connector returned an invalid event lease", "CONNECTOR_BAD_RESPONSE"); diff --git a/server/src/services/provider-retry-after.test.ts b/server/src/services/provider-retry-after.test.ts new file mode 100644 index 00000000000..b10fa764f27 --- /dev/null +++ b/server/src/services/provider-retry-after.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { providerRetryAt } from "./provider-retry-after.js"; +import { githubContinuityRevision, githubGrantContinuityDeadline } from "./tool-access/github-grant-continuity.js"; +const now = Date.parse("2026-01-01T00:00:00Z"); +describe("provider and credential deadlines", () => { + it("accepts bounded Retry-After seconds/dates and ignores malformed or expired advice", () => { + expect(providerRetryAt("120", now)).toBe(now + 120_000); + expect(providerRetryAt(new Date(now + 60_000).toUTCString(), now)).toBe(now + 60_000); + expect(providerRetryAt("999999999999", now)).toBe(now + 24 * 3600_000); + for (const value of [null, "", "unknown", "-3", new Date(now - 1).toUTCString()]) expect(providerRetryAt(value, now)).toBeUndefined(); + }); + it("reconstructs expiry, continuity and saved retries and preempts a retry when credentials change", () => { + const connection = { config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } } }; + const github = { userId: "42", login: "owner", installationCount: 1, repositoryCount: 1, repositorySelection: "all" as const, + installationIds: ["101"], installationOwnerLogins: ["owner"], lastAccessRefreshAt: new Date(now).toISOString() }; + const grant = { providerTenant: { github, oauth: { accessTokenExpiresAt: new Date(now + 8 * 3600_000).toISOString(), refreshedAt: new Date(now).toISOString() } } }; + expect(githubGrantContinuityDeadline(connection, grant)).toBe(now + 7 * 3600_000); + const retried = { providerTenant: { ...grant.providerTenant, github: { ...github, + continuityRetry: { at: new Date(now + 10 * 3600_000).toISOString(), attempts: 2, revision: githubContinuityRevision(grant) } } } }; + expect(githubGrantContinuityDeadline(connection, retried)).toBe(now + 10 * 3600_000); + retried.providerTenant.oauth.accessTokenExpiresAt = new Date(now + 2 * 3600_000).toISOString(); + expect(githubGrantContinuityDeadline(connection, retried)).toBe(now + 3600_000); + expect(githubGrantContinuityDeadline(connection, { providerTenant: { github } })).toBe(now + 30 * 24 * 3600_000); + expect(githubGrantContinuityDeadline({ config: {} }, grant)).toBeNull(); + }); +}); diff --git a/server/src/services/provider-retry-after.ts b/server/src/services/provider-retry-after.ts new file mode 100644 index 00000000000..c90364ea7e6 --- /dev/null +++ b/server/src/services/provider-retry-after.ts @@ -0,0 +1,6 @@ +export function providerRetryAt(value: string | null, now = Date.now()): number | undefined { + if (!value?.trim()) return undefined; + const seconds = /^\d+$/.test(value.trim()) ? Number(value) : Number.NaN; + const deadline = Number.isFinite(seconds) ? now + seconds * 1000 : Date.parse(value); + return Number.isFinite(deadline) && deadline > now ? Math.min(deadline, now + 24 * 3600_000) : undefined; +} diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index b7754387c00..611f07f1a41 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,5 +1,5 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index 622149badeb..a9c12fa915f 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,5 +1,5 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { chatConversations, issues, type Db } from "@paperclipai/db"; diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 0b1cc80f3bf..ee3717e231d 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js"; import { connectionHealthSchedule } from "./tool-access/catalog/health-schedule.js"; @@ -787,7 +788,7 @@ export function toolAccessService( const refreshLeases = createOAuthRefreshLeases(db, now, getConnectionRow, getOAuthGrantRow); const maybeRefreshOAuthCredentials = createLegacyOAuthRefresh(db, { now, secrets, store: credentialStore, leases: refreshLeases, getConnectionRow, exchangeOAuthToken }); const refreshOAuthGrantCredentials = createOAuthGrantRefresh(db, { now, store: credentialStore, leases: refreshLeases, currentCloudConnector, getConnectionRow, exchangeOAuthToken }); - const { refreshManagedGitHubGrantAccess, sweepGitHubConnectionContinuity } = createGitHubGrantContinuity(db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }); + const { refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt, githubConnectionContinuitySchedule } = createGitHubGrantContinuity(db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }); const vercelCredentials = createVercelCredentialService(db, now, vercelConnect); const { resolveVercelCredentialHeaders, vercelGrantForConnection } = vercelCredentials; const credentialAccess = createConnectionCredentialAccess(db, { secrets, audit, vercel: vercelCredentials, resolveOAuthGrantSecret, maybeRefreshOAuthCredentials, refreshOAuthGrantCredentials }); @@ -811,7 +812,7 @@ export function toolAccessService( const finishOAuthCatalogWithRecommendedDefaults = createOAuthCatalogCompletion(db, finishGalleryAppConnection); const startOAuth = createOAuthAuthorizationStart(db, { now, getConnectionRow, setupPolicy, fixedPersonalIdentityForReconnect, currentCloudConnector, vercelConnect, ensureOAuthClient, oauthEndpointsForConnection, assertOAuthEndpointUrl }); const completeOAuthCallback = createDirectOAuthCallback(db, { now, getConnectionRow, state: oauthState, store: credentialStore, setupPolicy, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, checkConnectionHealth, ensureDefaultOrganizationGrant, oauthEndpointsForConnection, exchangeOAuthToken }); - const completePaperclipCloudConnectorCallback = createCloudOAuthCallback(db, { now, getConnectionRow, state: oauthState, store: credentialStore, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, currentCloudConnector }); + const completePaperclipCloudConnectorCallback = createCloudOAuthCallback(db, { now, getConnectionRow, state: oauthState, store: credentialStore, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, currentCloudConnector, syncGitHubGrantBindings }); const completeVercelConnectCallback = createVercelOAuthCallback(db, { now, getConnectionRow, state: oauthState, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, vercelConnect }); const finalizeOAuthAccess = createOAuthAccessFinalization(db, { getConnectionRow, secrets, store: credentialStore, assertAgentsInCompany, finishGalleryAppConnection }); const runtimeHealth = createToolRuntimeHealth(db, options, now, trustedRuntimeHost); @@ -3155,6 +3156,7 @@ export function toolAccessService( await tx.select({ id: toolApplications.id }).from(toolApplications) .where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, companyId))) .for("update"); + await notifyGitHubCredentialChange(tx, connection); await notifyToolConnectionHealth(tx, connection); const [row] = await tx.update(toolConnections).set({ config, transportConfig: config, status: "active", enabled: true, ...(input.agentInstructions !== undefined ? { agentInstructions: input.agentInstructions } : {}), updatedAt: new Date() }) .where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId))).returning(); @@ -3513,6 +3515,7 @@ export function toolAccessService( }, tx, ); + await notifyGitHubCredentialChange(tx, connection); await notifyToolConnectionHealth(tx, connection); const [updatedConnection] = await tx .update(toolConnections) @@ -4676,6 +4679,10 @@ export function toolAccessService( return rows; }) : await writeToolConnectionHealthChange(db, existing, async tx => { + if (input.enabled !== undefined || input.status !== undefined || input.config !== undefined || input.transportConfig !== undefined || input.credentialSecretRefs !== undefined || input.credentialRefs !== undefined) { + await notifyGitHubCredentialChange(tx, existing); + await notifyGitHubCredentialChange(tx, { companyId: existing.companyId, config }); + } if (existing.transport === "voice") await notifyVoiceChange(tx, { kind: "connection", id: existing.id }); const rows = await write(tx); await syncCredentialBindings(rows[0]!, [], tx); @@ -4735,6 +4742,8 @@ export function toolAccessService( scheduledHealth: connectionHealthSchedule(db, checkConnectionHealth), sweepGitHubConnectionContinuity, + nextGitHubConnectionContinuityAt, + githubConnectionContinuitySchedule, ...catalogReader, ...toolActionReviewQueue, ...profileService, diff --git a/server/src/services/tool-access/connections/grants.ts b/server/src/services/tool-access/connections/grants.ts index 1836a6a52fc..572bb0e8856 100644 --- a/server/src/services/tool-access/connections/grants.ts +++ b/server/src/services/tool-access/connections/grants.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import type { ConnectionManagementContext } from "../../connection-management.js"; import { connectionActorUserId } from "../actor.js"; import { toConnectionGrant } from "./projection.js"; @@ -537,6 +538,7 @@ replaceConnectionGrantMembers: async ( providerRevocation = "local_only_shared_client"; } const grant = await db.transaction(async (tx) => { + await notifyGitHubCredentialChange(tx, connection); const [lockedGrant] = await tx.select({ id: connectionGrants.id }).from(connectionGrants) .where(and(eq(connectionGrants.id, grantId), eq(connectionGrants.companyId, connection.companyId), eq(connectionGrants.connectionId, connection.id))) .for("update"); diff --git a/server/src/services/tool-access/connections/removal.ts b/server/src/services/tool-access/connections/removal.ts index a9c201f9984..be9bea0dd78 100644 --- a/server/src/services/tool-access/connections/removal.ts +++ b/server/src/services/tool-access/connections/removal.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import { notifyVoiceChange } from "../../voice/voice-recovery-notifications.js"; import { notifyChatEndpointWork } from "../../chat-work-notifications.js"; import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js"; @@ -543,6 +544,7 @@ async function removeConnection( const archived = await db.transaction(async (tx) => { if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id, connection.companyId); if (connection.transport === "voice") await notifyVoiceChange(tx, { kind: "connection", id: connection.id }); + await notifyGitHubCredentialChange(tx, connection); await notifyToolConnectionHealth(tx, connection); await tx.select({ id: toolApplications.id }).from(toolApplications) .where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, connection.companyId))) diff --git a/server/src/services/tool-access/github-access-discovery.ts b/server/src/services/tool-access/github-access-discovery.ts index 4f0912b000e..2d45377ab6e 100644 --- a/server/src/services/tool-access/github-access-discovery.ts +++ b/server/src/services/tool-access/github-access-discovery.ts @@ -1,3 +1,4 @@ +import { providerRetryAt } from "../provider-retry-after.js"; import { randomUUID } from "node:crypto"; import { unprocessable } from "../../errors.js"; @@ -106,6 +107,7 @@ export async function loadGitHubGrantMetadata( throw unprocessable( "GitHub could not verify this account's installation access", { + retryAt: providerRetryAt(response.headers.get("retry-after")), code: response.status === 401 ? "oauth_reauthorization_required" diff --git a/server/src/services/tool-access/github-grant-continuity.ts b/server/src/services/tool-access/github-grant-continuity.ts index 568a0a00a99..13aaec986e7 100644 --- a/server/src/services/tool-access/github-grant-continuity.ts +++ b/server/src/services/tool-access/github-grant-continuity.ts @@ -1,12 +1,14 @@ +import { randomUUID } from "node:crypto"; +import { notifyGitHubCredentialChange } from "../github-connection-notifications.js"; import type { ActorInfo } from "./types.js"; import { loadGitHubGrantMetadata } from "./github-access-discovery.js"; import { measureRunPhase } from "../execution-preparation-timing.js"; import { measureRunTransaction } from "../execution-transaction-timing.js"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { connectionGrants, toolConnections } from "@paperclipai/db"; +import { companies, connectionGrants, toolConnections } from "@paperclipai/db"; import { conflict, HttpError, notFound, unprocessable } from "../../errors.js"; -import { type PaperclipCloudConnector } from "../paperclip-cloud-connector.js"; +import { PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../paperclip-cloud-connector.js"; import type { createOAuthGrantRefresh } from "./oauth/grant-refresh.js"; import type { OAuthCredentialStore } from "./oauth/credential-store.js"; @@ -16,6 +18,8 @@ function asRecord(value: unknown): Record { return {}; } +type RecordGitHubAttempt = (grant: typeof connectionGrants.$inferSelect) => void; + type GitHubContinuityDependencies = { now: () => Date; currentCloudConnector: () => PaperclipCloudConnector | null; @@ -24,11 +28,11 @@ type GitHubContinuityDependencies = { }; export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }: GitHubContinuityDependencies) { - let nextGitHubContinuitySweepAt = 0; async function refreshManagedGitHubGrantAccess( connection: typeof toolConnections.$inferSelect, initialGrant: typeof connectionGrants.$inferSelect, actor?: ActorInfo, + recordAttempt?: RecordGitHubAttempt, ) { await actor?.managementContext?.reauthorize(); return measureRunPhase("connectors.github.refresh", async () => { @@ -37,6 +41,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector connection, initialGrant, actor, + recordAttempt, ); } catch (error) { if ( @@ -55,7 +60,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector ), )); if (!current || current.status !== "active") throw error; - return refreshManagedGitHubGrantAccessOnce(connection, current, actor); + return refreshManagedGitHubGrantAccessOnce(connection, current, actor, recordAttempt); } }); } @@ -64,6 +69,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector connection: typeof toolConnections.$inferSelect, initialGrant: typeof connectionGrants.$inferSelect, actor?: ActorInfo, + recordAttempt?: RecordGitHubAttempt, ) { await actor?.managementContext?.reauthorize(); let grant = await measureRunPhase("connectors.github.credentials", () => refreshOAuthGrantCredentials({ @@ -72,6 +78,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector grantId: initialGrant.id, actor, })); + recordAttempt?.(grant); const resolveAccessToken = async () => measureRunPhase("connectors.github.credentials", async () => { const accessRef = grant.credentialSecretRefs.find( (ref) => ref.configPath === "oauth.access_token", @@ -112,6 +119,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector forceRefresh: true, actor, })); + recordAttempt?.(grant); accessToken = await resolveAccessToken(); try { metadata = await measureRunPhase("connectors.github.repositories", () => loadGitHubGrantMetadata( @@ -138,7 +146,8 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector throw retryError; } } - const { updated, previousGitHub } = await measureRunTransaction(db, "connectors.github.refresh", async (tx) => { + const { updated } = await measureRunTransaction(db, "connectors.github.refresh", async (tx) => { + await notifyGitHubCredentialChange(tx, connection); const [currentGrant] = await measureRunPhase("connectors.github.grant_lock", () => tx .select() .from(connectionGrants) @@ -170,6 +179,13 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector ...(currentGrant.providerTenant ?? {}), github: { ...metadata, + bindingSync: { + id: randomUUID(), + removeInstallationIds: [...new Set([ + ...(previousGitHub?.bindingSync?.removeInstallationIds ?? []), + ...(previousGitHub?.installationIds ?? []), + ])].filter(id => !metadata.installationIds.includes(id)), + }, ...(previousGitHub?.lastWebhookAt ? { lastWebhookAt: previousGitHub.lastWebhookAt } : {}), @@ -195,111 +211,187 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector return { updated, previousGitHub }; }); - const cloudConnector = currentCloudConnector(); - const subject = - updated.kind === "agent" && updated.subjectAgentId - ? `agent:${updated.subjectAgentId}` - : updated.kind === "user" && updated.subjectUserId - ? updated.subjectUserId - : null; - if (cloudConnector && subject) { - const previous = new Set(previousGitHub?.installationIds ?? []); - const current = new Set(metadata.installationIds); - await measureRunPhase("connectors.github.webhooks", () => Promise.all([ - ...metadata.installationIds.map((installationId) => - cloudConnector.setWebhookBinding({ - subject, - companyId: connection.companyId, - id: `${updated.id}_${installationId}`, - installationId, - connectionId: connection.id, - grantId: updated.id, - active: true, - accessToken, - }), - ), - ...[...previous] - .filter((installationId) => !current.has(installationId)) - .map((installationId) => - cloudConnector.setWebhookBinding({ - subject, - companyId: connection.companyId, - id: `${updated.id}_${installationId}`, - installationId, - connectionId: connection.id, - grantId: updated.id, - active: false, - }), - ), - ])); - } - return updated; + return measureRunPhase("connectors.github.webhooks", () => syncGitHubGrantBindings(connection, updated, actor, accessToken, recordAttempt)); } - async function sweepGitHubConnectionContinuity() { - if (now().getTime() < nextGitHubContinuitySweepAt) { - return { checked: 0, due: 0, refreshed: 0, failed: 0 }; - } - nextGitHubContinuitySweepAt = now().getTime() + 60 * 60_000; - const cutoff = now().getTime() - 30 * 24 * 60 * 60_000; - const rows = await db + async function syncGitHubGrantBindings( + connection: typeof toolConnections.$inferSelect, + initialGrant: typeof connectionGrants.$inferSelect, + actor?: ActorInfo, + accessToken?: string, + recordAttempt?: RecordGitHubAttempt, + ) { + await actor?.managementContext?.reauthorize(); + const cloudConnector = currentCloudConnector(); + if (!initialGrant.providerTenant?.github?.bindingSync) return initialGrant; + if (!cloudConnector) throw unprocessable("GitHub webhook delivery is not configured"); + const grant = accessToken || initialGrant.providerTenant.github.installationIds.length === 0 ? initialGrant : await refreshOAuthGrantCredentials({ + companyId: connection.companyId, connectionId: connection.id, grantId: initialGrant.id, actor, + }); + const accessRef = grant.credentialSecretRefs.find(ref => ref.configPath === "oauth.access_token"); + if (!accessRef && grant.providerTenant?.github?.installationIds.length) throw unprocessable("GitHub authorization must be reconnected", { code: "oauth_reauthorization_required" }); + const token = accessToken ?? (accessRef ? (await resolveOAuthGrantSecret(connection, grant, accessRef, actor, undefined)).value : undefined); + // Serialize external binding changes with grant/connection lifecycle writes. + // Requests occur only for saved debt; no idle lock or connection is retained. + return db.transaction(async tx => { + // Company deletion locks its parent before cascading to grants/connections. + // Keep that order while retaining admission through the external handoff. + const [company] = await tx.select({ status: companies.status }).from(companies) + .where(eq(companies.id, connection.companyId)).for("share").limit(1); + if (!company) return grant; + const [current] = await tx.select().from(connectionGrants).where(and( + eq(connectionGrants.id, grant.id), eq(connectionGrants.companyId, connection.companyId), + eq(connectionGrants.connectionId, connection.id), + )).for("update").limit(1); + if (!current || current.status !== "active") return current ?? grant; + const [liveConnection] = await tx.select().from(toolConnections).where(and( + eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId), + )).for("update").limit(1); + if (!liveConnection || liveConnection.status !== "active" || !liveConnection.enabled) return current; + if (company.status !== "active") return current; + const github = current.providerTenant?.github; + if (!github?.bindingSync || github.bindingSync.id !== grant.providerTenant?.github?.bindingSync?.id) return current; + const subject = current.kind === "agent" && current.subjectAgentId ? `agent:${current.subjectAgentId}` + : current.kind === "user" ? current.subjectUserId : null; + if (!subject) throw unprocessable("GitHub webhook subject is unavailable"); + recordAttempt?.(grant); + const changes = [ + ...github.installationIds.map(installationId => ({ installationId, active: true as const })), + ...github.bindingSync.removeInstallationIds.filter(id => !github.installationIds.includes(id)) + .map(installationId => ({ installationId, active: false as const })), + ]; + for (let offset = 0; offset < changes.length; offset += 4) { + const outcomes = await Promise.allSettled(changes.slice(offset, offset + 4).map(change => cloudConnector.setWebhookBinding({ + subject, companyId: connection.companyId, id: `${current.id}_${change.installationId}`, + installationId: change.installationId, connectionId: connection.id, grantId: current.id, + ...change.active ? { active: true, accessToken: token! } : { active: false }, + }))); + const failure = outcomes.find(outcome => outcome.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + } + await notifyGitHubCredentialChange(tx, connection); + const [settled] = await tx.update(connectionGrants).set({ + providerTenant: sql`${connectionGrants.providerTenant} #- '{github,bindingSync}' #- '{github,continuityRetry}'`, + updatedAt: now(), + }).where(and(eq(connectionGrants.id, current.id), eq(connectionGrants.companyId, connection.companyId))).returning(); + return settled!; + }); + } + + async function retainRetry(connection: typeof toolConnections.$inferSelect, grantId: string, attemptedRevision: string, error?: unknown, minimumMs = 30_000) { + await db.transaction(async tx => { + const [current] = await tx.select().from(connectionGrants).where(and( + eq(connectionGrants.id, grantId), eq(connectionGrants.companyId, connection.companyId), + eq(connectionGrants.connectionId, connection.id), eq(connectionGrants.status, "active"), + )).for("update").limit(1); + // Backoff belongs to the attempted credential/installation revision. + if (!current?.providerTenant?.github || githubContinuityRevision(current) !== attemptedRevision) return; + const attempts = Math.min(16, (current.providerTenant.github.continuityRetry?.attempts ?? 0) + 1); + const advised = error instanceof PaperclipCloudConnectorError ? error.retryAt + : error instanceof HttpError ? asRecord(error.details).retryAt : undefined; + const at = new Date(Math.max(now().getTime() + Math.max(minimumMs, Math.min(60 * 60_000, 30_000 * 2 ** Math.min(attempts - 1, 7))), + typeof advised === "number" ? advised : 0)).toISOString(); + await notifyGitHubCredentialChange(tx, connection); + await tx.update(connectionGrants).set({ + providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,continuityRetry}', ${JSON.stringify({ + at, attempts, revision: attemptedRevision, + })}::jsonb)`, + updatedAt: now(), + }).where(and(eq(connectionGrants.id, current.id), eq(connectionGrants.companyId, connection.companyId))); + }); + } + + async function continuityRows(companyId?: string) { + return db .select({ grant: connectionGrants, connection: toolConnections }) .from(connectionGrants) - .innerJoin( - toolConnections, - and( - eq(toolConnections.id, connectionGrants.connectionId), - eq(toolConnections.companyId, connectionGrants.companyId), - ), - ) - .where( - and( - eq(connectionGrants.status, "active"), - eq(toolConnections.status, "active"), - eq(toolConnections.enabled, true), - ), - ); + .innerJoin(toolConnections, and( + eq(toolConnections.id, connectionGrants.connectionId), + eq(toolConnections.companyId, connectionGrants.companyId), + )) + .where(and( + companyId ? eq(connectionGrants.companyId, companyId) : undefined, + eq(connectionGrants.status, "active"), + eq(toolConnections.status, "active"), + eq(toolConnections.enabled, true), + sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, + sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`, + sql`${connectionGrants.providerTenant}->'github' is not null`, + sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status = 'active')`, + )); + } + + async function nextGitHubConnectionContinuityAt(companyId?: string): Promise { + const deadlines = (await continuityRows(companyId)).flatMap(({ grant, connection }) => { + const deadline = githubGrantContinuityDeadline(connection, grant); + return deadline === null ? [] : [deadline]; + }); + return deadlines.length ? Math.min(...deadlines) : null; + } + + async function sweepGitHubConnectionContinuity(companyId?: string) { + const rows = await continuityRows(companyId); const due = rows.filter(({ grant, connection }) => { - const config = asRecord(connection.config); - const oauth = asRecord(config.oauth); - if ( - config.sourceTemplateKey !== "github" || - oauth.connectorProfile !== "github.code" || - !grant.providerTenant?.github - ) - return false; - const expiresAt = grant.providerTenant.oauth?.accessTokenExpiresAt; - const refreshedAt = grant.providerTenant.oauth?.refreshedAt - ? Date.parse(grant.providerTenant.oauth.refreshedAt) - : Number.NaN; - const accessCheckedAt = grant.providerTenant.github.lastAccessRefreshAt - ? Date.parse(grant.providerTenant.github.lastAccessRefreshAt) - : Number.NaN; - if (typeof expiresAt === "string") { - const expiry = Date.parse(expiresAt); - return ( - !Number.isFinite(expiry) || - expiry <= now().getTime() + 60 * 60_000 || - !Number.isFinite(refreshedAt) || - refreshedAt <= cutoff - ); - } - return !Number.isFinite(accessCheckedAt) || accessCheckedAt <= cutoff; + const deadline = githubGrantContinuityDeadline(connection, grant); + return deadline !== null && deadline <= now().getTime(); }); let refreshed = 0; let failed = 0; for (const row of due) { + let attemptedRevision = githubContinuityRevision(row.grant); + const recordAttempt: RecordGitHubAttempt = grant => { attemptedRevision = githubContinuityRevision(grant); }; try { - await refreshManagedGitHubGrantAccess(row.connection, row.grant, { - actorType: "system", - actorId: "github-connection-continuity", - }); + const actor: ActorInfo = { actorType: "system", actorId: "github-connection-continuity" }; + const updated = row.grant.providerTenant?.github?.bindingSync + ? await syncGitHubGrantBindings(row.connection, row.grant, actor, undefined, recordAttempt) + : await refreshManagedGitHubGrantAccess(row.connection, row.grant, actor, recordAttempt); + // A successful provider response can still leave an unchanged/short-lived + // credential due. Retain that obligation with bounded durable backoff. + const next = githubGrantContinuityDeadline(row.connection, updated); + if (next !== null && next <= now().getTime()) await retainRetry(row.connection, updated.id, githubContinuityRevision(updated), undefined, 60 * 60_000); refreshed += 1; - } catch { + } catch (error) { + await retainRetry(row.connection, row.grant.id, attemptedRevision, error); failed += 1; } } return { checked: rows.length, due: due.length, refreshed, failed }; } - return { refreshManagedGitHubGrantAccess, sweepGitHubConnectionContinuity }; + async function githubConnectionContinuitySchedule(companyId?: string) { + const schedule = new Map(); + for (const { grant, connection } of await continuityRows(companyId)) { + const dueAt = githubGrantContinuityDeadline(connection, grant); + if (dueAt !== null) schedule.set(connection.companyId, Math.min(schedule.get(connection.companyId) ?? Infinity, dueAt)); + } + return [...schedule].map(([companyId, dueAt]) => ({ id: companyId, companyId, dueAt })); + } + return { githubConnectionContinuitySchedule, refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt }; +} + +/** Credential expiry and provider continuity are obligations, not a scan cadence. */ +export function githubGrantContinuityDeadline( + connection: Pick, + grant: Pick, +): number | null { + const config = asRecord(connection.config); + if (config.sourceTemplateKey !== "github" || asRecord(config.oauth).connectorProfile !== "github.code" + || !grant.providerTenant?.github) return null; + const oauth = grant.providerTenant.oauth; + const access = grant.providerTenant.github; + const parsed = (value: string | undefined) => { + const time = value ? Date.parse(value) : Number.NaN; + return Number.isFinite(time) ? time : 0; + }; + const continuityMs = 30 * 24 * 60 * 60_000; + const credentialDeadline = typeof oauth?.accessTokenExpiresAt === "string" + ? Math.min(parsed(oauth.accessTokenExpiresAt) - 60 * 60_000, oauth.refreshedAt ? parsed(oauth.refreshedAt) + continuityMs : 0) + : access.lastAccessRefreshAt ? parsed(access.lastAccessRefreshAt) + continuityMs : 0; + const deadline = access.bindingSync ? 0 : credentialDeadline; + return access.continuityRetry?.revision === githubContinuityRevision(grant) + ? Math.max(deadline, parsed(access.continuityRetry.at)) : deadline; +} + +export function githubContinuityRevision(grant: Pick): string { + return JSON.stringify([grant.providerTenant?.oauth?.accessTokenExpiresAt, grant.providerTenant?.oauth?.refreshedAt, + grant.providerTenant?.github?.lastAccessRefreshAt, grant.providerTenant?.github?.bindingSync?.id]); } diff --git a/server/src/services/tool-access/oauth/access-finalization.ts b/server/src/services/tool-access/oauth/access-finalization.ts index a6ad361dde8..4d0ab9ed010 100644 --- a/server/src/services/tool-access/oauth/access-finalization.ts +++ b/server/src/services/tool-access/oauth/access-finalization.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js"; import { connectionActorUserId } from "../actor.js"; import type { ActorInfo } from "../types.js"; @@ -116,6 +117,7 @@ async function finalizeOAuthAccess( throw conflict("This connection already uses a company identity"); } connection = await writeToolConnectionHealthChange(db, connection, async tx => { + await notifyGitHubCredentialChange(tx, connection); const [updated] = await tx .update(toolConnections) .set({ diff --git a/server/src/services/tool-access/oauth/cloud-callback.ts b/server/src/services/tool-access/oauth/cloud-callback.ts index fac5bce24ba..ebba51b0b92 100644 --- a/server/src/services/tool-access/oauth/cloud-callback.ts +++ b/server/src/services/tool-access/oauth/cloud-callback.ts @@ -1,3 +1,6 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; +import type { createGitHubGrantContinuity } from "../github-grant-continuity.js"; +import { randomUUID } from "node:crypto"; import { notifyIssueReviewState } from "../../issue-review-state-notifications.js"; import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js"; import { managedConnectorProfile } from "./client-settings.js"; @@ -32,6 +35,7 @@ type Dependencies = { finishOAuthCatalogWithRecommendedDefaults: OAuthCatalogCompletion; store: Pick; currentCloudConnector: () => PaperclipCloudConnector | null; + syncGitHubGrantBindings: ReturnType["syncGitHubGrantBindings"]; }; export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { @@ -301,6 +305,8 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { accessRef, ...(refreshRef ? [refreshRef] : []), ]; + await notifyGitHubCredentialChange(tx, connection); + await notifyToolConnectionHealth(tx, connection); const grantValues = { providerTenant: { name: githubMetadata?.login ?? providerName, @@ -319,7 +325,12 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { ? { refreshedAt: now().toISOString() } : {}), }, - ...(githubMetadata ? { github: githubMetadata } : {}), + ...(githubMetadata ? { github: { ...githubMetadata, bindingSync: { + id: randomUUID(), removeInstallationIds: [...new Set([ + ...(existingCredentialGrant?.providerTenant?.github?.bindingSync?.removeInstallationIds ?? []), + ...(existingCredentialGrant?.providerTenant?.github?.installationIds ?? []), + ])].filter(id => !githubMetadata.installationIds.includes(id)), + } } } : {}), }, credentialSecretRefs, status: "active" as const, @@ -358,7 +369,6 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { scopes: [...profile.scopes], }, }; - await notifyToolConnectionHealth(tx, connection); [connection] = await tx .update(toolConnections) .set({ @@ -438,42 +448,8 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { // Paperclip Cloud connector callback is the managed variant of an OAuth // callback completion. emitConnectionUpdated(connection, preCloudCallbackLifecycle, "oauth_callback"); - if (githubMetadata) { - const [githubGrant] = await db - .select({ id: connectionGrants.id }) - .from(connectionGrants) - .where( - and( - eq(connectionGrants.companyId, connection.companyId), - eq(connectionGrants.connectionId, connection.id), - subjectAgentId - ? and( - eq(connectionGrants.kind, "agent"), - eq(connectionGrants.subjectAgentId, subjectAgentId), - ) - : and( - eq(connectionGrants.kind, "user"), - eq(connectionGrants.subjectUserId, subjectUserId!), - ), - ), - ) - .limit(1); - if (!githubGrant) throw new Error("GitHub grant was not persisted"); - await Promise.all( - githubMetadata.installationIds.map((installationId) => - cloudConnector.setWebhookBinding({ - subject: connectorSubject, - companyId: connection.companyId, - id: `${githubGrant.id}_${installationId}`, - installationId, - connectionId: connection.id, - grantId: githubGrant.id, - active: true, - accessToken: credentials.accessToken, - }), - ), - ); - } + // Binding registration is saved above before Cloud I/O. The continuity + // worker drains it after activation, including after a callback/server crash. const refresh = await refreshCatalog(connection.id, input.actor, { enableAllByDefault: false, quarantineManagedOAuthDraft: shouldFinalizeManagedDefaults, @@ -494,6 +470,15 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { actor: input.actor, }) : null; + if (githubMetadata) { + const [grant] = await db.select().from(connectionGrants).where(and( + eq(connectionGrants.companyId, connection.companyId), eq(connectionGrants.connectionId, connection.id), + subjectAgentId ? and(eq(connectionGrants.kind, "agent"), eq(connectionGrants.subjectAgentId, subjectAgentId)) + : and(eq(connectionGrants.kind, "user"), eq(connectionGrants.subjectUserId, subjectUserId!)), + )).limit(1); + if (!grant) throw new Error("GitHub grant was not persisted"); + await dependencies.syncGitHubGrantBindings(await getConnectionRow(connection.id, connection.companyId), grant, input.actor, credentials.accessToken); + } const activatedCatalogEntryIds = new Set( shouldFinalizeManagedDefaults ? refresh.catalog diff --git a/server/src/services/tool-access/oauth/grant-refresh.ts b/server/src/services/tool-access/oauth/grant-refresh.ts index 48e09cf27b9..096b5d4f9c9 100644 --- a/server/src/services/tool-access/oauth/grant-refresh.ts +++ b/server/src/services/tool-access/oauth/grant-refresh.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js"; import type { ActorInfo } from "../types.js"; import { createOAuthClientProtocol, storedOAuthTokenEndpointAuthMethod } from "./client.js"; @@ -13,7 +14,7 @@ import { secretService } from "../../secrets.js"; import { isPaperclipCloudConnectorStrategy, PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../../paperclip-cloud-connector.js"; import { actorForSecret, type OAuthCredentialStore } from "./credential-store.js"; -import { withoutOAuthRefreshLease, withoutOAuthGrantRefreshLease, type OAuthRefreshLeases } from "./refresh-leases.js"; +import { withoutOAuthRefreshLease, type OAuthRefreshLeases } from "./refresh-leases.js"; import { managedConnectorProfile, oauthGrantConfig, oauthGrantExpiresAtMs } from "./client-settings.js"; import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js"; import { singleFlight } from "../single-flight.js"; @@ -178,10 +179,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo grant.kind === "user" && grant.subjectUserId ? { actorType: "user", actorId: grant.subjectUserId } : input.actor; - const providerTenant = { - ...(grant.providerTenant ?? {}), - oauth: { - ...withoutOAuthGrantRefreshLease(oauthGrantConfig(grant)), + const oauthUpdates = { strategy: "paperclip_cloud_connector", accessTokenExpiresAt: credentials.accessTokenExpiresAt, scopes: credentials.scopes, @@ -190,9 +188,9 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo ...(credentials.refreshTokenExpiresAt ? { refreshTokenExpiresAt: credentials.refreshTokenExpiresAt } : {}), - }, }; const updated = await db.transaction(async (tx) => { + await notifyGitHubCredentialChange(tx, connection); const [current] = await tx.select({ id: connectionGrants.id }).from(connectionGrants) .where(and(eq(connectionGrants.id, grant.id), eq(connectionGrants.companyId, grant.companyId), eq(connectionGrants.status, "active"), @@ -217,7 +215,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo const [committed] = await tx .update(connectionGrants) .set({ - providerTenant, + providerTenant: sql`jsonb_set(coalesce(${connectionGrants.providerTenant}, '{}'::jsonb), '{oauth}', (coalesce(${connectionGrants.providerTenant}->'oauth', '{}'::jsonb) || ${JSON.stringify(oauthUpdates)}::jsonb) - 'refreshLease')`, status: "active", updatedAt: now(), }) @@ -342,18 +340,11 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo ref.configPath !== "oauth.access_token" && ref.configPath !== "oauth.refresh_token", ); - const providerTenant = { - ...(grant.providerTenant ?? {}), - oauth: { - ...withoutOAuthGrantRefreshLease(grantOauth), - accessTokenExpiresAt: undefined, - }, - }; const [marked] = await db .update(connectionGrants) .set({ status: "needs_reauthorization", - providerTenant, + providerTenant: sql`${connectionGrants.providerTenant} #- '{oauth,refreshLease}' #- '{oauth,accessTokenExpiresAt}'`, credentialSecretRefs: retainedCredentialSecretRefs, updatedAt: now(), }) @@ -523,26 +514,22 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo if (connection.config.sourceTemplateKey === "google-workspace") { refreshed.scopes = googleWorkspaceEffectiveScopes(normalizeOauthScopes(grantOauth.requestedScopes ?? grantOauth.scopes), refreshed.scopes); } - const providerTenant = { - ...(grant.providerTenant ?? {}), - oauth: { - ...withoutOAuthGrantRefreshLease(grantOauth), + const oauthUpdates = { strategy: typeof grantOauth.strategy === "string" ? grantOauth.strategy : "direct_oauth", - accessTokenExpiresAt: expiresAt ?? undefined, + accessTokenExpiresAt: expiresAt ?? null, scopes: refreshed.scopes, scopeSource: refreshed.scopeSource, unrequestedScopes: refreshed.unrequestedScopes, tokenType: token.tokenType, refreshedAt: now().toISOString(), - }, }; - const [updated] = await db + const [updated] = await writeToolConnectionHealthChange(db, connection, tx => tx .update(connectionGrants) .set({ - providerTenant, + providerTenant: sql`jsonb_set(coalesce(${connectionGrants.providerTenant}, '{}'::jsonb), '{oauth}', (coalesce(${connectionGrants.providerTenant}->'oauth', '{}'::jsonb) || ${JSON.stringify(oauthUpdates)}::jsonb) - 'refreshLease')`, credentialSecretRefs: nextCredentialSecretRefs, status: "active", updatedAt: now(), @@ -555,7 +542,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo sql`${connectionGrants.providerTenant} -> 'oauth' -> 'refreshLease' ->> 'id' = ${lease.leaseId}`, ), ) - .returning(); + .returning()); if (!updated) { throw conflict( "OAuth credentials changed while refresh was in progress. Retry the request.", diff --git a/server/src/services/tool-access/oauth/refresh-leases.ts b/server/src/services/tool-access/oauth/refresh-leases.ts index 8250da26e65..c3040b8828d 100644 --- a/server/src/services/tool-access/oauth/refresh-leases.ts +++ b/server/src/services/tool-access/oauth/refresh-leases.ts @@ -43,15 +43,11 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR ) { const latest = await getConnectionRow(connection.id, connection.companyId); if (oauthRefreshLeaseId(latest) !== leaseId) return latest; - const nextConfig = { - ...latest.config, - oauth: withoutOAuthRefreshLease(oauthConfig(latest)), - }; const [updated] = await db .update(toolConnections) .set({ - config: nextConfig, - transportConfig: nextConfig, + config: sql`${toolConnections.config} #- '{oauth,refreshLease}'`, + transportConfig: sql`${toolConnections.transportConfig} #- '{oauth,refreshLease}'`, updatedAt: now(), }) .where( @@ -165,14 +161,10 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR }); const oauth = oauthGrantConfig(latest); if (asRecord(oauth.refreshLease).id !== leaseId) return latest; - const providerTenant = { - ...(latest.providerTenant ?? {}), - oauth: withoutOAuthGrantRefreshLease(oauth), - }; const [updated] = await db .update(connectionGrants) .set({ - providerTenant, + providerTenant: sql`${connectionGrants.providerTenant} #- '{oauth,refreshLease}'`, updatedAt: now(), }) .where( @@ -202,7 +194,7 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR leaseId: string | null; }> { const waitDeadline = Date.now() + OAUTH_REFRESH_LEASE_WAIT_MS; - const initialUpdatedAt = grant.updatedAt.getTime(); + const initialOAuth = JSON.stringify(withoutOAuthGrantRefreshLease(oauthGrantConfig(grant))); while (true) { const latest = await getOAuthGrantRow({ companyId: connection.companyId, @@ -221,7 +213,7 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR (!forceRefresh && (expiresAtMs === null || expiresAtMs > Date.now() + 60_000)) || (forceRefresh && - latest.updatedAt.getTime() > initialUpdatedAt && + JSON.stringify(withoutOAuthGrantRefreshLease(oauthGrantConfig(latest))) !== initialOAuth && !asRecord(oauthGrantConfig(latest).refreshLease).id) ) { return { grant: latest, leaseId: null }; @@ -266,6 +258,9 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR eq(connectionGrants.id, latest.id), eq(connectionGrants.companyId, latest.companyId), eq(connectionGrants.status, "active"), + // SQL NULL and JSON null both represent absent optional metadata. + // Keep object snapshots exact so concurrent metadata writes win the CAS. + sql`coalesce(${connectionGrants.providerTenant}, 'null'::jsonb) = ${JSON.stringify(latest.providerTenant)}::jsonb`, sql`${connectionGrants.providerTenant} #>> '{oauth,refreshLease,id}' is null`, ), )