import { notifyChatEndpointConfiguration } from "../services/chat-channels/configuration-work-notifications.js"; import { deleteCompany } from "../services/company-deletion.js"; import { toolAccessService } from "../services/tool-access.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { randomUUID, generateKeyPairSync, verify, createHash, diffieHellman, hkdfSync, randomBytes, createCipheriv } from "node:crypto"; import { createServer, type Server } from "node:http"; import { once } from "node:events"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; import { applyPendingMigrations, companies, activityLog, companySecrets, connectionEventDeliveries, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { createGitHubGrantContinuity, githubGrantContinuityDeadline, githubContinuityRevision } from "../services/tool-access/github-grant-continuity.js"; import { createPaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; import { githubConnectionEventService } from "../services/github-connection-events.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerGitHubConnectionContinuityWork, registerGitHubConnectionEventWork } from "../services/github-connection-work.js"; import { notifyGitHubCredentialChange } from "../services/github-connection-notifications.js"; import { createOAuthCredentialStore } from "../services/tool-access/oauth/credential-store.js"; import { createOAuthRefreshLeases } from "../services/tool-access/oauth/refresh-leases.js"; import { createOAuthGrantRefresh } from "../services/tool-access/oauth/grant-refresh.js"; import { secretService } from "../services/secrets.js"; import { writeConnectionCredential } from "../services/connection-credentials.js"; import { GITHUB_CONNECTOR_PROFILES } from "@paperclipai/shared"; import { notifyToolConnectionHealth } from "../services/tool-connection-health-notifications.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_WORK_TEST_DATABASE_URL"); const embeddedSupport = databaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeDatabase = embeddedSupport.supported ? describe : describe.skip; describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP (simulation)", () => { let db: ReturnType; let temporaryDatabase: Awaited> | undefined; let server: Server, origin: string; let leaseTimes: number[] = []; let bindings: Array>, leases: number, failBindings: boolean, githubRequests: number; const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); let releaseRefresh: (() => void) | undefined, refreshArrived: (() => void) | undefined; let refreshGate: Promise | undefined; let discoveryFailureGate: Promise | undefined, discoveryArrived: (() => void) | undefined; let heldBindingCompany: string | undefined, bindingGate: Promise | undefined; let bindingArrived: (() => void) | undefined, bindingClosed: (() => void) | undefined; const realFetch = globalThis.fetch; let companyIds: string[] = []; beforeAll(async () => { if (databaseUrl) await applyPendingMigrations(databaseUrl); else temporaryDatabase = await startEmbeddedPostgresTestDatabase("paperclip-github-work-"); // Only scheduler time is virtual here. Driver housekeeping must not expire // a real socket connection while that fake clock advances across idle windows. db = createDb(databaseUrl ?? temporaryDatabase!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0, }); // This explicitly assigned external test database is exclusive to this suite. await db.transaction(async tx => { await tx.execute(sql`SET LOCAL client_min_messages = warning`); await tx.execute(sql`TRUNCATE companies CASCADE`); }); server = createServer(async (req, res) => { const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); const raw = Buffer.concat(chunks).toString(); if (req.url!.startsWith("/v1/connector/")) { const body = JSON.parse(raw), [header, payload, signature] = body.request.split("."); expect(verify(null, Buffer.from(`${header}.${payload}`), signing.publicKey, Buffer.from(signature, "base64url"))).toBe(true); const claims = JSON.parse(Buffer.from(payload, "base64url").toString()); expect(claims.iss).toBe("instance"); expect(claims.env).toBe("test"); expect(claims.aud).toBe(`${origin}${req.url}`); if (req.url!.endsWith("/refresh")) { refreshArrived?.(); await refreshGate; const scopes = [...GITHUB_CONNECTOR_PROFILES["github.code"].scopes]; const algorithm = "X25519-HKDF-SHA256-A256GCM"; const purpose = "access"; const aad = Buffer.from([1, algorithm, purpose, "instance", "test", "github", "github.code", scopes.sort().join(" ")].join("\n")); const ephemeral = generateKeyPairSync("x25519"); const epk = Buffer.from(ephemeral.publicKey.export({ format: "jwk" }).x!, "base64url"); const recipient = Buffer.from(sealing.publicKey.export({ format: "jwk" }).x!, "base64url"); const key = Buffer.from(hkdfSync("sha256", diffieHellman({ privateKey: ephemeral.privateKey, publicKey: sealing.publicKey }), Buffer.concat([epk, recipient]), aad, 32)); const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad); const credentials = { v: 1, accessToken: "rotated-access", refreshToken: "rotated-refresh", tokenType: "Bearer", scopes, accessTokenExpiresAt: new Date(Date.now() + 8 * 3600_000).toISOString(), refreshTokenExpiresAt: null, subject: claims.sub, companyId: claims.cid, instanceId: "instance", environment: "test", provider: "github", profile: "github.code" }; const ct = Buffer.concat([cipher.update(JSON.stringify(credentials)), cipher.final(), cipher.getAuthTag()]); res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ sealed: { v: 1, alg: algorithm, purpose, provider: "github", profile: "github.code", epk: epk.toString("base64url"), iv: iv.toString("base64url"), ct: ct.toString("base64url") } })); return; } if (req.url!.endsWith("/events/lease")) { leases++; leaseTimes.push(Date.now()); res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ events: [], ...(claims.ewv === 1 ? { eventWakeVersion: 1 } : {}) })); return; } expect(claims.sh).toBe(createHash("sha256").update(body.binding).digest("base64url")); bindings.push(JSON.parse(body.binding)); if (claims.cid === heldBindingCompany) { res.writeHead(200, { "content-type": "application/json" }); res.write("{"); res.once("close", () => bindingClosed?.()); bindingArrived?.(); await bindingGate; res.end("}"); return; } res.writeHead(failBindings ? 503 : 200, { "content-type": "application/json" }).end("{}"); return; } if (req.url === "/oauth/token") { expect(new URLSearchParams(raw).get("grant_type")).toBe("refresh_token"); expect(new URLSearchParams(raw).get("refresh_token")).toBe("saved-refresh"); res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ access_token: "renewed-access", refresh_token: "renewed-refresh", token_type: "Bearer", expires_in: 3600 })); return; } githubRequests++; if (req.url === "/user" && discoveryFailureGate) { discoveryArrived?.(); await discoveryFailureGate; res.writeHead(503, { "content-type": "application/json", "retry-after": "86400" }).end("{}"); return; } const body = req.url === "/user" ? { id: 42, login: "octocat" } : req.url!.startsWith("/user/installations?") ? { installations: [{ id: 101, repository_selection: "all", account: { login: "octocat" } }] } : { total_count: 1, repositories: [{ id: 999, full_name: "octocat/repo" }] }; res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(body)); }); server.listen(0, "127.0.0.1"); await once(server, "listening"); const address = server.address(); if (!address || typeof address === "string") throw new Error("HTTP fixture did not start"); origin = `http://127.0.0.1:${address.port}`; }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); for (const id of companyIds) { await db.delete(activityLog).where(eq(activityLog.companyId, id)); await db.delete(companySecrets).where(eq(companySecrets.companyId, id)); await db.delete(companies).where(eq(companies.id, id)); } companyIds = []; }); afterAll(async () => { server.close(); await once(server, "close"); await db.$client.end({ timeout: 1 }); await temporaryDatabase?.cleanup(); }); async function fixture(ids: string[] = []) { bindings = []; leases = 0; leaseTimes = []; failBindings = false; githubRequests = 0; const companyId = randomUUID(); companyIds.push(companyId); const [company] = await db.insert(companies).values({ id: companyId, name: "GitHub work test", issuePrefix: `GW${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}` }).returning(); const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: "GitHub", type: "mcp_http", status: "active" }).returning(); const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: app!.id, uid: randomUUID(), name: "GitHub", status: "active", enabled: true, transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_user", config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } }, transportConfig: {} }).returning(); const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection!.id, kind: "user", subjectUserId: "owner", status: "active", credentialSecretRefs: [{ configPath: "oauth.access_token", secretId: randomUUID(), label: "Access" }], providerTenant: { oauth: { accessTokenExpiresAt: new Date(Date.now() + 8 * 3600_000).toISOString(), refreshedAt: new Date().toISOString() }, github: { userId: "42", login: "octocat", installationCount: ids.length, repositoryCount: 1, repositorySelection: "all", installationIds: ids, installationOwnerLogins: ["octocat"], accessRevision: randomUUID(), lastAccessRefreshAt: new Date().toISOString(), webhookHealth: "healthy", } } }).returning(); const connector = createPaperclipCloudConnector({ config: { baseUrl: origin, instanceId: "instance", environment: "test", signPrivateKey: signing.privateKey.export({ format: "jwk" }).d!, sealPrivateKey: sealing.privateKey.export({ format: "jwk" }).d! } }); type Dependencies = Parameters[1]; const dependencies: Dependencies = { now: () => new Date(), currentCloudConnector: () => connector, refreshOAuthGrantCredentials: async input => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, input.grantId)))[0]!, resolveOAuthGrantSecret: (async () => ({ value: "controlled-token", latestVersion: 1 })) as Dependencies["resolveOAuthGrantSecret"], }; const service = createGitHubGrantContinuity(db, dependencies); return { company: company!, connection: connection!, grant: grant!, connector, service, dependencies }; } const fakeClock = () => vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); async function settled(coordinator: ReturnType) { await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); expect(coordinator.nextWakeAt()).not.toBeNull(); } it("ordinary grant refresh activates events from quiet, commits durable result, then performs no owner SQL/HTTP across old cadence windows", async () => { const f = await fixture(); fakeClock(); vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); const errors: unknown[] = []; const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); const events = githubConnectionEventService(db, { connector: f.connector }); const ew = registerGitHubConnectionEventWork(db, coordinator, events, () => true); const cw = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); try { await Promise.all([ew.ready, cw.ready]); expect(leases).toBe(1); const startupLeases = leases; await f.service.refreshManagedGitHubGrantAccess(f.connection, f.grant, { actorType: "system", actorId: "test" }); await vi.advanceTimersByTimeAsync(6_000); await vi.waitFor(() => expect(leases).toBeGreaterThan(startupLeases)); await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(saved!.providerTenant!.github!.installationIds).toEqual(["101"]); expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); expect(bindings).toContainEqual(expect.objectContaining({ active: true, installationId: "101", id: `${f.grant.id}_101` })); const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"), execute = vi.spyOn(db, "execute"); const before = { leases, githubRequests, bindings: bindings.length }; await vi.advanceTimersByTimeAsync(10 * 60_000); expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect({ leases, githubRequests, bindings: bindings.length }).toEqual(before); expect(errors).toEqual([]); } finally { await ew.stop(); await cw.stop(); await coordinator.stop(); } }); it("failed binding removal persists retry/debt through restart, uses current lease deadline and settles without refetching GitHub metadata", async () => { const f = await fixture(["101"]); fakeClock(); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } } }).where(eq(connectionGrants.id, f.grant.id)); failBindings = true; expect(await f.service.sweepGitHubConnectionContinuity()).toMatchObject({ due: 1, failed: 1 }); const [failed] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(failed!.providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["202"]); const retryAt = Date.parse(failed!.providerTenant!.github!.continuityRetry!.at); const attemptedRemovals = bindings.filter(binding => binding.installationId === "202").length; const restarted = createGitHubGrantContinuity(db, f.dependencies); expect(await restarted.nextGitHubConnectionContinuityAt()).toBe(retryAt); await vi.advanceTimersByTimeAsync(29_000); expect(await restarted.sweepGitHubConnectionContinuity()).toMatchObject({ due: 0 }); failBindings = false; await vi.advanceTimersByTimeAsync(2_000); await Promise.all([restarted.sweepGitHubConnectionContinuity(), restarted.sweepGitHubConnectionContinuity()]); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); expect(saved!.providerTenant!.github!.continuityRetry).toBeUndefined(); expect(bindings.filter(binding => binding.installationId === "202")).toHaveLength(attemptedRemovals + 1); expect(githubRequests).toBe(0); expect(await restarted.nextGitHubConnectionContinuityAt()).toBeGreaterThan(Date.now()); await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, f.connection.id)); expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); await db.update(toolConnections).set({ enabled: true }).where(eq(toolConnections.id, f.connection.id)); await db.update(companies).set({ status: "archived" }).where(eq(companies.id, f.company.id)); expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); }); it.each(["credential", "installation"])("does not attach a withheld provider failure to newer %s work", async change => { const f = await fixture(["101"]); vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); let release!: () => void; const arrived = new Promise(resolve => { discoveryArrived = resolve; }); discoveryFailureGate = new Promise(resolve => { release = resolve; }); const pending = f.service.sweepGitHubConnectionContinuity(f.company.id); const finished = Promise.allSettled([pending]); try { await Promise.race([arrived, pending.then(() => { throw new Error("Expected held provider request"); })]); const revision = randomUUID(); const expiry = new Date(Date.now() + 10 * 60_000).toISOString(); await db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); await tx.update(connectionGrants).set({ providerTenant: change === "credential" ? sql`jsonb_set(${connectionGrants.providerTenant}, '{oauth,accessTokenExpiresAt}', ${JSON.stringify(expiry)}::jsonb)` : sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify({ id: revision, removeInstallationIds: ["202"] })}::jsonb)`, updatedAt: new Date() }).where(eq(connectionGrants.id, f.grant.id)); }); release(); expect(await pending).toMatchObject({ due: 1, failed: 1 }); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(saved!.providerTenant!.github!.continuityRetry).toBeUndefined(); if (change === "credential") expect(saved!.providerTenant!.oauth!.accessTokenExpiresAt).toBe(expiry); else expect(saved!.providerTenant!.github!.bindingSync!.id).toBe(revision); expect(githubGrantContinuityDeadline(f.connection, saved!)).toBeLessThanOrEqual(Date.now()); discoveryFailureGate = undefined; const restarted = createGitHubGrantContinuity(db, f.dependencies); expect(await restarted.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, refreshed: 1, failed: 0 }); expect(bindings).toContainEqual(expect.objectContaining({ installationId: "101", active: true })); if (change === "installation") expect(bindings).toContainEqual(expect.objectContaining({ installationId: "202", active: false })); const [settled] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(settled!.providerTenant!.github!.bindingSync).toBeUndefined(); const next = await restarted.nextGitHubConnectionContinuityAt(f.company.id); expect(next).toBeLessThan(Date.now() + 24 * 60 * 60_000); } finally { release(); await finished; discoveryFailureGate = undefined; discoveryArrived = undefined; } }); it("retains provider backoff for the actual attempted revision after its own credential rotation", async () => { const f = await fixture(["101"]); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); const owner = createGitHubGrantContinuity(db, { ...f.dependencies, refreshOAuthGrantCredentials: async () => { const [rotated] = await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{oauth}', ${JSON.stringify({ accessTokenExpiresAt: new Date(Date.now() + 10 * 60_000).toISOString(), refreshedAt: new Date().toISOString(), })}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)).returning(); return rotated!; } }); discoveryFailureGate = Promise.resolve(); try { expect(await owner.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, failed: 1 }); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(saved!.providerTenant!.github!.continuityRetry!.revision).toBe(githubContinuityRevision(saved!)); expect(await owner.nextGitHubConnectionContinuityAt(f.company.id)).toBeGreaterThan(Date.now() + 23 * 60 * 60_000); const calls = githubRequests; expect(await createGitHubGrantContinuity(db, f.dependencies).sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 0 }); expect(githubRequests).toBe(calls); } finally { discoveryFailureGate = undefined; } }); it.each([["suspend", "unsuspend"], ["unsuspend", "suspend"]])("reconciles same-batch installation order %s then %s and duplicate replay", async (first, last) => { const f = await fixture(["101"]); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } } }).where(eq(connectionGrants.id, f.grant.id)); const events = [first, last].map(action => ({ id: randomUUID(), provider: "github" as const, event: "installation", action, installationId: "101", repositoryId: null, createdAt: new Date(Date.now() + 1000).toISOString(), bindingIds: [`${f.grant.id}_101`], payload: { installationId: "101" } })); const acknowledge = vi.fn(async () => events.length); const ingress = githubConnectionEventService(db, { connector: { ...f.connector, leaseEvents: async () => ({ leaseId: "installation-batch", events }), acknowledgeEvents: acknowledge } }); expect(await ingress.pollOnce({ forDelivery: true })).toMatchObject({ processed: 2, failed: 0 }); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); const active = last === "unsuspend"; expect(saved!.providerTenant!.github!.installationIds).toEqual(active ? ["101"] : []); expect(saved!.providerTenant!.github!.bindingSync!.removeInstallationIds.sort()).toEqual(active ? ["202"] : ["101", "202"]); expect(githubGrantContinuityDeadline(f.connection, saved!)).toBe(0); await f.service.syncGitHubGrantBindings(f.connection, saved!); expect(bindings.filter(binding => binding.installationId === "101")).toEqual([expect.objectContaining({ active })]); expect(bindings.filter(binding => binding.installationId === "202")).toEqual([expect.objectContaining({ active: false })]); const [settled] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(settled!.providerTenant!.github!.bindingSync).toBeUndefined(); // Batch duplicates must retain terminal receipts and emit no registration changes. expect(await ingress.pollOnce({ forDelivery: true })).toMatchObject({ processed: 0, duplicate: active ? 2 : 0, ignored: active ? 0 : 2, failed: 0 }); await f.service.syncGitHubGrantBindings(f.connection, saved!); expect(bindings).toHaveLength(2); expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)))[0]!.providerTenant).toEqual(settled!.providerTenant); expect((await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.companyId, f.company.id))).map(receipt => receipt.status)).toEqual(["processed", "processed"]); expect(acknowledge).toHaveBeenCalledTimes(2); }); it("does not retire active installations from a legacy conflicting binding intent", async () => { const f = await fixture(["101"]); const [saved] = await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101", "202"] } } } }).where(eq(connectionGrants.id, f.grant.id)).returning(); await f.service.syncGitHubGrantBindings(f.connection, saved!); expect(bindings.filter(binding => binding.installationId === "101")).toEqual([expect.objectContaining({ active: true })]); expect(bindings.filter(binding => binding.installationId === "202")).toEqual([expect.objectContaining({ active: false })]); }); it("preserves webhook removal debt across an actual signed, held Cloud token rotation and lease cleanup", async () => { const f = await fixture(["101"]); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); const refs = []; for (const path of ["oauth.access_token", "oauth.refresh_token"]) { const saved = await writeConnectionCredential(db, { companyId: f.company.id, connectionName: "GitHub", configPath: path, label: path, value: path, ownerUserId: "owner" }); refs.push({ configPath: path, secretId: saved.secret.id, versionSelector: "latest" as const }); } const [connection] = await db.update(toolConnections).set({ credentialSource: "paperclip_vault", config: { ...f.connection.config, oauth: { strategy: "paperclip_cloud_connector", connectorProfile: "github.code" } } }).where(eq(toolConnections.id, f.connection.id)).returning(); const [grant] = await db.update(connectionGrants).set({ credentialSecretRefs: refs }).where(eq(connectionGrants.id, f.grant.id)).returning(); const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)))[0]!; const store = createOAuthCredentialStore(db, secretService(db)); await store.syncCredentialBindings(connection!, refs); const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, store.getOAuthGrantRow); const rotation = createOAuthGrantRefresh(db, { now: () => new Date(), store, leases, getConnectionRow: getConnection, currentCloudConnector: () => f.connector, exchangeOAuthToken: vi.fn() }); const arrived = new Promise(resolve => { refreshArrived = resolve; }); refreshGate = new Promise(resolve => { releaseRefresh = resolve; }); const pending = rotation({ companyId: f.company.id, connectionId: connection!.id, grantId: grant!.id, forceRefresh: true }); const result = Promise.allSettled([pending]); try { await Promise.race([arrived, pending.then(() => { throw new Error("Rotation returned before the Cloud HTTP boundary"); })]); const connector = { ...f.connector, acknowledgeEvents: async () => 1, leaseEvents: async () => ({ leaseId: "removal", events: [{ id: randomUUID(), provider: "github" as const, event: "installation", action: "deleted", installationId: "101", repositoryId: null, createdAt: new Date(Date.now() + 1000).toISOString(), bindingIds: [`${grant!.id}_101`], payload: { installationId: "101" } }] }) }; expect(await githubConnectionEventService(db, { connector }).pollOnce()).toMatchObject({ processed: 1 }); const [before] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); expect(before!.providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["101"]); releaseRefresh!(); const [completion] = await result; if (completion.status === "rejected") throw completion.reason; const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); expect(saved!.providerTenant!.github).toEqual(before!.providerTenant!.github); expect(saved!.providerTenant!.oauth).toMatchObject({ tokenType: "Bearer" }); expect(saved!.providerTenant!.oauth!.refreshLease).toBeUndefined(); await f.service.syncGitHubGrantBindings(connection!, saved!); expect(bindings).toContainEqual(expect.objectContaining({ installationId: "101", active: false })); expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)))[0]!.providerTenant!.github!.bindingSync).toBeUndefined(); } finally { releaseRefresh?.(); await result; refreshGate = undefined; refreshArrived = undefined; vi.unstubAllEnvs(); } }); it("bounds a held Cloud body without releasing revocation admission, while another company progresses and restart retains removal debt", async () => { const f = await fixture([]), other = await fixture([]); for (const row of [f, other]) await db.update(connectionGrants).set({ providerTenant: { ...row.grant.providerTenant, github: { ...row.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, } }).where(eq(connectionGrants.id, row.grant.id)); const latest = async (id: string) => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, id)))[0]!; heldBindingCompany = f.company.id; let release!: () => void; bindingGate = new Promise(resolve => { release = resolve; }); const arrived = new Promise(resolve => { bindingArrived = resolve; }); const closed = new Promise(resolve => { bindingClosed = resolve; }); const timeout = AbortSignal.timeout.bind(AbortSignal); const deadlines = vi.spyOn(AbortSignal, "timeout").mockImplementation(ms => timeout(ms === 15_000 ? 3_000 : ms)); let completed = false; const attempt = f.service.syncGitHubGrantBindings(f.connection, await latest(f.grant.id)); const outcome = attempt.then(value => ({ value, error: null }), error => ({ value: null, error })).finally(() => { completed = true; }); try { await arrived; const otherResult = await other.service.syncGitHubGrantBindings(other.connection, await latest(other.grant.id)); expect(otherResult.providerTenant!.github!.bindingSync).toBeUndefined(); await db.update(companies).set({ name: "Unrelated company changed while body held" }).where(eq(companies.id, other.company.id)); expect(completed).toBe(false); expect((await outcome).error).toMatchObject({ code: "CONNECTOR_BAD_RESPONSE" }); await closed; expect(deadlines).toHaveBeenCalledWith(15_000); expect((await latest(f.grant.id)).providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["101"]); heldBindingCompany = undefined; release(); const restarted = createGitHubGrantContinuity(db, f.dependencies); expect(await restarted.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, refreshed: 1, failed: 0 }); expect((await latest(f.grant.id)).providerTenant!.github!.bindingSync).toBeUndefined(); } finally { heldBindingCompany = undefined; release(); await outcome; bindingGate = undefined; bindingArrived = undefined; bindingClosed = undefined; } }); it("serializes binding handoff behind canonical company deletion without a child lock cycle", async () => { const f = await fixture([]); const [pending] = await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, } }).where(eq(connectionGrants.id, f.grant.id)).returning(); let release!: () => void, entered!: (pid: number) => void; const gate = new Promise(resolve => { release = resolve; }); const companyLocked = new Promise(resolve => { entered = resolve; }); const transaction = db.transaction.bind(db); // Pause after the canonical deletion's actual company lock, before its child writes. const spy = vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: unknown) => Promise) => transaction(async tx => { const select = tx.select.bind(tx); tx.select = ((...args: Parameters) => { const query = select(...args), from = query.from.bind(query); query.from = ((table: unknown) => { const builder = from(table as typeof companies); if (table === companies) { const lock = builder.for.bind(builder); builder.for = ((strength: Parameters[0]) => { const locked = lock(strength); if (strength === "no key update") { const then = locked.then.bind(locked); locked.then = ((resolve: (rows: unknown) => unknown, reject: (error: unknown) => unknown) => then(async rows => { const [owner] = await tx.execute(sql`select pg_backend_pid() as pid`); entered(Number(owner!.pid)); await gate; return resolve(rows); }, reject)) as typeof locked.then; } return locked; }) as typeof builder.for; } return builder; }) as typeof query.from; return query; }) as typeof tx.select; return callback(tx); })) as typeof db.transaction); const deletion = deleteCompany(db, f.company.id); const deleted = Promise.allSettled([deletion]); let handoff: Promise | undefined; try { const deletionPid = await companyLocked; handoff = f.service.syncGitHubGrantBindings(f.connection, pending!); const completed = Promise.allSettled([deletion, handoff]); await vi.waitFor(async () => { const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() and query like '%"companies"%' and query like '%for share%' and ${deletionPid} = any(pg_blocking_pids(pid))`); expect(waiting).toHaveLength(1); }, { timeout: 5_000, interval: 20 }); release(); const results = await completed; expect(results.map(row => row.status), JSON.stringify(results)).toEqual(["fulfilled", "fulfilled"]); expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(0); expect(await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id))).toHaveLength(0); expect(bindings).toHaveLength(0); } finally { release(); await deleted; await handoff?.catch(() => {}); spy.mockRestore(); } }); it("lets an admitted binding finish before canonical company deletion", async () => { const f = await fixture([]); const [pending] = await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, } }).where(eq(connectionGrants.id, f.grant.id)).returning(); heldBindingCompany = f.company.id; let release!: () => void; bindingGate = new Promise(resolve => { release = resolve; }); const arrived = new Promise(resolve => { bindingArrived = resolve; }); const handoff = f.service.syncGitHubGrantBindings(f.connection, pending!); const handedOff = Promise.allSettled([handoff]); let deletion: Promise | undefined; try { await arrived; deletion = deleteCompany(db, f.company.id); const deleted = Promise.allSettled([deletion]); await vi.waitFor(async () => { const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() and query like '%"companies"%' and query like '%for no key update%' and cardinality(pg_blocking_pids(pid)) > 0`); expect(waiting).toHaveLength(1); }, { timeout: 5_000, interval: 20 }); release(); expect((await handedOff)[0]!.status).toBe("fulfilled"); expect((await deleted)[0]!.status).toBe("fulfilled"); expect(bindings).toEqual([expect.objectContaining({ grantId: f.grant.id, connectionId: f.connection.id, installationId: "101", active: false })]); expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(0); expect(await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id))).toHaveLength(0); } finally { release(); await handedOff; await deletion?.catch(() => {}); heldBindingCompany = undefined; bindingGate = undefined; bindingArrived = undefined; } }); it.each(["revocation", "revision", "rollback"] as const)("revalidates a late %s after waiting for company admission", async change => { const f = await fixture([]), revision = randomUUID(); const [pending] = await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, } }).where(eq(connectionGrants.id, f.grant.id)).returning(); let entered!: (pid: number) => void, release!: () => void; const locked = new Promise(resolve => { entered = resolve; }); const gate = new Promise(resolve => { release = resolve; }); const mutation = db.transaction(async tx => { await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update"); const [owner] = await tx.execute(sql`select pg_backend_pid() as pid`); entered(Number(owner!.pid)); await gate; if (change === "revision") await tx.update(connectionGrants).set({ providerTenant: { ...pending!.providerTenant, github: { ...pending!.providerTenant!.github!, bindingSync: { id: revision, removeInstallationIds: ["202"] } }, } }).where(eq(connectionGrants.id, f.grant.id)); else await tx.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id)); if (change === "rollback") throw new Error("rollback late revocation"); }); const mutated = Promise.allSettled([mutation]); let handoff: Promise | undefined; try { const ownerPid = await locked; handoff = f.service.syncGitHubGrantBindings(f.connection, pending!); const handedOff = Promise.allSettled([handoff]); await vi.waitFor(async () => { const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() and query like '%"companies"%' and query like '%for share%' and ${ownerPid} = any(pg_blocking_pids(pid))`); expect(waiting).toHaveLength(1); }, { timeout: 5_000, interval: 20 }); release(); expect((await mutated)[0]!.status).toBe(change === "rollback" ? "rejected" : "fulfilled"); expect((await handedOff)[0]!.status).toBe("fulfilled"); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); if (change === "rollback") { expect(bindings).toEqual([expect.objectContaining({ grantId: f.grant.id, installationId: "101", active: false })]); expect(saved!.status).toBe("active"); expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); } else { expect(bindings).toHaveLength(0); if (change === "revocation") expect(saved!.status).toBe("revoked"); else expect(saved!.providerTenant!.github!.bindingSync).toEqual({ id: revision, removeInstallationIds: ["202"] }); } } finally { release(); await mutated; await handoff?.catch(() => {}); } }); it("renews an installation created without metadata using the saved connection expiry", async () => { const f = await fixture(); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); try { const refs = []; for (const path of ["oauth.access_token", "oauth.refresh_token"]) { const saved = await writeConnectionCredential(db, { companyId: f.company.id, connectionName: "OAuth", configPath: path, label: path, value: path === "oauth.refresh_token" ? "saved-refresh" : "expired-access" }); refs.push({ configPath: path, secretId: saved.secret.id, versionSelector: "latest" as const }); } await db.update(toolConnections).set({ credentialSource: "paperclip_vault", credentialPolicy: "shared", config: { oauth: { provider: "fixture", tokenUrl: `${origin}/oauth/token`, clientId: "fixture-client", clientRegistrationSource: "dcr", clientTokenEndpointAuthMethod: "none", expiresAt: new Date(Date.now() - 60_000).toISOString(), } } }).where(eq(toolConnections.id, f.connection.id)); const api = toolAccessService(db); const installation = await api.addConnectionInstallation(f.connection.id, { credentialSecretRefs: refs }); const [before] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)); expect(before!.providerTenant).toBeNull(); expect((await db.execute(sql`select provider_tenant is null as absent from connection_grants where id = ${installation.id}`))[0]!.absent).toBe(true); const renewed = await api.refreshOAuthGrantCredentials({ companyId: f.company.id, connectionId: f.connection.id, grantId: installation.id }); expect(renewed.providerTenant!.oauth).toMatchObject({ tokenType: "Bearer", accessTokenExpiresAt: expect.any(String) }); expect(Date.parse(renewed.providerTenant!.oauth!.accessTokenExpiresAt!)).toBeGreaterThan(Date.now() + 3500_000); expect(renewed.providerTenant!.oauth!.refreshLease).toBeUndefined(); expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)))[0]!.providerTenant).toEqual(renewed.providerTenant); } finally { vi.unstubAllEnvs(); } }, 40_000); it.each(["sql-null", "json-null", "object"] as const)("claims current %s metadata, fences stale snapshots and rolls back lease writes", async mode => { const f = await fixture(); const api = toolAccessService(db); const installation = await api.addConnectionInstallation(f.connection.id, {}); const metadata = mode === "sql-null" ? sql`null` : mode === "json-null" ? sql`'null'::jsonb` : sql`'{"name":"current"}'::jsonb`; await db.update(connectionGrants).set({ providerTenant: metadata }).where(eq(connectionGrants.id, installation.id)); const getGrant = async () => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)))[0]!; const initial = await getGrant(); const leases = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, getGrant); const claimed = await leases.acquireOAuthGrantRefreshLease(f.connection, initial, true); expect(claimed.leaseId).toBeTruthy(); expect(claimed.grant.providerTenant!.oauth!.refreshLease!.id).toBe(claimed.leaseId); if (mode === "object") expect(claimed.grant.providerTenant!.name).toBe("current"); await leases.clearOAuthGrantRefreshLease(claimed.grant, claimed.leaseId!); await db.update(connectionGrants).set({ providerTenant: metadata }).where(eq(connectionGrants.id, installation.id)); const newer = { name: "concurrent", github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } }; let raced = false; const staleRead = async () => { const snapshot = await getGrant(); if (!raced) { raced = true; await db.update(connectionGrants).set({ providerTenant: newer }).where(eq(connectionGrants.id, installation.id)); } return snapshot; }; const racingLeases = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, staleRead); const retained = await racingLeases.acquireOAuthGrantRefreshLease(f.connection, initial, true); expect(retained.grant.providerTenant).toMatchObject(newer); expect(retained.leaseId).toBeTruthy(); await racingLeases.clearOAuthGrantRefreshLease(retained.grant, retained.leaseId!); await db.update(connectionGrants).set({ providerTenant: metadata }).where(eq(connectionGrants.id, installation.id)); await expect(db.transaction(async tx => { const read = async () => (await tx.select().from(connectionGrants).where(eq(connectionGrants.id, installation.id)))[0]!; const transactional = createOAuthRefreshLeases(tx as unknown as typeof db, () => new Date(), async () => f.connection, read); const lease = await transactional.acquireOAuthGrantRefreshLease(f.connection, await read(), true); expect(lease.leaseId).toBeTruthy(); throw new Error("rollback lease"); })).rejects.toThrow("rollback lease"); expect((await getGrant()).providerTenant).toEqual(initial.providerTenant); const retried = await leases.acquireOAuthGrantRefreshLease(f.connection, initial, true); expect(retried.leaseId).toBeTruthy(); await leases.clearOAuthGrantRefreshLease(retried.grant, retried.leaseId!); }, 40_000); it("lease acquire and cleanup cannot replace a concurrent webhook metadata revision", async () => { const f = await fixture(["101"]); let interleave = true; const removal = { id: randomUUID(), removeInstallationIds: ["202"] }; const getGrant = async () => { const [snapshot] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); if (interleave) { interleave = false; await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify(removal)}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)); } return snapshot!; }; const leaseOwner = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, getGrant); const lease = await leaseOwner.acquireOAuthGrantRefreshLease(f.connection, f.grant, true); expect(lease.leaseId).toBeTruthy(); expect(lease.grant.providerTenant!.github!.bindingSync).toEqual(removal); // Cleanup deliberately reads a stale snapshot immediately before a new delta. const second = { ...removal, id: randomUUID(), removeInstallationIds: ["202", "303"] }; const cleanup = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, async () => { const snapshot = await getGrant(); await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify(second)}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)); return snapshot; }); const cleared = await cleanup.clearOAuthGrantRefreshLease(lease.grant, lease.leaseId!); expect(cleared.providerTenant!.github!.bindingSync).toEqual(second); expect(cleared.providerTenant!.oauth!.refreshLease).toBeUndefined(); }); it("ignores unrelated health write bursts and scopes credential reconstruction to its owning company", async () => { const f = await fixture(["101"]), other = await fixture(["202"]); fakeClock(); // A large unrelated company set must not enter a targeted credential reload. for (let index = 0; index < 50; index++) { const id = randomUUID(); companyIds.push(id); await db.insert(companies).values({ id, name: "Unrelated credential owner", issuePrefix: `U${index}${id.slice(0, 6)}` }); } const [unrelatedConnection] = await db.update(toolConnections).set({ config: {}, transportConfig: {}, transport: "rest_api", authKind: "none", credentialRefs: [], credentialSecretRefs: [] }).where(eq(toolConnections.id, other.connection.id)).returning(); await db.update(connectionGrants).set({ credentialSecretRefs: [] }).where(eq(connectionGrants.id, other.grant.id)); const api = toolAccessService(db); const loadTimes: number[] = []; const schedule = f.service.githubConnectionContinuitySchedule.bind(f.service); const load = vi.spyOn(f.service, "githubConnectionContinuitySchedule").mockImplementation(companyId => { loadTimes.push(Date.now()); return schedule(companyId); }); const sweep = vi.spyOn(f.service, "sweepGitHubConnectionContinuity"); const errors: unknown[] = []; const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); const continuity = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); const events = registerGitHubConnectionEventWork(db, coordinator, githubConnectionEventService(db, { connector: f.connector }), () => true); try { await Promise.all([continuity.ready, events.ready]); await vi.waitFor(() => expect(leases).toBe(1)); await vi.waitFor(() => expect(load).toHaveBeenCalledTimes(1)); load.mockClear(); loadTimes.length = 0; sweep.mockClear(); for (let i = 0; i < 30; i++) { await db.transaction(async tx => { await notifyToolConnectionHealth(tx, other.connection); await notifyChatEndpointConfiguration(tx, { id: randomUUID(), provider: "discord" }); }); await api.updateConnection(unrelatedConnection!.id, { name: `Unrelated patch ${i}` }); } await vi.advanceTimersByTimeAsync(10_000); expect(load).not.toHaveBeenCalled(); expect(sweep).not.toHaveBeenCalled(); expect(leases).toBe(1); expect(errors).toEqual([]); for (let index = 0; index < 30; index++) await db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); }); expect(leases).toBeLessThanOrEqual(2); await vi.advanceTimersByTimeAsync(6_000); await vi.waitFor(() => expect(load).toHaveBeenCalledWith(f.company.id)); expect(load.mock.calls.every(([companyId]) => companyId === f.company.id)).toBe(true); expect(sweep).not.toHaveBeenCalled(); await vi.waitFor(() => expect(leases).toBeGreaterThanOrEqual(2)); // The first hint can run immediately; changes during it get one spaced pass. expect(leases).toBeLessThanOrEqual(3); expect(load.mock.calls.length).toBeLessThanOrEqual(2); if (loadTimes.length === 2) expect(loadTimes[1]! - loadTimes[0]!).toBeGreaterThanOrEqual(5_000); if (leaseTimes.length === 3) expect(leaseTimes[2]! - leaseTimes[1]!).toBeGreaterThanOrEqual(5_000); // A hint arriving during the first SQL pass retains one spaced generation. // Start the empty-window assertion after that concrete work has settled. await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); expect(leases).toBeLessThanOrEqual(3); expect(load.mock.calls.length).toBeLessThanOrEqual(2); if (loadTimes.length === 2) expect(loadTimes[1]! - loadTimes[0]!).toBeGreaterThanOrEqual(5_000); const quiet = { leases, loads: load.mock.calls.length }; await vi.advanceTimersByTimeAsync(60_000); expect({ leases, loads: load.mock.calls.length }).toEqual(quiet); expect(errors).toEqual([]); } finally { await events.stop(); await continuity.stop(); await coordinator.stop(); } }); it("settles only the company and grant that own saved binding debt", async () => { const f = await fixture(["101"]), other = await fixture(["202"]); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: [] } } } }).where(eq(connectionGrants.id, f.grant.id)); await f.service.sweepGitHubConnectionContinuity(); expect(bindings).toHaveLength(1); expect(bindings[0]).toMatchObject({ grantId: f.grant.id, installationId: "101" }); const [untouched] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, other.grant.id)); expect(untouched!.providerTenant).toEqual(other.grant.providerTenant); }); it("retains a real deadline when successful renewal leaves a short-lived credential unchanged", async () => { const f = await fixture(["101"]); fakeClock(); vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); expect(await f.service.sweepGitHubConnectionContinuity()).toMatchObject({ due: 1, refreshed: 1 }); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); const retryAt = Date.parse(saved!.providerTenant!.github!.continuityRetry!.at); expect(retryAt).toBeGreaterThanOrEqual(Date.now() + 59 * 60_000); const restarted = createGitHubGrantContinuity(db, f.dependencies); expect(await restarted.nextGitHubConnectionContinuityAt()).toBe(retryAt); const requests = githubRequests; await vi.advanceTimersByTimeAsync(10 * 60_000); expect(await restarted.sweepGitHubConnectionContinuity()).toMatchObject({ due: 0 }); expect(githubRequests).toBe(requests); }); it("outer commit, rollback and first producer during drain retain the deadline and resume obligation without cross-company work", async () => { const f = await fixture(["101"]); fakeClock(); let enabled = false; const errors: unknown[] = []; const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => enabled, canReconcile: () => true, onError: error => errors.push(error) }); const worker = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => enabled); try { await worker.ready; const dueAt = Date.now() + 90 * 60_000; let release!: () => void; const held = new Promise(resolve => { release = resolve; }); const transaction = db.transaction(async tx => { await tx.transaction(async nested => { await notifyGitHubCredentialChange(nested, f.connection); await nested.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(dueAt + 3600_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); }); await held; }); await new Promise(resolve => setImmediate(resolve)); expect(bindings).toHaveLength(0); release(); await transaction; await vi.advanceTimersByTimeAsync(31_000); expect(idleWorkSnapshot().active).toBe(0); expect(bindings).toHaveLength(0); enabled = true; await vi.advanceTimersByTimeAsync(31_000); await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(dueAt)); await expect(db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); await tx.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id)); throw new Error("rollback"); })).rejects.toThrow("rollback"); await vi.advanceTimersByTimeAsync(31_000); await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(dueAt)); expect(errors).toEqual([]); } finally { await worker.stop(); await coordinator.stop(); } }); });