diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index 122188c49f7..84e9679737e 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -13,6 +13,7 @@ import { companyMemberships, heartbeatRuns, instanceUserRoles, + principalPermissionGrants, } from "@paperclipai/db"; import { MAX_ISSUE_PREFIX_ATTEMPTS, @@ -44,6 +45,7 @@ export { import { syncCloudTenantWrites } from "./cloud-tenant-sync.js"; import { resolveOwnerInstanceAdmin } from "../services/cloud-tenant-authority.js"; import { ensureHumanRoleDefaultGrants } from "../services/principal-access-compatibility.js"; +import { grantsForHumanRole, normalizeHumanRole } from "../services/company-member-roles.js"; import { forbidden, unauthorized, unprocessable } from "../errors.js"; export { isCloudManagedInstance } from "../services/cloud-instance.js"; @@ -154,11 +156,45 @@ async function loadCloudTenantMembershipsWithStaleAdmin(db: Db, userId: string) status: companyMemberships.status, staleInstanceAdmin: sql`exists (select 1 from ${instanceUserRoles} where ${instanceUserRoles.userId} = ${userId} and ${instanceUserRoles.role} = 'instance_admin')`, + // Lets a request with an unchanged membership skip the write sync only + // when the role's default grants are also present. + grantedPermissionKeys: sql`(select array_agg(${principalPermissionGrants.permissionKey}) + from ${principalPermissionGrants} + where ${principalPermissionGrants.companyId} = ${companyMemberships.companyId} + and ${principalPermissionGrants.principalType} = 'user' + and ${principalPermissionGrants.principalId} = ${userId})`, }) .from(companyMemberships) .where(activeUserMembershipFilter(userId)); } +/** A lock held elsewhere fails the repair quickly instead of holding the request. */ +const CLOUD_TENANT_GRANT_REPAIR_LOCK_TIMEOUT_MS = 2_000; +const CLOUD_TENANT_GRANT_REPAIR_STATEMENT_TIMEOUT_MS = 5_000; + +function missesDefaultGrants(grantedPermissionKeys: unknown, membershipRole: string) { + if (!Array.isArray(grantedPermissionKeys)) return true; + const granted = new Set(grantedPermissionKeys); + return grantsForHumanRole(normalizeHumanRole(membershipRole, "operator")) + .some((grant) => !granted.has(grant.permissionKey)); +} + +async function repairCloudTenantDefaultGrants( + db: Db, + input: { companyId: string; userId: string; membershipRole: string }, +) { + await db.transaction(async (tx) => { + await tx.execute(sql`select set_config('lock_timeout', ${`${CLOUD_TENANT_GRANT_REPAIR_LOCK_TIMEOUT_MS}ms`}, true), + set_config('statement_timeout', ${`${CLOUD_TENANT_GRANT_REPAIR_STATEMENT_TIMEOUT_MS}ms`}, true)`); + await ensureHumanRoleDefaultGrants(tx as unknown as Db, { + companyId: input.companyId, + principalId: input.userId, + membershipRole: input.membershipRole, + grantedByUserId: null, + }); + }); +} + async function auditAgentJwtRunHeaderMismatch( db: Db, input: { companyId: string; agentId: string; claimRunId: string; headerRunId: string; method: string; url: string }, @@ -656,81 +692,107 @@ async function resolveCloudTenantActorOnce( (rows) => rows.length === 0 || rows.some((row) => row.staleInstanceAdmin) ? purgeStaleInstanceAdmin() : undefined, () => purgeStaleInstanceAdmin(), ); - const tenantSync = (async () => { - await syncCloudTenantWrites(db, userId, syncFingerprint, async () => { - const now = new Date(); - await db - .insert(authUsers) - .values({ - id: userId, - name: userName, - email: userEmail, - emailVerified: true, - image: null, - createdAt: now, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: authUsers.id, - set: { name: userName, email: userEmail, emailVerified: true, updatedAt: now }, - setWhere: sql`${authUsers.name} is distinct from ${userName} - or ${authUsers.email} is distinct from ${userEmail} - or ${authUsers.emailVerified} is distinct from true`, - }); - - await insertCloudTenantCompany(db, { companyId, companyName, now }); - if (paperclipCompanyName) { - await repairCloudTenantCompanyName(db, { - companyId, - paperclipCompanyId, - paperclipCompanyName, - now, - }); - } - // Derive any legacy prefix from the repaired name, preserving the helper's - // exact legacy-marker guards and the operator's own edits. - await repairCloudTenantCompanyProvisionDefaults(db, { companyId, stackId, now }); - - effectiveMembership = await db.transaction(async tx => { - const [existing] = await tx.select({ - status: companyMemberships.status, - membershipRole: companyMemberships.membershipRole, - }).from(companyMemberships).where(and( - eq(companyMemberships.companyId, companyId), - eq(companyMemberships.principalType, "user"), - eq(companyMemberships.principalId, userId), - )).for("update"); - // Serialize authority restoration and its notice while retaining the - // guarded upsert that leaves unchanged memberships untouched. - if (!existing || existing.status !== "active" || existing.membershipRole === "viewer") { - await notifyChatCompanyAuthorityChange(tx, { companyId }); - await notifyMuseContactVerification(tx, { companyId, operatorId: userId }); - } - return tx - .insert(companyMemberships) - .values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole, updatedAt: now }) - .onConflictDoUpdate({ - target: [companyMemberships.companyId, companyMemberships.principalType, companyMemberships.principalId], - set: { status: "active", membershipRole, updatedAt: now }, - setWhere: sql`${companyMemberships.status} is distinct from 'active' - or ${companyMemberships.membershipRole} is distinct from ${membershipRole}`, - }) - .returning({ companyId: companyMemberships.companyId, membershipRole: companyMemberships.membershipRole, status: companyMemberships.status }) - // PostgreSQL returns no row for an unchanged ON CONFLICT ... WHERE. - // That condition itself proves the existing role/status match the input. - .then((rows) => rows[0] ?? { companyId, membershipRole, status: "active" }); + // The same promise for every request that shares one pending per-user sync. + const tenantSync = syncCloudTenantWrites(db, userId, syncFingerprint, async () => { + const now = new Date(); + await db + .insert(authUsers) + .values({ + id: userId, + name: userName, + email: userEmail, + emailVerified: true, + image: null, + createdAt: now, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: authUsers.id, + set: { name: userName, email: userEmail, emailVerified: true, updatedAt: now }, + setWhere: sql`${authUsers.name} is distinct from ${userName} + or ${authUsers.email} is distinct from ${userEmail} + or ${authUsers.emailVerified} is distinct from true`, }); - // An unchanged membership can still lack default grants after a partial - // sync or an older deployment; keep this repair inside every actual sync. - await ensureHumanRoleDefaultGrants(db, { + await insertCloudTenantCompany(db, { companyId, companyName, now }); + if (paperclipCompanyName) { + await repairCloudTenantCompanyName(db, { companyId, - principalId: userId, - membershipRole: effectiveMembership.membershipRole ?? membershipRole, - grantedByUserId: null, + paperclipCompanyId, + paperclipCompanyName, + now, }); + } + // Derive any legacy prefix from the repaired name, preserving the helper's + // exact legacy-marker guards and the operator's own edits. + await repairCloudTenantCompanyProvisionDefaults(db, { companyId, stackId, now }); + + effectiveMembership = await db.transaction(async tx => { + const [existing] = await tx.select({ + status: companyMemberships.status, + membershipRole: companyMemberships.membershipRole, + }).from(companyMemberships).where(and( + eq(companyMemberships.companyId, companyId), + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, userId), + )).for("update"); + // Serialize authority restoration and its notice while retaining the + // guarded upsert that leaves unchanged memberships untouched. + if (!existing || existing.status !== "active" || existing.membershipRole === "viewer") { + await notifyChatCompanyAuthorityChange(tx, { companyId }); + await notifyMuseContactVerification(tx, { companyId, operatorId: userId }); + } + return tx + .insert(companyMemberships) + .values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole, updatedAt: now }) + .onConflictDoUpdate({ + target: [companyMemberships.companyId, companyMemberships.principalType, companyMemberships.principalId], + set: { status: "active", membershipRole, updatedAt: now }, + setWhere: sql`${companyMemberships.status} is distinct from 'active' + or ${companyMemberships.membershipRole} is distinct from ${membershipRole}`, + }) + .returning({ companyId: companyMemberships.companyId, membershipRole: companyMemberships.membershipRole, status: companyMemberships.status }) + // PostgreSQL returns no row for an unchanged ON CONFLICT ... WHERE. + // That condition itself proves the existing role/status match the input. + .then((rows) => rows[0] ?? { companyId, membershipRole, status: "active" }); }); - })(); + + // An unchanged membership can still lack default grants after a partial + // sync or an older deployment; keep this repair inside every actual sync. + await ensureHumanRoleDefaultGrants(db, { + companyId, + principalId: userId, + membershipRole: effectiveMembership.membershipRole ?? membershipRole, + grantedByUserId: null, + }); + }); + // Handled at once: the decision below attaches its handler only after the + // membership read returns. + tenantSync.catch(() => {}); + // A request waits for the sync only when its authority depends on it: a new + // user, or a role or status change. When the stored primary membership + // already grants exactly this authority, the sync cannot change the actor, + // so it finishes in the background. Otherwise one open transaction holding + // this user's rows queues every request of the user, static files included, + // behind the shared per-user sync. + const tenantSyncForRequest = membershipRowsRead.then( + (rows) => { + const primary = rows.find((row) => row.companyId === companyId && row.status === "active" + && row.membershipRole === membershipRole); + if (!primary) return tenantSync; + finishCloudTenantSyncInBackground(tenantSync, { userId, stackId }); + // Missing default grants deny the user, so this request waits for the + // grant insert alone, bounded, and never for the rest of the sync. + // A failed repair never fails the request: authorization still denies + // what a missing grant covers, and the background sync retries it. + return missesDefaultGrants(primary.grantedPermissionKeys, membershipRole) + ? repairCloudTenantDefaultGrants(db, { companyId, userId, membershipRole }).catch((err) => { + logger.warn({ err, userId, stackId }, "Cloud tenant default grant repair failed; the sync retries it"); + }) + : undefined; + }, + () => tenantSync, + ); // The stack's seeded company is only where Cloud provisioned this user. // Companies created afterwards on the instance (imports, in-app company @@ -768,7 +830,7 @@ async function resolveCloudTenantActorOnce( // and the owner-settings read can insert the missing settings row. const [purged, synced, membershipsRead, ownerRead] = await Promise.allSettled([ purge, - tenantSync, + tenantSyncForRequest, additionalMembershipsRead, ownerInstanceAdminRead, ]); @@ -799,6 +861,20 @@ async function resolveCloudTenantActorOnce( }; } +// Requests of one user share one sync promise. Track and report each shared +// sync once, so a long-blocked sync does not gain a handler per request. +const backgroundCloudTenantSyncs = new WeakSet>(); + +function finishCloudTenantSyncInBackground(sync: Promise, context: { userId: string; stackId: string }) { + if (backgroundCloudTenantSyncs.has(sync)) return; + backgroundCloudTenantSyncs.add(sync); + // Idle and drain checks still count the writes until they settle. + const finish = beginIdleTrackedWork(); + sync + .catch((err) => logger.warn({ err, ...context }, "Cloud tenant sync failed; the next request retries it")) + .finally(finish); +} + function requiredCloudHeader(req: CloudActorHeaderSource, name: string): string { const value = req.header(name)?.trim(); if (!value) { diff --git a/server/src/middleware/cloud-tenant-actor.test.ts b/server/src/middleware/cloud-tenant-actor.test.ts index f13be1579d1..ee447d1a4a1 100644 --- a/server/src/middleware/cloud-tenant-actor.test.ts +++ b/server/src/middleware/cloud-tenant-actor.test.ts @@ -65,6 +65,8 @@ function createFakeDb(options: { } const db = { transaction: async (work: (tx: Db) => Promise) => work(db), + // Session settings such as the grant repair's lock timeout. + execute: async () => [], insert: (table: unknown) => { insertedTables.push(table); return writeChain(table); @@ -590,4 +592,5 @@ describe("resolveCloudTenantActor (shared-pool hardening)", () => { expect(deletedTables).toContain(instanceUserRoles); }); }); + }); diff --git a/server/src/middleware/cloud-tenant-sync-postgres.test.ts b/server/src/middleware/cloud-tenant-sync-postgres.test.ts index ce7ce28f464..59c5103cba3 100644 --- a/server/src/middleware/cloud-tenant-sync-postgres.test.ts +++ b/server/src/middleware/cloud-tenant-sync-postgres.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { and, eq, sql } from "drizzle-orm"; import { agents, applyPendingMigrations, authUsers, companies, companyMemberships, createDb, instanceSettings, @@ -12,6 +12,8 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./au import { errorHandler } from "./error-handler.js"; import { userProfileRoutes } from "../routes/user-profiles.js"; import { invalidateInstanceSettingsCache } from "../services/instance-settings-cache.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { logger } from "./logger.js"; const support = await getEmbeddedPostgresTestSupport(); const aliasFields = ["name", "email-local", "email", "principal"] as const; @@ -138,7 +140,9 @@ const aliasFields = ["name", "email-local", "email", "principal"] as const; expect(await db.select().from(authUsers).where(eq(authUsers.id, userId))).toEqual([beforeUser]); expect(await db.select().from(companyMemberships).where(membershipWhere)).toEqual([beforeMembership]); expect(await db.execute(sql`select * from cloud_sync_write_probe`)).toEqual([]); - expect((await db.select().from(principalPermissionGrants).where(and(eq(principalPermissionGrants.companyId, companyId), eq(principalPermissionGrants.principalId, userId)))).length).toBeGreaterThan(0); + // An unchanged membership does not hold the request; the repair finishes after it. + await vi.waitFor(async () => expect((await db.select().from(principalPermissionGrants).where(and(eq(principalPermissionGrants.companyId, companyId), eq(principalPermissionGrants.principalId, userId)))).length).toBeGreaterThan(0)); + expect(await db.execute(sql`select * from cloud_sync_write_probe`)).toEqual([]); await resolve(freshDb(), { ...value, "x-paperclip-cloud-user-name": "Changed profile", "x-paperclip-cloud-stack-role": "member" }); const [changedUser] = await db.select().from(authUsers).where(eq(authUsers.id, userId)); const [changedMembership] = await db.select().from(companyMemberships).where(membershipWhere); @@ -148,6 +152,163 @@ const aliasFields = ["name", "email-local", "email", "principal"] as const; expect(changedMembership.updatedAt.getTime()).toBeGreaterThan(oldUpdatedAt.getTime()); }); + it("authenticates an already-synced user while another transaction holds that user's rows", async () => { + const value = headers(); + const actor = await resolve(db, value); + const companyId = actor!.companyIds![0]!; + const userId = value["x-paperclip-cloud-user-id"]; + const holder = freshDb(); + let release!: () => void; + const released = new Promise(done => { release = done; }); + let locked!: () => void; + const lockHeld = new Promise(done => { locked = done; }); + // Another open transaction that touched this user: it inserted a row that + // references the user and holds the membership row. Staging hung every + // request of one signed-in user for 6.5 minutes behind such a transaction. + const holding = holder.transaction(async tx => { + await tx.execute(sql`select id from "user" where id = ${userId} for key share`); + await tx.execute(sql`select id from company_memberships where company_id = ${companyId} + and principal_type = 'user' and principal_id = ${userId} for update`); + locked(); + await released; + }); + await lockHeld; + try { + const next = freshDb(); // a new client has no process-local write debounce + const outcome = await Promise.race([ + resolve(next, value).then(result => ({ result })), + new Promise<"timed_out">(done => setTimeout(() => done("timed_out"), 3_000)), + ]); + expect(outcome).not.toBe("timed_out"); + const result = (outcome as { result: Awaited> }).result; + expect(result?.companyIds).toEqual(actor!.companyIds); + expect(result?.memberships?.[0]).toEqual({ companyId, membershipRole: "owner", status: "active" }); + } finally { + release(); + await holding; + } + }); + + it("tracks one blocked shared sync once across many requests and reports its failure once", async () => { + const value = headers(); + const actor = await resolve(db, value); + const companyId = actor!.companyIds![0]!; + const userId = value["x-paperclip-cloud-user-id"]; + const holder = freshDb(); + let release!: () => void; + const released = new Promise(done => { release = done; }); + let locked!: () => void; + const lockHeld = new Promise(done => { locked = done; }); + const holding = holder.transaction(async tx => { + await tx.execute(sql`select id from "user" where id = ${userId} for key share`); + await tx.execute(sql`select id from company_memberships where company_id = ${companyId} + and principal_type = 'user' and principal_id = ${userId} for update`); + locked(); + await released; + }); + await lockHeld; + const warn = vi.spyOn(logger, "warn"); + const before = idleWorkSnapshot().active; + try { + const next = freshDb(); // one server's client, past the write debounce + for (let index = 0; index < 5; index += 1) { + expect((await resolve(next, value))?.companyIds).toEqual(actor!.companyIds); + } + expect(idleWorkSnapshot().active).toBe(before + 1); + // End the blocked sync with an error, as a lock or statement timeout would. + await vi.waitFor(async () => { + const cancelled = await db.execute(sql`select pg_cancel_backend(pid) as cancelled from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock'`); + expect(cancelled.length).toBeGreaterThan(0); + }); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(before)); + expect(warn.mock.calls.filter(call => call[1] === "Cloud tenant sync failed; the next request retries it")).toHaveLength(1); + } finally { + warn.mockRestore(); + release(); + await holding; + } + }); + + it("restores missing default grants before an unchanged membership skips a blocked sync", async () => { + const value = headers(); + const actor = await resolve(db, value); + const companyId = actor!.companyIds![0]!; + const userId = value["x-paperclip-cloud-user-id"]; + const grantsOf = () => db.select().from(principalPermissionGrants).where(and( + eq(principalPermissionGrants.companyId, companyId), eq(principalPermissionGrants.principalId, userId))); + const granted = (await grantsOf()).length; + expect(granted).toBeGreaterThan(0); + // An earlier sync wrote the membership and failed before its grants. + await db.delete(principalPermissionGrants).where(and( + eq(principalPermissionGrants.companyId, companyId), eq(principalPermissionGrants.principalId, userId))); + const holder = freshDb(); + let release!: () => void; + const released = new Promise(done => { release = done; }); + let locked!: () => void; + const lockHeld = new Promise(done => { locked = done; }); + const holding = holder.transaction(async tx => { + await tx.execute(sql`select id from "user" where id = ${userId} for key share`); + await tx.execute(sql`select id from company_memberships where company_id = ${companyId} + and principal_type = 'user' and principal_id = ${userId} for update`); + locked(); + await released; + }); + await lockHeld; + try { + const outcome = await Promise.race([ + resolve(freshDb(), value).then(result => ({ result })), + new Promise<"timed_out">(done => setTimeout(() => done("timed_out"), 3_000)), + ]); + expect(outcome).not.toBe("timed_out"); + // The grants exist when the request proceeds, not some time after it. + expect((await grantsOf()).length).toBe(granted); + } finally { + release(); + await holding; + } + }); + + it("still authenticates when the default grant repair times out on a held lock", async () => { + const value = headers(); + const actor = await resolve(db, value); + const companyId = actor!.companyIds![0]!; + const userId = value["x-paperclip-cloud-user-id"]; + await db.delete(principalPermissionGrants).where(and( + eq(principalPermissionGrants.companyId, companyId), eq(principalPermissionGrants.principalId, userId))); + const holder = freshDb(); + let release!: () => void; + const released = new Promise(done => { release = done; }); + let locked!: () => void; + const lockHeld = new Promise(done => { locked = done; }); + // The company row is locked FOR UPDATE, so the grant insert's foreign-key + // check waits until its lock timeout. + const holding = holder.transaction(async tx => { + await tx.execute(sql`select id from companies where id = ${companyId} for update`); + await tx.execute(sql`select id from company_memberships where company_id = ${companyId} + and principal_type = 'user' and principal_id = ${userId} for update`); + locked(); + await released; + }); + await lockHeld; + const warn = vi.spyOn(logger, "warn"); + try { + const outcome = await Promise.race([ + resolve(freshDb(), value).then(result => ({ result }), error => ({ error })), + new Promise<"timed_out">(done => setTimeout(() => done("timed_out"), 6_000)), + ]); + expect(outcome).not.toBe("timed_out"); + expect(outcome).not.toHaveProperty("error"); + expect((outcome as { result: Awaited> }).result?.memberships?.[0]) + .toEqual({ companyId, membershipRole: "owner", status: "active" }); + expect(warn.mock.calls.some(call => call[1] === "Cloud tenant default grant repair failed; the sync retries it")).toBe(true); + } finally { + warn.mockRestore(); + release(); + await holding; + } + }, 15_000); + it("preserves runtime invalidation on real membership changes while no-op conflicts produce no invalidation", async () => { const value = headers(); const actor = await resolve(db, value); diff --git a/server/src/middleware/cloud-tenant-sync.ts b/server/src/middleware/cloud-tenant-sync.ts index 9ed0bccbb8b..5d364a410a2 100644 --- a/server/src/middleware/cloud-tenant-sync.ts +++ b/server/src/middleware/cloud-tenant-sync.ts @@ -32,8 +32,10 @@ function prune(states: Map, now: number) { * Only the last queued context can be shared: A -> B -> A must still apply A * after B. Failed/partial syncs never populate the successful-write debounce. * This caches writes only; callers must still resolve current authority. + * Callers that share a pending sync receive the same promise object, so they + * can track that sync once. */ -export async function syncCloudTenantWrites( +export function syncCloudTenantWrites( db: Db, userId: string, fingerprint: string,