diff --git a/doc/plans/2026-10-24-recurring-server-sleep-stack.md b/doc/plans/2026-10-24-recurring-server-sleep-stack.md index bd37f78a858..c1fdba19aae 100644 --- a/doc/plans/2026-10-24-recurring-server-sleep-stack.md +++ b/doc/plans/2026-10-24-recurring-server-sleep-stack.md @@ -1688,3 +1688,13 @@ six action-worker tests, server typecheck/build. No provider authorization was relaxed. Cloud trusted-header membership sync and board claim remain alternate writers to cover before production registration/poll removal. Older parent findings and the complete original stack remain open. + + +Slice 28 now covers Cloud trusted-header sync and board claim. The missing +Cloud notice and absent board-claim intents were reproduced before repair. +All 39 post-close cases plus 30 provisioning/auth/claim cases pass; server types +and build pass. Unchanged Cloud profile synchronization and active board membership +stay silent. See the milestone audit for transaction boundaries and retained logs. +The previous pushed head 0f38d1079d has the recurring E2E shard-3 cancellation +failure; process-identity event append deadlock evidence is retained for diagnosis. +Production registration/poll removal and the entire remaining stack are still open. diff --git a/doc/plans/2026-10-25-chat-run-milestone-sleep-audit.md b/doc/plans/2026-10-25-chat-run-milestone-sleep-audit.md index 52ad57b1d50..1c24306eab9 100644 --- a/doc/plans/2026-10-25-chat-run-milestone-sleep-audit.md +++ b/doc/plans/2026-10-25-chat-run-milestone-sleep-audit.md @@ -1413,3 +1413,51 @@ runtime fence; old sources must not regain authority from a new credential fingerprint. Keep the existing generation/chronology denial tests. Remaining producer dispositions, production wiring, full remote gates and original stack scope are still required; this checkpoint is not a completed sleep migration. + + +## Cloud sign-in and board-claim authority restoration + +Cloud trusted-header membership synchronization now owns a short transaction for +its existing membership upsert. It locks the existing membership while comparing +its authority and registers a company-scoped milestone intent before a new, +reactivated, or formerly viewer membership becomes active. Profile-only syncs +remain silent even when the debounce is bypassed by a changed profile. The +trusted-header roles already exclude viewer; existing authentication, company +provisioning, stale instance-admin removal and default-grant behavior are retained. +This adds one indexed membership read and a transaction per actual debounced sync, +not an idle timer. Provider calls are not wrapped in a transaction. + +Board claim uses its existing transaction and signals only actual membership +insertion/reactivation. Existing active membership behavior is unchanged. Its +real database tests verify that nested savepoint completion emits intent, while +settlement waits for the caller's outer commit and the outside connection still +sees the old membership. This is producer/transaction proof; the Cloud integration +case additionally proves the real post-close milestone becomes publishable. + +Evidence retained: +- /tmp/sleep28-cloud-sync-before.log: restored Cloud authority missed its exact + working notice before repair, after the worker had exhausted startup discovery. +- /tmp/sleep28-board-claim-before.log: new and suspended membership cases failed + because no intent was emitted; unchanged active membership passed. +- /tmp/sleep28-cloud-authority-acceptance.log: all 39 post-close integration cases + pass, including Cloud restoration and a forced second profile sync with no + milestone signal. Existing NOWAIT admission retries remain visible in the log. +- /tmp/sleep28-cloud-owner-acceptance.log: all 30 Cloud provisioning, auth route, + transient retry and board-claim cases pass. Dedicated external test DB support + preserves the embedded fallback and closes the owning clients. +- /tmp/sleep28-cloud-authority-typecheck.log retained the initial redundant + viewer comparison failure. Cloud's validated role union excludes viewer; the + redundant condition was removed. Final typecheck and build pass in + /tmp/sleep28-cloud-authority-final-typecheck.log and + /tmp/sleep28-cloud-authority-build.log. Diff whitespace checks pass. + +The pre-push remote head 0f38d1079d fails E2E shard 3 and aggregate in run +40210763612, job 121221379705. The complete completed-job log is retained at +/tmp/sleep28-0f38-e2e3-failure.log. It again fails split-handoff cleanup at +agent-chat-projects.spec.ts:161 (HTTP 500), followed by the experiment-state +assertion at line 568. The log also records a process-identity event append +transaction deadlock at heartbeat.ts:3409 / run-lifecycle.ts:583. This is evidence +for the unresolved cancellation investigation, not proof of its entire lock cycle. +No blind rerun or masking was performed. Production milestone registration, +final producer dispositions, native progress as its separate slice, remote gates, +and every remaining original stack duty remain outstanding. diff --git a/server/src/__tests__/auth-session-route.test.ts b/server/src/__tests__/auth-session-route.test.ts index e30c09896ac..1ab75bf346f 100644 --- a/server/src/__tests__/auth-session-route.test.ts +++ b/server/src/__tests__/auth-session-route.test.ts @@ -10,12 +10,16 @@ import { import { errorHandler } from "../middleware/error-handler.js"; import { assertCompanyAccess } from "../routes/authz.js"; +function queryResult(rows: T) { + return Object.assign(Promise.resolve(rows), { for: () => Promise.resolve(rows) }); +} + function createSelectChain(rows: unknown[]) { return { from() { return { where() { - return Promise.resolve(rows); + return queryResult(rows); }, }; }, @@ -102,6 +106,7 @@ describe("actorMiddleware authenticated session profile", () => { delete: vi.fn(() => ({ where: () => Promise.resolve(undefined) })), select: vi.fn(() => createSelectChain([])), } as any; + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { @@ -176,7 +181,7 @@ describe("actorMiddleware authenticated session profile", () => { select: vi.fn(() => ({ from: (table: unknown) => ({ where: () => - Promise.resolve( + queryResult( table === companyMemberships ? [{ companyId: importedCompanyId, membershipRole: "member", status: "active" }] : [], @@ -186,6 +191,7 @@ describe("actorMiddleware authenticated session profile", () => { insert: vi.fn(() => insertChain), delete: vi.fn(() => ({ where: () => Promise.resolve(undefined) })), } as any; + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { @@ -252,7 +258,7 @@ describe("actorMiddleware authenticated session profile", () => { select: vi.fn(() => ({ from: (table: unknown) => ({ where: () => - Promise.resolve( + queryResult( table === companies ? [{ name: "paperclip-stack-purple-rain" }] : [], @@ -280,7 +286,7 @@ describe("actorMiddleware authenticated session profile", () => { })), delete: vi.fn(() => ({ where: () => Promise.resolve(undefined) })), } as any; - db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { @@ -349,7 +355,7 @@ describe("actorMiddleware authenticated session profile", () => { select: vi.fn(() => ({ from: (table: unknown) => ({ where: () => - Promise.resolve( + queryResult( table === instanceUserRoles && state.staleInstanceAdminRow ? [{ id: "stale-role-row" }] : [], ), }), @@ -362,6 +368,7 @@ describe("actorMiddleware authenticated session profile", () => { }, })), } as any; + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { diff --git a/server/src/__tests__/board-claim.test.ts b/server/src/__tests__/board-claim.test.ts index f408a8fbc08..b103cbb7d1a 100644 --- a/server/src/__tests__/board-claim.test.ts +++ b/server/src/__tests__/board-claim.test.ts @@ -1,3 +1,5 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -8,6 +10,7 @@ import { createDb, instanceUserRoles, principalPermissionGrants, + subscribeDatabaseWork, } from "@paperclipai/db"; import { claimBoardOwnership, @@ -20,7 +23,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_BOARD_CLAIM_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; describeEmbeddedPostgres("board claim", () => { @@ -28,8 +32,8 @@ describeEmbeddedPostgres("board claim", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-board-claim-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-board-claim-"); + db = createDb(external ?? tempDb!.connectionString); }, 20_000); afterEach(async () => { @@ -42,10 +46,11 @@ describeEmbeddedPostgres("board claim", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); - it("lets a signed-in user claim a local-board-only authenticated instance", async () => { + it.each(["new", "suspended", "active"] as const)("claims a local-board-only instance with %s membership and wakes only restored authority", async prior => { const now = new Date(); const userId = `claim-user-${randomUUID()}`; const company = await db @@ -70,6 +75,10 @@ describeEmbeddedPostgres("board claim", () => { role: "instance_admin", }); + if (prior !== "new") await db.insert(companyMemberships).values({ + companyId: company.id, principalType: "user", principalId: userId, + status: prior, membershipRole: "owner", + }); await initializeBoardClaimChallenge(db, { deploymentMode: "authenticated" }); const warningUrl = getBoardClaimWarningUrl("127.0.0.1", 3197); expect(warningUrl).toBeTruthy(); @@ -84,12 +93,19 @@ describeEmbeddedPostgres("board claim", () => { claimedByUserId: null, }); - await expect( - claimBoardOwnership(db, { token, code, userId }), - ).resolves.toEqual({ - status: "claimed", - claimedByUserId: userId, - }); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => events.push(event)); + try { + await db.transaction(async tx => { + await expect(claimBoardOwnership(tx as unknown as typeof db, { token, code, userId })) + .resolves.toEqual({ status: "claimed", claimedByUserId: userId }); + expect(events).toEqual(prior === "active" ? [] : ["intent"]); + // A nested claim must not publish a commit before its caller settles. + const outside = await db.select().from(companyMemberships).where(eq(companyMemberships.companyId, company.id)); + expect(outside.map(row => row.status)).toEqual(prior === "new" ? [] : [prior]); + }); + expect(events).toEqual(prior === "active" ? [] : ["intent", "settled"]); + } finally { unsubscribe(); } await expect( db diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 45d6c3a3640..93e66b3f4ca 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1,3 +1,5 @@ +import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "../middleware/auth.js"; +import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js"; import { accessService } from "../services/access.js"; import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; import { restoreNativeChatReviewPresentationInTransaction } from "../services/native-runtime/native-chat-review-presentation.js"; @@ -1134,8 +1136,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const companyPrefixAttempts = 8; const freshCompanyPrefix = () => `C${randomUUID().replaceAll("-", "").slice(0, 7).toUpperCase()}`; - async function seedCompany(nextIssuePrefix = freshCompanyPrefix) { - const companyId = randomUUID(); + async function seedCompany(nextIssuePrefix = freshCompanyPrefix, companyId = randomUUID()) { const assignedAgentId = randomUUID(); const replacementAgentId = randomUUID(); let inserted = false; @@ -51338,8 +51339,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const registeredCommandId = "888888888888888819"; let interactionSequence = 0n; - async function commandFixture() { - const fixture = await seedCompany(); + async function commandFixture(companyId?: string) { + const fixture = await seedCompany(undefined, companyId); const applicationId = uniqueDiscordApplicationId(); let holdWork = false; const queued: Array<() => void> = []; @@ -52433,8 +52434,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { : lastFreshDiscordMessageId + 1n; return lastFreshDiscordMessageId.toString(); }; - async function reopenedFixture(includeDuringConfirmationSource = false) { - const f = await commandFixture(); + async function reopenedFixture(includeDuringConfirmationSource = false, companyId?: string) { + const f = await commandFixture(companyId); const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; const admitMessage = async (message: ReturnType) => { await deliverMessage({ @@ -52621,9 +52622,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } finally { await coordinator.stop(); await f.close(); } }); - it.each(["connection", "resource", "unlinked", "identity", "member", "member_permissions", "ensure_membership", "company_access", "visibility"] as const)( + it.each(["connection", "resource", "unlinked", "identity", "member", "member_permissions", "ensure_membership", "company_access", "visibility", "cloud_sync"] as const)( "wakes a post-close milestone when %s authority is restored", async owner => { - const f = await reopenedFixture(); + const stackId = randomUUID(); + const f = await reopenedFixture(false, owner === "cloud_sync" ? cloudTenantPrimaryCompanyId(stackId) : undefined); + const previousToken = process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; const onError = vi.fn(); const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); const notice = () => db.select().from(chatPublications).where(and( @@ -52662,6 +52665,20 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await db.update(companyMemberships).set({ status: "suspended" }).where(eq(companyMemberships.id, membership.id)); if (owner === "member") restore = () => access.updateMember(f.fixture.companyId, membership.id, { status: "active" }); else if (owner === "member_permissions") restore = () => access.updateMemberAndPermissions(f.fixture.companyId, membership.id, { status: "active", grants: [] }, "owner-user"); + else if (owner === "cloud_sync") { + process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "milestone-cloud-sync-test"; + let request = 0; + restore = () => resolveCloudTenantActor(db, cloudActorHeaderSourceFromHeaders({ + "x-paperclip-cloud-tenant-token": "milestone-cloud-sync-test", + "x-paperclip-cloud-user-id": "owner-user", + "x-paperclip-cloud-user-email": "owner@example.test", + // Change the profile to force an actual sync on replay. The + // debounce must not hide repeated unchanged membership wakes. + "x-paperclip-cloud-user-name": `Owner ${request++}`, + "x-paperclip-cloud-stack-id": stackId, + "x-paperclip-cloud-stack-role": "owner", + })); + } else if (owner === "ensure_membership") restore = () => access.ensureMembership(f.fixture.companyId, "user", "owner-user", membership.membershipRole, "active"); else { const existing = await db.select().from(companyMemberships).where(and( @@ -52686,7 +52703,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(await notice()).toHaveLength(1); } expect(onError).not.toHaveBeenCalled(); - } finally { await coordinator.stop(); await f.close(); } + } finally { + if (previousToken === undefined) delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; + else process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = previousToken; + await coordinator.stop(); await f.close(); + } }, ); diff --git a/server/src/__tests__/cloud-tenant-company-provisioning.test.ts b/server/src/__tests__/cloud-tenant-company-provisioning.test.ts index c57dde6d63e..3c08f90e7e7 100644 --- a/server/src/__tests__/cloud-tenant-company-provisioning.test.ts +++ b/server/src/__tests__/cloud-tenant-company-provisioning.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { createHash } from "node:crypto"; import { afterAll, afterEach, beforeEach, beforeAll, describe, expect, it } from "vitest"; import { and, eq } from "drizzle-orm"; @@ -18,7 +19,8 @@ import { } from "./helpers/embedded-postgres.js"; import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "../middleware/auth.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CLOUD_PROVISIONING_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -59,8 +61,8 @@ describeEmbeddedPostgres("cloud tenant company provisioning", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-tenant-provisioning-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-tenant-provisioning-"); + db = createDb(external ?? tempDb!.connectionString); }, 20_000); beforeEach(() => { @@ -80,6 +82,7 @@ describeEmbeddedPostgres("cloud tenant company provisioning", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/board-claim.ts b/server/src/board-claim.ts index c4784406646..275dfdf0180 100644 --- a/server/src/board-claim.ts +++ b/server/src/board-claim.ts @@ -4,6 +4,7 @@ import type { Db } from "@paperclipai/db"; import { companies, companyMemberships, instanceUserRoles } from "@paperclipai/db"; import type { DeploymentMode } from "@paperclipai/shared"; import { ensureHumanRoleDefaultGrants } from "./services/principal-access-compatibility.js"; +import { notifyChatRunMilestones } from "./services/chat-run-milestone-notifications.js"; const LOCAL_BOARD_USER_ID = "local-board"; const CLAIM_TTL_MS = 1000 * 60 * 60 * 24; @@ -124,6 +125,7 @@ export async function claimBoardOwnership( .then((rows) => rows[0] ?? null); if (!existing) { + await notifyChatRunMilestones(tx, { companyId: company.id }); await tx.insert(companyMemberships).values({ companyId: company.id, principalType: "user", @@ -135,6 +137,7 @@ export async function claimBoardOwnership( } if (existing.status !== "active") { + await notifyChatRunMilestones(tx, { companyId: company.id }); await tx .update(companyMemberships) .set({ status: "active", membershipRole: "owner", updatedAt: new Date() }) diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index 8504fc3c853..6142e13ef81 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -30,6 +30,7 @@ import { captureRunIdentity } from "../services/run-identity.js"; import { boardAuthService } from "../services/board-auth.js"; import { retryIdempotentDatabaseOperation } from "../database-retry.js"; import { beginIdleTrackedWork } from "../services/task-admission.js"; +import { notifyChatRunMilestones } from "../services/chat-run-milestone-notifications.js"; export { isTransientDbConnectionError, @@ -685,34 +686,51 @@ async function resolveCloudTenantActorOnce( await repairCloudTenantCompanyProvisionDefaults(db, { companyId, stackId, now }); } - effectiveMembership = shouldSync ? await db - .insert(companyMemberships) - .values({ - companyId, - principalType: "user", - principalId: userId, - status: "active", - membershipRole, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [ - companyMemberships.companyId, - companyMemberships.principalType, - companyMemberships.principalId, - ], - set: { - status: "active", - membershipRole, - updatedAt: now, - }, - }) - .returning() - .then((rows) => rows[0] ?? { - companyId, - membershipRole, - status: "active", - }) : { companyId, membershipRole, status: "active" as const }; + if (shouldSync) { + effectiveMembership = await db.transaction(async tx => { + const [existing] = await tx.select({ + status: companyMemberships.status, + membershipRole: companyMemberships.membershipRole, + }).from(companyMemberships).where(and( + eq(companyMemberships.companyId, companyId), + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, userId), + )).for("update"); + // A trusted-header sync can restore publication authority. Serialize the + // comparison with the upsert so unchanged profile syncs remain quiet. + if (!existing || existing.status !== "active" || existing.membershipRole === "viewer") { + await notifyChatRunMilestones(tx, { companyId }); + } + return tx + .insert(companyMemberships) + .values({ + companyId, + principalType: "user", + principalId: userId, + status: "active", + membershipRole, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + companyMemberships.companyId, + companyMemberships.principalType, + companyMemberships.principalId, + ], + set: { + status: "active", + membershipRole, + updatedAt: now, + }, + }) + .returning() + .then((rows) => rows[0] ?? { + companyId, + membershipRole, + status: "active", + }); + }); + } // Without instance-admin elevation, cloud tenant users are authorized purely // through company-scoped permission grants — seed the same role defaults the