diff --git a/doc/plans/2026-10-30-deferred-execution-sleep.md b/doc/plans/2026-10-30-deferred-execution-sleep.md index e84d9abf165..ded551cdf22 100644 --- a/doc/plans/2026-10-30-deferred-execution-sleep.md +++ b/doc/plans/2026-10-30-deferred-execution-sleep.md @@ -25,3 +25,5 @@ A resume that first records a new execution-wait reason arms its existing 30-sec Persisted budget and agent refusals are normalized by the existing deferred-wait owner. The wait writer reports whether its locked conditional update saved the receipt, and only that successful path can produce the internal `PersistedExecutionWaitRefusal` subtype. It remains an HTTP409 with the original message and details. Receipt recovery parks that outcome instead of creating failure debt; the committed first/changed reason still grants its one stored30-second grace. Unrelated409s, failed writes and unexpected provider/infrastructure errors retain failure backoff. This adds no admission policy and changes no native/task-chat ownership. Production-path controls use actual task Retry, heartbeat, PostgreSQL budget/lifecycle services and a child process writing saved stdout. The lifecycle host completion boundary is controlled explicitly. One set of real budget/agent refusal fixtures consumes its finite grace and becomes quiet for25 minutes of timer advancement with zero driver SQL and no execution. Independent warm-worker fixtures keep the real PostgreSQL clock throughout admission, actual gate-producer release, controller authority and child-process exit, proving exactly one saved execution. Long-idle and provider-release evidence are separate; no clock reset or restart is used to claim warm recovery. A PostgreSQL trigger fault proves failed wait persistence retains recovery backoff, and an unrelated unbranded409 remains a failure. + +Project budget admission locks and re-reads the company-scoped project before publishing pause intent. An already-paused project emits no new eligibility hint and preserves its pause reason and timestamp, including a manual pause. The actual transition still publishes dispatch and deferred eligibility intent before updating the project in the accounting transaction. Concurrent admission checks serialize through that authority. Project-scoped saved Retry controls cover its one grace followed by25 minutes with zero driver SQL, plus a separate real-clock budget increase through one succeeded process with saved stdout. diff --git a/doc/plans/2026-10-30-github-connection-sleep.md b/doc/plans/2026-10-30-github-connection-sleep.md index 6d381373b19..9369d76cf4b 100644 --- a/doc/plans/2026-10-30-github-connection-sleep.md +++ b/doc/plans/2026-10-30-github-connection-sleep.md @@ -43,7 +43,11 @@ A grant stores a unique `bindingSync` revision and removed installation IDs befo Cloud registration calls. Successful delivery clears only the locked current revision. Failure retains the saved intent and an exponential retry deadline, including provider Retry-After advice. A restart loads that deadline from the -grant. Binding-only retries use the existing credential refresh lease and current +grant. A retry write compares the attempted credential/installation revision under +the grant lock and cannot delay work committed by a newer revision. Restoration +creates a new binding revision and removes active installations from the pending +retirement list; binding commands also reject legacy conflicting retirements. +Binding-only retries use the existing credential refresh lease and current secret resolver without repeating account/repository discovery. The initial OAuth callback uses the same saved debt and settlement path after connection activation. diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index 3f3b8933cbc..97566de3bc4 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -155,3 +155,8 @@ closed-range evaluations perform one conversion, not another minute walk. Both caches use bounded LRU retention and hold no company/card data. Tests prove eviction, skipped/repeated hours, half-hour DST, date-line changes, rollback, strict boundaries, and actual twenty-card source delivery followed by quiet. + + +Refresh decisions use the card's PostgreSQL row revision as a private compare-and-swap fence. If a card changes during watched-source reads, the owner rereads it and reconsiders the current policy, for at most three attempts. A replacement future interval deadline remains unchanged; a replacement manual policy does not run automatically. The same fence protects no-change, paused-policy and generation claims. Existing issue idempotency, generation ownership and losing-claim cancellation still apply. The row revision is not exposed through card APIs or shared scheduling contracts. + +Company source hints do not preempt a future deadline owned by a failed evaluation (`retry`). Membership and policy deadlines remain eligible for source-driven rechecks, and actual due retries still run. An explicit policy edit can replace the retry through the existing card mutation path. Real PostgreSQL regressions hold the actual search while committing an ordinary policy edit, then verify the saved interval still fires. Persistent search failures stay quiet through issue hints and restart, retry at their saved deadline, and recover through an ordinary policy edit. diff --git a/server/src/__tests__/budgets-service.test.ts b/server/src/__tests__/budgets-service.test.ts index 0ea5a94dfc6..1b2495cf7b6 100644 --- a/server/src/__tests__/budgets-service.test.ts +++ b/server/src/__tests__/budgets-service.test.ts @@ -1,4 +1,5 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -264,6 +265,31 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => { }); }); + it.each([false, true])("only signals a real project budget pause with concurrent admission checks (already paused: %s)", async alreadyPaused => { + const { companyId, agentId, projectId } = await createBudgetFixture(); + await db.insert(budgetPolicies).values({ companyId, scopeType: "project", scopeId: projectId, + metric: "billed_cents", windowKind: "lifetime", amount: 100, hardStopEnabled: true, notifyEnabled: false }); + await insertCostEvent({ companyId, agentId, projectId, costCents: 125 }); + const pausedAt = new Date("2020-01-01T00:00:00.000Z"); + if (alreadyPaused) await db.update(projects).set({ pausedAt, pauseReason: "manual" }).where(eq(projects.id, projectId)); + const dispatch = vi.fn(), deferred = vi.fn(); + const unsubscribe = [subscribeDeliveryWork(db, DELIVERY_QUEUES.queuedRunDispatch, dispatch), + subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred)]; + try { + const service = budgetService(db); + const blocks = await Promise.all(Array.from({ length: 4 }, () => service.getInvocationBlock(companyId, agentId, { projectId }))); + expect(blocks.every(block => block?.scopeType === "project" && block.scopeId === projectId)).toBe(true); + const [paused] = await db.select().from(projects).where(eq(projects.id, projectId)); + expect(paused).toMatchObject({ pauseReason: alreadyPaused ? "manual" : "budget", pausedAt: alreadyPaused ? pausedAt : expect.any(Date) }); + expect(dispatch).toHaveBeenCalledTimes(alreadyPaused ? 0 : 1); + expect(deferred).toHaveBeenCalledTimes(alreadyPaused ? 0 : 1); + await Promise.all(Array.from({ length: 4 }, () => service.getInvocationBlock(companyId, agentId, { projectId }))); + expect((await db.select().from(projects).where(eq(projects.id, projectId)))[0]).toEqual(paused); + expect(dispatch).toHaveBeenCalledTimes(alreadyPaused ? 0 : 1); + expect(deferred).toHaveBeenCalledTimes(alreadyPaused ? 0 : 1); + } finally { unsubscribe.forEach(stop => stop()); } + }); + it("hard-stops project work until a valid budget raise resumes it and overview reconciles ledger spend", async () => { const { companyId, agentId, projectId } = await createBudgetFixture(); const cancelWorkForScope = vi.fn().mockResolvedValue(undefined); diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index 18af0f94f97..78c56ff9a83 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -13,7 +13,7 @@ import { issueTreeControlService } from "../services/issue-tree-control.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, costEvents, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; +import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, costEvents, createDb, projects, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -142,17 +142,22 @@ describe("deferred receipt durable dispatch", () => { expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1); await quiet(); }); - async function blockedRetry(gate: "budget" | "paused agent", clocked = false) { + async function blockedRetry(gate: "budget" | "project budget" | "paused agent", clocked = false) { const f = await fixture(), lifecycle = createAgentLifecycle(db), budgets = budgetService(db); // Host lifecycle completion is controlled; all state, admission and the // eventual process adapter execution use their production implementations. const lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runHost: async () => "complete", runPlugin: async () => "complete" }); lifecycleWorkers.add(lifecycleWorker); - const policy = { scopeType: "agent" as const, scopeId: f.agent.id, amount: 100, + const projectId = gate === "project budget" ? randomUUID() : null; + if (projectId) { + await db.insert(projects).values({ id: projectId, companyId: f.companyId, name: "Retry budget project" }); + await db.update(issues).set({ projectId }).where(eq(issues.id, f.issue.id)); + } + const policy = { scopeType: projectId ? "project" as const : "agent" as const, scopeId: projectId ?? f.agent.id, amount: 100, metric: "billed_cents" as const, windowKind: "lifetime" as const, hardStopEnabled: true, notifyEnabled: false }; - if (gate === "budget") { - await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agent.id, + if (gate !== "paused agent") { + await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agent.id, projectId, provider: "test", model: "receipt-proof", costCents: 200, occurredAt: new Date() }); await budgets.upsertPolicy(f.companyId, policy, f.actorId); } else await lifecycle.pauseAgent(f.agent.id); @@ -166,18 +171,18 @@ describe("deferred receipt durable dispatch", () => { expect(request.accepted).toBe(true); const read = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0]; await vi.waitFor(async () => expect((await read()).payload?.executionWait).toMatchObject({ - reason: gate === "budget" ? "budget.blocked" : "agent.not_invokable", + reason: gate !== "paused agent" ? "budget.blocked" : "agent.not_invokable", }), { timeout: 10_000 }); const refusals: unknown[] = []; await heartbeat.resumeExecutionWaitComments({ companyId: f.companyId, requestId: request.requestId!, onError: error => refusals.push(error) }); expect(refusals).toHaveLength(1); expect(refusals[0]).toBeInstanceOf(HttpError); expect(refusals[0]).toMatchObject({ status: 409, - details: gate === "budget" ? { scopeType: "agent", scopeId: f.agent.id } : { status: "paused", reason: "paused" } }); + details: gate !== "paused agent" ? { scopeType: policy.scopeType, scopeId: policy.scopeId } : { status: "paused", reason: "paused" } }); return { f, lifecycle, budgets, policy, lifecycleWorker, request, read }; } - it.each(["budget", "paused agent"] as const)("parks real task Retry behind unchanged %s without recurring SQL", async gate => { + it.each(["budget", "project budget", "paused agent"] as const)("parks real task Retry behind unchanged %s without recurring SQL", async gate => { const { request, read } = await blockedRetry(gate, true); // The clock was installed before worker registration and every timer. // No execution exists while we observe actual driver SQL across25min. @@ -190,11 +195,11 @@ describe("deferred receipt durable dispatch", () => { expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0); }); - it.each(["budget", "paused agent"] as const)("releases real %s Retry into exactly one saved process execution", async gate => { + it.each(["budget", "project budget", "paused agent"] as const)("releases real %s Retry into exactly one saved process execution", async gate => { const { f, lifecycle, budgets, policy, lifecycleWorker, request, read } = await blockedRetry(gate); // This independent warm-worker fixture stays on the real PostgreSQL clock // through gate release, controller authority and actual child-process exit. - if (gate === "budget") await budgets.upsertPolicy(f.companyId, { ...policy, amount: 1000 }, f.actorId); + if (gate !== "paused agent") await budgets.upsertPolicy(f.companyId, { ...policy, amount: 1000 }, f.actorId); else await lifecycle.resumeAgent(f.agent.id); await lifecycleWorker.process(f.agent.id); let runId: string | null = null; diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index f0c6480a027..419580a1fbb 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -1,4 +1,3 @@ -import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index 1a268d57605..783ed45de43 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -6,9 +6,9 @@ import { createServer, type Server } from "node:http"; import { once } from "node:events"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { applyPendingMigrations, companies, activityLog, companySecrets, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; +import { applyPendingMigrations, companies, activityLog, companySecrets, connectionEventDeliveries, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; -import { createGitHubGrantContinuity, githubGrantContinuityDeadline } from "../services/tool-access/github-grant-continuity.js"; +import { createGitHubGrantContinuity, githubGrantContinuityDeadline, githubContinuityRevision } from "../services/tool-access/github-grant-continuity.js"; import { createPaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; import { githubConnectionEventService } from "../services/github-connection-events.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -36,6 +36,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); let releaseRefresh: (() => void) | undefined, refreshArrived: (() => void) | undefined; let refreshGate: Promise | undefined; + let discoveryFailureGate: Promise | undefined, discoveryArrived: (() => void) | undefined; let heldBindingCompany: string | undefined, bindingGate: Promise | undefined; let bindingArrived: (() => void) | undefined, bindingClosed: (() => void) | undefined; const realFetch = globalThis.fetch; @@ -86,6 +87,10 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( res.writeHead(failBindings ? 503 : 200, { "content-type": "application/json" }).end("{}"); return; } githubRequests++; + if (req.url === "/user" && discoveryFailureGate) { + discoveryArrived?.(); await discoveryFailureGate; + res.writeHead(503, { "content-type": "application/json", "retry-after": "86400" }).end("{}"); return; + } const body = req.url === "/user" ? { id: 42, login: "octocat" } : req.url!.startsWith("/user/installations?") ? { installations: [{ id: 101, repository_selection: "all", account: { login: "octocat" } }] } : { total_count: 1, repositories: [{ id: 999, full_name: "octocat/repo" }] }; @@ -187,6 +192,108 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); }); + it.each(["credential", "installation"])("does not attach a withheld provider failure to newer %s work", async change => { + const f = await fixture(["101"]); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + let release!: () => void; + const arrived = new Promise(resolve => { discoveryArrived = resolve; }); + discoveryFailureGate = new Promise(resolve => { release = resolve; }); + const pending = f.service.sweepGitHubConnectionContinuity(f.company.id); + const finished = Promise.allSettled([pending]); + try { + await Promise.race([arrived, pending.then(() => { throw new Error("Expected held provider request"); })]); + const revision = randomUUID(); + const expiry = new Date(Date.now() + 10 * 60_000).toISOString(); + await db.transaction(async tx => { + await notifyGitHubCredentialChange(tx, f.connection); + await tx.update(connectionGrants).set({ providerTenant: change === "credential" + ? sql`jsonb_set(${connectionGrants.providerTenant}, '{oauth,accessTokenExpiresAt}', ${JSON.stringify(expiry)}::jsonb)` + : sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify({ id: revision, removeInstallationIds: ["202"] })}::jsonb)`, + updatedAt: new Date() }).where(eq(connectionGrants.id, f.grant.id)); + }); + release(); + expect(await pending).toMatchObject({ due: 1, failed: 1 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.continuityRetry).toBeUndefined(); + if (change === "credential") expect(saved!.providerTenant!.oauth!.accessTokenExpiresAt).toBe(expiry); + else expect(saved!.providerTenant!.github!.bindingSync!.id).toBe(revision); + expect(githubGrantContinuityDeadline(f.connection, saved!)).toBeLessThanOrEqual(Date.now()); + discoveryFailureGate = undefined; + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, refreshed: 1, failed: 0 }); + expect(bindings).toContainEqual(expect.objectContaining({ installationId: "101", active: true })); + if (change === "installation") expect(bindings).toContainEqual(expect.objectContaining({ installationId: "202", active: false })); + const [settled] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(settled!.providerTenant!.github!.bindingSync).toBeUndefined(); + const next = await restarted.nextGitHubConnectionContinuityAt(f.company.id); + expect(next).toBeLessThan(Date.now() + 24 * 60 * 60_000); + } finally { release(); await finished; discoveryFailureGate = undefined; discoveryArrived = undefined; } + }); + + it("retains provider backoff for the actual attempted revision after its own credential rotation", async () => { + const f = await fixture(["101"]); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + const owner = createGitHubGrantContinuity(db, { ...f.dependencies, refreshOAuthGrantCredentials: async () => { + const [rotated] = await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{oauth}', ${JSON.stringify({ + accessTokenExpiresAt: new Date(Date.now() + 10 * 60_000).toISOString(), refreshedAt: new Date().toISOString(), + })}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)).returning(); + return rotated!; + } }); + discoveryFailureGate = Promise.resolve(); + try { + expect(await owner.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, failed: 1 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.continuityRetry!.revision).toBe(githubContinuityRevision(saved!)); + expect(await owner.nextGitHubConnectionContinuityAt(f.company.id)).toBeGreaterThan(Date.now() + 23 * 60 * 60_000); + const calls = githubRequests; + expect(await createGitHubGrantContinuity(db, f.dependencies).sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 0 }); + expect(githubRequests).toBe(calls); + } finally { discoveryFailureGate = undefined; } + }); + + it.each([["suspend", "unsuspend"], ["unsuspend", "suspend"]])("reconciles same-batch installation order %s then %s and duplicate replay", async (first, last) => { + const f = await fixture(["101"]); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } } }).where(eq(connectionGrants.id, f.grant.id)); + const events = [first, last].map(action => ({ id: randomUUID(), provider: "github" as const, event: "installation", action, + installationId: "101", repositoryId: null, createdAt: new Date(Date.now() + 1000).toISOString(), bindingIds: [`${f.grant.id}_101`], payload: { installationId: "101" } })); + const acknowledge = vi.fn(async () => events.length); + const ingress = githubConnectionEventService(db, { connector: { ...f.connector, leaseEvents: async () => ({ leaseId: "installation-batch", events }), acknowledgeEvents: acknowledge } }); + expect(await ingress.pollOnce({ forDelivery: true })).toMatchObject({ processed: 2, failed: 0 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + const active = last === "unsuspend"; + expect(saved!.providerTenant!.github!.installationIds).toEqual(active ? ["101"] : []); + expect(saved!.providerTenant!.github!.bindingSync!.removeInstallationIds.sort()).toEqual(active ? ["202"] : ["101", "202"]); + expect(githubGrantContinuityDeadline(f.connection, saved!)).toBe(0); + await f.service.syncGitHubGrantBindings(f.connection, saved!); + expect(bindings.filter(binding => binding.installationId === "101")).toEqual([expect.objectContaining({ active })]); + expect(bindings.filter(binding => binding.installationId === "202")).toEqual([expect.objectContaining({ active: false })]); + const [settled] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(settled!.providerTenant!.github!.bindingSync).toBeUndefined(); + // Batch duplicates must retain terminal receipts and emit no registration changes. + expect(await ingress.pollOnce({ forDelivery: true })).toMatchObject({ processed: 0, duplicate: active ? 2 : 0, ignored: active ? 0 : 2, failed: 0 }); + await f.service.syncGitHubGrantBindings(f.connection, saved!); + expect(bindings).toHaveLength(2); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)))[0]!.providerTenant).toEqual(settled!.providerTenant); + expect((await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.companyId, f.company.id))).map(receipt => receipt.status)).toEqual(["processed", "processed"]); + expect(acknowledge).toHaveBeenCalledTimes(2); + }); + + it("does not retire active installations from a legacy conflicting binding intent", async () => { + const f = await fixture(["101"]); + const [saved] = await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + github: { ...f.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101", "202"] } } } }).where(eq(connectionGrants.id, f.grant.id)).returning(); + await f.service.syncGitHubGrantBindings(f.connection, saved!); + expect(bindings.filter(binding => binding.installationId === "101")).toEqual([expect.objectContaining({ active: true })]); + expect(bindings.filter(binding => binding.installationId === "202")).toEqual([expect.objectContaining({ active: false })]); + }); + it("preserves webhook removal debt across an actual signed, held Cloud token rotation and lease cleanup", async () => { const f = await fixture(["101"]); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); @@ -327,6 +434,11 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( expect(leases).toBeLessThanOrEqual(3); expect(load.mock.calls.length).toBeLessThanOrEqual(2); if (loadTimes.length === 2) expect(loadTimes[1]! - loadTimes[0]!).toBeGreaterThanOrEqual(5_000); if (leaseTimes.length === 3) expect(leaseTimes[2]! - leaseTimes[1]!).toBeGreaterThanOrEqual(5_000); + // A hint arriving during the first SQL pass retains one spaced generation. + // Start the empty-window assertion after that concrete work has settled. + await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); + expect(leases).toBeLessThanOrEqual(3); expect(load.mock.calls.length).toBeLessThanOrEqual(2); + if (loadTimes.length === 2) expect(loadTimes[1]! - loadTimes[0]!).toBeGreaterThanOrEqual(5_000); const quiet = { leases, loads: load.mock.calls.length }; await vi.advanceTimersByTimeAsync(60_000); expect({ leases, loads: load.mock.calls.length }).toEqual(quiet); diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index d012a239efd..7c0f010949d 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -588,6 +588,88 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(wakeup).toHaveBeenCalledOnce(); }); + it.each([false, true])("rechecks current policy after a committed edit during held search (changes=%s)", async changed => { + const created = await card(); + if (changed) await issueService(db).create(companyId, { title: "Policy race blocker", status: "blocked", priority: "medium" }); + const entered = Promise.withResolvers(), release = Promise.withResolvers(); + const factory = companySearch.companySearchService; + let hold = true; + const searches = vi.fn(); + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const service = factory(owner), actual = service.search; + service.search = async (...args) => { + searches(); + const result = await actual(...args); + if (hold) { hold = false; entered.resolve(); await release.promise; } + return result; + }; + return service; + }); + const service = statusCardEvaluationService(db, { wakeup }); + const pass = service.run(new AbortController().signal); + await entered.promise; + const cards = statusCardService(db); + let scheduled!: Date; + try { + scheduled = (await cards.update((await cards.getById(created.id))!, { refreshPolicy: policy("interval") }, actor)).nextEvalAt!; + } finally { release.resolve(); } + await pass; + const current = (await cards.getById(created.id))!; + expect(current.refreshPolicy.mode).toBe("interval"); + expect(current.nextEvalAt!.getTime()).toBe(scheduled.getTime()); + expect(current.nextEvalReason).toBe("refresh"); + expect(searches).toHaveBeenCalledOnce(); + expect(wakeup).not.toHaveBeenCalled(); + const intervalCard = (await cards.getById(created.id))!; + await issueService(db).create(companyId, { title: "Next interval blocker", status: "blocked", priority: "medium" }); + vi.setSystemTime(intervalCard.nextEvalAt!); + await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("keeps future failure retries fenced from ordinary company hints and permits explicit policy recovery", async () => { + const created = await card(); + const factory = companySearch.companySearchService; + let failing = true; + const searches = vi.fn(); + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const service = factory(owner), actual = service.search; + service.search = async (...args) => { searches(); if (failing) throw new Error("Controlled search unavailable"); return actual(...args); }; + return service; + }); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const failed = (await statusCardService(db).getById(created.id))!; + expect(failed.nextEvalReason).toBe("retry"); + const retryAt = failed.nextEvalAt!.getTime(); + expect(searches).toHaveBeenCalledOnce(); + for (let index = 0; index < 5; index++) { + await issueService(db).create(companyId, { title: `Retry blocker ${index}`, status: "blocked", priority: "medium" }); + service.changed(new Set([`changes:${companyId}`])); + await service.run(new AbortController().signal); + } + expect(searches).toHaveBeenCalledOnce(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt!.getTime()).toBe(retryAt); + const restarted = statusCardEvaluationService(db, { wakeup }); + restarted.changed(new Set([`changes:${companyId}`])); + await restarted.run(new AbortController().signal); + expect(searches).toHaveBeenCalledOnce(); + expect(await restarted.next()).toBe(retryAt); + vi.setSystemTime(retryAt); + await restarted.run(new AbortController().signal); + expect(searches).toHaveBeenCalledTimes(2); + const cards = statusCardService(db); + failing = false; + await cards.update((await cards.getById(created.id))!, { refreshPolicy: { ...policy(), debounceSeconds: 1 } }, actor); + await restarted.run(new AbortController().signal); + expect(searches).toHaveBeenCalledTimes(3); + const pending = (await cards.getById(created.id))!; + expect(pending.nextEvalReason).toBe("policy"); + vi.setSystemTime(pending.nextEvalAt!); + await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => { const created = await card(); await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index a32789305ff..3d5c2c00352 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -304,6 +304,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { + // Admission rechecks the same hard stop. Lock current eligibility before + // publishing intent so a no-op pause cannot reactivate its own receipts. + const [project] = await db.select({ pausedAt: projects.pausedAt }).from(projects) + .where(and(eq(projects.id, policy.scopeId), eq(projects.companyId, policy.companyId))).for("update"); + if (!project || project.pausedAt !== null) return; await notifyRunDispatch(db, { companyId: policy.companyId }); await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await db @@ -313,7 +318,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica pausedAt: now, updatedAt: now, }) - .where(and(eq(projects.id, policy.scopeId), isNull(projects.pausedAt))); + .where(and(eq(projects.id, policy.scopeId), eq(projects.companyId, policy.companyId), isNull(projects.pausedAt))); return; } diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 2e08d887dee..a70213dc44b 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -330,9 +330,9 @@ export function githubConnectionEventService( // Lifecycle webhooks carry IDs, not the user token’s complete repository view. // Discard the snapshot until Refresh access verifies it again. accessRevision: randomUUID(), - ...(unavailable ? { bindingSync: { + ...(unavailable || github.bindingSync || !github.installationIds.includes(binding.installationId) ? { bindingSync: { id: randomUUID(), removeInstallationIds: [...new Set([ - ...(github.bindingSync?.removeInstallationIds ?? []), binding.installationId, + ...(github.bindingSync?.removeInstallationIds ?? []), ...(unavailable ? [binding.installationId] : []), ])].filter(id => !installationIds.includes(id)), } } : {}), repositories: undefined, diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index d4660227c28..862d9d64ec4 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -643,14 +643,17 @@ export function statusCardService( ? membershipAt : interval; } - async function requestRefresh(cardId: string, input: { + async function refreshCurrentCard(cardId: string, input: { full?: boolean; trigger?: "manual" | "interval" | "reactive" | "restore"; actor?: StatusCardActor; now?: Date; - } = {}) { - const card = await getById(cardId); - if (!card) throw notFound("Status card not found"); + } = {}, superseded = false) { + const [source] = await db.select({ card: statusCards, revision: sql`${statusCards}.xmin::text` }) + .from(statusCards).where(eq(statusCards.id, cardId)); + if (!source) throw notFound("Status card not found"); + const { card, revision } = source; + const currentRevision = sql`${statusCards}.xmin::text = ${revision}`; if (card.archivedAt) throw unprocessable("Archived status cards cannot be refreshed"); if (card.queries.length === 0) throw conflict("Compile the status-card query before refreshing it"); if (card.generatingIssueId) { @@ -663,6 +666,10 @@ export function statusCardService( } const now = input.now ?? new Date(); + if (superseded && input.trigger !== undefined && input.trigger !== "manual" && input.trigger !== "restore" + && (card.refreshPolicy.mode === "manual" || (card.refreshPolicy.mode === "interval" && card.nextEvalAt && card.nextEvalAt > now))) { + return { card, generatingIssue: null, alreadyGenerating: false, enqueued: false }; + } let membershipAt: Date | null = null; const snapshot = await executeQueries(card, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const fingerprint = buildStatusCardFingerprint(snapshot); @@ -679,8 +686,9 @@ export function statusCardService( state: "active", nextEvalAt, nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, - }).where(eq(statusCards.id, card.id)).returning()); - return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; + }).where(and(eq(statusCards.id, card.id), currentRevision)).returning()); + if (!next) return undefined; + return { card: next, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } const hourAgo = new Date(now.getTime() - 60 * 60 * 1000); @@ -712,8 +720,9 @@ export function statusCardService( completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt), tokensToday: Number(daily[0]?.tokens ?? 0), }), - }).where(eq(statusCards.id, card.id)).returning()); - return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; + }).where(and(eq(statusCards.id, card.id), currentRevision)).returning()); + if (!next) return undefined; + return { card: next, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } const history = await listUpdates(card.id); @@ -765,13 +774,14 @@ export function statusCardService( nextEvalAt, nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, failureReason: null, - }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); + }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), currentRevision, or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); if (!next) { const winner = await getById(card.id); if (!winner?.generatingIssueId) { if (!TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } + if (winner && !winner.archivedAt) return undefined; throw conflict("Status-card refresh claim was lost"); } if (generationIssue!.id !== winner.generatingIssueId && !TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { @@ -794,6 +804,14 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } + async function requestRefresh(cardId: string, input: Parameters[1] = {}) { + for (let attempt = 0; attempt < 3; attempt += 1) { + const result = await refreshCurrentCard(cardId, input, attempt > 0); + if (result) return result; + } + throw conflict("Status-card refresh was superseded by concurrent edits"); + } + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "changed", reconstruct = false) { const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; @@ -811,7 +829,9 @@ export function statusCardService( .innerJoin(companies, eq(companies.id, statusCards.companyId)) .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), or(and(isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now)), - and(reactiveScope, ne(statusCards.state, "error"), ne(statusCards.state, "compiling"), sql`${statusCards.refreshPolicy}->>'mode' = 'reactive'`, sql`jsonb_array_length(${statusCards.queries}) > 0`)))) + and(reactiveScope, + or(isNull(statusCards.nextEvalReason), ne(statusCards.nextEvalReason, "retry"), isNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now)), + ne(statusCards.state, "error"), ne(statusCards.state, "compiling"), sql`${statusCards.refreshPolicy}->>'mode' = 'reactive'`, sql`jsonb_array_length(${statusCards.queries}) > 0`)))) .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; diff --git a/server/src/services/tool-access/github-grant-continuity.ts b/server/src/services/tool-access/github-grant-continuity.ts index e3095597d4b..c0f56c96f3c 100644 --- a/server/src/services/tool-access/github-grant-continuity.ts +++ b/server/src/services/tool-access/github-grant-continuity.ts @@ -18,6 +18,8 @@ function asRecord(value: unknown): Record { return {}; } +type RecordGitHubAttempt = (grant: typeof connectionGrants.$inferSelect) => void; + type GitHubContinuityDependencies = { now: () => Date; currentCloudConnector: () => PaperclipCloudConnector | null; @@ -30,6 +32,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector connection: typeof toolConnections.$inferSelect, initialGrant: typeof connectionGrants.$inferSelect, actor?: ActorInfo, + recordAttempt?: RecordGitHubAttempt, ) { await actor?.managementContext?.reauthorize(); return measureRunPhase("connectors.github.refresh", async () => { @@ -38,6 +41,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector connection, initialGrant, actor, + recordAttempt, ); } catch (error) { if ( @@ -56,7 +60,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector ), )); if (!current || current.status !== "active") throw error; - return refreshManagedGitHubGrantAccessOnce(connection, current, actor); + return refreshManagedGitHubGrantAccessOnce(connection, current, actor, recordAttempt); } }); } @@ -65,6 +69,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector connection: typeof toolConnections.$inferSelect, initialGrant: typeof connectionGrants.$inferSelect, actor?: ActorInfo, + recordAttempt?: RecordGitHubAttempt, ) { await actor?.managementContext?.reauthorize(); let grant = await measureRunPhase("connectors.github.credentials", () => refreshOAuthGrantCredentials({ @@ -73,6 +78,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector grantId: initialGrant.id, actor, })); + recordAttempt?.(grant); const resolveAccessToken = async () => measureRunPhase("connectors.github.credentials", async () => { const accessRef = grant.credentialSecretRefs.find( (ref) => ref.configPath === "oauth.access_token", @@ -113,6 +119,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector forceRefresh: true, actor, })); + recordAttempt?.(grant); accessToken = await resolveAccessToken(); try { metadata = await measureRunPhase("connectors.github.repositories", () => loadGitHubGrantMetadata( @@ -204,7 +211,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector return { updated, previousGitHub }; }); - return measureRunPhase("connectors.github.webhooks", () => syncGitHubGrantBindings(connection, updated, actor, accessToken)); + return measureRunPhase("connectors.github.webhooks", () => syncGitHubGrantBindings(connection, updated, actor, accessToken, recordAttempt)); } async function syncGitHubGrantBindings( @@ -212,6 +219,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector initialGrant: typeof connectionGrants.$inferSelect, actor?: ActorInfo, accessToken?: string, + recordAttempt?: RecordGitHubAttempt, ) { await actor?.managementContext?.reauthorize(); const cloudConnector = currentCloudConnector(); @@ -243,9 +251,11 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector const subject = current.kind === "agent" && current.subjectAgentId ? `agent:${current.subjectAgentId}` : current.kind === "user" ? current.subjectUserId : null; if (!subject) throw unprocessable("GitHub webhook subject is unavailable"); + recordAttempt?.(grant); const changes = [ ...github.installationIds.map(installationId => ({ installationId, active: true as const })), - ...github.bindingSync.removeInstallationIds.map(installationId => ({ installationId, active: false as const })), + ...github.bindingSync.removeInstallationIds.filter(id => !github.installationIds.includes(id)) + .map(installationId => ({ installationId, active: false as const })), ]; for (let offset = 0; offset < changes.length; offset += 4) { const outcomes = await Promise.allSettled(changes.slice(offset, offset + 4).map(change => cloudConnector.setWebhookBinding({ @@ -265,13 +275,14 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector }); } - async function retainRetry(connection: typeof toolConnections.$inferSelect, grantId: string, error?: unknown, minimumMs = 30_000) { + async function retainRetry(connection: typeof toolConnections.$inferSelect, grantId: string, attemptedRevision: string, error?: unknown, minimumMs = 30_000) { await db.transaction(async tx => { const [current] = await tx.select().from(connectionGrants).where(and( eq(connectionGrants.id, grantId), eq(connectionGrants.companyId, connection.companyId), eq(connectionGrants.connectionId, connection.id), eq(connectionGrants.status, "active"), )).for("update").limit(1); - if (!current?.providerTenant?.github) return; + // Backoff belongs to the attempted credential/installation revision. + if (!current?.providerTenant?.github || githubContinuityRevision(current) !== attemptedRevision) return; const attempts = Math.min(16, (current.providerTenant.github.continuityRetry?.attempts ?? 0) + 1); const advised = error instanceof PaperclipCloudConnectorError ? error.retryAt : error instanceof HttpError ? asRecord(error.details).retryAt : undefined; @@ -280,7 +291,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector await notifyGitHubCredentialChange(tx, connection); await tx.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,continuityRetry}', ${JSON.stringify({ - at, attempts, revision: githubContinuityRevision(current), + at, attempts, revision: attemptedRevision, })}::jsonb)`, updatedAt: now(), }).where(and(eq(connectionGrants.id, current.id), eq(connectionGrants.companyId, connection.companyId))); @@ -324,18 +335,20 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector let refreshed = 0; let failed = 0; for (const row of due) { + let attemptedRevision = githubContinuityRevision(row.grant); + const recordAttempt: RecordGitHubAttempt = grant => { attemptedRevision = githubContinuityRevision(grant); }; try { const actor: ActorInfo = { actorType: "system", actorId: "github-connection-continuity" }; const updated = row.grant.providerTenant?.github?.bindingSync - ? await syncGitHubGrantBindings(row.connection, row.grant, actor) - : await refreshManagedGitHubGrantAccess(row.connection, row.grant, actor); + ? await syncGitHubGrantBindings(row.connection, row.grant, actor, undefined, recordAttempt) + : await refreshManagedGitHubGrantAccess(row.connection, row.grant, actor, recordAttempt); // A successful provider response can still leave an unchanged/short-lived // credential due. Retain that obligation with bounded durable backoff. const next = githubGrantContinuityDeadline(row.connection, updated); - if (next !== null && next <= now().getTime()) await retainRetry(row.connection, updated.id, undefined, 60 * 60_000); + if (next !== null && next <= now().getTime()) await retainRetry(row.connection, updated.id, githubContinuityRevision(updated), undefined, 60 * 60_000); refreshed += 1; } catch (error) { - await retainRetry(row.connection, row.grant.id, error); + await retainRetry(row.connection, row.grant.id, attemptedRevision, error); failed += 1; } }