diff --git a/packages/db/src/migrations/0444_github_access_refresh_runtime_bookkeeping.sql b/packages/db/src/migrations/0444_github_access_refresh_runtime_bookkeeping.sql new file mode 100644 index 00000000000..97bdd71c35f --- /dev/null +++ b/packages/db/src/migrations/0444_github_access_refresh_runtime_bookkeeping.sql @@ -0,0 +1,114 @@ +-- A GitHub access refresh (scheduled health checks and continuity renewal) +-- rewrites the grant's access fence, refresh time, and webhook binding debt even +-- when the installation and repository view is unchanged. Those fields are +-- provider bookkeeping, like webhook health and refresh leases, so they must +-- not advance every agent's runtime revision. Repository, installation, and +-- credential changes still do. +CREATE OR REPLACE FUNCTION paperclip_connector_runtime_changed() RETURNS trigger +LANGUAGE plpgsql SET search_path = public, pg_temp AS $$ +DECLARE company uuid; query text; target record; binding record; candidate uuid; + old_projection text := '(to_jsonb(o) - $1)'; new_projection text := '(to_jsonb(n) - $1)'; + ignored text[] := ARRAY['updated_at', 'created_at', 'last_used_at', 'last_resolved_at', 'last_health_at', + 'health_status', 'health_checked_at', 'health_message', 'last_catalog_refresh_at', 'last_seen_at', 'last_error', 'last_event_at', 'last_publication_at', + 'last_sync_at', 'sync_checkpoint', 'activation_at', 'lease_token', 'lease_expires_at']; +BEGIN + IF TG_TABLE_NAME = 'tool_profiles' AND TG_OP = 'INSERT' THEN RETURN NULL; END IF; + IF TG_TABLE_NAME = 'connection_grant_delegations' THEN + IF TG_OP = 'INSERT' THEN query := 'SELECT DISTINCT company_id, agent_id FROM new_rows ORDER BY company_id, agent_id'; + ELSIF TG_OP = 'DELETE' THEN query := 'SELECT DISTINCT company_id, agent_id FROM old_rows ORDER BY company_id, agent_id'; + ELSE query := 'WITH changed AS ( + SELECT to_jsonb(o) AS previous, to_jsonb(n) AS current FROM old_rows o FULL JOIN new_rows n ON o.id = n.id + WHERE (to_jsonb(o) - $1) IS DISTINCT FROM (to_jsonb(n) - $1)), targets AS ( + SELECT previous AS target FROM changed UNION SELECT current FROM changed) + SELECT DISTINCT (target->>''company_id'')::uuid AS company_id, (target->>''agent_id'')::uuid AS agent_id + FROM targets WHERE target IS NOT NULL ORDER BY company_id, agent_id'; + END IF; + FOR binding IN EXECUTE query USING ignored LOOP + PERFORM paperclip_invalidate_connector_agent(binding.company_id, binding.agent_id); + END LOOP; + RETURN NULL; + END IF; + IF TG_TABLE_NAME = 'company_secret_bindings' THEN + IF TG_OP = 'INSERT' THEN query := 'SELECT DISTINCT company_id, target_type, target_id FROM new_rows ORDER BY company_id, target_id'; + ELSIF TG_OP = 'DELETE' THEN query := 'SELECT DISTINCT company_id, target_type, target_id FROM old_rows ORDER BY company_id, target_id'; + ELSE query := 'WITH changed AS ( + SELECT to_jsonb(o) AS previous, to_jsonb(n) AS current FROM old_rows o FULL JOIN new_rows n ON o.id = n.id + WHERE (to_jsonb(o) - $1) IS DISTINCT FROM (to_jsonb(n) - $1)), targets AS ( + SELECT previous AS target FROM changed UNION SELECT current FROM changed) + SELECT DISTINCT (target->>''company_id'')::uuid AS company_id, target->>''target_type'' AS target_type, target->>''target_id'' AS target_id + FROM targets WHERE target IS NOT NULL ORDER BY company_id, target_id'; + END IF; + FOR binding IN EXECUTE query USING ignored LOOP + IF binding.target_type = 'agent' THEN + SELECT id INTO candidate FROM agents WHERE company_id = binding.company_id AND id::text = binding.target_id; + IF FOUND THEN PERFORM paperclip_record_agent_runtime(binding.company_id, candidate); END IF; + ELSE PERFORM paperclip_invalidate_connector_company(binding.company_id); + END IF; + END LOOP; + RETURN NULL; + END IF; + IF TG_TABLE_NAME = 'connection_grants' THEN + old_projection := old_projection || ' #- ''{provider_tenant,oauth,refreshLease}'' #- ''{external_credential,lastVerifiedAt}'' #- ''{provider_tenant,github,lastWebhookAt}'' #- ''{provider_tenant,github,webhookHealth}'' + #- ''{provider_tenant,github,accessRevision}'' #- ''{provider_tenant,github,lastAccessRefreshAt}'' #- ''{provider_tenant,github,bindingSync}'' #- ''{provider_tenant,github,continuityRetry}'''; + new_projection := new_projection || ' #- ''{provider_tenant,oauth,refreshLease}'' #- ''{external_credential,lastVerifiedAt}'' #- ''{provider_tenant,github,lastWebhookAt}'' #- ''{provider_tenant,github,webhookHealth}'' + #- ''{provider_tenant,github,accessRevision}'' #- ''{provider_tenant,github,lastAccessRefreshAt}'' #- ''{provider_tenant,github,bindingSync}'' #- ''{provider_tenant,github,continuityRetry}'''; + END IF; + IF TG_TABLE_NAME = 'tool_policies' THEN + old_projection := old_projection || ' #- ''{config,trustRule,hitCount}'' #- ''{config,trustRule,lastHitAt}'''; + new_projection := new_projection || ' #- ''{config,trustRule,hitCount}'' #- ''{config,trustRule,lastHitAt}'''; + END IF; + IF TG_TABLE_NAME IN ('tool_profile_bindings', 'tool_connection_installs', 'tool_profile_entries', 'tool_policies') THEN + IF TG_TABLE_NAME IN ('tool_profile_entries', 'tool_policies') THEN + IF TG_OP = 'INSERT' THEN query := 'SELECT to_jsonb(n) AS row_data FROM new_rows n'; + ELSIF TG_OP = 'DELETE' THEN query := 'SELECT to_jsonb(o) AS row_data FROM old_rows o'; + ELSE query := format('WITH changed AS ( + SELECT to_jsonb(o) AS previous, to_jsonb(n) AS current FROM old_rows o FULL JOIN new_rows n ON o.%I = n.%I + WHERE %s IS DISTINCT FROM %s) + SELECT previous AS row_data FROM changed WHERE previous IS NOT NULL UNION + SELECT current AS row_data FROM changed WHERE current IS NOT NULL', TG_ARGV[0], TG_ARGV[0], old_projection, new_projection); + END IF; + IF TG_TABLE_NAME = 'tool_profile_entries' THEN + query := 'SELECT b.company_id, b.target_type, b.target_id FROM (' || query || ') changed + JOIN tool_profile_bindings b ON b.company_id = (row_data->>''company_id'')::uuid + AND b.profile_id = (row_data->>''profile_id'')::uuid'; + ELSE + query := 'SELECT (row_data->>''company_id'')::uuid AS company_id, t.target_type, t.target_id + FROM (' || query || ') changed CROSS JOIN LATERAL + paperclip_connector_policy_targets((row_data->>''company_id'')::uuid, row_data->''selectors'') t'; + END IF; + ELSIF TG_OP = 'INSERT' THEN query := 'SELECT company_id, target_type, target_id FROM new_rows'; + ELSIF TG_OP = 'DELETE' THEN query := 'SELECT company_id, target_type, target_id FROM old_rows'; + ELSE query := format('WITH changed AS ( + SELECT o.company_id AS old_company, o.target_type AS old_type, o.target_id AS old_target, + n.company_id AS new_company, n.target_type AS new_type, n.target_id AS new_target + FROM old_rows o FULL JOIN new_rows n ON o.%I = n.%I WHERE %s IS DISTINCT FROM %s) + SELECT old_company AS company_id, old_type AS target_type, old_target AS target_id FROM changed WHERE old_company IS NOT NULL UNION + SELECT new_company AS company_id, new_type AS target_type, new_target AS target_id FROM changed WHERE new_company IS NOT NULL', + TG_ARGV[0], TG_ARGV[0], old_projection, new_projection); + END IF; + FOR target IN EXECUTE 'SELECT DISTINCT company_id, target_type, target_id FROM (' || query || ') targets + WHERE target_type IN (''agent'', ''company'') + ORDER BY company_id, target_type DESC, target_id' USING ignored LOOP + IF target.target_type = 'company' AND target.target_id = target.company_id::text THEN + PERFORM paperclip_invalidate_connector_company(target.company_id); + ELSIF target.target_type = 'agent' AND target.target_id ~ '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' THEN + SELECT id INTO candidate FROM agents WHERE company_id = target.company_id AND id = target.target_id::uuid; + IF FOUND THEN PERFORM paperclip_invalidate_connector_agent(target.company_id, candidate); END IF; + END IF; + END LOOP; + RETURN NULL; + END IF; + IF TG_OP = 'INSERT' THEN query := 'SELECT DISTINCT company_id FROM new_rows ORDER BY company_id'; + ELSIF TG_OP = 'DELETE' THEN query := 'SELECT DISTINCT company_id FROM old_rows ORDER BY company_id'; + ELSE query := format('WITH changed AS ( + SELECT o.company_id AS old_company, n.company_id AS new_company FROM old_rows o FULL JOIN new_rows n ON o.%I = n.%I + WHERE %s IS DISTINCT FROM %s) + SELECT old_company AS company_id FROM changed WHERE old_company IS NOT NULL UNION + SELECT new_company AS company_id FROM changed WHERE new_company IS NOT NULL ORDER BY company_id', TG_ARGV[0], TG_ARGV[0], old_projection, new_projection); + END IF; + FOR company IN EXECUTE query USING ignored LOOP + PERFORM paperclip_invalidate_connector_company(company); + END LOOP; + RETURN NULL; +END; +$$; diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 5c2c5b66634..d0917c105c4 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3095,6 +3095,13 @@ "when": 1794076124161, "tag": "0443_native_warm_follow_up_default", "breakpoints": true + }, + { + "idx": 444, + "version": "7", + "when": 1794080714388, + "tag": "0444_github_access_refresh_runtime_bookkeeping", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/src/__tests__/agent-runtime-connector-access.test.ts b/server/src/__tests__/agent-runtime-connector-access.test.ts index 6adefe53a1c..885d142a5ee 100644 --- a/server/src/__tests__/agent-runtime-connector-access.test.ts +++ b/server/src/__tests__/agent-runtime-connector-access.test.ts @@ -654,6 +654,22 @@ describeDatabase("connector access runtime synchronization", () => { await db.update(connectionGrants).set({ providerTenant: { github: { ...github, accessRevision: "changed", repositoryCount: 0, repositorySelection: "none", installationIds: [] } } }).where(eq(connectionGrants.id, grant.id)); expect(await events()).toHaveLength(before.length + 2); }); + it("ignores GitHub access refresh bookkeeping while retaining repository access changes", async () => { + const github = { userId: "fixture-user", login: "fixture", installationCount: 1, repositoryCount: 1, repositorySelection: "selected" as const, installationIds: ["1"], installationOwnerLogins: ["fixture"], + repositories: [{ id: "10", fullName: "fixture/repo", installationId: "1" }], accessRevision: "initial", lastAccessRefreshAt: "2000-01-01T00:00:00.000Z", webhookHealth: "healthy" as const }; + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId, kind: "user", subjectUserId: "alice", providerTenant: { github } }).returning(); + const before = await events(); + // A scheduled health check rereads unchanged access: it stamps a fresh fence + // and refresh time, records binding debt, then settles that debt. + const refreshed = { ...github, accessRevision: "refreshed", lastAccessRefreshAt: "2000-01-02T00:00:00.000Z" }; + await db.update(connectionGrants).set({ providerTenant: { github: { ...refreshed, bindingSync: { id: "fixture-sync", removeInstallationIds: [] } } } }).where(eq(connectionGrants.id, grant.id)); + await db.update(connectionGrants).set({ providerTenant: { github: { ...refreshed, continuityRetry: { at: "2000-01-02T00:00:30.000Z", attempts: 1, revision: "fixture" } } } }).where(eq(connectionGrants.id, grant.id)); + await db.update(connectionGrants).set({ providerTenant: { github: refreshed } }).where(eq(connectionGrants.id, grant.id)); + expect(await events()).toEqual(before); + await db.update(connectionGrants).set({ providerTenant: { github: { ...refreshed, accessRevision: "changed", repositoryCount: 2, + repositories: [...github.repositories, { id: "11", fullName: "fixture/other", installationId: "1" }] } } }).where(eq(connectionGrants.id, grant.id)); + expect(await events()).toHaveLength(before.length + 2); + }); it("ignores OAuth grant lease acquisition and release while retaining access changes", async () => { const oauth = { scopes: ["read"], tokenEndpoint: "https://oauth.example.test/token" }; const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId, kind: "organization", providerTenant: { oauth } }).returning(); diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index 885c64f564f..ac4e2486e7f 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -6,8 +6,8 @@ import { randomUUID, generateKeyPairSync, verify, createHash, diffieHellman, hkd import { createServer, type Server } from "node:http"; import { once } from "node:events"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { eq, sql } from "drizzle-orm"; -import { applyPendingMigrations, companies, activityLog, companySecrets, connectionEventDeliveries, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; +import { and, eq, sql } from "drizzle-orm"; +import { applyPendingMigrations, agents, companies, activityLog, companySecrets, connectionEventDeliveries, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, resourceLifecycleEvents, toolApplications, toolConnections } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { createGitHubGrantContinuity, githubGrantContinuityDeadline, githubContinuityRevision } from "../services/tool-access/github-grant-continuity.js"; import { createPaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; @@ -174,6 +174,32 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( } finally { await ew.stop(); await cw.stop(); await coordinator.stop(); } }); + it("rereads unchanged access on a scheduled refresh without advancing agent runtime revisions", async () => { + const f = await fixture(["101"]); + const agentId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId: f.company.id, name: "Runtime fixture", adapterType: "paperclip_runner" }); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + const actor = { actorType: "system", actorId: "tool_health_sweep" } as const; + const runtimeEvents = () => db.select().from(resourceLifecycleEvents).where(and( + eq(resourceLifecycleEvents.companyId, f.company.id), eq(resourceLifecycleEvents.resourceType, "agent_runtime"))); + const grant = async () => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)))[0]!; + try { + // The first refresh records the repository list the fixture lacked. + await f.service.refreshManagedGitHubGrantAccess(f.connection, f.grant, actor); + const first = await grant(); + const before = await runtimeEvents(); + expect(before.length).toBeGreaterThan(0); + await f.service.refreshManagedGitHubGrantAccess(f.connection, first, actor); + const second = await grant(); + expect(second.providerTenant!.github!.accessRevision).not.toBe(first.providerTenant!.github!.accessRevision); + expect(second.providerTenant!.github!.bindingSync).toBeUndefined(); + expect(bindings.filter(binding => binding.active === true)).toHaveLength(2); + expect(await runtimeEvents()).toEqual(before); + } finally { + await db.delete(agents).where(eq(agents.id, agentId)); + } + }); it("failed binding removal persists retry/debt through restart, uses current lease deadline and settles without refetching GitHub metadata", async () => { const f = await fixture(["101"]); fakeClock(); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!,