From 7aa8bb147b9814f8ddbe6d6ac46a9e65432f247d Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 8 Oct 2026 06:15:25 -0500 Subject: [PATCH] fix(connections): isolate Google service discovery failures Exercise managed and customer token refresh through gateway retries, reject invalid consent evidence, and retain healthy services during partial outages. Co-Authored-By: Paperclip --- doc/connections/GOOGLE-WORKSPACE.md | 6 + .../src/__tests__/tool-access-service.test.ts | 121 ++++++++++++++++++ .../paperclip-cloud-connector.test.ts | 6 + server/src/services/tool-access.ts | 46 +++++-- 4 files changed, 170 insertions(+), 9 deletions(-) diff --git a/doc/connections/GOOGLE-WORKSPACE.md b/doc/connections/GOOGLE-WORKSPACE.md index 0fe3bb93299..f4ae3dd9cbd 100644 --- a/doc/connections/GOOGLE-WORKSPACE.md +++ b/doc/connections/GOOGLE-WORKSPACE.md @@ -20,6 +20,12 @@ permissions; discovery skips services without grants and excludes unsupported actions. The gateway checks actual grant scopes again before each call, including after refresh. Requested scopes alone never authorize an action. +Discovery isolates service outages. Healthy services remain usable while the +connection's health message identifies unavailable services and recommends a +catalog refresh. Failed services' cached actions are disabled until rediscovered; +an outage affecting every requested service still fails the refresh. Loss of the +shared OAuth grant never degrades into a successful partial discovery. + Existing individual product connections and their direct setup routes remain supported. They are not automatically merged, upgraded or reauthorized. The Sheets robot-account method remains separate because it uses explicitly shared files, diff --git a/server/src/__tests__/tool-access-service.test.ts b/server/src/__tests__/tool-access-service.test.ts index 20dc810fabf..241f7bc915e 100644 --- a/server/src/__tests__/tool-access-service.test.ts +++ b/server/src/__tests__/tool-access-service.test.ts @@ -8101,6 +8101,127 @@ describeEmbeddedPostgres("tool access service", () => { expect(JSON.stringify(completed)).not.toContain("workspace-access-token"); }); + describe("combined Workspace service isolation and refresh consent", () => { + async function connectWorkspace(mode: "managed" | "customer") { + const company = await createCompany(db); + const userId = `workspace-refresh-${randomUUID()}`; + await grantBoardUser(db, company.id, userId, [], "owner"); + const actor = { actorType: "user" as const, actorId: userId }; + const initialScopes = ["documents.readonly", "gmail.readonly"].map((scope) => `https://www.googleapis.com/auth/${scope}`); + let refreshScopes: string[] | undefined = initialScopes; + const connector = fakeGoogleWorkspaceConnector(company.id, userId, "workspace.all"); + const originalClaim = connector.claim; + connector.claim = vi.fn(async (input) => ({ ...await originalClaim(input), scopes: initialScopes, + accessTokenExpiresAt: new Date(Date.now() + 86_400_000).toISOString() })); + connector.refresh = vi.fn(async () => ({ ...await connector.claim({ subject: userId, companyId: company.id, + profile: "workspace.all", claimId: "fixture", redemptionId: "fixture" }), scopes: refreshScopes ?? [] })); + const service = createTestToolAccessService(db, { paperclipCloudConnector: connector }); + const calls: string[] = []; + const unavailable = new Set(); + let unauthorizedNextCall = false; + const tokenRefreshes: string[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (url, init) => { + const href = String(url); + if (href === "https://oauth2.googleapis.com/token") { + const refreshing = new URLSearchParams(String(init?.body)).get("grant_type") === "refresh_token"; + if (refreshing) tokenRefreshes.push(href); + return Response.json({ access_token: "workspace-refresh-access", refresh_token: "workspace-refresh-token", + token_type: "Bearer", expires_in: 86_400, scope: (refreshing ? refreshScopes : initialScopes)?.join(" ") }); + } + const host = new URL(href).hostname; + if (unavailable.has(host)) return new Response("unavailable", { status: 503 }); + const body = JSON.parse(String(init?.body)); + if (body.method === "tools/call") { + calls.push(body.params.name); + if (unauthorizedNextCall) { unauthorizedNextCall = false; return new Response("expired", { status: 401 }); } + return Response.json({ jsonrpc: "2.0", id: body.id, result: { content: [{ type: "text", text: "ok" }] } }); + } + return mcpHttpResponse({ jsonrpc: "2.0", id: body.id, result: { tools: [{ + name: host === "docsmcp.googleapis.com" ? "read_doc" : "get_message", annotations: { readOnlyHint: true }, + inputSchema: { type: "object", properties: {} }, + }] } }); + }); + const connected = await service.connectGalleryApp(company.id, { + galleryKey: "google-workspace", grantKind: "organization", + connectionMethodKey: mode === "managed" ? "paperclip-workspace" : "customer-workspace-oauth", + ...(mode === "customer" ? { oauthClient: { clientId: "workspace-client", clientSecret: "workspace-client-secret" } } : {}), + }, actor); + const redirectUri = mode === "managed" ? "https://paperclip.example/api/tools/oauth/cloud-connector/callback" : "https://paperclip.example/api/tools/oauth/callback"; + const started = await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor }); + const state = new URL(started.authorizationUrl).searchParams.get("state")!; + const completed = mode === "managed" + ? await service.completePaperclipCloudConnectorCallback({ state, claimId: "fixture", actor }) + : await service.completeOAuthCallback({ state, code: "fixture", redirectUri, actor }); + const agent = await createAgent(db, company.id); + await service.finishGalleryAppConnection(company.id, connected.connectionId, { + enabledCatalogEntryIds: completed.catalog.map((entry) => entry.id), askFirstCatalogEntryIds: [], access: { agentIds: [agent.id] }, + }, actor); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + invocationSource: "on_demand", status: "running", contextSnapshot: {} }).returning(); + const gateway = createToolGatewayService(db, { toolActionSigningSecret: "test-secret", + oauthGrantRefresher: service.refreshOAuthGrantCredentials }); + const session = await gateway.createSession({ companyId: company.id, agentId: agent.id, runId: run.id }); + return { company, actor, service, connected, gateway, session, calls, unavailable, connector, tokenRefreshes, + expireNextCall: (scopes: string[] | undefined) => { refreshScopes = scopes; unauthorizedNextCall = true; } }; + } + + it("keeps healthy service tools visible during a partial outage and restores recovered services", async () => { + const fixture = await connectWorkspace("managed"); + fixture.unavailable.add("gmailmcp.googleapis.com"); + const refreshed = await fixture.service.refreshCatalog(fixture.connected.connectionId, fixture.actor); + expect(refreshed.connection).toMatchObject({ healthStatus: "ok", healthMessage: expect.stringContaining("gmail") }); + expect(refreshed.catalog.map((entry) => entry.toolName)).toEqual(["docs__read_doc"]); + const tools = await fixture.gateway.listToolsForSession(fixture.session.token); + const doc = tools.find((tool) => tool.upstreamToolName === "docs__read_doc")!; + expect(doc).toBeTruthy(); + expect(tools.some((tool) => tool.upstreamToolName === "gmail__get_message")).toBe(false); + await expect(fixture.gateway.executeTool({ sessionToken: fixture.session.token, tool: doc.name, parameters: {} })).resolves.toMatchObject({ status: "completed" }); + const health = await fixture.service.checkHealth(fixture.connected.connectionId, fixture.actor); + expect(health.connection).toMatchObject({ healthStatus: "ok", healthMessage: expect.stringContaining("gmail") }); + const events = await db.select().from(toolAccessAuditEvents).where(eq(toolAccessAuditEvents.connectionId, fixture.connected.connectionId)); + expect(events).toContainEqual(expect.objectContaining({ reasonCode: "google_workspace_service_unavailable", details: { service: "gmail" } })); + fixture.unavailable.clear(); + const recovered = await fixture.service.refreshCatalog(fixture.connected.connectionId, fixture.actor); + expect(recovered.catalog.map((entry) => entry.toolName).sort()).toEqual(["docs__read_doc", "gmail__get_message"]); + expect(recovered.connection.healthMessage).not.toContain("unavailable"); + fixture.unavailable.add("gmailmcp.googleapis.com"); + fixture.unavailable.add("docsmcp.googleapis.com"); + await expect(fixture.service.refreshCatalog(fixture.connected.connectionId, fixture.actor)).rejects.toMatchObject({ details: { code: "google_workspace_services_unavailable" } }); + }); + + it.each(["managed", "customer"] as const)("blocks the auth retry and updates listing after %s refresh narrows consent", async (mode) => { + const fixture = await connectWorkspace(mode); + const tool = (await fixture.gateway.listToolsForSession(fixture.session.token)).find((item) => item.upstreamToolName === "docs__read_doc")!; + expect(tool).toBeTruthy(); + const scopes = ["https://www.googleapis.com/auth/gmail.readonly"]; + fixture.expireNextCall(scopes); + await expect(fixture.gateway.executeTool({ sessionToken: fixture.session.token, tool: tool.name, parameters: {} })).rejects.toMatchObject({ reasonCode: "oauth_insufficient_scope" }); + expect(fixture.calls).toEqual(["read_doc"]); // No second tools/call after the 401 refresh removed Docs permission. + const [grant] = await db.select().from(connectionGrants).where(and(eq(connectionGrants.connectionId, fixture.connected.connectionId), eq(connectionGrants.status, "active"))); + expect(grant.providerTenant?.oauth?.scopes).toEqual(scopes); + if (mode === "managed") expect(fixture.connector.refresh).toHaveBeenCalled(); + else expect(fixture.tokenRefreshes).toHaveLength(1); + const tools = await fixture.gateway.listToolsForSession(fixture.session.token); + expect(tools.some((item) => item.upstreamToolName === "docs__read_doc")).toBe(false); + const gmail = tools.find((item) => item.upstreamToolName === "gmail__get_message")!; + expect(gmail).toBeTruthy(); + await expect(fixture.gateway.executeTool({ sessionToken: fixture.session.token, tool: gmail.name, parameters: {} })).resolves.toMatchObject({ status: "completed" }); + }); + + it.each((["managed", "customer"] as const).flatMap((mode) => [undefined, [], ["https://www.googleapis.com/auth/gmail.send"]].map((scopes) => ({ mode, scopes }))))( + "fails closed on invalid refreshed permissions ($mode, $scopes)", async ({ mode, scopes }) => { + const fixture = await connectWorkspace(mode); + const tool = (await fixture.gateway.listToolsForSession(fixture.session.token)).find((item) => item.upstreamToolName === "docs__read_doc")!; + fixture.expireNextCall(scopes); + await expect(fixture.gateway.executeTool({ sessionToken: fixture.session.token, tool: tool.name, parameters: {} })).rejects.toMatchObject({ reasonCode: "oauth_reauthorization_required" }); + expect(fixture.calls).toEqual(["read_doc"]); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.connectionId, fixture.connected.connectionId)); + expect(grant.status).toBe("needs_reauthorization"); + expect((await fixture.gateway.listToolsForSession(fixture.session.token)).some((item) => item.connectionId === fixture.connected.connectionId)).toBe(false); + }, + ); + }); + it("routes a managed Drive callback into the personal vault, filtered catalog, and provider-specific activity", async () => { const company = await createCompany(db); const userId = `drive-member-${randomUUID()}`; diff --git a/server/src/services/paperclip-cloud-connector.test.ts b/server/src/services/paperclip-cloud-connector.test.ts index 3e6f73bdc74..3ebfb1faaef 100644 --- a/server/src/services/paperclip-cloud-connector.test.ts +++ b/server/src/services/paperclip-cloud-connector.test.ts @@ -64,11 +64,17 @@ describe("Paperclip Cloud connector", () => { const connector = createPaperclipCloudConnector({ config: keys.config, request: vi.fn(async () => Response.json({ sealed })) as typeof fetch }); await expect(connector.claim({ subject, companyId, profile: "workspace.all", claimId: "test-claim", redemptionId: "test-state" })).resolves.toEqual(credentials); + sealed = seal(credentials, keys.sealPublicKey, "access", keys.config, "workspace.all"); + await expect(connector.refresh({ subject, companyId, profile: "workspace.all", refreshToken: "test-refresh" })).resolves.toEqual(credentials); for (const invalid of [[], [...scopes, "https://www.googleapis.com/auth/gmail.send"]]) { credentials.scopes = invalid; sealed = seal(credentials, keys.sealPublicKey, "initial", keys.config, "workspace.all"); await expect(connector.claim({ subject, companyId, profile: "workspace.all", claimId: "test-claim", redemptionId: "test-state" })).rejects.toMatchObject({ code: "REAUTHORIZATION_REQUIRED" }); + sealed = seal(credentials, keys.sealPublicKey, "access", keys.config, "workspace.all"); + await expect(connector.refresh({ subject, companyId, profile: "workspace.all", refreshToken: "test-refresh" })).rejects.toMatchObject({ code: "REAUTHORIZATION_REQUIRED" }); } + sealed = seal({ ...credentials, scopes: undefined }, keys.sealPublicKey, "access", keys.config, "workspace.all"); + await expect(connector.refresh({ subject, companyId, profile: "workspace.all", refreshToken: "test-refresh" })).rejects.toBeInstanceOf(PaperclipCloudConnectorError); }); async function rejection(response: Response) { const connector = createPaperclipCloudConnector({ diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 3ebdca23012..9a186544d29 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -2921,7 +2921,7 @@ function sanitizeHttpFailure(error: unknown): { if (code === "grant_credential_invalid" || code === "oauth_insufficient_scope") { return { status: code === "grant_credential_invalid" ? "missing_secret" : "degraded", message: error.message, code }; } - if (code === "slack_mcp_access_disabled") { + if (code === "slack_mcp_access_disabled" || code === "google_workspace_services_unavailable") { return { status: "error", message: error.message, code }; } if (code === "user_authorization_required") { @@ -7122,6 +7122,7 @@ export function toolAccessService( credentialHeaders?: Record, actor?: ActorInfo, workspaceEndpoint?: string, + unavailableServices: string[] = [], ): Promise { assertSupportedConnection(connection); let headers = credentialHeaders ?? { @@ -7134,16 +7135,31 @@ export function toolAccessService( const services = (Object.keys(GOOGLE_WORKSPACE_SERVICES) as GoogleWorkspaceService[]) .filter((service) => googleWorkspaceServiceGranted(service, scopes)); // One credential authority, no child connections or implied service grants. - const catalogs = await Promise.all(services.map(async (service) => { + const catalogs = await Promise.allSettled(services.map(async (service) => { const profile = GOOGLE_WORKSPACE_CONNECTOR_PROFILES[GOOGLE_WORKSPACE_SERVICES[service]]; const tools = await remoteTools(connection, headers, actor, profile.serverUrl); return tools.map((tool) => ({ ...tool, name: googleWorkspaceToolName(service, tool.name) })) .filter((tool) => isGoogleWorkspaceToolGranted(tool.name, scopes)); })); + for (const [index, catalog] of catalogs.entries()) { + if (catalog.status === "fulfilled") continue; + const service = services[index]!; + unavailableServices.push(service); + await audit({ companyId: connection.companyId, connectionId: connection.id, + action: "tool_connection.catalog_refresh", outcome: "failure", actor, + reasonCode: "google_workspace_service_unavailable", details: { service } }); + } + const currentGrant = await vaultGrantForConnection(connection, actor); + if (!currentGrant) throw unprocessable("OAuth authorization must be reconnected", { code: "oauth_reauthorization_required" }); + if (catalogs.length > 0 && catalogs.every((catalog) => catalog.status === "rejected")) { + throw new HttpError(502, `Google services unavailable: ${unavailableServices.join(", ")}. Retry refreshing actions.`, + { code: "google_workspace_services_unavailable" }); + } // A service's authentication retry may have refreshed into a narrower // grant. Never publish actions using the pre-refresh permission snapshot. - const currentScopes = googleWorkspaceGrantedScopes(await vaultGrantForConnection(connection, actor)); - return catalogs.flat().filter((tool) => isGoogleWorkspaceToolGranted(tool.name, currentScopes)); + const currentScopes = googleWorkspaceGrantedScopes(currentGrant); + return catalogs.flatMap((catalog) => catalog.status === "fulfilled" ? catalog.value : []) + .filter((tool) => isGoogleWorkspaceToolGranted(tool.name, currentScopes)); } const endpoint = workspaceEndpoint ?? await resolvedRemoteEndpoint(connection, actor); // Pinned to the address the guard approved: `config.url` is operator-supplied, @@ -7474,6 +7490,7 @@ export function toolAccessService( connection: typeof toolConnections.$inferSelect, credentialHeaders?: Record, actor?: ActorInfo, + unavailableServices: string[] = [], ): Promise { assertSupportedConnection(connection); if (connection.connectionPurpose === "ai") throw unprocessable("AI connections provide runtime authentication, not tool actions"); @@ -7487,7 +7504,7 @@ export function toolAccessService( return []; } if (connection.transport === "mcp_remote") - return remoteTools(connection, credentialHeaders, actor); + return remoteTools(connection, credentialHeaders, actor, undefined, unavailableServices); if (connection.transport !== "local_stdio") { throw unsupportedToolConnectionTransport(); } @@ -7591,6 +7608,7 @@ export function toolAccessService( ): Promise { const connection = await getConnectionRow(connectionId); if (connection.connectionPurpose === "ai") return { connection: toConnection(connection), runtimeSlot: null }; + const unavailableServices: string[] = []; try { assertSupportedConnection(connection); const config = asRecord(connection.config); @@ -7652,7 +7670,7 @@ export function toolAccessService( : canProbeWithoutAuthorization ? {} : undefined; - await remoteTools(connection, credentialHeaders, actor); + await remoteTools(connection, credentialHeaders, actor, undefined, unavailableServices); } else if (connection.transport === "local_stdio") { await resolveCredentialHeaders(connection); await stdioTemplateId(connection.companyId, connection.config); @@ -7663,7 +7681,9 @@ export function toolAccessService( const updated = await updateConnectionHealth( connection, "ok", - config.sourceTemplateKey === "github" && + unavailableServices.length > 0 + ? `Connected. Google services unavailable: ${unavailableServices.join(", ")}. Other services remain available; retry refreshing actions.` + : config.sourceTemplateKey === "github" && oauth.connectorProfile === "github.code" ? "GitHub account, installation, and repository access are available." : isBrowserUseConnection(connection) @@ -7734,12 +7754,14 @@ export function toolAccessService( const connection = await getConnectionRow(connectionId); if (connection.connectionPurpose === "ai") throw unprocessable("AI connections do not have a tool catalog"); const refreshedAt = now(); + const unavailableServices: string[] = []; let descriptors: McpToolDescriptor[]; try { descriptors = await discoverTools( connection, refreshOptions.credentialHeaders, actor, + unavailableServices, ); } catch (error) { if ( @@ -7776,7 +7798,8 @@ export function toolAccessService( const existingByName = new Map( existingRows.map((entry) => [entry.toolName, entry]), ); - if (isRemoteMcpConnectorMethod(connection.config.sourceTemplateKey, connection.config.connectionMethodKey)) { + if (connection.config.sourceTemplateKey === "google-workspace" + || isRemoteMcpConnectorMethod(connection.config.sourceTemplateKey, connection.config.connectionMethodKey)) { const discoveredNames = new Set(descriptors.map((descriptor) => descriptor.name)); const removedIds = existingRows.filter((entry) => !discoveredNames.has(entry.toolName) && entry.status !== "disabled").map((entry) => entry.id); if (removedIds.length) await db.update(toolCatalogEntries) @@ -7938,7 +7961,9 @@ export function toolAccessService( config: normalizedConfig, transportConfig: normalizedTransportConfig, healthStatus: "ok", - healthMessage: isAgentMailConnection(connection) + healthMessage: unavailableServices.length > 0 + ? `Actions refreshed. Google services unavailable: ${unavailableServices.join(", ")}. Other services remain available; retry refreshing actions.` + : isAgentMailConnection(connection) ? "AgentMail API key is connected." : "Tool catalog refreshed.", healthCheckedAt: refreshedAt, @@ -11463,6 +11488,9 @@ export function toolAccessService( profile: profile.id, refreshToken: refreshSecret.value, }); + if (profile.id === "workspace.all" && !isGoogleWorkspaceScopeGrant(credentials.scopes)) { + throw new PaperclipCloudConnectorError("Google did not return verifiable permissions", "REAUTHORIZATION_REQUIRED"); + } } catch (error) { if ( error instanceof PaperclipCloudConnectorError &&