mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-22 03:29:18 +01:00
Narrow exact connected tool reads before loading schemas (#22457)
This commit is contained in:
1 parent
2b8d92c769
commit
7f4cfae5e1
3 files changed
+226
-17
No files matched your search
@@ -74,6 +74,8 @@ import {
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { grantLegacyPluginTools } from "./helpers/legacy-plugin-tools.js";
|
||||
import { recordingDb } from "./helpers/tool-gateway-listing-fixture.js";
|
||||
import { githubCredentialsFixture } from "./helpers/github-credentials-fixture.js";
|
||||
|
||||
const externalTestDatabase = resolveExternalTestDatabaseUrl("PAPERCLIP_TOOL_GATEWAY_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalTestDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
@@ -2527,6 +2529,158 @@ rl.on("line", (line) => {
|
||||
]));
|
||||
});
|
||||
|
||||
it("loads only candidate catalog rows for a live exact connected name", async () => {
|
||||
const company = await createCompany(db);
|
||||
const agent = await createAgent(db, company.id);
|
||||
const { run } = await createIssueAndRun(db, company.id, agent.id);
|
||||
const fake = await startFakeRemoteMcpServer(({ body }) => ({
|
||||
body: { jsonrpc: "2.0", id: body?.id, result: { content: [{ type: "text", text: "found" }] } },
|
||||
}));
|
||||
try {
|
||||
const { application, connection, catalogEntry } = await createRemoteMcpTool(db, company.id, {
|
||||
applicationKey: "exact-demo", toolName: " READ__Item ", url: fake.url, riskLevel: "read",
|
||||
});
|
||||
const connectionPrefix = connection.id.slice(0, 8);
|
||||
const peerId = connectionPrefix + randomUUID().slice(8);
|
||||
await db.insert(toolConnections).values({
|
||||
...connection, id: peerId, uid: "exact-lookup-peer", name: "Name collision peer",
|
||||
});
|
||||
const [peer, unicode, natural] = await db.insert(toolCatalogEntries).values([
|
||||
{ ...catalogEntry, id: randomUUID(), connectionId: peerId, name: "read-item", toolName: "read-item" },
|
||||
{ ...catalogEntry, id: randomUUID(), name: "unicode-name", toolName: "Key_İtem" },
|
||||
{ ...catalogEntry, id: randomUUID(), name: "natural-suffix", toolName: "natural-deadbeef" },
|
||||
{ ...catalogEntry, id: randomUUID(), name: "unrelated-name", toolName: "unrelated-name" },
|
||||
]).returning();
|
||||
await allowAllToolsForAgent(db, company.id, agent.id);
|
||||
const recorded = recordingDb(db);
|
||||
const gateway = createTestToolGatewayService(recorded.db);
|
||||
const session = await gateway.createSession({ companyId: company.id, agentId: agent.id, runId: run.id });
|
||||
const call = (tool: string) => gateway.executeTool({ sessionToken: session.token, tool, parameters: {} });
|
||||
const baseName = expectedConnectedToolName({ applicationKey: application.applicationKey,
|
||||
connectionId: connection.id, toolName: "read_item" });
|
||||
const collidingName = baseName + "-" + catalogEntry.id.slice(0, 8);
|
||||
const list = await gateway.listToolsForSession(session.token);
|
||||
expect(list.find((tool) => tool.catalogEntryId === catalogEntry.id)?.name).toBe(collidingName);
|
||||
expect(list.find((tool) => tool.catalogEntryId === peer.id)?.name).toBe(baseName + "-" + peer.id.slice(0, 8));
|
||||
expect(list.find((tool) => tool.catalogEntryId === unicode.id)?.name).toBe(`mcp.exact-demo-${connectionPrefix}:key-i-tem`);
|
||||
|
||||
recorded.statements.length = 0;
|
||||
recorded.statementParams.length = 0;
|
||||
await expect(call(collidingName)).resolves.toMatchObject({ status: "completed" });
|
||||
const catalogRead = recorded.statements.findIndex((sql) => sql.includes('"input_schema"') &&
|
||||
sql.includes('order by "tool_connections"."name", "tool_catalog_entries"."name"'));
|
||||
expect(catalogRead).toBeGreaterThanOrEqual(0);
|
||||
expect(recorded.statements[catalogRead]).toContain("regexp_replace");
|
||||
expect(recorded.statementParams[catalogRead]).toEqual(expect.arrayContaining([
|
||||
connectionPrefix, "read-item", collidingName.split(":")[1],
|
||||
]));
|
||||
for (const entry of [unicode, natural]) {
|
||||
await expect(call(list.find((tool) => tool.catalogEntryId === entry.id)!.name))
|
||||
.resolves.toMatchObject({ status: "completed" });
|
||||
expect(fake.requests.at(-1)?.body).toMatchObject({ params: { name: entry.toolName } });
|
||||
}
|
||||
|
||||
const summary = await gateway.summarizeConnectionAccessForAgent({ companyId: company.id,
|
||||
connectionId: connection.id, agentId: agent.id, heartbeatRunId: run.id });
|
||||
expect(summary.tools.map((tool) => tool.gatewayToolName)).toEqual(expect.arrayContaining([collidingName]));
|
||||
expect(summary.tools.some((tool) => tool.gatewayToolName === baseName + "-" + peer.id.slice(0, 8))).toBe(false);
|
||||
|
||||
// A natural slug can also be another entry's disambiguated name.
|
||||
const [duplicate] = await db.insert(toolCatalogEntries).values({
|
||||
...catalogEntry, id: randomUUID(), name: "zz-duplicate-alias", toolName: collidingName.split(":")[1]!,
|
||||
}).returning();
|
||||
await expect(call(collidingName)).resolves.toMatchObject({ status: "completed" });
|
||||
expect(fake.requests.at(-1)?.body).toMatchObject({ params: { name: catalogEntry.toolName } });
|
||||
await db.delete(toolCatalogEntries).where(eq(toolCatalogEntries.id, duplicate.id));
|
||||
|
||||
// With the peer quarantined, the surviving entry loses its suffix.
|
||||
await db.update(toolCatalogEntries).set({ status: "quarantined", quarantinedAt: new Date() })
|
||||
.where(eq(toolCatalogEntries.id, peer.id));
|
||||
const versionHash = randomUUID();
|
||||
await db.update(toolCatalogEntries).set({ toolName: "Read.Item", versionHash, schemaHash: randomUUID() })
|
||||
.where(eq(toolCatalogEntries.id, catalogEntry.id));
|
||||
await expect(call(collidingName)).rejects.toMatchObject({ reasonCode: "tool_not_found" });
|
||||
const updated = await call(baseName);
|
||||
expect(updated.status).toBe("completed");
|
||||
const [invocation] = await db.select().from(toolInvocations).where(eq(toolInvocations.id, updated.invocationId));
|
||||
expect(invocation?.catalogVersionHash).toBe(versionHash);
|
||||
expect(fake.requests.at(-1)?.body).toMatchObject({ params: { name: "Read.Item" } });
|
||||
} finally {
|
||||
await fake.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("finds the same GitHub account's tool on a different connection after credential failure", async () => {
|
||||
const { seed, grant } = githubCredentialsFixture(() => db);
|
||||
const context = await seed();
|
||||
const { companyId, agentId, runId, issueId } = context;
|
||||
const older = await grant(context, "A");
|
||||
const newer = await grant(context, "A");
|
||||
const alternateCredential = "synthetic-alternate-credential";
|
||||
const secrets = secretService(db);
|
||||
await secrets.rotateCurrentUserSecretValue(companyId, "A", older.secretId, { value: alternateCredential });
|
||||
await secrets.syncUserSecretDeclarationsForTarget(companyId,
|
||||
{ targetType: "tool_connection", targetId: older.connectionId },
|
||||
[{ definitionKey: older.definitionId, configPath: "oauth.access_token", envKey: "Authorization", required: true }],
|
||||
{ replaceAll: true });
|
||||
const upstreamToolName = "list_issues";
|
||||
const olderUrl = "https://github-mcp-fixture.invalid/older";
|
||||
const newerUrl = "https://github-mcp-fixture.invalid/newer";
|
||||
let requestedName = "";
|
||||
for (const [candidate, url, createdAt] of [
|
||||
[older, olderUrl, new Date(0)],
|
||||
[newer, newerUrl, new Date(1)],
|
||||
] as const) {
|
||||
const [connection] = await db.update(toolConnections)
|
||||
.set({ authKind: "oauth", config: { sourceTemplateKey: "github", url } })
|
||||
.where(eq(toolConnections.id, candidate.connectionId)).returning();
|
||||
await db.update(connectionGrants).set({ createdAt }).where(eq(connectionGrants.id, candidate.id));
|
||||
await db.insert(toolCatalogEntries).values({
|
||||
companyId, applicationId: connection.applicationId, connectionId: connection.id,
|
||||
name: upstreamToolName, toolName: upstreamToolName, riskLevel: "read", versionHash: randomUUID(),
|
||||
});
|
||||
if (candidate === newer) {
|
||||
const [application] = await db.select().from(toolApplications).where(eq(toolApplications.id, connection.applicationId));
|
||||
requestedName = expectedConnectedToolName({
|
||||
applicationKey: application.applicationKey ?? connection.name, connectionId: connection.id, toolName: upstreamToolName,
|
||||
});
|
||||
}
|
||||
}
|
||||
await allowAllToolsForAgent(db, companyId, agentId);
|
||||
const refresh = vi.fn(async ({ grantId }: { grantId: string }) => {
|
||||
if (grantId === newer.id) throw new Error("Synthetic credential acquisition failure");
|
||||
const [current] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grantId));
|
||||
return current;
|
||||
});
|
||||
const remoteHttpRequest = vi.fn(async (_url: string, init: RequestInit) => {
|
||||
const request = JSON.parse(String(init.body));
|
||||
expect(request.params.name).toBe(upstreamToolName);
|
||||
expect(new Headers(init.headers).get("Authorization")).toBe(`Bearer ${alternateCredential}`);
|
||||
return Response.json({ jsonrpc: "2.0", id: request.id, result: { content: [{ type: "text", text: "alternate ok" }] } });
|
||||
});
|
||||
const recorded = recordingDb(db);
|
||||
const catalogReads = () => recorded.statements.filter((query) => query.includes('"input_schema"') &&
|
||||
query.includes('order by "tool_connections"."name", "tool_catalog_entries"."name"'));
|
||||
const gateway = createTestToolGatewayService(recorded.db, { oauthGrantRefresher: refresh, remoteHttpRequest });
|
||||
const session = await gateway.createSession({ companyId, agentId, runId, issueId });
|
||||
const result = await gateway.executeTool({ sessionToken: session.token, tool: requestedName, parameters: {} });
|
||||
|
||||
expect(result).toMatchObject({ status: "completed", result: { content: "alternate ok" } });
|
||||
expect(refresh.mock.calls.map(([input]) => input.grantId)).toEqual([newer.id, older.id]);
|
||||
expect(remoteHttpRequest.mock.calls.map(([url]) => url)).toEqual([olderUrl]);
|
||||
const [invocation] = await db.select().from(toolInvocations).where(eq(toolInvocations.id, result.invocationId));
|
||||
expect(invocation?.connectionId).toBe(older.connectionId);
|
||||
expect(catalogReads()).toHaveLength(2);
|
||||
expect(catalogReads().every((query) => query.includes("regexp_replace"))).toBe(true);
|
||||
|
||||
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, newer.id));
|
||||
recorded.statements.length = 0;
|
||||
recorded.statementParams.length = 0;
|
||||
await expect(gateway.executeTool({ sessionToken: session.token, tool: invocation.toolName, parameters: {} }))
|
||||
.resolves.toMatchObject({ status: "completed" });
|
||||
expect(catalogReads()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("dispatches a bounded alias to the original long upstream tool name", async () => {
|
||||
const company = await createCompany(db);
|
||||
const agent = await createAgent(db, company.id);
|
||||
@@ -3514,6 +3668,13 @@ rl.on("line", (line) => {
|
||||
}),
|
||||
}),
|
||||
]));
|
||||
|
||||
await db.update(toolConnections).set({ config: { url: fake.url } })
|
||||
.where(eq(toolConnections.id, remoteTool.connection.id));
|
||||
await expect(gateway.executeTool({ sessionToken: session.token, tool: "search_tools", parameters: { query: "kv" } }))
|
||||
.rejects.toMatchObject({ reasonCode: "tool_not_found" });
|
||||
await expect(gateway.executeTool({ sessionToken: session.token, tool: targetToolName,
|
||||
parameters: { key: "direct-key", value: "direct-value" } })).resolves.toMatchObject({ status: "completed" });
|
||||
} finally {
|
||||
await fake.close();
|
||||
}
|
||||
|
||||
@@ -4,11 +4,17 @@ import { createHash } from "node:crypto";
|
||||
// Gateway punctuation is normalized one-for-one by the MCP client.
|
||||
export const ASSIGNED_MCP_SERVER_NAME = "paperclip-assigned";
|
||||
export const MAX_GATEWAY_TOOL_NAME_LENGTH = 128 - `mcp__${ASSIGNED_MCP_SERVER_NAME}__`.length;
|
||||
const BOUNDED_NAME_HASH_LENGTH = 16;
|
||||
const BOUNDED_NAME_SUFFIX = new RegExp(`-[a-f0-9]{${BOUNDED_NAME_HASH_LENGTH}}$`);
|
||||
|
||||
export function mayBeBoundedMcpToolName(name: string): boolean {
|
||||
return BOUNDED_NAME_SUFFIX.test(name);
|
||||
}
|
||||
|
||||
/** Preserve existing short names; aliases still resolve through gateway metadata. */
|
||||
export function boundedMcpToolName(name: string, identity: readonly string[]): string {
|
||||
if (name.length <= MAX_GATEWAY_TOOL_NAME_LENGTH) return name;
|
||||
const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 16);
|
||||
const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, BOUNDED_NAME_HASH_LENGTH);
|
||||
const separator = name.indexOf(":");
|
||||
const namespace = name.slice(0, separator < 0 ? name.length : separator).slice(0, 32);
|
||||
const tool = separator < 0 ? "tool" : name.slice(separator + 1);
|
||||
|
||||
@@ -41,7 +41,7 @@ import { githubAccessDeniedInstructions, githubAccessError } from "./tool-access
|
||||
import { githubDefaultOwnerApplies, githubDefaultOwnerArguments } from "./tool-access/github-default-owner.js";
|
||||
import { toolSourceTemplateKeySql } from "./tool-access/source-template-key.js";
|
||||
import { usesClientCredentials } from "./tool-access/oauth/client-credentials.js";
|
||||
import { boundedMcpToolName } from "./mcp-tool-names.js";
|
||||
import { boundedMcpToolName, mayBeBoundedMcpToolName } from "./mcp-tool-names.js";
|
||||
import { browserUseService } from "./browser-use.js";
|
||||
import { isBrowserUseConnection } from "./browser-use-client.js";
|
||||
import { COGNEE_STDIO_TEMPLATE, cogneeCloudUrl, callCogneeCloud } from "./cognee-connection.js";
|
||||
@@ -1079,6 +1079,11 @@ function riskFromCatalogEntry(
|
||||
return "read";
|
||||
}
|
||||
|
||||
const SLUG_SEGMENT_SEPARATOR = /[^a-z0-9]+/g;
|
||||
const SLUG_SEGMENT_LIMIT = 64;
|
||||
const CONNECTED_TOOL_SLUG_FALLBACK = "tool";
|
||||
const NON_ASCII_TOOL_NAME = /[^\x01-\x7f]/;
|
||||
|
||||
export function slugSegment(
|
||||
value: string | null | undefined,
|
||||
fallback: string,
|
||||
@@ -1086,9 +1091,9 @@ export function slugSegment(
|
||||
const slug = String(value ?? "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(SLUG_SEGMENT_SEPARATOR, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 64);
|
||||
.slice(0, SLUG_SEGMENT_LIMIT);
|
||||
return slug || fallback;
|
||||
}
|
||||
|
||||
@@ -1828,15 +1833,40 @@ export function createToolGatewayService(
|
||||
);
|
||||
}
|
||||
|
||||
function exactConnectedToolScope(name: string) {
|
||||
// Hashed aliases may have lost the UUID prefix when their namespace was truncated.
|
||||
if (mayBeBoundedMcpToolName(name)) return undefined;
|
||||
const match = /^(?:app|mcp)\.[a-z0-9-]+-([a-f0-9]{8}):([a-z0-9-]+)$/.exec(name);
|
||||
if (!match) return undefined;
|
||||
const [, prefix, slug] = match;
|
||||
if (!prefix || !slug) return undefined;
|
||||
// A suffix can be part of the original slug or the ID of a colliding entry.
|
||||
const base = slug.replace(/-[a-f0-9]{8}$/, "");
|
||||
return { prefix, slugs: base === slug ? [slug] : [slug, base] };
|
||||
}
|
||||
|
||||
async function connectedMcpToolsForCompany(
|
||||
companyId: string,
|
||||
scope?: { connectionIds: ReadonlySet<string>; preserveToolNames?: true },
|
||||
exact?: ReturnType<typeof exactConnectedToolScope>,
|
||||
): Promise<ToolGatewayDescriptor[]> {
|
||||
if (scope?.connectionIds.size === 0) return [];
|
||||
// Equal gateway base names require the same short connection UUID. Include
|
||||
// these peers in the catalog snapshot so their eligible tools still affect naming.
|
||||
let connectionScope: SQL | undefined;
|
||||
if (scope?.preserveToolNames) {
|
||||
if (exact) {
|
||||
// Match slugSegment for ASCII and keep Unicode candidates for the JS name builder.
|
||||
const normalizedSlug = sql<string>`coalesce(nullif(left(btrim(regexp_replace(
|
||||
lower(${toolCatalogEntries.toolName} collate "C"), ${SLUG_SEGMENT_SEPARATOR.source}, '-', 'g'
|
||||
), '-'), ${SLUG_SEGMENT_LIMIT}), ''), ${CONNECTED_TOOL_SLUG_FALLBACK})`;
|
||||
connectionScope = and(
|
||||
sql`split_part(${toolConnections.id}::text, '-', 1) = ${exact.prefix}`,
|
||||
or(
|
||||
inArray(normalizedSlug, exact.slugs),
|
||||
sql`${toolCatalogEntries.toolName} collate "C" ~ ${NON_ASCII_TOOL_NAME.source}`,
|
||||
),
|
||||
);
|
||||
} else if (scope?.preserveToolNames) {
|
||||
const prefixes = [...new Set(Array.from(scope.connectionIds, shortStableId))];
|
||||
connectionScope = sql`split_part(${toolConnections.id}::text, '-', 1) = any(${sql.param(prefixes)}::text[])`;
|
||||
} else if (scope) {
|
||||
@@ -1953,7 +1983,7 @@ export function createToolGatewayService(
|
||||
}>): ToolGatewayDescriptor[] {
|
||||
const baseNames = eligibleRows.map(
|
||||
({ catalogEntry, connection, application }) => {
|
||||
const toolSlug = slugSegment(catalogEntry.toolName, "tool");
|
||||
const toolSlug = slugSegment(catalogEntry.toolName, CONNECTED_TOOL_SLUG_FALLBACK);
|
||||
return `${connectionToolNamespace({
|
||||
connectionId: connection.id, transport: connection.transport, connectionName: connection.name,
|
||||
config: connection.config, applicationKey: application.applicationKey, applicationName: application.name,
|
||||
@@ -2051,10 +2081,15 @@ export function createToolGatewayService(
|
||||
async function connectedMcpToolsForConnection(
|
||||
companyId: string,
|
||||
connectionId: string,
|
||||
upstreamToolName?: string | null,
|
||||
): Promise<ToolGatewayDescriptor[]> {
|
||||
return (await connectedMcpToolsForCompany(companyId)).filter(
|
||||
(tool) => tool.connectionId === connectionId,
|
||||
);
|
||||
const exact = typeof upstreamToolName === "string"
|
||||
? { prefix: shortStableId(connectionId), slugs: [slugSegment(upstreamToolName, CONNECTED_TOOL_SLUG_FALLBACK)] }
|
||||
: undefined;
|
||||
return connectedMcpToolsForCompany(companyId, {
|
||||
connectionIds: new Set([connectionId]),
|
||||
preserveToolNames: true,
|
||||
}, exact);
|
||||
}
|
||||
|
||||
async function assertAgentInCompany(
|
||||
@@ -3387,8 +3422,13 @@ export function createToolGatewayService(
|
||||
session: ToolGatewaySession,
|
||||
toolName: string,
|
||||
): Promise<ToolGatewayDescriptor> {
|
||||
const connectedTools = (await connectedMcpToolsForCompany(session.companyId)).filter(isModelVisibleTool);
|
||||
const hasOnDemandTargets = connectedTools.some(isOnDemandRemoteTool);
|
||||
const exact = exactConnectedToolScope(toolName);
|
||||
let connectedTools = (await connectedMcpToolsForCompany(session.companyId, undefined, exact)).filter(isModelVisibleTool);
|
||||
if (exact && connectedTools.filter((candidate) => candidate.name === toolName).length > 1) {
|
||||
// Preserve the existing lookup order for duplicate canonical aliases.
|
||||
connectedTools = (await connectedMcpToolsForCompany(session.companyId)).filter(isModelVisibleTool);
|
||||
}
|
||||
const hasOnDemandTargets = !exact && connectedTools.some(isOnDemandRemoteTool);
|
||||
const virtualTools = hasOnDemandTargets ? VIRTUAL_TOOLS : [];
|
||||
const githubBotTools = await githubBotToolsForSession(db, session);
|
||||
const tool = [...await toolsForAgent(session.companyId, session.agentId), ...connectedTools, ...virtualTools, ...githubBotTools, ...await slackToolsForSession(db, session), ...await spekoToolsForSession(db, session)]
|
||||
@@ -3444,12 +3484,14 @@ export function createToolGatewayService(
|
||||
// gets its own catalog descriptor and policy checks; never replay a call.
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
const grant = selected.grant!;
|
||||
const target = connectedTools.find(
|
||||
(candidate) =>
|
||||
candidate.connectionId === grant.connectionId &&
|
||||
candidate.upstreamToolName === tool.upstreamToolName &&
|
||||
candidate.providerType === tool.providerType,
|
||||
);
|
||||
const matchesTarget = (candidate: ToolGatewayDescriptor) =>
|
||||
candidate.connectionId === grant.connectionId &&
|
||||
candidate.upstreamToolName === tool.upstreamToolName &&
|
||||
candidate.providerType === tool.providerType &&
|
||||
isModelVisibleTool(candidate);
|
||||
const target = connectedTools.find(matchesTarget) ?? (exact
|
||||
? (await connectedMcpToolsForConnection(session.companyId, grant.connectionId, tool.upstreamToolName)).find(matchesTarget)
|
||||
: undefined);
|
||||
if (!target)
|
||||
throw new ToolGatewayHttpError(
|
||||
404,
|
||||
|
||||
Reference in new issue
Block a user