diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index a458977f205..e34de3e36df 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -10,3 +10,4 @@ export { chatChannelExternalCases } from "./external-chat-channels.js"; export { appCatalogExternalCases } from "./external-app-catalog.js"; export { pluginExternalCases } from "./external-plugins.js"; export { assistantDeviceExternalCases } from "./external-assistant-device.js"; +export { gatewayClientExternalCases } from "./external-gateway-client.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-gateway-client.ts b/tests/runner-e2e/chat-catalog-domains/external-gateway-client.ts new file mode 100644 index 00000000000..0fdc0ad5e1a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-gateway-client.ts @@ -0,0 +1,97 @@ +import { randomBytes } from "node:crypto"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { ToolProfileWithDetails, ToolProfileEffectiveSummary, ToolRunDecisionLookup } from "../../../packages/shared/src/types/tool-access.js"; +import { pollUntil } from "../api.js"; +import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +import { connectNotes, startNotesProvider, type NotesProvider } from "./external-provider.js"; + +interface PrivateFixture { provider: NotesProvider; key?: { id: string; token: string }; session?: { sessionId: string; token: string } } +const fixtures = new WeakMap(); +async function privateRequest(ctx: ChatCatalogContext, pathname: string, token: string, body?: unknown, session = false) { + const url = new URL(pathname, ctx.input.api.baseURL); + if (url.origin !== ctx.input.api.baseURL || !url.pathname.startsWith("/api/tool-gateway/")) throw new Error("Scoped session probe escaped its fixture API"); + return fetch(url, { method: body === undefined ? "GET" : "POST", redirect: "error", signal: AbortSignal.timeout(15_000), headers: { + ...(session ? { "x-paperclip-tool-gateway-token": token } : { Authorization: `Bearer ${token}` }), "content-type": "application/json", + }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); +} +export function gradeGatewayClient(value: unknown) { + const c = checks(), o = externalObservation(value); + c.add("complete-scoped-client-evidence", !!o, "Actual assigned profile, run/task admission, independent discovery/denial and provider-read receipts are required."); if (!o) return c.result(); + const effective = o.state.effective as ToolProfileEffectiveSummary | undefined; + c.add("intended-effective-profile", effective?.agentId === o.leadId && effective.profiles.some(row => row.id === o.state.profileId) + && effective.allowedTools.some(row => row.id === o.state.readEntryId) && !effective.allowedTools.some(row => row.id === o.state.writeEntryId), + "The saved launch profile grants its read action to this agent while excluding the write action."); + c.add("actual-active-run-scoped-session", o.state.sessionCreated === true && o.runs.some(run => run.id === o.state.probeRunId && run.agentId === o.leadId && run.companyId === o.companyId) + && o.state.discoveredRead === true && o.state.discoveredWrite === false, + "A real agent bearer creates a session for the admitted source run/task and discovers only authorized actions."); + const decisions = o.state.decisions as ToolRunDecisionLookup[] | undefined; + c.add("model-performed-bounded-read", !!decisions && decisions.some(row => row.decisions.some(decision => decision.invocation.runId === row.runId + && decision.invocation.agentId === o.leadId && decision.invocation.issueId === o.state.sourceIssueId && decision.invocation.connectionId === o.state.connectionId + && decision.invocation.catalogEntryId === o.state.readEntryId && decision.invocation.status === "succeeded")) + && Number(o.state.providerReadCalls) >= 1 && Number(o.state.providerReadCalls) <= 3 && typeof o.state.marker === "string" && o.reply.includes(o.state.marker), + "The actual model run uses the real provider read and reports its unpredictable result; the oracle never performs that read."); + c.add("disallowed-write-denied", o.state.writeProbeStatus === 403 && ["deny_default", "deny_policy_block"].includes(String(o.state.writeProbeReason)) && o.state.providerWriteCalls === 0 && o.state.written === null, + "A real scoped-session write attempt is denied before provider execution."); + c.add("no-private-token-disclosure", o.state.tokenScanComplete === true && o.state.privateTokenLeaked === false, "Scoped session and probe credentials never enter the persisted reply/run result."); + return c.result(); +} +export const gatewayClientExternalCases = [definition("gateways-profiles.connect-client", async ctx => { + const marker = `Launch fixture result ${randomBytes(16).toString("hex")}`; + const provider = await startNotesProvider(marker); const fixture: PrivateFixture = { provider }; fixtures.set(ctx, fixture); + try { + const connection = await connectNotes(ctx, provider, "read-only"); + const profile = await ctx.input.api.post(`${root(ctx)}/tools/profiles`, { profileKey: "launch", name: "Launch profile", defaultAction: "deny", entries: [ + { selectorType: "catalog_entry", effect: "include", connectionId: connection.connectionId, catalogEntryId: connection.read[0] }, + { selectorType: "catalog_entry", effect: "exclude", connectionId: connection.connectionId, catalogEntryId: connection.write[0] }, + ] }); + await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profile.id}/bind`, { targetType: "agent", targetId: ctx.input.fixtures.agent.id }); + const access = await ctx.input.api.get<{ access: { tools: Array<{ toolName: string; gatewayToolName: string }> } }>(`/api/tool-connections/${connection.connectionId}/test-agents/${ctx.input.fixtures.agent.id}/access`); + const write = access.access.tools.find(row => row.toolName === "launch_update"); if (!write) throw new Error("The excluded write action has no independently discoverable catalog identity"); + return { connectionId: connection.connectionId, profileId: profile.id, readEntryId: connection.read[0], writeEntryId: connection.write[0], writeGatewayName: write.gatewayToolName, marker }; + } catch (error) { fixtures.delete(ctx); await provider.close(); throw error; } +}, gradeGatewayClient, { + sourceFiles: ["chat-catalog-domains/external-gateway-client.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/gateways-profiles.yaml"], + async act(ctx, state) { + const fixture = fixtures.get(ctx)!; + const active = await pollUntil({ label: "admitted source run for independent scoped denial probe", deadlineAt: Date.now() + 60_000, intervalMs: 100, + load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, accept: rows => rows.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "running"), + reject: rows => rows.length && rows.every(row => ["succeeded", "failed", "cancelled", "timed_out"].includes(row.status)) ? "Source run settled before scoped-session probe admission" : undefined }); + const run = active.find(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "running")!; + const source = ctx.issue(); if (!source || !ctx.input.privateSecrets) throw new Error("Scoped client requires source identity and private redaction"); + state.probeRunId = run.id; state.sourceIssueId = source.id; + const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `scoped-client-probe-${ctx.input.nonce}` }); fixture.key = key; ctx.input.privateSecrets.push(key.token); + const admitted = await privateRequest(ctx, "/api/tool-gateway/sessions", key.token, { companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, runId: run.id, issueId: source.id }); + if (admitted.status !== 201) { await admitted.body?.cancel(); throw new Error("Active-run scoped session admission failed; response withheld"); } + fixture.session = await admitted.json() as { sessionId: string; token: string }; ctx.input.privateSecrets.push(fixture.session.token); state.sessionCreated = true; + const listed = await privateRequest(ctx, "/api/tool-gateway/tools", fixture.session.token, undefined, true); + if (!listed.ok) { await listed.body?.cancel(); throw new Error("Scoped tool discovery failed; response withheld"); } + const tools = await listed.json() as Array<{ catalogEntryId?: string }>; + state.discoveredRead = tools.some(row => row.catalogEntryId === state.readEntryId); state.discoveredWrite = tools.some(row => row.catalogEntryId === state.writeEntryId); + const denied = await privateRequest(ctx, "/api/tool-gateway/tools/call", fixture.session.token, { tool: state.writeGatewayName, parameters: { text: "This oracle write must be denied" } }, true); + state.writeProbeStatus = denied.status; + // Retain only the allowlisted policy code, never an arbitrary error body. + const denial = await denied.json() as { reasonCode?: unknown }; + state.writeProbeReason = ["deny_default", "deny_policy_block"].includes(String(denial.reasonCode)) ? denial.reasonCode : "other"; + }, + async observe(ctx, state) { + const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Scoped synthetic fixture missing"); + state.effective = await ctx.input.api.get(`${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`); + state.decisions = await Promise.all((await ctx.allRuns()).map(run => ctx.input.api.get(`${root(ctx)}/tools/runs/${run.id}/decisions`))); + state.providerReadCalls = fixture.provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_read" && row.authorized).length; + state.providerWriteCalls = fixture.provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_update").length; state.written = fixture.provider.written() ?? null; + const o = await observeExternal(ctx, state); + const tokens = [fixture.key?.token, fixture.session?.token].filter((token): token is string => typeof token === "string"); + state.tokenScanComplete = tokens.length === 2; state.privateTokenLeaked = tokens.some(token => JSON.stringify([o.reply, o.runs]).includes(token)); + return { ...o, state }; + }, + async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); }, + async cleanup(ctx, state) { + const fixture = fixtures.get(ctx); fixtures.delete(ctx); + try { + if (fixture?.session && fixture.key) { const response = await privateRequest(ctx, `/api/tool-gateway/sessions/${fixture.session.sessionId}/revoke`, fixture.key.token, { companyId: ctx.input.fixtures.company.id }); if (!response.ok && response.status !== 404) throw new Error("Scoped fixture session revocation failed"); await response.body?.cancel(); } + if (fixture?.key) await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${fixture.key.id}`, { allowNotFound: true }); + if (typeof state.profileId === "string") await ctx.input.api.post(`${root(ctx)}/tools/profiles/${state.profileId}/unbind`, { targetType: "agent", targetId: ctx.input.fixtures.agent.id }); + if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true }); + } finally { await fixture?.provider.close(); } + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index 4adba4ba2e1..6b85dd1cdc2 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { assistantConnectionExternalCases } from "./external-assistant-connections.js"; import { chatChannelExternalCases } from "./external-chat-channels.js"; +import { gatewayClientExternalCases } from "./external-gateway-client.js"; import { assistantDeviceExternalCases } from "./external-assistant-device.js"; import { pluginExternalCases } from "./external-plugins.js"; import { appCatalogExternalCases } from "./external-app-catalog.js"; @@ -12,7 +13,7 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js"; import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js"; import type { ExternalObservation } from "./external-support.js"; -const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases]; +const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases]; function base(scenarioId: string): ExternalObservation { return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" }, phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], @@ -20,7 +21,13 @@ function base(scenarioId: string): ExternalObservation { } function valid(id: string) { const o = base(id); - if (id === "assistant-connections.device-login") { + if (id === "gateways-profiles.connect-client") { + Object.assign(o.state, { profileId: "profile", readEntryId: "read", writeEntryId: "write", connectionId: "connection", sourceIssueId: "source", probeRunId: "initial", + effective: { agentId: "ceo", profiles: [{ id: "profile" }], allowedTools: [{ id: "read" }] }, sessionCreated: true, discoveredRead: true, discoveredWrite: false, + decisions: [{ runId: "initial", decisions: [{ invocation: { runId: "initial", agentId: "ceo", issueId: "source", connectionId: "connection", catalogEntryId: "read", status: "succeeded" } }] }], + providerReadCalls: 1, marker: "Unpredictable launch result", writeProbeStatus: 403, writeProbeReason: "deny_default", providerWriteCalls: 0, written: null, tokenScanComplete: true, privateTokenLeaked: false }); + o.reply = "The launch profile returned Unpredictable launch result."; + } else if (id === "assistant-connections.device-login") { Object.assign(o.state, { verificationPath: "/mcp-device", currentCodeVisible: true, companyConsentVisible: true, readScopeVisible: true, humanApproved: true, clientCompanyId: "company", clientReadOnly: true, pendingError: "authorization_pending", wrongResourceDenied: true, wrongClientDenied: true, correctExchangeSucceeded: true, reusedCodeDenied: true, expirationWaitMs: 600000, elapsedBeforeExpiryProbe: 600501, expiredCodeDenied: true, expiredPageVisible: true }); @@ -78,6 +85,7 @@ function valid(id: string) { return o; } const wrong: Record void> = { + "gateways-profiles.connect-client": o => { o.state.providerWriteCalls = 1; }, "assistant-connections.device-login": o => { o.state.elapsedBeforeExpiryProbe = 500; }, "plugins.configure-plugin": o => { o.state.configuredReadback = false; }, "plugins.disable-plugin": o => { o.state.previewStatusAfter = 200; }, @@ -94,6 +102,12 @@ const wrong: Record void> = { "connection-setup.notion-declined": o => { o.interactions[0]!.resolvedByUserId = "someone-else"; }, }; describe("external domain independent oracles", () => { + it("an inactive or expired gateway session cannot prove profile write denial", () => { + for (const reason of ["run_inactive", "session_run_inactive", "gateway_token_expired", "tool_not_found", undefined]) { + const o = valid("gateways-profiles.connect-client"); o.state.writeProbeReason = reason; + expect(gatewayClientExternalCases[0]!.grade(o).passed).toBe(false); + } + }); for (const definition of definitions) it(`${definition.scenarioId} accepts valid, rejects plausible wrong and missing evidence`, () => { const good = valid(definition.scenarioId); expect(definition.grade(good), JSON.stringify(definition.grade(good))).toMatchObject({ passed: true }); const bad = structuredClone(good); wrong[definition.scenarioId]!(bad); expect(definition.grade(bad).passed).toBe(false);