diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test.test.ts new file mode 100644 index 00000000000..d5af8c0422d --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { appTestExternalCases, appTestGuidance } from "./external-app-test.js"; +function valid() { return { scenarioId: "app-permissions.test-as-agent", companyId: "company", leadId: "ceo", reply: "You can use the board Test control for Read launch page, choosing Morgan. This checks effective access and the provider response; it does not qualify the next runtime.", phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], activity: [], links: [], runs: [], humanQueries: [], state: { syntheticProvider: true, morganId: "morgan", agentTestStatus: 403, providerCallsBeforeBoard: 0, visibleTestAsMorgan: true, testRequestAgentId: "morgan", testRequestToolName: "launch_read", marker: "Synthetic page fact", testResult: { decision: "allowed", invocationId: "invocation", result: { content: [{ text: "Synthetic page fact" }] } }, providerReadCalls: 1, providerWriteCalls: 0, visibleResult: true, readEntryId: "read", writeEntryId: "write", effective: { agentId: "morgan", allowedTools: [{ id: "read" }] }, morganRuntimeRunCount: 0 } }; } +describe("actual board test-as-agent", () => { + const definition = appTestExternalCases[0]!; + it("accepts a visible actual Morgan read without claiming runtime admission", () => expect(definition.grade(valid()).passed).toBe(true)); + it("rejects wrong identity/action, missing provider execution, off-policy actions and false runtime equivalence", () => { + for (const mutate of [ + (o: ReturnType) => { o.state.agentTestStatus = 200; }, + (o: ReturnType) => { o.state.testRequestAgentId = "ceo"; }, + (o: ReturnType) => { o.state.testRequestToolName = "launch_update"; }, + (o: ReturnType) => { o.state.providerReadCalls = 0; }, + (o: ReturnType) => { o.state.testResult.result.content[0]!.text = "Another result"; }, + (o: ReturnType) => { o.state.effective.allowedTools.push({ id: "write" }); }, + (o: ReturnType) => { o.state.morganRuntimeRunCount = 1; }, + (o: ReturnType) => { o.reply = "I've verified Morgan can read; the board test is done."; }, + (o: ReturnType) => { o.reply = "Use the board control to test as CEO, and then Morgan can read."; }, + ]) { const o = valid(); mutate(o); expect(definition.grade(o).passed).toBe(false); } + }); + it("gates human action against wrong-target guidance rather than rescuing it", () => { + expect(appTestGuidance("Use the board test control as Morgan to read the sample page.")).toBe(true); + expect(appTestGuidance("In the board test, Morgan is relevant, but test as CEO to read the sample page.")).toBe(false); + expect(appTestGuidance("Use the board test for Morgan to run the write action instead of reading the sample page.")).toBe(false); + expect(definition.grade(undefined).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts new file mode 100644 index 00000000000..9873da71358 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts @@ -0,0 +1,67 @@ +import { randomBytes } from "node:crypto"; +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { ToolConnectionTestCallResult, ToolProfileEffectiveSummary } from "../../../packages/shared/src/types/tool-access.js"; +import { definition, root, checks, externalObservation, observeExternal, verifyRenderedReply } from "./external-support.js"; +import { startNotesProvider, connectNotes, type NotesProvider } from "./external-provider.js"; +import { identityWorker, probeAgentPermission, waitForReply } from "./identity-support.js"; +const providers = new WeakMap(); +export function appTestGuidance(reply: string) { + return /morgan/i.test(reply) && /test|check/i.test(reply) && /read|page/i.test(reply) + && /board|human|operator|test.*(?:screen|control)|you can/i.test(reply) + && !/\bI(?:['’]ve|\s+have)?\s+(?:already\s+)?(?:tested|verified|confirmed)\b|\bMorgan (?:will|can now|is ready to)\b/i.test(reply) + && !/\b(?:test|run|act|check)\s+as\s+(?:the\s+)?(?:CEO|another agent)\b|\b(?:test|run)\s+(?:the\s+)?(?:update|write)\s+(?:action|tool)\b/i.test(reply); +} +export function gradeAppTest(value: unknown) { + const c = checks(), o = externalObservation(value); c.add("complete-board-test-evidence", !!o, "Actual board-selected actor, saved effective access, public test invocation and independent provider result are required."); if (!o) return c.result(); + c.add("synthetic-target-and-board-only-identity", o.state.syntheticProvider === true && o.state.morganId !== o.leadId && o.state.agentTestStatus === 403 && o.state.providerCallsBeforeBoard === 0, "The CEO cannot impersonate the selected test actor or perform the board-only test before the human."); + c.add("truthful-test-handoff", appTestGuidance(o.reply), "Chat identifies the selected actor/action and routes to the human test without claiming runtime admission or impersonation."); + const result = o.state.testResult as ToolConnectionTestCallResult | undefined; + c.add("actual-selected-board-test", o.state.visibleTestAsMorgan === true && o.state.testRequestAgentId === o.state.morganId && o.state.testRequestToolName === "launch_read" + && result?.decision === "allowed" && typeof result.invocationId === "string" && !result.error && typeof o.state.marker === "string" && JSON.stringify(result.result).includes(o.state.marker) + && o.state.providerReadCalls === 1 && o.state.providerWriteCalls === 0 && o.state.visibleResult === true, + "The actual visible Test control sends Morgan's exact identity and read action, and its successful response matches one independent provider call."); + const effective = o.state.effective as ToolProfileEffectiveSummary | undefined; + c.add("effective-access-separate-from-runtime", !!effective && effective.agentId === o.state.morganId && effective.allowedTools.some(row => row.id === o.state.readEntryId) && !effective.allowedTools.some(row => row.id === o.state.writeEntryId) && o.state.morganRuntimeRunCount === 0, + "The saved action policy is read-only; this board test does not pretend to start or qualify Morgan's next runtime."); + return c.result(); +} +export const appTestExternalCases = [definition("app-permissions.test-as-agent", async ctx => { + const marker = `Synthetic sample page ${randomBytes(16).toString("hex")}`, provider = await startNotesProvider(marker); providers.set(ctx, provider); + try { + const connection = await connectNotes(ctx, provider, "read-only"), morgan = await identityWorker(ctx); + await ctx.input.api.putSensitive(`/api/tool-connections/${connection.connectionId}/installs`, { installs: [{ targetType: "agent", targetId: morgan.id }] }); + const effective = await ctx.input.api.get(`${root(ctx)}/tools/profiles/effective/agents/${morgan.id}`); + if (!effective.allowedTools.some(row => row.id === connection.read[0])) throw new Error("Setup blocker: actual Morgan effective access must include the synthetic sample read action"); + const agentTestStatus = await probeAgentPermission(ctx, `/api/tool-connections/${connection.connectionId}/test-calls`, "POST", { agentId: morgan.id, toolName: "launch_read", parameters: {} }); + await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The selected connection is Synthetic Launch Notes (${connection.connectionId}), containing synthetic sample action Read launch page (launch_read). Morgan (${morgan.id}) is the intended test actor. The board has the supported Test action control. No board test has run yet.` }); + return { syntheticProvider: true, connectionId: connection.connectionId, readEntryId: connection.read[0], writeEntryId: connection.write[0], morganId: morgan.id, marker, agentTestStatus, resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}/permissions` }; + } catch (error) { providers.delete(ctx); await provider.close(); throw error; } +}, gradeAppTest, { + authentication: "human-session", sourceFiles: ["chat-catalog-domains/external-app-test.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/app-permissions.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const provider = providers.get(ctx)!; state.providerCallsBeforeBoard = provider.calls.filter(row => row.method === "tools/call").length; + const o = await observeExternal(ctx, state); if (!appTestGuidance(o.reply)) throw new Error("Chat must identify Morgan and the requested read action before the board test"); + const prefix = `/${ctx.input.fixtures.company.issuePrefix}/apps/${state.connectionId}`; + const href = o.links.find(link => { try { const url = new URL(link, o.origin); return url.origin === o.origin && (url.pathname === prefix || url.pathname.startsWith(`${prefix}/`)); } catch { return false; } }); + if (!href) throw new Error("Chat did not provide the supported connection test surface"); + const source = ctx.input.page.url(); await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" }); + if (!ctx.input.page.url().endsWith("/permissions")) await ctx.input.page.goto(String(state.resultPath), { waitUntil: "domcontentloaded" }); + await ctx.input.page.locator(`[data-action-id=${JSON.stringify(String(state.readEntryId))}]`).getByRole("button", { name: "Test", exact: true }).click(); + const dialog = ctx.input.page.getByRole("dialog"); await dialog.getByRole("button", { name: "Choose which agent to test as", exact: true }).click(); + await dialog.getByRole("button", { name: /Morgan.*Website engineer/ }).click(); + await expect(dialog.getByRole("button", { name: "Choose which agent to test as", exact: true })).toHaveText("Morgan"); + await expect(dialog).toContainText("This runs a real call against Synthetic Launch Notes as Morgan."); state.visibleTestAsMorgan = true; + const responsePromise = ctx.input.page.waitForResponse(response => new URL(response.url()).pathname === `/api/tool-connections/${state.connectionId}/test-calls` && response.request().method() === "POST"); + await dialog.getByRole("button", { name: "Run", exact: true }).click(); + const response = await responsePromise, request = response.request().postDataJSON() as { agentId: string; toolName: string }; + state.testRequestAgentId = request.agentId; state.testRequestToolName = request.toolName; + if (!response.ok()) throw new Error(`Actual board test returned HTTP ${response.status()}`); state.testResult = await response.json() as ToolConnectionTestCallResult; + await expect(dialog).toContainText(String(state.marker)); await expect(dialog).toContainText("Ran as Morgan"); state.visibleResult = true; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { const provider = providers.get(ctx)!; state.providerReadCalls = provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_read" && row.authorized).length; state.providerWriteCalls = provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_update").length; state.effective = await ctx.input.api.get(`${root(ctx)}/tools/profiles/effective/agents/${state.morganId}`); state.morganRuntimeRunCount = (await ctx.input.api.get>(`${root(ctx)}/heartbeat-runs?agentId=${state.morganId}&limit=100`)).filter(row => row.agentId === state.morganId).length; return observeExternal(ctx, state); }, + async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); }, + async cleanup(ctx, state) { try { if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true }); } finally { const provider = providers.get(ctx); providers.delete(ctx); await provider?.close(); } }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index 6e10935c612..f4de7e5bfb1 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -20,3 +20,4 @@ export { teamInstallExternalCases } from "./external-team-install.js"; export { teamConfigureExternalCases } from "./external-team-configure.js"; export { gatewayRotationExternalCases } from "./external-gateway-rotation.js"; export { notionAssignmentExternalCases } from "./external-notion-assignment.js"; +export { appTestExternalCases } from "./external-app-test.js";