test(evals): exercise actual board test-as-agent and provider outcome

tests/runner-e2e/chat-catalog-domains/external-app-test.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from "vitest";
+import { appTestExternalCases, appTestGuidance } from "./external-app-test.js";
+function valid() { return { scenarioId: "app-permissions.test-as-agent", companyId: "company", leadId: "ceo", reply: "You can use the board Test control for Read launch page, choosing Morgan. This checks effective access and the provider response; it does not qualify the next runtime.", phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], activity: [], links: [], runs: [], humanQueries: [], state: { syntheticProvider: true, morganId: "morgan", agentTestStatus: 403, providerCallsBeforeBoard: 0, visibleTestAsMorgan: true, testRequestAgentId: "morgan", testRequestToolName: "launch_read", marker: "Synthetic page fact", testResult: { decision: "allowed", invocationId: "invocation", result: { content: [{ text: "Synthetic page fact" }] } }, providerReadCalls: 1, providerWriteCalls: 0, visibleResult: true, readEntryId: "read", writeEntryId: "write", effective: { agentId: "morgan", allowedTools: [{ id: "read" }] }, morganRuntimeRunCount: 0 } }; }
+describe("actual board test-as-agent", () => {
+  const definition = appTestExternalCases[0]!;
+  it("accepts a visible actual Morgan read without claiming runtime admission", () => expect(definition.grade(valid()).passed).toBe(true));
+  it("rejects wrong identity/action, missing provider execution, off-policy actions and false runtime equivalence", () => {
+    for (const mutate of [
+      (o: ReturnType<typeof valid>) => { o.state.agentTestStatus = 200; },
+      (o: ReturnType<typeof valid>) => { o.state.testRequestAgentId = "ceo"; },
+      (o: ReturnType<typeof valid>) => { o.state.testRequestToolName = "launch_update"; },
+      (o: ReturnType<typeof valid>) => { o.state.providerReadCalls = 0; },
+      (o: ReturnType<typeof valid>) => { o.state.testResult.result.content[0]!.text = "Another result"; },
+      (o: ReturnType<typeof valid>) => { o.state.effective.allowedTools.push({ id: "write" }); },
+      (o: ReturnType<typeof valid>) => { o.state.morganRuntimeRunCount = 1; },
+      (o: ReturnType<typeof valid>) => { o.reply = "I've verified Morgan can read; the board test is done."; },
+      (o: ReturnType<typeof valid>) => { o.reply = "Use the board control to test as CEO, and then Morgan can read."; },
+    ]) { const o = valid(); mutate(o); expect(definition.grade(o).passed).toBe(false); }
+  });
+  it("gates human action against wrong-target guidance rather than rescuing it", () => {
+    expect(appTestGuidance("Use the board test control as Morgan to read the sample page.")).toBe(true);
+    expect(appTestGuidance("In the board test, Morgan is relevant, but test as CEO to read the sample page.")).toBe(false);
+    expect(appTestGuidance("Use the board test for Morgan to run the write action instead of reading the sample page.")).toBe(false);
+    expect(definition.grade(undefined).passed).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/external-app-test.ts

@@ -0,0 +1,67 @@
+import { randomBytes } from "node:crypto";
+import { expect } from "@playwright/test";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { ToolConnectionTestCallResult, ToolProfileEffectiveSummary } from "../../../packages/shared/src/types/tool-access.js";
+import { definition, root, checks, externalObservation, observeExternal, verifyRenderedReply } from "./external-support.js";
+import { startNotesProvider, connectNotes, type NotesProvider } from "./external-provider.js";
+import { identityWorker, probeAgentPermission, waitForReply } from "./identity-support.js";
+const providers = new WeakMap<ChatCatalogContext, NotesProvider>();
+export function appTestGuidance(reply: string) {
+  return /morgan/i.test(reply) && /test|check/i.test(reply) && /read|page/i.test(reply)
+    && /board|human|operator|test.*(?:screen|control)|you can/i.test(reply)
+    && !/\bI(?:['’]ve|\s+have)?\s+(?:already\s+)?(?:tested|verified|confirmed)\b|\bMorgan (?:will|can now|is ready to)\b/i.test(reply)
+    && !/\b(?:test|run|act|check)\s+as\s+(?:the\s+)?(?:CEO|another agent)\b|\b(?:test|run)\s+(?:the\s+)?(?:update|write)\s+(?:action|tool)\b/i.test(reply);
+}
+export function gradeAppTest(value: unknown) {
+  const c = checks(), o = externalObservation(value); c.add("complete-board-test-evidence", !!o, "Actual board-selected actor, saved effective access, public test invocation and independent provider result are required."); if (!o) return c.result();
+  c.add("synthetic-target-and-board-only-identity", o.state.syntheticProvider === true && o.state.morganId !== o.leadId && o.state.agentTestStatus === 403 && o.state.providerCallsBeforeBoard === 0, "The CEO cannot impersonate the selected test actor or perform the board-only test before the human.");
+  c.add("truthful-test-handoff", appTestGuidance(o.reply), "Chat identifies the selected actor/action and routes to the human test without claiming runtime admission or impersonation.");
+  const result = o.state.testResult as ToolConnectionTestCallResult | undefined;
+  c.add("actual-selected-board-test", o.state.visibleTestAsMorgan === true && o.state.testRequestAgentId === o.state.morganId && o.state.testRequestToolName === "launch_read"
+    && result?.decision === "allowed" && typeof result.invocationId === "string" && !result.error && typeof o.state.marker === "string" && JSON.stringify(result.result).includes(o.state.marker)
+    && o.state.providerReadCalls === 1 && o.state.providerWriteCalls === 0 && o.state.visibleResult === true,
+    "The actual visible Test control sends Morgan's exact identity and read action, and its successful response matches one independent provider call.");
+  const effective = o.state.effective as ToolProfileEffectiveSummary | undefined;
+  c.add("effective-access-separate-from-runtime", !!effective && effective.agentId === o.state.morganId && effective.allowedTools.some(row => row.id === o.state.readEntryId) && !effective.allowedTools.some(row => row.id === o.state.writeEntryId) && o.state.morganRuntimeRunCount === 0,
+    "The saved action policy is read-only; this board test does not pretend to start or qualify Morgan's next runtime.");
+  return c.result();
+}
+export const appTestExternalCases = [definition("app-permissions.test-as-agent", async ctx => {
+  const marker = `Synthetic sample page ${randomBytes(16).toString("hex")}`, provider = await startNotesProvider(marker); providers.set(ctx, provider);
+  try {
+    const connection = await connectNotes(ctx, provider, "read-only"), morgan = await identityWorker(ctx);
+    await ctx.input.api.putSensitive(`/api/tool-connections/${connection.connectionId}/installs`, { installs: [{ targetType: "agent", targetId: morgan.id }] });
+    const effective = await ctx.input.api.get<ToolProfileEffectiveSummary>(`${root(ctx)}/tools/profiles/effective/agents/${morgan.id}`);
+    if (!effective.allowedTools.some(row => row.id === connection.read[0])) throw new Error("Setup blocker: actual Morgan effective access must include the synthetic sample read action");
+    const agentTestStatus = await probeAgentPermission(ctx, `/api/tool-connections/${connection.connectionId}/test-calls`, "POST", { agentId: morgan.id, toolName: "launch_read", parameters: {} });
+    await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
+    await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The selected connection is Synthetic Launch Notes (${connection.connectionId}), containing synthetic sample action Read launch page (launch_read). Morgan (${morgan.id}) is the intended test actor. The board has the supported Test action control. No board test has run yet.` });
+    return { syntheticProvider: true, connectionId: connection.connectionId, readEntryId: connection.read[0], writeEntryId: connection.write[0], morganId: morgan.id, marker, agentTestStatus, resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}/permissions` };
+  } catch (error) { providers.delete(ctx); await provider.close(); throw error; }
+}, gradeAppTest, {
+  authentication: "human-session", sourceFiles: ["chat-catalog-domains/external-app-test.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/app-permissions.yaml"],
+  async act(ctx, state) {
+    await waitForReply(ctx); const provider = providers.get(ctx)!; state.providerCallsBeforeBoard = provider.calls.filter(row => row.method === "tools/call").length;
+    const o = await observeExternal(ctx, state); if (!appTestGuidance(o.reply)) throw new Error("Chat must identify Morgan and the requested read action before the board test");
+    const prefix = `/${ctx.input.fixtures.company.issuePrefix}/apps/${state.connectionId}`;
+    const href = o.links.find(link => { try { const url = new URL(link, o.origin); return url.origin === o.origin && (url.pathname === prefix || url.pathname.startsWith(`${prefix}/`)); } catch { return false; } });
+    if (!href) throw new Error("Chat did not provide the supported connection test surface");
+    const source = ctx.input.page.url(); await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" });
+    if (!ctx.input.page.url().endsWith("/permissions")) await ctx.input.page.goto(String(state.resultPath), { waitUntil: "domcontentloaded" });
+    await ctx.input.page.locator(`[data-action-id=${JSON.stringify(String(state.readEntryId))}]`).getByRole("button", { name: "Test", exact: true }).click();
+    const dialog = ctx.input.page.getByRole("dialog"); await dialog.getByRole("button", { name: "Choose which agent to test as", exact: true }).click();
+    await dialog.getByRole("button", { name: /Morgan.*Website engineer/ }).click();
+    await expect(dialog.getByRole("button", { name: "Choose which agent to test as", exact: true })).toHaveText("Morgan");
+    await expect(dialog).toContainText("This runs a real call against Synthetic Launch Notes as Morgan."); state.visibleTestAsMorgan = true;
+    const responsePromise = ctx.input.page.waitForResponse(response => new URL(response.url()).pathname === `/api/tool-connections/${state.connectionId}/test-calls` && response.request().method() === "POST");
+    await dialog.getByRole("button", { name: "Run", exact: true }).click();
+    const response = await responsePromise, request = response.request().postDataJSON() as { agentId: string; toolName: string };
+    state.testRequestAgentId = request.agentId; state.testRequestToolName = request.toolName;
+    if (!response.ok()) throw new Error(`Actual board test returned HTTP ${response.status()}`); state.testResult = await response.json() as ToolConnectionTestCallResult;
+    await expect(dialog).toContainText(String(state.marker)); await expect(dialog).toContainText("Ran as Morgan"); state.visibleResult = true;
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  async observe(ctx, state) { const provider = providers.get(ctx)!; state.providerReadCalls = provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_read" && row.authorized).length; state.providerWriteCalls = provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_update").length; state.effective = await ctx.input.api.get<ToolProfileEffectiveSummary>(`${root(ctx)}/tools/profiles/effective/agents/${state.morganId}`); state.morganRuntimeRunCount = (await ctx.input.api.get<Array<{ agentId: string }>>(`${root(ctx)}/heartbeat-runs?agentId=${state.morganId}&limit=100`)).filter(row => row.agentId === state.morganId).length; return observeExternal(ctx, state); },
+  async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); },
+  async cleanup(ctx, state) { try { if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true }); } finally { const provider = providers.get(ctx); providers.delete(ctx); await provider?.close(); } },
+})];

tests/runner-e2e/chat-catalog-domains/external-definitions.ts

@@ -20,3 +20,4 @@ export { teamInstallExternalCases } from "./external-team-install.js";
 export { teamConfigureExternalCases } from "./external-team-configure.js";
 export { gatewayRotationExternalCases } from "./external-gateway-rotation.js";
 export { notionAssignmentExternalCases } from "./external-notion-assignment.js";
+export { appTestExternalCases } from "./external-app-test.js";