From 481722349ed0ec1fffca957f0ea8a538571016df Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 21:02:12 -0600 Subject: [PATCH] fix(evals): open scoped app review in composer takeover Co-Authored-By: Paperclip --- .../external-app-review-navigation.test.ts | 35 +++++++++++++++++++ .../external-app-review.ts | 11 +++++- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts new file mode 100644 index 00000000000..5b1b0e68b22 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts @@ -0,0 +1,35 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +vi.mock("../chat-flow.js",()=>({sendChatMessage:vi.fn()})); +vi.mock("@playwright/test",()=>({expect:(node:any)=>({toBeVisible:async()=>{if(!node.visible())throw Error("Missing selected UI");},toContainText:async(text:string)=>{if(!node.text.includes(text))throw Error("Wrong selected action");}})})); +vi.mock("./external-provider.js",()=>({startNotesProvider:async()=>({calls:[],failWrites:vi.fn(),close:vi.fn()}),connectNotes:async()=>({connectionId:"connection",write:["write"]})})); +vi.mock("./identity-support.js",()=>({openStandingChat:async()=>({id:"source"}),probeAgentPermission:async()=>403})); +vi.mock("../chat-query-attachments.js",()=>({uploadFactualQueryAttachment:async()=>({issueId:"source"}),chatQueryWithAttachments:(q:string)=>q,verifyChatQueryAttachments:async()=>true})); +import { appReviewExternalCases } from "./external-app-review.js"; +afterEach(()=>vi.restoreAllMocks()); +function fixture(fault?:"wrong preview"|"missing takeover"|"wrong app"|"wrong action"){ + let opened=false,declined=false;const events:string[]=[]; + const action={connectionId:"connection",actionRequestId:"action",invocationId:"invocation",appDisplayName:"Synthetic Launch Notes",toolName:"launch_update",toolDisplayName:"Update launch page",risk:"write",rememberActionScope:"Same agent, action, connection and project",previewMarkdown:"The public launch is November 8, 2030."}; + const card={id:"selected",kind:"request_confirmation",status:"pending",sourceRunId:"source-native",payload:{toolAction:action}}; + const timeline={visible:()=>fault!=="wrong preview",text:"Update launch page",getByRole:(_role:string,opts:{name:string})=>({click:async()=>{ + if(opts.name!=="Review request")throw Error("Timeline preview has no decision controls");events.push("open exact review");opened=true; + }})}; + const review={visible:()=>opened&&fault!=="missing takeover",text:fault==="wrong action"?"Read launch page":"Update launch page",getByRole:(role:string,opts:{name:string})=>role==="img"?{visible:()=>opened&&fault!=="wrong app"&&opts.name===action.appDisplayName}:{click:async()=>{ + if(!opened||opts.name!=="Decline")throw Error("Wrong decision target");declined=true;events.push("selected decline");throw Error("Stop after selected decline"); + }}}; + const page={locator:(selector:string)=>{events.push(selector);return timeline;},getByTestId:(id:string)=>{expect(id).toBe("task-chat-composer-takeover");return review;}}; + const ctx={scenario:{id:"app-permissions.ask-before-action",query:"Add this launch update to our notes app."},input:{page,fixtures:{company:{id:"company"},agent:{id:"ceo"}},api:{ + get:async(path:string)=>path==="/api/auth/get-session"?{user:{id:"person"}}:path.endsWith("/tools/trust-rules")?{trustRules:[]}:path.endsWith("/interactions")?[card]:Promise.reject(Error("Unexpected fixture read")), + post:async()=>({id:"project"}),patch:async()=>({}),},evidence:async()=>{}},refreshIssue:async()=>{},issue:()=>({id:"source",conversationState:"waiting"}),allRuns:async()=>[{id:"source-native",status:"succeeded",contextSnapshot:{issueId:"source"}}]} as unknown as ChatCatalogContext; + const definition=appReviewExternalCases[0]!;return{ctx,definition,events,declined:()=>declined}; +} +it("the actual approval caller opens its selected timeline request and decides only in the real composer takeover",async()=>{ + const f=fixture(),state=await f.definition.setup(f.ctx); + await expect(f.definition.act!(f.ctx,state)).rejects.toThrow("Stop after selected decline"); + expect(f.declined()).toBe(true);expect(f.events).toEqual(['[id="interaction-selected"]',"open exact review","selected decline"]); + expect(state.agentApproveStatus).toBe(403); +}); +it.each(["wrong preview","missing takeover","wrong app","wrong action"] as const)("does not decide a %s",async fault=>{ + const f=fixture(fault),state=await f.definition.setup(f.ctx); + await expect(f.definition.act!(f.ctx,state)).rejects.toThrow();expect(f.declined()).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts index a9a1d17225c..a1b3a26334e 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts @@ -370,6 +370,8 @@ export const appReviewExternalCases = [ "../../server/src/services/tool-action-delivery.ts", "../../server/src/services/tool-access-policy.ts", "../../ui/src/components/IssueThreadInteractionCard.tsx", + "../../ui/src/components/task-chat/TaskChatInteractionCard.tsx", + "../../ui/src/components/task-chat/TaskChatComposer.tsx", "chat-scenarios/app-permissions.yaml", ], async act(ctx, state) { @@ -445,10 +447,17 @@ export const appReviewExternalCases = [ toolName: action.toolName, applicationId: action.applicationId, }; - const view = ctx.input.page.locator( + const timeline = ctx.input.page.locator( `[id=${JSON.stringify(`interaction-${card.id}`)}]`, ); + await expect(timeline).toBeVisible(); + await expect(timeline).toContainText("Update launch page"); + // The timeline is a preview. Open this exact pending request before + // using the production decision controls in the composer takeover. + await timeline.getByRole("button", { name: "Review request", exact: true }).click(); + const view = ctx.input.page.getByTestId("task-chat-composer-takeover"); await expect(view).toBeVisible(); + await expect(view.getByRole("img", { name: action.appDisplayName || action.toolDisplayName, exact: true })).toBeVisible(); await expect(view).toContainText("Update launch page"); phase.reviewVisible = true; if (index === 0) {