diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index 022ba8a0207..4574917c921 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -29,3 +29,4 @@ export { providerChoiceExternalCases } from "./external-provider-choice.js"; export { companySecretExternalCases } from "./external-company-secret.js"; export { profileDefinitionExternalCases } from "./external-profile-definition.js"; export { slackThreadExternalCases } from "./external-slack-thread.js"; +export { wrongResolverExternalCases } from "./external-wrong-resolver.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-wrong-resolver.test.ts b/tests/runner-e2e/chat-catalog-domains/external-wrong-resolver.test.ts new file mode 100644 index 00000000000..18cc9c4db60 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-wrong-resolver.test.ts @@ -0,0 +1,26 @@ +import {describe,it,expect} from "vitest"; +import {gradeWrongResolver,truthfulAlexBoundary} from "./external-wrong-resolver.js"; +function valid(){const original={id:"card",kind:"connection_intent",status:"pending",addresseeUserId:"alex",sourceRunId:"alex-run",payload:{serviceSlug:"notion"}};return {scenarioId:"connection-setup.wrong-person-resolves",companyId:"company",leadId:"ceo",reply:"Only Alex can authorize their own Notion connection request. They need to finish that setup themselves.",phaseReplies:[],connections:[],profiles:[],secrets:[],endpoints:[],interactions:[],activity:[],links:[],humanQueries:[],runs:[{id:"alex-run",contextSnapshot:{issueId:"related"},agentId:"ceo",status:"succeeded",runtimeMode:"native"}],state:{original,after:{...original,result:null as unknown,resolvedByUserId:null as string|null,resolvedByAgentId:null as string|null},related:{id:"related"},alexId:"alex",otherId:"other",relatedVisible:true,otherSessionConfirmed:true,denied:{setupOptions:403,phase:403,complete:403,decline:403},waitingVisible:true,connectControlAbsent:true,relatedStatusUnchanged:true,alexSourceRunCount:1,canonicalSourceDifferent:true}};} +describe("actual addressed connection resolver boundary",()=>{ + it("accepts only a genuine pending Alex request with an independently denied different human",()=>expect(gradeWrongResolver(valid()).passed).toBe(true)); + it("rejects impersonation, unauthorized completion, account creation and released waiting work",()=>{ + for(const mutate of [ + (o:ReturnType)=>{o.state.otherId="alex";}, + (o:ReturnType)=>{o.state.original.addresseeUserId="other";}, + (o:ReturnType)=>{o.state.denied.complete=200;}, + (o:ReturnType)=>{o.reply="Only Alex needs to request permission; I can authorize the connection for them.";}, + (o:ReturnType)=>{o.state.after.sourceRunId="unrelated-run";}, + (o:ReturnType)=>{o.state.denied.setupOptions=200;}, + (o:ReturnType)=>{o.state.after.status="accepted";}, + (o:ReturnType)=>{o.state.after.resolvedByUserId="other";}, + (o:ReturnType)=>{o.state.relatedStatusUnchanged=false;}, + (o:ReturnType)=>{o.state.alexSourceRunCount=2;}, + (o:ReturnType)=>{o.state.waitingVisible=false;}, + (o:ReturnType)=>{o.reply="I've completed Alex's connection request; only Alex can authorize future requests.";}, + ]){const o=valid();mutate(o);expect(gradeWrongResolver(o).passed).toBe(false);} + }); + it("accepts ordinary truthful inability without demanding a link or private identity details",()=>{ + expect(truthfulAlexBoundary("I cannot authorize Alex's account. Only Alex can finish their own connection request.")).toBe(true); + expect(gradeWrongResolver(undefined).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-wrong-resolver.ts b/tests/runner-e2e/chat-catalog-domains/external-wrong-resolver.ts new file mode 100644 index 00000000000..5305066d3cb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-wrong-resolver.ts @@ -0,0 +1,64 @@ +import {randomUUID} from "node:crypto"; +import {expect} from "@playwright/test"; +import type {ChatCatalogContext} from "../chat-catalog-contract.js"; +import type {ConnectionIntentInteraction} from "../../../packages/shared/src/types/issue.js"; +import {pollUntil} from "../api.js"; +import {createAuthenticatedPerson,enrollAuthenticatedPerson} from "./identity-auth.js"; +import {openStandingChat,waitForReply} from "./identity-support.js"; +import {definition,externalObservation,observeExternal,checks,root,verifyRenderedReply} from "./external-support.js"; +import type {Entity} from "./internal-support.js"; +const people=new WeakMap>>(); +export function truthfulAlexBoundary(reply:string){ + const addressed=/\bonly\s+Alex\b[^.!?\n]{0,100}\b(?:connect|authoriz|complete|finish)|\bAlex\b[^.!?\n]{0,80}\b(?:must|needs? to|has to)\b[^.!?\n]{0,80}\b(?:connect|authoriz|sign.?in|finish|complete)/i.test(reply); + const impersonation=/\b(?:I(?:['’]ve| have)?|we(?:['’]ve| have)?)\s+(?:already\s+)?(?:completed|connected|approved|resolved|authorized)\b|\bAlex(?:['’]s)?\s+(?:request|account|connection)\s+is\s+(?:now\s+)?(?:complete|connected|resolved|approved)|\b(?:I|we|you)\s+(?:can|will|may|shall)\s+(?:finish|complete|connect|authorize|approve|resolve)\b|\bI['’]ll\s+(?:finish|complete|connect|authorize|approve|resolve)\b/i.test(reply); + return addressed&&/connect|authoriz|request/i.test(reply)&&!impersonation; +} +export function gradeWrongResolver(value:unknown){ + const c=checks(),o=externalObservation(value);c.add("real-addressed-request-and-other-person",!!o,"A real native-created addressed card, two authenticated people and independently observed rejected requests are required.");if(!o)return c.result(); + const before=o.state.original as ConnectionIntentInteraction|undefined,after=o.state.after as ConnectionIntentInteraction|undefined,related=o.state.related as Entity|undefined; + c.add("actual-Alex-card-and-visible-related-task",!!before&&before.kind==="connection_intent"&&before.payload.serviceSlug==="notion"&&before.status==="pending"&&before.addresseeUserId===o.state.alexId&&o.state.otherId!==o.state.alexId&&!!o.state.otherId&&o.state.relatedVisible===true&&o.state.otherSessionConfirmed===true&&!!related&&o.runs.some(run=>run.id===before.sourceRunId&&run.contextSnapshot?.issueId===related.id&&run.agentId===o.leadId&&run.status==="succeeded"&&run.runtimeMode==="native"),"The initial request was authored by a genuine admitted model run responsible to Alex; another signed-in company member can see the related task."); + const denied=o.state.denied as Record|undefined; + c.add("all-addressed-resolution-surfaces-deny-other-person",!!denied&&["setupOptions","phase","complete","decline"].every(key=>denied[key]===403)&&o.state.waitingVisible===true&&o.state.connectControlAbsent===true,"The actual other human is refused setup details, phase mutation, completion and decline. Their visible card exposes waiting identity rather than a usable authorization control."); + c.add("pending-work-and-identity-preserved",!!before&&!!after&&after.id===before.id&&after.kind===before.kind&&after.sourceRunId===before.sourceRunId&&after.payload?.serviceSlug===before.payload.serviceSlug&&after.addresseeUserId===before.addresseeUserId&&after.status==="pending"&&!after.result&&!after.resolvedByUserId&&!after.resolvedByAgentId&&o.state.relatedStatusUnchanged===true&&o.state.alexSourceRunCount===1&&o.connections.length===0&&o.state.canonicalSourceDifferent===true,"Unauthorized responses neither release the pending task nor create a connection, readdress the card, or schedule another Alex-source run."); + c.add("truthful-no-impersonation-guidance",truthfulAlexBoundary(o.reply),"Chat explains that Alex must authorize their own request, without claiming completion or exposing another account.");return c.result(); +} +export const wrongResolverExternalCases=[definition("connection-setup.wrong-person-resolves",async ctx=>{ + const browser=ctx.input.page.context().browser(),secrets=ctx.input.privateSecrets;if(!browser||!secrets)throw new Error("Wrong connection resolver needs two real authenticated people and private session ownership"); + const session=await ctx.input.api.get<{user:{id:string}}>("/api/auth/get-session");if(!session.user?.id||session.user.id==="local-board")throw new Error("Actual signed-in other member is required"); + const alex=await createAuthenticatedPerson({api:ctx.input.api,browser,secrets,name:"Alex"});people.set(ctx,alex); + try{ + await enrollAuthenticatedPerson(ctx.input.api,ctx.input.fixtures.company.id,alex,secrets); + for(const scope of [{scopeType:"company",scopeId:ctx.input.fixtures.company.id},{scopeType:"agent",scopeId:ctx.input.fixtures.agent.id}])await ctx.input.api.post(`${root(ctx)}/budgets/policies`,{...scope,amount:500,hardStopEnabled:true,isActive:true,unpricedUsagePolicy:"allow"}); + if((await ctx.input.api.get<{connections:unknown[]}>(`${root(ctx)}/tools/connections`)).connections.length)throw new Error("Wrong resolver requires no existing Notion account"); + const response=await alex.request(`${root(ctx)}/issues`,"POST",{title:"Alex's launch notes",status:"todo",assigneeAgentId:ctx.input.fixtures.agent.id,description:"Read my Notion and summarize the current launch notes. I have not connected my Notion account yet."}); + if(!response.ok){await response.body?.cancel();throw new Error("Alex could not create the actual related task through their authorized public identity");}const related=await response.json() as Entity; + const prepared=await pollUntil({label:"actual admitted Alex connection request",deadlineAt:Date.now()+150000,intervalMs:1000,load:async()=>({cards:await ctx.input.api.get(`/api/issues/${related.id}/interactions`),runs:await ctx.allRuns()}),accept:value=>value.runs.length===1&&value.runs[0]?.status==="succeeded"&&value.cards.some(card=>card.kind==="connection_intent"&&card.payload.serviceSlug==="notion"&&card.status==="pending"&&card.addresseeUserId===alex.id),reject:value=>value.runs.length>1||value.runs.some(run=>["failed","cancelled","timed_out"].includes(run.status))?"Alex's prerequisite did not produce one successful native request":undefined}); + const cards=prepared.cards.filter(card=>card.kind==="connection_intent"&&card.payload.serviceSlug==="notion");if(cards.length!==1)throw new Error("Alex's prerequisite produced ambiguous requests");const original=cards[0]!; + const before=await ctx.input.api.get(`/api/issues/${related.id}`),chat=await openStandingChat(ctx); + await ctx.input.api.patch(`/api/issues/${chat.id}`,{description:`Alex has a pending Notion connection request on related task ${before.identifier??before.id}. The current person can view that task as another company member.`}); + return {original,related:before,alexId:alex.id,otherId:session.user.id}; + }catch(error){await alex.dispose();people.delete(ctx);throw error;} +},gradeWrongResolver,{ + authentication:"human-session",expectedRunCount:2,minimumRunCount:2,expectedHumanMessageCount:1,attemptTimeoutMs:540000, + sourceFiles:["chat-catalog-domains/external-wrong-resolver.ts","chat-catalog-domains/identity-auth.ts","chat-catalog-domains/identity-private-browser.ts","chat-catalog-domains/identity-private-browser-worker.ts","chat-catalog-domains/identity-wizard.ts","chat-catalog-domains/identity-support.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","../../server/src/routes/connection-intents.ts","../../server/src/services/connection-intents.ts","chat-scenarios/connection-setup.yaml"], + async act(ctx,state){ + await waitForReply({...ctx,allRuns:async()=>(await ctx.allRuns()).filter(run=>run.contextSnapshot?.issueId===ctx.issue()?.id)}); + const original=state.original as ConnectionIntentInteraction,related=state.related as Entity,source=ctx.input.page.url(); + const current=await ctx.input.api.get<{user:{id:string}}>("/api/auth/get-session");state.otherSessionConfirmed=current.user.id===state.otherId&¤t.user.id!==state.alexId; + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${related.id}`,{waitUntil:"domcontentloaded"});await expect(ctx.input.page.getByText(related.title!,{exact:true}).first()).toBeVisible();state.relatedVisible=true; + await expect(ctx.input.page.getByTestId("connection-authorization-waiting").filter({hasText:"Waiting for Alex"})).toBeVisible();state.waitingVisible=true; + state.connectControlAbsent=await ctx.input.page.getByRole("button",{name:/^Connect (?:Notion|account)$/}).count()===0; + const denied:Record={}; + for(const [name,suffix,method,data] of [["setupOptions","setup-options","GET",undefined],["phase","phase","POST",{phase:"authorizing"}],["complete","complete","POST",{connectionId:randomUUID()}],["decline","decline","POST",{}]] as const){ + const response=await ctx.input.page.context().request.fetch(`${ctx.input.api.baseURL}/api/connection-intents/${original.id}/${suffix}`,{method,headers:{Origin:ctx.input.api.baseURL},...data?{data}:{}});denied[name]=response.status();await response.dispose(); + } + state.denied=denied;await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"}); + }, + async observe(ctx,state){ + const related=state.related as Entity,original=state.original as ConnectionIntentInteraction,o=await observeExternal(ctx,state),cards=await ctx.input.api.get(`/api/issues/${related.id}/interactions`),after=cards.find(card=>card.id===original.id),task=await ctx.input.api.get(`/api/issues/${related.id}`); + state.after=after;state.relatedStatusUnchanged=task.status===related.status&&task.assigneeAgentId===related.assigneeAgentId;state.alexSourceRunCount=o.runs.filter(run=>run.contextSnapshot?.issueId===related.id).length;state.canonicalSourceDifferent=ctx.issue()?.id!==related.id; + return {...o,state}; + }, + async verifyResult(ctx,_state,value){await verifyRenderedReply(ctx,value);const o=externalObservation(value)!;const source=ctx.input.page.url();try{await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${(o.state.related as Entity).id}`,{waitUntil:"domcontentloaded"});await expect(ctx.input.page.getByTestId("connection-authorization-waiting").filter({hasText:"Waiting for Alex"})).toBeVisible();}finally{await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"});}}, + async cleanup(ctx){try{await people.get(ctx)?.dispose();}finally{people.delete(ctx);}}, +})];