diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index f70a4350051..b3b59f07f31 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -830,7 +830,7 @@ "../../ui/src/components/task-chat/TaskChatComposer.tsx", "chat-scenarios/app-permissions.yaml" ], - "oracleVersion": "paperclip.chat-catalog.app-review.v3", + "oracleVersion": "paperclip.chat-catalog.app-review.v4", "expectedRunCount": 6, "expectedHumanMessageCount": 3, "minimumRunCount": 6, diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts index c61027068a7..9e71bc0c57c 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review-navigation.test.ts @@ -4,7 +4,7 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js"; vi.mock("../chat-flow.js",()=>({sendChatMessage:vi.fn()})); vi.mock("@playwright/test",()=>({expect:(node:any)=>({toBeVisible:async()=>{if(!node.visible())throw Error("Missing selected UI");},toContainText:async(text:string)=>{if(!node.text.includes(text))throw Error("Wrong selected action");}})})); vi.mock("./external-provider.js",()=>({startNotesProvider:async()=>({calls:[],failWrites:vi.fn(),close:vi.fn()}),connectNotes:async()=>({connectionId:"connection",write:["write"]})})); -vi.mock("./identity-support.js",()=>({openStandingChat:async()=>({id:"source"}),probeAgentPermission:async()=>403})); +vi.mock("./identity-support.js",()=>({openStandingChat:async()=>({id:"source"}),probeAgentPermission:async()=>403,cleanupIdentityProbes:vi.fn()})); vi.mock("../chat-query-attachments.js",()=>({uploadFactualQueryAttachment:async()=>({issueId:"source"}),chatQueryWithAttachments:(q:string)=>q,verifyChatQueryAttachments:async()=>true})); vi.mock("./external-app-review-probe.js", () => ({ probeReviewApproval: vi.fn(async () => ({status: 403, code: "interaction_governed_action_denied", boundaryUnchanged: true, keyRevoked: true})) })); import { probeReviewApproval } from "./external-app-review-probe.js"; @@ -12,6 +12,7 @@ import { appReviewExternalCases } from "./external-app-review.js"; afterEach(()=>vi.restoreAllMocks()); function fixture(fault?:"wrong preview"|"missing takeover"|"wrong app"|"wrong action"|"missing disclosure"|"wrong full body"|"wrong hash"|"wrong date"){ let opened=false,declined=false;const events:string[]=[]; + let runtimeConfig={retained:true,heartbeat:{enabled:false,maxDailyRuns:3,maxConcurrentRuns:1,maxDailyCostCents:500}}; const proposalText = "# Launch update\n\n" + "Owned destination context. ".repeat(7) + "The public launch is November 8, 2030."; const argumentsSummaryJson = JSON.stringify({text: fault === "wrong date" ? proposalText.replace("November 8", "November 9") : proposalText}); const argumentsHash = createHash("sha256").update(argumentsSummaryJson).digest("hex"); @@ -26,9 +27,9 @@ function fixture(fault?:"wrong preview"|"missing takeover"|"wrong app"|"wrong ac }}}; const page={locator:(selector:string)=>{events.push(selector);return timeline;},getByTestId:(id:string)=>{expect(id).toBe("task-chat-composer-takeover");return review;}}; const ctx={scenario:{id:"app-permissions.ask-before-action",query:"Add this launch update to our notes app."},input:{page,fixtures:{company:{id:"company"},agent:{id:"ceo"}},api:{ - get:async(path:string)=>path==="/api/auth/get-session"?{user:{id:"person"}}:path.endsWith("/tools/trust-rules")?{trustRules:[]}:path.endsWith("/interactions")?[card]:path.endsWith("/runs/source-native/decisions")?{runId:"source-native",decisions:[{invocation:{id:"invocation",companyId:"company",agentId:"ceo",issueId:"source",runId:"source-native",connectionId:"connection",catalogEntryId:"write",argumentsHash,status:"awaiting_approval"},actionRequest:{id:"action",status:"pending",canonicalArgumentsHash:argumentsHash}}]}:Promise.reject(Error("Unexpected fixture read")), - post:async()=>({id:"project"}),patch:async()=>({}),},evidence:async()=>{}},refreshIssue:async()=>{},issue:()=>({id:"source",conversationState:"waiting"}),allRuns:async()=>[{id:"source-native",companyId:"company",agentId:"ceo",status:"succeeded",contextSnapshot:{issueId:"source"}}]} as unknown as ChatCatalogContext; - const definition=appReviewExternalCases[0]!;return{ctx,definition,events,declined:()=>declined}; + get:async(path:string)=>path==="/api/agents/ceo"?{runtimeConfig}:path==="/api/auth/get-session"?{user:{id:"person"}}:path.endsWith("/tools/trust-rules")?{trustRules:[]}:path.endsWith("/interactions")?[card]:path.endsWith("/runs/source-native/decisions")?{runId:"source-native",decisions:[{invocation:{id:"invocation",companyId:"company",agentId:"ceo",issueId:"source",runId:"source-native",connectionId:"connection",catalogEntryId:"write",argumentsHash,status:"awaiting_approval"},actionRequest:{id:"action",status:"pending",canonicalArgumentsHash:argumentsHash}}]}:Promise.reject(Error("Unexpected fixture read")), + post:async()=>({id:"project"}),patch:async(path:string,body:any)=>{if(path==="/api/agents/ceo")runtimeConfig=body.runtimeConfig;return{};},},evidence:async()=>{}},refreshIssue:async()=>{},issue:()=>({id:"source",conversationState:"waiting"}),allRuns:async()=>[{id:"source-native",companyId:"company",agentId:"ceo",status:"succeeded",contextSnapshot:{issueId:"source"}}]} as unknown as ChatCatalogContext; + const definition=appReviewExternalCases[0]!;return{ctx,definition,events,declined:()=>declined,runtimeConfig:()=>runtimeConfig}; } it("the actual approval caller opens its selected timeline request and decides only in the real composer takeover",async()=>{ const f=fixture(),state=await f.definition.setup(f.ctx); @@ -41,3 +42,15 @@ it.each(["wrong preview","missing takeover","wrong app","wrong action","missing const f=fixture(fault),state=await f.definition.setup(f.ctx); await expect(f.definition.act!(f.ctx,state)).rejects.toThrow();expect(f.declined()).toBe(false); }); + +it("provisions the declared six native review turns while retaining finite cost and concurrency limits",async()=>{ + const f=fixture();await f.definition.setup(f.ctx); + expect(f.definition.expectedRunCount).toBe(6); + expect(f.runtimeConfig()).toEqual({retained:true,heartbeat:{enabled:false,maxDailyRuns:6,maxConcurrentRuns:1,maxDailyCostCents:500}}); +}); + +it("does not run the review journey when the six-turn fixture allowance was not saved",async()=>{ + const f=fixture();vi.spyOn(f.ctx.input.api,"patch").mockResolvedValue({}); + await expect(f.definition.setup(f.ctx)).rejects.toThrow("declared six-turn daily allowance"); + expect(f.runtimeConfig().heartbeat.maxDailyRuns).toBe(3); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts index 49af0878e89..18b27a61ef6 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts @@ -299,6 +299,21 @@ export const appReviewExternalCases = [ try { const connection = await connectNotes(ctx, provider, "ask-first"); provider.failWrites(); + // Three human intents each need a proposal turn and an outcome turn. + // Keep the daily admission gate finite and consistent with this case. + const agentPath = `/api/agents/${ctx.input.fixtures.agent.id}`; + const agent = await ctx.input.api.get<{ + runtimeConfig: { heartbeat?: Record; [key: string]: unknown }; + }>(agentPath); + await ctx.input.api.patch(agentPath, { + runtimeConfig: { + ...agent.runtimeConfig, + heartbeat: { ...agent.runtimeConfig.heartbeat, maxDailyRuns: 6 }, + }, + }); + const savedAgent = await ctx.input.api.get(agentPath); + if (savedAgent.runtimeConfig.heartbeat?.maxDailyRuns !== 6) + throw new Error("Tool review fixture requires its declared six-turn daily allowance"); const project = await ctx.input.api.post<{ id: string }>( `${root(ctx)}/projects`, { name: "Launch notes review", status: "in_progress" }, @@ -347,7 +362,7 @@ export const appReviewExternalCases = [ gradeAppReview, { authentication: "human-session", - oracleVersion: "paperclip.chat-catalog.app-review.v3", + oracleVersion: "paperclip.chat-catalog.app-review.v4", queryAttachments: [{ stateKey: "factualContextAttachment", filename: "launch-update.md" }], expectedRunCount: 6, minimumRunCount: 6,