diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-tree-budget-fixture.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-tree-budget-fixture.test.ts new file mode 100644 index 00000000000..df776c46926 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-tree-budget-fixture.test.ts @@ -0,0 +1,48 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const owner=vi.hoisted(()=>({id:"manual-worker",companyId:"company"})); +vi.mock("./runtime-support.js", async importOriginal => ({ ...await importOriginal(), runtimeWorker: vi.fn(async () => ({...owner})) })); +import { runtimeTreeCases } from "./runtime-tree.js"; +afterEach(() => { vi.useRealTimers();owner.id="manual-worker";owner.companyId="company"; }); +it.each([[0,true],[1,true],[0,false]] as const)("requires real positive-cap independent budget prerequisite for tree case%s (hold=%s)",async(index,budgetHeld)=>{ + vi.useFakeTimers();vi.setSystemTime(100); + const events:string[]=[];let imported=false,positiveCap=false; + const api={post:vi.fn(async(route:string,body:Record)=>{ + if(route.endsWith("/issues"))return {id:body.parentId?"manual-child":"parent"}; + if(route.endsWith("/pause")){events.push("manual-pause");return {};} + if(route.endsWith("/budgets/policies")){positiveCap=Number(body.amount)>0;expect(body.amount).toBe(100);expect(body.scopeId).toBe("manual-worker");events.push("positive-policy");return {};} + if(route.endsWith("/cost-events")){ + expect(body).toMatchObject({agentId:"manual-worker",provider:"synthetic-eval-import",biller:"synthetic-eval-import",model:"imported-fixture-receipt",costStatus:"reported",costCents:125,inputTokens:0,outputTokens:0}); + expect(body.heartbeatRunId).toBeUndefined();imported=true;events.push("synthetic-accounting-import");return {id:"receipt"}; + } + if(route.endsWith("/tree-holds")){events.push("actual-budget-observed");throw new Error("stop-after-budget-prerequisite");} + throw new Error("Unexpected fixture mutation"); + }),get:vi.fn(async()=>({status:"paused",lifecycleHolds:budgetHeld&&positiveCap&&imported?["manual","budget"]:["manual"]}))}; + const evidence=vi.fn(async(_file:string,value:{modelProviderUsage:string;importedFixtureCostCents:number;hardCapCents:number})=>{ + expect(value.modelProviderUsage).toContain("Not model/provider usage");expect(value.importedFixtureCostCents).toBe(125);expect(value.hardCapCents).toBe(100);events.push("accounting-provenance"); + }); + const ctx={scenario:{id:runtimeTreeCases[index]!.scenarioId},observationDeadlineAt:900000,input:{nonce:"fixture",fixtures:{company:{id:"company"},agent:{id:"ceo"}},api,evidence}} as unknown as ChatCatalogContext; + const result=expect(runtimeTreeCases[index]!.setup(ctx)).rejects.toThrow(budgetHeld?"stop-after-budget-prerequisite":"Timed out waiting"); + await vi.advanceTimersByTimeAsync(46000);await result; + expect(events).toEqual(["manual-pause","positive-policy","synthetic-accounting-import","accounting-provenance",...(budgetHeld?["actual-budget-observed"]:[])]); + expect(api.post.mock.calls.filter(([route])=>route.includes("cost-events"))).toHaveLength(1); + expect(api.post.mock.calls.filter(([route])=>route.includes("budgets/policies"))).toHaveLength(1); +}); + +it.each(["foreign company","source CEO"])("rejects %s as the independent held worker",async invalid=>{ + if(invalid==="foreign company")owner.companyId="foreign";else owner.id="ceo"; + const post=vi.fn(async(_route:string,_body:unknown)=>({id:"parent"}));const ctx={input:{fixtures:{company:{id:"company"},agent:{id:"ceo"}},api:{post}}} as unknown as ChatCatalogContext; + await expect(runtimeTreeCases[0]!.setup(ctx)).rejects.toThrow("distinct same-company worker"); + expect(post).toHaveBeenCalledOnce();expect(post.mock.calls[0]?.[0]).not.toContain("budgets"); +}); +it("preserves the independent policy and rejects starving any active branch",async()=>{ + expect(runtimeTreeCases[0]!.setupPolicy?.funding).toBe("preserve-scenario"); + const policy=(scopeType:string,scopeId:string,amount:number)=>({scopeType,scopeId,amount,isActive:true,hardStopEnabled:true,unpricedUsagePolicy:"allow"}); + let holdPresent=true; + const policies=[policy("company","company",500),...['ceo','first','second'].map(id=>policy("agent",id,500)),policy("agent","manual",100)]; + const state={worker:{id:"first"},workerRunId:"first-run",workerWasReallyWaiting:true,second:{worker:{id:"second"},workerRunId:"second-run",workerWasReallyWaiting:true},manualWorker:{id:"manual"},manualChild:{id:"manual-child"},manualHold:{id:"hold"}}; + const ctx={input:{fixtures:{company:{id:"company"},agent:{id:"ceo"}},api:{get:async(route:string)=>route.endsWith("/budgets/overview")?{policies}:route.endsWith("/tree-control/state")?{activePauseHold:{holdId:"hold"}}:route==="/api/agents/ceo"?{lifecycleState:"ready",status:"idle"}:{id:"manual",status:"paused",lifecycleHolds:holdPresent?["manual","budget"]:["manual"]}}},allRuns:async()=>[{id:"first-run",agentId:"first",status:"running",companyId:"company"},{id:"second-run",agentId:"second",status:"running",companyId:"company"}]} as unknown as ChatCatalogContext; + expect((await runtimeTreeCases[0]!.verifySetup!(ctx,state)).passed).toBe(true); + holdPresent=false;expect((await runtimeTreeCases[0]!.verifySetup!(ctx,state)).passed).toBe(false);holdPresent=true; + for(const p of policies){const before=p.amount;p.amount=0;expect((await runtimeTreeCases[0]!.verifySetup!(ctx,state)).passed).toBe(false);p.amount=before;} +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-tree.ts b/tests/runner-e2e/chat-catalog-domains/runtime-tree.ts index f936aba631b..1712676ab64 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-tree.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-tree.ts @@ -39,30 +39,44 @@ export function gradeTreePause(value: unknown) { async function setupTree(ctx: Parameters[0]): Promise { const parent = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Launch", status: "backlog", assigneeAgentId: ctx.input.fixtures.agent.id }); const manualWorker = await runtimeWorker(ctx, "Jamie"); + if (manualWorker.companyId !== ctx.input.fixtures.company.id || manualWorker.id === ctx.input.fixtures.agent.id) throw new Error("Independent tree budget fixture requires a distinct same-company worker"); const manualChild = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Launch copy", status: "backlog", parentId: parent.id, assigneeAgentId: manualWorker.id }); await ctx.input.api.post(`/api/agents/${manualWorker.id}/pause`, {}); await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: manualWorker.id, - amount: 0, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "block" }); + amount: 100, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "allow" }); + // A zero amount disables enforcement. Seed an explicit synthetic accounting + // fixture through the public import path; this is not model/provider spend. + const budgetReceipt = await ctx.input.api.post<{ id: string }>(`/api/companies/${ctx.input.fixtures.company.id}/cost-events`, { + agentId: manualWorker.id, provider: "synthetic-eval-import", biller: "synthetic-eval-import", model: "imported-fixture-receipt", + billingType: "metered_api", costStatus: "reported", costCents: 125, inputTokens: 0, outputTokens: 0, + occurredAt: new Date().toISOString(), idempotencyKey: `tree-held-prerequisite:${ctx.input.nonce}` }); + if (typeof budgetReceipt.id !== "string" || !budgetReceipt.id) throw new Error("Missing public synthetic accounting fixture receipt"); + await ctx.input.evidence("runtime-tree-imported-budget-context.json", { agentId: manualWorker.id, receiptId: budgetReceipt.id, + origin: "public-imported-synthetic-fixture", importedFixtureCostCents: 125, hardCapCents: 100, + modelProviderUsage: "Not model/provider usage; explicit public synthetic accounting fixture" }); await pollUntil({ label: "Real independent budget hold on the manually paused child owner", deadlineAt: Date.now() + 45_000, load: () => ctx.input.api.get(`/api/agents/${manualWorker.id}`), accept: row => row.status === "paused" && row.lifecycleHolds?.includes("budget") === true }); const manual = await ctx.input.api.post<{ hold: IssueTreeHold }>(`/api/issues/${manualChild.id}/tree-holds`, { mode: "pause", reason: "Independent human copy pause", releasePolicy: { strategy: "manual" } }); const first = await busyRuntimeWorker(ctx), riley = await runtimeWorker(ctx, "Riley"), second = await busyRuntimeWorker(ctx, { worker: riley }); for (const state of [first, second]) await ctx.input.api.patch(`/api/issues/${state.task.id}`, { parentId: parent.id }); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); - return { ...first, second, parent, manualWorker, manualChild, manualHold: manual.hold, humanId: session.user.id }; + return { ...first, second, parent, manualWorker, manualChild, manualHold: manual.hold, humanId: session.user.id, syntheticBudgetReceiptId: budgetReceipt.id, syntheticAccountingCostCents: 125, syntheticAccountingOrigin: "public-imported-synthetic-fixture" }; } export const runtimeTreeCases = ["task-coordination.pause-tree", "steering.stop-versus-pause"].map(id => runtimeDefinition(id, setupTree, gradeTreePause, { authentication: "human-session", expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 2 } }, attemptTimeoutMs: 900_000, sourceFiles: ["chat-catalog-domains/runtime-tree.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", `chat-scenarios/${id.split(".")[0]}.yaml`], - setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "two-real-native-workers", "independent-child-pause-budget"] }, + setupPolicy: { readiness: "preserve-scenario", funding: "preserve-scenario", invariantCheckIds: ["lead-ready", "two-real-native-workers", "independent-child-pause-budget", "active-branches-funded"] }, async verifySetup(ctx, state) { const c = checks(), lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`), runs = await ctx.allRuns(); c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual CEO is ready to answer the canonical query."); c.add("two-real-native-workers", [state, state.second as State].every(s => s.workerWasReallyWaiting === true && runs.some(row => row.id === s.workerRunId && row.agentId === (s.worker as Entity).id && row.status === "running" && row.companyId === ctx.input.fixtures.company.id)), "Two actual native workers are independently active before chat."); + const funding = await ctx.input.api.get<{ policies: Array<{ scopeType: string; scopeId: string; amount: number; isActive: boolean; hardStopEnabled: boolean; unpricedUsagePolicy: string }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`); + const activeScopes = [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[ctx.input.fixtures.agent.id, (state.worker as Entity).id, ((state.second as State).worker as Entity).id].map(scopeId => ({ scopeType: "agent", scopeId }))]; + c.add("active-branches-funded", activeScopes.every(scope => funding.policies.some(policy => policy.scopeType === scope.scopeType && policy.scopeId === scope.scopeId && policy.amount === 500 && policy.isActive && policy.hardStopEnabled && policy.unpricedUsagePolicy === "allow")), "The real company, CEO and both active branches keep their positive execution budgets while the independent branch remains held."); const manual = await ctx.input.api.get(`/api/agents/${(state.manualWorker as Entity).id}`); const hold = await ctx.input.api.get(`/api/issues/${(state.manualChild as Entity).id}/tree-control/state`); - c.add("independent-child-pause-budget", manual.status === "paused" && manual.lifecycleHolds?.includes("budget") === true && hold.activePauseHold?.holdId === (state.manualHold as IssueTreeHold).id, + c.add("independent-child-pause-budget", manual.status === "paused" && manual.lifecycleHolds?.includes("budget") === true && funding.policies.some(policy => policy.scopeType === "agent" && policy.scopeId === manual.id && policy.amount === 100 && policy.isActive && policy.hardStopEnabled) && hold.activePauseHold?.holdId === (state.manualHold as IssueTreeHold).id, "The independent manual leaf pause and effective budget hold are genuine saved prerequisites."); return c.result(); }, async act(ctx, state) {