Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-04 19:28:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(evals): establish and preserve independent tree budget holds

Browse Source 
Use the existing positive-cap and explicitly synthetic accounting-import fixture pattern instead of zero, which disables enforcement. Preserve the held branch policy while independently verifying company, CEO and active branches remain funded; retain all real tree stop/pause/hold oracles.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 20:18:31 -06:00
1 parent 6e5a430118
commit 0697887371
2 files changed
+66 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-domains/runtime-tree-budget-fixture.test.ts
+48
View File
@@ -0,0 +1,48 @@
import { afterEach, expect, it, vi } from "vitest";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
const owner=vi.hoisted(()=>({id:"manual-worker",companyId:"company"}));
vi.mock("./runtime-support.js", async importOriginal => ({ ...await importOriginal<typeof import("./runtime-support.js")>(), runtimeWorker: vi.fn(async () => ({...owner})) }));
import { runtimeTreeCases } from "./runtime-tree.js";
afterEach(() => { vi.useRealTimers();owner.id="manual-worker";owner.companyId="company"; });
it.each([[0,true],[1,true],[0,false]] as const)("requires real positive-cap independent budget prerequisite for tree case%s (hold=%s)",async(index,budgetHeld)=>{
vi.useFakeTimers();vi.setSystemTime(100);
const events:string[]=[];let imported=false,positiveCap=false;
const api={post:vi.fn(async(route:string,body:Record<string,unknown>)=>{
if(route.endsWith("/issues"))return {id:body.parentId?"manual-child":"parent"};
if(route.endsWith("/pause")){events.push("manual-pause");return {};}
if(route.endsWith("/budgets/policies")){positiveCap=Number(body.amount)>0;expect(body.amount).toBe(100);expect(body.scopeId).toBe("manual-worker");events.push("positive-policy");return {};}
if(route.endsWith("/cost-events")){
expect(body).toMatchObject({agentId:"manual-worker",provider:"synthetic-eval-import",biller:"synthetic-eval-import",model:"imported-fixture-receipt",costStatus:"reported",costCents:125,inputTokens:0,outputTokens:0});
expect(body.heartbeatRunId).toBeUndefined();imported=true;events.push("synthetic-accounting-import");return {id:"receipt"};
}
if(route.endsWith("/tree-holds")){events.push("actual-budget-observed");throw new Error("stop-after-budget-prerequisite");}
throw new Error("Unexpected fixture mutation");
}),get:vi.fn(async()=>({status:"paused",lifecycleHolds:budgetHeld&&positiveCap&&imported?["manual","budget"]:["manual"]}))};
const evidence=vi.fn(async(_file:string,value:{modelProviderUsage:string;importedFixtureCostCents:number;hardCapCents:number})=>{
expect(value.modelProviderUsage).toContain("Not model/provider usage");expect(value.importedFixtureCostCents).toBe(125);expect(value.hardCapCents).toBe(100);events.push("accounting-provenance");
});
const ctx={scenario:{id:runtimeTreeCases[index]!.scenarioId},observationDeadlineAt:900000,input:{nonce:"fixture",fixtures:{company:{id:"company"},agent:{id:"ceo"}},api,evidence}} as unknown as ChatCatalogContext;
const result=expect(runtimeTreeCases[index]!.setup(ctx)).rejects.toThrow(budgetHeld?"stop-after-budget-prerequisite":"Timed out waiting");
await vi.advanceTimersByTimeAsync(46000);await result;
expect(events).toEqual(["manual-pause","positive-policy","synthetic-accounting-import","accounting-provenance",...(budgetHeld?["actual-budget-observed"]:[])]);
expect(api.post.mock.calls.filter(([route])=>route.includes("cost-events"))).toHaveLength(1);
expect(api.post.mock.calls.filter(([route])=>route.includes("budgets/policies"))).toHaveLength(1);
});
it.each(["foreign company","source CEO"])("rejects %s as the independent held worker",async invalid=>{
if(invalid==="foreign company")owner.companyId="foreign";else owner.id="ceo";
const post=vi.fn(async(_route:string,_body:unknown)=>({id:"parent"}));const ctx={input:{fixtures:{company:{id:"company"},agent:{id:"ceo"}},api:{post}}} as unknown as ChatCatalogContext;
await expect(runtimeTreeCases[0]!.setup(ctx)).rejects.toThrow("distinct same-company worker");
expect(post).toHaveBeenCalledOnce();expect(post.mock.calls[0]?.[0]).not.toContain("budgets");
});
it("preserves the independent policy and rejects starving any active branch",async()=>{
expect(runtimeTreeCases[0]!.setupPolicy?.funding).toBe("preserve-scenario");
const policy=(scopeType:string,scopeId:string,amount:number)=>({scopeType,scopeId,amount,isActive:true,hardStopEnabled:true,unpricedUsagePolicy:"allow"});
let holdPresent=true;
const policies=[policy("company","company",500),...['ceo','first','second'].map(id=>policy("agent",id,500)),policy("agent","manual",100)];
const state={worker:{id:"first"},workerRunId:"first-run",workerWasReallyWaiting:true,second:{worker:{id:"second"},workerRunId:"second-run",workerWasReallyWaiting:true},manualWorker:{id:"manual"},manualChild:{id:"manual-child"},manualHold:{id:"hold"}};
const ctx={input:{fixtures:{company:{id:"company"},agent:{id:"ceo"}},api:{get:async(route:string)=>route.endsWith("/budgets/overview")?{policies}:route.endsWith("/tree-control/state")?{activePauseHold:{holdId:"hold"}}:route==="/api/agents/ceo"?{lifecycleState:"ready",status:"idle"}:{id:"manual",status:"paused",lifecycleHolds:holdPresent?["manual","budget"]:["manual"]}}},allRuns:async()=>[{id:"first-run",agentId:"first",status:"running",companyId:"company"},{id:"second-run",agentId:"second",status:"running",companyId:"company"}]} as unknown as ChatCatalogContext;
expect((await runtimeTreeCases[0]!.verifySetup!(ctx,state)).passed).toBe(true);
holdPresent=false;expect((await runtimeTreeCases[0]!.verifySetup!(ctx,state)).passed).toBe(false);holdPresent=true;
for(const p of policies){const before=p.amount;p.amount=0;expect((await runtimeTreeCases[0]!.verifySetup!(ctx,state)).passed).toBe(false);p.amount=before;}
});
tests/runner-e2e/chat-catalog-domains/runtime-tree.ts
+18 -4
View File
@@ -39,30 +39,44 @@ export function gradeTreePause(value: unknown) {
async function setupTree(ctx: Parameters<typeof busyRuntimeWorker>[0]): Promise<State> {
const parent = await ctx.input.api.post<Entity>(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Launch", status: "backlog", assigneeAgentId: ctx.input.fixtures.agent.id });
const manualWorker = await runtimeWorker(ctx, "Jamie");
if (manualWorker.companyId !== ctx.input.fixtures.company.id || manualWorker.id === ctx.input.fixtures.agent.id) throw new Error("Independent tree budget fixture requires a distinct same-company worker");
const manualChild = await ctx.input.api.post<Entity>(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Launch copy", status: "backlog", parentId: parent.id, assigneeAgentId: manualWorker.id });
await ctx.input.api.post(`/api/agents/${manualWorker.id}/pause`, {});
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: manualWorker.id,
amount: 0, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "block" });
amount: 100, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "allow" });
// A zero amount disables enforcement. Seed an explicit synthetic accounting
// fixture through the public import path; this is not model/provider spend.
const budgetReceipt = await ctx.input.api.post<{ id: string }>(`/api/companies/${ctx.input.fixtures.company.id}/cost-events`, {
agentId: manualWorker.id, provider: "synthetic-eval-import", biller: "synthetic-eval-import", model: "imported-fixture-receipt",
billingType: "metered_api", costStatus: "reported", costCents: 125, inputTokens: 0, outputTokens: 0,
occurredAt: new Date().toISOString(), idempotencyKey: `tree-held-prerequisite:${ctx.input.nonce}` });
if (typeof budgetReceipt.id !== "string" || !budgetReceipt.id) throw new Error("Missing public synthetic accounting fixture receipt");
await ctx.input.evidence("runtime-tree-imported-budget-context.json", { agentId: manualWorker.id, receiptId: budgetReceipt.id,
origin: "public-imported-synthetic-fixture", importedFixtureCostCents: 125, hardCapCents: 100,
modelProviderUsage: "Not model/provider usage; explicit public synthetic accounting fixture" });
await pollUntil({ label: "Real independent budget hold on the manually paused child owner", deadlineAt: Date.now() + 45_000,
load: () => ctx.input.api.get<RuntimeAgent>(`/api/agents/${manualWorker.id}`), accept: row => row.status === "paused" && row.lifecycleHolds?.includes("budget") === true });
const manual = await ctx.input.api.post<{ hold: IssueTreeHold }>(`/api/issues/${manualChild.id}/tree-holds`, { mode: "pause", reason: "Independent human copy pause", releasePolicy: { strategy: "manual" } });
const first = await busyRuntimeWorker(ctx), riley = await runtimeWorker(ctx, "Riley"), second = await busyRuntimeWorker(ctx, { worker: riley });
for (const state of [first, second]) await ctx.input.api.patch(`/api/issues/${state.task.id}`, { parentId: parent.id });
const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
return { ...first, second, parent, manualWorker, manualChild, manualHold: manual.hold, humanId: session.user.id };
return { ...first, second, parent, manualWorker, manualChild, manualHold: manual.hold, humanId: session.user.id, syntheticBudgetReceiptId: budgetReceipt.id, syntheticAccountingCostCents: 125, syntheticAccountingOrigin: "public-imported-synthetic-fixture" };
}
export const runtimeTreeCases = ["task-coordination.pause-tree", "steering.stop-versus-pause"].map(id => runtimeDefinition(id, setupTree, gradeTreePause, {
authentication: "human-session", expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 2 } }, attemptTimeoutMs: 900_000,
sourceFiles: ["chat-catalog-domains/runtime-tree.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", `chat-scenarios/${id.split(".")[0]}.yaml`],
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "two-real-native-workers", "independent-child-pause-budget"] },
setupPolicy: { readiness: "preserve-scenario", funding: "preserve-scenario", invariantCheckIds: ["lead-ready", "two-real-native-workers", "independent-child-pause-budget", "active-branches-funded"] },
async verifySetup(ctx, state) {
const c = checks(), lead = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`), runs = await ctx.allRuns();
c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual CEO is ready to answer the canonical query.");
c.add("two-real-native-workers", [state, state.second as State].every(s => s.workerWasReallyWaiting === true && runs.some(row => row.id === s.workerRunId
&& row.agentId === (s.worker as Entity).id && row.status === "running" && row.companyId === ctx.input.fixtures.company.id)), "Two actual native workers are independently active before chat.");
const funding = await ctx.input.api.get<{ policies: Array<{ scopeType: string; scopeId: string; amount: number; isActive: boolean; hardStopEnabled: boolean; unpricedUsagePolicy: string }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`);
const activeScopes = [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[ctx.input.fixtures.agent.id, (state.worker as Entity).id, ((state.second as State).worker as Entity).id].map(scopeId => ({ scopeType: "agent", scopeId }))];
c.add("active-branches-funded", activeScopes.every(scope => funding.policies.some(policy => policy.scopeType === scope.scopeType && policy.scopeId === scope.scopeId && policy.amount === 500 && policy.isActive && policy.hardStopEnabled && policy.unpricedUsagePolicy === "allow")), "The real company, CEO and both active branches keep their positive execution budgets while the independent branch remains held.");
const manual = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${(state.manualWorker as Entity).id}`);
const hold = await ctx.input.api.get<TreeState>(`/api/issues/${(state.manualChild as Entity).id}/tree-control/state`);
c.add("independent-child-pause-budget", manual.status === "paused" && manual.lifecycleHolds?.includes("budget") === true && hold.activePauseHold?.holdId === (state.manualHold as IssueTreeHold).id,
c.add("independent-child-pause-budget", manual.status === "paused" && manual.lifecycleHolds?.includes("budget") === true && funding.policies.some(policy => policy.scopeType === "agent" && policy.scopeId === manual.id && policy.amount === 100 && policy.isActive && policy.hardStopEnabled) && hold.activePauseHold?.holdId === (state.manualHold as IssueTreeHold).id,
"The independent manual leaf pause and effective budget hold are genuine saved prerequisites."); return c.result();
},
async act(ctx, state) {