test(evals): qualify budget prerequisites and company package upload

Preserve the original failures while distinguishing nonretryable native billing debt and stopping before a human action can mask unauthorized budget changes. Admit only the exact declared company-preview human upload with existing independent receipt checks.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { accountingIdentityCases } from "./identity-accounting.js";
+import { accountingIdentityCases, nativeAccountingPrerequisite } from "./identity-accounting.js";
 import type { IdentityObservation } from "./identity-support.js";
 function valid(scenarioId: string): IdentityObservation {
   const repair = scenarioId.endsWith("repair-cost-recording");
@@ -33,3 +33,24 @@ describe("actual saved native accounting contexts", () => {
   it("no hidden human receipt or missing phase may pass", () => { const o = valid("budgets-costs.repair-cost-recording"); o.humanQueries = []; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
   it("an actual agent retry bypass fails", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.agentRetryStatus = 200; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
 });
+
+describe("saved-run retry prerequisite classification", () => {
+  const run = { id: "original", status: "succeeded" as const, usageJson: { inputTokens: 10 } };
+  const item = { id: "original", runId: "original", agentId: "morgan", agentName: "Morgan", occurredAt: "2026-10-27T00:00:00Z", provider: "openai", model: "actual" };
+  const quality = { canRetry: true, runs: [{ ...item, reason: "retryable" as const }], events: [], hasMoreRuns: false, hasMoreEvents: false };
+  it("accepts genuine pending saved usage exposed by the public retry control", () => {
+    expect(nativeAccountingPrerequisite(run, quality, true)).toMatchObject({ eligible: true, disposition: "eligible", retryControlAvailable: true });
+  });
+  it("does not mistake acknowledged unknown billing for settled or retryable accounting", () => {
+    const q = { ...quality, runs: [], events: [{ ...item, reason: "unknown_billing" as const }] };
+    expect(nativeAccountingPrerequisite(run, q, true)).toMatchObject({ eligible: false, disposition: "unresolved-without-saved-run-retry", unresolvedReasons: ["unknown_billing"] });
+    expect(nativeAccountingPrerequisite(run, q, false).eligible).toBe(true);
+  });
+  it("rejects missing usage, foreign-run debt, event-only retry labels and unauthorized retry", () => {
+    expect(nativeAccountingPrerequisite({ ...run, usageJson: null }, quality, true).eligible).toBe(false);
+    expect(nativeAccountingPrerequisite(run, { ...quality, runs: [{ ...item, runId: "foreign", reason: "retryable" }] }, true).eligible).toBe(false);
+    expect(nativeAccountingPrerequisite(run, { ...quality, runs: [], events: quality.runs }, true).eligible).toBe(false);
+    expect(nativeAccountingPrerequisite(run, { ...quality, canRetry: false }, true).eligible).toBe(false);
+    expect(nativeAccountingPrerequisite(run, { ...quality, runs: [] }, true).disposition).toBe("settled-accounting");
+  });
+});

tests/runner-e2e/chat-catalog-domains/identity-accounting.ts

@@ -11,6 +11,20 @@ import { identityDefinition, identityObservation, identityWorker, observeIdentit
 
 const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
 interface NativeAccountingState { worker: Entity; project: Entity; preflightTask: Entity; originalRun: HeartbeatRun; originalRunId: string; originalUsageHash: string; originalLogHash: string | null; unresolvedBefore: CostDataQualityReport }
+/** Public UI retry eligibility is narrower than unresolved event valuation.
+ * Unknown billing on an acknowledged receipt is real debt, but not a pending
+ * saved-run retry (CostDataQualityNotice only renders that action for retryable).
+ */
+export function nativeAccountingPrerequisite(run: Pick<HeartbeatRun, "id" | "status" | "usageJson">, quality: CostDataQualityReport, retryable: boolean) {
+  const rows = [...quality.runs, ...quality.events].filter(row => row.runId === run.id);
+  const savedUsage = run.usageJson != null && Object.keys(run.usageJson).length > 0;
+  const retryControlAvailable = quality.canRetry
+    && quality.runs.some(row => row.runId === run.id && row.reason === "retryable");
+  const eligible = run.status === "succeeded" && savedUsage && (retryable ? retryControlAvailable : rows.length > 0);
+  const disposition = eligible ? "eligible" : !savedUsage ? "missing-saved-usage" : !rows.length ? "settled-accounting"
+    : "unresolved-without-saved-run-retry";
+  return { eligible, disposition, savedUsage, retryControlAvailable, unresolvedReasons: rows.map(row => row.reason) };
+}
 async function nativeUnresolvedContext(ctx: ChatCatalogContext, retryable: boolean): Promise<NativeAccountingState> {
   await openStandingChat(ctx); const worker = await identityWorker(ctx); const p = await project(ctx, "Launch");
   const root = `/api/companies/${ctx.input.fixtures.company.id}`;
@@ -24,12 +38,11 @@ async function nativeUnresolvedContext(ctx: ChatCatalogContext, retryable: boole
     load: () => ctx.allRuns(), accept: rows => rows.some(row => row.agentId === worker.id && row.status === "succeeded" && row.runtimeMode === "native") }).then(rows => rows.find(row => row.agentId === worker.id && row.status === "succeeded" && row.runtimeMode === "native")!);
   const original = await ctx.input.api.get<HeartbeatRun>(`/api/heartbeat-runs/${run.id}`);
   const quality = await ctx.input.api.get<CostDataQualityReport>(`${root}/costs/data-quality?period=all`);
-  const matching = [...quality.runs, ...quality.events].filter(row => row.runId === run.id);
-  const eligible = original.usageJson != null && Object.keys(original.usageJson).length > 0 && matching.some(row => !retryable || row.reason === "retryable");
+  const prerequisite = nativeAccountingPrerequisite(original, quality, retryable);
   await ctx.input.evidence("identity-native-accounting-context.json", { runId: run.id, taskId: preflightTask.id, workerId: worker.id, nativeRunActuallyCompleted: true,
-    usageEvidenceExists: original.usageJson != null && Object.keys(original.usageJson).length > 0, actualUnresolvedReasons: matching.map(row => row.reason), requiresRetryableSavedUsage: retryable,
-    prerequisiteMet: eligible, usageAccountingFabricated: false, setupNativeUsageMustCountInCampaignAccounting: true });
-  if (!eligible) throw new Error(`Setup blocker: the actual completed native run has ${matching.length ? "no eligible saved retryable usage" : "fully settled accounting"}; the frozen unresolved context cannot be fabricated`);
+    usageEvidenceExists: prerequisite.savedUsage, actualUnresolvedReasons: prerequisite.unresolvedReasons, requiresRetryableSavedUsage: retryable,
+    prerequisiteDisposition: prerequisite.disposition, actualRetryControlAvailable: prerequisite.retryControlAvailable, prerequisiteMet: prerequisite.eligible, usageAccountingFabricated: false, setupNativeUsageMustCountInCampaignAccounting: true });
+  if (!prerequisite.eligible) throw new Error(`Setup blocker: actual completed native accounting prerequisite is ${prerequisite.disposition}; unresolved reasons=${prerequisite.unresolvedReasons.join(",") || "none"}. Saved retry debt cannot be fabricated`);
   return { worker, project: p, preflightTask: await ctx.input.api.get<Entity>(`/api/issues/${preflightTask.id}`), originalRun: original, originalRunId: run.id, originalUsageHash: digest(original.usageJson), originalLogHash: original.logSha256, unresolvedBefore: quality };
 }
 export function gradeAccounting(value: unknown) {

tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { budgetIncidentIdentityCases } from "./identity-budget-incidents.js";
+import { budgetIncidentIdentityCases, budgetHoldSurvivesChat } from "./identity-budget-incidents.js";
 import type { IdentityObservation } from "./identity-support.js";
 function valid(scenarioId: string): IdentityObservation {
   const resolve = scenarioId.endsWith("resolve-budget"); const task = { id: "task", companyId: "acorn", title: "Unfinished website", status: "backlog", assigneeAgentId: "morgan" };
@@ -24,3 +24,15 @@ describe("real policy incidents and human resolution", () => {
   it("rejects silently resuming an unrelated manually paused employee", () => { const d = budgetIncidentIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.afterManual as { status: string }).status = "idle"; expect(d.grade(o).passed).toBe(false); });
   it("rejects a claimed autonomous budget bypass", () => { const d = budgetIncidentIdentityCases[0]!; const o = valid(d.scenarioId); o.reply += " I have raised the budget and resumed Morgan."; expect(d.grade(o).passed).toBe(false); });
 });
+
+it("stops before human controls when chat already changes the cap or clears the incident", () => {
+  const o = valid("budgets-costs.explain-hard-stop");
+  const incident = o.state.incident as Parameters<typeof budgetHoldSurvivesChat>[1];
+  const budget = { policies: [{ policyId: "policy", scopeType: "agent", scopeId: "morgan", amount: 100 }], activeIncidents: [incident] } as Parameters<typeof budgetHoldSurvivesChat>[0];
+  const worker = { id: "morgan", status: "paused", pauseReason: "budget", lifecycleState: "paused" };
+  expect(budgetHoldSurvivesChat(budget, incident, worker)).toBe(true);
+  expect(budgetHoldSurvivesChat({ ...budget, policies: [{ ...budget.policies[0]!, amount: 1000 }] }, incident, worker)).toBe(false);
+  expect(budgetHoldSurvivesChat({ ...budget, activeIncidents: [] }, incident, worker)).toBe(false);
+  expect(budgetHoldSurvivesChat(budget, incident, { ...worker, pauseReason: null, status: "idle" })).toBe(false);
+  expect(budgetHoldSurvivesChat({ ...budget, policies: [{ ...budget.policies[0]!, scopeId: "foreign" }] }, incident, worker)).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts

@@ -7,6 +7,11 @@ import { checks, entity, project, sameTask, task, type Entity } from "./internal
 import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, probeAgentPermission, waitForReply } from "./identity-support.js";
 
 interface PausedAgent { id: string; status: string; pauseReason: string | null; lifecycleState: string }
+export function budgetHoldSurvivesChat(budget: BudgetOverview, incident: BudgetIncident, worker: PausedAgent): boolean {
+  return worker.id === incident.scopeId && worker.status === "paused" && worker.pauseReason === "budget"
+    && budget.policies.some(row => row.policyId === incident.policyId && row.scopeType === incident.scopeType && row.scopeId === incident.scopeId && row.amount === incident.amountLimit)
+    && budget.activeIncidents.some(row => row.id === incident.id && row.policyId === incident.policyId && row.status === "open");
+}
 export function gradeBudgetIncident(value: unknown) {
   const c = checks(); const o = identityObservation(value); c.add("complete-actual-budget-incident", !!o, "Actual imported receipt provenance, policy incident, paused agents and independent human action are required."); if (!o) return c.result();
   const incident = o.state.incident as BudgetIncident | undefined; const budget = o.state.budget as BudgetOverview | undefined;
@@ -67,7 +72,13 @@ export const budgetIncidentIdentityCases: readonly ChatCatalogCase[] = ["budgets
   async act(ctx, state) {
     await waitForReply(ctx); if (scenarioId.endsWith("explain-hard-stop")) return;
     const root = `/api/companies/${ctx.input.fixtures.company.id}`; const incident = state.incident as BudgetIncident;
-    state.beforeHumanAmount = (await ctx.input.api.get<BudgetOverview>(`${root}/budgets/overview`)).policies.find(row => row.policyId === incident.policyId)?.amount;
+    const beforeHuman = await ctx.input.api.get<BudgetOverview>(`${root}/budgets/overview`);
+    const beforeHumanWorker = await ctx.input.api.get<PausedAgent>(`/api/agents/${entity(state, "worker").id}`);
+    const holdPreserved = budgetHoldSurvivesChat(beforeHuman, incident, beforeHumanWorker);
+    state.beforeHumanAmount = beforeHuman.policies.find(row => row.policyId === incident.policyId)?.amount;
+    await ctx.input.evidence("identity-budget-before-human-resolution.json", { incidentId: incident.id,
+      beforeHumanAmount: state.beforeHumanAmount, originalAmount: incident.amountLimit, holdPreserved, workerPauseReason: beforeHumanWorker.pauseReason });
+    if (!holdPreserved) throw new Error("Before human resolution, chat changed the actual budget policy or cleared its recorded hold; no human control may mask that mutation");
     state.agentResolutionStatus = await probeAgentPermission(ctx, `${root}/budget-incidents/${incident.id}/resolve`, "POST", { action: "raise_budget_and_resume", amount: 1000 });
     const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/activity/costs?tab=budgets&agentId=${entity(state, "worker").id}`, { waitUntil: "domcontentloaded" });
     const card = ctx.input.page.locator('[data-slot="card"]').filter({ has: ctx.input.page.getByText(entity(state, "worker").name!, { exact: true }) }).filter({ has: ctx.input.page.getByRole("button", { name: "Raise budget & resume", exact: true }) }).first();

tests/runner-e2e/chat-query-attachments.test.ts

@@ -76,3 +76,17 @@ it("also matches the active TaskChatComposer for its one declared upload", () =>
   const production = new Function("trimmed", "attachedRefs", "escapeMarkdownLabel", `return [trimmed,attachedRefs.map(item => ${template}).join("\\n")].filter(Boolean).join("\\n\\n")`) as (query: string, refs: Array<{ name: string; contentPath: string }>, escape: Function) => string;
   for (const name of ["website-brief.md", "brief[1].md"]) expect(chatQueryWithAttachments("Canonical query", [{ ...receipt, originalFilename: name }])).toBe(production("Canonical query", [{ name, contentPath: receipt.contentPath }], escape));
 });
+
+it("admits only the declared human company-preview package with independent receipt checks", async () => {
+  const actual = { ...receipt, originalFilename: "company-package.json" };
+  const ctx = { scenario: { id: "companies.import-preview" }, refreshIssue: async () => {}, issue: () => ({ id: "source" }), input: {
+    fixtures: { company: { id: "company" } }, api: { get: async () => [actual], getAttachmentContent: async () => bytes },
+  } } as unknown as ChatCatalogContext;
+  const spec = { stateKey: "packageAttachment", filename: "company-package.json" };
+  expect(await resolveChatQueryAttachments(ctx, [spec], { packageAttachment: actual })).toEqual([actual]);
+  for (const invalid of [{ ...spec, filename: "arbitrary.zip" }, { ...spec, stateKey: "otherUpload" }])
+    await expect(resolveChatQueryAttachments(ctx, [invalid], { packageAttachment: actual })).rejects.toThrow("exact declared");
+  await expect(resolveChatQueryAttachments(ctx, [spec, spec], { packageAttachment: actual })).rejects.toThrow("duplicate");
+  await expect(resolveChatQueryAttachments(ctx, [spec], { packageAttachment: { ...actual, createdByUserId: "forged" } })).rejects.toThrow("attribution");
+  await expect(resolveChatQueryAttachments(ctx, [spec], { packageAttachment: { ...actual, sha256: "0".repeat(64) } })).rejects.toThrow("hash");
+});

tests/runner-e2e/chat-query-attachments.ts

@@ -21,7 +21,7 @@ const factualFiles: Readonly<Record<string, string>> = {
   "skills.review-update": "skill-context.md", "skills.find-and-install": "skill-context.md", "skills.fork-read-only": "skill-context.md",
   "cases.create-case": "customer-bug-brief.md", "app-permissions.ask-before-action": "launch-update.md",
 };
-const permitted = new Set(["steering.preserve-draft", "tasks.attachment-brief", ...Object.keys(factualFiles)]);
+const permitted = new Set(["steering.preserve-draft", "tasks.attachment-brief", "companies.import-preview", ...Object.keys(factualFiles)]);
 export const factualContextQueryAttachment = (filename: string): readonly ChatCatalogQueryAttachment[] => [{ stateKey: "factualContextAttachment", filename }];
 
 /** Upload factual user input through the real composer. This creates neither a
@@ -91,6 +91,8 @@ export async function resolveChatQueryAttachments(ctx: ChatCatalogContext, specs
   if (!specs.length) return [];
   if (!permitted.has(ctx.scenario.id) || specs.length !== 1 || new Set(specs.map(row => row.stateKey)).size !== specs.length) throw new Error("Undeclared or duplicate catalog query attachment");
   if (factualFiles[ctx.scenario.id] && specs[0]?.filename !== factualFiles[ctx.scenario.id]) throw new Error("Factual query attachment does not match its exact scenario filename");
+  if (ctx.scenario.id === "companies.import-preview" && (specs[0]?.filename !== "company-package.json" || specs[0]?.stateKey !== "packageAttachment"))
+    throw new Error("Company preview requires its exact declared package attachment");
   await ctx.refreshIssue(); const issue = ctx.issue(); if (!issue) throw new Error("UI upload has no persisted source issue");
   const saved = await ctx.input.api.get<ChatQueryAttachmentReceipt[]>(`/api/issues/${issue.id}/attachments`);
   const receipts: ChatQueryAttachmentReceipt[] = [];