fix(evals): qualify case permissions and cancelled-run accounting

tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts

@@ -1,3 +1,4 @@
+import { grantCaseTaskAssignment } from "./integrator-case-prerequisites.js";
 import type { cases } from "../../../packages/db/src/schema/cases.js";
 import type { CaseHandoff, HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js";
 import type { Issue, IssueDocument, DocumentRevision, IssueBlockerDiagnosticsResponse } from "../../../packages/shared/src/types/issue.js";
@@ -57,7 +58,7 @@ export const integratorCaseCloseCases = [integratorDefinition("cases", "cases.cl
   const prior = await ctx.input.api.get<{ enableCases: boolean }>("/api/instance/settings/experimental"); settings.set(ctx, prior.enableCases);
   try {
     await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: true }); const p = await project(ctx, "Launch"); const worker = await fixtureWorker(ctx);
-    await ctx.input.api.patch(`/api/agents/${worker.id}`, { permissions: { canAssignTasks: true } });
+    await grantCaseTaskAssignment(ctx, worker.id);
     for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
     const kickoff = await ctx.input.api.post<HandoffCaseResult>(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, {
       idempotencyKey: `close-review-${ctx.input.nonce}`, case: { create: { caseType: "customer", title: "Accepted checkout fix", projectId: p.id, fields: { reviewRequirement: "Verify supplied correction and retain review document" } } }, toAgentId: worker.id, phase: "review",

tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts

@@ -1,3 +1,4 @@
+import { waitCaseCancellationAccounting } from "./integrator-case-prerequisites.js";
 import { randomUUID } from "node:crypto";
 import type { cases } from "../../../packages/db/src/schema/cases.js";
 import type { HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js";
@@ -75,6 +76,7 @@ export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.up
       load: async () => ({ task: await ctx.input.api.get<Issue>(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns() }),
       accept: v => v.runs.length === 1 && v.runs[0]?.status === "cancelled" && v.task.status === "in_progress" && !v.task.executionRunId && !v.task.checkoutRunId });
     gate.release();
+    await waitCaseCancellationAccounting(ctx, { companyId: ctx.input.fixtures.company.id, agentId: worker.id, issueId: kickoff.taskId, runId: initial.runs[0]!.id });
     return { initialRunId: initial.runs[0]!.id, project: p, worker: { id: worker.id }, reviewer: { id: reviewer.id }, case: await ctx.input.api.get<CaseRecord>(`/api/cases/${kickoff.caseId}`),
       brief: await ctx.input.api.get<IssueDocument>(`/api/issues/${kickoff.taskId}/documents/brief`), collaboratorValue: `support-${ctx.input.nonce}` };
   } catch (error) { await cleanup(ctx); throw error; }

tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts

@@ -1,3 +1,4 @@
+import { waitCaseCancellationAccounting } from "./integrator-case-prerequisites.js";
 import { randomUUID } from "node:crypto";
 import type { cases } from "../../../packages/db/src/schema/cases.js";
 import type { CaseHandoff, HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js";
@@ -89,6 +90,7 @@ export const integratorCaseHandoffCases = [integratorDefinition("cases", "cases.
       load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get<OwnedTask>(`/api/issues/${kickoff.taskId}`) }),
       accept: value => value.runs.length === 1 && value.runs[0]?.status === "cancelled" && value.task.status === "in_progress" && !value.task.executionRunId && !value.task.checkoutRunId });
     gate.release();
+    await waitCaseCancellationAccounting(ctx, { companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, issueId: kickoff.taskId, runId: phase.runs[0]!.id });
     return { project: p, reviewer: { id: reviewer.id, name: "Riley" }, respondentId: ctx.input.fixtures.agent.id, task: await ctx.input.api.get<OwnedTask>(`/api/issues/${kickoff.taskId}`),
       case: await ctx.input.api.get<CaseRecord>(`/api/cases/${kickoff.caseId}`), findings, interruptedTask: phase.task, interruptedRunId: phase.runs[0]!.id };
   } catch (error) { await cleanup(ctx); throw error; }

tests/runner-e2e/chat-catalog-domains/integrator-case-history.ts

@@ -1,3 +1,4 @@
+import { grantCaseTaskAssignment } from "./integrator-case-prerequisites.js";
 import type { cases, caseEvents } from "../../../packages/db/src/schema/cases.js";
 import type { CaseHandoff, HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js";
 import type { Issue, IssueDocument, DocumentRevision } from "../../../packages/shared/src/types/issue.js";
@@ -57,8 +58,9 @@ export const integratorCaseHistoryCases = [integratorDefinition("cases", "cases.
   try {
     await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: true }); const p = await project(ctx, "Launch");
     const worker = await fixtureWorker(ctx); const reviewer = await fixtureWorker(ctx);
-    await ctx.input.api.patch(`/api/agents/${reviewer.id}`, { name: "Riley", permissions: { canAssignTasks: true } });
-    await ctx.input.api.patch(`/api/agents/${worker.id}`, { permissions: { canAssignTasks: true } });
+    await ctx.input.api.patch(`/api/agents/${reviewer.id}`, { name: "Riley" });
+    await grantCaseTaskAssignment(ctx, reviewer.id);
+    await grantCaseTaskAssignment(ctx, worker.id);
     for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[worker.id, reviewer.id].map(scopeId => ({ scopeType: "agent", scopeId }))])
       await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
     const kickoff = await ctx.input.api.post<HandoffCaseResult>(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, {

tests/runner-e2e/chat-catalog-domains/integrator-case-prerequisites.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it, vi } from "vitest";
+import type { BudgetOverview } from "../../../packages/shared/src/types/budget.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { caseCancellationAccountingReady, grantCaseTaskAssignment } from "./integrator-case-prerequisites.js";
+const expected = { companyId: "company", agentId: "worker", issueId: "primary", runId: "cancelled-run" };
+function ready() {
+  return { run: { id: expected.runId, companyId: "company", agentId: "worker", issueId: "primary", status: "cancelled" as const, finishedAt: new Date() },
+    nativeRun: { id: expected.runId, companyId: "company", agentId: "worker", status: "cancelled", runtimeMode: "native", contextSnapshot: { issueId: "primary" } },
+    overview: { companyId: "company", policies: ["company", "agent"].map(scopeType => ({ companyId: "company", scopeType, scopeId: scopeType === "company" ? "company" : "worker", isActive: true, hardStopEnabled: true, amount: 500, remainingAmount: 499, unpricedUsagePolicy: "allow", pendingRunCount: 0, status: "ok", paused: false })) } as BudgetOverview };
+}
+describe("case prerequisite cancellation accounting", () => {
+  it("accepts only matching genuinely cancelled native execution and observed funded scopes", () => { expect(caseCancellationAccountingReady(expected, ready())).toBe(true); });
+  it("does not equate terminal cancellation or receipt arrival with settled accounting", () => {
+    for (const index of [0, 1]) { const v = ready(); v.overview.policies[index]!.pendingRunCount = 1; expect(caseCancellationAccountingReady(expected, v)).toBe(false); }
+  });
+  it("rejects wrong company, run, agent and task identities", () => {
+    for (const field of ["companyId", "agentId", "issueId", "id"] as const) { const v = ready(); v.run[field] = "other"; expect(caseCancellationAccountingReady(expected, v)).toBe(false); }
+    const v = ready(); v.nativeRun.companyId = "other"; expect(caseCancellationAccountingReady(expected, v)).toBe(false);
+  });
+  it("never forces readiness from missing scope, legacy run, unfinished run, paused or exhausted budget", () => {
+    const variants = [ready(), ready(), ready(), ready(), ready()];
+    variants[0]!.overview.policies.pop(); variants[1]!.nativeRun.runtimeMode = "legacy";
+    variants[2]!.run.finishedAt = null as unknown as Date; variants[3]!.overview.policies[0]!.paused = true; variants[4]!.overview.policies[0]!.remainingAmount = 0;
+    for (const v of variants) expect(caseCancellationAccountingReady(expected, v)).toBe(false);
+  });
+});
+describe("public case assignment permission prerequisite", () => {
+  function context(readback = true, companyId = "company") {
+    const before = { id: "worker", companyId, permissions: { canCreateAgents: false, canCreateSkills: false, canAssignTasks: false } };
+    const api = { get: vi.fn().mockResolvedValueOnce(before).mockResolvedValueOnce({ ...before, permissions: { ...before.permissions, canAssignTasks: readback } }), patch: vi.fn().mockResolvedValue({}) };
+    return { ctx: { input: { api, fixtures: { company: { id: "company" } }, evidence: vi.fn() } } as unknown as ChatCatalogContext, api };
+  }
+  it("uses dedicated route with required fields preserving unrelated authority", async () => {
+    const { ctx, api } = context(); await grantCaseTaskAssignment(ctx, "worker");
+    expect(api.patch).toHaveBeenCalledExactlyOnceWith("/api/agents/worker/permissions", { canCreateAgents: false, canCreateSkills: false, canAssignTasks: true }); expect(api.get).toHaveBeenCalledTimes(2);
+  });
+  it("fails closed on false readback and foreign agent before mutation", async () => {
+    const wrong = context(false); await expect(grantCaseTaskAssignment(wrong.ctx, "worker")).rejects.toThrow("readback");
+    const foreign = context(true, "other"); await expect(grantCaseTaskAssignment(foreign.ctx, "worker")).rejects.toThrow("scope"); expect(foreign.api.patch).not.toHaveBeenCalled();
+  });
+});

tests/runner-e2e/chat-catalog-domains/integrator-case-prerequisites.ts

@@ -0,0 +1,43 @@
+import type { Agent } from "../../../packages/shared/src/types/agent.js";
+import type { BudgetOverview } from "../../../packages/shared/src/types/budget.js";
+import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js";
+import { updateAgentPermissionsSchema } from "../../../packages/shared/src/validators/agent.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { ChatRun } from "../chat-flow.js";
+import { pollUntil } from "../api.js";
+
+/** Use the permission-specific public route; ordinary agent updates reject permissions. */
+export async function grantCaseTaskAssignment(ctx: ChatCatalogContext, agentId: string) {
+  const before = await ctx.input.api.get<Agent>(`/api/agents/${agentId}`);
+  if (before.id !== agentId || before.companyId !== ctx.input.fixtures.company.id) throw new Error("Case assignment prerequisite agent scope mismatch");
+  const permissions = updateAgentPermissionsSchema.parse({ ...before.permissions, canAssignTasks: true });
+  await ctx.input.api.patch(`/api/agents/${agentId}/permissions`, permissions);
+  const after = await ctx.input.api.get<Agent>(`/api/agents/${agentId}`);
+  if (after.id !== before.id || after.companyId !== before.companyId || !after.permissions.canAssignTasks
+    || after.permissions.canCreateAgents !== before.permissions.canCreateAgents) throw new Error("Case assignment prerequisite public readback failed");
+  await ctx.input.evidence(`chat-case-assignment-${agentId}.json`, { agentId, companyId: before.companyId, beforePermissions: before.permissions, requestedPermissions: permissions, afterPermissions: after.permissions });
+}
+
+type CancellationIdentity = { companyId: string; agentId: string; issueId: string; runId: string };
+type CancellationSnapshot = { run: Pick<HeartbeatRun, "id" | "companyId" | "agentId" | "issueId" | "status" | "finishedAt">; nativeRun: ChatRun | undefined; overview: BudgetOverview };
+export function caseCancellationAccountingReady(expected: CancellationIdentity, value: CancellationSnapshot): boolean {
+  const { run, nativeRun, overview } = value;
+  if (run.id !== expected.runId || run.companyId !== expected.companyId || run.agentId !== expected.agentId || run.issueId !== expected.issueId
+    || run.status !== "cancelled" || !run.finishedAt || nativeRun?.id !== run.id || nativeRun.companyId !== run.companyId
+    || nativeRun.agentId !== run.agentId || nativeRun.runtimeMode !== "native" || nativeRun.status !== "cancelled"
+    || nativeRun.contextSnapshot?.issueId !== expected.issueId || overview.companyId !== expected.companyId) return false;
+  const relevant = overview.policies.filter(policy => policy.isActive && (policy.scopeType === "company" && policy.scopeId === expected.companyId || policy.scopeType === "agent" && policy.scopeId === expected.agentId));
+  // Public pendingRunCount includes terminal runs awaiting final accounting. Both
+  // funded scopes must be observable; absence of policies is not readiness.
+  return ["company", "agent"].every(scope => relevant.some(policy => policy.scopeType === scope))
+    && relevant.every(policy => policy.companyId === expected.companyId && policy.hardStopEnabled && policy.amount > 0
+      && policy.unpricedUsagePolicy === "allow" && policy.pendingRunCount === 0 && policy.status !== "hard_stop" && !policy.paused && policy.remainingAmount > 0);
+}
+
+export async function waitCaseCancellationAccounting(ctx: ChatCatalogContext, expected: CancellationIdentity) {
+  const snapshot = await pollUntil({ label: "owned cancelled case prerequisite final accounting before canonical admission", deadlineAt: Date.now() + 60_000, intervalMs: 1000,
+    load: async (): Promise<CancellationSnapshot> => ({ run: await ctx.input.api.get<HeartbeatRun>(`/api/heartbeat-runs/${expected.runId}`),
+      nativeRun: (await ctx.allRuns()).find(run => run.id === expected.runId), overview: await ctx.input.api.get<BudgetOverview>(`/api/companies/${expected.companyId}/budgets/overview`) }),
+    accept: value => caseCancellationAccountingReady(expected, value) });
+  await ctx.input.evidence("chat-case-cancelled-prerequisite-accounting.json", { expected, ...snapshot });
+}