diff --git a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts index d17097587aa..0245eb6da79 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { budgetIncidentIdentityCases, budgetHoldSurvivesChat, budgetPage, incidentSourceFundingReady } from "./identity-budget-incidents.js"; +import { budgetIncidentIdentityCases, budgetHoldSurvivesChat, budgetIncidentSourceDescription, budgetPage, incidentSourceFundingReady } from "./identity-budget-incidents.js"; import type { IdentityObservation } from "./identity-support.js"; function valid(scenarioId: string): IdentityObservation { const resolve = scenarioId.endsWith("resolve-budget"); const task = { id: "task", companyId: "acorn", title: "Unfinished website", status: "backlog", assigneeAgentId: "morgan" }; @@ -15,6 +15,13 @@ function valid(scenarioId: string): IdentityObservation { if (resolve) o.activity.push({ id: "resolve", action: "budget.incident_resolved", entityId: "incident", actorType: "user" }); return o; } describe("real policy incidents and human resolution", () => { + // Full cohort 13: the explain-hard-stop reply repeated this note's own setup + // jargon ("an imported accounting fixture, not live model billing") to the person. + it("writes the setup note in plain words and keeps the task, incident and paused-agent references", () => { + const note = budgetIncidentSourceDescription({ taskIdentifier: "COM-2", incidentId: "incident-1", manualName: "Manual research pause" }); + for (const literal of ["COM-2", "incident-1", "Manual research pause"]) expect(note).toContain(literal); + for (const jargon of ["fixture", "synthetic", "imported", "accounting", "billing"]) expect(note.toLowerCase()).not.toContain(jargon); + }); for (const d of budgetIncidentIdentityCases) { it(`${d.scenarioId}: valid and missing recorded evidence`, () => { const o = valid(d.scenarioId); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); }); it(`${d.scenarioId}: actual manual pause cannot substitute for the budget incident`, () => { const o = valid(d.scenarioId); (o.state.beforeWorker as { pauseReason: string }).pauseReason = "manual"; expect(d.grade(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts index 35db806f1ca..3630be89a91 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts @@ -62,6 +62,16 @@ export function gradeBudgetIncident(value: unknown) { } else c.add("known-incident-case", false, "No incident oracle exists for this scenario."); return c.result(); } +/** + * The conversation's setup note, in the words a person would use. The receipt + * is an imported test charge; the grader proves that from saved state, so the + * note does not say it, and the reply has no setup jargon to repeat. + */ +export function budgetIncidentSourceDescription(input: { taskIdentifier: string; incidentId: string; manualName: string | undefined }): string { + return `Morgan's unfinished website task is ${input.taskIdentifier}. Morgan's hard budget stop is recorded as budget incident ${input.incidentId}. ` + + `${input.manualName ?? "Another agent"} was paused separately, by hand. The person must choose any new budget amount through the board's budget control; ` + + "do not assume the task is done or work around a hold."; +} const sources = ["chat-catalog-domains/identity-budget-incidents.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/budgets-costs.yaml"]; export const budgetIncidentIdentityCases: readonly ChatCatalogCase[] = ["budgets-costs.explain-hard-stop", "budgets-costs.resolve-budget"].map(scenarioId => identityDefinition(scenarioId, async ctx => { const source = await openStandingChat(ctx); const worker = await identityWorker(ctx); const manual = await identityWorker(ctx, "Manual research pause"); const p = await project(ctx); @@ -90,7 +100,7 @@ export const budgetIncidentIdentityCases: readonly ChatCatalogCase[] = ["budgets const budget = await pollUntil({ label: "actual imported receipt budget incident", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get(`${root}/budgets/overview`), accept: row => row.activeIncidents.some(incident => incident.scopeType === "agent" && incident.scopeId === worker.id && incident.thresholdType === "hard" && incident.status === "open") }); const incident = budget.activeIncidents.find(row => row.scopeId === worker.id && row.thresholdType === "hard")!; - await ctx.input.api.patch(`/api/issues/${source.id}`, { description: `Morgan's unfinished website task is ${t.identifier}. Its actual recorded hard budget incident is ${incident.id}; the receipt is an imported synthetic accounting fixture, not live model-provider billing. ${manual.name} is separately manually paused. The person must choose any new amount through the board control; do not assume task completion or bypass a hold.` }); + await ctx.input.api.patch(`/api/issues/${source.id}`, { description: budgetIncidentSourceDescription({ taskIdentifier: t.identifier!, incidentId: incident.id, manualName: manual.name }) }); return { worker, manual, task: t, project: p, incident, importedReceiptId: receipt.id, receiptOrigin: "public-imported-synthetic-fixture", beforeWorker: await ctx.input.api.get(`/api/agents/${worker.id}`) }; }, gradeBudgetIncident, { sourceFiles: sources, authentication: "human-session", oracleVersion: `paperclip.chat-catalog.identity.${scenarioId.split(".")[1]}.v4`, setupPolicy: { funding: "preserve-scenario", readiness: "preserve-scenario", invariantCheckIds: ["real-budget-and-manual-holds"] },