fix(evals): preserve catalog intent in held correction set

tests/runner-e2e/chat-catalog-domains/internal-activity.ts

@@ -33,11 +33,8 @@ export function grade_activity(value: unknown) {
     case "activity.audit-change": {
       const priorityEvents = (o.state.priorityEvents as Entity[] | undefined) ?? [];
       c.add("real-two-change-history", priorityEvents.length === 2 && priorityEvents.every(event => o.activity.some(row => row.id === event.id)), "Both fixture priority changes remain visible on the exact task.");
-      c.add("attribution-and-values", priorityEvents.length === 2 && priorityEvents.every(event => !!event.actorId
-        && mentionsRecordedActor(o.reply, event.actorId)
-        && o.reply.split(/[;\n]|\.(?:\s|$)/).some(part => new RegExp(`\\b${String(event.details?.priority)}\\b`, "i").test(part) && mentionsRecordedActor(part, event.actorId!))
-        && o.reply.toLowerCase().includes(String(event.details?.priority).toLowerCase())),
-        "Editor identity is the recorded actor ID or the canonical Local board label, never a generic operator inference.");
+      c.add("attribution-and-values", attributedPriorityChanges(o.reply, priorityEvents, before),
+        "Recorded actors bind each actual transition, including a truthful common actor header; wrong actors, tasks or changes cannot pass.");
       c.add("recorded-event-times", priorityEvents.length === 2 && priorityEvents.every(event => mentionsRecordedTime(o.reply, event.createdAt,
         String(o.state.observedAt ?? ""), String(o.state.timezone ?? "UTC"), priorityEvents.map(row => row.createdAt))),
         "Reported calendar and clock times match the actual retained events in UTC or the browser's declared timezone.");
@@ -127,3 +124,44 @@ function mentionsRecordedActor(reply: string, actorId: string): boolean {
       || new RegExp(`(?:^|\\n)\\s*(?:#{1,6}\\s*|\\*\\*)?${name}(?:\\*\\*)?\\s*:`, "i").test(reply);
   });
 }
+
+/** A common actor header scopes following bullets; repeating the actor in every
+ * line is unnecessary. Explicit contrary actor/transition claims still fail. */
+export function attributedPriorityChanges(reply: string, events: Entity[], target?: Entity): boolean {
+  if (events.length !== 2 || events.some(event => !event.actorId || !event.details?.priority)) return false;
+  const plain = reply.replace(/[`*]/g, "");
+  const taskRefs = plain.match(/\b[A-Z]{2,10}-\d+\b/g) ?? [];
+  if (target?.identifier && taskRefs.some(id => id !== target.identifier)) return false;
+  let cursor = 0;
+  const parts = plain.split(/[;\n]|\.(?:\s|$)/).filter(part => part.trim()).map(text => { const start = plain.indexOf(text, cursor); cursor = start + text.length; return { text, start, end: cursor }; });
+  const tokens = [...plain.matchAll(/\b(low|medium|high|critical)\b/gi)];
+  const pairs: Array<{ from: string; to: string; start: number; end: number }> = [];
+  for (let i = 0; i + 1 < tokens.length; i++) {
+    const left = tokens[i]!, right = tokens[i + 1]!;
+    if (/^\s*(?:→|->|to)\s*$/i.test(plain.slice(left.index! + left[0].length, right.index))) pairs.push({ from: left[1]!.toLowerCase(), to: right[1]!.toLowerCase(), start: left.index!, end: right.index! });
+  }
+  const change = (event: Entity) => {
+    const changes = event.details?.changes as { priority?: { from?: unknown } } | undefined;
+    const previous = event.details?._previous as { priority?: unknown } | undefined;
+    const raw = changes?.priority?.from ?? previous?.priority;
+    return { to: String(event.details!.priority).toLowerCase(), from: typeof raw === "string" ? raw.toLowerCase() : undefined };
+  };
+  if (pairs.some(pair => !events.some(event => { const expected = change(event); return pair.to === expected.to && (!expected.from || pair.from === expected.from); }))) return false;
+  return events.every(event => {
+    const actor = event.actorId!, expected = change(event);
+    const headerAt = parts.findIndex(part => mentionsRecordedActor(part.text, actor) && /\b(?:twice|both|two|priority|edits?|changes?)\b/i.test(part.text));
+    const aliases = actor === "local-board" ? [actor, "board"] : [actor.toLowerCase()];
+    return parts.some((entry, partIndex) => {
+      const part = entry.text;
+      if (!new RegExp(`\\b${expected.to}\\b`, "i").test(part)) return false;
+      const contraryActor = [...part.matchAll(/\b([a-z0-9_-]+)\s+(?:changed|updated|set|made|raised|lowered|edited)\b/gi)]
+        .some(match => !aliases.includes(match[1]!.toLowerCase()) && !["priority", "it", "they", "both", "changes", "edits"].includes(match[1]!.toLowerCase()));
+      if (contraryActor) return false;
+      if (!mentionsRecordedActor(part, actor) && !(headerAt >= 0 && partIndex >= headerAt)) return false;
+      // A priority named only as the starting value of another transition is
+      // not a report that this priority change occurred.
+      return tokens.some(token => token[1]!.toLowerCase() === expected.to && token.index! >= entry.start && token.index! < entry.end
+        && (!pairs.some(pair => pair.start === token.index) || pairs.some(pair => pair.end === token.index)));
+    });
+  });
+}

tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts

@@ -18,8 +18,7 @@ export function grade_documents_artifacts(value: unknown) {
   switch (o.scenarioId) {
     case "documents-artifacts.write-plan": {
       const plans = o.documents.filter(row => row.key === "plan"); const plan = plans[0];
-      c.add("saved-implementation-plan", plans.length === 1 && !!plan && !!plan.latestRevisionId && /\b(?:implement|build|develop)\b/i.test(plan.body ?? "")
-        && /\b(?:test|verify|validation)\b/i.test(plan.body ?? "") && (plan.body ?? "").includes(String(o.state.requirement)), "A durable plan reflects the website's actual prerequisite brief and verification, rather than merely chat prose.");
+      c.add("saved-implementation-plan", plans.length === 1 && !!plan && !!plan.latestRevisionId && meaningfulOrchidPlan(plan.body ?? ""), "A durable plan reflects the website's actual prerequisite brief and verification, rather than merely chat prose.");
       c.add("versioned-author", !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.body === plan.body && row.createdByAgentId === o.leadId
         && o.runs.some(run => run.id === row.createdByRunId)), "The current document revision is attributed to this real agent run.");
       c.add("plan-audit", !!before && authored("issue.document_created", before.id), "The API audit records the CEO creating the plan after the request.");
@@ -67,7 +66,7 @@ export const documents_artifactsCases: readonly ChatCatalogCase[] = [
   ...privateDownloadCases,
   resolveFeedback(),
   define("documents-artifacts.write-plan", async ctx => {
-    const p = await project(ctx); const requirement = `Orchid ${ctx.input.nonce}`;
+    const p = await project(ctx); const requirement = "Orchid";
     const t = await task(ctx, "Website task", p.id, { assigneeAgentId: ctx.input.fixtures.agent.id,
       description: `Build a single-page website for ${requirement}, with accessible signup and responsive layout. Verify keyboard navigation and mobile presentation.` });
     return { project: p, task: t, requirement };
@@ -112,3 +111,10 @@ function resolveFeedback(): ChatCatalogCase {
   } };
 }
 import { observeInternal } from "./internal-support.js";
+
+export function meaningfulOrchidPlan(body: string): boolean {
+  return /\b(?:implement|build|develop)\b/i.test(body) && /\b(?:test|verify|verification|validation)\b/i.test(body)
+    && /\bOrchid\b/i.test(body) && /\b(?:single|one)[- ]page\b/i.test(body)
+    && /\b(?:signup|sign.up|registration)\b/i.test(body) && /\b(?:accessible|accessibility|labels?|semantic)\b/i.test(body)
+    && /\b(?:responsive|breakpoints?)\b/i.test(body) && /\bkeyboard\b/i.test(body) && /\bmobile\b/i.test(body);
+}

tests/runner-e2e/chat-catalog-domains/internal-projects.ts

@@ -1,3 +1,4 @@
+import { expect } from "@playwright/test";
 import { projectPrivacyCases } from "./internal-privacy.js";
 import { repositoryProjectCases } from "./internal-repositories.js";
 import { projectWorkCases } from "./internal-task-creation.js";
@@ -31,7 +32,8 @@ export function grade_projects(value: unknown) {
       c.add("history-retained", !!before && sameTask(before, current) && o.documents.some(row => row.body === o.state.originalBody)
         && o.activity.some(row => row.id === o.state.originalActivityId), "The original task, document bytes and creation audit survive archiving.");
       c.add("attributed-project-update", !!projectNow && authored("project.updated", projectNow.id), "The saved lifecycle change belongs to the acting agent.");
-      c.add("historical-link", !!projectNow && hasLink(o, "projects", projectNow), "Chat supplies a reachable historical project.");
+      c.add("historical-api-reachability", !!projectNow && o.state.historicalProjectReadId === projectNow.id && o.state.historicalTaskReadId === before?.id, "The actual archived project and retained task remain independently readable through their public resource APIs.");
+      c.add("truthful-archive-report", /\barchived\b/i.test(o.reply) && !assertsDestructiveArchiveChange(o.reply), "The response reports saved archiving without falsely claiming resource deletion or task cancellation.");
       break;
     }
     default: c.add("implemented-scenario", false, "No independent oracle is implemented for this scenario.");
@@ -45,14 +47,7 @@ export const projectsCases: readonly ChatCatalogCase[] = [
   ...repositoryProjectCases,
   ...projectWorkCases,
   retryDefinition("projects.create-project", grade_projects, initialProjectCreation),
-  define("projects.archive-project", async ctx => {
-    const p = await project(ctx, "Launch", { status: "completed" });
-    const t = await task(ctx, "Website task", p.id, { status: "done" });
-    const body = `# Delivered website\nHistorical receipt ${ctx.input.nonce}.`;
-    await putDocument(ctx, t.id, body);
-    const activity = await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`);
-    return { project: p, task: t, originalBody: body, originalActivityId: activity.find(row => row.action === "issue.created")?.id };
-  }),
+  archivedProject(),
 
 ];
 
@@ -63,3 +58,38 @@ function initialProjectCreation(o: InternalObservation | undefined): boolean {
     && o.activity.some(row => row.action === "project.created" && row.entityId === p.id && row.agentId === o.leadId)
     && !/\b(?:deployed|website is ready|repository is ready)\b/i.test(o.reply);
 }
+
+function assertsDestructiveArchiveChange(reply: string) {
+  for (const match of reply.matchAll(/\b(?:delet(?:e|ed|ing)|cancel(?:led|ed|ling|ing))\b/gi)) {
+    const before = reply.slice(Math.max(0, match.index! - 75), match.index);
+    if (!/(?:\b(?:not|never|nothing|without|no resources?|no tasks?)|n['’]t|rather than)\s+(?:(?:was|were|is|are|been|be|being|have|has|had)\s+)*$/i.test(before)) return true;
+  }
+  return false;
+}
+function archivedProject(): ChatCatalogCase {
+  const base = define("projects.archive-project", async ctx => {
+    const p = await project(ctx, "Launch", { status: "completed" });
+    const t = await task(ctx, "Website task", p.id, { status: "done" });
+    const body = `# Delivered website\nHistorical receipt ${ctx.input.nonce}.`;
+    await putDocument(ctx, t.id, body);
+    const activity = await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`);
+    return { project: p, task: t, originalBody: body, originalActivityId: activity.find(row => row.action === "issue.created")?.id };
+  });
+  return { ...base,
+    async observe(ctx, state) {
+      const [p, t] = await Promise.all([ctx.input.api.get<Entity>(`/api/projects/${entity(state, "project").id}`), ctx.input.api.get<Entity>(`/api/issues/${entity(state, "task").id}`)]);
+      state.historicalProjectReadId = p.id; state.historicalTaskReadId = t.id;
+      return base.observe!(ctx, state);
+    },
+    async verifyResult(ctx, state) {
+      const source = ctx.input.page.url();
+      for (const [kind, row] of [["projects", entity(state, "project")], ["issues", entity(state, "task")]] as const) {
+        await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/${kind}/${row.id}`, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" });
+        await expect(ctx.input.page.getByRole("heading", { name: row.name ?? row.title!, exact: true }).first()).toBeVisible();
+        await ctx.input.evidence(`chat-catalog-archived-history-${kind}.json`, { id: row.id, actualUrl: ctx.input.page.url(), actualHeading: row.name ?? row.title });
+      }
+      await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+      await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
+    },
+  };
+}

tests/runner-e2e/chat-catalog-domains/internal.test.ts

@@ -203,6 +203,7 @@ function valid(scenario: string): InternalObservation {
     }
     case "projects.archive-project": {
       o.projects[0] = { ...o.projects[0]!, archivedAt: "2026-10-26T12:00:00Z" }; o.state.originalBody = "Delivered"; o.state.originalActivityId = "original";
+      o.state.historicalProjectReadId = "p"; o.state.historicalTaskReadId = "task"; o.reply = "Archived the Launch project. Its tasks and history remain intact; nothing was deleted.";
       o.documents = [{ id: "doc", body: "Delivered" }]; o.activity = [{ id: "original", action: "issue.created", entityId: "task" }, event("project.updated", "p")]; break;
     }
     case "tasks.website-task":
@@ -264,7 +265,7 @@ function valid(scenario: string): InternalObservation {
     }
     case "documents-artifacts.write-plan": {
       o.state.requirement = "Orchid 42";
-      const body = "Implement the Orchid 42 website, then verify accessible signup.";
+      const body = "Implement the single-page Orchid website with accessible signup and responsive layout, then verify keyboard navigation and mobile presentation.";
       o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "revision" }];
       o.revisions = [{ id: "revision", body, createdByAgentId: "ceo", createdByRunId: "run" }]; o.activity = [event("issue.document_created", "task")]; break;
     }
@@ -365,7 +366,7 @@ describe("internal chat scenario independent oracles", () => {
       expect(gradeInternal(wrong).passed).toBe(false);
       expect(gradeInternal(undefined).passed).toBe(false);
       const noLinks = structuredClone(good); noLinks.links = [];
-      expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive", "documents-artifacts.restore-document"].includes(definition.scenarioId));
+      expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.archive-project", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive", "documents-artifacts.restore-document"].includes(definition.scenarioId));
     });
   }
   it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-partial-failure")).passed).toBe(false));
@@ -710,3 +711,40 @@ describe("personal inbox operation calibration", () => {
     expect(gradeInternal(o).passed).toBe(false);
   });
 });
+
+describe("measured initial counterexamples calibrated for coherent B", () => {
+  it("accepts saved archival without a mandatory model hyperlink, with real history readbacks", () => {
+    const o = valid("projects.archive-project"); o.links = [];
+    expect(gradeInternal(o)).toMatchObject({ passed: true });
+    delete o.state.historicalProjectReadId; expect(gradeInternal(o).passed).toBe(false);
+  });
+  it.each(["Archived Launch and deleted its tasks.", "Archived Launch; I cancelled all unfinished execution."])("rejects false destructive archival claim: %s", reply => {
+    const o = valid("projects.archive-project"); o.reply = reply; expect(gradeInternal(o).passed).toBe(false);
+  });
+  it("accepts an actor header over both actual priority transitions", () => {
+    const o = valid("activity.audit-change");
+    o.state.task = { ...(o.state.task as Entity), identifier: "RUN-1" };
+    const events = o.state.priorityEvents as Entity[];
+    events[0]!.details = { priority: "high", _previous: { priority: "medium" } };
+    events[1]!.details = { priority: "low", _previous: { priority: "high" } };
+    o.reply = "The user actor **`local-board`** changed RUN-1’s priority twice:\n\n- Medium → High\n- High → Low\n\nBoth changes occurred at 12:00:00 UTC.";
+    expect(gradeInternal(o), JSON.stringify(gradeInternal(o).checks)).toMatchObject({ passed: true });
+    for (const reply of [
+      "Morgan changed RUN-1’s priority twice: Medium → High; High → Low. At 12:00 UTC.",
+      "Board changed RUN-99’s priority twice: Medium → High; High → Low. At 12:00 UTC.",
+      "Board changed RUN-1’s priority twice: Medium → Low; Low → High. At 12:00 UTC.",
+      "Board changed RUN-1’s priority: High → Low. At 12:00 UTC.",
+      "Board changed RUN-1’s priority twice: Medium → High; High → Low. At 13:00 UTC.",
+      "Board changed RUN-1’s priority twice:\nMorgan changed Medium → High\nBoard changed High → Low. At 12:00 UTC.",
+    ]) { o.reply = reply; expect(gradeInternal(o).passed, reply).toBe(false); }
+  });
+  it("accepts the observed substantive plan without requiring a random nonce", () => {
+    const o = valid("documents-artifacts.write-plan");
+    const body = "# Implementation Plan\n1. Define the single-page structure and Orchid visual direction, including clear content hierarchy and responsive breakpoints.\n2. Build the semantic page sections and an accessible signup form with explicit labels, useful validation states, and clear success/error feedback.\n3. Add responsive styling for mobile through desktop, ensuring readable typography, comfortable spacing, and usable touch targets.\n4. Verify the complete keyboard path, visible focus states, form behavior, and mobile presentation; fix any accessibility or layout regressions found.";
+    o.state.requirement = "Orchid old-random-nonce"; o.documents[0]!.body = body; o.revisions[0]!.body = body;
+    expect(gradeInternal(o), JSON.stringify(gradeInternal(o).checks)).toMatchObject({ passed: true });
+    for (const changed of [body.replaceAll("Orchid", "Otherbrand"), "Implement Orchid website. Verify it.", body.replaceAll(/accessible|accessibility|semantic|labels/g, "plain"), body.replaceAll(/responsive|breakpoints/g, "fixed"), body.replaceAll("keyboard", "pointer"), body.replaceAll("mobile", "desktop"), body.replaceAll("single-page", "multi-page")]) {
+      o.documents[0]!.body = changed; o.revisions[0]!.body = changed; expect(gradeInternal(o).passed, changed).toBe(false);
+    }
+  });
+});

tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect } from "vitest";
-import { gradeRuntimeAgents, runtimeAgentsCases } from "./runtime-agents.js";
+import { gradeRuntimeAgents, hiringDeniedPermissions, runtimeAgentsCases } from "./runtime-agents.js";
+import { updateAgentPermissionsSchema } from "../../../packages/shared/src/validators/agent.js";
 import type { RuntimeObservation } from "./runtime-support.js";
 
 function valid(id: string): RuntimeObservation {
@@ -39,6 +40,14 @@ function valid(id: string): RuntimeObservation {
   return o;
 }
 describe("runtime agent oracles", () => {
+  it("disables hiring with the public permission contract while preserving existing task authority", () => {
+    for (const canAssignTasks of [true, false]) {
+      const payload = hiringDeniedPermissions({ canCreateAgents: true, canAssignTasks, canCreateSkills: true });
+      expect(updateAgentPermissionsSchema.safeParse(payload).success).toBe(true);
+      expect(payload).toMatchObject({ canCreateAgents: false, canAssignTasks, canCreateSkills: true });
+    }
+    expect(() => hiringDeniedPermissions({ canCreateAgents: true })).toThrow();
+  });
   it("accepts substantive paraphrases for each authored agent outcome", () => {
     const replies: Record<string, string> = {
       "agents.change-manager": "Morgan now reports to Riley; the reporting relationship is saved.",

tests/runner-e2e/chat-catalog-domains/runtime-agents.ts

@@ -2,6 +2,11 @@ import { checks, sameTask } from "./internal-support.js";
 import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, agentLink, verifyRuntimeResult, type RuntimeObservation, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
 import { pollUntil } from "../api.js";
 import { expect } from "@playwright/test";
+import { updateAgentPermissionsSchema } from "../../../packages/shared/src/validators/agent.js";
+
+export function hiringDeniedPermissions(permissions: RuntimeAgent["permissions"]) {
+  return updateAgentPermissionsSchema.parse({ ...permissions, canCreateAgents: false });
+}
 
 export function gradeRuntimeAgents(value: unknown) {
   const c = checks(); const o = runtimeObservation(value);
@@ -120,7 +125,8 @@ export const runtimeAgentsCases = [
     await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
   } }),
   runtimeDefinition("agents.hire-denied", async ctx => {
-    await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}/permissions`, { canCreateAgents: false });
+    const original = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
+    await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}/permissions`, hiringDeniedPermissions(original.permissions));
     const { probeAgentPermission } = await import("./identity-support.js");
     // Invalid payload would only prove validation. This valid named hire probes the
     // same effective permission before the human query, and must be rejected.