test(evals): grade read-history per task against its recorded outcome

The reply check required outcome keywords in a sentence naming each task, so a
correct "COM-2 published successfully and is done" failed while a reply that
swapped the outcomes passed. Attribute reply lines to exactly one linked task,
pair each with the status the product records (blocked/failed vs
done/completed), and reject a unit that gives a task the other outcome.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4688,7 +4688,7 @@
       "chat-catalog-domains/integrator-support.ts",
       "chat-scenarios/routines.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v2",
+    "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v3",
     "expectedRunCount": 3,
     "expectedHumanMessageCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts

@@ -126,3 +126,24 @@ describe("complete normal accounting stays mandatory", () => {
     const foreign = phase(true); foreign.identity.runId = "other"; expect(routineHistoryAccountingReady(foreign.identity, [foreign.snapshot.run!], overview())).toBe(false);
   });
 });
+// Recorded reply from run 42584001490 (prefix renamed); it failed v2 only because "published successfully and is done" missed a keyword.
+const RECORDED = "Yes \u2014 the **Morning Launch summary** routine ran twice this week, both today (Nov 9):\n\n- **Failed:** run `51e8bf51` at 08:20 UTC. [ACORN-1](/ACORN/issues/ACORN-1) was blocked because the Synthetic Launch Notes `launch_update` publish returned `Synthetic write failed; page unchanged`. It remains blocked; owner is the Synthetic Launch Notes provider, and the unblock action is restoring `launch_update`.\n- **Completed:** run `4b180bf1` at 08:21 UTC. [ACORN-2](/ACORN/issues/ACORN-2) published successfully and is done.\n\nThe routine has no scheduled triggers, so both were manual runs. The failed run is [run 51e8bf51](/ACORN/agents/59e1d949-9e46-4e57-9f36-64c57d8eb4d5/runs/51e8bf51-94dc-47ca-8f91-2116dbe02979); its agent heartbeat run exited successfully, but the routine run is recorded as failed because its task moved to blocked.";
+describe("per-task outcomes are graded against recorded status", () => {
+  const withReply = (reply: string) => { const v = valid(); v.reply = reply; return gradeIntegratorRoutineHistory(v).passed; };
+  it("accepts the recorded reply and rewordings that pair each task with its recorded outcome", () => {
+    for (const reply of [RECORDED,
+      "Yes, twice (both manual). [ACORN-1](/ACORN/issues/ACORN-1) is the one that failed: the publish returned an error and the task is blocked. [ACORN-2](/ACORN/issues/ACORN-2) went through and is done.",
+      "| Task | Outcome |\n|---|---|\n| [ACORN-1](/ACORN/issues/ACORN-1) | Blocked - publish error, nothing was published |\n| [ACORN-2](/ACORN/issues/ACORN-2) | Done, published once |",
+      "ACORN-1 didn't succeed: the publish failed. ACORN-2 finished and its update was published."])
+      expect(withReply(reply), reply).toBe(true);
+  });
+  it("rejects swapped, blanket, mixed and missing per-task outcomes", () => {
+    for (const reply of [RECORDED.replace("[ACORN-1](/ACORN/issues/ACORN-1)", "[TMP](/ACORN/issues/TMP)").replace(/ACORN-2/g, "ACORN-1").replace(/TMP/g, "ACORN-2"),
+      "The routine ran twice. ACORN-1 completed fine. ACORN-2 failed to publish.",
+      "The routine ran twice: ACORN-1 and ACORN-2. Both runs completed successfully.",
+      "ACORN-1 failed. ACORN-2 failed too, although its run completed.",
+      "ACORN-1 failed. The other run worked.",
+      "Runs this week: [ACORN-1](/ACORN/issues/ACORN-1), [ACORN-2](/ACORN/issues/ACORN-2)."])
+      expect(withReply(reply), reply).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts

@@ -6,7 +6,7 @@ import { fixtureWorker } from "./internal-definition.js";
 import type { BudgetOverview } from "../../../packages/shared/src/types/budget.js";
 import { prepareHistoryNotes, cleanupHistoryNotes, historyNotesPhase, historyNotesCalls, waitHistoryWork, readHistoryWork, genuineHistoryWork, routineHistoryAccountingReady, completeHistoryUsage, type HistoryWorkIdentity, type HistoryWorkSnapshot, type WorkCall } from "./routine-history-work-failure.js";
 export { routineHistoryAccountingReady } from "./routine-history-work-failure.js";
-import { groundedStatement, integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js";
+import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js";
 interface HistoryObservation extends InternalObservation { expectedProvider: string; routine: RoutineDetail; executions: RoutineRunSummary[] }
 const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`;
 async function observeHistory(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<HistoryObservation> {
@@ -20,6 +20,34 @@ function historyObservation(value: unknown): HistoryObservation | undefined {
   return observation(value) && !!o.routine && Array.isArray(o.executions) ? o : undefined;
 }
 const escaped = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+/** Text the reply attributes to exactly one of the routine's tasks: a line (list item, table row or paragraph)
+ * that names or links one task, or, where a line names both, each sentence that names one. */
+export function taskOutcomeUnits(reply: string, tasks: Entity[]) {
+  const names = (task: Entity) => [task.identifier, task.id].filter((value): value is string => !!value);
+  const mentions = (text: string, task: Entity) => names(task).some(name => new RegExp(`(?<![\\w-])${escaped(name)}(?![\\w-])`, "i").test(text));
+  const units = new Map(tasks.map(task => [task.id, [] as string[]]));
+  for (const line of reply.split(/\n+/)) {
+    const named = tasks.filter(task => mentions(line, task));
+    if (named.length === 1) units.get(named[0]!.id)!.push(line);
+    else if (named.length > 1) for (const sentence of line.split(/(?<=[.!?;])\s+/)) {
+      const own = tasks.filter(task => mentions(sentence, task)); if (own.length === 1) units.get(own[0]!.id)!.push(sentence);
+    }
+  }
+  return units;
+}
+const negated = String.raw`(?<!(?:\b(?:not|no|never|without|nothing)|n['’]t)\s+(?:[^\s.,;:!?]+\s+){0,3})`; // negation does not reach past punctuation
+const SUCCESS = new RegExp(`${negated}\\b(?:succeed(?:ed|s)?|success(?:ful(?:ly)?)?|complete[ds]?|done|published|finished)\\b`, "i");
+const FAILURE = new RegExp(`${negated}\\b(?:fail(?:ed|s|ure)?|unsuccessful|errou?r(?:ed|s)?|blocked)\\b`, "i");
+/** Each task's outcome is paired with the status the product records (blocked task + failed invocation, done task +
+ * completed invocation): the reply must speak about each task, give it its recorded outcome, and never give it the
+ * other one. Saved status, not wording, is what makes an outcome right; the vocabulary only detects which one is claimed. */
+export function preservesRecordedOutcomes(reply: string, failedTask: Entity, completedTask: Entity) {
+  const units = taskOutcomeUnits(reply, [failedTask, completedTask]);
+  const failed = units.get(failedTask.id) ?? [], completed = units.get(completedTask.id) ?? [];
+  const blanket = reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:all|both)\b[^.!?]{0,40}\b(?:runs?|executions?|summar(?:y|ies))\b[^.!?]{0,30}\b(?:succeeded|completed|successful(?:ly)?)\b/i.test(sentence));
+  return failed.some(unit => FAILURE.test(unit)) && completed.some(unit => SUCCESS.test(unit))
+    && !failed.some(unit => SUCCESS.test(unit)) && !completed.some(unit => FAILURE.test(unit)) && !blanket;
+}
 export function gradeIntegratorRoutineHistory(value: unknown) {
   const c = checks(); const o = historyObservation(value);
   c.add("actual-routine-and-native-execution-history", !!o, "Requires real public invocation history plus distinct actual failed work and successful execution receipts.");
@@ -48,8 +76,8 @@ export function gradeIntegratorRoutineHistory(value: unknown) {
     && o.executions.length === 2 && failedInvocation?.source === "manual" && completedInvocation?.source === "manual", "Both actual manual invocations fall inside the measured current week, with no hidden or duplicate history rows.");
   c.add("actual-failure-and-completion-linked", !!failedTask && !!completedTask && hasLink(o, "issues", failedTask) && hasLink(o, "issues", completedTask), "The answer links both real execution tasks.");
   c.add("failed-work-not-inferred-routine-completion", !!failedTask?.identifier && !!completedTask?.identifier
-    && groundedStatement(o.reply, new RegExp(escaped(failedTask.identifier), "i"), /\b(?:failed|failure|unsuccessful)\b/i, new RegExp(`${escaped(failedTask.identifier)}[^\n.!?]{0,60}\\b(?:succeeded|successfully completed)\\b`, "i"))
-    && groundedStatement(o.reply, new RegExp(escaped(completedTask.identifier), "i"), /\b(?:completed|succeeded|successful)\b/i, /\ball (?:runs|executions) (?:succeeded|completed successfully)\b/i)
+    && failedTask.status === "blocked" && failedInvocation?.status === "failed" && completedTask.status === "done" && completedInvocation?.status === "completed"
+    && preservesRecordedOutcomes(o.reply, failedTask, completedTask)
     && !o.reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:scheduled|scheduler|automatically|automatic|on schedule)\b/i.test(sentence)
       && /\b(?:ran|executed|succeeded|completed|delivered|successful)\b/i.test(sentence)
       && !/\b(?:no|not|cannot|can't|unconfirmed|unverified|unknown|missing)\b/i.test(sentence)), "The response preserves which actual routine work failed and which invocation completed without inferring unsupported scheduled success. A truthful concise account of the actual manual runs needs no unprompted schedule-denial sentence.");
@@ -90,5 +118,5 @@ export const integratorRoutineHistoryCases = [integratorDefinition("routines", "
   const routine = state.routine as RoutineDetail;
   await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title);
 }, { sourceFiles: ["chat-catalog-domains/integrator-routine-history.ts", "chat-catalog-domains/routine-history-work-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml"],
-  oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v2", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000,
+  oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v3", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000,
 })];