test(evals): ignore quoted tool output when reading read-history outcomes

Run 42594125035 replied correctly but quoted the provider's `success: false`
on the failed task's line, which the v3 oracle read as a success claim. Quoted
spans are cited data, not the reply's own outcome (task ids in quotes are kept).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -697,7 +697,7 @@
       "chat-catalog-domains/identity-support.ts",
       "chat-scenarios/agents.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v3",
+    "oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v4",
     "coverageGaps": [],
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,
@@ -4688,7 +4688,7 @@
       "chat-catalog-domains/integrator-support.ts",
       "chat-scenarios/routines.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v3",
+    "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v4",
     "expectedRunCount": 3,
     "expectedHumanMessageCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts

@@ -128,10 +128,12 @@ describe("complete normal accounting stays mandatory", () => {
 });
 // Recorded reply from run 42584001490 (prefix renamed); it failed v2 only because "published successfully and is done" missed a keyword.
 const RECORDED = "Yes \u2014 the **Morning Launch summary** routine ran twice this week, both today (Nov 9):\n\n- **Failed:** run `51e8bf51` at 08:20 UTC. [ACORN-1](/ACORN/issues/ACORN-1) was blocked because the Synthetic Launch Notes `launch_update` publish returned `Synthetic write failed; page unchanged`. It remains blocked; owner is the Synthetic Launch Notes provider, and the unblock action is restoring `launch_update`.\n- **Completed:** run `4b180bf1` at 08:21 UTC. [ACORN-2](/ACORN/issues/ACORN-2) published successfully and is done.\n\nThe routine has no scheduled triggers, so both were manual runs. The failed run is [run 51e8bf51](/ACORN/agents/59e1d949-9e46-4e57-9f36-64c57d8eb4d5/runs/51e8bf51-94dc-47ca-8f91-2116dbe02979); its agent heartbeat run exited successfully, but the routine run is recorded as failed because its task moved to blocked.";
+// Second recorded reply (run 42594125035): v3 first misread the quoted tool output `success: false` as a success claim.
+const RECORDED_2 = "Yes \u2014 the \"Morning Launch summary\" routine ran this week, twice this morning (Mon, Nov 9).\n\n- **Failed run:** `f0c38bf6\u2026` at 09:41 \u2014 its task moved to blocked because the Synthetic Launch Notes `launch_update` returned `success: false` (\"Synthetic write failed; page unchanged\"). [ACORN-1](/ACORN/issues/ACORN-1) remains blocked.\n- **Succeeded run:** `b050bd96\u2026` at 09:42 \u2014 published successfully; [ACORN-2](/ACORN/issues/ACORN-2) is done.\n\nNo agent heartbeat/execution runs are marked failed \u2014 all three recorded runs show `succeeded`; the failure is recorded at the routine-run level.";
 describe("per-task outcomes are graded against recorded status", () => {
   const withReply = (reply: string) => { const v = valid(); v.reply = reply; return gradeIntegratorRoutineHistory(v).passed; };
   it("accepts the recorded reply and rewordings that pair each task with its recorded outcome", () => {
-    for (const reply of [RECORDED,
+    for (const reply of [RECORDED, RECORDED_2, "`ACORN-1` failed. `ACORN-2` is done.",
       "Yes, twice (both manual). [ACORN-1](/ACORN/issues/ACORN-1) is the one that failed: the publish returned an error and the task is blocked. [ACORN-2](/ACORN/issues/ACORN-2) went through and is done.",
       "| Task | Outcome |\n|---|---|\n| [ACORN-1](/ACORN/issues/ACORN-1) | Blocked - publish error, nothing was published |\n| [ACORN-2](/ACORN/issues/ACORN-2) | Done, published once |",
       "ACORN-1 didn't succeed: the publish failed. ACORN-2 finished and its update was published."])
@@ -143,6 +145,7 @@ describe("per-task outcomes are graded against recorded status", () => {
       "The routine ran twice: ACORN-1 and ACORN-2. Both runs completed successfully.",
       "ACORN-1 failed. ACORN-2 failed too, although its run completed.",
       "ACORN-1 failed. The other run worked.",
+      RECORDED_2.replace("[ACORN-1](/ACORN/issues/ACORN-1) remains blocked", "[ACORN-1](/ACORN/issues/ACORN-1) is done"),
       "Runs this week: [ACORN-1](/ACORN/issues/ACORN-1), [ACORN-2](/ACORN/issues/ACORN-2)."])
       expect(withReply(reply), reply).toBe(false);
   });

tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts

@@ -42,9 +42,14 @@ const FAILURE = new RegExp(`${negated}\\b(?:fail(?:ed|s|ure)?|unsuccessful|errou
  * completed invocation): the reply must speak about each task, give it its recorded outcome, and never give it the
  * other one. Saved status, not wording, is what makes an outcome right; the vocabulary only detects which one is claimed. */
 export function preservesRecordedOutcomes(reply: string, failedTask: Entity, completedTask: Entity) {
-  const units = taskOutcomeUnits(reply, [failedTask, completedTask]);
+  // Quoted tool output (`success: false`, "Synthetic write failed") is data the reply cites, not its own claim.
+  const prose = reply.replace(/`([^`\n]*)`|"([^"\n]*)"|“([^”\n]*)”/g, (_quote, ...spans: Array<string | undefined>) => {
+    const text = spans.slice(0, 3).find(span => span !== undefined) ?? "";
+    return /^[A-Z][A-Z0-9]*-\d+$/.test(text.trim()) ? text : " ";
+  });
+  const units = taskOutcomeUnits(prose, [failedTask, completedTask]);
   const failed = units.get(failedTask.id) ?? [], completed = units.get(completedTask.id) ?? [];
-  const blanket = reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:all|both)\b[^.!?]{0,40}\b(?:runs?|executions?|summar(?:y|ies))\b[^.!?]{0,30}\b(?:succeeded|completed|successful(?:ly)?)\b/i.test(sentence));
+  const blanket = prose.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:all|both)\b[^.!?]{0,40}\b(?:runs?|executions?|summar(?:y|ies))\b[^.!?]{0,30}\b(?:succeeded|completed|successful(?:ly)?)\b/i.test(sentence));
   return failed.some(unit => FAILURE.test(unit)) && completed.some(unit => SUCCESS.test(unit))
     && !failed.some(unit => SUCCESS.test(unit)) && !completed.some(unit => FAILURE.test(unit)) && !blanket;
 }
@@ -118,5 +123,5 @@ export const integratorRoutineHistoryCases = [integratorDefinition("routines", "
   const routine = state.routine as RoutineDetail;
   await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title);
 }, { sourceFiles: ["chat-catalog-domains/integrator-routine-history.ts", "chat-catalog-domains/routine-history-work-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml"],
-  oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v3", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000,
+  oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v4", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000,
 })];