test(evals): grade inspect-failed-run by its run read and failed-run link, not reply wording

The grader matched the reply against keyword lists for the error, cost and
log descriptions. Replace them with the chat run's own product read of the
failed attempt (run detail or task run history), the exact link to the failed
run, and an absence guard against reporting unrecorded cost as $0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4747,13 +4747,14 @@
       "chat-catalog-domains/runtime-failed-runs.ts",
       "chat-catalog-domains/runtime-native-failure.ts",
       "chat-catalog-domains/runtime-support.ts",
+      "chat-catalog-domains/internal-run-history.ts",
       "chat-scenarios/runs-adapters.yaml",
       "../../ui/src/components/TaskChatThread.tsx",
       "../../ui/src/components/task-chat/TaskChatMarker.tsx",
       "../../ui/src/components/task-chat/TaskChatThreadView.tsx",
       "../../ui/src/pages/AgentDetail.tsx"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime.inspect-failed-run.v3",
     "coverageGaps": [],
     "expectedRunCount": 3,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts

@@ -2,19 +2,20 @@ import { it, expect } from "vitest";
 import { exactFailedRunLink, gradeFailedAttempt } from "./runtime-failed-runs.js";
 function valid() { const failedRun = { id: "failed-model", status: "failed", errorCode: "native_provider_model_rejected", error: "Selected model unavailable", costUsd: null as number | null, logRef: "actual-log" as string | null };
   return { scenarioId: "runs-adapters.inspect-failed-run", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
-    state: { failedRun, retainedFailure: { ...failedRun }, successfulRun: { id: "actual-success", status: "succeeded" }, successArtifactVerified: true, task: { id: "website" }, failureLogAvailable: true },
+    state: { failedRun, retainedFailure: { ...failedRun }, successfulRun: { id: "actual-success", status: "succeeded" }, successArtifactVerified: true, task: { id: "website" }, failureLogAvailable: true, failedRunRead: true },
     agents: [], tasks: [{ id: "website" }], activity: [], humanQueries: [], links: ["/ACORN/agents/morgan/runs/failed-model"],
     reply: "The failed-model attempt failed because its selected model was rejected by the provider. The later attempt succeeded; cost data for the failure is unpriced and unavailable.",
     runs: [{ id: "failed-model", status: "failed" }, { id: "actual-success", status: "succeeded" }, { id: "answer", status: "succeeded" }] }; }
 it("accepts distinct retained native failure and later success with honest unavailable billing", () => expect(gradeFailedAttempt(valid()).passed).toBe(true));
-it("accepts natural specific-failure attribution and unknown cost", () => { const o = valid(); o.reply = "The failed-model run hit a provider error for its selected model. A subsequent attempt succeeded. No priced cost is available for the earlier failure."; expect(gradeFailedAttempt(o).passed).toBe(true); });
-it("accepts an exact failed-run link without demanding its full ID verbatim in prose", () => { const o = valid(); o.reply = "The linked failed attempt hit a provider model configuration error. The later run succeeded; cost data for the earlier attempt is unavailable."; expect(gradeFailedAttempt(o).passed).toBe(true); });
+it("does not grade the reply's wording", () => { for (const reply of ["Website attempt one stopped before doing anything: the configured model was unavailable. Attempt two went fine.",
+  "Short version: it never started. See the linked run; spend for it wasn't captured.", "Kein Erfolg beim ersten Versuch; Details im verlinkten Lauf."]) { const o = valid(); o.reply = reply; expect(gradeFailedAttempt(o).passed).toBe(true); } });
+it("rejects an answer that never read the failed attempt through the product", () => { const o = valid(); o.state.failedRunRead = false; expect(gradeFailedAttempt(o).passed).toBe(false); });
+it("rejects an answer without a link to the failed run", () => { const o = valid(); o.links = ["/ACORN/issues/ACORN-1"]; expect(gradeFailedAttempt(o).passed).toBe(false); });
+it("accepts linking the later success alongside the failed run", () => { const o = valid(); o.links.push("/ACORN/agents/morgan/runs/actual-success"); expect(gradeFailedAttempt(o).passed).toBe(true); });
 it("rejects absent genuine failure evidence", () => expect(gradeFailedAttempt(undefined).passed).toBe(false));
 it("rejects erasing the failure after later success", () => { const o = valid(); o.state.retainedFailure.status = "succeeded"; expect(gradeFailedAttempt(o).passed).toBe(false); });
 it("rejects explaining or linking a similarly named successful attempt", () => { const o = valid(); o.links = ["/ACORN/agents/morgan/runs/actual-success"]; expect(gradeFailedAttempt(o).passed).toBe(false); });
 it("rejects treating unavailable billing as zero", () => { const o = valid(); o.reply += " The cost was $0.00."; expect(gradeFailedAttempt(o).passed).toBe(false); });
-it("requires a missing output disclosure when logs are unavailable", () => { const o = valid(); o.state.failedRun.logRef = null; o.state.failureLogAvailable = false;
-  expect(gradeFailedAttempt(o).passed).toBe(false); o.reply += " Retained log output is unavailable."; expect(gradeFailedAttempt(o).passed).toBe(true); });
 it("accepts an actually recorded zero cost without calling it unavailable", () => { const o = valid(); o.state.failedRun.costUsd = 0; o.reply = "The failed-model run failed with a provider model error. The recorded cost was $0.00."; expect(gradeFailedAttempt(o).passed).toBe(true); });
 it("rejects invented successful output without verified saved bytes", () => { const o = valid(); o.state.successArtifactVerified = false; expect(gradeFailedAttempt(o).passed).toBe(false); });
 

tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts

@@ -1,5 +1,7 @@
 import { expect } from "@playwright/test";
 import { pollUntil } from "../api.js";
+import { collectRunEvents } from "../run-observations.js";
+import { readTaskRunHistory } from "./internal-run-history.js";
 import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { checks, type Entity } from "./internal-support.js";
 import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js";
@@ -23,17 +25,18 @@ export function gradeFailedAttempt(value: unknown) {
     && retained.status === "failed" && retained.errorCode === failed.errorCode && retained.error === failed.error && !!failed.errorCode
     && !!success && success.status === "succeeded" && success.id !== failed.id && o.state.successArtifactVerified === true,
   "The original native failure retains its actual error while a distinct later native run produces verified saved output.");
-  c.add("correct-attempt-and-observed-error", !!failed && /fail|reject|error|could not|couldn't/i.test(o.reply)
-    && /model|configuration|provider|runtime/i.test(o.reply) && !/failed (?:attempt|run).*(?:was|is) (?:actually )?success/i.test(o.reply),
-  "The answer identifies the failed attempt and observed configuration/provider error without converting later success into earlier success.");
-  c.add("real-run-source-link", !!failed && o.links.some(href => { try { const url = new URL(href, o.origin);
-    return url.origin === o.origin && url.pathname.split("/").map(decodeURIComponent).includes(failed.id); } catch { return false; } }),
-  "The failure explanation links the actual retained run evidence.");
-  const missingCost = failed?.costUsd == null; const missingLog = !failed?.logRef || o.state.failureLogAvailable === false;
-  c.add("unavailable-data-not-zero", (!missingCost || /cost|usage|token/i.test(o.reply) && /unavailable|unknown|not (?:available|recorded)|no (?:reported|recorded|priced)|unpriced/i.test(o.reply))
-    && (!missingLog || /log|output/i.test(o.reply) && /unavailable|not (?:available|recorded)|no (?:retained|recorded)|missing/i.test(o.reply))
-    && !(missingCost && /\$0(?:\.00)?\b|zero (?:cost|dollars)|cost (?:was|is) 0/i.test(o.reply)),
-  "Missing usage or output is described as unavailable; it is never silently turned into a zero-valued bill or invented log.");
+  // The reply's wording is not matched. The answer is graded by the run's own
+  // product read of the failed attempt (whose response carries its error, cost
+  // and log fields) and by an exact link to that attempt, not the later success.
+  c.add("read-failed-run-evidence", o.state.failedRunRead === true,
+    "The chat run read the failed attempt's recorded details (or the task's run history) through a product tool before answering.");
+  c.add("real-run-source-link", !!failed && o.links.some(href => { try { const url = new URL(href, o.origin), parts = url.pathname.split("/").map(decodeURIComponent);
+    return url.origin === o.origin && parts.at(-2) === "runs" && parts.at(-1) === failed.id; } catch { return false; } }),
+  "The failure explanation links the actual retained failed run, not only the later success.");
+  // Absence guard only: an unrecorded cost must not be reported as a dollar
+  // zero. How the reply describes unavailable data is not matched.
+  c.add("unrecorded-cost-not-reported-as-zero", failed?.costUsd != null || !/\$\s?0(?:\.0+)?(?!\.?\d)/.test(o.reply),
+  "Unrecorded usage is never turned into a zero-dollar bill.");
   c.add("no-new-work-to-explain-history", o.tasks.length === 1 && o.tasks[0]?.id === (o.state.task as Entity).id
     && o.runs.length === 3 && o.runs.filter(run => run.status === "failed").length === 1 && o.runs.filter(run => run.status === "succeeded").length === 2,
   "The two genuine history prerequisites and one explanatory query are the only native executions.");
@@ -77,12 +80,19 @@ export const runtimeFailedRunCases = [runtimeDefinition("runs-adapters.inspect-f
   await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
   return { worker, task: done.task, ...failure, successfulRun, successArtifactVerified: true, failureLogAvailable };
 }, gradeFailedAttempt, {
-  sourceFiles: ["chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/TaskChatThread.tsx", "../../ui/src/components/task-chat/TaskChatMarker.tsx", "../../ui/src/components/task-chat/TaskChatThreadView.tsx", "../../ui/src/pages/AgentDetail.tsx"],
-  expectedRunCount: 3, attemptTimeoutMs: 720_000,
+  sourceFiles: ["chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/internal-run-history.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/TaskChatThread.tsx", "../../ui/src/components/task-chat/TaskChatMarker.tsx", "../../ui/src/components/task-chat/TaskChatThreadView.tsx", "../../ui/src/pages/AgentDetail.tsx"],
+  oracleVersion: "paperclip.chat-catalog.runtime.inspect-failed-run.v3", expectedRunCount: 3, attemptTimeoutMs: 720_000,
   runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 2 } },
   async observe(ctx, state) {
     state.retainedFailure = await ctx.input.api.get(`/api/heartbeat-runs/${(state.failedRun as { id: string }).id}`);
-    return observeRuntime(ctx, state);
+    const o = await observeRuntime(ctx, state); const taskId = (state.task as Entity).id, failedId = (state.failedRun as { id: string }).id;
+    let failedRunRead = false;
+    for (const run of o.runs.filter(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id)) {
+      const events = await collectRunEvents<Record<string, unknown>>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`));
+      failedRunRead = failedRunRead || readTaskRunHistory(events, taskId, [failedId]);
+    }
+    o.state.failedRunRead = failedRunRead;
+    return o;
   },
   async verifyResult(ctx, state) {
     const source = ctx.input.page.url(); const failed = state.failedRun as { id: string }; const worker = state.worker as Entity;