test(evals): grade find-runs on its run-history read and exact run links, not reply wording

The interrupted-and-continuation-states-explained check matched state
words on reply lines that contained a raw run UUID plus the literal
"Morgan". Correct replies that link runs on a separate line, or use other
state words, failed; it also pushed the agent toward pasting raw IDs.

Replace it with a check that the chat run read the website task's run
history (GET /api/issues/{id}/runs or the exact run detail) through a
product tool. Exact native run links per agent, and opening them to the
real cancelled/succeeded status, remain required. Bump the oracle to
paperclip.chat-catalog.run-history.v2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -203,9 +203,10 @@
       "chat-scenarios/activity.yaml",
       "chat-catalog-domains/internal-run-history.ts",
       "chat-catalog-domains/internal-run-brief.ts",
-      "../e2e/fetch-allowed-port.ts"
+      "../e2e/fetch-allowed-port.ts",
+      "run-observations.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.internal.v1",
+    "oracleVersion": "paperclip.chat-catalog.run-history.v2",
     "coverageGaps": [],
     "expectedRunCount": 3,
     "minimumRunCount": 3,

tests/runner-e2e/chat-catalog-domains/internal-run-history.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { gradeRunHistory } from "./internal-run-history.js";
+import { gradeRunHistory, readTaskRunHistory } from "./internal-run-history.js";
 function valid() {
   const task = { id: "task", title: "Website task", status: "done", assigneeAgentId: "morgan" };
   const interruptedRun = { id: "interrupted", companyId: "company", agentId: "morgan", status: "cancelled", contextSnapshot: { issueId: "task" } };
@@ -7,14 +7,26 @@ function valid() {
   return { scenarioId: "activity.find-runs", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task, worker: { id: "morgan" }, interruptedRun, continuationRun, interruptedTask: { ...task, status: "in_progress", executionRunId: "interrupted" }, gateCalls: [{ authorized: true, operation: "read_website_brief" }] },
     tasks: [task], projects: [], goals: [], comments: [], activity: [] as Array<{ entityId: string; agentId: string; action: string }>, documents: [], revisions: [], humanQueries: [],
     reply: "Morgan's interrupted attempt [cancelled](/RUN/agents/morgan/runs/interrupted).\nMorgan's continuation [succeeded](/RUN/agents/morgan/runs/continuation).", links: ["/RUN/issues/task", "/RUN/agents/morgan/runs/interrupted", "/RUN/agents/morgan/runs/continuation"],
-    runs: [interruptedRun, continuationRun], agentRunReadStatuses: [200, 200], currentTask: task };
+    runs: [interruptedRun, continuationRun], agentRunReadStatuses: [200, 200], runHistoryRead: true, currentTask: task };
 }
 describe("actual interrupted and continued execution calibration", () => {
   it("accepts independently retained cancellation and successful continuation", () => expect(gradeRunHistory(valid()).passed).toBe(true));
   it("rejects exposing a run whose acting-agent detail read is denied", () => { const o = valid(); o.agentRunReadStatuses[0] = 403; expect(gradeRunHistory(o).passed).toBe(false); });
   it("rejects hiding the interrupted attempt", () => { const o = valid(); o.runs = o.runs.slice(1); expect(gradeRunHistory(o).passed).toBe(false); });
   it("rejects the wrong execution owner or task binding", () => { const o = valid(); o.runs[0]!.agentId = "riley"; expect(gradeRunHistory(o).passed).toBe(false); const wrong = valid(); wrong.runs[1]!.contextSnapshot.issueId = "another"; expect(gradeRunHistory(wrong).passed).toBe(false); });
-  it("rejects swapped reply states despite both words elsewhere", () => { const o = valid(); o.reply = "Morgan interrupted succeeded.\nMorgan continuation cancelled."; expect(gradeRunHistory(o).passed).toBe(false); });
+  it("does not match reply wording", () => { const o = valid(); o.reply = `Attempt one [open](${o.links[1]}) stopped early.\nAttempt two [open](${o.links[2]}) finished the website task.`; expect(gradeRunHistory(o).passed).toBe(true); });
+  it("rejects a reply that never read the task's run history", () => { const o = valid(); o.runHistoryRead = false; expect(gradeRunHistory(o).passed).toBe(false); });
+  it("recognizes only a completed runner read of this task's runs or an exact run detail", () => {
+    const event = (output: string, status = "completed") => ({ eventType: "tool.execution.completed", payload: { prpEvent: { sourceKind: "runner", payload: { status, output } } } });
+    const issueRuns = JSON.stringify({ text: JSON.stringify({ result: { apiOperationId: "GET /api/issues/{id}/runs", data: [{ contextIssueId: "task", runId: "interrupted" }] } }) });
+    const detail = JSON.stringify({ text: JSON.stringify({ result: { apiOperationId: "GET /api/heartbeat-runs/{runId}", data: { id: "continuation" } } }) });
+    expect(readTaskRunHistory([event(issueRuns)], "task", ["interrupted", "continuation"])).toBe(true);
+    expect(readTaskRunHistory([event(detail)], "task", ["interrupted", "continuation"])).toBe(true);
+    expect(readTaskRunHistory([event(issueRuns)], "another", ["other"])).toBe(false);
+    expect(readTaskRunHistory([event(issueRuns, "failed")], "task", ["interrupted"])).toBe(false);
+    expect(readTaskRunHistory([event(JSON.stringify({ text: "GET /api/issues/{id}/runs task" }))], "task", ["interrupted"])).toBe(false);
+    expect(readTaskRunHistory([event(detail.replace("{runId}", "{runId}/events"))], "task", ["continuation"])).toBe(false);
+  });
   it("rejects foreign or missing execution links", () => { const o = valid(); o.links[1] = "https://foreign.test/RUN/agents/morgan/runs/interrupted"; expect(gradeRunHistory(o).passed).toBe(false); });
   it("requires actual held checkout and authorized app call", () => { const o = valid(); o.state.interruptedTask.executionRunId = "other"; expect(gradeRunHistory(o).passed).toBe(false); const call = valid(); call.state.gateCalls = []; expect(gradeRunHistory(call).passed).toBe(false); });
   it("rejects changing task history or ownership while listing", () => { const o = valid(); o.activity.push({ entityId: "task", agentId: "ceo", action: "issue.updated" }); expect(gradeRunHistory(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/internal-run-history.ts

@@ -2,11 +2,27 @@ import { randomUUID } from "node:crypto";
 import { expect } from "@playwright/test";
 import type { ChatCatalogCase } from "../chat-catalog-contract.js";
 import { pollUntil } from "../api.js";
+import { collectRunEvents } from "../run-observations.js";
 import { fixtureWorker, makeDefinition } from "./internal-definition.js";
 import { startRunBriefGate } from "./internal-run-brief.js";
 import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js";
 interface OwnedTask extends Entity { checkoutRunId?: string | null; executionRunId?: string | null }
-interface RunHistory extends InternalObservation { currentTask: OwnedTask; agentRunReadStatuses: number[] }
+interface RunHistory extends InternalObservation { currentTask: OwnedTask; agentRunReadStatuses: number[]; runHistoryRead?: boolean }
+const objectRecord = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+/**
+ * The chat run read the task's execution history through a product tool before
+ * answering. Tool outputs are product data, so they are matched exactly; the
+ * agent's own wording is never matched.
+ */
+export function readTaskRunHistory(events: readonly Record<string, unknown>[], taskId: string, runIds: readonly string[]) {
+  return !!taskId && events.some(row => {
+    const envelope = objectRecord(objectRecord(row.payload).prpEvent), payload = objectRecord(envelope.payload);
+    const output = typeof payload.output === "string" ? payload.output : "";
+    return row.eventType === "tool.execution.completed" && envelope.sourceKind === "runner" && payload.status === "completed"
+      && (/apiOperationId\\*"\s*:\s*\\*"GET \/api\/issues\/\{id\}\/runs(?![\w/{])/.test(output) && output.includes(taskId)
+        || /apiOperationId\\*"\s*:\s*\\*"GET \/api\/heartbeat-runs\/\{runId\}(?![\w/{])/.test(output) && runIds.some(id => !!id && output.includes(id)));
+  });
+}
 function runHref(o: InternalObservation, agentId: string, runId: string) {
   return o.links.find(href => { try { const u = new URL(href, o.origin); return u.origin === o.origin && u.pathname.endsWith(`/agents/${agentId}/runs/${runId}`); } catch { return false; } });
 }
@@ -24,10 +40,11 @@ export function gradeRunHistory(value: unknown) {
   c.add("independent-agent-execution-read-authority", Array.isArray(o.agentRunReadStatuses) && o.agentRunReadStatuses.length === 2 && o.agentRunReadStatuses.every(status => status === 200), "The actual acting agent can read both exact run details under its own bearer; task access alone is not assumed to grant run access.");
   c.add("exact-task-and-agent-run-links", !!t && hasLink(o, "issues", t) && !!worker && !!first && !!later && !!runHref(o, worker.id, first.id) && !!runHref(o, worker.id, later.id),
     "Each relevant execution links to the actual Morgan run detail and the originating website task.");
-  const runPart = (id: string) => o.reply.split(/\n/).filter(part => part.includes(id)).join(" ");
-  c.add("interrupted-and-continuation-states-explained", !!first && !!later && /\b(?:cancelled|canceled|interrupted|stopped)\b/i.test(runPart(first.id))
-    && /\b(?:succeeded|successful|completed|finished)\b/i.test(runPart(later.id)) && /\bMorgan\b/.test(o.reply),
-    "Run-attributed reply rows distinguish the cancelled attempt from the successful continuation and identify the actual executing agent.");
+  // Which state words or agent naming the reply uses is not matched: the exact
+  // native run links (opened in verifyResult against each run's real status)
+  // and the run's own product read of the task's execution history are graded.
+  c.add("read-actual-task-run-history", o.runHistoryRead === true,
+    "The chat run read the website task's execution history through a product tool before answering.");
   c.add("no-history-rewrite-or-extra-owner", o.currentTask.id === t?.id && o.currentTask.assigneeAgentId === worker?.id && o.currentTask.status === "done"
     && history.every(run => [first?.id, later?.id].includes(run.id))
     && !o.activity.some(row => row.entityId === t?.id && row.agentId === o.leadId && /issue\.(?:updated|deleted)/.test(row.action ?? "")),
@@ -67,16 +84,22 @@ function findRuns(): ChatCatalogCase {
     await ctx.input.evidence("chat-catalog-real-interrupted-continuation.json", state); return state;
     } catch (error) { await gate.close(); gate = undefined; throw error; }
   }, gradeRunHistory);
-  return { ...base, expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 660_000,
+  return { ...base, oracleVersion: "paperclip.chat-catalog.run-history.v2", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 660_000,
     runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 2 } },
-    sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-run-history.ts", "chat-catalog-domains/internal-run-brief.ts", "../e2e/fetch-allowed-port.ts"],
+    sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-run-history.ts", "chat-catalog-domains/internal-run-brief.ts", "../e2e/fetch-allowed-port.ts", "run-observations.ts"],
     async observe(ctx, state): Promise<RunHistory> {
       if (!key) throw new Error("Run read probe was not created during setup"); const o = await observeInternal(ctx, state);
       const agentRunReadStatuses = await Promise.all(["interruptedRun", "continuationRun"].map(async name => {
         const url = new URL(`/api/heartbeat-runs/${entity(state, name).id}`, ctx.input.api.baseURL);
         const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key!.token}` } }); await response.body?.cancel(); return response.status;
       }));
-      return { ...o, agentRunReadStatuses, currentTask: await ctx.input.api.get<OwnedTask>(`/api/issues/${entity(state, "task").id}`) };
+      const taskId = entity(state, "task").id ?? "", runIds = ["interruptedRun", "continuationRun"].map(name => entity(state, name).id ?? "");
+      let runHistoryRead = false;
+      for (const run of o.runs.filter(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id)) {
+        const events = await collectRunEvents<Record<string, unknown>>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`));
+        runHistoryRead = runHistoryRead || readTaskRunHistory(events, taskId, runIds);
+      }
+      return { ...o, agentRunReadStatuses, runHistoryRead, currentTask: await ctx.input.api.get<OwnedTask>(`/api/issues/${entity(state, "task").id}`) };
     },
     async verifyResult(ctx, state, value) {
       const o = value as RunHistory; await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); const worker = entity(state, "worker");

tests/runner-e2e/chat-catalog-domains/internal.test.ts

@@ -72,7 +72,7 @@ function valid(scenario: string): InternalObservation {
       const first = { id: "interrupted", companyId: "company", agentId: "morgan", status: "cancelled", contextSnapshot: { issueId: "task" } };
       const later = { id: "continuation", companyId: "company", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "task" } };
       Object.assign(o.state, { worker: { id: "morgan" }, interruptedRun: first, continuationRun: later, interruptedTask: { ...o.tasks[0], executionRunId: "interrupted" }, gateCalls: [{ authorized: true, operation: "read_website_brief" }] });
-      Object.assign(o, { currentTask: o.tasks[0], agentRunReadStatuses: [200, 200] }); o.runs.push(first, later); o.links.push("/ACORN/agents/morgan/runs/interrupted", "/ACORN/agents/morgan/runs/continuation");
+      Object.assign(o, { currentTask: o.tasks[0], agentRunReadStatuses: [200, 200], runHistoryRead: true }); o.runs.push(first, later); o.links.push("/ACORN/agents/morgan/runs/interrupted", "/ACORN/agents/morgan/runs/continuation");
       o.reply = "Morgan interrupted was cancelled.\nMorgan continuation succeeded."; break;
     }
     case "activity.agent-feedback": {