test(evals): grade routines.run-now on saved state and tool reads, not reply wording

The later-result check required keyword combinations in the follow-up reply and
rejected any mention of 'failed', so a correct answer saying 'nothing failed'
failed, while an acceptance-only reply could pass through the
'summary ... is' alternative. The routine output check likewise required
specific words in the routine's own comment.

Grade instead that the routine's native run read the actual Launch project
and saved its summary, and that the follow-up answer came from a new chat run
which read the completed result task after the invocation finished, with the
task linked. The first-reply guard is now sentence-scoped and ignores
future/conditional framing. Bump the case oracle to
integrator.routine-invocation.v2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4702,11 +4702,12 @@
     "scenarioId": "routines.run-now",
     "sourceFiles": [
       "chat-catalog-domains/integrator-routine-invocation.ts",
+      "run-observations.ts",
       "chat-catalog-domains/integrator-support.ts",
       "chat-catalog-domains/internal-support.ts",
       "chat-scenarios/routines.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.integrator.v1",
+    "oracleVersion": "paperclip.chat-catalog.integrator.routine-invocation.v2",
     "expectedRunCount": 3,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts

@@ -2,21 +2,37 @@ import { describe, expect, it } from "vitest";
 import { gradeIntegratorRoutineInvocation } from "./integrator-routine-invocation.js";
 function valid() {
   const task = { id: "task", identifier: "ACN-2", projectId: "project", assigneeAgentId: "ceo", status: "done", title: "Launch summary" };
-  const invocation = { id: "invocation", routineId: "routine", source: "manual", status: "completed", linkedIssueId: "task", idempotencyKey: "delivery", triggerPayload: { variables: { projectName: "Launch" } } };
+  const invocation = { id: "invocation", routineId: "routine", source: "manual", status: "completed", linkedIssueId: "task", idempotencyKey: "delivery",
+    completedAt: "2026-01-01T00:02:00.000Z", triggerPayload: { variables: { projectName: "Launch" } } };
+  const chatRun = { id: "chat-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "chat" } };
+  const outputRun = { id: "output-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "task" } };
+  const followUpRun = { id: "follow-up-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "chat" } };
   const base = { scenarioId: "routines.run-now", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" } },
     tasks: [task], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], links: ["/ACN/issues/ACN-2"],
     routine: { id: "routine" }, invocations: [invocation], outputComments: [{ id: "output", authorAgentId: "ceo", createdByRunId: "output-run", body: "Launch: payment approval is pending." }],
-    runs: [{ id: "output-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "task" } }], reply: "Accepted the Launch summary invocation; its task is waiting." };
+    runs: [chatRun, outputRun], reply: "Accepted the Launch summary invocation; its task is waiting.",
+    toolReads: [
+      { runId: "output-run", name: "paperclip_list_projects", at: "2026-01-01T00:01:30.000Z", output: '{"projects":[{"id":"project"}]}' },
+      { runId: "follow-up-run", name: "paperclip_get_task", at: "2026-01-01T00:03:00.000Z", output: '{"data":{"id":"task","status":"done"}}' },
+    ] };
   const first = structuredClone(base); first.invocations[0]!.status = "issue_created"; first.tasks[0]!.status = "todo";
-  return { ...base, state: { ...base.state, replayRunId: "invocation", replayIssueId: "task", firstInvocation: first },
+  return { ...base, runs: [...base.runs, followUpRun], state: { ...base.state, replayRunId: "invocation", replayIssueId: "task", firstInvocation: first },
     reply: `${first.reply}\nThe summary completed: Launch is still waiting on payment approval. Here is its result task.` };
 }
+type Observation = ReturnType<typeof valid>;
+const withLater = (o: Observation, later: string) => ({ ...o, reply: `${o.state.firstInvocation.reply}\n${later}` });
 describe("real manual invocation and delivery distinction", () => {
   it("accepts substantive actual result and refuses missing receipts", () => {
     expect(gradeIntegratorRoutineInvocation(valid()).passed).toBe(true);
     expect(gradeIntegratorRoutineInvocation(undefined).passed).toBe(false);
     const o = valid(); o.outputComments = [];
     expect(gradeIntegratorRoutineInvocation(o).passed).toBe(false);
+    const noReads = valid(); delete (noReads as Partial<Observation>).toolReads;
+    expect(gradeIntegratorRoutineInvocation(noReads).passed).toBe(false);
+  });
+  it("does not depend on the wording of a correct later result", () => {
+    expect(gradeIntegratorRoutineInvocation(withLater(valid(), "RUN-2 is closed and nothing failed. Per its saved comment, the payment sign-off has not come through yet.")).passed).toBe(true);
+    expect(gradeIntegratorRoutineInvocation(withLater(valid(), "It's done — approval for payment is outstanding.")).passed).toBe(true);
   });
   it("rejects duplicate delivery, wrong input and unrelated native output", () => {
     const duplicate = valid(); duplicate.invocations.push({ ...duplicate.invocations[0]!, id: "another" });
@@ -25,16 +41,28 @@ describe("real manual invocation and delivery distinction", () => {
     expect(gradeIntegratorRoutineInvocation(wrongInput).passed).toBe(false);
     const wrongReplay = valid(); wrongReplay.state.replayIssueId = "other-task";
     expect(gradeIntegratorRoutineInvocation(wrongReplay).passed).toBe(false);
-    const wrongRun = valid(); wrongRun.runs[0]!.contextSnapshot.issueId = "other-task";
+    const wrongRun = valid(); wrongRun.runs[1]!.contextSnapshot.issueId = "other-task";
     expect(gradeIntegratorRoutineInvocation(wrongRun).passed).toBe(false);
+    const unread = valid(); unread.toolReads = unread.toolReads.filter(row => row.runId !== "output-run");
+    expect(gradeIntegratorRoutineInvocation(unread).passed).toBe(false);
   });
-  it("rejects false early completion, later acceptance-only and invented approved outcome", () => {
+  it("rejects false early completion and a later answer not grounded on the completed result", () => {
     const early = valid(); early.state.firstInvocation.reply = "The Launch summary completed successfully.";
     early.reply = `${early.state.firstInvocation.reply}\nThe summary completed; payment approval remains pending for Launch.`;
     expect(gradeIntegratorRoutineInvocation(early).passed).toBe(false);
-    const o = valid();
-    expect(gradeIntegratorRoutineInvocation({ ...o, reply: `${o.state.firstInvocation.reply}\nI accepted the Launch summary, waiting for payment approval.` }).passed).toBe(false);
-    expect(gradeIntegratorRoutineInvocation({ ...o, reply: `${o.state.firstInvocation.reply}\nThe Launch summary completed. Payment is approved.` }).passed).toBe(false);
+    const future = valid(); future.state.firstInvocation.reply = "I started the routine; RUN-2 will be marked done once the summary is finished.";
+    future.reply = `${future.state.firstInvocation.reply}\nThe summary completed; payment approval remains pending.`;
+    expect(gradeIntegratorRoutineInvocation(future).passed).toBe(true);
+    const recalled = valid(); recalled.toolReads = recalled.toolReads.filter(row => row.runId !== "follow-up-run");
+    expect(gradeIntegratorRoutineInvocation(recalled).passed).toBe(false);
+    const stale = valid(); stale.toolReads[1]!.at = "2026-01-01T00:01:00.000Z";
+    expect(gradeIntegratorRoutineInvocation(stale).passed).toBe(false);
+    const noFollowUp = valid(); noFollowUp.runs = noFollowUp.runs.filter(run => run.id !== "follow-up-run");
+    expect(gradeIntegratorRoutineInvocation(noFollowUp).passed).toBe(false);
+    const silent = valid(); silent.reply = silent.state.firstInvocation.reply;
+    expect(gradeIntegratorRoutineInvocation(silent).passed).toBe(false);
+    const unlinked = valid(); unlinked.links = [];
+    expect(gradeIntegratorRoutineInvocation(unlinked).passed).toBe(false);
   });
 });
 

tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts

@@ -3,51 +3,82 @@ import { createRoutineSchema } from "../../../packages/shared/src/validators/rou
 import type { RoutineDetail, RoutineRunSummary } from "../../../packages/shared/src/types/routine.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import { pollUntil } from "../api.js";
+import { collectRunEvents } from "../run-observations.js";
 import { checks, hasLink, observation, project, type Entity, type InternalObservation } from "./internal-support.js";
 import { integratorDefinition, observeInternal, inspectResult, resultHref } from "./integrator-support.js";
 
-interface InvocationObservation extends InternalObservation { routine: RoutineDetail; invocations: RoutineRunSummary[]; outputComments: Entity[] }
-async function observeInvocation(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<InvocationObservation> {
+/** One completed product tool execution, as recorded in the run's durable event stream. */
+export interface InvocationToolRead { runId: string; name: string; at: string; output: string }
+interface InvocationObservation extends InternalObservation { routine: RoutineDetail; invocations: RoutineRunSummary[]; outputComments: Entity[]; toolReads?: InvocationToolRead[] }
+async function invocationToolReads(ctx: ChatCatalogContext, runs: InternalObservation["runs"]): Promise<InvocationToolRead[]> {
+  const reads: InvocationToolRead[] = [];
+  for (const run of runs) {
+    const events = await collectRunEvents<{ seq?: number; createdAt?: string; payload?: Record<string, unknown> | null }>((afterSeq, limit) =>
+      ctx.input.api.getRunObservation(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`));
+    for (const event of events) {
+      const prp = event.payload?.prpEvent as { eventType?: string; payload?: { name?: unknown; status?: unknown; output?: unknown } } | undefined;
+      if (prp?.eventType !== "tool.execution.completed" || prp.payload?.status !== "completed" || typeof prp.payload.name !== "string") continue;
+      const output = typeof prp.payload.output === "string" ? prp.payload.output : JSON.stringify(prp.payload.output ?? "");
+      reads.push({ runId: run.id, name: prp.payload.name, at: event.createdAt ?? "", output: output.slice(0, 8192) });
+    }
+  }
+  return reads;
+}
+async function observeInvocation(ctx: ChatCatalogContext, state: Record<string, unknown>, withToolReads = false): Promise<InvocationObservation> {
   const base = await observeInternal(ctx, state); const id = (state.routine as RoutineDetail).id;
   const invocations = await ctx.input.api.get<RoutineRunSummary[]>(`/api/routines/${id}/runs`);
   return { ...base, routine: await ctx.input.api.get<RoutineDetail>(`/api/routines/${id}`), invocations,
-    outputComments: invocations[0]?.linkedIssueId ? await ctx.input.api.get<Entity[]>(`/api/issues/${invocations[0].linkedIssueId}/comments?order=asc`) : [] };
+    outputComments: invocations[0]?.linkedIssueId ? await ctx.input.api.get<Entity[]>(`/api/issues/${invocations[0].linkedIssueId}/comments?order=asc`) : [],
+    ...(withToolReads ? { toolReads: await invocationToolReads(ctx, base.runs) } : {}) };
 }
 function invocationObservation(value: unknown): InvocationObservation | undefined {
   const o = value as InvocationObservation;
   return observation(value) && !!o.routine && Array.isArray(o.invocations) && Array.isArray(o.outputComments) ? o : undefined;
 }
-function falseCompletion(reply: string) {
-  return /\b(?:summary|execution|task|run)\b[^.!?\n]{0,50}\b(?:completed|finished|succeeded|is ready)\b/i.test(reply)
-    && !/\b(?:not|hasn't|has not|isn't|not yet|pending|waiting)\b/i.test(reply);
+/** Narrow contradiction guard for the first answer only: a sentence that states the summary,
+ * run or task as already done, without negation or a future/conditional frame. It never makes
+ * a valid answer depend on particular wording; it only refuses an explicit premature claim. */
+function prematureCompletionClaim(reply: string) {
+  return reply.split(/(?<=[.!?])\s+|\n+/).some(sentence =>
+    /\b(?:summary|execution|task|run)\b[^.!?\n]{0,50}\b(?:completed|finished|succeeded|is ready|is done)\b/i.test(sentence)
+    && !/\bnot\b|n't\b|\b(?:pending|waiting|will|once|when|after|until|should|expected|queued|started|in progress)\b/i.test(sentence));
+}
+/** The run read a product record naming `id` at or after `since`. Call-API payloads are redacted
+ * in run evidence, so only tool outputs that expose the record count as proof of the read. */
+function readRecordAfter(o: InvocationObservation, runId: string | undefined, id: string | undefined, since?: string | null) {
+  return !!runId && !!id && (o.toolReads ?? []).some(row => row.runId === runId && row.output.includes(id)
+    && (!since || Date.parse(row.at) >= Date.parse(since)));
 }
 export function gradeIntegratorRoutineInvocation(value: unknown) {
   const c = checks(); const o = invocationObservation(value);
-  c.add("actual-invocation-and-output", !!o, "Requires public invocation, task, native output and original response evidence.");
+  c.add("actual-invocation-and-output", !!o && Array.isArray(o.toolReads), "Requires public invocation, task, native output, tool-read and original response evidence.");
   if (!o) return c.result();
   const invocation = o.invocations[0]; const task = o.tasks.find(row => row.id === invocation?.linkedIssueId);
   const first = invocationObservation(o.state.firstInvocation);
   const firstInvocation = first?.invocations[0];
+  const projectId = (o.state.project as Entity | undefined)?.id;
   c.add("one-manual-launch-input", o.invocations.length === 1 && invocation?.source === "manual" && invocation.routineId === o.routine.id
     && invocation.triggerPayload?.variables != null && (invocation.triggerPayload.variables as Record<string, unknown>).projectName === "Launch"
-    && !!task && task.projectId === (o.state.project as Entity | undefined)?.id && task.assigneeAgentId === o.leadId,
+    && !!task && task.projectId === projectId && task.assigneeAgentId === o.leadId,
   "Exactly one manual invocation retains the supplied project-name input and creates the intended owned Launch task.");
   c.add("same-delivery-is-not-new-intent", !!firstInvocation?.idempotencyKey && o.state.replayRunId === invocation?.id
     && o.state.replayIssueId === task?.id && firstInvocation.id === invocation?.id && o.invocations.length === 1,
   "Replaying the committed public invocation with its original key retains the same invocation and task IDs.");
-  c.add("accepted-versus-completed-at-first-reply", !!first && first.invocations.length === 1
-    && (!falseCompletion(first.reply) || firstInvocation?.status === "completed" && first.tasks.some(row => row.id === firstInvocation.linkedIssueId && row.status === "done")),
+  c.add("accepted-versus-completed-at-first-reply", !!first && first.invocations.length === 1 && first.reply.trim().length > 0
+    && (!prematureCompletionClaim(first.reply) || firstInvocation?.status === "completed" && first.tasks.some(row => row.id === firstInvocation.linkedIssueId && row.status === "done")),
   "The first answer cannot claim a completed summary before the saved invocation and task actually complete.");
   const outputRun = o.runs.find(run => run.contextSnapshot?.issueId === task?.id && run.agentId === o.leadId && run.status === "succeeded" && run.runtimeMode === "native");
-  const output = o.outputComments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId === outputRun?.id).map(row => row.body ?? "").join("\n");
-  c.add("actual-completed-summary", !!task && invocation?.status === "completed" && task.status === "done" && !!outputRun && /launch/i.test(output)
-    && /payment/i.test(output) && /\b(?:pending|waiting|await|approval)\b/i.test(output),
-  "The routine's own native task actually saved the Launch summary using the pending payment fact and completed normally.");
-  const later = first && o.reply.startsWith(first.reply) ? o.reply.slice(first.reply.length) : "";
-  c.add("later-actual-result-and-task-link", !!task && hasLink(o, "issues", task) && /launch/i.test(later) && /payment/i.test(later)
-    && /\b(?:pending|waiting|await|approval)\b/i.test(later) && /\b(?:completed|finished)\b|\b(?:summary|result)\b[^.!?\n]{0,25}(?:\b(?:reports?|shows?|contains?|says?|found|is)\b|:)/i.test(later)
-    && !/\b(?:failed|not completed|not finished|still running|only accepted|payment (?:is )?approved)\b/i.test(later),
-  "The explicit later user request receives the real summary result and its actual execution task, not another acceptance or an invented payment approval.");
+  const output = o.outputComments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId === outputRun?.id && (row.body ?? "").trim().length > 0);
+  c.add("actual-completed-summary", !!task && invocation?.status === "completed" && task.status === "done" && !!outputRun && output.length > 0
+    && readRecordAfter(o, outputRun?.id, projectId),
+  "The routine's own native task run read the actual Launch project, saved its summary on the task and completed normally.");
+  // The follow-up answer comes from a chat run that did not exist at the first reply and is not the routine's own task run.
+  const followUp = o.runs.find(run => run.agentId === o.leadId && run.status === "succeeded" && run.contextSnapshot?.issueId !== task?.id
+    && !!first && !first.runs.some(row => row.id === run.id));
+  const later = first && o.reply.startsWith(first.reply) ? o.reply.slice(first.reply.length).trim() : "";
+  c.add("later-actual-result-and-task-link", !!task && hasLink(o, "issues", task) && later.length > 0 && !!followUp
+    && readRecordAfter(o, followUp?.id, task.id, invocation?.completedAt ? new Date(invocation.completedAt).toISOString() : undefined),
+  "The explicit later request is answered by a new chat run that read the completed result task after the invocation finished, and the conversation links that task.");
   return c.result();
 }
 /** Validate prerequisite input against the same supported public creation schema. */
@@ -60,12 +91,13 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines"
   const p = await project(ctx, "Launch", { description: "Synthetic Launch project. Payment approval is pending; no approval has been received." });
   const routine = await ctx.input.api.post<RoutineDetail>(`/api/companies/${ctx.input.fixtures.company.id}/routines`, launchSummaryRoutineInput(p.id, ctx.input.fixtures.agent.id));
   return { project: p, routine };
-}, observeInvocation, gradeIntegratorRoutineInvocation, async (ctx, _state, value) => {
+}, (ctx, state) => observeInvocation(ctx, state, true), gradeIntegratorRoutineInvocation, async (ctx, _state, value) => {
   const o = invocationObservation(value); const task = o?.tasks.find(row => row.id === o.invocations[0]?.linkedIssueId);
   if (!o || !task) throw new Error("Missing actual routine result task");
   await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, task.title!, resultHref(o, task.identifier ?? task.id, "issues") ?? resultHref(o, task.id, "issues"));
 }, {
-  sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"],
+  oracleVersion: "paperclip.chat-catalog.integrator.routine-invocation.v2",
+  sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"],
   expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000,
   async act(ctx, state) {
     const first = await pollUntil({ label: "routine invocation first actual reply", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000,