test(evals): grade the pause-agent case by saved state and the followed profile link

pending-human-step-honest matched reply keywords. It is replaced with
structural checks: Morgan and the worker run were still running when the
person took over, no agent-authored lifecycle write to Morgan succeeded,
the reply rendered a link to Morgan's profile, and the scripted person
reached the Pause control by clicking that link. The typed-URL fallback
is gone, and the actions menu is scoped to the profile header because the
sidebar has a same-named menu.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -713,7 +713,7 @@
       "chat-catalog-domains/runtime-support.ts",
       "chat-scenarios/agents.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime-pause.v3",
     "coverageGaps": [],
     "expectedRunCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts

@@ -2,14 +2,23 @@ import { it, expect } from "vitest";
 import { gradePauseAgent } from "./runtime-agent-lifecycle.js";
 function valid() {
   return { scenarioId: "agents.pause-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
-    state: { worker: { id: "morgan" }, workerRunId: "worker-run", workerWasReallyWaiting: true, beforeHumanStatus: "running", agentPauseStatus: 403, humanPaused: true },
+    state: { worker: { id: "morgan" }, workerRunId: "worker-run", workerWasReallyWaiting: true, beforeHumanStatus: "running", beforeHumanLifecycleState: "ready",
+      workerRunStatusBeforeHuman: "running", agentPauseStatus: 403, humanPaused: true, profileLinkFollowed: "/ACORN/agents/morgan" } as Record<string, unknown>,
     agents: [{ id: "morgan", companyId: "company", status: "paused", lifecycleState: "paused" }], tasks: [], links: ["/ACORN/agents/morgan"],
-    reply: "Open Morgan's profile and use the Pause control. Existing running work stops after cleanup; the pause remains until resumed.",
-    activity: [{ id: "pause", action: "agent.paused", entityId: "morgan", actorType: "user" }],
+    // The run 42557891748 reply, with its profile link.
+    reply: "I couldn't pause Morgan myself: the pause action is board-only and returned 403 \"Board access required,\" which is a final boundary for my credentials.\n\nA board user needs to do it from [Morgan's profile](/ACORN/agents/morgan) (Pause). Morgan is still running until that's done; tell me when it's set and I'll keep them out of any delegation.",
+    activity: [{ id: "pause", action: "agent.paused", entityId: "morgan", actorType: "user" } as Record<string, unknown>],
     runs: [{ id: "worker-run", status: "cancelled" }], humanQueries: [] };
 }
-it("accepts an actual human pause with stopped worker evidence", () => expect(gradePauseAgent(valid()).passed).toBe(true));
-it("accepts natural sustained-pause guidance", () => { const o = valid(); o.reply = "You can pause Morgan from the profile actions. Let the running turn finish its cleanup; Morgan then stays suspended until you resume."; expect(gradePauseAgent(o).passed).toBe(true); });
+it("accepts an actual human pause reached through the reply's profile link", () => expect(gradePauseAgent(valid()).passed).toBe(true));
+it("accepts the same outcome reworded, with no grading of the reply's words", () => {
+  const o = valid(); o.reply = "Only a board member can suspend Morgan. Head to [Morgan](/ACORN/agents/morgan) and choose it from the actions menu; the turn in flight is cancelled.";
+  expect(gradePauseAgent(o).passed).toBe(true);
+});
+it("accepts the profile link written with Morgan's URL key", () => {
+  const o = valid(); (o.state.worker as Record<string, unknown>).urlKey = "morgan-2"; o.links = ["/ACORN/agents/morgan-2"]; o.state.profileLinkFollowed = "/ACORN/agents/morgan-2";
+  expect(gradePauseAgent(o).passed).toBe(true);
+});
 it("rejects missing pause evidence", () => expect(gradePauseAgent(undefined).passed).toBe(false));
 it("rejects acknowledgement without a saved human pause", () => {
   const o = valid(); o.state.humanPaused = false; expect(gradePauseAgent(o).passed).toBe(false);
@@ -20,7 +29,18 @@ it("rejects a paused label while the old process still runs", () => {
 it("rejects an agent-authored pause attribution", () => {
   const o = valid(); o.activity[0]!.actorType = "agent"; expect(gradePauseAgent(o).passed).toBe(false);
 });
-
-it("accepts correct human pause guidance without requiring a profile hyperlink", () => {
-  const o = valid(); o.links = []; expect(gradePauseAgent(o).passed).toBe(true);
+it("rejects Morgan already paused before the person acted", () => {
+  const o = valid(); o.state.beforeHumanStatus = "paused"; expect(gradePauseAgent(o).passed).toBe(false);
+  const p = valid(); p.state.workerRunStatusBeforeHuman = "cancelled"; expect(gradePauseAgent(p).passed).toBe(false);
+});
+it("rejects a successful agent status write, even beside the human pause", () => {
+  const o = valid(); o.activity.push({ id: "patch", action: "agent.updated", entityId: "morgan", actorType: "agent", agentId: "ceo" });
+  expect(gradePauseAgent(o).passed).toBe(false);
+});
+it("rejects a reply with no rendered link to Morgan's profile", () => {
+  const o = valid(); o.links = []; expect(gradePauseAgent(o).passed).toBe(false);
+  const p = valid(); p.links = ["/ACORN/agents/ceo"]; expect(gradePauseAgent(p).passed).toBe(false);
+});
+it("rejects a human pause that did not follow the reply's link", () => {
+  const o = valid(); delete o.state.profileLinkFollowed; expect(gradePauseAgent(o).passed).toBe(false);
 });

tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts

@@ -3,7 +3,7 @@ import { expect } from "@playwright/test";
 import { pollUntil } from "../api.js";
 import { checks } from "./internal-support.js";
 import { probeAgentPermission } from "./identity-support.js";
-import { runtimeDefinition, runtimeObservation, observeRuntime, openRuntimeAgentProfile, type RuntimeAgent } from "./runtime-support.js";
+import { agentLink, runtimeDefinition, runtimeObservation, observeRuntime, openRuntimeAgentProfile, type RuntimeAgent } from "./runtime-support.js";
 import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
 export function gradePauseAgent(value: unknown) {
   const c = checks(); const o = runtimeObservation(value);
@@ -17,15 +17,22 @@ export function gradePauseAgent(value: unknown) {
     "The real profile Pause control persists suspension and confirms the original process stopped.");
   c.add("human-attribution", o.activity.some(row => row.action === "agent.paused" && row.entityId === worker.id
     && row.actorType === "user" && !row.agentId), "The pause audit belongs to the authorized human.");
-  c.add("pending-human-step-honest", /profile|pause button|pause control|you can pause|you.*pause/i.test(o.reply)
-    && /stop|settle|current|existing|running/i.test(o.reply) && !/I(?:'ve| have)? (?:now )?paused/i.test(o.reply),
-    "The reply identifies the exact human step and explains how existing execution settles without falsely claiming the pause.");
+  // Graded from saved state and the rendered link, never from reply wording.
+  c.add("still-running-before-human", o.state.beforeHumanStatus !== "paused" && o.state.beforeHumanLifecycleState !== "paused"
+    && o.state.workerRunStatusBeforeHuman === "running", "Morgan was still running when the person took over, so chat did not pause Morgan.");
+  c.add("no-agent-pause-write", !o.activity.some(row => row.entityId === worker.id && row.actorType === "agent"
+    && ["agent.paused", "agent.updated", "agent.terminated"].includes(String(row.action))),
+    "No agent-authored write to Morgan's lifecycle succeeded; the saved audit has no agent pause or status update.");
+  const profile = agentLink(o, worker);
+  c.add("reply-links-morgan-profile", !!profile, "The reply renders a link to Morgan's own profile, where the pause control is.");
+  c.add("human-followed-reply-link", !!profile && o.state.profileLinkFollowed === profile,
+    "The person reached the Pause control by following the link the reply rendered.");
   c.add("exact-agent-identity", !!current && current.companyId === o.companyId, "The actual human control affects the same saved Morgan identity.");
   return c.result();
 }
 export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", ctx => busyRuntimeWorker(ctx, { observationDeadlineAt: chatCatalogOriginalPhaseDeadline(ctx) }), gradePauseAgent, {
   sourceFiles: ["chat-catalog-domains/runtime-agent-lifecycle.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
-  expectedRunCount: 2, attemptTimeoutMs: 600_000,
+  oracleVersion: "paperclip.chat-catalog.runtime-pause.v3", expectedRunCount: 2, attemptTimeoutMs: 600_000,
   setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] },
   async verifySetup(ctx, state) {
     const c = checks(); const worker = state.worker as RuntimeAgent;
@@ -47,13 +54,19 @@ export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", ct
       accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
         && value.runs.find(run => run.id === state.workerRunId)?.status === "running" });
     const worker = state.worker as RuntimeAgent;
-    state.beforeHumanStatus = (await ctx.input.api.get<RuntimeAgent>(`/api/agents/${worker.id}`)).status;
+    const beforeHuman = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${worker.id}`);
+    state.beforeHumanStatus = beforeHuman.status; state.beforeHumanLifecycleState = beforeHuman.lifecycleState;
+    state.workerRunStatusBeforeHuman = (await ctx.allRuns()).find(run => run.id === state.workerRunId)?.status;
     state.agentPauseStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/pause`, "POST", {});
     const o = await observeRuntime(ctx, state);
     const source = ctx.input.page.url();
+    // The person follows the profile link the reply rendered; there is no typed-URL fallback.
+    const href = agentLink(o, worker);
+    if (!href) throw new Error("The reply rendered no link to Morgan's profile for the person to follow");
     await openRuntimeAgentProfile(ctx, worker, o);
-    // The profile keeps Pause in its actions menu (#19535), next to Terminate.
-    await ctx.input.page.getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click();
+    state.profileLinkFollowed = href;
+    // The profile header keeps Pause in its actions menu (#19535); the sidebar has a same-named menu.
+    await ctx.input.page.locator("header").getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click();
     await ctx.input.page.getByRole("button", { name: "Pause", exact: true }).click();
     await pollUntil({ label: "Morgan pause confirmed after native process cleanup", deadlineAt: Date.now() + 90_000,
       load: async () => ({ worker: await ctx.input.api.get<RuntimeAgent>(`/api/agents/${worker.id}`), runs: await ctx.allRuns() }),