test(evals): grade hire-needs-approval by each reply's exact approval link, not its wording

The pending-phase check matched reply wording with regular expressions.
Replace it with structural evidence: each request's own reply renders a
link to the exact pending approval the person must decide. Early worker
execution stays graded from saved state and runs. Bump the case oracle to
paperclip.chat-catalog.runtime-hiring.v3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -675,7 +675,7 @@
       "chat-catalog-domains/runtime-support.ts",
       "chat-scenarios/agents.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime-hiring.v3",
     "coverageGaps": [],
     "expectedRunCount": 4,
     "expectedHumanMessageCount": 2,

tests/runner-e2e/chat-catalog-domains/runtime-hiring.test.ts

@@ -1,5 +1,8 @@
 import { it, expect } from "vitest";
 import { gradeGovernedHire } from "./runtime-hiring.js";
+function replyLink(commentId: string, href: string) {
+  return { threadTestId: "task-chat-thread" as const, href, source: "reply" as const, conversationId: "source", anchorCommentId: commentId, commentId };
+}
 function valid() {
   const proposal = (id: string, name: string) => ({ id, companyId: "company", type: "hire_agent", status: "pending", requestedByAgentId: "ceo", decidedByUserId: null as string | null,
     payload: { agentId: name.toLowerCase(), name, role: "qa", adapterType: "runner", adapterConfig: { model: "admitted" } } });
@@ -9,26 +12,28 @@ function valid() {
     state: { firstProposal, secondProposal, approvalsAfter: [{ ...firstProposal, status: "rejected", decidedByUserId: "human" }, { ...secondProposal, status: "approved", decidedByUserId: "human" }],
       secondRequestAttributed: true, rejectedBeforeChangedRequest: true, noRepeatedProposalBeforeChangedRequest: true, acceptedBefore,
       noWorkerBeforeFirstDecision: true, noWorkerBeforeSecondDecision: true, requestRunIds: ["canonical", "changed"],
-      firstReply: "Morgan's QA proposal is pending board approval.", secondReply: "Jules's QA proposal is awaiting approval." },
+      firstReply: "Morgan's QA proposal is pending board approval.", secondReply: "Jules's QA proposal is awaiting approval.",
+      firstReplyCommentIds: ["reply-1"], secondReplyCommentIds: ["reply-2"] },
+    renderedLinkEvidence: [replyLink("reply-1", "/RUN/approvals/first"), replyLink("reply-1", "/RUN/agents/morgan"), replyLink("reply-2", "/RUN/approvals/second")],
     agents: [{ ...acceptedBefore, status: "idle", lifecycleState: "ready" }], tasks: [], links: [], activity: [], humanQueries: [], reply: "",
     runs: [{ id: "canonical", agentId: "ceo", status: "succeeded", contextSnapshot: {} as Record<string, unknown> }, { id: "changed", agentId: "ceo", status: "succeeded", contextSnapshot: {} as Record<string, unknown> }] };
 }
 it("accepts distinct real rejection and acceptance branches after an explicit changed request", () => expect(gradeGovernedHire(valid()).passed).toBe(true));
-it("accepts natural phase-specific proposal language", () => { const o = valid(); o.state.firstReply = "The board must approve Morgan's QA hire proposal before work can begin."; o.state.secondReply = "Jules's replacement proposal is pending the board's approval decision."; expect(gradeGovernedHire(o).passed).toBe(true); });
+it("does not grade reply wording: a reworded reply with the same approval links passes", () => { const o = valid(); o.state.firstReply = "Morgan is set up as a QA hire; open the approval below."; o.state.secondReply = "Here is Jules. Decide on the card."; expect(gradeGovernedHire(o).passed).toBe(true); });
 it("rejects missing governance evidence", () => expect(gradeGovernedHire(undefined).passed).toBe(false));
 it("rejects a working employee before the decision", () => { const o = valid(); o.state.noWorkerBeforeFirstDecision = false; expect(gradeGovernedHire(o).passed).toBe(false); });
 it("rejects repeating the rejected proposal without changed human direction", () => { const o = valid(); o.state.noRepeatedProposalBeforeChangedRequest = false; expect(gradeGovernedHire(o).passed).toBe(false); });
 it("rejects accepting a replacement identity instead of the proposed hire", () => { const o = valid(); o.agents[0]!.id = "replacement"; expect(gradeGovernedHire(o).passed).toBe(false); });
-it("rejects readiness language before human approval", () => { const o = valid(); o.state.firstReply = "Morgan is now ready and working."; expect(gradeGovernedHire(o).passed).toBe(false); });
+it("rejects a request reply that does not link its own pending approval", () => {
+  const o = valid(); o.renderedLinkEvidence = o.renderedLinkEvidence.filter(link => link.href !== "/RUN/approvals/second"); expect(gradeGovernedHire(o).passed).toBe(false);
+  const swapped = valid(); swapped.state.secondReplyCommentIds = ["reply-1"]; expect(gradeGovernedHire(swapped).passed).toBe(false);
+  const foreign = valid(); foreign.renderedLinkEvidence = [replyLink("reply-1", "/RUN/approvals/first"), replyLink("reply-2", "https://elsewhere.invalid/RUN/approvals/second")]; expect(gradeGovernedHire(foreign).passed).toBe(false);
+  const api = valid(); api.renderedLinkEvidence = [replyLink("reply-1", "/RUN/approvals/first"), replyLink("reply-2", "/api/approvals/second")]; expect(gradeGovernedHire(api).passed).toBe(false);
+});
 it("rejects a hidden nudge classified as a decision notification", () => { const o = valid(); o.runs.push({ id: "extra", agentId: "ceo", status: "succeeded", contextSnapshot: { source: "approval.approved", approvalId: "foreign" } }); expect(gradeGovernedHire(o).passed).toBe(false); });
 it("accepts a separately counted exact approval notification", () => { const o = valid(); o.runs.push({ id: "extra", agentId: "ceo", status: "succeeded", contextSnapshot: { source: "approval.approved", approvalId: "second" } }); expect(gradeGovernedHire(o).passed).toBe(true); });
 
-it("accepts the retained request-ready answer with an actual pending proposal and explicit activation gate", () => {
-  const o = valid(); o.state.firstReply = "Morgan’s QA Engineer hire request is ready. Morgan will report to the CEO. Approve Morgan’s hire to activate the agent.";
+it("accepts an absolute same-origin approval link", () => {
+  const o = valid(); o.renderedLinkEvidence = [replyLink("reply-1", "http://fixture.invalid/RUN/approvals/first"), replyLink("reply-2", "/RUN/approvals/second")];
   expect(gradeGovernedHire(o).passed).toBe(true);
 });
-it("rejects pending language mixed with a named employee readiness claim", () => {
-  const o = valid(); o.state.firstReply = "The proposal is pending, but Morgan is now ready and working.";
-  expect(gradeGovernedHire(o).passed).toBe(false);
-  o.state.firstReply = "Approve the proposal; the QA engineer is active."; expect(gradeGovernedHire(o).passed).toBe(false);
-});

tests/runner-e2e/chat-catalog-domains/runtime-hiring.ts

@@ -3,7 +3,7 @@ import { expect } from "@playwright/test";
 import { pollUntil } from "../api.js";
 import { sendChatMessage } from "../chat-flow.js";
 import { checks, type Entity } from "./internal-support.js";
-import { runtimeDefinition, runtimeObservation, observeRuntime, unchangedAgent, type RuntimeAgent } from "./runtime-support.js";
+import { runtimeDefinition, runtimeObservation, observeRuntime, unchangedAgent, type RuntimeAgent, type RuntimeObservation } from "./runtime-support.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 interface HireApproval { id: string; companyId: string; type: string; status: string; requestedByAgentId: string | null;
   decidedByUserId: string | null; payload: { agentId?: string; name?: string; role?: string; title?: string; reportsTo?: string; adapterType?: string; adapterConfig?: unknown } }
@@ -17,12 +17,20 @@ async function pendingHire(ctx: ChatCatalogContext, name: string) {
       && value.runs.every(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded"),
     reject: value => value.runs.length > 4 || value.runs.some(run => run.agentId !== ctx.input.fixtures.agent.id || ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Governed hire admitted unexpected work or failed" : undefined });
 }
-export function pendingHireGuidance(reply: string, name: "Morgan" | "Jules") {
-  if (!/pending|approv(?:al|e)|propos|await|board/i.test(reply) || /successfully hired/i.test(reply)) return false;
-  // A hire request can be ready for the board while its intended employee
-  // remains pending. Do not turn that into a general exemption for ready.
-  const employeeClaim = new RegExp(`\\b(?:${name}|(?:the )?(?:QA engineer|employee|agent|hire)|they|he|she)\\s+(?:is|are|has|have)\\s+(?:now |already )?(?:been )?(?:ready(?! for (?:board )?(?:approval|review))|working|started|active|running)\\b`, "i");
-  return !employeeClaim.test(reply);
+/**
+ * The reply reports the pending approval by giving the person the exact
+ * approval to open and decide. Its wording is not graded; whether the hire ran
+ * early is graded from saved state and runs.
+ */
+export function repliesLinkApproval(o: RuntimeObservation, commentIds: unknown, approvalId: string | undefined) {
+  const ids = Array.isArray(commentIds) ? commentIds.map(String) : [];
+  return !!approvalId && ids.length > 0 && (o.renderedLinkEvidence ?? []).some(link => {
+    if (link.source !== "reply" || !link.commentId || !ids.includes(link.commentId)) return false;
+    try {
+      const url = new URL(link.href, o.origin); const parts = url.pathname.split("/").map(decodeURIComponent);
+      return url.origin === new URL(o.origin).origin && parts.length === 4 && !!parts[1] && parts[1] !== "api" && parts[2] === "approvals" && parts[3] === approvalId;
+    } catch { return false; }
+  });
 }
 export function gradeGovernedHire(value: unknown) {
   const c = checks(); const o = runtimeObservation(value);
@@ -52,9 +60,9 @@ export function gradeGovernedHire(value: unknown) {
   c.add("bounded-authoritative-decision-notifications", extra.every(run => ["approval.approved", "approval.rejected"].includes(String(run.contextSnapshot?.source))
     && [rejected?.id, accepted?.id].includes(String(run.contextSnapshot?.approvalId))),
   "Any extra provider turn is independently tied to one of these exact real human approval decisions.");
-  c.add("honest-pending-phase-guidance", pendingHireGuidance(String(o.state.firstReply), "Morgan")
-    && pendingHireGuidance(String(o.state.secondReply), "Jules"),
-  "Each actual reply describes an approvable proposal, without claiming that its employee is ready or working before approval.");
+  c.add("reply-links-each-pending-approval", repliesLinkApproval(o, o.state.firstReplyCommentIds, rejected?.id)
+    && repliesLinkApproval(o, o.state.secondReplyCommentIds, accepted?.id),
+  "Each request's own reply links the exact pending approval the person must decide.");
   return c.result();
 }
 export const runtimeHiringCases = [runtimeDefinition("agents.hire-needs-approval", async ctx => {
@@ -64,6 +72,7 @@ export const runtimeHiringCases = [runtimeDefinition("agents.hire-needs-approval
   return { companyRequiresApproval: true };
 }, gradeGovernedHire, {
   sourceFiles: ["chat-catalog-domains/runtime-hiring.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
+  oracleVersion: "paperclip.chat-catalog.runtime-hiring.v3",
   expectedHumanMessageCount: 2, expectedRunCount: 4, minimumRunCount: 2, attemptTimeoutMs: 720_000,
   async act(ctx, state) {
     const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
@@ -90,7 +99,10 @@ export const runtimeHiringCases = [runtimeDefinition("agents.hire-needs-approval
     const profiles = await Promise.all(next.map(run => ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`)));
     state.secondRequestAttributed = human.length === 1 && next.length === 1 && profiles.some(run => run.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id));
     state.requestRunIds = [...firstRequestRuns, ...next.map(run => run.id)];
-    state.secondReply = comments.filter(row => next.some(run => run.id === row.createdByRunId) && row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body ?? "").join("\n");
+    const leadReplies = (runIds: string[]) => comments.filter(row => runIds.includes(String(row.createdByRunId)) && row.authorAgentId === ctx.input.fixtures.agent.id);
+    state.firstReplyCommentIds = leadReplies(firstRequestRuns).map(row => row.id);
+    state.secondReplyCommentIds = leadReplies(next.map(run => run.id)).map(row => row.id);
+    state.secondReply = leadReplies(next.map(run => run.id)).map(row => row.body ?? "").join("\n");
     state.acceptedBefore = secondPhase.agents.find(row => row.id === second.payload.agentId);
     state.worker = state.acceptedBefore;
     await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/approvals/${second.id}`, { waitUntil: "domcontentloaded" });