test(evals): grade fast-responses by following the reply's own link to the control

The scripted human typed /company/settings/connections instead of clicking
the reply's link, and the reply checks accepted any "settings" mention, so a
reply that linked /company/settings (no Fast response section) passed.

The person now clicks each rendered reply link and must land on a page that
shows the Fast response region; with no such link the act stops before
configuring. The reply keyword checks for this case are removed. Other
services in the module keep their existing checks and oracle version.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -2058,7 +2058,7 @@
       "chat-catalog-domains/internal-support.ts",
       "chat-scenarios/companies.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.identity.v1",
+    "oracleVersion": "paperclip.chat-catalog.identity.fast-responses.v2",
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,
     "attemptTimeoutMs": 420000,

tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts

@@ -3,6 +3,7 @@ import { companyModelIdentityCases, requestedAuxiliaryServiceModel, saveFastResp
 import type { IdentityObservation } from "./identity-support.js";
 vi.mock("./identity-support.js", async importOriginal => ({
   ...await importOriginal<typeof import("./identity-support.js")>(), probeAgentPermission: async () => 403,
+  observeIdentity: async () => ({ links: ["/RUN/company/settings/connections"], origin: "https://fixture.test" }), followRenderedLink: async () => {},
 }));
 function valid(scenarioId: string): IdentityObservation {
   const model = scenarioId === "companies.configure-decision-model" ? "typesafe/jev-1.13" : requestedAuxiliaryServiceModel;
@@ -14,7 +15,10 @@ function valid(scenarioId: string): IdentityObservation {
       saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model }, modelBeforeTest: model, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403,
       persistedAfterReload: true, testHttpStatus: 200, visibleTestReceipt: true, history: [history], testCostEvents: 2 } };
   if (scenarioId.endsWith("configure-decision-model")) o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage };
-  else if (scenarioId.endsWith("fast-responses")) o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage };
+  else if (scenarioId.endsWith("fast-responses")) {
+    o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage };
+    o.links = ["/acorn/company/settings/connections"]; o.state.replyLinkedControlHref = "/acorn/company/settings/connections";
+  }
   else o.state.testResult = { passed: true, samples: [
     { name: "Benign source code", expectedViolation: false, actualViolation: false, passed: true, invocationId: "benign", durationMs: 80, usage },
     { name: "Malicious instructions", expectedViolation: true, actualViolation: true, passed: true, invocationId: "malicious", durationMs: 90, usage },
@@ -62,12 +66,12 @@ describe("company model service configuration and actual tests", () => {
   it.each(["companies.prompt-injection-guard", "companies.fast-responses"])("%s actual human act stops before test if saved model differs", async id => {
     const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!;
     const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), test = vi.fn(async () => {});
-    const section = { getByLabel: () => ({ fill }), getByRole: (_role: string, options: { name: string }) => ({
+    const section = { waitFor: async () => {}, getByLabel: () => ({ fill }), getByRole: (_role: string, options: { name: string }) => ({
       click: options.name.startsWith("Test") ? test : save, setChecked: async () => {},
     }) };
     const get = vi.fn(async () => ({ settings: { enabled: false, model: "openai/wrong-default" } }));
     const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }],
-      observationDeadlineAt: Date.now() + 390_000, input: { fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } };
+      observationDeadlineAt: Date.now() + 390_000, input: { evidence: async () => {}, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } };
     await expect(definition.act!(ctx as never, valid(id).state)).rejects.toThrow(/Requested auxiliary model saved before any service test/);
     expect(fill).toHaveBeenCalledExactlyOnceWith(requestedAuxiliaryServiceModel); expect(save).toHaveBeenCalledTimes(2);
     expect(get).toHaveBeenCalledTimes(id.endsWith("fast-responses") ? 1 : 2); expect(test).not.toHaveBeenCalled();
@@ -75,15 +79,31 @@ describe("company model service configuration and actual tests", () => {
   it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); });
   it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); });
   it("rejects classifier model samples without their independent cost events", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); o.state.testCostEvents = 0; expect(d.grade(o).passed).toBe(false); });
-  it("does not accept an instruction-only acknowledgement change as Fast response configuration guidance", () => {
+  it("requires a reply link whose page showed the Fast response control, not reply wording", () => {
     const d = companyModelIdentityCases.find(row => row.scenarioId === "companies.fast-responses")!;
-    const o = valid(d.scenarioId);
-    o.reply = "I updated my persistent instructions to acknowledge chat requests immediately.";
-    const grade = d.grade(o);
-    expect(grade.passed).toBe(false);
-    expect(grade.checks.find(check => check.id === "honest-human-settings-guidance")?.passed).toBe(false);
+    const reworded = valid(d.scenarioId); reworded.reply = "A board member can switch this on here: [Connections](/acorn/company/settings/connections).";
+    expect(d.grade(reworded).passed).toBe(true);
+    const unlinked = valid(d.scenarioId); unlinked.state.replyLinkedControlHref = null;
+    expect(d.grade(unlinked).checks.find(check => check.id === "reply-link-opens-fast-response-control")?.passed).toBe(false);
+    const notRendered = valid(d.scenarioId); notRendered.links = ["/acorn/company/settings"];
+    expect(d.grade(notRendered).passed).toBe(false);
+  });
+  it("fast response act follows only the reply's links and stops when none shows the control", async () => {
+    const definition = companyModelIdentityCases.find(row => row.scenarioId === "companies.fast-responses")!;
+    const support = await import("./identity-support.js");
+    const observe = vi.spyOn(support, "observeIdentity").mockResolvedValue({ links: ["/RUN/company/settings", "https://elsewhere.test/RUN/company/settings/connections"], origin: "https://fixture.test" } as never);
+    const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), goto = vi.fn(async () => {});
+    const section = { waitFor: async () => { throw new Error("not visible"); }, getByLabel: () => ({ fill }), getByRole: () => ({ click: save, setChecked: save }) };
+    const evidence = vi.fn(async () => {});
+    const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }],
+      observationDeadlineAt: Date.now() + 390_000, input: { evidence, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get: async () => ({ settings: { enabled: false } }) },
+        page: { url: () => "https://fixture.test/RUN/issues/RUN-1", goto, getByRole: () => section } } };
+    const state = valid("companies.fast-responses").state; delete state.replyLinkedControlHref;
+    await definition.act!(ctx as never, state);
+    expect(state.replyLinkedControlHref).toBeNull(); expect(fill).not.toHaveBeenCalled(); expect(save).not.toHaveBeenCalled();
+    expect(goto.mock.calls.map(call => (call as unknown[])[0])).not.toContain("/RUN/company/settings/connections");
+    observe.mockRestore();
   });
-  it("rejects fast response work-completion claims", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); o.reply = "Open Settings for fast responses. Your task is finished."; expect(d.grade(o).passed).toBe(false); });
   it("rejects an acknowledgement whose independent invocation belongs to a different human", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); (o.state.history as Array<{ responsibleUserId: string }>)[0]!.responsibleUserId = "other-person"; expect(d.grade(o).passed).toBe(false); });
 });
 

tests/runner-e2e/chat-catalog-domains/identity-company-models.ts

@@ -7,7 +7,8 @@ import type { PromptInjectionPreventionSettings, PromptInjectionTestResult } fro
 import type { FastResponseSettings, FastResponseTestResult } from "../../../packages/shared/src/fast-response.js";
 import type { CostSummary } from "../../../packages/shared/src/types/cost.js";
 import { checks } from "./internal-support.js";
-import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js";
+import { followRenderedLink, identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js";
+import type { Locator } from "@playwright/test";
 
 export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017";
 
@@ -38,6 +39,23 @@ export async function saveFastResponseModelBeforeTest(ctx: ChatCatalogContext, c
   return persisted;
 }
 
+/**
+ * The person follows the reply's own rendered links, in order, and stops at the first page that shows the service control.
+ * There is no typed-URL fallback: a reply that links a page without the control leaves the person without it.
+ */
+export async function openServiceControlFromReply(ctx: ChatCatalogContext, links: readonly string[], origin: string, control: () => Locator): Promise<string | undefined> {
+  const source = ctx.input.page.url();
+  for (const href of links) {
+    let url: URL; try { url = new URL(href, origin); } catch { continue; }
+    if (url.origin !== new URL(origin).origin) continue;
+    await followRenderedLink(ctx, href);
+    const shown = await control().waitFor({ state: "visible", timeout: Math.min(15_000, remainingCompanyServiceObservationBudget(ctx)) }).then(() => true, () => false);
+    if (shown) return href;
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  }
+  return undefined;
+}
+
 type Settings = DecisionModelSettings | PromptInjectionPreventionSettings | FastResponseSettings;
 const fastExperiments = new WeakMap<ChatCatalogContext, boolean>();
 const specs = [
@@ -54,7 +72,11 @@ export function gradeCompanyModel(value: unknown) {
     && saved.companyId === o.companyId && saved.connectionId === connection?.id && saved.grantId === connection?.grantId && o.state.persistedAfterReload === true
     && o.state.testHttpStatus === 200 && o.state.visibleTestReceipt === true,
     "The canonical chat leaves the disabled board-managed service unchanged; the actual human settings controls save the intended grant and the result survives reload.");
-  c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform.");
+  if (o.scenarioId === "companies.fast-responses") {
+    const href = o.state.replyLinkedControlHref;
+    c.add("reply-link-opens-fast-response-control", typeof href === "string" && o.links.includes(href),
+      "The person clicked a link rendered in the reply and that page showed the Fast response control; a settings page without it does not qualify.");
+  } else c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform.");
   if (["companies.prompt-injection-guard", "companies.fast-responses"].includes(o.scenarioId)) {
     c.add("requested-service-model-persisted-before-test", o.state.modelBeforeTest === requestedAuxiliaryServiceModel && saved?.model === requestedAuxiliaryServiceModel,
       "The actual public human configuration uses the requested OpenRouter API model before service testing and after readback; default auxiliary models are not substituted.");
@@ -80,7 +102,6 @@ export function gradeCompanyModel(value: unknown) {
       && row.connectionId === connection?.id && row.model === requestedAuxiliaryServiceModel && row.responsibleUserId === o.state.humanId && row.feature === "settings.test" && row.finishedAt != null
       && row.inputTokens === result.usage.inputTokens && row.outputTokens === result.usage.outputTokens && row.costStatus === result.usage.costStatus),
       "The real settings test generates an actual model acknowledgement and matching durable usage/failure history; elapsed worker time alone cannot qualify.");
-    c.add("acknowledgement-is-not-work", /acknowledg|receipt|quick|fast/i.test(o.reply) && !/\b(?:work|task)\s+(?:is|has been)\s+(?:finished|completed|started)/i.test(o.reply), "The response does not present a fast acknowledgement as task completion or proof of worker execution.");
   } else c.add("known-service", false, "No oracle exists for this service.");
   return c.result();
 }
@@ -104,14 +125,21 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s
   } catch (error) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } throw error; }
 }, gradeCompanyModel, {
   sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000,
+  ...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v2" } : {}),
   async act(ctx, state) {
     await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
     state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled;
     state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId,
       ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) });
     state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {});
-    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" });
     const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true });
+    if (spec.path === "fast-response") {
+      const o = await observeIdentity(ctx, state);
+      const href = await openServiceControlFromReply(ctx, o.links, o.origin, () => section);
+      state.replyLinkedControlHref = href ?? null;
+      await ctx.input.evidence("identity-company-model-reply-link.json", { links: o.links, opened: href ?? null, url: ctx.input.page.url() });
+      if (!href) { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return; }
+    } else await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" });
     await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click();
     if (spec.path !== "decision-model") await section.getByLabel("Model", { exact: true }).fill(requestedAuxiliaryServiceModel);
     await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true);