test(evals): grade update-case-facts by the saved browser value and a non-contradicting reply

The saved field was matched by keywords anywhere in the JSON ("only" in any
field satisfied it), so a correct "iOS Safari" failed and a desktop-and-mobile
value passed. Grade the setup's browser field by normalized tokens (Safari plus
a mobile qualifier and nothing else). Grade the follow-up reply (text after the
first checkpoint) by the case reference, the setup-chosen escalation literal and
non-contradiction instead of a keyword sentence.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -1442,7 +1442,7 @@
       "chat-catalog-domains/internal-definition.ts",
       "chat-scenarios/cases.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.integrator.v1",
+    "oracleVersion": "paperclip.chat-catalog.integrator.case-facts.v2",
     "expectedRunCount": 4,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts

@@ -1,22 +1,28 @@
 import { describe, expect, it } from "vitest";
-import { gradeIntegratorCaseFacts } from "./integrator-case-facts.js";
+import { corrected, gradeIntegratorCaseFacts } from "./integrator-case-facts.js";
+const FIRST = "Updated case COM-C1: browser is now **\"Mobile Safari only\"**, with customer, expected result, and severity preserved. See [COM-1](/COM/issues/COM-1).";
+// Recorded follow-up reply from run 42584001490 (escalation value substituted).
+const FOLLOW_UP = "Yes. The case's saved facts still include your correction: `browser` = **Mobile Safari only**. The other facts are unchanged: customer **Acorn Test Shop**, expected **Pay should submit a single order**, severity **normal**.\n\nThe saved support escalation is **support-123**.\n\nOne caveat: the separate reproduction brief document still says \"The affected device has not yet been confirmed.\" Case COM-C1 is linked to [COM-1](/COM/issues/COM-1).";
+function withFollowUp(reply: string) { const v = valid(); v.state.collaborationCheckpoint.reply = `${FIRST}\n${reply}`; return v; }
 function valid() {
-  const original = { id: "case", workIssueId: "primary", handoffVersion: 1, fields: { browser: "Safari; device unknown", customer: "Acorn", severity: "normal", expected: "one order" } };
+  const original = { id: "case", identifier: "COM-C1", workIssueId: "primary", handoffVersion: 1, fields: { browser: "Safari; device unknown", customer: "Acorn", severity: "normal", expected: "one order" } };
   const brief = { id: "brief", issueId: "primary", latestRevisionId: "v1", body: "Customer reproduction" };
   const fields = { ...original.fields, browser: "Mobile Safari only", supportEscalation: "support-123" };
   const base = { scenarioId: "cases.update-case-facts", origin: "https://fixture.test", companyId: "company", leadId: "ceo",
     state: { initialRunId: "initial", worker: { id: "morgan" }, case: original, brief, collaboratorValue: "support-123", reviewer: { id: "riley" } }, caseRecord: { ...original, fields }, brief,
-    briefRevisions: [{ id: "v1", issueId: "primary", body: brief.body }], projects: [], tasks: [], goals: [], documents: [], revisions: [], comments: [], links: [], activity: [], humanQueries: [],
+    briefRevisions: [{ id: "v1", issueId: "primary", body: brief.body }], projects: [], tasks: [{ id: "primary", identifier: "COM-1" }], goals: [], documents: [], revisions: [], comments: [], links: [], activity: [], humanQueries: [],
     reply: "The reproduction occurs exclusively on mobile Safari. Support escalation is support-123.",
     runs: [{ id: "initial", status: "cancelled", runtimeMode: "native", agentId: "morgan", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: "{}" } } }, { id: "review", status: "succeeded", runtimeMode: "native", agentId: "riley", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: JSON.stringify({ fields }) } } }] };
-  return { ...base, caseRecord: { ...base.caseRecord, handoffVersion: 2 }, state: { ...base.state, firstCorrection: structuredClone(base), collaborationCheckpoint: structuredClone(base) } };
+  const first = { ...structuredClone(base), reply: FIRST };
+  return { ...base, caseRecord: { ...base.caseRecord, handoffVersion: 2 }, state: { ...base.state, firstCorrection: first, collaborationCheckpoint: { ...structuredClone(base), reply: `${FIRST}\n${FOLLOW_UP}` } } };
 }
 describe("current case facts and real subsequent context", () => {
   it("accepts a substantive correction, preserved collaborator field and actual native projection", () => {
     expect(gradeIntegratorCaseFacts(valid()).passed).toBe(true);
     expect(gradeIntegratorCaseFacts(undefined).passed).toBe(false);
-    const natural = valid(); natural.state.collaborationCheckpoint.reply = "This affects mobile Safari only. The current support escalation is support-123.";
-    expect(gradeIntegratorCaseFacts(natural).passed).toBe(true);
+    for (const reply of ["COM-C1 still limits the bug to iOS Safari — nothing was overwritten. The current support escalation is support-123.",
+      "Your correction survived support's edit on [COM-1](/COM/issues/COM-1): affected browser is Safari on iPhone. Escalation saved: support-123."])
+      expect(gradeIntegratorCaseFacts(withFollowUp(reply)).passed).toBe(true);
     const structured = valid(); structured.caseRecord.fields.browser = "mobile Safari"; expect(gradeIntegratorCaseFacts(structured).passed).toBe(true);
   });
   it("rejects lost unrelated fields and an initially wrong correction repaired later", () => {
@@ -31,7 +37,23 @@ describe("current case facts and real subsequent context", () => {
     const missing = valid(); missing.briefRevisions = []; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false);
   });
   it("rejects a plausible contrary chat report and missing collaborative acknowledgement", () => {
-    const falseReport = valid(); falseReport.state.collaborationCheckpoint.reply = "It is not only mobile Safari. Support escalation is support-123."; expect(gradeIntegratorCaseFacts(falseReport).passed).toBe(false);
-    const missing = valid(); missing.state.collaborationCheckpoint.reply = "Mobile Safari only."; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false);
+    for (const reply of ["No. Your mobile Safari only correction on COM-C1 was overwritten by support. Escalation: support-123.",
+      "COM-C1 no longer includes your browser correction. Support escalation is support-123.",
+      "Yes, COM-C1 still says mobile Safari only. The escalation is support-999.",
+      "Yes, COM-C1 still says mobile Safari only.",
+      "Yes, it still says mobile Safari only. Support escalation is support-123.",
+      `${FIRST}`])
+      expect(gradeIntegratorCaseFacts(withFollowUp(reply)).passed, reply).toBe(false);
+    // The first reply already named the case; the follow-up must answer on its own.
+    const reusedFirst = valid(); reusedFirst.state.collaborationCheckpoint.reply = `${FIRST}\nYes, it still says mobile Safari only. Support escalation is support-123.`;
+    expect(gradeIntegratorCaseFacts(reusedFirst).passed).toBe(false);
+  });
+  it("grades the saved browser by normalized tokens, not keywords anywhere in the fields", () => {
+    for (const browser of ["Mobile Safari only", "mobile Safari", "iOS Safari", "Safari on iPhone/iPad only", "Safari (iOS)"]) expect(corrected({ browser }), browser).toBe(true);
+    expect(corrected({ browser: "Safari", platform: "mobile only" })).toBe(true);
+    expect(corrected({ affectedBrowsers: ["iOS Safari"] })).toBe(true);
+    for (const fields of [{ browser: "Safari (desktop and mobile Safari)", note: "only intermittent" }, { browser: "Safari; affected device not yet confirmed" }, { browser: "Desktop Safari only" },
+      { browser: "not mobile Safari" }, { browser: "Safari", note: "mobile only" }, { browser: "Chrome and mobile Safari" }, { affectedBrowsers: ["iOS Safari", "Chrome"] }, { browser: "Safari; device unknown", affectedBrowser: "iOS Safari" }, {}])
+      expect(corrected(fields), JSON.stringify(fields)).toBe(false);
   });
 });

tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts

@@ -19,11 +19,42 @@ async function observeFacts(ctx: ChatCatalogContext, state: Record<string, unkno
   return { ...base, caseRecord: await ctx.input.api.get<CaseRecord>(`/api/cases/${row.id}`), brief: await ctx.input.api.get<IssueDocument>(`/api/issues/${row.workIssueId}/documents/brief`), briefRevisions: await ctx.input.api.get<DocumentRevision[]>(`/api/issues/${row.workIssueId}/documents/brief/revisions`) };
 }
 function factsObservation(value: unknown): FactsObservation | undefined { const o = value as FactsObservation; return observation(value) && !!o.caseRecord && !!o.brief && Array.isArray(o.briefRevisions) ? o : undefined; }
-function corrected(fields: unknown) {
-  const text = JSON.stringify(fields); const record = fields && typeof fields === "object" ? fields as Record<string, unknown> : {};
-  const singleBrowser = [record.browser, record.affectedBrowser].some(value => typeof value === "string" && /^mobile Safari$/i.test(value.trim()))
-    || Array.isArray(record.affectedBrowsers) && record.affectedBrowsers.length === 1 && /^mobile Safari$/i.test(String(record.affectedBrowsers[0]));
-  return /mobile Safari/i.test(text) && (singleBrowser || /\bonly\b|\bexclusive/i.test(text)) && !/not (?:only |exclusive[^ ]* )?mobile Safari|desktop Safari only/i.test(text);
+/** The setup records the reproduction browser in `browser`; the person's correction is "only on mobile Safari".
+ * A corrected value is graded by its normalized tokens: Safari plus a mobile-platform qualifier and nothing
+ * else (filler such as "only"/"on" is ignored). "iOS Safari" and "Safari on iPhone/iPad" are the same fact;
+ * any other token (desktop, "not", "and", a second browser, the stale "not yet confirmed") is a different one. */
+const FILLER = new Set(["only", "on", "exclusively", "just", "the", "browser", "browsers", "device", "devices", "affected"]);
+const MOBILE = new Set(["mobile", "ios", "iphone", "ipad", "ipados"]);
+const tokens = (value: string) => value.toLowerCase().split(/[^a-z0-9]+/).filter(token => token && !FILLER.has(token));
+function mobileSafariOnly(value: unknown) {
+  if (typeof value !== "string") return false; const t = tokens(value);
+  return t.includes("safari") && t.some(token => MOBILE.has(token)) && t.every(token => token === "safari" || MOBILE.has(token));
+}
+function mobilePlatformOnly(value: unknown) { if (typeof value !== "string") return false; const t = tokens(value); return t.length > 0 && t.every(token => MOBILE.has(token)); }
+export function corrected(fields: unknown) {
+  const record = fields && typeof fields === "object" ? fields as Record<string, unknown> : {};
+  const platformSplit = Object.entries(record).some(([key, value]) => /^(?:platform|os|device)$/i.test(key) && mobilePlatformOnly(value));
+  const browserValues = [record.browser, record.affectedBrowser].filter(value => value !== undefined);
+  if (record.affectedBrowsers !== undefined) browserValues.push(Array.isArray(record.affectedBrowsers) && record.affectedBrowsers.length === 1 ? record.affectedBrowsers[0] : null);
+  return browserValues.length > 0 && browserValues.every(value => mobileSafariOnly(value)
+    || platformSplit && typeof value === "string" && tokens(value).join(" ") === "safari");
+}
+/** The follow-up reply is the text added after the first checkpoint. It must name the case (its identifier or its
+ * primary task) and the setup-chosen support escalation, and must not contradict the saved state: no other
+ * escalation value, no "No" answer, and no sentence saying the correction was lost. Saved fields are the proof. */
+export function latestReply(fresh: InternalObservation, first: InternalObservation | undefined) {
+  return first && fresh.reply.startsWith(first.reply) ? fresh.reply.slice(first.reply.length) : fresh.reply;
+}
+const escapeRe = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+export function reportsCurrentFacts(reply: string, o: FactsObservation) {
+  const caseRow = o.state.case as CaseRecord & { identifier?: string }; const task = o.tasks.find(row => row.id === caseRow.workIssueId);
+  const names = [caseRow.identifier, task?.identifier, caseRow.id, caseRow.workIssueId].filter((value): value is string => !!value);
+  const refersToCase = names.some(name => new RegExp(`(?<![\\w-])${escapeRe(name)}(?![\\w-])`, "i").test(reply));
+  const value = String(o.state.collaboratorValue); const otherEscalation = (reply.match(/\bsupport-[\w-]+/gi) ?? []).some(found => found !== value);
+  const answersNo = /^\W*no\b/i.test(reply.trim());
+  const lost = reply.split(/\n+|(?<=[.!?])\s+/).some(sentence => /\b(?:overwr[io]te|overwritten|lost|removed|reverted|dropped|discarded|no longer|missing|(?:is|was|are|were)n['’]?t (?:saved|kept|included|there|present)|not (?:saved|kept|included|there|present)|does(?:n['’]t| not) (?:include|contain|have))\b/i.test(sentence)
+    && /\b(?:correction|mobile Safari|browser|reproduction step)\b/i.test(sentence) && !/\b(?:nothing|not|never|wasn['’]?t|none)\b[^.!?]{0,30}\b(?:overwr|lost|removed|reverted|dropped|discarded)/i.test(sentence));
+  return refersToCase && reply.includes(value) && !otherEscalation && !answersNo && !lost;
 }
 function retained(o: FactsObservation, concurrent: boolean) {
   const original = o.state.case as CaseRecord; const fields = o.caseRecord.fields as Record<string, unknown>;
@@ -38,8 +69,7 @@ export function gradeIntegratorCaseFacts(value: unknown) {
   const first = factsObservation(o.state.firstCorrection); const fresh = factsObservation(o.state.collaborationCheckpoint);
   c.add("initial-correction-preserves-unrelated-fields", !!first && retained(first, false), "The unchanged first query must already correct the saved facts without discarding unrelated fields; a later turn cannot repair an initially wrong mutation.");
   c.add("fresh-collaborative-change-not-stale-replacement", !!fresh && retained(fresh, true) && retained(o, true)
-    && fresh.reply.split(/\n+|(?<=[.!?])\s+/).some(sentence => /mobile Safari/i.test(sentence) && /only|exclusiv/i.test(sentence) && !/not (?:only |exclusive[^ ]* )?mobile Safari/i.test(sentence))
-    && fresh.reply.includes(String(o.state.collaboratorValue)), "A real independently saved support change survives the explicit reread, and chat reports both the correction and that actual concurrent field.");
+    && reportsCurrentFacts(latestReply(fresh, first), o), "A real independently saved support change survives the explicit reread; the follow-up reply names the case and the actual concurrent field and does not contradict the saved correction.");
   const before = o.state.brief as IssueDocument; const original = o.state.case as CaseRecord;
   c.add("ordinary-task-document-ownership-and-history", o.brief.id === before.id && o.brief.issueId === original.workIssueId
     && o.briefRevisions.some(revision => revision.id === before.latestRevisionId && revision.issueId === original.workIssueId && revision.body === before.body), "The original ordinary-task document and its actual saved revision remain owned and accessible on the same primary task.");
@@ -84,7 +114,7 @@ export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.up
   const row = state.case as CaseRecord; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title);
   const task = await ctx.input.api.get<Issue>(`/api/issues/${row.workIssueId}`);
   await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier}`, task.title);
-}, { expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } },
+}, { oracleVersion: "paperclip.chat-catalog.integrator.case-facts.v2", expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } },
   sourceFiles: ["chat-catalog-domains/integrator-case-facts.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], cleanup,
   async act(ctx, state) {
     const settle = (count: number) => pollUntil({ label: `saved case correction phase ${count}`, deadlineAt: Date.now() + 180_000, intervalMs: 1000,