diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index ef7d693bdda..9bbfbec174e 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -1442,7 +1442,7 @@ "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.integrator.case-facts.v2", "expectedRunCount": 4, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts index c9bb771c05a..7958beae8d5 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts @@ -1,22 +1,28 @@ import { describe, expect, it } from "vitest"; -import { gradeIntegratorCaseFacts } from "./integrator-case-facts.js"; +import { corrected, gradeIntegratorCaseFacts } from "./integrator-case-facts.js"; +const FIRST = "Updated case COM-C1: browser is now **\"Mobile Safari only\"**, with customer, expected result, and severity preserved. See [COM-1](/COM/issues/COM-1)."; +// Recorded follow-up reply from run 42584001490 (escalation value substituted). +const FOLLOW_UP = "Yes. The case's saved facts still include your correction: `browser` = **Mobile Safari only**. The other facts are unchanged: customer **Acorn Test Shop**, expected **Pay should submit a single order**, severity **normal**.\n\nThe saved support escalation is **support-123**.\n\nOne caveat: the separate reproduction brief document still says \"The affected device has not yet been confirmed.\" Case COM-C1 is linked to [COM-1](/COM/issues/COM-1)."; +function withFollowUp(reply: string) { const v = valid(); v.state.collaborationCheckpoint.reply = `${FIRST}\n${reply}`; return v; } function valid() { - const original = { id: "case", workIssueId: "primary", handoffVersion: 1, fields: { browser: "Safari; device unknown", customer: "Acorn", severity: "normal", expected: "one order" } }; + const original = { id: "case", identifier: "COM-C1", workIssueId: "primary", handoffVersion: 1, fields: { browser: "Safari; device unknown", customer: "Acorn", severity: "normal", expected: "one order" } }; const brief = { id: "brief", issueId: "primary", latestRevisionId: "v1", body: "Customer reproduction" }; const fields = { ...original.fields, browser: "Mobile Safari only", supportEscalation: "support-123" }; const base = { scenarioId: "cases.update-case-facts", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { initialRunId: "initial", worker: { id: "morgan" }, case: original, brief, collaboratorValue: "support-123", reviewer: { id: "riley" } }, caseRecord: { ...original, fields }, brief, - briefRevisions: [{ id: "v1", issueId: "primary", body: brief.body }], projects: [], tasks: [], goals: [], documents: [], revisions: [], comments: [], links: [], activity: [], humanQueries: [], + briefRevisions: [{ id: "v1", issueId: "primary", body: brief.body }], projects: [], tasks: [{ id: "primary", identifier: "COM-1" }], goals: [], documents: [], revisions: [], comments: [], links: [], activity: [], humanQueries: [], reply: "The reproduction occurs exclusively on mobile Safari. Support escalation is support-123.", runs: [{ id: "initial", status: "cancelled", runtimeMode: "native", agentId: "morgan", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: "{}" } } }, { id: "review", status: "succeeded", runtimeMode: "native", agentId: "riley", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: JSON.stringify({ fields }) } } }] }; - return { ...base, caseRecord: { ...base.caseRecord, handoffVersion: 2 }, state: { ...base.state, firstCorrection: structuredClone(base), collaborationCheckpoint: structuredClone(base) } }; + const first = { ...structuredClone(base), reply: FIRST }; + return { ...base, caseRecord: { ...base.caseRecord, handoffVersion: 2 }, state: { ...base.state, firstCorrection: first, collaborationCheckpoint: { ...structuredClone(base), reply: `${FIRST}\n${FOLLOW_UP}` } } }; } describe("current case facts and real subsequent context", () => { it("accepts a substantive correction, preserved collaborator field and actual native projection", () => { expect(gradeIntegratorCaseFacts(valid()).passed).toBe(true); expect(gradeIntegratorCaseFacts(undefined).passed).toBe(false); - const natural = valid(); natural.state.collaborationCheckpoint.reply = "This affects mobile Safari only. The current support escalation is support-123."; - expect(gradeIntegratorCaseFacts(natural).passed).toBe(true); + for (const reply of ["COM-C1 still limits the bug to iOS Safari — nothing was overwritten. The current support escalation is support-123.", + "Your correction survived support's edit on [COM-1](/COM/issues/COM-1): affected browser is Safari on iPhone. Escalation saved: support-123."]) + expect(gradeIntegratorCaseFacts(withFollowUp(reply)).passed).toBe(true); const structured = valid(); structured.caseRecord.fields.browser = "mobile Safari"; expect(gradeIntegratorCaseFacts(structured).passed).toBe(true); }); it("rejects lost unrelated fields and an initially wrong correction repaired later", () => { @@ -31,7 +37,23 @@ describe("current case facts and real subsequent context", () => { const missing = valid(); missing.briefRevisions = []; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false); }); it("rejects a plausible contrary chat report and missing collaborative acknowledgement", () => { - const falseReport = valid(); falseReport.state.collaborationCheckpoint.reply = "It is not only mobile Safari. Support escalation is support-123."; expect(gradeIntegratorCaseFacts(falseReport).passed).toBe(false); - const missing = valid(); missing.state.collaborationCheckpoint.reply = "Mobile Safari only."; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false); + for (const reply of ["No. Your mobile Safari only correction on COM-C1 was overwritten by support. Escalation: support-123.", + "COM-C1 no longer includes your browser correction. Support escalation is support-123.", + "Yes, COM-C1 still says mobile Safari only. The escalation is support-999.", + "Yes, COM-C1 still says mobile Safari only.", + "Yes, it still says mobile Safari only. Support escalation is support-123.", + `${FIRST}`]) + expect(gradeIntegratorCaseFacts(withFollowUp(reply)).passed, reply).toBe(false); + // The first reply already named the case; the follow-up must answer on its own. + const reusedFirst = valid(); reusedFirst.state.collaborationCheckpoint.reply = `${FIRST}\nYes, it still says mobile Safari only. Support escalation is support-123.`; + expect(gradeIntegratorCaseFacts(reusedFirst).passed).toBe(false); + }); + it("grades the saved browser by normalized tokens, not keywords anywhere in the fields", () => { + for (const browser of ["Mobile Safari only", "mobile Safari", "iOS Safari", "Safari on iPhone/iPad only", "Safari (iOS)"]) expect(corrected({ browser }), browser).toBe(true); + expect(corrected({ browser: "Safari", platform: "mobile only" })).toBe(true); + expect(corrected({ affectedBrowsers: ["iOS Safari"] })).toBe(true); + for (const fields of [{ browser: "Safari (desktop and mobile Safari)", note: "only intermittent" }, { browser: "Safari; affected device not yet confirmed" }, { browser: "Desktop Safari only" }, + { browser: "not mobile Safari" }, { browser: "Safari", note: "mobile only" }, { browser: "Chrome and mobile Safari" }, { affectedBrowsers: ["iOS Safari", "Chrome"] }, { browser: "Safari; device unknown", affectedBrowser: "iOS Safari" }, {}]) + expect(corrected(fields), JSON.stringify(fields)).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts index 187eae6bb92..6274e5b8684 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts @@ -19,11 +19,42 @@ async function observeFacts(ctx: ChatCatalogContext, state: Record(`/api/cases/${row.id}`), brief: await ctx.input.api.get(`/api/issues/${row.workIssueId}/documents/brief`), briefRevisions: await ctx.input.api.get(`/api/issues/${row.workIssueId}/documents/brief/revisions`) }; } function factsObservation(value: unknown): FactsObservation | undefined { const o = value as FactsObservation; return observation(value) && !!o.caseRecord && !!o.brief && Array.isArray(o.briefRevisions) ? o : undefined; } -function corrected(fields: unknown) { - const text = JSON.stringify(fields); const record = fields && typeof fields === "object" ? fields as Record : {}; - const singleBrowser = [record.browser, record.affectedBrowser].some(value => typeof value === "string" && /^mobile Safari$/i.test(value.trim())) - || Array.isArray(record.affectedBrowsers) && record.affectedBrowsers.length === 1 && /^mobile Safari$/i.test(String(record.affectedBrowsers[0])); - return /mobile Safari/i.test(text) && (singleBrowser || /\bonly\b|\bexclusive/i.test(text)) && !/not (?:only |exclusive[^ ]* )?mobile Safari|desktop Safari only/i.test(text); +/** The setup records the reproduction browser in `browser`; the person's correction is "only on mobile Safari". + * A corrected value is graded by its normalized tokens: Safari plus a mobile-platform qualifier and nothing + * else (filler such as "only"/"on" is ignored). "iOS Safari" and "Safari on iPhone/iPad" are the same fact; + * any other token (desktop, "not", "and", a second browser, the stale "not yet confirmed") is a different one. */ +const FILLER = new Set(["only", "on", "exclusively", "just", "the", "browser", "browsers", "device", "devices", "affected"]); +const MOBILE = new Set(["mobile", "ios", "iphone", "ipad", "ipados"]); +const tokens = (value: string) => value.toLowerCase().split(/[^a-z0-9]+/).filter(token => token && !FILLER.has(token)); +function mobileSafariOnly(value: unknown) { + if (typeof value !== "string") return false; const t = tokens(value); + return t.includes("safari") && t.some(token => MOBILE.has(token)) && t.every(token => token === "safari" || MOBILE.has(token)); +} +function mobilePlatformOnly(value: unknown) { if (typeof value !== "string") return false; const t = tokens(value); return t.length > 0 && t.every(token => MOBILE.has(token)); } +export function corrected(fields: unknown) { + const record = fields && typeof fields === "object" ? fields as Record : {}; + const platformSplit = Object.entries(record).some(([key, value]) => /^(?:platform|os|device)$/i.test(key) && mobilePlatformOnly(value)); + const browserValues = [record.browser, record.affectedBrowser].filter(value => value !== undefined); + if (record.affectedBrowsers !== undefined) browserValues.push(Array.isArray(record.affectedBrowsers) && record.affectedBrowsers.length === 1 ? record.affectedBrowsers[0] : null); + return browserValues.length > 0 && browserValues.every(value => mobileSafariOnly(value) + || platformSplit && typeof value === "string" && tokens(value).join(" ") === "safari"); +} +/** The follow-up reply is the text added after the first checkpoint. It must name the case (its identifier or its + * primary task) and the setup-chosen support escalation, and must not contradict the saved state: no other + * escalation value, no "No" answer, and no sentence saying the correction was lost. Saved fields are the proof. */ +export function latestReply(fresh: InternalObservation, first: InternalObservation | undefined) { + return first && fresh.reply.startsWith(first.reply) ? fresh.reply.slice(first.reply.length) : fresh.reply; +} +const escapeRe = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +export function reportsCurrentFacts(reply: string, o: FactsObservation) { + const caseRow = o.state.case as CaseRecord & { identifier?: string }; const task = o.tasks.find(row => row.id === caseRow.workIssueId); + const names = [caseRow.identifier, task?.identifier, caseRow.id, caseRow.workIssueId].filter((value): value is string => !!value); + const refersToCase = names.some(name => new RegExp(`(? found !== value); + const answersNo = /^\W*no\b/i.test(reply.trim()); + const lost = reply.split(/\n+|(?<=[.!?])\s+/).some(sentence => /\b(?:overwr[io]te|overwritten|lost|removed|reverted|dropped|discarded|no longer|missing|(?:is|was|are|were)n['’]?t (?:saved|kept|included|there|present)|not (?:saved|kept|included|there|present)|does(?:n['’]t| not) (?:include|contain|have))\b/i.test(sentence) + && /\b(?:correction|mobile Safari|browser|reproduction step)\b/i.test(sentence) && !/\b(?:nothing|not|never|wasn['’]?t|none)\b[^.!?]{0,30}\b(?:overwr|lost|removed|reverted|dropped|discarded)/i.test(sentence)); + return refersToCase && reply.includes(value) && !otherEscalation && !answersNo && !lost; } function retained(o: FactsObservation, concurrent: boolean) { const original = o.state.case as CaseRecord; const fields = o.caseRecord.fields as Record; @@ -38,8 +69,7 @@ export function gradeIntegratorCaseFacts(value: unknown) { const first = factsObservation(o.state.firstCorrection); const fresh = factsObservation(o.state.collaborationCheckpoint); c.add("initial-correction-preserves-unrelated-fields", !!first && retained(first, false), "The unchanged first query must already correct the saved facts without discarding unrelated fields; a later turn cannot repair an initially wrong mutation."); c.add("fresh-collaborative-change-not-stale-replacement", !!fresh && retained(fresh, true) && retained(o, true) - && fresh.reply.split(/\n+|(?<=[.!?])\s+/).some(sentence => /mobile Safari/i.test(sentence) && /only|exclusiv/i.test(sentence) && !/not (?:only |exclusive[^ ]* )?mobile Safari/i.test(sentence)) - && fresh.reply.includes(String(o.state.collaboratorValue)), "A real independently saved support change survives the explicit reread, and chat reports both the correction and that actual concurrent field."); + && reportsCurrentFacts(latestReply(fresh, first), o), "A real independently saved support change survives the explicit reread; the follow-up reply names the case and the actual concurrent field and does not contradict the saved correction."); const before = o.state.brief as IssueDocument; const original = o.state.case as CaseRecord; c.add("ordinary-task-document-ownership-and-history", o.brief.id === before.id && o.brief.issueId === original.workIssueId && o.briefRevisions.some(revision => revision.id === before.latestRevisionId && revision.issueId === original.workIssueId && revision.body === before.body), "The original ordinary-task document and its actual saved revision remain owned and accessible on the same primary task."); @@ -84,7 +114,7 @@ export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.up const row = state.case as CaseRecord; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title); const task = await ctx.input.api.get(`/api/issues/${row.workIssueId}`); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier}`, task.title); -}, { expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } }, +}, { oracleVersion: "paperclip.chat-catalog.integrator.case-facts.v2", expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } }, sourceFiles: ["chat-catalog-domains/integrator-case-facts.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], cleanup, async act(ctx, state) { const settle = (count: number) => pollUntil({ label: `saved case correction phase ${count}`, deadlineAt: Date.now() + 180_000, intervalMs: 1000,