From fdd772c7a7b853485a2b57d7815844a955f10326 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 21:58:21 -0500 Subject: [PATCH] test: cover corrected case facts and concurrent reread --- .../integrator-case-facts.test.ts | 37 ++++++++ .../integrator-case-facts.ts | 94 +++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts new file mode 100644 index 00000000000..d0b4063e016 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { gradeIntegratorCaseFacts } from "./integrator-case-facts.js"; +function valid() { + const original = { id: "case", workIssueId: "primary", handoffVersion: 1, fields: { browser: "Safari; device unknown", customer: "Acorn", severity: "normal", expected: "one order" } }; + const brief = { id: "brief", issueId: "primary", latestRevisionId: "v1", body: "Customer reproduction" }; + const fields = { ...original.fields, browser: "Mobile Safari only", supportEscalation: "support-123" }; + const base = { scenarioId: "cases.update-case-facts", origin: "https://fixture.test", companyId: "company", leadId: "ceo", + state: { case: original, brief, collaboratorValue: "support-123", reviewer: { id: "riley" } }, caseRecord: { ...original, fields }, brief, + briefRevisions: [{ id: "v1", issueId: "primary", body: brief.body }], projects: [], tasks: [], goals: [], documents: [], revisions: [], comments: [], links: [], activity: [], humanQueries: [], + reply: "The reproduction occurs exclusively on mobile Safari. Support escalation is support-123.", + runs: [{ id: "review", status: "succeeded", runtimeMode: "native", agentId: "riley", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: JSON.stringify({ fields }) } } }] }; + return { ...base, caseRecord: { ...base.caseRecord, handoffVersion: 2 }, state: { ...base.state, firstCorrection: structuredClone(base), collaborationCheckpoint: structuredClone(base) } }; +} +describe("current case facts and real subsequent context", () => { + it("accepts a substantive correction, preserved collaborator field and actual native projection", () => { + expect(gradeIntegratorCaseFacts(valid()).passed).toBe(true); + expect(gradeIntegratorCaseFacts(undefined).passed).toBe(false); + const natural = valid(); natural.state.collaborationCheckpoint.reply = "This affects mobile Safari only. The current support escalation is support-123."; + expect(gradeIntegratorCaseFacts(natural).passed).toBe(true); + const structured = valid(); structured.caseRecord.fields.browser = "mobile Safari"; expect(gradeIntegratorCaseFacts(structured).passed).toBe(true); + }); + it("rejects lost unrelated fields and an initially wrong correction repaired later", () => { + const lost = valid(); lost.caseRecord.fields.customer = "Other"; expect(gradeIntegratorCaseFacts(lost).passed).toBe(false); + const repaired = valid(); repaired.state.firstCorrection.caseRecord.fields.browser = "Desktop Safari only"; expect(gradeIntegratorCaseFacts(repaired).passed).toBe(false); + const stale = valid(); stale.state.collaborationCheckpoint.caseRecord.fields.supportEscalation = "stale"; expect(gradeIntegratorCaseFacts(stale).passed).toBe(false); + }); + it("rejects stale handoff context, chat-only projection and moved document ownership", () => { + const stale = valid(); stale.runs[0]!.contextSnapshot.paperclipCase.data = JSON.stringify({ fields: stale.state.case.fields }); expect(gradeIntegratorCaseFacts(stale).passed).toBe(false); + const chat = valid(); chat.runs[0]!.contextSnapshot.issueId = "chat"; expect(gradeIntegratorCaseFacts(chat).passed).toBe(false); + const moved = valid(); moved.brief.issueId = "case"; expect(gradeIntegratorCaseFacts(moved).passed).toBe(false); + const missing = valid(); missing.briefRevisions = []; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false); + }); + it("rejects a plausible contrary chat report and missing collaborative acknowledgement", () => { + const falseReport = valid(); falseReport.state.collaborationCheckpoint.reply = "It is not only mobile Safari. Support escalation is support-123."; expect(gradeIntegratorCaseFacts(falseReport).passed).toBe(false); + const missing = valid(); missing.state.collaborationCheckpoint.reply = "Mobile Safari only."; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts new file mode 100644 index 00000000000..ad441b6f71a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts @@ -0,0 +1,94 @@ +import type { cases } from "../../../packages/db/src/schema/cases.js"; +import type { HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js"; +import type { Issue } from "../../../packages/shared/src/types/issue.js"; +import type { IssueDocument, DocumentRevision } from "../../../packages/shared/src/types/issue.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { checks, observation, project, type Entity, type InternalObservation } from "./internal-support.js"; +import { fixtureWorker } from "./internal-definition.js"; +import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; +type CaseRecord = typeof cases.$inferSelect; +interface FactsObservation extends InternalObservation { caseRecord: CaseRecord; brief: IssueDocument; briefRevisions: DocumentRevision[] } +const settings = new WeakMap(); +async function cleanup(ctx: ChatCatalogContext) { const prior = settings.get(ctx); if (prior === undefined) return; await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: prior }); settings.delete(ctx); } +async function observeFacts(ctx: ChatCatalogContext, state: Record): Promise { + const base = await observeInternal(ctx, state); const row = state.case as CaseRecord; + return { ...base, caseRecord: await ctx.input.api.get(`/api/cases/${row.id}`), brief: await ctx.input.api.get(`/api/issues/${row.workIssueId}/documents/brief`), briefRevisions: await ctx.input.api.get(`/api/issues/${row.workIssueId}/documents/brief/revisions`) }; +} +function factsObservation(value: unknown): FactsObservation | undefined { const o = value as FactsObservation; return observation(value) && !!o.caseRecord && !!o.brief && Array.isArray(o.briefRevisions) ? o : undefined; } +function corrected(fields: unknown) { + const text = JSON.stringify(fields); const record = fields && typeof fields === "object" ? fields as Record : {}; + const singleBrowser = [record.browser, record.affectedBrowser].some(value => typeof value === "string" && /^mobile Safari$/i.test(value.trim())) + || Array.isArray(record.affectedBrowsers) && record.affectedBrowsers.length === 1 && /^mobile Safari$/i.test(String(record.affectedBrowsers[0])); + return /mobile Safari/i.test(text) && (singleBrowser || /\bonly\b|\bexclusive/i.test(text)) && !/not (?:only |exclusive[^ ]* )?mobile Safari|desktop Safari only/i.test(text); +} +function retained(o: FactsObservation, concurrent: boolean) { + const original = o.state.case as CaseRecord; const fields = o.caseRecord.fields as Record; + const before = original.fields as Record; + return o.caseRecord.id === original.id && o.caseRecord.workIssueId === original.workIssueId && corrected(fields) + && fields.customer === before.customer && fields.severity === before.severity && fields.expected === before.expected + && (!concurrent || fields.supportEscalation === o.state.collaboratorValue); +} +export function gradeIntegratorCaseFacts(value: unknown) { + const c = checks(); const o = factsObservation(value); c.add("saved-case-and-task-document-evidence", !!o, "Requires real case facts, ordinary-task document revisions, phase checkpoints and native handoff context."); if (!o) return c.result(); + const first = factsObservation(o.state.firstCorrection); const fresh = factsObservation(o.state.collaborationCheckpoint); + c.add("initial-correction-preserves-unrelated-fields", !!first && retained(first, false), "The unchanged first query must already correct the saved facts without discarding unrelated fields; a later turn cannot repair an initially wrong mutation."); + c.add("fresh-collaborative-change-not-stale-replacement", !!fresh && retained(fresh, true) && retained(o, true) + && fresh.reply.split(/\n+|(?<=[.!?])\s+/).some(sentence => /mobile Safari/i.test(sentence) && /only|exclusiv/i.test(sentence) && !/not (?:only |exclusive[^ ]* )?mobile Safari/i.test(sentence)) + && fresh.reply.includes(String(o.state.collaboratorValue)), "A real independently saved support change survives the explicit reread, and chat reports both the correction and that actual concurrent field."); + const before = o.state.brief as IssueDocument; const original = o.state.case as CaseRecord; + c.add("ordinary-task-document-ownership-and-history", o.brief.id === before.id && o.brief.issueId === original.workIssueId + && o.briefRevisions.some(revision => revision.id === before.latestRevisionId && revision.issueId === original.workIssueId && revision.body === before.body), "The original ordinary-task document and its actual saved revision remain owned and accessible on the same primary task."); + const reviewerId = (o.state.reviewer as Entity).id; + const run = o.runs.find(row => row.agentId === reviewerId && row.contextSnapshot?.issueId === original.workIssueId); + const context = run?.contextSnapshot?.paperclipCase as { id?: string; taskId?: string; data?: string } | undefined; + let projected: { fields?: unknown } | undefined; try { projected = JSON.parse(context?.data ?? "null") ?? undefined; } catch { /* malformed public projection fails below */ } + const projectedFields = projected?.fields as Record | undefined; + c.add("subsequent-real-handoff-carries-current-facts", !!run && run.status === "succeeded" && run.runtimeMode === "native" && context?.id === original.id && context.taskId === original.workIssueId + && corrected(projectedFields) && projectedFields?.supportEscalation === o.state.collaboratorValue + && projectedFields?.customer === (original.fields as Record).customer && o.caseRecord.handoffVersion === original.handoffVersion + 1, + "A subsequent genuinely admitted reviewer on the same primary receives the saved correction plus collaboration change in its actual case context, not a fixture-synthesized context."); + return c.result(); +} +export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.update-case-facts", async ctx => { + const prior = await ctx.input.api.get<{ enableCases: boolean }>("/api/instance/settings/experimental"); settings.set(ctx, prior.enableCases); + try { + await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: true }); const p = await project(ctx, "Launch"); + const worker = await fixtureWorker(ctx); const reviewer = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${reviewer.id}`, { name: "Riley" }); + for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[worker.id, reviewer.id].map(scopeId => ({ scopeType: "agent", scopeId }))]) + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + const kickoff = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, { + idempotencyKey: `facts-${ctx.input.nonce}`, case: { create: { caseType: "customer", title: "Checkout reproduction case", projectId: p.id, + fields: { browser: "Safari; affected device not yet confirmed", customer: "Acorn Test Shop", severity: "normal", expected: "Pay should submit a single order" } } }, toAgentId: worker.id, phase: "investigation", + instructions: "Read the saved customer brief. The affected device is not yet confirmed: record the missing reproduction detail in a task comment and mark this ordinary investigation blocked awaiting the customer's correction. Do not invent the affected device or alter case facts.", + workflow_document_text: "Morgan investigates this ordinary primary. A corrected customer reproduction detail must remain in saved case facts alongside unrelated customer fields. Riley reviews the current saved case when a later actual review handoff is requested, records a task comment, and completes the review task without closing the case.", + initialDocuments: [{ key: "brief", title: "Customer reproduction brief", body: "Acorn Test Shop: Safari checkout Pay sometimes does not submit. The affected device has not yet been confirmed. Expected result is one submitted order." }], + }); + await pollUntil({ label: "actual active case awaiting missing reproduction detail", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns() }), accept: v => v.runs.length === 1 && v.runs[0]?.status === "succeeded" && v.task.status === "blocked" }); + return { project: p, worker: { id: worker.id }, reviewer: { id: reviewer.id }, case: await ctx.input.api.get(`/api/cases/${kickoff.caseId}`), + brief: await ctx.input.api.get(`/api/issues/${kickoff.taskId}/documents/brief`), collaboratorValue: `support-${ctx.input.nonce}` }; + } catch (error) { await cleanup(ctx); throw error; } +}, observeFacts, gradeIntegratorCaseFacts, async (ctx, state) => { + const row = state.case as CaseRecord; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title); + const task = await ctx.input.api.get(`/api/issues/${row.workIssueId}`); + await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier}`, task.title); +}, { expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, + sourceFiles: ["chat-catalog-domains/integrator-case-facts.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], cleanup, + async act(ctx, state) { + const settle = (count: number) => pollUntil({ label: `saved case correction phase ${count}`, deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: () => observeFacts(ctx, state), accept: o => o.runs.length === count && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const first = await settle(2); state.firstCorrection = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-case-initial-correction.json", state.firstCorrection); + if (!retained(first, false)) throw new Error("Initial case correction was incorrect; a follow-up cannot repair this phase"); + const row = first.caseRecord; + await ctx.input.api.patch(`/api/cases/${row.id}`, { fields: { ...(row.fields as Record), supportEscalation: state.collaboratorValue } }); + const { sendChatMessage } = await import("../chat-flow.js"); + await sendChatMessage(ctx.input.page, "Support updated this case while we were talking. Read its current saved facts: does it still include my corrected reproduction step, and what support escalation is saved now?"); + const fresh = await settle(3); state.collaborationCheckpoint = { ...fresh, state: structuredClone(state) }; await ctx.input.evidence("chat-case-concurrent-change.json", state.collaborationCheckpoint); + if (!retained(fresh, true)) throw new Error("Chat overwrote or lost the separately saved collaboration change"); + const task = await ctx.input.api.get(`/api/issues/${row.workIssueId}`); + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, { case: { id: row.id }, expectedHandoffVersion: fresh.caseRecord.handoffVersion, + expectedIssueStatusVersion: task.statusVersion, idempotencyKey: `facts-review-${ctx.input.nonce}`, toAgentId: (state.reviewer as Entity).id, phase: "review", + instructions: "Review the current saved case facts and task-owned brief, record your assessment in a task comment, and complete this ordinary review task. Keep the case open; no terminal handoff is requested." }); + }, +})];