From c80f926546cfc2ed1a9f246993f5f17380046098 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 20:09:27 -0500 Subject: [PATCH] test(evals): add real case kickoff and saved card generation checks --- .../integrator-case-create.test.ts | 37 ++++++++ .../integrator-case-create.ts | 89 +++++++++++++++++++ .../integrator-status-cards.test.ts | 34 ++++++- .../integrator-status-cards.ts | 43 ++++++++- 4 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts new file mode 100644 index 00000000000..55284c814fb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { customerBugFacts, preservesCustomerBugFacts, gradeIntegratorCaseCreation } from "./integrator-case-create.js"; +function valid() { + const brief = { id: "brief", issueId: "primary", companyId: "company", key: "brief", body: Object.values(customerBugFacts).join("\n") }; + const workflow = { id: "workflow", issueId: "primary", companyId: "company", key: "workflow", body: "Morgan investigates, saves findings, then hands this same primary task to its next owner for review." }; + const revision = (doc: typeof brief) => ({ id: `${doc.id}-revision`, documentId: doc.id, issueId: "primary", key: doc.key, body: doc.body, createdByAgentId: "ceo", createdByRunId: "source-run", createdAt: "2026-10-26T12:00:00Z" }); + return { scenarioId: "cases.create-case", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" }, task: { id: "source-brief" }, worker: { id: "morgan" } }, + tasks: [{ id: "source-brief", title: "Customer bug brief" }, { id: "primary", companyId: "company", title: "Customer checkout bug", assigneeAgentId: "morgan" }], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], reply: "Created the case and started Morgan's investigation on its primary task.", links: ["/ACORN/cases/case", "/ACORN/issues/primary"], + cases: [{ id: "case", companyId: "company", projectId: "project", workIssueId: "primary", fields: structuredClone(customerBugFacts) }], + handoffs: { case: [{ id: "kickoff", caseId: "case", version: 1, issueId: "primary", toAgentId: "morgan", actorId: "ceo", sourceRunId: "source-run", state: "delivered" }] }, taskDocuments: { primary: [workflow, brief] }, + documentVersions: { "primary:workflow": [revision(workflow)], "primary:brief": [revision(brief)] }, + runs: [{ id: "source-run", agentId: "ceo", status: "succeeded" }, { id: "investigation", agentId: "morgan", status: "running", contextSnapshot: { issueId: "primary" }, startedAt: "2026-10-26T12:01:00Z" }] }; +} +describe("real case kickoff independent controls", () => { + it("requires the real primary task, first Morgan dispatch and original saved documents", () => { + expect(gradeIntegratorCaseCreation(valid()).passed).toBe(true); + expect(gradeIntegratorCaseCreation(undefined).passed).toBe(false); + const passive = valid(); passive.cases[0]!.workIssueId = ""; + expect(gradeIntegratorCaseCreation(passive).passed).toBe(false); + const late = valid(); late.documentVersions["primary:brief"][0]!.createdAt = "2026-10-26T12:02:00Z"; + expect(gradeIntegratorCaseCreation(late).passed).toBe(false); + const otherOwner = valid(); otherOwner.taskDocuments.primary[1]!.issueId = "source-brief"; + expect(gradeIntegratorCaseCreation(otherOwner).passed).toBe(false); + const wrongWorker = valid(); wrongWorker.handoffs.case[0]!.toAgentId = "riley"; + expect(gradeIntegratorCaseCreation(wrongWorker).passed).toBe(false); + const repaired = valid(); repaired.documentVersions["primary:brief"][0]!.body = "Investigate a bug."; + expect(gradeIntegratorCaseCreation(repaired).passed).toBe(false); + expect(gradeIntegratorCaseCreation({ ...valid(), links: ["/ACORN/cases/case"] }).passed).toBe(false); + }); + it("accepts changed wording of factual fields but rejects swapped expected/observed behavior", () => { + const paraphrase = { customer: "Acorn Test Shop", issue: "Checkout stops responding", browser: "Safari 18 on iPhone 15", steps: ["Add the trial item", "Fill the test address", "Press Pay"], expectedBehavior: "An order confirmation should be shown", observedBehavior: "Without confirmation; customer is not charged" }; + expect(preservesCustomerBugFacts(paraphrase)).toBe(true); + expect(preservesCustomerBugFacts({ ...paraphrase, expectedBehavior: "No order confirmation appears" })).toBe(false); + expect(preservesCustomerBugFacts({ ...paraphrase, browser: "Chrome on Android" })).toBe(false); + expect(preservesCustomerBugFacts({ ...paraphrase, observedBehavior: "Order confirmation appears and customer is charged" })).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts new file mode 100644 index 00000000000..be4862e4463 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts @@ -0,0 +1,89 @@ +import type { cases } from "../../../packages/db/src/schema/cases.js"; +import type { CaseHandoff } from "../../../packages/shared/src/case-handoffs.js"; +import type { IssueDocument, DocumentRevision } from "../../../packages/shared/src/types/issue.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { checks, hasLink, observation, project, task, putDocument, type Entity, type InternalObservation } from "./internal-support.js"; +import { fixtureWorker } from "./internal-definition.js"; +import { integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; +type CaseRecord = typeof cases.$inferSelect; +interface CaseCreationObservation extends InternalObservation { cases: CaseRecord[]; handoffs: Record; taskDocuments: Record; documentVersions: Record } +export const customerBugFacts = { customer: "Acorn Test Shop", symptom: "Checkout button becomes unresponsive", browser: "Safari 18", device: "iPhone 15", reproduction: "Add the synthetic trial item, enter the test address, then press Pay", expected: "Order confirmation appears", actual: "No confirmation and no charge" }; +export function preservesCustomerBugFacts(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + const text = JSON.stringify(value); const entries: Array<{ path: string; text: string }> = []; + const walk = (node: unknown, path: string) => { + if (typeof node === "string") entries.push({ path, text: node }); + else if (node && typeof node === "object") for (const [key, nested] of Object.entries(node)) walk(nested, `${path}.${key}`); + }; walk(value, ""); + const expected = entries.filter(entry => /expect|should/i.test(entry.path)).map(entry => entry.text).join(" "); + const actual = entries.filter(entry => /actual|observ|result/i.test(entry.path)).map(entry => entry.text).join(" "); + return /Acorn Test Shop/i.test(text) && /checkout/i.test(text) && /unresponsive|non.responsive|does not respond|stops responding|no response/i.test(text) + && /Safari\s*18/i.test(text) && /iPhone\s*15/i.test(text) && /trial item/i.test(text) && /test address/i.test(text) && /\bPay\b/i.test(text) + && /order confirmation/i.test(expected) && !/\b(?:no|not|without|never)\b/i.test(expected) + && /(?:no|without) (?:order )?confirmation|confirmation (?:does not|doesn't|never) appear/i.test(actual) + && /no charge|not charged|without (?:a )?charge/i.test(actual); +} +const briefHasFacts = (body: string) => /Acorn Test Shop/i.test(body) && /checkout/i.test(body) && /unresponsive|non.responsive|does not respond|stops responding|no response/i.test(body) + && /Safari\s*18/i.test(body) && /iPhone\s*15/i.test(body) && /trial item/i.test(body) && /test address/i.test(body) && /\bPay\b/i.test(body) + && /order confirmation/i.test(body) && /(?:no|without) (?:order )?confirmation|confirmation (?:does not|doesn't|never) appear/i.test(body) && /no charge|not charged|without (?:a )?charge/i.test(body); +async function observeCreation(ctx: ChatCatalogContext, state: Record): Promise { + const o = await observeInternal(ctx, state); const api = ctx.input.api; + const cases = await api.get(`/api/companies/${o.companyId}/cases`); + const handoffs: Record = {}; const taskDocuments: Record = {}; const documentVersions: Record = {}; + for (const row of cases) { + handoffs[row.id] = (await api.get<{ items: CaseHandoff[] }>(`/api/cases/${row.id}/handoffs?limit=100`)).items; + if (!row.workIssueId) continue; + const summaries = await api.get(`/api/issues/${row.workIssueId}/documents`); + taskDocuments[row.workIssueId] = await Promise.all(summaries.map(doc => api.get(`/api/issues/${row.workIssueId}/documents/${encodeURIComponent(doc.key)}`))); + for (const doc of summaries) documentVersions[`${row.workIssueId}:${doc.key}`] = await api.get(`/api/issues/${row.workIssueId}/documents/${encodeURIComponent(doc.key)}/revisions`); + } + return { ...o, cases, handoffs, taskDocuments, documentVersions }; +} +function creationObservation(value: unknown): CaseCreationObservation | undefined { + const o = value as CaseCreationObservation; + return observation(value) && Array.isArray(o.cases) && !!o.handoffs && !!o.taskDocuments && !!o.documentVersions ? o : undefined; +} +export function gradeIntegratorCaseCreation(value: unknown) { + const c = checks(); const o = creationObservation(value); + c.add("public-case-kickoff-and-document-evidence", !!o, "Requires actual public case, handoff, primary task and saved document revision evidence."); + if (!o) return c.result(); + const row = o.cases[0]; const worker = o.state.worker as Entity | undefined; + const primary = row?.workIssueId ? o.tasks.find(task => task.id === row.workIssueId) : undefined; + c.add("one-case-with-one-primary-task", o.cases.length === 1 && !!row && row.companyId === o.companyId && row.projectId === (o.state.project as Entity | undefined)?.id + && !!primary && o.tasks.filter(task => task.id !== (o.state.task as Entity | undefined)?.id).length === 1, "One actual company case has one primary task; the prerequisite customer brief remains a separate original record."); + c.add("structured-customer-facts", preservesCustomerBugFacts(row?.fields), "Structured case fields preserve the actual customer, device/browser, reproduction and expected versus observed behavior; prose may be paraphrased."); + const kickoff = row && o.handoffs[row.id]?.filter(handoff => handoff.version === 1); + c.add("real-first-morgan-kickoff", !!worker && kickoff?.length === 1 && kickoff[0]?.caseId === row?.id && kickoff[0]?.issueId === primary?.id && kickoff[0]?.toAgentId === worker.id + && kickoff[0]?.actorId === o.leadId && !!kickoff[0]?.sourceRunId && o.runs.some(run => run.id === kickoff[0]?.sourceRunId && run.agentId === o.leadId) + && ["delivered", "completed"].includes(kickoff[0]?.state ?? "") + && o.runs.some(run => run.agentId === worker.id && run.contextSnapshot?.issueId === primary?.id && ["queued", "running", "succeeded"].includes(run.status)), "The real measured CEO starts the primary task with Morgan as its first actual investigation recipient."); + const docs = primary ? o.taskDocuments[primary.id] ?? [] : []; + const workflow = docs.find(doc => doc.key === "workflow"); const brief = docs.find(doc => doc.key !== "workflow" && briefHasFacts(doc.body)); + const workerRun = o.runs.find(run => run.agentId === worker?.id && run.contextSnapshot?.issueId === primary?.id); + const savedBeforeStart = (doc: IssueDocument | undefined, content: (body: string) => boolean) => !!doc && !!primary && doc.issueId === primary.id && doc.companyId === o.companyId + && !!workerRun && o.documentVersions[`${primary.id}:${doc.key}`]?.some(revision => revision.issueId === primary.id && revision.documentId === doc.id && revision.createdByAgentId === o.leadId + && content(revision.body) && revision.createdByRunId === kickoff?.[0]?.sourceRunId && Date.parse(String(revision.createdAt)) <= Date.parse(String(workerRun.startedAt))); + c.add("required-documents-persist-before-investigation", savedBeforeStart(workflow, body => /Morgan/i.test(body) && /\b(?:review|handoff|hand.off|owner)\b/i.test(body)) && savedBeforeStart(brief, briefHasFacts), "The primary task owns an attributed saved brief and workflow describing Morgan, owners and later handoffs before Morgan's run begins; a later repaired document cannot rescue a missing initial brief."); + c.add("case-and-primary-task-links", !!row && !!primary && !!(resultHref(o, row.id, "cases") ?? resultHref(o, row.identifier, "cases")) && hasLink(o, "issues", primary), "Chat links the actual case and primary task instead of a passive record with a fictional investigation claim."); + return c.result(); +} +const previous = new WeakMap(); +export const integratorCaseCreationCases = [integratorDefinition("cases", "cases.create-case", async ctx => { + const settings = await ctx.input.api.get<{ enableCases: boolean }>("/api/instance/settings/experimental"); previous.set(ctx, settings.enableCases); + try { + await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: true }); + const p = await project(ctx, "Launch"); const worker = await fixtureWorker(ctx); const t = await task(ctx, "Customer bug brief", p.id); + await putDocument(ctx, t.id, `# Customer bug brief\n\n${Object.entries(customerBugFacts).map(([key, value]) => `${key}: ${value}`).join("\n")}\n\nMorgan is the eligible initial investigator. Investigation should retain evidence for the later review handoff.`); + return { project: p, task: t, worker }; + } catch (error) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: settings.enableCases }); previous.delete(ctx); throw error; } +}, observeCreation, gradeIntegratorCaseCreation, async (ctx, _state, value) => { + const o = creationObservation(value); const row = o?.cases[0]; if (!o || !row) throw new Error("Missing created case result"); + const href = resultHref(o, row.id, "cases") ?? resultHref(o, row.identifier, "cases"); if (!href) throw new Error("Missing real case link"); + await inspectResult(ctx, new URL(href, o.origin).pathname, row.title, href); + const primary = o.tasks.find(task => task.id === row.workIssueId); if (!primary) throw new Error("Missing primary case task"); + const taskHref = o.links.find(link => hasLink({ ...o, links: [link] }, "issues", primary)); if (!taskHref) throw new Error("Missing primary task link"); + await inspectResult(ctx, new URL(taskHref, o.origin).pathname, primary.title!, taskHref); +}, { sourceFiles: ["chat-catalog-domains/integrator-case-create.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], + expectedRunCount: 2, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, attemptTimeoutMs: 420_000, + async cleanup(ctx) { const value = previous.get(ctx); if (value !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: value }); previous.delete(ctx); } }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts index e74439099b4..14a512850ea 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts @@ -6,11 +6,27 @@ function valid(id: string) { const update = { id: "update", kind: "full", status: "ok", runId: "worker-run", generationIssueId: "generation-task" }; const summary = { id: "summary", body: card.summaryBody }; const source = { id: "source", title: "Payment approval blocks launch", status: "blocked" }; - return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" }, worker: { id: "morgan" }, sourceTasks: [structuredClone(source)], ...(creation ? {} : { card: { ...card, archivedAt: null, archivedByAgentId: null }, originalUpdates: [structuredClone(update)], originalSummaries: [structuredClone(summary)], ...(archive ? { otherCard: { ...card, id: "other", interestPrompt: "Research blockers", archivedAt: null, archivedByAgentId: null } } : {}) }) }, + const result = { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" }, worker: { id: "morgan" }, sourceTasks: [structuredClone(source)], ...(creation ? {} : { card: { ...card, archivedAt: null, archivedByAgentId: null }, originalUpdates: [structuredClone(update)], originalSummaries: [structuredClone(summary)], ...(archive ? { otherCard: { ...card, id: "other", interestPrompt: "Research blockers", archivedAt: null, archivedByAgentId: null } } : {}) }) }, tasks: [source], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: ["/ACORN/status/card"], runs: [{ id: "source-run", agentId: "ceo", status: "succeeded" }, { id: "worker-run", agentId: "morgan", status: "succeeded" }], activity: creation ? [{ id: "create", action: "status_card.created", entityId: "card", agentId: "ceo" }] : [], cardsEnabled: true, cards: archive ? [card, { ...card, id: "other", interestPrompt: "Research blockers", archivedAt: null, archivedByAgentId: null }] : [card], updates: { card: [update] }, summaries: { card: [summary] }, reply: creation ? "Created the Launch card; the generated summary is ready." : archive ? "The Launch card is archived. Restore it from archived cards and refresh its older content." : "I cannot manage Morgan's card: only its author or a human operator can change the refresh policy." }; + if (id === "status-cards.explain-card") { + result.tasks[0]!.status = "done"; result.state.sourceTasks[0]!.status = "done"; + result.reply = "The card reflects its older observation: payment approval blocked Launch. The source task is now done and the payment approval is resolved."; + result.links.push("/ACORN/issues/source"); + } + if (id === "status-cards.refresh-card") { + const first = structuredClone(result); first.reply = "The refresh was requested; generation is pending."; + Object.assign(result.state, { firstRefresh: first }); + result.cards[0]!.summaryBody = "The remaining blocker is mobile Safari checkout."; + result.cards[0]!.lastGeneratedAt = "2026-10-26T12:00:00Z"; + result.updates.card.push({ ...update, id: "new-update", runId: "new-worker-run", generationIssueId: "refresh-task" }); + result.summaries.card.push({ id: "new-summary", body: result.cards[0]!.summaryBody }); + result.runs.push({ id: "new-worker-run", agentId: "morgan", status: "succeeded" }); + result.reply = `${first.reply}\nThe refreshed summary identifies mobile Safari checkout as the remaining blocker.`; + } + return result; } describe("saved card state and truthful generation controls", () => { for (const definition of integratorStatusCardsCases) it(`${definition.scenarioId}: actual-shaped outcome and missing evidence`, () => { @@ -40,4 +56,20 @@ describe("saved card state and truthful generation controls", () => { const stolen = structuredClone(o); stolen.cards[0]!.createdByAgentId = "ceo"; expect(gradeIntegratorCards(stolen).passed).toBe(false); }); + it("rejects an older blocker presented as current and missing source grounding", () => { + const o = valid("status-cards.explain-card"); + expect(gradeIntegratorCards({ ...o, reply: "The card is current and up-to-date. Payment approval still blocks Launch although the task is done." }).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, links: ["/ACORN/status/card"] }).passed).toBe(false); + const fabricated = structuredClone(o); fabricated.cards[0]!.lastGeneratedAt = "2026-10-26T12:00:00Z"; + expect(gradeIntegratorCards(fabricated).passed).toBe(false); + }); + it("rejects stale acknowledgement described as fresh, duplicate refresh and no later result", () => { + const o = valid("status-cards.refresh-card"); + const first = (o.state as Record).firstRefresh as ReturnType; + const dishonest = structuredClone(o); ((dishonest.state as Record).firstRefresh as ReturnType).reply = "The card is updated and ready with a fresh summary."; + expect(gradeIntegratorCards(dishonest).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, reply: first.reply }).passed).toBe(false); + const duplicate = structuredClone(o); duplicate.updates.card.push({ ...duplicate.updates.card[1]!, id: "duplicate" }); + expect(gradeIntegratorCards(duplicate).passed).toBe(false); + }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts index 630eb84fa76..603cc5845f4 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts @@ -1,7 +1,7 @@ import type { StatusCard, StatusCardUpdate, StatusCardSummaryRevision } from "../../../packages/shared/src/validators/status-card.js"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; -import { checks, observation, project, task, sameTask, type Entity, type InternalObservation } from "./internal-support.js"; +import { checks, observation, project, task, sameTask, hasLink, type Entity, type InternalObservation } from "./internal-support.js"; import { fixtureWorker } from "./internal-definition.js"; import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record; summaries: Record; cardsEnabled: boolean } @@ -51,6 +51,27 @@ export function gradeIntegratorCards(value: unknown) { && JSON.stringify(o.updates[card.id]) === JSON.stringify(o.state.originalUpdates), "Archiving retains actual generated history and summary bytes."); c.add("archive-restoration-truthful", groundedStatement(o.reply, /\b(?:card|launch)\b/i, /\barchived\b/i, /\b(?:deleted|permanently removed|not archived)\b/i) && /\brestor(?:e|ing|ation)\b/i.test(o.reply) && /\brefresh\b/i.test(o.reply), "Chat reports archive rather than deletion and explains restoration plus possible refresh."); + } else if (o.scenarioId === "status-cards.explain-card") { + const source = originals[0]; + c.add("saved-observation-is-stale", !!before && !!card && !!source && card.summaryBody === before.summaryBody && card.lastGeneratedAt === before.lastGeneratedAt + && card.fingerprint?.[source.id]?.status === "blocked" && source.status === "done" && freshSummary(o, card), "The real previous generated blocker summary is retained while the actual source task is now done."); + c.add("saved-versus-live-explained", groundedStatement(o.reply, /\b(?:card|summary|observation)\b/i, /\b(?:stale|older|previous|last|outdated)\b/i, + /\b(?:still|currently|remains)\s+(?:is\s+)?blocked\b|\b(?:summary|card)\s+(?:is\s+)?(?:current|up.to.date|fresh)\b/i) + && /payment approval/i.test(o.reply) && /\b(?:done|resolved|completed|approved)\b/i.test(o.reply), "The explanation grounds the saved blocker in its older generation and distinguishes the resolved live source state."); + c.add("saved-card-and-live-task-links", !!card && !!source && !!cardLink(o, card.id) && hasLink(o, "issues", source), "The explanation links the actual saved card and its changed source task."); + } else if (o.scenarioId === "status-cards.refresh-card") { + const initial = cardsObservation(o.state.firstRefresh); const oldIds = new Set((o.state.originalUpdates as StatusCardUpdate[] | undefined)?.map(row => row.id)); + const changes = card ? o.updates[card.id]?.filter(row => !oldIds.has(row.id) && row.kind !== "compile") ?? [] : []; + c.add("one-real-new-refresh-output", !!before && !!card && card.createdByAgentId === o.leadId && changes.length === 1 && changes[0]?.status === "ok" + && freshSummary(o, card) && card.lastGeneratedAt !== before.lastGeneratedAt && card.summaryBody !== before.summaryBody + && o.summaries[card.id]?.length === (o.state.originalSummaries as StatusCardSummaryRevision[] | undefined)?.length! + 1 + && /mobile Safari/i.test(card.summaryBody ?? ""), "Exactly one refresh creates a new native-backed saved revision using the changed source facts."); + const initialCard = initial?.cards.find(row => row.id === before?.id); + c.add("initial-acknowledgement-not-false-freshness", !!initial && !!initialCard && (!/\b(?:summary|card|blockers?)\b[^.!?\n]{0,45}\b(?:fresh|updated|generated|ready|complete)\b/i.test(initial.reply) + || initialCard.lastGeneratedAt !== before?.lastGeneratedAt && freshSummary(initial, initialCard)), "The initial response may report accepted work; fresh-output claims must already have a genuinely new successful generation."); + const finalReply = initial && o.reply.startsWith(initial.reply) ? o.reply.slice(initial.reply.length) : ""; + c.add("actual-result-reported-after-generation", /mobile Safari/i.test(finalReply) && /\b(?:generated|refreshed|updated|summary|blocker)\b/i.test(finalReply) + && !/\b(?:still waiting|not refreshed|refresh failed|could not refresh)\b/i.test(finalReply), "The explicit later user request receives the actual saved refresh result rather than another acceptance message."); } else if (o.scenarioId === "status-cards.other-authors-card") { c.add("morgan-card-not-evaded", !!before && !!card && card.createdByAgentId === (o.state.worker as Entity | undefined)?.id && o.cards.length === 1 && card.interestPrompt === before.interestPrompt && JSON.stringify(card.refreshPolicy) === JSON.stringify(before.refreshPolicy) && card.archivedAt === before.archivedAt @@ -103,6 +124,11 @@ function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial ctx.allRuns(), accept: rows => rows.length === 2 && rows.every(row => row.status === "succeeded") }); } + if (["status-cards.refresh-card", "status-cards.explain-card"].includes(id)) { + sourceTasks[0] = await ctx.input.api.patch(`/api/issues/${t.id}`, id === "status-cards.explain-card" + ? { status: "done", description: "The payment approval has been received; this blocker is resolved." } + : { description: "Payment approval has been received. The remaining blocker is the mobile Safari checkout regression." }); + } return { project: p, sourceTasks, worker, card, otherCard, originalUpdates: await ctx.input.api.get(`/api/status-cards/${card.id}/updates`), originalSummaries: await ctx.input.api.get(`/api/status-cards/${card.id}/summary-revisions`) }; } catch (error) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableStatusCards: previous.enableStatusCards }); experiments.delete(ctx); throw error; } }, observeCards, gradeIntegratorCards, verifyCard, { expectedRunCount: 2, attemptTimeoutMs: 540_000, ...extra, @@ -112,4 +138,19 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [ define("status-cards.create-card", "none"), define("status-cards.archive-card", "ceo", { expectedRunCount: 3, attemptTimeoutMs: 720_000 }), define("status-cards.other-authors-card", "morgan"), + define("status-cards.explain-card", "ceo", { coverageGaps: ["missing-source-access"] }), + define("status-cards.refresh-card", "ceo", { expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, + coverageGaps: ["failed-generation-retains-prior-summary"], + async act(ctx, state) { + const first = await pollUntil({ label: "initial actual refresh acknowledgement", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: () => observeCards(ctx, state), accept: o => o.runs.filter(row => row.agentId === o.leadId && row.status === "succeeded").length === 1 && ctx.issue()?.conversationState === "waiting" }); + state.firstRefresh = { ...first, state: structuredClone(state) }; + await ctx.input.evidence("chat-card-refresh-initial-response.json", state.firstRefresh); + const before = state.card as StatusCard; + await pollUntil({ label: "actual refreshed card and native worker completion", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: () => observeCards(ctx, state), accept: o => { const card = o.cards.find(row => row.id === before.id); return !!card && card.lastGeneratedAt !== before.lastGeneratedAt && freshSummary(o, card) && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded"); } }); + const { sendChatMessage } = await import("../chat-flow.js"); + await sendChatMessage(ctx.input.page, "What did the refresh actually produce?"); + }, + }), ];