diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 3c2bee1bf2b..b2ff82b7946 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -145,7 +145,10 @@ export type ChatCatalogRunPolicy = * The case oracle must bind every run above the minimum to a declared role. */ maximumTerminalStatusCounts?: Partial> } | { kind: "source-settled-with-active-work"; maximumActiveWorkerRuns: number; - expectedWorkerCancellation?: "owned-task-reassignment" }; + expectedWorkerCancellation?: "owned-task-reassignment"; + /** Lead runs outside the source woken by a case handoff back to the lead (a worker returning its case). + * Defaults to 0; the case oracle must bind each one to the worker's actual handoff. */ + maximumLeadCaseHandbackRuns?: number }; export interface ChatCatalogExpectedWorkerCancellation { runId: string; taskId: string; workerId: string; sourceRunId: string; } diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 69c069fd693..81936ec35f9 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -1368,15 +1368,16 @@ "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", - "expectedRunCount": 2, + "oracleVersion": "paperclip.chat-catalog.integrator.case-create.v2", + "expectedRunCount": 3, "expectedHumanMessageCount": 1, "minimumRunCount": 2, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, "runPolicy": { "kind": "source-settled-with-active-work", - "maximumActiveWorkerRuns": 1 + "maximumActiveWorkerRuns": 1, + "maximumLeadCaseHandbackRuns": 1 }, "modulePath": "./chat-catalog-domains/integrator-case-create.js", "exportName": "integratorCaseCreationCases", @@ -1448,7 +1449,7 @@ "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.case-facts.v7", + "oracleVersion": "paperclip.chat-catalog.integrator.case-facts.v8", "expectedRunCount": 6, "minimumRunCount": 4, "expectedHumanMessageCount": 2, @@ -5677,7 +5678,7 @@ "run-observations.ts", "chat-scenarios/task-coordination.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.block-on-design.v8", + "oracleVersion": "paperclip.chat-catalog.runtime.block-on-design.v9", "coverageGaps": [], "expectedRunCount": 4, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts index de0ffdeba35..0685da2c25f 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.test.ts @@ -9,9 +9,10 @@ function valid() { return { scenarioId: "cases.create-case", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" }, task: { id: "source-brief" }, worker: { id: "morgan" } }, tasks: [{ id: "source-brief", title: "Customer bug brief" }, { id: "primary", companyId: "company", title: "Customer checkout bug", assigneeAgentId: "morgan" }], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], reply: "Created the case and started Morgan's investigation on its primary task.", links: ["/ACORN/cases/case", "/ACORN/issues/primary"], cases: [{ id: "case", companyId: "company", projectId: "project", workIssueId: "primary", fields: structuredClone(customerBugFacts) }], - handoffs: { case: [{ id: "kickoff", caseId: "case", version: 1, issueId: "primary", toAgentId: "morgan", actorId: "ceo", sourceRunId: "source-run", state: "delivered" }] }, taskDocuments: { primary: [workflow, brief] }, + handoffs: { case: [{ id: "kickoff", caseId: "case", version: 1, issueId: "primary", toAgentId: "morgan", actorId: "ceo", sourceRunId: null as string | null, state: "delivered", createdAt: "2026-10-26T12:00:30Z" }] as Array> }, taskDocuments: { primary: [workflow, brief] }, documentVersions: { "primary:workflow": [revision(workflow)], "primary:brief": [revision(brief)] }, - runs: [{ id: "source-run", agentId: "ceo", status: "succeeded" }, { id: "investigation", agentId: "morgan", status: "running", contextSnapshot: { issueId: "primary" }, startedAt: "2026-10-26T12:01:00Z" }] }; + runs: [{ id: "source-run", agentId: "ceo", status: "succeeded", contextSnapshot: { issueId: "source-brief" } as Record, startedAt: "2026-10-26T11:59:00Z", finishedAt: "2026-10-26T12:00:40Z" }, + { id: "investigation", agentId: "morgan", status: "running", contextSnapshot: { issueId: "primary" } as Record, startedAt: "2026-10-26T12:01:00Z" } as Record] }; } describe("real case kickoff independent controls", () => { it("requires the real primary task, first Morgan dispatch and original saved documents", () => { @@ -29,6 +30,33 @@ describe("real case kickoff independent controls", () => { expect(gradeIntegratorCaseCreation(repaired).passed).toBe(false); expect(gradeIntegratorCaseCreation({ ...valid(), links: ["/ACORN/cases/case"] }).passed).toBe(false); }); + // gha-43101125481: a kickoff that creates its primary task has no sourceRunId (that field names the run holding the + // task being handed off). The kickoff and its documents are bound to the CEO chat run active when it was saved. + it("binds the kickoff and its documents to the CEO chat run that saved them, not to sourceRunId", () => { + const failed = (value: unknown) => gradeIntegratorCaseCreation(value).checks.filter(check => !check.passed).map(check => check.id); + expect(failed(valid())).toEqual([]); + const otherRun = valid(); for (const key of Object.keys(otherRun.documentVersions)) otherRun.documentVersions[key as keyof typeof otherRun.documentVersions][0]!.createdByRunId = "elsewhere"; + expect(failed(otherRun)).toContain("required-documents-persist-before-investigation"); + const outside = valid(); outside.handoffs.case[0]!.createdAt = "2026-10-26T12:05:00Z"; + expect(failed(outside)).toEqual(expect.arrayContaining(["real-first-morgan-kickoff", "required-documents-persist-before-investigation"])); + const byWorker = valid(); byWorker.handoffs.case[0]!.actorId = "morgan"; + expect(failed(byWorker)).toContain("real-first-morgan-kickoff"); + const otherChat = valid(); otherChat.runs[0]!.contextSnapshot = { issueId: "primary" }; + expect(failed(otherChat)).toContain("real-first-morgan-kickoff"); + }); + it("accepts one CEO run woken by Morgan's own case handoff back, and nothing else", () => { + const failed = (value: unknown) => gradeIntegratorCaseCreation(value).checks.filter(check => !check.passed).map(check => check.id); + const handback = (o: ReturnType, wakeReason = "case_handoff", issueId = "primary") => { + o.runs.push({ id: "handback", agentId: "ceo", status: "running", contextSnapshot: { issueId, wakeReason }, startedAt: "2026-10-26T12:03:00Z" }); + return o; + }; + const returned = (o: ReturnType) => { o.handoffs.case.push({ id: "back", caseId: "case", version: 2, issueId: "primary", toAgentId: "ceo", actorId: "morgan", state: "delivered" }); return o; }; + expect(failed(returned(handback(valid())))).toEqual([]); + expect(failed(handback(valid()))).toContain("only-declared-case-handback-to-ceo"); + expect(failed(returned(handback(valid(), "issue_commented")))).toContain("only-declared-case-handback-to-ceo"); + expect(failed(returned(handback(valid(), "case_handoff", "elsewhere")))).toContain("only-declared-case-handback-to-ceo"); + expect(failed(returned(handback(handback(valid()))))).toContain("only-declared-case-handback-to-ceo"); + }); it("accepts changed wording of factual fields but rejects swapped expected/observed behavior", () => { const paraphrase = { customer: "Acorn Test Shop", issue: "Checkout stops responding", browser: "Safari 18 on iPhone 15", steps: ["Add the trial item", "Fill the test address", "Press Pay"], expectedBehavior: "An order confirmation should be shown", observedBehavior: "Without confirmation; customer is not charged" }; expect(preservesCustomerBugFacts(paraphrase)).toBe(true); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts index 6bd2b0c7e2d..a26136fc13e 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-create.ts @@ -40,6 +40,13 @@ async function observeCreation(ctx: ChatCatalogContext, state: Record run.agentId === o.leadId && run.contextSnapshot?.issueId === sourceId && Date.parse(String(run.startedAt)) <= at + && (!(run as { finishedAt?: string | null }).finishedAt || at <= Date.parse(String((run as { finishedAt?: string | null }).finishedAt)))); +} function creationObservation(value: unknown): CaseCreationObservation | undefined { const o = value as CaseCreationObservation; return observation(value) && Array.isArray(o.cases) && !!o.handoffs && !!o.taskDocuments && !!o.documentVersions ? o : undefined; @@ -54,8 +61,11 @@ export function gradeIntegratorCaseCreation(value: unknown) { && !!primary && o.tasks.filter(task => task.id !== (o.state.task as Entity | undefined)?.id).length === 1, "One actual company case has one primary task; the prerequisite customer brief remains a separate original record."); c.add("structured-customer-facts", preservesCustomerBugFacts(row?.fields), "Structured case fields preserve the actual customer, device/browser, reproduction and expected versus observed behavior; prose may be paraphrased."); const kickoff = row && o.handoffs[row.id]?.filter(handoff => handoff.version === 1); + // A handoff's sourceRunId names the run that held the task being handed off, so a kickoff that creates its + // primary task has none (gha-43101125481). Bind the kickoff to the measured CEO chat run active when it was saved. + const kickoffRun = kickoffSourceRun(o, kickoff?.[0]); c.add("real-first-morgan-kickoff", !!worker && kickoff?.length === 1 && kickoff[0]?.caseId === row?.id && kickoff[0]?.issueId === primary?.id && kickoff[0]?.toAgentId === worker.id - && kickoff[0]?.actorId === o.leadId && !!kickoff[0]?.sourceRunId && o.runs.some(run => run.id === kickoff[0]?.sourceRunId && run.agentId === o.leadId) + && kickoff[0]?.actorId === o.leadId && !!kickoffRun && ["delivered", "completed"].includes(kickoff[0]?.state ?? "") && o.runs.some(run => run.agentId === worker.id && run.contextSnapshot?.issueId === primary?.id && ["queued", "running", "succeeded"].includes(run.status)), "The real measured CEO starts the primary task with Morgan as its first actual investigation recipient."); const docs = primary ? o.taskDocuments[primary.id] ?? [] : []; @@ -63,8 +73,14 @@ export function gradeIntegratorCaseCreation(value: unknown) { const workerRun = o.runs.find(run => run.agentId === worker?.id && run.contextSnapshot?.issueId === primary?.id); const savedBeforeStart = (doc: IssueDocument | undefined, content: (body: string) => boolean) => !!doc && !!primary && doc.issueId === primary.id && doc.companyId === o.companyId && !!workerRun && o.documentVersions[`${primary.id}:${doc.key}`]?.some(revision => revision.issueId === primary.id && revision.documentId === doc.id && revision.createdByAgentId === o.leadId - && content(revision.body) && revision.createdByRunId === kickoff?.[0]?.sourceRunId && Date.parse(String(revision.createdAt)) <= Date.parse(String(workerRun.startedAt))); + && content(revision.body) && !!kickoffRun && revision.createdByRunId === kickoffRun.id && Date.parse(String(revision.createdAt)) <= Date.parse(String(workerRun.startedAt))); c.add("required-documents-persist-before-investigation", savedBeforeStart(workflow, body => /Morgan/i.test(body) && /\b(?:review|handoff|hand.off|owner)\b/i.test(body)) && savedBeforeStart(brief, briefHasFacts), "The primary task owns an attributed saved brief and workflow describing Morgan, owners and later handoffs before Morgan's run begins; a later repaired document cannot rescue a missing initial brief."); + // Morgan may legitimately hand the case back to the CEO (e.g. blocked without app access); that wakes the CEO on the + // primary task once. Any other CEO run outside this conversation is undeclared work. + const handbacks = o.runs.filter(run => run.agentId === o.leadId && run.contextSnapshot?.issueId !== (o.state.task as Entity | undefined)?.id); + c.add("only-declared-case-handback-to-ceo", handbacks.length <= 1 && handbacks.every(run => run.contextSnapshot?.issueId === primary?.id && run.contextSnapshot?.wakeReason === "case_handoff" + && !!row && (o.handoffs[row.id] ?? []).some(handoff => handoff.version > 1 && handoff.issueId === primary?.id && handoff.toAgentId === o.leadId && handoff.actorId === worker?.id)), + "Beyond its chat turn, the CEO runs at most once, on the primary task, woken by Morgan's own case handoff back to it."); c.add("case-and-primary-task-links", !!row && !!primary && !!(resultHref(o, row.id, "cases") ?? resultHref(o, row.identifier, "cases")) && hasLink(o, "issues", primary), "Chat links the actual case and primary task instead of a passive record with a fictional investigation claim."); return c.result(); } @@ -93,6 +109,7 @@ export const integratorCaseCreationCases = [integratorDefinition("cases", "cases const taskHref = o.links.find(link => hasLink({ ...o, links: [link] }, "issues", primary)); if (!taskHref) throw new Error("Missing primary task link"); await inspectResult(ctx, new URL(taskHref, o.origin).pathname, primary.title!, taskHref); }, { authentication: "human-session", queryAttachments: factualContextQueryAttachment("customer-bug-brief.md"), sourceFiles: ["chat-catalog-domains/integrator-case-create.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], - expectedRunCount: 2, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, attemptTimeoutMs: 420_000, + oracleVersion: "paperclip.chat-catalog.integrator.case-create.v2", + expectedRunCount: 3, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1, maximumLeadCaseHandbackRuns: 1 }, attemptTimeoutMs: 420_000, async cleanup(ctx) { const value = previous.get(ctx); if (value !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: value }); previous.delete(ctx); } }, })]; diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts index 462e6f6d270..a6ce2312bb6 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { corrected, gradeIntegratorCaseFacts } from "./integrator-case-facts.js"; +import { corrected, gradeIntegratorCaseFacts, SETUP_BROWSER } from "./integrator-case-facts.js"; const FIRST = "Updated case COM-C1: browser is now **\"Mobile Safari only\"**, with customer, expected result, and severity preserved. See [COM-1](/COM/issues/COM-1)."; // Recorded follow-up reply from run 42584001490 (escalation value substituted). const FOLLOW_UP = "Yes. The case's saved facts still include your correction: `browser` = **Mobile Safari only**. The other facts are unchanged: customer **Acorn Test Shop**, expected **Pay should submit a single order**, severity **normal**.\n\nThe saved support escalation is **support-123**.\n\nOne caveat: the separate reproduction brief document still says \"The affected device has not yet been confirmed.\" Case COM-C1 is linked to [COM-1](/COM/issues/COM-1)."; @@ -29,7 +29,7 @@ describe("current case facts and real subsequent context", () => { }); it("rejects lost unrelated fields and an initially wrong correction repaired later", () => { const lost = valid(); lost.caseRecord.fields.customer = "Other"; expect(gradeIntegratorCaseFacts(lost).passed).toBe(false); - const repaired = valid(); repaired.state.firstCorrection.caseRecord.fields.browser = "Desktop Safari only"; expect(gradeIntegratorCaseFacts(repaired).passed).toBe(false); + const repaired = valid(); repaired.state.firstCorrection.caseRecord.fields.browser = repaired.state.case.fields.browser; expect(gradeIntegratorCaseFacts(repaired).passed).toBe(false); const stale = valid(); stale.state.collaborationCheckpoint.caseRecord.fields.supportEscalation = "stale"; expect(gradeIntegratorCaseFacts(stale).passed).toBe(false); }); it("rejects stale handoff context, chat-only projection and moved document ownership", () => { @@ -48,17 +48,14 @@ describe("current case facts and real subsequent context", () => { const reusedFirst = valid(); reusedFirst.state.collaborationCheckpoint.reply = `${FIRST}\nYes, it still says mobile Safari only. Support escalation is support-123.`; expect(gradeIntegratorCaseFacts(reusedFirst).passed).toBe(false); }); - it("grades the saved browser by normalized tokens, not keywords anywhere in the fields", () => { - for (const browser of ["Mobile Safari only", "mobile Safari", "iOS Safari", "Safari on iPhone/iPad only", "Safari (iOS)"]) expect(corrected({ browser }), browser).toBe(true); - expect(corrected({ browser: "Safari", platform: "mobile only" })).toBe(true); - // A clause that only rules another platform out is a clarification of the same fact. - for (const browser of ["Mobile Safari only (iPhone/iPad); desktop Safari unaffected", "Mobile Safari only - desktop Safari and Chrome are not affected", "iOS Safari only, works on desktop", "Mobile Safari only; not on Chrome", "Mobile Safari only; desktop Safari excluded"]) + it("grades the saved correction structurally, leaving the meaning of its wording to a person", () => { + // Any rewording of the correction passes, including the retained gha-43088288923 value. + for (const browser of ["Mobile Safari only (bug reproduces only on mobile Safari)", "Mobile Safari only", "iOS Safari", "Safari on iPhone/iPad only"]) expect(corrected({ browser }), browser).toBe(true); - for (const browser of ["Mobile Safari; desktop Safari also affected", "Mobile Safari only (not yet confirmed)", "Mobile Safari, not iOS", "Mobile Safari; desktop Safari not excluded", "Mobile Safari; desktop Safari is not excluded", "Safari; desktop unaffected", "Mobile Safari and desktop Chrome"]) - expect(corrected({ browser }), browser).toBe(false); - expect(corrected({ affectedBrowsers: ["iOS Safari"] })).toBe(true); - for (const fields of [{ browser: "Safari (desktop and mobile Safari)", note: "only intermittent" }, { browser: "Safari; affected device not yet confirmed" }, { browser: "Desktop Safari only" }, - { browser: "not mobile Safari" }, { browser: "Safari", note: "mobile only" }, { browser: "Chrome and mobile Safari" }, { affectedBrowsers: ["iOS Safari", "Chrome"] }, { browser: "Safari; device unknown", affectedBrowser: "iOS Safari" }, {}]) + expect(corrected({ browser: "Safari", platform: "mobile only" })).toBe(true); + // Broken saves fail: the unchanged setup value, the stale setup literal kept in any field, or the facts removed. + for (const fields of [{ browser: SETUP_BROWSER }, { browser: "Mobile Safari; affected device not yet confirmed" }, { browser: "Mobile Safari", note: "device not yet confirmed" }, { browser: "" }, {}, + { browser: "", customer: "Acorn", severity: "normal", expected: "one order" }, { customer: "Acorn", severity: "normal", expected: "one order", supportEscalation: "support-1" }, { browser: " ", platform: "mobile" }]) expect(corrected(fields), JSON.stringify(fields)).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts index 5df2bee4fc5..854563d03d1 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts @@ -33,34 +33,22 @@ export function caseFactsRoles(o: Pick !initial.includes(run) && !ceo.includes(run) && !continuation?.includes(run) && !review.includes(run)); return { ceo, continuation: continuation ?? [], review, valid: !!continuation && initial.length === 1 && review.length <= 1 && unexplained.length === 0 }; } -/** The setup records the reproduction browser in `browser`; the person's correction is "only on mobile Safari". - * A corrected value is graded by its normalized tokens: Safari plus a mobile-platform qualifier and nothing - * else (filler such as "only"/"on" is ignored). "iOS Safari" and "Safari on iPhone/iPad" are the same fact; - * any other token (desktop, "not", "and", a second browser, the stale "not yet confirmed") is a different one. */ -const FILLER = new Set(["only", "on", "exclusively", "just", "the", "browser", "browsers", "device", "devices", "affected"]); -const MOBILE = new Set(["mobile", "ios", "iphone", "ipad", "ipados"]); -const tokens = (value: string) => value.toLowerCase().split(/[^a-z0-9]+/).filter(token => token && !FILLER.has(token)); -/** A value may add clauses that only rule other platforms out ("desktop Safari unaffected", - * "not on Chrome"): such a clause carries a negation and names a non-mobile platform or browser and - * no mobile one. Every other clause together must still say exactly mobile Safari. */ -const EXCLUDES = /\b(?:unaffected|excluded|not (?:affected|impacted|reproducible)|(?:does not|doesn't|never) (?:occur|happen|reproduce)|works(?: fine)?|is fine)\b|^\s*(?:not|never|no)\b/i; -const OTHER_PLATFORM = new Set(["desktop", "mac", "macos", "windows", "linux", "android", "chrome", "firefox", "edge", "laptop"]); -function mobileSafariOnly(value: unknown) { - if (typeof value !== "string") return false; - const clauses = value.split(/[;,()]|\s[-\u2013\u2014]\s|\bbut\b/i).filter(clause => tokens(clause).length > 0); - const exclusion = (clause: string) => { const t = tokens(clause); - return EXCLUDES.test(clause) && !/\bnot\s+(?:excluded|unaffected)\b/i.test(clause) && t.some(token => OTHER_PLATFORM.has(token)) && !t.some(token => MOBILE.has(token)); }; - const t = clauses.filter(clause => !exclusion(clause)).flatMap(tokens); - return t.includes("safari") && t.some(token => MOBILE.has(token)) && t.every(token => token === "safari" || MOBILE.has(token)); -} -function mobilePlatformOnly(value: unknown) { if (typeof value !== "string") return false; const t = tokens(value); return t.length > 0 && t.every(token => MOBILE.has(token)); } -export function corrected(fields: unknown) { - const record = fields && typeof fields === "object" ? fields as Record : {}; - const platformSplit = Object.entries(record).some(([key, value]) => /^(?:platform|os|device)$/i.test(key) && mobilePlatformOnly(value)); - const browserValues = [record.browser, record.affectedBrowser].filter(value => value !== undefined); - if (record.affectedBrowsers !== undefined) browserValues.push(Array.isArray(record.affectedBrowsers) && record.affectedBrowsers.length === 1 ? record.affectedBrowsers[0] : null); - return browserValues.length > 0 && browserValues.every(value => mobileSafariOnly(value) - || platformSplit && typeof value === "string" && tokens(value).join(" ") === "safari"); +/** The setup records `browser: "Safari; affected device not yet confirmed"`; the person's correction is "only on mobile + * Safari". The saved correction is graded structurally, never by its wording (an allow-list rejected the correct "Mobile + * Safari only (bug reproduces only on mobile Safari)" in gha-43088288923, and a deny-list accepts wrong facts): the saved + * facts changed from the setup value and no field keeps the setup-chosen stale literal. Whether the new wording says + * mobile Safari is free-form text that a person reads in the saved case and the reply. */ +export const SETUP_BROWSER = "Safari; affected device not yet confirmed"; +const STALE = "not yet confirmed"; +/** Facts the correction must leave unchanged; they cannot stand in for the saved replacement fact. */ +const UNRELATED = new Set(["customer", "severity", "expected", "supportEscalation"]); +export function corrected(fields: unknown, setupBrowser: unknown = SETUP_BROWSER) { + const record = fields && typeof fields === "object" && !Array.isArray(fields) ? fields as Record : {}; + const strings = (entries: unknown[]) => entries.flatMap(value => Array.isArray(value) ? value : [value]).filter((value): value is string => typeof value === "string"); + const values = strings(Object.values(record)); + const replacement = strings(Object.entries(record).filter(([key]) => !UNRELATED.has(key)).map(([, value]) => value)).filter(value => value.trim().length > 0); + return replacement.length > 0 && (record.browser === undefined || strings([record.browser]).some(value => value.trim().length > 0)) + && record.browser !== setupBrowser && !values.some(value => value.toLowerCase().includes(STALE)); } /** The follow-up reply is the text added after the first checkpoint. It must name the case (its identifier or its * primary task) and the setup-chosen support escalation literal. Whether its wording is right is free-form text that a @@ -78,7 +66,7 @@ export function reportsCurrentFacts(reply: string, o: FactsObservation) { function retained(o: FactsObservation, concurrent: boolean) { const original = o.state.case as CaseRecord; const fields = o.caseRecord.fields as Record; const before = original.fields as Record; - return o.caseRecord.id === original.id && o.caseRecord.workIssueId === original.workIssueId && corrected(fields) + return o.caseRecord.id === original.id && o.caseRecord.workIssueId === original.workIssueId && corrected(fields, before.browser) && fields.customer === before.customer && fields.severity === before.severity && fields.expected === before.expected && (!concurrent || fields.supportEscalation === o.state.collaboratorValue); } @@ -101,7 +89,7 @@ export function gradeIntegratorCaseFacts(value: unknown) { c.add("role-bound-runs", roles.valid && roles.ceo.length === 2 && roles.review.length === 1, "Two CEO chat turns, the stopped setup run and one Riley review run; Morgan may continue only when a CEO turn's comment on the case task wakes Morgan, once per turn. No other run is admitted."); c.add("subsequent-real-handoff-carries-current-facts", !!run && run.status === "succeeded" && run.runtimeMode === "native" && context?.id === original.id && context.taskId === original.workIssueId - && corrected(projectedFields) && projectedFields?.supportEscalation === o.state.collaboratorValue + && corrected(projectedFields, (original.fields as Record).browser) && projectedFields?.supportEscalation === o.state.collaboratorValue && projectedFields?.customer === (original.fields as Record).customer && o.caseRecord.handoffVersion === original.handoffVersion + 1, "A subsequent genuinely admitted reviewer on the same primary receives the saved correction plus collaboration change in its actual case context, not a fixture-synthesized context."); return c.result(); @@ -117,7 +105,7 @@ export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.up settings.get(ctx)!.gate = gate; const kickoff = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, { idempotencyKey: `facts-${ctx.input.nonce}`, case: { create: { caseType: "customer", title: "Checkout reproduction case", projectId: p.id, - fields: { browser: "Safari; affected device not yet confirmed", customer: "Acorn Test Shop", severity: "normal", expected: "Pay should submit a single order" } } }, toAgentId: worker.id, phase: "investigation", + fields: { browser: SETUP_BROWSER, customer: "Acorn Test Shop", severity: "normal", expected: "Pay should submit a single order" } } }, toAgentId: worker.id, phase: "investigation", instructions: "Read the saved customer brief. The affected device is not yet confirmed: record the missing reproduction detail in a task comment, then call the connected Website Brief read_website_brief tool to wait for the remaining customer evidence. Keep this ordinary investigation active; do not complete it, invent the affected device, or alter case facts.", workflow_document_text: "Morgan investigates this ordinary primary. A corrected customer reproduction detail must remain in saved case facts alongside unrelated customer fields. Riley reviews the current saved case when a later actual review handoff is requested, records a task comment, and completes the review task without closing the case.", initialDocuments: [{ key: "brief", title: "Customer reproduction brief", body: "Acorn Test Shop: Safari checkout Pay sometimes does not submit. The affected device has not yet been confirmed. Expected result is one submitted order." }], @@ -137,7 +125,7 @@ export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.up const row = state.case as CaseRecord; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title); const task = await ctx.input.api.get(`/api/issues/${row.workIssueId}`); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier}`, task.title); -}, { oracleVersion: "paperclip.chat-catalog.integrator.case-facts.v7", expectedRunCount: 6, minimumRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, +}, { oracleVersion: "paperclip.chat-catalog.integrator.case-facts.v8", expectedRunCount: 6, minimumRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 }, maximumTerminalStatusCounts: { succeeded: 5 } }, sourceFiles: ["chat-catalog-domains/integrator-case-facts.ts", "chat-catalog-domains/run-roles.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], cleanup, async act(ctx, state) { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-dependency.ts b/tests/runner-e2e/chat-catalog-domains/runtime-dependency.ts index 0f1c2fd50e4..82f1bd88481 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-dependency.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-dependency.ts @@ -70,7 +70,7 @@ export const runtimeDependencyCases = [runtimeDefinition("task-coordination.bloc description: `Implement the approved Design as a small website plan. Once eligible, read the actual Design task and save a task document with the exact reference ${marker}, accessible labels and a clear primary action, then finish normally. Do not deploy, create other tasks or invent approval.` }); return { worker, task, design, marker }; }, gradeDesignDependency, { - expectedHumanMessageCount: 2, expectedRunCount: 4, minimumRunCount: 3, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.block-on-design.v8", + expectedHumanMessageCount: 2, expectedRunCount: 4, minimumRunCount: 3, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.block-on-design.v9", sourceFiles: ["chat-catalog-domains/runtime-dependency.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/runtime-support.ts", "run-observations.ts", "chat-scenarios/task-coordination.yaml"], async act(ctx, state) { const workerId = (state.worker as Entity | undefined)?.id ?? "", websiteTask = state.task as Entity, gate = state.design as Entity; @@ -86,6 +86,10 @@ export const runtimeDependencyCases = [runtimeDefinition("task-coordination.bloc const source = ctx.input.page.url(), task = state.task as Entity, design = state.design as Entity, websiteRoot = `/api/issues/${task.id}`; state.waitingTask = await ctx.input.api.get(websiteRoot); state.designUnfinishedAtWait = (await ctx.input.api.get(`/api/issues/${design.id}`)).status !== "done"; + // The scripted human checks the blocker in the task UI only after it is saved. Without the saved edge the UI + // step can only time out on a missing row (gha-43088288923), which reads as a harness defect; stop with the cause. + if (!(state.waitingTask as Issue).blockedBy?.some(row => row.id === design.id)) + throw new Error("The CEO did not save Design as a blocker on Website before approval; a reply or comment alone does not hold the work"); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); // A backlog task shows its blockers in the properties panel; the thread's blocker banner appears only once it is Blocked. const blockedBy = ctx.input.page.locator('[data-property-row]:not([inert] *):not([aria-hidden="true"] *)') diff --git a/tests/runner-e2e/chat-catalog-run-policy.test.ts b/tests/runner-e2e/chat-catalog-run-policy.test.ts index b6f55f6cc2f..eb6931760cc 100644 --- a/tests/runner-e2e/chat-catalog-run-policy.test.ts +++ b/tests/runner-e2e/chat-catalog-run-policy.test.ts @@ -19,6 +19,16 @@ it("admits declared separate active work while rejecting active CEO, wrong scope expect(gradeChatCatalogRuns({ ...scope, implementation, runs: [succeeded, bad] }).passed).toBe(false); expect(gradeChatCatalogRuns({ ...scope, implementation, runs: [succeeded, worker, { ...worker, id: "extra" }] }).passed).toBe(false); }); +it("admits a lead run outside the source only as a declared case handback", () => { + const handback: ChatRun = { id: "handback", companyId: "company", agentId: "ceo", status: "running", runtimeMode: "native", contextSnapshot: { issueId: "task", wakeReason: "case_handoff" } }; + const done = { ...worker, status: "succeeded" }; + const declared = { expectedRunCount: 3, runPolicy: { kind: "source-settled-with-active-work" as const, maximumActiveWorkerRuns: 1, maximumLeadCaseHandbackRuns: 1 } }; + expect(gradeChatCatalogRuns({ ...scope, implementation: declared, runs: [succeeded, done, handback] }).passed).toBe(true); + // Undeclared (default 0), a lead run woken another way, and a second handback all stay rejected. + expect(gradeChatCatalogRuns({ ...scope, implementation: { ...declared, runPolicy: { kind: "source-settled-with-active-work" as const, maximumActiveWorkerRuns: 1 } }, runs: [succeeded, done, handback] }).passed).toBe(false); + expect(gradeChatCatalogRuns({ ...scope, implementation: declared, runs: [succeeded, done, { ...handback, contextSnapshot: { issueId: "task", wakeReason: "issue_commented" } }] }).passed).toBe(false); + expect(gradeChatCatalogRuns({ ...scope, implementation: { ...declared, expectedRunCount: 4 }, runs: [succeeded, done, handback, { ...handback, id: "again" }] }).passed).toBe(false); +}); function reassignment() { const receipt = { runId: "worker-run", taskId: "task", workerId: "worker", sourceRunId: "final-source" }; diff --git a/tests/runner-e2e/chat-catalog-run-policy.ts b/tests/runner-e2e/chat-catalog-run-policy.ts index 97684647397..102c4699128 100644 --- a/tests/runner-e2e/chat-catalog-run-policy.ts +++ b/tests/runner-e2e/chat-catalog-run-policy.ts @@ -68,7 +68,8 @@ export function gradeChatCatalogRuns(input: { detail: "The independently observed ledger matches every declared successful/cancelled/failed terminal outcome count (or its declared role-bound range)." }); } else if (c.runPolicy?.kind === "source-settled-with-active-work") { const sourceRuns = runs.filter(run => taskId(run) === source?.id); - const workers = runs.filter(run => taskId(run) !== source?.id); + const handbacks = runs.filter(run => taskId(run) !== source?.id && run.agentId === input.leadId && run.contextSnapshot?.wakeReason === "case_handoff"); + const workers = runs.filter(run => taskId(run) !== source?.id && !handbacks.includes(run)); const cancelled = runs.filter(run => run.status === "cancelled"); const declared = permitsReassignment(c); let expected: ReturnType>; @@ -91,8 +92,9 @@ export function gradeChatCatalogRuns(input: { checks.push({ id: "catalog-settled-source-separate-work", passed: !!source && source.companyId === input.companyId && source.conversationState === "waiting" && sourceRuns.length > 0 && sourceRuns.every(run => run.agentId === input.leadId && run.status === "succeeded") && workers.every(run => typeof taskId(run) === "string" && run.agentId !== input.leadId) - && workers.filter(run => ["queued", "running"].includes(run.status)).length <= c.runPolicy.maximumActiveWorkerRuns, - detail: "The CEO source is settled; unfinished worker runs are separately task-bound and bounded. The case oracle must additionally bind their exact task and assignee." }); + && workers.filter(run => ["queued", "running"].includes(run.status)).length <= c.runPolicy.maximumActiveWorkerRuns + && handbacks.length <= (c.runPolicy.maximumLeadCaseHandbackRuns ?? 0) && handbacks.every(run => typeof taskId(run) === "string"), + detail: "The CEO source is settled; unfinished worker runs are separately task-bound and bounded, and any CEO run outside the source is a declared case handback. The case oracle must additionally bind their exact task and assignee." }); } return { passed: checks.every(check => check.passed), checks }; }