diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts index d0b4063e016..c9bb771c05a 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.test.ts @@ -5,10 +5,10 @@ function valid() { const brief = { id: "brief", issueId: "primary", latestRevisionId: "v1", body: "Customer reproduction" }; const fields = { ...original.fields, browser: "Mobile Safari only", supportEscalation: "support-123" }; const base = { scenarioId: "cases.update-case-facts", origin: "https://fixture.test", companyId: "company", leadId: "ceo", - state: { case: original, brief, collaboratorValue: "support-123", reviewer: { id: "riley" } }, caseRecord: { ...original, fields }, brief, + state: { initialRunId: "initial", worker: { id: "morgan" }, case: original, brief, collaboratorValue: "support-123", reviewer: { id: "riley" } }, caseRecord: { ...original, fields }, brief, briefRevisions: [{ id: "v1", issueId: "primary", body: brief.body }], projects: [], tasks: [], goals: [], documents: [], revisions: [], comments: [], links: [], activity: [], humanQueries: [], reply: "The reproduction occurs exclusively on mobile Safari. Support escalation is support-123.", - runs: [{ id: "review", status: "succeeded", runtimeMode: "native", agentId: "riley", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: JSON.stringify({ fields }) } } }] }; + runs: [{ id: "initial", status: "cancelled", runtimeMode: "native", agentId: "morgan", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: "{}" } } }, { id: "review", status: "succeeded", runtimeMode: "native", agentId: "riley", contextSnapshot: { issueId: "primary", paperclipCase: { id: "case", taskId: "primary", data: JSON.stringify({ fields }) } } }] }; return { ...base, caseRecord: { ...base.caseRecord, handoffVersion: 2 }, state: { ...base.state, firstCorrection: structuredClone(base), collaborationCheckpoint: structuredClone(base) } }; } describe("current case facts and real subsequent context", () => { @@ -25,8 +25,8 @@ describe("current case facts and real subsequent context", () => { const stale = valid(); stale.state.collaborationCheckpoint.caseRecord.fields.supportEscalation = "stale"; expect(gradeIntegratorCaseFacts(stale).passed).toBe(false); }); it("rejects stale handoff context, chat-only projection and moved document ownership", () => { - const stale = valid(); stale.runs[0]!.contextSnapshot.paperclipCase.data = JSON.stringify({ fields: stale.state.case.fields }); expect(gradeIntegratorCaseFacts(stale).passed).toBe(false); - const chat = valid(); chat.runs[0]!.contextSnapshot.issueId = "chat"; expect(gradeIntegratorCaseFacts(chat).passed).toBe(false); + const stale = valid(); stale.runs[1]!.contextSnapshot.paperclipCase.data = JSON.stringify({ fields: stale.state.case.fields }); expect(gradeIntegratorCaseFacts(stale).passed).toBe(false); + const chat = valid(); chat.runs[1]!.contextSnapshot.issueId = "chat"; expect(gradeIntegratorCaseFacts(chat).passed).toBe(false); const moved = valid(); moved.brief.issueId = "case"; expect(gradeIntegratorCaseFacts(moved).passed).toBe(false); const missing = valid(); missing.briefRevisions = []; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts index ad441b6f71a..762f9cca790 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-facts.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import type { cases } from "../../../packages/db/src/schema/cases.js"; import type { HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js"; import type { Issue } from "../../../packages/shared/src/types/issue.js"; @@ -5,12 +6,13 @@ import type { IssueDocument, DocumentRevision } from "../../../packages/shared/s import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; import { checks, observation, project, type Entity, type InternalObservation } from "./internal-support.js"; +import { startRunBriefGate } from "./internal-run-brief.js"; import { fixtureWorker } from "./internal-definition.js"; import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; type CaseRecord = typeof cases.$inferSelect; interface FactsObservation extends InternalObservation { caseRecord: CaseRecord; brief: IssueDocument; briefRevisions: DocumentRevision[] } -const settings = new WeakMap(); -async function cleanup(ctx: ChatCatalogContext) { const prior = settings.get(ctx); if (prior === undefined) return; await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: prior }); settings.delete(ctx); } +const settings = new WeakMap> }>(); +async function cleanup(ctx: ChatCatalogContext) { const prior = settings.get(ctx); if (prior === undefined) return; try { await prior.gate?.close(); } finally { await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: prior.experimental }); settings.delete(ctx); } } async function observeFacts(ctx: ChatCatalogContext, state: Record): Promise { const base = await observeInternal(ctx, state); const row = state.case as CaseRecord; return { ...base, caseRecord: await ctx.input.api.get(`/api/cases/${row.id}`), brief: await ctx.input.api.get(`/api/issues/${row.workIssueId}/documents/brief`), briefRevisions: await ctx.input.api.get(`/api/issues/${row.workIssueId}/documents/brief/revisions`) }; @@ -31,6 +33,7 @@ function retained(o: FactsObservation, concurrent: boolean) { } export function gradeIntegratorCaseFacts(value: unknown) { const c = checks(); const o = factsObservation(value); c.add("saved-case-and-task-document-evidence", !!o, "Requires real case facts, ordinary-task document revisions, phase checkpoints and native handoff context."); if (!o) return c.result(); + c.add("genuine-paused-active-primary-prerequisite", o.runs.some(run => run.id === o.state.initialRunId && run.agentId === (o.state.worker as Entity).id && run.contextSnapshot?.issueId === (o.state.case as CaseRecord).workIssueId && run.status === "cancelled" && run.runtimeMode === "native"), "The initial ordinary active task is released by an actual operator stop, allowing a later normal board handoff without silently overriding a blocked-task gate."); const first = factsObservation(o.state.firstCorrection); const fresh = factsObservation(o.state.collaborationCheckpoint); c.add("initial-correction-preserves-unrelated-fields", !!first && retained(first, false), "The unchanged first query must already correct the saved facts without discarding unrelated fields; a later turn cannot repair an initially wrong mutation."); c.add("fresh-collaborative-change-not-stale-replacement", !!fresh && retained(fresh, true) && retained(o, true) @@ -51,33 +54,39 @@ export function gradeIntegratorCaseFacts(value: unknown) { return c.result(); } export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.update-case-facts", async ctx => { - const prior = await ctx.input.api.get<{ enableCases: boolean }>("/api/instance/settings/experimental"); settings.set(ctx, prior.enableCases); + const prior = await ctx.input.api.get<{ enableCases: boolean }>("/api/instance/settings/experimental"); settings.set(ctx, { experimental: prior.enableCases }); try { await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: true }); const p = await project(ctx, "Launch"); const worker = await fixtureWorker(ctx); const reviewer = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${reviewer.id}`, { name: "Riley" }); for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[worker.id, reviewer.id].map(scopeId => ({ scopeType: "agent", scopeId }))]) await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + const gate = await startRunBriefGate(ctx, worker.id); settings.get(ctx)!.gate = gate; const kickoff = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, { idempotencyKey: `facts-${ctx.input.nonce}`, case: { create: { caseType: "customer", title: "Checkout reproduction case", projectId: p.id, fields: { browser: "Safari; affected device not yet confirmed", customer: "Acorn Test Shop", severity: "normal", expected: "Pay should submit a single order" } } }, toAgentId: worker.id, phase: "investigation", - instructions: "Read the saved customer brief. The affected device is not yet confirmed: record the missing reproduction detail in a task comment and mark this ordinary investigation blocked awaiting the customer's correction. Do not invent the affected device or alter case facts.", + instructions: "Read the saved customer brief. The affected device is not yet confirmed: record the missing reproduction detail in a task comment, then call the connected Website Brief read_website_brief tool to wait for the remaining customer evidence. Keep this ordinary investigation active; do not complete it, invent the affected device, or alter case facts.", workflow_document_text: "Morgan investigates this ordinary primary. A corrected customer reproduction detail must remain in saved case facts alongside unrelated customer fields. Riley reviews the current saved case when a later actual review handoff is requested, records a task comment, and completes the review task without closing the case.", initialDocuments: [{ key: "brief", title: "Customer reproduction brief", body: "Acorn Test Shop: Safari checkout Pay sometimes does not submit. The affected device has not yet been confirmed. Expected result is one submitted order." }], }); - await pollUntil({ label: "actual active case awaiting missing reproduction detail", deadlineAt: Date.now() + 180_000, intervalMs: 1000, - load: async () => ({ task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns() }), accept: v => v.runs.length === 1 && v.runs[0]?.status === "succeeded" && v.task.status === "blocked" }); - return { project: p, worker: { id: worker.id }, reviewer: { id: reviewer.id }, case: await ctx.input.api.get(`/api/cases/${kickoff.caseId}`), + const initial = await pollUntil({ label: "actual active case waiting for missing reproduction evidence", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns() }), accept: v => v.runs.length === 1 && v.runs[0]?.status === "running" && v.task.status === "in_progress" && gate.calls.some(call => call.authorized) }); + await ctx.input.api.post(`/api/heartbeat-runs/${initial.runs[0]!.id}/cancel`, { cancellationRequestId: randomUUID() }); + await pollUntil({ label: "actual operator stop releases primary without changing task status", deadlineAt: Date.now() + 90_000, intervalMs: 1000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns() }), + accept: v => v.runs.length === 1 && v.runs[0]?.status === "cancelled" && v.task.status === "in_progress" && !v.task.executionRunId && !v.task.checkoutRunId }); + gate.release(); + return { initialRunId: initial.runs[0]!.id, project: p, worker: { id: worker.id }, reviewer: { id: reviewer.id }, case: await ctx.input.api.get(`/api/cases/${kickoff.caseId}`), brief: await ctx.input.api.get(`/api/issues/${kickoff.taskId}/documents/brief`), collaboratorValue: `support-${ctx.input.nonce}` }; } catch (error) { await cleanup(ctx); throw error; } }, observeFacts, gradeIntegratorCaseFacts, async (ctx, state) => { const row = state.case as CaseRecord; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title); const task = await ctx.input.api.get(`/api/issues/${row.workIssueId}`); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier}`, task.title); -}, { expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, - sourceFiles: ["chat-catalog-domains/integrator-case-facts.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], cleanup, +}, { expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } }, + sourceFiles: ["chat-catalog-domains/integrator-case-facts.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], cleanup, async act(ctx, state) { const settle = (count: number) => pollUntil({ label: `saved case correction phase ${count}`, deadlineAt: Date.now() + 180_000, intervalMs: 1000, - load: () => observeFacts(ctx, state), accept: o => o.runs.length === count && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + load: () => observeFacts(ctx, state), accept: o => o.runs.length === count && o.runs.every(run => run.status === (run.id === state.initialRunId ? "cancelled" : "succeeded")) && ctx.issue()?.conversationState === "waiting" }); const first = await settle(2); state.firstCorrection = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-case-initial-correction.json", state.firstCorrection); if (!retained(first, false)) throw new Error("Initial case correction was incorrect; a follow-up cannot repair this phase"); const row = first.caseRecord;