From be6437c648e37c6150bbfc11f43c7bdf84a675fd Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 23:44:25 -0600 Subject: [PATCH] test(evals): keep the existing post-stop grader tests and cover the held-cleanup interval Co-Authored-By: Claude Opus 5.5 --- .../runtime-post-stop.test.ts | 72 +++++++++---------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts index bef431e3891..ce46729279d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts @@ -1,43 +1,37 @@ -import { expect, it } from "vitest"; -import { gradePostStop, sameSourcePostStopChronology } from "./runtime-post-stop.js"; - -const query = "Now make the website a single page instead."; -const ids = { company: "company", agent: "lead", human: "user", issue: "11111111-1111-1111-1111-111111111111", old: "22222222-2222-2222-2222-222222222222", next: "33333333-3333-3333-3333-333333333333", comment: "comment-2" }; -const intentId = "native-cancellation:44444444-4444-4444-4444-444444444444"; -function input(times: { sent: string; ack: string; released: string; started: string; ended?: string }) { - const boundary = { runId: ids.old, issueId: ids.issue, intentId, intentRecordedAt: "2026-11-07T10:00:00.000Z" }; - const old = { id: ids.old, nativeIssueId: ids.issue, companyId: ids.company, agentId: ids.agent, runtimeMode: "native", status: "cancelled", finishedAt: times.ended ?? times.ack, - resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId, issueId: ids.issue, companyId: ids.company, runId: ids.old, scope: "run", intentAuditId: "a1", acknowledgementAuditId: "a2", - recordedAt: boundary.intentRecordedAt, dispatchState: "acknowledged", dispatched: true, acknowledgedAt: times.ack } } }; - const next = { id: ids.next, nativeIssueId: ids.issue, companyId: ids.company, agentId: ids.agent, runtimeMode: "native", status: "succeeded", startedAt: times.started }; - const human = { id: ids.comment, body: query, issueId: ids.issue, companyId: ids.company, authorAgentId: null, authorUserId: ids.human, createdAt: times.sent }; - return { boundary, old, next, human, query, companyId: ids.company, agentId: ids.agent, humanId: ids.human, prompt: `${ids.comment} ${query}`, wakeCommentIds: [ids.comment], heldReleasedAt: times.released } as unknown as Parameters[0]; +import { expect, it, vi } from "vitest"; +import { gradePostStop, sameSourcePostStopChronology, releasePostStopGateAfterCleanup } from "./runtime-post-stop.js"; +function chronology() { + const old = { id: "old", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "cancelled", finishedAt: "2026-10-26T12:00:02Z", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: "intent", intentAuditId: "audit-intent", acknowledgementAuditId: "audit-ack", issueId: "source", companyId: "company", runId: "old", scope: "run", recordedAt: "2026-10-26T12:00:00Z", acknowledgedAt: "2026-10-26T12:00:02Z", dispatchState: "acknowledged", dispatched: true } } }; + return { old, next: { id: "new", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-10-26T12:00:03Z" }, boundary: { runId: "old", issueId: "source", intentId: "intent", intentRecordedAt: "2026-10-26T12:00:00Z" }, human: { id: "canonical", body: "Now make the website a single page instead.", issueId: "source", companyId: "company", authorAgentId: null, authorUserId: "person", createdAt: "2026-10-26T12:00:01Z" }, query: "Now make the website a single page instead.", companyId: "company", agentId: "ceo", humanId: "person", prompt: "canonical\nNow make the website a single page instead.", wakeCommentIds: ["canonical"], heldReleasedAt: "2026-10-26T12:00:02.500Z" } as unknown as Parameters[0]; } - -it("accepts a message saved while the old provider is still held in cleanup, even after the Stop ACK", () => { - // The product ACKs a Stop after a bounded cleanup grace while the provider may still be closing. - expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", sent: "2026-11-07T10:00:02.300Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(true); - expect(sameSourcePostStopChronology(input({ sent: "2026-11-07T10:00:01.000Z", ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(true); -}); - -it("rejects a message sent after cleanup was released and a successor started during cleanup", () => { - expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", sent: "2026-11-07T10:00:03.500Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(false); - expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", sent: "2026-11-07T10:00:02.300Z", started: "2026-11-07T10:00:02.600Z", released: "2026-11-07T10:00:03.000Z" }))).toBe(false); - expect(sameSourcePostStopChronology(input({ sent: "2026-11-07T09:59:59.000Z", ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(false); -}); - -function observation(state: Record) { - return { reply: "", companyId: ids.company, agents: [], links: [], activity: [], leadId: ids.agent, humanQueries: [{}, {}], tasks: [], runs: [{ status: "cancelled", agentId: ids.agent }, { status: "succeeded", agentId: ids.agent }], - state: { stopStatus: 200, messageAcceptedDuringHeldCleanup: true, noSuccessorWhileHeld: true, gateReleasedAfterCleanup: true, chronologyBound: true, providerExited: true, initialHumanRetained: true, - singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "Done: [website.html](/PAP/issues/PAP-1#document-website)", ...state } }; +function valid() { + return { companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: ["initial", "canonical"], reply: "", tasks: [], activity: [], runs: [{ id: "old", agentId: "ceo", status: "cancelled" }, { id: "new", agentId: "ceo", status: "succeeded" }], state: { stopStatus: 200, messageAcceptedDuringHeldCleanup: true, noSuccessorWhileHeld: true, gateReleasedAfterCleanup: true, chronologyBound: true, providerExited: true, initialHumanRetained: true, singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "The single-page website is built and saved here." } }; } +it("binds the real unchanged canonical human instruction between same-source Stop intent and the end of held cleanup", () => expect(sameSourcePostStopChronology(chronology())).toBe(true)); +it("rejects wrong-source, early, late, duplicate, resumed or unacknowledged deliveries", () => { + for (const change of [(v: ReturnType) => { v.human.issueId = "other"; }, (v: ReturnType) => { v.old.nativeIssueId = "other"; }, (v: ReturnType) => { v.next.agentId = "other"; }, (v: ReturnType) => { v.human.createdAt = "2026-10-26T12:00:03Z" as never; }, (v: ReturnType) => { v.next.startedAt = "2026-10-26T12:00:02.400Z" as never; }, (v: ReturnType) => { v.next.startedAt = "2026-10-26T12:00:01Z" as never; }, (v: ReturnType) => { v.wakeCommentIds.push("initial"); }, (v: ReturnType) => { v.prompt = "a different instruction"; }, (v: ReturnType) => { v.old.status = "succeeded"; }, (v: ReturnType) => { (v.old.resultJson!.nativeCancellation as Record).dispatchState = "pending"; }]) { const v = chronology(); change(v); expect(sameSourcePostStopChronology(v)).toBe(false); } +}); +it("accepts real post-cleanup delivery and a natural substantive result paraphrase", () => { + expect(gradePostStop(valid()).passed).toBe(true); const o = valid(); o.state.canonicalReply = "Acorn Notes is ready, with its signup action and all the content together. The artifact is saved in this conversation."; expect(gradePostStop(o).passed).toBe(true); +}); +it("rejects polite completion without actual same-source cleanup, delivery or rendered output", () => { + for (const key of ["messageAcceptedDuringHeldCleanup", "noSuccessorWhileHeld", "gateReleasedAfterCleanup", "chronologyBound", "providerExited", "initialHumanRetained", "singleSuccessor", "sameCurrentAssignment", "oldEvidenceRetained", "actualSimplePage", "artifactRunBound"] as const) { const o = valid(); o.state[key] = false; expect(gradePostStop(o).passed).toBe(false); } + const o = valid(); o.state.canonicalReply = ""; expect(gradePostStop(o).passed).toBe(false); expect(gradePostStop(undefined).passed).toBe(false); +}); -it("grades the saved page, not the reply wording", () => { - const passed = (state: Record) => gradePostStop(observation(state)).checks.every(check => check.passed); - expect(passed({})).toBe(true); - expect(passed({ canonicalReply: "Switched it over — one scrolling layout, no tabs anymore." })).toBe(true); - expect(passed({ actualSimplePage: false })).toBe(false); - expect(passed({ canonicalReply: "" })).toBe(false); - expect(passed({ messageAcceptedDuringHeldCleanup: false })).toBe(false); - expect(passed({ noSuccessorWhileHeld: false })).toBe(false); +it("releases the neutral command gate after real cleanup without waiting for successor success", async () => { + const release = vi.fn(async () => {}), old = chronology().old; + await expect(releasePostStopGateAfterCleanup(old, false, release)).rejects.toThrow("cleanup"); + await expect(releasePostStopGateAfterCleanup({ ...old, status: "running" }, true, release)).rejects.toThrow("cleanup"); + const pending = chronology().old; (pending.resultJson!.nativeCancellation as Record).dispatchState = "pending"; + await expect(releasePostStopGateAfterCleanup(pending, true, release)).rejects.toThrow("cleanup"); expect(release).not.toHaveBeenCalled(); + // No successor outcome is an input: it may still be running and retrying the + // legitimate initial command. Releasing only its neutral result avoids a cycle. + await releasePostStopGateAfterCleanup(old, true, release); expect(release).toHaveBeenCalledOnce(); +}); + +it("accepts a message saved after the bounded Stop ACK while the old provider is still held in cleanup", () => { + // The product ACKs Stop after a bounded cleanup grace; the provider may still be closing then. + const v = chronology(); v.human.createdAt = "2026-10-26T12:00:02.200Z" as never; expect(sameSourcePostStopChronology(v)).toBe(true); + v.human.createdAt = "2026-10-26T12:00:02.600Z" as never; expect(sameSourcePostStopChronology(v)).toBe(false); });