From caff11daf6b128ea6c4bf0117e63dfd1857186ad Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 23:50:53 -0500 Subject: [PATCH] test(evals): release post-stop gate only after cleanup proof --- .../runtime-post-stop.test.ts | 19 +++++++++++++++---- .../chat-catalog-domains/runtime-post-stop.ts | 16 +++++++++++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts index 2fe371e7feb..c7a557717de 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts @@ -1,11 +1,11 @@ -import { expect, it } from "vitest"; -import { gradePostStop, sameSourcePostStopChronology } from "./runtime-post-stop.js"; +import { expect, it, vi } from "vitest"; +import { gradePostStop, sameSourcePostStopChronology, releasePostStopGateAfterCleanup } from "./runtime-post-stop.js"; function chronology() { const old = { id: "old", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "cancelled", finishedAt: "2026-10-26T12:00:02Z", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: "intent", intentAuditId: "audit-intent", acknowledgementAuditId: "audit-ack", issueId: "source", companyId: "company", runId: "old", scope: "run", recordedAt: "2026-10-26T12:00:00Z", acknowledgedAt: "2026-10-26T12:00:02Z", dispatchState: "acknowledged", dispatched: true } } }; return { old, next: { id: "new", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-10-26T12:00:03Z" }, boundary: { runId: "old", issueId: "source", intentId: "intent", intentRecordedAt: "2026-10-26T12:00:00Z" }, human: { id: "canonical", body: "Now make the website a single page instead.", issueId: "source", companyId: "company", authorAgentId: null, authorUserId: "person", createdAt: "2026-10-26T12:00:01Z" }, query: "Now make the website a single page instead.", companyId: "company", agentId: "ceo", humanId: "person", prompt: "canonical\nNow make the website a single page instead.", wakeCommentIds: ["canonical"] } as unknown as Parameters[0]; } function valid() { - return { companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: ["initial", "canonical"], reply: "", tasks: [], activity: [], runs: [{ id: "old", agentId: "ceo", status: "cancelled" }, { id: "new", agentId: "ceo", status: "succeeded" }], state: { stopStatus: 200, chronologyBound: true, providerExited: true, initialHumanRetained: true, singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "The single-page website is built and saved here." } }; + return { companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: ["initial", "canonical"], reply: "", tasks: [], activity: [], runs: [{ id: "old", agentId: "ceo", status: "cancelled" }, { id: "new", agentId: "ceo", status: "succeeded" }], state: { stopStatus: 200, gateReleasedAfterCleanup: true, chronologyBound: true, providerExited: true, initialHumanRetained: true, singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "The single-page website is built and saved here." } }; } it("binds the real unchanged canonical human instruction between same-source Stop intent and cleanup ACK", () => expect(sameSourcePostStopChronology(chronology())).toBe(true)); it("rejects wrong-source, early, late, duplicate, resumed or unacknowledged deliveries", () => { @@ -15,6 +15,17 @@ it("accepts real post-cleanup delivery and a natural substantive result paraphra expect(gradePostStop(valid()).passed).toBe(true); const o = valid(); o.state.canonicalReply = "Acorn Notes is ready, with its signup action and all the content together. The artifact is saved in this conversation."; expect(gradePostStop(o).passed).toBe(true); }); it("rejects polite completion without actual same-source cleanup, delivery or rendered output", () => { - for (const key of ["chronologyBound", "providerExited", "initialHumanRetained", "singleSuccessor", "sameCurrentAssignment", "oldEvidenceRetained", "actualSimplePage", "artifactRunBound"] as const) { const o = valid(); o.state[key] = false; expect(gradePostStop(o).passed).toBe(false); } + for (const key of ["gateReleasedAfterCleanup", "chronologyBound", "providerExited", "initialHumanRetained", "singleSuccessor", "sameCurrentAssignment", "oldEvidenceRetained", "actualSimplePage", "artifactRunBound"] as const) { const o = valid(); o.state[key] = false; expect(gradePostStop(o).passed).toBe(false); } const o = valid(); o.state.canonicalReply = "The website is still queued and not built."; expect(gradePostStop(o).passed).toBe(false); expect(gradePostStop(undefined).passed).toBe(false); }); + +it("releases the neutral command gate after real cleanup without waiting for successor success", async () => { + const release = vi.fn(async () => {}), old = chronology().old; + await expect(releasePostStopGateAfterCleanup(old, false, release)).rejects.toThrow("cleanup"); + await expect(releasePostStopGateAfterCleanup({ ...old, status: "running" }, true, release)).rejects.toThrow("cleanup"); + const pending = chronology().old; (pending.resultJson!.nativeCancellation as Record).dispatchState = "pending"; + await expect(releasePostStopGateAfterCleanup(pending, true, release)).rejects.toThrow("cleanup"); expect(release).not.toHaveBeenCalled(); + // No successor outcome is an input: it may still be running and retrying the + // legitimate initial command. Releasing only its neutral result avoids a cycle. + await releasePostStopGateAfterCleanup(old, true, release); expect(release).toHaveBeenCalledOnce(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts index 901181f8008..4cf9d0c2614 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts @@ -26,11 +26,17 @@ export function sameSourcePostStopChronology(input: { boundary: NativeStopBounda && before <= sent && sent < ack && next.id !== old.id && next.nativeIssueId === b.issueId && next.companyId === input.companyId && next.agentId === input.agentId && next.runtimeMode === "native" && next.status === "succeeded" && started >= ack && started >= ended && input.wakeCommentIds.length === 1 && input.wakeCommentIds[0] === human.id && input.prompt.includes(human.id) && input.prompt.includes(input.query); } +export async function releasePostStopGateAfterCleanup(old: Pick, providerExited: boolean, release: () => Promise) { + const intent = old.resultJson?.nativeCancellation as StopIntent | undefined; + if (old.status !== "cancelled" || intent?.dispatchState !== "acknowledged" || intent.dispatched !== true || !intent.acknowledgedAt || !intent.acknowledgementAuditId || !providerExited) + throw new Error("The old native cleanup must be independently proven before releasing the original neutral tool gate"); + await release(); +} export function gradePostStop(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("actual-same-source-stop-continuation-evidence", !!o, "A real setup human request, actual native provider cleanup, canonical human instruction and normal continuation must be independently observed."); if (!o) return c.result(); const s = o.state; - c.add("human-stop-cleanup-and-exact-message-chronology", s.stopStatus === 200 && s.chronologyBound === true && s.providerExited === true && s.initialHumanRetained === true, + c.add("human-stop-cleanup-and-exact-message-chronology", s.stopStatus === 200 && s.gateReleasedAfterCleanup === true && s.chronologyBound === true && s.providerExited === true && s.initialHumanRetained === true, "The same actual source run has a real Stop intent and ACK; the canonical human message persisted in between, unchanged, and only its eligible successor consumes it after cleanup."); c.add("one-eligible-current-owner-with-no-revival-or-duplicate", o.runs.length === 2 && o.runs.filter(row => row.status === "cancelled").length === 1 && o.runs.filter(row => row.status === "succeeded").length === 1 && o.runs.every(row => row.agentId === o.leadId) && s.singleSuccessor === true && s.sameCurrentAssignment === true && s.oldEvidenceRetained === true && o.humanQueries.length === 2 && o.tasks.length === 0, @@ -81,6 +87,11 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa return { runId: run.id, issueId: ctx.issue()!.id, intentId: intent.intentId, intentRecordedAt: intent.recordedAt }; }, async act(ctx, state) { + const cleaned = await pollUntil({ label: "Actual old source cancellation ACK and physical provider exit before neutral gate release", deadlineAt: Date.now() + 90000, + load: async () => ({ old: await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), providerExited: await ownedCleanupProviderHasExited(ctx) }), + accept: v => v.old.status === "cancelled" && (v.old.resultJson?.nativeCancellation as StopIntent | undefined)?.dispatchState === "acknowledged" && v.providerExited }); + await releasePostStopGateAfterCleanup(cleaned.old, cleaned.providerExited, () => writeFile((state.gate as { gate: string }).gate, "The pending fixture tool has been released.", "utf8")); + state.gateReleasedAfterCleanup = true; const settled = await pollUntil({ label: "One normal same-source continuation after actual stopped-provider cleanup", deadlineAt: Date.now() + 300000, load: async () => { await ctx.refreshIssue(); return { runs: await ctx.allRuns(), source: ctx.issue() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.find(row => row.id === state.workerRunId)?.status === "cancelled" && v.runs.find(row => row.id !== state.workerRunId)?.status === "succeeded", @@ -97,8 +108,7 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa const events = await collectRunEvents<{ seq?: number; eventType?: string; createdAt?: string; payload?: { schema?: string; runId?: string; nativeSessionId?: string } }>((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${old.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); state.oldEvidenceRetained = events.length > 0 && old.status === "cancelled"; state.providerExited = await ownedCleanupProviderHasExited(ctx); - if (old.errorCode === "native_session_cleanup_quarantined") state.chronologyBound = state.chronologyBound === true && events.some(event => event.eventType === "native.stopped_conversation_verified" && event.payload?.schema === "paperclip.stopped_native_conversation.v1" && event.payload.runId === old.id && event.payload.nativeSessionId === old.nativeSessionId && Date.parse(event.createdAt ?? "") <= Date.parse(String(next.startedAt))); - state.chronologyBound = state.chronologyBound === true && !!next.nativeSessionId && next.nativeSessionId !== old.nativeSessionId; + if (old.errorCode === "native_session_cleanup_quarantined") state.chronologyBound = state.chronologyBound === true && !!next.nativeSessionId && next.nativeSessionId !== old.nativeSessionId && events.some(event => event.eventType === "native.stopped_conversation_verified" && event.payload?.schema === "paperclip.stopped_native_conversation.v1" && event.payload.runId === old.id && event.payload.nativeSessionId === old.nativeSessionId && Date.parse(event.createdAt ?? "") <= Date.parse(String(next.startedAt))); state.canonicalReply = comments.filter(row => row.createdByRunId === next.id && row.authorAgentId === next.agentId).map(row => row.body).join("\n"); state.successorRunId = next.id; await releaseOwnedCleanupProvider(ctx); },