Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-10 12:37:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): keep the existing post-stop grader tests and cover the held-cleanup interval

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-07 23:44:25 -06:00
1 parent cf99e8a3e5
commit be6437c648
1 file changed
+33 -39
Download Patch File Download Diff File Expand all files Collapse all files
tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts
+33 -39
View File
@@ -1,43 +1,37 @@
import { expect, it } from "vitest";
import { gradePostStop, sameSourcePostStopChronology } from "./runtime-post-stop.js";
const query = "Now make the website a single page instead.";
const ids = { company: "company", agent: "lead", human: "user", issue: "11111111-1111-1111-1111-111111111111", old: "22222222-2222-2222-2222-222222222222", next: "33333333-3333-3333-3333-333333333333", comment: "comment-2" };
const intentId = "native-cancellation:44444444-4444-4444-4444-444444444444";
function input(times: { sent: string; ack: string; released: string; started: string; ended?: string }) {
const boundary = { runId: ids.old, issueId: ids.issue, intentId, intentRecordedAt: "2026-11-07T10:00:00.000Z" };
const old = { id: ids.old, nativeIssueId: ids.issue, companyId: ids.company, agentId: ids.agent, runtimeMode: "native", status: "cancelled", finishedAt: times.ended ?? times.ack,
resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId, issueId: ids.issue, companyId: ids.company, runId: ids.old, scope: "run", intentAuditId: "a1", acknowledgementAuditId: "a2",
recordedAt: boundary.intentRecordedAt, dispatchState: "acknowledged", dispatched: true, acknowledgedAt: times.ack } } };
const next = { id: ids.next, nativeIssueId: ids.issue, companyId: ids.company, agentId: ids.agent, runtimeMode: "native", status: "succeeded", startedAt: times.started };
const human = { id: ids.comment, body: query, issueId: ids.issue, companyId: ids.company, authorAgentId: null, authorUserId: ids.human, createdAt: times.sent };
return { boundary, old, next, human, query, companyId: ids.company, agentId: ids.agent, humanId: ids.human, prompt: `${ids.comment} ${query}`, wakeCommentIds: [ids.comment], heldReleasedAt: times.released } as unknown as Parameters<typeof sameSourcePostStopChronology>[0];
import { expect, it, vi } from "vitest";
import { gradePostStop, sameSourcePostStopChronology, releasePostStopGateAfterCleanup } from "./runtime-post-stop.js";
function chronology() {
const old = { id: "old", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "cancelled", finishedAt: "2026-10-26T12:00:02Z", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: "intent", intentAuditId: "audit-intent", acknowledgementAuditId: "audit-ack", issueId: "source", companyId: "company", runId: "old", scope: "run", recordedAt: "2026-10-26T12:00:00Z", acknowledgedAt: "2026-10-26T12:00:02Z", dispatchState: "acknowledged", dispatched: true } } };
return { old, next: { id: "new", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-10-26T12:00:03Z" }, boundary: { runId: "old", issueId: "source", intentId: "intent", intentRecordedAt: "2026-10-26T12:00:00Z" }, human: { id: "canonical", body: "Now make the website a single page instead.", issueId: "source", companyId: "company", authorAgentId: null, authorUserId: "person", createdAt: "2026-10-26T12:00:01Z" }, query: "Now make the website a single page instead.", companyId: "company", agentId: "ceo", humanId: "person", prompt: "canonical\nNow make the website a single page instead.", wakeCommentIds: ["canonical"], heldReleasedAt: "2026-10-26T12:00:02.500Z" } as unknown as Parameters<typeof sameSourcePostStopChronology>[0];
}
it("accepts a message saved while the old provider is still held in cleanup, even after the Stop ACK", () => {
// The product ACKs a Stop after a bounded cleanup grace while the provider may still be closing.
expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", sent: "2026-11-07T10:00:02.300Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(true);
expect(sameSourcePostStopChronology(input({ sent: "2026-11-07T10:00:01.000Z", ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(true);
});
it("rejects a message sent after cleanup was released and a successor started during cleanup", () => {
expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", sent: "2026-11-07T10:00:03.500Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(false);
expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", sent: "2026-11-07T10:00:02.300Z", started: "2026-11-07T10:00:02.600Z", released: "2026-11-07T10:00:03.000Z" }))).toBe(false);
expect(sameSourcePostStopChronology(input({ sent: "2026-11-07T09:59:59.000Z", ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(false);
});
function observation(state: Record<string, unknown>) {
return { reply: "", companyId: ids.company, agents: [], links: [], activity: [], leadId: ids.agent, humanQueries: [{}, {}], tasks: [], runs: [{ status: "cancelled", agentId: ids.agent }, { status: "succeeded", agentId: ids.agent }],
state: { stopStatus: 200, messageAcceptedDuringHeldCleanup: true, noSuccessorWhileHeld: true, gateReleasedAfterCleanup: true, chronologyBound: true, providerExited: true, initialHumanRetained: true,
singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "Done: [website.html](/PAP/issues/PAP-1#document-website)", ...state } };
function valid() {
return { companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: ["initial", "canonical"], reply: "", tasks: [], activity: [], runs: [{ id: "old", agentId: "ceo", status: "cancelled" }, { id: "new", agentId: "ceo", status: "succeeded" }], state: { stopStatus: 200, messageAcceptedDuringHeldCleanup: true, noSuccessorWhileHeld: true, gateReleasedAfterCleanup: true, chronologyBound: true, providerExited: true, initialHumanRetained: true, singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "The single-page website is built and saved here." } };
}
it("binds the real unchanged canonical human instruction between same-source Stop intent and the end of held cleanup", () => expect(sameSourcePostStopChronology(chronology())).toBe(true));
it("rejects wrong-source, early, late, duplicate, resumed or unacknowledged deliveries", () => {
for (const change of [(v: ReturnType<typeof chronology>) => { v.human.issueId = "other"; }, (v: ReturnType<typeof chronology>) => { v.old.nativeIssueId = "other"; }, (v: ReturnType<typeof chronology>) => { v.next.agentId = "other"; }, (v: ReturnType<typeof chronology>) => { v.human.createdAt = "2026-10-26T12:00:03Z" as never; }, (v: ReturnType<typeof chronology>) => { v.next.startedAt = "2026-10-26T12:00:02.400Z" as never; }, (v: ReturnType<typeof chronology>) => { v.next.startedAt = "2026-10-26T12:00:01Z" as never; }, (v: ReturnType<typeof chronology>) => { v.wakeCommentIds.push("initial"); }, (v: ReturnType<typeof chronology>) => { v.prompt = "a different instruction"; }, (v: ReturnType<typeof chronology>) => { v.old.status = "succeeded"; }, (v: ReturnType<typeof chronology>) => { (v.old.resultJson!.nativeCancellation as Record<string, unknown>).dispatchState = "pending"; }]) { const v = chronology(); change(v); expect(sameSourcePostStopChronology(v)).toBe(false); }
});
it("accepts real post-cleanup delivery and a natural substantive result paraphrase", () => {
expect(gradePostStop(valid()).passed).toBe(true); const o = valid(); o.state.canonicalReply = "Acorn Notes is ready, with its signup action and all the content together. The artifact is saved in this conversation."; expect(gradePostStop(o).passed).toBe(true);
});
it("rejects polite completion without actual same-source cleanup, delivery or rendered output", () => {
for (const key of ["messageAcceptedDuringHeldCleanup", "noSuccessorWhileHeld", "gateReleasedAfterCleanup", "chronologyBound", "providerExited", "initialHumanRetained", "singleSuccessor", "sameCurrentAssignment", "oldEvidenceRetained", "actualSimplePage", "artifactRunBound"] as const) { const o = valid(); o.state[key] = false; expect(gradePostStop(o).passed).toBe(false); }
const o = valid(); o.state.canonicalReply = ""; expect(gradePostStop(o).passed).toBe(false); expect(gradePostStop(undefined).passed).toBe(false);
});
it("grades the saved page, not the reply wording", () => {
const passed = (state: Record<string, unknown>) => gradePostStop(observation(state)).checks.every(check => check.passed);
expect(passed({})).toBe(true);
expect(passed({ canonicalReply: "Switched it over — one scrolling layout, no tabs anymore." })).toBe(true);
expect(passed({ actualSimplePage: false })).toBe(false);
expect(passed({ canonicalReply: "" })).toBe(false);
expect(passed({ messageAcceptedDuringHeldCleanup: false })).toBe(false);
expect(passed({ noSuccessorWhileHeld: false })).toBe(false);
it("releases the neutral command gate after real cleanup without waiting for successor success", async () => {
const release = vi.fn(async () => {}), old = chronology().old;
await expect(releasePostStopGateAfterCleanup(old, false, release)).rejects.toThrow("cleanup");
await expect(releasePostStopGateAfterCleanup({ ...old, status: "running" }, true, release)).rejects.toThrow("cleanup");
const pending = chronology().old; (pending.resultJson!.nativeCancellation as Record<string, unknown>).dispatchState = "pending";
await expect(releasePostStopGateAfterCleanup(pending, true, release)).rejects.toThrow("cleanup"); expect(release).not.toHaveBeenCalled();
// No successor outcome is an input: it may still be running and retrying the
// legitimate initial command. Releasing only its neutral result avoids a cycle.
await releasePostStopGateAfterCleanup(old, true, release); expect(release).toHaveBeenCalledOnce();
});
it("accepts a message saved after the bounded Stop ACK while the old provider is still held in cleanup", () => {
// The product ACKs Stop after a bounded cleanup grace; the provider may still be closing then.
const v = chronology(); v.human.createdAt = "2026-10-26T12:00:02.200Z" as never; expect(sameSourcePostStopChronology(v)).toBe(true);
v.human.createdAt = "2026-10-26T12:00:02.600Z" as never; expect(sameSourcePostStopChronology(v)).toBe(false);
});