From 3a8822988af1669bc323660b32f31a7ad001ff1d Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 9 Nov 2026 03:54:45 -0600 Subject: [PATCH] test(evals): bind interrupt-with-direction's delivery to the successor's website-delivery revision, not a copied plan marker The successor saved website-delivery in its own words. Requiring the plan's setup marker inside it graded copying, not delivery. The page content is graded from the registered website.html. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../runtime-interruption.test.ts | 17 ++++++++++++++++- .../runtime-interruption.ts | 16 +++++++++++++--- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 4069681880d..ef7d693bdda 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5483,7 +5483,7 @@ "registered-artifact.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.interrupt-with-direction.v4", + "oracleVersion": "paperclip.chat-catalog.runtime.interrupt-with-direction.v5", "coverageGaps": [], "expectedRunCount": 3, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.test.ts index 6a2e7d8ca7d..de015ac67c6 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeInterruption } from "./runtime-interruption.js"; +import { deliveryRevisionBound, gradeInterruption } from "./runtime-interruption.js"; function valid() { return { scenarioId: "steering.interrupt-with-direction", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website", status: "done" }], activity: [], runs: [{ id: "old", status: "cancelled" }, { id: "source", status: "succeeded" }, { id: "new", status: "succeeded" }], @@ -29,3 +29,18 @@ it("rejects rewriting the accepted plan after the interruption", () => { const o it("rejects a friendly completion with no real simpler page", () => { const o = valid(); o.state.actualSimplePage = false; expect(gradeInterruption(o).passed).toBe(false); }); it("rejects lost original run history", () => { const o = valid(); o.state.oldEvidenceRetained = false; expect(gradeInterruption(o).passed).toBe(false); }); it("rejects missing actual interruption evidence", () => expect(gradeInterruption(undefined).passed).toBe(false)); +// Shaped like run 42592781763: the successor saved website-delivery in its own +// words, without the plan's setup marker. +function delivery() { + return { documents: [{ id: "plan-doc", key: "plan", issueId: "website", latestRevisionId: "plan-r1" }, { id: "brief-doc", key: "website-brief", issueId: "website", latestRevisionId: "brief-r1" }, { id: "delivery-doc", key: "website-delivery", issueId: "website", latestRevisionId: "delivery-r1" }], + revisions: [{ id: "plan-r1", documentId: "plan-doc", createdByRunId: null, createdByAgentId: null, body: "# Approved simpler website direction\nSimpler layout nonce-1" }, + { id: "brief-r1", documentId: "brief-doc", createdByRunId: "new", createdByAgentId: "morgan", body: "# Website brief" }, + { id: "delivery-r1", documentId: "delivery-doc", createdByRunId: "new", createdByAgentId: "morgan", body: "# Website delivery\n- Deliverable: website.html, single semantic page." }] }; +} +it("binds the successor's own current website-delivery revision without a copied setup marker", () => expect(deliveryRevisionBound(delivery(), "website", "new", "morgan")).toBe(true)); +it("accepts reworded delivery prose", () => { const o = delivery(); (o.revisions[2] as { body: string }).body = "Shipped the one-page Acorn Notes site."; expect(deliveryRevisionBound(o, "website", "new", "morgan")).toBe(true); }); +it("rejects a delivery document written by the stopped attempt", () => { const o = delivery(); o.revisions[2]!.createdByRunId = "old"; expect(deliveryRevisionBound(o, "website", "new", "morgan")).toBe(false); }); +it("rejects a delivery written by another agent", () => { const o = delivery(); o.revisions[2]!.createdByAgentId = "ceo"; expect(deliveryRevisionBound(o, "website", "new", "morgan")).toBe(false); }); +it("rejects a superseded successor revision", () => { const o = delivery(); o.documents[2]!.latestRevisionId = "later"; expect(deliveryRevisionBound(o, "website", "new", "morgan")).toBe(false); }); +it("rejects only some other successor document", () => { const o = delivery(); o.documents.pop(); expect(deliveryRevisionBound(o, "website", "new", "morgan")).toBe(false); }); +it("rejects a delivery document on a different task", () => { const o = delivery(); o.documents[2]!.issueId = "other"; expect(deliveryRevisionBound(o, "website", "new", "morgan")).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts index 40d48fa47f2..e97df83d4a2 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts @@ -18,6 +18,16 @@ import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime function issueApiPaths(task: Entity, suffix: string) { return new Set([task.id, (task as { identifier?: string | null }).identifier].filter((key): key is string => !!key).map(key => `/api/issues/${encodeURIComponent(key)}${suffix}`)); } +// The plan asks for a task document keyed website-delivery. The successor's +// own revision must be that document's current content on the original task; +// its prose is free-form, so no setup literal has to be copied into it. The +// delivered page itself is graded separately from the registered website.html. +type DeliveryDocument = { id?: string; key?: string | null; issueId?: string | null; latestRevisionId?: string | null }; +type DeliveryRevision = { id?: string; documentId?: string | null; createdByRunId?: string | null; createdByAgentId?: string | null }; +export function deliveryRevisionBound(o: { documents: DeliveryDocument[]; revisions: DeliveryRevision[] }, taskId: string, successorRunId: string, workerId: string) { + return o.documents.some(doc => doc.key === "website-delivery" && doc.issueId === taskId && !!doc.latestRevisionId + && o.revisions.some(row => row.id === doc.latestRevisionId && row.documentId === doc.id && row.createdByRunId === successorRunId && row.createdByAgentId === workerId)); +} export function gradeInterruption(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("actual-fresh-direction-native-interruption-evidence", !!o, "A genuine native active owner, saved accepted plan direction, supported human Interrupt and one eligible successor are required."); if (!o) return c.result(); @@ -41,7 +51,7 @@ export function gradeInterruption(value: unknown) { } export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-with-direction", async ctx => { const state: Record = await busyRuntimeWorker(ctx), task = state.task as Entity, source = ctx.input.page.url(); - const marker = `Simpler layout ${ctx.input.nonce}`, body = `# Approved simpler website direction\n${marker}\nReplace the complex multi-panel dashboard layout with one semantic HTML page. Include a visible Acorn Notes heading, one short product paragraph and one labelled signup action. No tabs or multi-page navigation. Register the actual website.html file and save a website-delivery task document with this reference. This applies only to the next eligible attempt; do not start another owner while the current execution remains active.\n`; + const marker = `Simpler layout ${ctx.input.nonce}`, body = `# Approved simpler website direction\n${marker}\nReplace the complex multi-panel dashboard layout with one semantic HTML page. Include a visible Acorn Notes heading, one short product paragraph and one labelled signup action. No tabs or multi-page navigation. Register the actual website.html file and save a task document with key website-delivery that records the delivery. This applies only to the next eligible attempt; do not start another owner while the current execution remains active.\n`; const plan = await ctx.input.api.putSensitive(`/api/issues/${task.id}/documents/plan`, { title: "Simpler Website direction", format: "markdown", body, baseRevisionId: null }); if (!plan.id || !plan.latestRevisionId || !plan.latestRevisionNumber) throw new Error("The real human-authored simpler plan has no retained revision identity"); const confirmation = await ctx.input.api.post(`/api/issues/${task.id}/interactions`, createIssueThreadInteractionSchema.parse({ kind: "request_confirmation", @@ -63,7 +73,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return { ...state, marker, plan, planBody: body, approvedDirection: approved, directionQueue: queue, planApprovedByHuman: approved.status === "accepted" && approved.resolvedByUserId === session.user.id && !approved.resolvedByAgentId, interruptControlEnabled: true }; }, gradeInterruption, { - oracleVersion: "paperclip.chat-catalog.runtime.interrupt-with-direction.v4", authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, + oracleVersion: "paperclip.chat-catalog.runtime.interrupt-with-direction.v5", authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 2 } }, sourceFiles: ["chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], @@ -117,7 +127,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w const revisions = await ctx.input.api.get(`/api/issues/${task.id}/documents/plan/revisions`); state.planRevisionRetained = revisions.some(row => row.id === plan.latestRevisionId && row.body === state.planBody); if (state.successorRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.successorRunId), ["website.html"]); state.actualSimplePage = artifact?.contentVerified === true && hasSimplerWebsite(await renderRuntimeArtifact(ctx, artifact.content)); - state.deliveryRevisionBound = o.revisions.some(row => row.createdByRunId === state.successorRunId && row.createdByAgentId === (state.worker as Entity).id && (row.body ?? "").includes(String(state.marker)) && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); } + state.deliveryRevisionBound = deliveryRevisionBound(o, task.id, String(state.successorRunId), (state.worker as Entity).id); } return o; }, async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Entity; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" });