From 417089f9f19b4c250f3b036a670013e14644535d Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 01:00:21 -0600 Subject: [PATCH] test(evals): record interrupt-with-direction grading change from the loosened rendered-artifact helper runtime-rendered-artifact.ts now accepts a single page without a main landmark and a tagline heading when the product is named on the page. runtime-interruption.ts grades through that helper, so bump its oracleVersion in code and the catalog descriptors. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 022b2176f6c..cb926b26df4 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5478,7 +5478,7 @@ "registered-artifact.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.interrupt-with-direction.v3", "coverageGaps": [], "expectedRunCount": 3, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts index b78fbb5788f..7d0594b2cb6 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts @@ -58,7 +58,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return { ...state, marker, plan, planBody: body, approvedDirection: approved, directionQueue: queue, planApprovedByHuman: approved.status === "accepted" && approved.resolvedByUserId === session.user.id && !approved.resolvedByAgentId, interruptControlEnabled: true }; }, gradeInterruption, { - authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, + oracleVersion: "paperclip.chat-catalog.runtime.interrupt-with-direction.v3", authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 2 } }, sourceFiles: ["chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"],