From 869595fed30a048d7eba8fdbbdec64ce18db8bbe Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 00:22:40 -0600 Subject: [PATCH] test(evals): accept a single page without a main landmark; keep the graded artifact The single-page check required a
element, which the request never asked for. Any visible product paragraph or section now counts. The post-stop case also records the graded artifact and its closed sub-checks so a rejection can be diagnosed. Co-Authored-By: Claude Opus 5.5 --- .../chat-catalog-domains/runtime-post-stop.ts | 8 ++++++-- .../runtime-rendered-artifact.test.ts | 1 + .../chat-catalog-domains/runtime-rendered-artifact.ts | 10 ++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts index 539a64e5f31..4653ec665ae 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts @@ -11,7 +11,7 @@ import { openStandingChat } from "./identity-support.js"; import { prepareRuntimeWorkerGate, readRuntimeTaskGate } from "./runtime-busy-worker.js"; import { suspendOwnedProviderForCleanup, releaseOwnedCleanupProvider, ownedCleanupProviderHasExited, ownedCleanupProviderIsHeld } from "./runtime-provider-cleanup.js"; import type { NativeFaultRun } from "./runtime-native-fault.js"; -import { hasSimplerWebsite, renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; +import { hasSimplerWebsite, renderRuntimeArtifact, simplerWebsiteChecks } from "./runtime-rendered-artifact.js"; import { observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js"; interface StopIntent { schema?: string; intentId?: string; intentAuditId?: string; companyId?: string; runId?: string; issueId?: string; scope?: string; dispatchState?: string; dispatched?: boolean; recordedAt?: string; acknowledgedAt?: string; acknowledgementAuditId?: string } interface StopHandle { page: Page; completed?: Promise } @@ -126,7 +126,11 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa async observe(ctx, state) { const o = await observeRuntime(ctx, state); if (state.successorRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, String(state.sourceId), String(state.successorRunId), ["website.html"]); - state.artifactRunBound = artifact?.contentVerified === true; state.actualSimplePage = !!artifact?.contentVerified && hasSimplerWebsite(await renderRuntimeArtifact(ctx, artifact.content)); } + state.artifactRunBound = artifact?.contentVerified === true; + const nodes = artifact?.contentVerified ? await renderRuntimeArtifact(ctx, artifact.content) : []; + state.actualSimplePage = !!artifact?.contentVerified && hasSimplerWebsite(nodes); + await ctx.input.evidence("runtime-post-stop-artifact.json", { contentVerified: artifact?.contentVerified === true, bytes: artifact ? Buffer.byteLength(artifact.content) : 0, + checks: simplerWebsiteChecks(nodes), visibleTags: [...new Set(nodes.map(node => node.tag))].sort(), content: artifact?.content ?? null }); } return o; }, async verifyResult(ctx) { await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts index b2e353e8f62..22ce41b831d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts @@ -5,6 +5,7 @@ import { hasSimplerWebsite, renderRuntimeArtifact, type RuntimeRenderedNode } fr const node = (tag: string, text: string, extra: Partial = {}): RuntimeRenderedNode => ({ tag, role: null, text, href: null, inNavigation: false, color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)", ...extra }); const simple = () => [node("MAIN", "Acorn Notes organizes research into clear summaries so teams can work without the busywork. Join the early access list."), node("H1", "Acorn Notes"), node("A", "Join early access", { href: "#signup" })]; it("accepts visible substantive natural single-page content and action", () => expect(hasSimplerWebsite(simple())).toBe(true)); +it("accepts a visible product paragraph without a main landmark", () => { const x = simple(); x[0] = node("P", x[0]!.text); expect(hasSimplerWebsite(x)).toBe(true); expect(hasSimplerWebsite([node("P", "Short."), x[1]!, x[2]!])).toBe(false); }); it("accepts a natural button signup equivalent", () => { const x = simple(); x[2] = node("BUTTON", "Subscribe"); expect(hasSimplerWebsite(x)).toBe(true); }); it("rejects comment-only/hidden heading absent from rendered evidence", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "H1"))).toBe(false)); it("rejects an invisible signup action", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "A"))).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts index 587baf6c1e1..36ecb7cb22b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts @@ -28,11 +28,17 @@ export async function renderRuntimeArtifact(ctx: ChatCatalogContext, html: strin }); } finally { await preview.close(); } } -export function hasSimplerWebsite(nodes: RuntimeRenderedNode[]) { +/** Closed sub-results of the single-page check, for diagnosis without retaining page text. */ +export function simplerWebsiteChecks(nodes: RuntimeRenderedNode[]) { const headline = nodes.some(node => (node.tag === "H1" || node.role === "heading") && /\bAcorn\s+Notes\b/i.test(node.text)); - const content = nodes.some(node => node.tag === "MAIN" && node.text.length >= 40); + // A visible product paragraph; the request never named a
landmark. + const content = nodes.some(node => ["MAIN", "SECTION", "P"].includes(node.tag) && node.text.length >= 40); const signup = nodes.some(node => ["BUTTON", "A"].includes(node.tag) && /sign.?up|subscribe|join|early access/i.test(node.text)); const tabbed = nodes.some(node => ["tab", "tablist", "tabpanel"].includes(node.role ?? "")); const multiPage = nodes.some(node => node.tag === "A" && node.inNavigation && !!node.href && !/^#|^\s*$/.test(node.href)); + return { headline, content, signup, tabbed, multiPage }; +} +export function hasSimplerWebsite(nodes: RuntimeRenderedNode[]) { + const { headline, content, signup, tabbed, multiPage } = simplerWebsiteChecks(nodes); return headline && content && signup && !tabbed && !multiPage; }