test(evals): accept a single page without a main landmark; keep the graded artifact

The single-page check required a <main> element, which the request never
asked for. Any visible product paragraph or section now counts. The
post-stop case also records the graded artifact and its closed sub-checks
so a rejection can be diagnosed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 869595fed3)

tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts

@@ -11,7 +11,7 @@ import { openStandingChat } from "./identity-support.js";
 import { prepareRuntimeWorkerGate, readRuntimeTaskGate } from "./runtime-busy-worker.js";
 import { suspendOwnedProviderForCleanup, releaseOwnedCleanupProvider, ownedCleanupProviderHasExited, ownedCleanupProviderIsHeld } from "./runtime-provider-cleanup.js";
 import type { NativeFaultRun } from "./runtime-native-fault.js";
-import { hasSimplerWebsite, renderRuntimeArtifact } from "./runtime-rendered-artifact.js";
+import { hasSimplerWebsite, renderRuntimeArtifact, simplerWebsiteChecks } from "./runtime-rendered-artifact.js";
 import { observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js";
 interface StopIntent { schema?: string; intentId?: string; intentAuditId?: string; companyId?: string; runId?: string; issueId?: string; scope?: string; dispatchState?: string; dispatched?: boolean; recordedAt?: string; acknowledgedAt?: string; acknowledgementAuditId?: string }
 interface StopHandle { page: Page; completed?: Promise<number> }
@@ -126,7 +126,11 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa
   async observe(ctx, state) {
     const o = await observeRuntime(ctx, state);
     if (state.successorRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, String(state.sourceId), String(state.successorRunId), ["website.html"]);
-      state.artifactRunBound = artifact?.contentVerified === true; state.actualSimplePage = !!artifact?.contentVerified && hasSimplerWebsite(await renderRuntimeArtifact(ctx, artifact.content)); }
+      state.artifactRunBound = artifact?.contentVerified === true;
+      const nodes = artifact?.contentVerified ? await renderRuntimeArtifact(ctx, artifact.content) : [];
+      state.actualSimplePage = !!artifact?.contentVerified && hasSimplerWebsite(nodes);
+      await ctx.input.evidence("runtime-post-stop-artifact.json", { contentVerified: artifact?.contentVerified === true, bytes: artifact ? Buffer.byteLength(artifact.content) : 0,
+        checks: simplerWebsiteChecks(nodes), visibleTags: [...new Set(nodes.map(node => node.tag))].sort(), content: artifact?.content ?? null }); }
     return o;
   },
   async verifyResult(ctx) { await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },

tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts

@@ -5,6 +5,7 @@ import { hasSimplerWebsite, renderRuntimeArtifact, type RuntimeRenderedNode } fr
 const node = (tag: string, text: string, extra: Partial<RuntimeRenderedNode> = {}): RuntimeRenderedNode => ({ tag, role: null, text, href: null, inNavigation: false, color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)", ...extra });
 const simple = () => [node("MAIN", "Acorn Notes organizes research into clear summaries so teams can work without the busywork. Join the early access list."), node("H1", "Acorn Notes"), node("A", "Join early access", { href: "#signup" })];
 it("accepts visible substantive natural single-page content and action", () => expect(hasSimplerWebsite(simple())).toBe(true));
+it("accepts a visible product paragraph without a main landmark", () => { const x = simple(); x[0] = node("P", x[0]!.text); expect(hasSimplerWebsite(x)).toBe(true); expect(hasSimplerWebsite([node("P", "Short."), x[1]!, x[2]!])).toBe(false); });
 it("accepts a natural button signup equivalent", () => { const x = simple(); x[2] = node("BUTTON", "Subscribe"); expect(hasSimplerWebsite(x)).toBe(true); });
 it("rejects comment-only/hidden heading absent from rendered evidence", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "H1"))).toBe(false));
 it("rejects an invisible signup action", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "A"))).toBe(false));

tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts

@@ -28,11 +28,17 @@ export async function renderRuntimeArtifact(ctx: ChatCatalogContext, html: strin
     });
   } finally { await preview.close(); }
 }
-export function hasSimplerWebsite(nodes: RuntimeRenderedNode[]) {
+/** Closed sub-results of the single-page check, for diagnosis without retaining page text. */
+export function simplerWebsiteChecks(nodes: RuntimeRenderedNode[]) {
   const headline = nodes.some(node => (node.tag === "H1" || node.role === "heading") && /\bAcorn\s+Notes\b/i.test(node.text));
-  const content = nodes.some(node => node.tag === "MAIN" && node.text.length >= 40);
+  // A visible product paragraph; the request never named a <main> landmark.
+  const content = nodes.some(node => ["MAIN", "SECTION", "P"].includes(node.tag) && node.text.length >= 40);
   const signup = nodes.some(node => ["BUTTON", "A"].includes(node.tag) && /sign.?up|subscribe|join|early access/i.test(node.text));
   const tabbed = nodes.some(node => ["tab", "tablist", "tabpanel"].includes(node.role ?? ""));
   const multiPage = nodes.some(node => node.tag === "A" && node.inNavigation && !!node.href && !/^#|^\s*$/.test(node.href));
+  return { headline, content, signup, tabbed, multiPage };
+}
+export function hasSimplerWebsite(nodes: RuntimeRenderedNode[]) {
+  const { headline, content, signup, tabbed, multiPage } = simplerWebsiteChecks(nodes);
   return headline && content && signup && !tabbed && !multiPage;
 }