test(evals): verify rendered native artifacts without post-query direction

tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts

@@ -7,6 +7,7 @@ import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { checks, type Entity } from "./internal-support.js";
 import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js";
 import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
+import { renderRuntimeArtifact } from "./runtime-rendered-artifact.js";
 import { runtimeQueueCases } from "./runtime-queue.js";
 import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js";
 export function isBlueColor(color: string) {
@@ -60,7 +61,7 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act
 }, gradeActiveSteering, {
   authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000,
   setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
-  sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"],
+  sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"],
   async act(ctx, state) {
     await pollUntil({ label: "Canonical steering guidance while the original native owner remains active", deadlineAt: Date.now() + 150_000,
       load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
@@ -117,17 +118,7 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act
     if (state.outputRunId) {
       const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.outputRunId), ["website.html"]);
       if (artifact?.contentVerified) {
-        const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Actual delivered color inspection requires the existing browser");
-        const preview = await browser.newContext({ javaScriptEnabled: false });
-        try {
-          const page = await preview.newPage(); await page.route("**/*", route => route.abort());
-          await page.setContent(artifact.content, { waitUntil: "domcontentloaded" });
-          const colors = await page.locator("body, main, section, header, h1, h2, h3, button, [role=button]").evaluateAll(nodes => nodes.filter(node => {
-            const box = node.getBoundingClientRect(), style = getComputedStyle(node);
-            return box.width > 0 && box.height > 0 && style.visibility !== "hidden" && style.display !== "none";
-          }).map(node => { const style = getComputedStyle(node); return { tag: node.tagName, role: node.getAttribute("role"), color: style.color, background: style.backgroundColor }; }));
-          state.actualBlueArtifact = hasVisibleBlueDesign(colors);
-        } finally { await preview.close(); }
+        state.actualBlueArtifact = hasVisibleBlueDesign(await renderRuntimeArtifact(ctx, artifact.content));
       }
       state.blueRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && /blue/i.test(row.body ?? "")
         && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id));

tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts

@@ -3,6 +3,7 @@ import { writeFile } from "node:fs/promises";
 import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js";
 import type { IssueQueuedCommentQueue, IssueThreadInteraction } from "../../../packages/shared/src/types/issue.js";
 import { pollUntil } from "../api.js";
+import { renderRuntimeArtifact, hasSimplerWebsite } from "./runtime-rendered-artifact.js";
 import { collectRunEvents } from "../run-observations.js";
 import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js";
@@ -61,7 +62,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w
   authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000,
   setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
   runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 2 } },
-  sourceFiles: ["chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"],
+  sourceFiles: ["chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"],
   async act(ctx, state) {
     await pollUntil({ label: "Canonical interruption guidance before the actual human action", deadlineAt: Date.now() + 150_000,
       load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
@@ -79,7 +80,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w
     state.nativeStopAcknowledged = cancellation?.runId === stopped.id && cancellation.issueId === task.id && cancellation.companyId === ctx.input.fixtures.company.id && cancellation.dispatched === true && cancellation.dispatchState === "acknowledged";
     // No gate value injects the simpler direction: it exists only in the
     // independently approved persisted plan and its actual queued response.
-    await writeFile((state.gate as { gate: string }).gate, "The human ended the original pending-brief tool. Keep any saved history; the supported fresh continuation must read its own actually accepted plan revision and implement that direction. Do not create tasks or external effects.", "utf8");
+    await writeFile((state.gate as { gate: string }).gate, "The pending fixture tool has been released.", "utf8");
     let neverConcurrent = true;
     const delivered = await pollUntil({ label: "One fresh native continuation after old-owner cleanup delivers simpler Website", deadlineAt: Date.now() + 240_000,
       load: async () => { const runs = await ctx.allRuns(); neverConcurrent &&= runs.filter(row => row.agentId === (state.worker as Entity).id && ["running", "queued"].includes(row.status)).length <= 1;
@@ -100,7 +101,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w
   async observe(ctx, state) { const o = await observeRuntime(ctx, state), task = state.task as Entity, plan = state.plan as Entity;
     const revisions = await ctx.input.api.get<Entity[]>(`/api/issues/${task.id}/documents/plan/revisions`); state.planRevisionRetained = revisions.some(row => row.id === plan.latestRevisionId && row.body === state.planBody);
     if (state.successorRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.successorRunId), ["website.html"]);
-      state.actualSimplePage = artifact?.contentVerified === true && /<h1\b[^>]*>[\s\S]*?Acorn Notes[\s\S]*?<\/h1>/i.test(artifact.content) && /<main\b/i.test(artifact.content) && /sign.?up|subscribe|join/i.test(artifact.content) && !/role=["']tab|<iframe\b/i.test(artifact.content);
+      state.actualSimplePage = artifact?.contentVerified === true && hasSimplerWebsite(await renderRuntimeArtifact(ctx, artifact.content));
       state.deliveryRevisionBound = o.revisions.some(row => row.createdByRunId === state.successorRunId && row.createdByAgentId === (state.worker as Entity).id && (row.body ?? "").includes(String(state.marker)) && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); }
     return o; },
   async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Entity;

tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact-canary-receipt.json

@@ -0,0 +1,18 @@
+{
+  "schemaVersion": 1,
+  "kind": "offline-synthetic-rendered-artifact-canary",
+  "canonicalScenarioCoverage": false,
+  "backendLaunched": false,
+  "providerCalled": false,
+  "browser": "Chromium",
+  "javascriptEnabled": false,
+  "networkRequests": "aborted",
+  "authenticatedStorage": false,
+  "command": "PAPERCLIP_CHAT_EVAL_RENDERED_ARTIFACT_CANARY=1 node node_modules/vitest/vitest.mjs run --config tests/runner-e2e/vitest.config.ts tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts",
+  "observedAt": "2026-10-26T22:35:01-05:00",
+  "testsPassed": 9,
+  "renderedCases": 8,
+  "outcomes": ["natural-inline-headline-and-link", "natural-button-action", "comment-only-wrong-page-rejected", "display-none-page-rejected", "zero-opacity-heading-rejected", "zero-opacity-action-rejected", "multi-page-navigation-rejected", "tabbed-dashboard-rejected"],
+  "initialOfflineFailure": "Preview context closed before awaiting the returned evaluation promise; fixed with return await before final passing canary.",
+  "note": "Explicit opt-in synthetic browser verification only. Cheap default Vitest skips the Chromium canary. This does not prove a canonical native run or count as a live campaign attempt."
+}

tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts

@@ -0,0 +1,33 @@
+import { expect, it } from "vitest";
+import { chromium } from "@playwright/test";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { hasSimplerWebsite, renderRuntimeArtifact, type RuntimeRenderedNode } from "./runtime-rendered-artifact.js";
+const node = (tag: string, text: string, extra: Partial<RuntimeRenderedNode> = {}): RuntimeRenderedNode => ({ tag, role: null, text, href: null, inNavigation: false, color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)", ...extra });
+const simple = () => [node("MAIN", "Acorn Notes organizes research into clear summaries so teams can work without the busywork. Join the early access list."), node("H1", "Acorn Notes"), node("A", "Join early access", { href: "#signup" })];
+it("accepts visible substantive natural single-page content and action", () => expect(hasSimplerWebsite(simple())).toBe(true));
+it("accepts a natural button signup equivalent", () => { const x = simple(); x[2] = node("BUTTON", "Subscribe"); expect(hasSimplerWebsite(x)).toBe(true); });
+it("rejects comment-only/hidden heading absent from rendered evidence", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "H1"))).toBe(false));
+it("rejects an invisible signup action", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "A"))).toBe(false));
+it("rejects an actual tabbed complex layout", () => expect(hasSimplerWebsite([...simple(), node("BUTTON", "Dashboard", { role: "tab" })])).toBe(false));
+it("rejects actual multi-page navigation", () => expect(hasSimplerWebsite([...simple(), node("A", "Pricing", { href: "pricing.html", inNavigation: true })])).toBe(false));
+it("accepts in-page navigation in a simple page", () => expect(hasSimplerWebsite([...simple(), node("A", "Signup", { href: "#signup", inNavigation: true })])).toBe(true));
+it("rejects the wrong visible product page", () => { const x = simple(); x[1]!.text = "Unrelated dashboard"; expect(hasSimplerWebsite(x)).toBe(false); });
+// Chromium remains explicit opt-in under the repository browser-suite contract.
+it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_RENDERED_ARTIFACT_CANARY !== "1")("qualifies actual rendered hidden/comment/complex controls without backend or provider calls", async () => {
+  const browser = await chromium.launch({ headless: true, ...(process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {}) });
+  const context = await browser.newContext(), page = await context.newPage();
+  const ctx = { input: { page } } as unknown as ChatCatalogContext;
+  const body = '<main><h1>Acorn <span>Notes</span></h1><p>Turn scattered research into a clear shared notebook for the whole team.</p><a href="#signup">Join early access</a></main>';
+  try {
+    for (const [html, valid] of [
+      [body, true],
+      [body.replace('<a href="#signup">Join early access</a>', '<button>Subscribe</button>'), true],
+      [`<!-- ${body} --><main><h1>Other page</h1><p>This unrelated content is not the intended product or a signup workflow.</p></main>`, false],
+      [body.replace('<main>', '<main style="display:none">'), false],
+      [body.replace('<h1>', '<h1 style="opacity:0">'), false],
+      [body.replace('<a href=', '<a style="opacity:0" href='), false],
+      [body + '<nav><a href="pricing.html">Pricing</a><a href="dashboard.html">Dashboard</a></nav>', false],
+      [body + '<div role="tablist"><button role="tab">Metrics</button><button role="tab">Dashboard</button></div>', false],
+    ] as const) expect(hasSimplerWebsite(await renderRuntimeArtifact(ctx, html))).toBe(valid);
+  } finally { await context.close(); await browser.close(); }
+}, 30_000);

tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts

@@ -0,0 +1,38 @@
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+export interface RuntimeRenderedNode { tag: string; role: string | null; text: string; href: string | null; inNavigation: boolean; color: string; background: string }
+/** Render only independently downloaded artifact bytes, with no scripts, network or authenticated storage. */
+export async function renderRuntimeArtifact(ctx: ChatCatalogContext, html: string): Promise<RuntimeRenderedNode[]> {
+  if (Buffer.byteLength(html) > 262_144) throw new Error("Rendered runtime artifact exceeds its registered bounded size");
+  const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Actual rendered artifact inspection requires the existing browser");
+  const preview = await browser.newContext({ javaScriptEnabled: false });
+  try {
+    const page = await preview.newPage(); await page.route("**/*", route => route.abort());
+    await page.setContent(html, { waitUntil: "domcontentloaded" });
+    return await page.locator("body, main, section, header, h1, h2, h3, p, button, a, nav, [role]").evaluateAll(elements => {
+      if (elements.length > 2_000) throw new Error("Rendered artifact exceeds its finite DOM inspection bound");
+      const visible = (element: Element) => {
+        const rect = element.getBoundingClientRect(); if (rect.width < 2 || rect.height < 2 || rect.right <= 0 || rect.left >= innerWidth) return false;
+        for (let node: Element | null = element; node; node = node.parentElement) { const style = getComputedStyle(node);
+          if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return false; }
+        return true;
+      };
+      const text = (element: Element) => {
+        const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT), parts: string[] = []; let node: Node | null; let count = 0;
+        while ((node = walker.nextNode())) { if (++count > 10_000) throw new Error("Rendered artifact exceeds its finite text inspection bound");
+          if (node.parentElement && visible(node.parentElement) && node.textContent?.trim()) parts.push(node.textContent.trim()); }
+        return parts.join(" ").replace(/\s+/g, " ").slice(0, 20_000);
+      };
+      return elements.filter(visible).map(element => { const style = getComputedStyle(element);
+        return { tag: element.tagName, role: element.getAttribute("role"), text: text(element) || element.getAttribute("aria-label") || "", href: element.getAttribute("href"),
+          inNavigation: !!element.closest('nav, [role="navigation"]'), color: style.color, background: style.backgroundColor }; });
+    });
+  } finally { await preview.close(); }
+}
+export function hasSimplerWebsite(nodes: RuntimeRenderedNode[]) {
+  const headline = nodes.some(node => (node.tag === "H1" || node.role === "heading") && /\bAcorn\s+Notes\b/i.test(node.text));
+  const content = nodes.some(node => node.tag === "MAIN" && node.text.length >= 40);
+  const signup = nodes.some(node => ["BUTTON", "A"].includes(node.tag) && /sign.?up|subscribe|join|early access/i.test(node.text));
+  const tabbed = nodes.some(node => ["tab", "tablist", "tabpanel"].includes(node.role ?? ""));
+  const multiPage = nodes.some(node => node.tag === "A" && node.inNavigation && !!node.href && !/^#|^\s*$/.test(node.href));
+  return headline && content && signup && !tabbed && !multiPage;
+}