From 7733c54e21f901d6573c71e23c8e91fd431dd73a Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 22:36:03 -0500 Subject: [PATCH] test(evals): verify rendered native artifacts without post-query direction --- .../runtime-active-steering.ts | 15 ++------ .../runtime-interruption.ts | 7 ++-- ...time-rendered-artifact-canary-receipt.json | 18 +++++++++ .../runtime-rendered-artifact.test.ts | 33 ++++++++++++++++ .../runtime-rendered-artifact.ts | 38 +++++++++++++++++++ 5 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact-canary-receipt.json create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts index f9a4e9e218a..ce890ad7084 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts @@ -7,6 +7,7 @@ import { readRegisteredArtifacts } from "../registered-artifact.js"; import { checks, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; export function isBlueColor(color: string) { @@ -60,7 +61,7 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act }, gradeActiveSteering, { authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, - sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { await pollUntil({ label: "Canonical steering guidance while the original native owner remains active", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, @@ -117,17 +118,7 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act if (state.outputRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.outputRunId), ["website.html"]); if (artifact?.contentVerified) { - const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Actual delivered color inspection requires the existing browser"); - const preview = await browser.newContext({ javaScriptEnabled: false }); - try { - const page = await preview.newPage(); await page.route("**/*", route => route.abort()); - await page.setContent(artifact.content, { waitUntil: "domcontentloaded" }); - const colors = await page.locator("body, main, section, header, h1, h2, h3, button, [role=button]").evaluateAll(nodes => nodes.filter(node => { - const box = node.getBoundingClientRect(), style = getComputedStyle(node); - return box.width > 0 && box.height > 0 && style.visibility !== "hidden" && style.display !== "none"; - }).map(node => { const style = getComputedStyle(node); return { tag: node.tagName, role: node.getAttribute("role"), color: style.color, background: style.backgroundColor }; })); - state.actualBlueArtifact = hasVisibleBlueDesign(colors); - } finally { await preview.close(); } + state.actualBlueArtifact = hasVisibleBlueDesign(await renderRuntimeArtifact(ctx, artifact.content)); } state.blueRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && /blue/i.test(row.body ?? "") && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts index db9f883d86a..868768e2dff 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts @@ -3,6 +3,7 @@ import { writeFile } from "node:fs/promises"; import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; import type { IssueQueuedCommentQueue, IssueThreadInteraction } from "../../../packages/shared/src/types/issue.js"; import { pollUntil } from "../api.js"; +import { renderRuntimeArtifact, hasSimplerWebsite } from "./runtime-rendered-artifact.js"; import { collectRunEvents } from "../run-observations.js"; import { readRegisteredArtifacts } from "../registered-artifact.js"; import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js"; @@ -61,7 +62,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 2 } }, - sourceFiles: ["chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { await pollUntil({ label: "Canonical interruption guidance before the actual human action", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, @@ -79,7 +80,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w state.nativeStopAcknowledged = cancellation?.runId === stopped.id && cancellation.issueId === task.id && cancellation.companyId === ctx.input.fixtures.company.id && cancellation.dispatched === true && cancellation.dispatchState === "acknowledged"; // No gate value injects the simpler direction: it exists only in the // independently approved persisted plan and its actual queued response. - await writeFile((state.gate as { gate: string }).gate, "The human ended the original pending-brief tool. Keep any saved history; the supported fresh continuation must read its own actually accepted plan revision and implement that direction. Do not create tasks or external effects.", "utf8"); + await writeFile((state.gate as { gate: string }).gate, "The pending fixture tool has been released.", "utf8"); let neverConcurrent = true; const delivered = await pollUntil({ label: "One fresh native continuation after old-owner cleanup delivers simpler Website", deadlineAt: Date.now() + 240_000, load: async () => { const runs = await ctx.allRuns(); neverConcurrent &&= runs.filter(row => row.agentId === (state.worker as Entity).id && ["running", "queued"].includes(row.status)).length <= 1; @@ -100,7 +101,7 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w async observe(ctx, state) { const o = await observeRuntime(ctx, state), task = state.task as Entity, plan = state.plan as Entity; const revisions = await ctx.input.api.get(`/api/issues/${task.id}/documents/plan/revisions`); state.planRevisionRetained = revisions.some(row => row.id === plan.latestRevisionId && row.body === state.planBody); if (state.successorRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.successorRunId), ["website.html"]); - state.actualSimplePage = artifact?.contentVerified === true && /]*>[\s\S]*?Acorn Notes[\s\S]*?<\/h1>/i.test(artifact.content) && / row.createdByRunId === state.successorRunId && row.createdByAgentId === (state.worker as Entity).id && (row.body ?? "").includes(String(state.marker)) && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); } return o; }, async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Entity; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact-canary-receipt.json b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact-canary-receipt.json new file mode 100644 index 00000000000..97d5ad61be2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact-canary-receipt.json @@ -0,0 +1,18 @@ +{ + "schemaVersion": 1, + "kind": "offline-synthetic-rendered-artifact-canary", + "canonicalScenarioCoverage": false, + "backendLaunched": false, + "providerCalled": false, + "browser": "Chromium", + "javascriptEnabled": false, + "networkRequests": "aborted", + "authenticatedStorage": false, + "command": "PAPERCLIP_CHAT_EVAL_RENDERED_ARTIFACT_CANARY=1 node node_modules/vitest/vitest.mjs run --config tests/runner-e2e/vitest.config.ts tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts", + "observedAt": "2026-10-26T22:35:01-05:00", + "testsPassed": 9, + "renderedCases": 8, + "outcomes": ["natural-inline-headline-and-link", "natural-button-action", "comment-only-wrong-page-rejected", "display-none-page-rejected", "zero-opacity-heading-rejected", "zero-opacity-action-rejected", "multi-page-navigation-rejected", "tabbed-dashboard-rejected"], + "initialOfflineFailure": "Preview context closed before awaiting the returned evaluation promise; fixed with return await before final passing canary.", + "note": "Explicit opt-in synthetic browser verification only. Cheap default Vitest skips the Chromium canary. This does not prove a canonical native run or count as a live campaign attempt." +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts new file mode 100644 index 00000000000..b2e353e8f62 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.test.ts @@ -0,0 +1,33 @@ +import { expect, it } from "vitest"; +import { chromium } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { hasSimplerWebsite, renderRuntimeArtifact, type RuntimeRenderedNode } from "./runtime-rendered-artifact.js"; +const node = (tag: string, text: string, extra: Partial = {}): RuntimeRenderedNode => ({ tag, role: null, text, href: null, inNavigation: false, color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)", ...extra }); +const simple = () => [node("MAIN", "Acorn Notes organizes research into clear summaries so teams can work without the busywork. Join the early access list."), node("H1", "Acorn Notes"), node("A", "Join early access", { href: "#signup" })]; +it("accepts visible substantive natural single-page content and action", () => expect(hasSimplerWebsite(simple())).toBe(true)); +it("accepts a natural button signup equivalent", () => { const x = simple(); x[2] = node("BUTTON", "Subscribe"); expect(hasSimplerWebsite(x)).toBe(true); }); +it("rejects comment-only/hidden heading absent from rendered evidence", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "H1"))).toBe(false)); +it("rejects an invisible signup action", () => expect(hasSimplerWebsite(simple().filter(n => n.tag !== "A"))).toBe(false)); +it("rejects an actual tabbed complex layout", () => expect(hasSimplerWebsite([...simple(), node("BUTTON", "Dashboard", { role: "tab" })])).toBe(false)); +it("rejects actual multi-page navigation", () => expect(hasSimplerWebsite([...simple(), node("A", "Pricing", { href: "pricing.html", inNavigation: true })])).toBe(false)); +it("accepts in-page navigation in a simple page", () => expect(hasSimplerWebsite([...simple(), node("A", "Signup", { href: "#signup", inNavigation: true })])).toBe(true)); +it("rejects the wrong visible product page", () => { const x = simple(); x[1]!.text = "Unrelated dashboard"; expect(hasSimplerWebsite(x)).toBe(false); }); +// Chromium remains explicit opt-in under the repository browser-suite contract. +it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_RENDERED_ARTIFACT_CANARY !== "1")("qualifies actual rendered hidden/comment/complex controls without backend or provider calls", async () => { + const browser = await chromium.launch({ headless: true, ...(process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {}) }); + const context = await browser.newContext(), page = await context.newPage(); + const ctx = { input: { page } } as unknown as ChatCatalogContext; + const body = '

Acorn Notes

Turn scattered research into a clear shared notebook for the whole team.

Join early access
'; + try { + for (const [html, valid] of [ + [body, true], + [body.replace('Join early access', ''), true], + [`

Other page

This unrelated content is not the intended product or a signup workflow.

`, false], + [body.replace('
', '
'), false], + [body.replace('

', '

'), false], + [body.replace('PricingDashboard', false], + [body + '
', false], + ] as const) expect(hasSimplerWebsite(await renderRuntimeArtifact(ctx, html))).toBe(valid); + } finally { await context.close(); await browser.close(); } +}, 30_000); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts new file mode 100644 index 00000000000..587baf6c1e1 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-rendered-artifact.ts @@ -0,0 +1,38 @@ +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +export interface RuntimeRenderedNode { tag: string; role: string | null; text: string; href: string | null; inNavigation: boolean; color: string; background: string } +/** Render only independently downloaded artifact bytes, with no scripts, network or authenticated storage. */ +export async function renderRuntimeArtifact(ctx: ChatCatalogContext, html: string): Promise { + if (Buffer.byteLength(html) > 262_144) throw new Error("Rendered runtime artifact exceeds its registered bounded size"); + const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Actual rendered artifact inspection requires the existing browser"); + const preview = await browser.newContext({ javaScriptEnabled: false }); + try { + const page = await preview.newPage(); await page.route("**/*", route => route.abort()); + await page.setContent(html, { waitUntil: "domcontentloaded" }); + return await page.locator("body, main, section, header, h1, h2, h3, p, button, a, nav, [role]").evaluateAll(elements => { + if (elements.length > 2_000) throw new Error("Rendered artifact exceeds its finite DOM inspection bound"); + const visible = (element: Element) => { + const rect = element.getBoundingClientRect(); if (rect.width < 2 || rect.height < 2 || rect.right <= 0 || rect.left >= innerWidth) return false; + for (let node: Element | null = element; node; node = node.parentElement) { const style = getComputedStyle(node); + if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return false; } + return true; + }; + const text = (element: Element) => { + const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT), parts: string[] = []; let node: Node | null; let count = 0; + while ((node = walker.nextNode())) { if (++count > 10_000) throw new Error("Rendered artifact exceeds its finite text inspection bound"); + if (node.parentElement && visible(node.parentElement) && node.textContent?.trim()) parts.push(node.textContent.trim()); } + return parts.join(" ").replace(/\s+/g, " ").slice(0, 20_000); + }; + return elements.filter(visible).map(element => { const style = getComputedStyle(element); + return { tag: element.tagName, role: element.getAttribute("role"), text: text(element) || element.getAttribute("aria-label") || "", href: element.getAttribute("href"), + inNavigation: !!element.closest('nav, [role="navigation"]'), color: style.color, background: style.backgroundColor }; }); + }); + } finally { await preview.close(); } +} +export function hasSimplerWebsite(nodes: RuntimeRenderedNode[]) { + const headline = nodes.some(node => (node.tag === "H1" || node.role === "heading") && /\bAcorn\s+Notes\b/i.test(node.text)); + const content = nodes.some(node => node.tag === "MAIN" && node.text.length >= 40); + const signup = nodes.some(node => ["BUTTON", "A"].includes(node.tag) && /sign.?up|subscribe|join|early access/i.test(node.text)); + const tabbed = nodes.some(node => ["tab", "tablist", "tabpanel"].includes(node.role ?? "")); + const multiPage = nodes.some(node => node.tag === "A" && node.inNavigation && !!node.href && !/^#|^\s*$/.test(node.href)); + return headline && content && signup && !tabbed && !multiPage; +}