diff --git a/tests/runner-e2e/product-journeys/campaign.test.ts b/tests/runner-e2e/product-journeys/campaign.test.ts index b1a111df3cc..0ede9070dc3 100644 --- a/tests/runner-e2e/product-journeys/campaign.test.ts +++ b/tests/runner-e2e/product-journeys/campaign.test.ts @@ -45,7 +45,7 @@ async function run(options:{authFailure?:boolean;onboardingFailure?:boolean;preA await writeFile(module, `import {readFile,writeFile} from 'node:fs/promises'; const identity={kind:'cloud',targetId:'qa',origin:'https://qa.example.test',commit:'${"a".repeat(40)}',cloudRelease:'v1',runtimeImage:'image',configurationHash:'config'}; let onboarded=false; -const context={identity,company:{id:'company-1',issuePrefix:'QA'},api:{origin:identity.origin,post:async()=>({id:'issue-fixture-1',identifier:'QA-1'})},startingState:'onboarding',capabilities:['company','board-user','task-write'],page:{url:()=>identity.origin+'/QA/issues/QA-1',locator:()=>({isVisible:async()=>true,screenshot:async()=>{}}) }}; +const context={identity,company:{id:'company-1',issuePrefix:'QA'},api:{origin:identity.origin,post:async()=>({id:'issue-fixture-1',identifier:'QA-1'})},startingState:'onboarding',capabilities:['company','board-user','task-write'],page:{url:()=>identity.origin+'/QA/issues/QA-1',locator:()=>({waitFor:async()=>{},isVisible:async()=>true,screenshot:async()=>{}}) }}; export const provider={version:1, async acquire(input){return { async authenticate(){${options.authFailure ? "throw {outcome:'failed',code:'login_code_expired'};" : ""}}, async onboard(evidence){${options.preAcceptanceFailure ? `evidence.onboardingPreAcceptance=${JSON.stringify(preAcceptanceEvidence)};throw {outcome:'failed',code:'onboarding_work_before_acceptance'};` : options.onboardingFailure ? "throw {outcome:'failed',code:'onboarding_failed'};" : `${options.missingArtifact ? "" : `evidence.artifact=${JSON.stringify(artifact)};`}evidence.checks.push(...['fresh-workspace','ui-connection-and-native-agent','first-task-complete','downloadable-artifact-verified'${options.missingCheck ? "" : ",'native-trace-verified'"}].map(id=>({id,passed:true})));onboarded=true;return context;`}}, diff --git a/tests/runner-e2e/product-journeys/clarification-campaign.test.ts b/tests/runner-e2e/product-journeys/clarification-campaign.test.ts index 95a93835e42..d4572b6323a 100644 --- a/tests/runner-e2e/product-journeys/clarification-campaign.test.ts +++ b/tests/runner-e2e/product-journeys/clarification-campaign.test.ts @@ -29,7 +29,7 @@ const identity = { kind: 'cloud', targetId: 'qa', origin: 'https://qa.example.te const context = { identity, company: { id: 'company', issuePrefix: 'QA' }, user: { id: 'board', name: 'QA' }, api: { origin: identity.origin }, startingState: 'recipe', capabilities: ['company', 'board-user', 'task-write', 'agent'], agent: { id: 'agent', name: 'QA', provider: 'opencode', model: 'model', runtimeMode: 'native' }, environmentId: 'daytona', - page: { url: () => identity.origin + '/QA/issues/QA-1', locator: () => ({ isVisible: async () => true, screenshot: async () => {} }) } }; + page: { url: () => identity.origin + '/QA/issues/QA-1', locator: () => ({ waitFor: async () => {}, isVisible: async () => true, screenshot: async () => {} }) } }; export const provider = { version: 1, async acquire() { return { async authenticate() {}, async prepare() { return context; }, async identity() { return identity; }, async collect() { return ${JSON.stringify(receipts)}; }, async own() {}, async release() { return 'released'; }, async stop() {} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 3f39f221870..1a905557ff6 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -206,14 +206,14 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri capture: async (_id, _label, file) => { const directory = path.join(outputDir, "private-stories", id); await mkdir(directory, { recursive: true, mode: 0o700 }); - await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file)); + await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); else await runAgentJourney(id, ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { const directory = path.join(outputDir, "private-stories", id); await mkdir(directory, { recursive: true, mode: 0o700 }); - await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file)); + await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); } else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); else await taskProperties(ready, session!, campaignId, evidence); @@ -226,7 +226,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.status = "passed"; const evidenceDir = path.join(outputDir, "evidence", result.executionId, "attempt-1"); await mkdir(evidenceDir, { recursive: true, mode: 0o700 }); - await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}.png`)); + await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}.png`), deadlineAt); result.screenshots = [{ id, label: productJourneys.find(j => j.id === id)!.label, file: `${id}.png` }]; } catch (error) { const failure = journeyFailure(error); @@ -242,7 +242,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri if (url.origin === ready.api.origin && /^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname)) { const evidenceDir = path.join(outputDir, "evidence", result.executionId, "attempt-1"); await mkdir(evidenceDir, { recursive: true, mode: 0o700 }); - await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}-failed.png`)); + await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}-failed.png`), deadlineAt); result.screenshots = [{ id: `${id}-failed`, label: "Failed journey screen", file: `${id}-failed.png` }]; } } catch { /* Preserve the original failure when its page is unavailable. */ } diff --git a/tests/runner-e2e/product-journeys/product-screenshot.test.ts b/tests/runner-e2e/product-journeys/product-screenshot.test.ts index 9d1cfd911cb..37f079e6a4f 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.test.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.test.ts @@ -1,20 +1,63 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { Page } from "@playwright/test"; import { captureProductScreenshot } from "./product-screenshot.js"; +const origin = "https://qa.example.test"; +afterEach(() => vi.restoreAllMocks()); +function fixture(options: { mounted?: boolean; redirect?: string; afterMount?(): void } = {}) { + let mounted = false, url = `${origin}/QA/issues/QA-1`; + const screenshot = vi.fn(async (_input: { path: string; timeout: number }) => {}); + const waitFor = vi.fn(async (_input: { state: string; timeout: number }) => { mounted = options.mounted !== false; if (options.redirect) url = options.redirect; options.afterMount?.(); }); + const locator = vi.fn(() => ({ waitFor, isVisible: async () => mounted, screenshot })); + const page = { url: () => url, locator } as unknown as Page; + return { page, locator, waitFor, screenshot }; +} describe("product screenshot boundary", () => { - it("excludes a sign-in screen rendered at an unchanged task URL", async () => { - const screenshot = vi.fn(); - const page = { url: () => "https://qa.example.test/QA/issues/QA-1", locator: () => ({ isVisible: async () => false, screenshot }) } as unknown as Page; - await expect(captureProductScreenshot(page, "https://qa.example.test", "shot.png")).rejects.toThrow("product_screen_unavailable"); - expect(screenshot).not.toHaveBeenCalled(); + it("waits for Layout to mount after the document navigation completes", async () => { + const f = fixture(); + await captureProductScreenshot(f.page, origin, "shot.png"); + expect(f.locator).toHaveBeenCalledWith("main#main-content"); + expect(f.waitFor).toHaveBeenCalledWith({ state: "visible", timeout: expect.any(Number) }); + expect(f.screenshot).toHaveBeenCalledTimes(1); + expect(f.waitFor.mock.invocationCallOrder[0]).toBeLessThan(f.screenshot.mock.invocationCallOrder[0]!); }); - it("captures only the signed-in product region and refuses other origins", async () => { - const screenshot = vi.fn(), locator = vi.fn(() => ({ isVisible: async () => true, screenshot })); - const page = { url: () => "https://qa.example.test/QA/issues/QA-1", locator } as unknown as Page; - await captureProductScreenshot(page, "https://qa.example.test", "shot.png"); - expect(locator).toHaveBeenCalledWith("main#main-content"); expect(screenshot).toHaveBeenCalledTimes(1); - await expect(captureProductScreenshot(page, "https://other.example.test", "shot.png")).rejects.toThrow("product_screen_unavailable"); - expect(screenshot).toHaveBeenCalledTimes(1); + it("excludes a sign-in screen rendered at an unchanged task URL", async () => { + const f = fixture({ mounted: false }); + await expect(captureProductScreenshot(f.page, origin, "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("rejects a product region that never mounts", async () => { + const f = fixture(); f.waitFor.mockRejectedValueOnce(new Error("Timeout")); + await expect(captureProductScreenshot(f.page, origin, "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it.each([`${origin}/login`, "https://other.example.test/QA/issues/QA-1"])("excludes a redirect during readiness to %s", async redirect => { + const f = fixture({ redirect }); + await expect(captureProductScreenshot(f.page, origin, "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("refuses another origin before waiting or taking a screenshot", async () => { + const f = fixture(); + await expect(captureProductScreenshot(f.page, "https://other.example.test", "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.waitFor).not.toHaveBeenCalled(); expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("bounds readiness and capture by the campaign deadline", async () => { + vi.spyOn(Date, "now").mockReturnValue(1000); + const f = fixture(); + await captureProductScreenshot(f.page, origin, "shot.png", 1050); + for (const call of [f.waitFor.mock.calls[0], f.screenshot.mock.calls[0]]) { + expect((call?.[0] as { timeout: number }).timeout).toBeGreaterThan(0); + expect((call?.[0] as { timeout: number }).timeout).toBeLessThanOrEqual(50); + } + }); + it("does not capture after readiness consumes the remaining deadline", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1000), f = fixture({ afterMount: () => { now.mockReturnValue(1050); } }); + await expect(captureProductScreenshot(f.page, origin, "shot.png", 1050)).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("never disables a timeout when the campaign deadline has elapsed", async () => { + const f = fixture(); + await expect(captureProductScreenshot(f.page, origin, "shot.png", Date.now())).rejects.toThrow("product_screen_unavailable"); + expect(f.waitFor).not.toHaveBeenCalled(); expect(f.screenshot).not.toHaveBeenCalled(); }); }); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.ts b/tests/runner-e2e/product-journeys/product-screenshot.ts index 14ae3d0a9c8..f8387e447e2 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.ts @@ -1,12 +1,22 @@ import type { Page } from "@playwright/test"; import { JourneyError } from "./types.js"; -export async function captureProductScreenshot(page: Page, origin: string, file: string) { - const url = new URL(page.url()); - if (url.origin !== origin || !/^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname)) throw new JourneyError("failed", "product_screen_unavailable"); +export async function captureProductScreenshot(page: Page, origin: string, file: string, deadlineAt = Date.now() + 10_000) { + const permitted = () => { + const url = new URL(page.url()); + return url.origin === origin && /^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname); + }; + const timeout = (maximum: number) => { + const remaining = deadlineAt - Date.now(); + if (remaining <= 0) throw new JourneyError("failed", "product_screen_unavailable"); + return Math.min(maximum, remaining); + }; + if (!permitted() || deadlineAt <= Date.now()) throw new JourneyError("failed", "product_screen_unavailable"); // CloudAccessGate replaces Layout on expiry without necessarily changing the // URL. Capture Layout's main element, so auth screens can never be included. const product = page.locator("main#main-content"); - if (!await product.isVisible()) throw new JourneyError("failed", "product_screen_unavailable"); - await product.screenshot({ path: file, timeout: 2000 }); + try { await product.waitFor({ state: "visible", timeout: timeout(10_000) }); } + catch { throw new JourneyError("failed", "product_screen_unavailable"); } + if (!permitted() || !await product.isVisible() || deadlineAt <= Date.now()) throw new JourneyError("failed", "product_screen_unavailable"); + await product.screenshot({ path: file, timeout: timeout(2000) }); }