diff --git a/tests/perf/collections/README.md b/tests/perf/collections/README.md index 1b675f08be7..25a2e875003 100644 --- a/tests/perf/collections/README.md +++ b/tests/perf/collections/README.md @@ -41,6 +41,26 @@ They allow 45 seconds for its bounded retry window before exercising manual Retry. Other cases keep the normal short assertion timeout. Timing samples do not accelerate the browser clock. +`mine-workspaces.spec.ts` guards Mine against waiting on a company-wide +execution-workspace list. Its fixture (`workspace-fixture.ts`) adds a company +with 6,000 execution workspaces, 4,000 of them created for finished tasks, and +12 Mine tasks that each run in their own isolated workspace. The API cannot +create execution workspaces, so this fixture writes those rows to the instance +database. Set `PAPERCLIP_CONFIG` to the disposable instance's `config.json` +(embedded PostgreSQL), or `PAPERCLIP_COLLECTION_PERF_DATABASE_URL` to its +loopback database. The fixture is retained in +`test-results/collections/workspace-fixture.json` with the same 404 rule as the +main fixture. The spec routes isolated workspaces on for its own pages. It +checks that Mine requests only `ids`-filtered workspace summaries for the +workspaces its rows reference, that rows appear while those reads are held when +neither the Workspace column nor workspace grouping is active, and that the +Workspace column still shows each row's workspace name. Its timing case turns on +the real isolated-workspaces setting, restores it, and records cold time from +navigation start to the first Mine row (two frames after mount) over +`PAPERCLIP_COLLECTION_PERF_TIMING_RUNS` runs (default and minimum 5). It records +a native scenario without request interception and a separate diagnostic scenario that delays +workspace reads by 3 seconds. The timings are attached evidence, not budgets. + Run these focused browser checks before the full collection suite: ```sh diff --git a/tests/perf/collections/mine-workspaces.spec.ts b/tests/perf/collections/mine-workspaces.spec.ts new file mode 100644 index 00000000000..8b9b6709e01 --- /dev/null +++ b/tests/perf/collections/mine-workspaces.spec.ts @@ -0,0 +1,143 @@ +import { expect, test, type Page } from "@playwright/test"; +import { isolatedWorkspaceSettings, workspaceFixture, WORKSPACE_COUNT, type WorkspaceFixture } from "./workspace-fixture"; + +// Mine must not wait on a company-wide execution-workspace list. With thousands +// of workspaces that list was megabytes and seconds long, and Mine held every +// row hidden until it finished. These checks run against a company with +// WORKSPACE_COUNT workspaces and Mine rows that each reference their own. +let fixture: WorkspaceFixture; +test.beforeAll(async ({ request }) => { + test.setTimeout(300_000); + fixture = await workspaceFixture(request); +}); + +const workspaceListPath = () => `/api/companies/${fixture.company.id}/execution-workspaces`; +const minePage = () => `/${fixture.company.issuePrefix}/issues?view=mine`; +const mineLink = (page: Page, identifier: string) => page.locator(`[data-inbox-issue-link][href$="/${identifier}"]`); +const mineRow = (page: Page, identifier: string) => page.locator("[data-inbox-item]", { has: mineLink(page, identifier) }); +const showWorkspaceColumn = (page: Page) => page.addInitScript(() => { + localStorage.setItem("paperclip:inbox:issue-columns", JSON.stringify(["status", "id", "workspace", "updated"])); +}); +function recordWorkspaceReads(page: Page) { + const reads: URL[] = []; + page.on("request", request => { + const url = new URL(request.url()); + if (url.pathname === workspaceListPath()) reads.push(url); + }); + return reads; +} +function holdWorkspaceReads(page: Page) { + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let pending = 0; + const ready = page.route(url => url.pathname === workspaceListPath(), async route => { + pending += 1; + await held; + await route.continue(); + }); + return { ready, release, pending: () => pending }; +} + +test("Mine reads only the execution workspaces its rows reference", async ({ page }) => { + await isolatedWorkspaceSettings(page); + const reads = recordWorkspaceReads(page); + await page.goto(minePage()); + for (const row of fixture.mine) await expect(mineLink(page, row.identifier)).toBeVisible(); + await expect.poll(() => reads.length).toBeGreaterThan(0); + // Late reads (background refetches) must follow the same rule. + await page.waitForTimeout(1_000); + const unfiltered = reads.filter(url => !url.searchParams.has("ids")).map(url => `${url.pathname}${url.search}`); + expect(unfiltered, `Mine requested the company-wide list of ${WORKSPACE_COUNT} workspaces`).toEqual([]); + const expected = fixture.mine.map(row => row.workspaceId).sort(); + expect([...new Set(reads.flatMap(url => url.searchParams.get("ids")!.split(",")))].sort()).toEqual(expected); + for (const url of reads) { + expect(url.searchParams.get("summary")).toBe("true"); + expect(url.searchParams.get("ids")!.split(",").every(id => expected.includes(id))).toBe(true); + } +}); + +test("Mine rows appear while workspace reads are pending when no workspace data is shown", async ({ page }) => { + await isolatedWorkspaceSettings(page); + const hold = holdWorkspaceReads(page); + await hold.ready; + try { + await page.goto(minePage()); + for (const row of fixture.mine) { + await expect(mineLink(page, row.identifier), "Mine rows waited for the workspace read").toBeVisible({ timeout: 15_000 }); + } + expect(hold.pending()).toBeGreaterThan(0); + } finally { hold.release(); } +}); + +test("the Workspace column resolves each Mine row's isolated workspace name", async ({ page }) => { + await isolatedWorkspaceSettings(page); + await showWorkspaceColumn(page); + await page.goto(minePage()); + for (const row of fixture.mine) await expect(mineRow(page, row.identifier)).toContainText(row.workspaceName); +}); + +// Cold time to first Mine row, from navigation start (two frames after the +// first row mounts, as in measure-mine-context.ts), in fresh browser contexts +// with the real instance setting. The native scenario uses no request +// interception. The delayed scenario holds every workspace read for +// WORKSPACE_DELAY_MS and is diagnostic evidence only: it shows whether rows wait +// on that read. Neither is a budget; the structural checks above are the guard. +const WORKSPACE_DELAY_MS = 3_000; +type Sample = { timeToRowsMs: number; workspaceReads: string[] }; +const median = (values: number[]) => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; +}; + +test("records cold time to Mine rows", async ({ browser, request, baseURL }, testInfo) => { + const runs = Number(process.env.PAPERCLIP_COLLECTION_PERF_TIMING_RUNS ?? 5); + expect(Number.isInteger(runs) && runs >= 5, "Record at least five samples").toBe(true); + test.setTimeout(60_000 + runs * 60_000); + const settingsUrl = "/api/instance/settings/experimental"; + const original = await (await request.get(settingsUrl)).json() as { enableIsolatedWorkspaces: boolean }; + const enable = await request.patch(settingsUrl, { data: { enableIsolatedWorkspaces: true } }); + expect(enable.ok(), await enable.text()).toBeTruthy(); + const sample = async (delayMs: number): Promise => { + const context = await browser.newContext({ baseURL, serviceWorkers: "block", viewport: { width: 1440, height: 900 } }); + try { + const page = await context.newPage(); + const reads = recordWorkspaceReads(page); + if (delayMs > 0) await page.route(url => url.pathname === workspaceListPath(), async route => { + await new Promise(resolve => setTimeout(resolve, delayMs)); + await route.continue(); + }); + await page.addInitScript(() => { + const state = window as unknown as { __mineRowsAt: number | null }; + state.__mineRowsAt = null; + const observer = new MutationObserver(() => { + if (!document.querySelector("[data-inbox-item]")) return; + observer.disconnect(); + requestAnimationFrame(() => requestAnimationFrame(() => { state.__mineRowsAt = performance.now(); })); + }); + observer.observe(document, { subtree: true, childList: true }); + }); + await page.goto(minePage()); + await page.waitForFunction(() => (window as unknown as { __mineRowsAt: number | null }).__mineRowsAt !== null, null, { timeout: 30_000 }); + const timeToRowsMs = Math.round(await page.evaluate(() => (window as unknown as { __mineRowsAt: number }).__mineRowsAt)); + await expect(mineLink(page, fixture.mine[0]!.identifier)).toBeVisible(); + return { timeToRowsMs, workspaceReads: reads.map(url => url.search) }; + } finally { await context.close(); } + }; + const scenarios = [ + { name: "native loopback, no interception", delayMs: 0, samples: [] as Sample[] }, + { name: `workspace reads delayed ${WORKSPACE_DELAY_MS} ms (diagnostic, intercepted)`, delayMs: WORKSPACE_DELAY_MS, samples: [] as Sample[] }, + ]; + try { + for (const scenario of scenarios) for (let run = 0; run < runs; run++) scenario.samples.push(await sample(scenario.delayMs)); + } finally { + const restore = await request.patch(settingsUrl, { data: { enableIsolatedWorkspaces: original.enableIsolatedWorkspaces } }); + expect(restore.ok(), await restore.text()).toBeTruthy(); + } + const report = scenarios.map(({ name, samples }) => { + const times = samples.map(entry => entry.timeToRowsMs); + console.log(`Mine cold time to rows, ${name}: median ${median(times)} ms over ${runs} runs (${[...times].sort((a, b) => a - b).join(", ")} ms)`); + return { scenario: name, workspaces: WORKSPACE_COUNT, mineRows: fixture.mine.length, medianMs: median(times), samples }; + }); + await testInfo.attach("mine-workspace-scale-timing", { contentType: "application/json", body: Buffer.from(JSON.stringify(report, null, 2)) }); +}); diff --git a/tests/perf/collections/workspace-fixture.ts b/tests/perf/collections/workspace-fixture.ts new file mode 100644 index 00000000000..c811f6606b0 --- /dev/null +++ b/tests/perf/collections/workspace-fixture.ts @@ -0,0 +1,132 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { expect, type APIRequestContext, type Page } from "@playwright/test"; +import { sql } from "../../../server/node_modules/drizzle-orm/index.js"; +import { closeRegisteredClients, createDb, executionWorkspaces, issues, projects } from "../../../packages/db/src/index.js"; + +// A company shaped like a long-lived instance: thousands of execution +// workspaces, most of them created for (now finished) tasks, and a few Mine +// rows that each run in their own isolated workspace. The API has no route to +// create execution workspaces, so the bulk rows are written to the disposable +// instance database directly. Mine rows are created through the API so Mine +// selects them exactly as it selects any board-created task. +export const WORKSPACE_COUNT = 6_000; +export const LINKED_WORKSPACE_COUNT = 4_000; +export const MINE_ROW_COUNT = 12; +export type WorkspaceFixture = { + company: { id: string; issuePrefix: string }; + mine: Array<{ id: string; identifier: string; workspaceId: string; workspaceName: string }>; +}; +export const workspaceFixturePath = "test-results/collections/workspace-fixture.json"; +const LOOPBACK_HOSTS = ["127.0.0.1", "localhost", "[::1]", "::1"]; + +async function databaseUrl() { + let url = process.env.PAPERCLIP_COLLECTION_PERF_DATABASE_URL; + if (!url && process.env.PAPERCLIP_CONFIG) { + const config = JSON.parse(await fs.readFile(process.env.PAPERCLIP_CONFIG, "utf8")); + url = config.database.mode === "postgres" ? config.database.connectionString + : `postgres://paperclip:paperclip@127.0.0.1:${(await fs.readFile( + path.join(config.database.embeddedPostgresDataDir, "postmaster.pid"), "utf8")).split("\n")[3]}/paperclip`; + } + if (!url) { + throw new Error("Workspace-scale checks seed execution workspaces directly. Set PAPERCLIP_COLLECTION_PERF_DATABASE_URL " + + "to the disposable instance database, or PAPERCLIP_CONFIG to its config.json."); + } + if (!LOOPBACK_HOSTS.includes(new URL(url).hostname)) throw new Error("Workspace-scale seeding only supports a loopback database."); + return url; +} + +async function seed(request: APIRequestContext): Promise { + const created = await request.post("/api/companies", { data: { name: "Mine workspace scale fixture" } }); + expect(created.ok(), await created.text()).toBeTruthy(); + const company = await created.json() as { id: string; issuePrefix: string }; + const preference = await request.put(`/api/companies/${company.id}/workspace-preferences/me`, { + data: { coordinatorWorkspaceEnabled: true }, + }); + expect(preference.ok(), await preference.text()).toBeTruthy(); + const mineTasks: Array<{ id: string; identifier: string; issueNumber: number }> = []; + for (let index = 0; index < MINE_ROW_COUNT; index++) { + const response = await request.post(`/api/companies/${company.id}/issues`, { data: { + title: `Workspace-scale Mine task ${index}`, status: "todo", + } }); + expect(response.ok(), await response.text()).toBeTruthy(); + mineTasks.push(await response.json()); + } + + const url = await databaseUrl(); + const db = createDb(url); + try { + const projectId = randomUUID(); + await db.insert(projects).values({ id: projectId, companyId: company.id, name: "Workspace scale project", status: "in_progress" }); + const firstNumber = Math.max(...mineTasks.map(task => task.issueNumber)) + 1; + const now = Date.now(); + // Finished tasks, each with the workspace it ran in. The remaining + // workspaces have no source task (reused, imported or orphaned). + const sourceTasks = Array.from({ length: LINKED_WORKSPACE_COUNT }, (_, index) => ({ + id: randomUUID(), companyId: company.id, projectId, title: `Finished workspace task ${index}`, status: "done", + issueNumber: firstNumber + index, identifier: `${company.issuePrefix}-${firstNumber + index}`, + createdAt: new Date(now - (index + 60) * 60_000), updatedAt: new Date(now - (index + 60) * 60_000), + })); + const workspace = (index: number, sourceIssueId: string | null, name = `scale-ws-${index}`) => ({ + id: randomUUID(), companyId: company.id, projectId, sourceIssueId, mode: "isolated_workspace", strategyType: "git_worktree", + providerType: "git_worktree", name, status: index % 3 === 0 ? "idle" : "archived", cwd: `/tmp/collections-perf/${name}`, + branchName: `perf/${name}`, lastUsedAt: new Date(now - (index + 60) * 60_000), + metadata: sourceIssueId ? { _issuePrivacySources: { [sourceIssueId]: true } } : null, + }); + const bulk = Array.from({ length: WORKSPACE_COUNT - MINE_ROW_COUNT }, (_, index) => + workspace(index, sourceTasks[index]?.id ?? null)); + const mineWorkspaces = mineTasks.map((task, index) => ({ + ...workspace(index, task.id, `mine-isolated-${index}`), status: "active", lastUsedAt: new Date(now - index * 1_000), + })); + for (let index = 0; index < sourceTasks.length; index += 500) await db.insert(issues).values(sourceTasks.slice(index, index + 500)); + const allWorkspaces = [...mineWorkspaces, ...bulk]; + for (let index = 0; index < allWorkspaces.length; index += 500) { + await db.insert(executionWorkspaces).values(allWorkspaces.slice(index, index + 500)); + } + await db.execute(sql`update issues set execution_workspace_id = w.id + from execution_workspaces w where w.source_issue_id = issues.id and w.company_id = ${company.id}`); + await db.execute(sql`update companies set issue_counter = ${firstNumber + LINKED_WORKSPACE_COUNT - 1} where id = ${company.id}`); + await db.execute(sql`analyze issues`); + await db.execute(sql`analyze execution_workspaces`); + const [counts] = await db.execute(sql`select + (select count(*)::int from execution_workspaces where company_id = ${company.id}) as workspaces, + (select count(*)::int from issues where company_id = ${company.id} and execution_workspace_id is not null) as linked`) as unknown as Array<{ workspaces: number; linked: number }>; + expect(counts).toEqual({ workspaces: WORKSPACE_COUNT, linked: LINKED_WORKSPACE_COUNT + MINE_ROW_COUNT }); + return { + company: { id: company.id, issuePrefix: company.issuePrefix }, + mine: mineTasks.map((task, index) => ({ + id: task.id, identifier: task.identifier, workspaceId: mineWorkspaces[index]!.id, workspaceName: mineWorkspaces[index]!.name, + })), + }; + } finally { + await closeRegisteredClients(url); + } +} + +// Retained like the main collection fixture: reused while its company exists, +// replaced only when the company is gone (404). Other errors fail. +export async function workspaceFixture(request: APIRequestContext): Promise { + const saved = await fs.readFile(workspaceFixturePath, "utf8").then(JSON.parse).catch(error => { + if (error.code !== "ENOENT") throw error; + return null; + }) as WorkspaceFixture | null; + if (saved) { + const existing = await request.get(`/api/companies/${saved.company.id}`); + if (existing.ok()) return saved; + expect(existing.status(), await existing.text()).toBe(404); + } + const fixture = await seed(request); + await fs.mkdir(path.dirname(workspaceFixturePath), { recursive: true }); + await fs.writeFile(workspaceFixturePath, JSON.stringify(fixture)); + return fixture; +} + +// Turns on isolated workspaces for this page only, so other specs keep the +// instance's real feature settings. +export async function isolatedWorkspaceSettings(page: Page) { + await page.route("**/api/instance/settings/experimental", async route => { + const response = await route.fetch(); + await route.fulfill({ json: { ...await response.json(), enableIsolatedWorkspaces: true } }); + }); +}