/** * Disposable synthetic PostgreSQL + real Git proof. Never accepts DATABASE_URL. * pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-workspace-preparation-history.ts \ * --baseline-ref=df80616d98119cbf5abd40549794d9980a5fabf6 --output=/tmp/workspace-history.json */ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { createRequire } from "node:module"; import { fileURLToPath, pathToFileURL } from "node:url"; import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; import { addReadyWorkspaceHistoryRepository, createWorkspacePreparationHistoryFixture, } from "../server/src/__tests__/helpers/workspace-preparation-history-fixture.js"; import * as treatment from "../server/src/services/execution-workspace-repositories.js"; const require = createRequire(new URL("../server/package.json", import.meta.url)); const { drizzle } = require("drizzle-orm/postgres-js"); const argument = (name: string) => process.argv.find(value => value.startsWith(`--${name}=`))?.slice(name.length + 3); const baselineRef = argument("baseline-ref") ?? "df80616d98119cbf5abd40549794d9980a5fabf6"; const baselineRevision = execFileSync("git", ["rev-parse", `${baselineRef}^{commit}`], { encoding: "utf8" }).trim(); const treatmentRevision = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); const samples = Number(argument("samples") ?? 5), warmups = Number(argument("warmups") ?? 2); assert.ok(Number.isInteger(samples) && samples >= 5, "At least five measured samples are required"); assert.ok(Number.isInteger(warmups) && warmups >= 2, "At least two warmups are required"); const selectedSizes = (argument("sizes") ?? "small,stress").split(","); assert.ok(selectedSizes.length && selectedSizes.every(size => size === "small" || size === "stress")); process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce"; const repositoryRoot = fileURLToPath(new URL("../", import.meta.url)); const originalSourceRoot = path.join(repositoryRoot, "server", "src"); const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-history-")); const baselineDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-history-baseline-")); const keepAlive = setInterval(() => {}, 1000); let database: Awaited> | undefined; let db: Db | undefined; type Fixture = Awaited>; type Observation = { ms: number; queries: string[]; queryFingerprints: string[]; realpaths: string[]; retiredTableQueries: number; historicalReaderQueries: number; companyInventories: number; }; async function sourceFiles(directory: string): Promise { const entries = await fs.readdir(directory, { withFileTypes: true }); return (await Promise.all(entries.map(entry => entry.isDirectory() ? sourceFiles(path.join(directory, entry.name)) : Promise.resolve([path.join(directory, entry.name)])))).flat(); } async function loadBaseline() { // Snapshot the complete relative server dependency graph. Holding just the top-level old // service while importing a changed authorization helper would invalidate the comparison. const archive = execFileSync("git", ["archive", "--format=tar", baselineRevision, "server/src", "server/package.json"], { cwd: repositoryRoot, maxBuffer: 256 * 1024 * 1024, }); execFileSync("tar", ["-xf", "-", "-C", baselineDirectory], { input: archive }); // Resolve unchanged workspace dependencies without writing a checkout or package-manager files. await fs.symlink(path.join(repositoryRoot, "server", "node_modules"), path.join(baselineDirectory, "node_modules"), "dir"); const snapshotSource = path.join(baselineDirectory, "server", "src"); for (const file of await sourceFiles(snapshotSource)) { if (!/\.[cm]?[jt]s$/.test(file)) continue; const original = path.join(originalSourceRoot, path.relative(snapshotSource, file)); const source = await fs.readFile(file, "utf8"); const rewritten = source.replace(/((?:from\s*|import\s*\(|import\s*|new\s+URL\s*\(\s*)["'])(\.{1,2}\/[^"']+)(["'])/g, (match, prefix: string, specifier: string, suffix: string) => { const destination = path.resolve(path.dirname(original), specifier); return destination.startsWith(`${originalSourceRoot}${path.sep}`) ? match : `${prefix}${destination}${suffix}`; }); if (source !== rewritten) await fs.writeFile(file, rewritten); } return import(pathToFileURL(path.join(snapshotSource, "services", "execution-workspace-repositories.ts")).href); } async function observe(action: (recorded: Db) => Promise, prepare?: (recorded: Db) => Promise<() => Promise>): Promise { assert.ok(db); const queries: string[] = [], realpaths: string[] = []; const recorded = drizzle(db.$client, { schema: db._.fullSchema, logger: { logQuery: (query: string) => queries.push(query) } }) as Db; const actualRealpath = fs.realpath; fs.realpath = (async (...args: Parameters) => { realpaths.push(String(args[0])); return actualRealpath(...args); }) as typeof fs.realpath; let start = performance.now(); try { const prepared = prepare ? await prepare(recorded) : undefined; queries.length = 0; realpaths.length = 0; start = performance.now(); if (prepared) await prepared(); else await action(recorded); } finally { fs.realpath = actualRealpath; } return { ms: performance.now() - start, queries, realpaths, queryFingerprints: [...new Set(queries.map(query => createHash("sha256").update(query).digest("hex")))], retiredTableQueries: queries.filter(query => /execution_workspace_repository_(?:access|roots)/.test(query)).length, historicalReaderQueries: queries.filter(query => /from "heartbeat_runs"/.test(query) && /\bin\s*\(/i.test(query)).length, companyInventories: queries.filter(query => /from "(?:execution_workspaces|project_workspaces)"/.test(query) && !/"(?:execution_workspaces|project_workspaces)"\."id"\s*=/.test(query)).length, }; } function stats(observations: Observation[]) { const ordered = observations.map(item => item.ms).sort((a, b) => a - b); return { samplesMs: observations.map(item => item.ms), medianMs: ordered[Math.floor(ordered.length / 2)], minMs: ordered[0], maxMs: ordered.at(-1), queryCounts: observations.map(item => item.queries.length), realpathCounts: observations.map(item => item.realpaths.length), retiredTableQueries: observations.map(item => item.retiredTableQueries), historicalReaderQueries: observations.map(item => item.historicalReaderQueries), companyInventories: observations.map(item => item.companyInventories), realpaths: [...new Set(observations.flatMap(item => item.realpaths))], sql: [...new Set(observations.flatMap(item => item.queries))], queryFingerprints: [...new Set(observations.flatMap(item => item.queryFingerprints))] }; } try { const baseline = await loadBaseline(); database = await startEmbeddedPostgresTestDatabase("workspace-preparation-history-benchmark-"); db = createDb(database.connectionString, { maxConnections: 5 }); await db.$client.unsafe("set statement_timeout = '120s'"); const results: Array<{ size: string; phase: string; baseline: ReturnType; treatment: ReturnType }> = []; const fixtureSummary = []; const sqlPlans: Array<{ size: string; phase: string; sql: string; plan: unknown }> = []; for (const size of selectedSizes) { const config = size === "stress" ? { workspaces: 8500, sharedRoots: 4800, sources: 5300, readers: 50 } : { workspaces: 8, sharedRoots: 5, sources: 6, readers: 2 }; const fixtures: Fixture[] = []; for (const variant of ["baseline", "treatment"]) { console.error(`Seeding ${size} ${variant}: ${config.workspaces} historical workspaces`); fixtures.push(await createWorkspacePreparationHistoryFixture(db, path.join(root, `${size}-${variant}`), config)); } fixtureSummary.push({ size, ...config, principals: 8, crossCompanyDecoys: 2 }); const modules = [baseline, treatment]; const phase = async (name: string, build: (module: any, recorded: Db, fixture: Fixture) => Promise, prepare?: (module: any, recorded: Db, fixture: Fixture) => Promise<() => Promise>) => { const observations: Observation[][] = [[], []]; for (let iteration = 0; iteration < warmups + samples; iteration++) { for (const variant of iteration % 2 ? [1, 0] : [0, 1]) { const fixture = fixtures[variant]; const observation = await observe(recorded => build(modules[variant], recorded, fixture), prepare ? recorded => prepare(modules[variant], recorded, fixture) : undefined); if (variant === 1) { assert.equal(observation.retiredTableQueries, 0, `${name} queried retired root tables`); assert.equal(observation.historicalReaderQueries, 0, `${name} loaded historical readers`); assert.equal(observation.companyInventories, 0, `${name} inventoried company workspaces`); assert.ok(observation.realpaths.every(cwd => cwd === fixture.cwd || cwd.startsWith(`${fixture.cwd}${path.sep}`)), `${name} canonicalized an unrelated path`); } if (iteration >= warmups) observations[variant].push(observation); } console.error(`${size} ${name}: ${iteration + 1}/${warmups + samples}`); } results.push({ size, phase: name, baseline: stats(observations[0]), treatment: stats(observations[1]) }); }; await phase("initial-local-source-guard", async (module, recorded, fixture) => { const service = module.executionWorkspaceRepositoryService(recorded); const method = service.prepareConfiguredRepositoryCopy ?? service.prepareConfiguredRepositorySource; assert.equal(typeof await method({ ...fixture.input, sourceCwd: fixture.cwd }), "function"); }); await phase("before-copy-revalidation", async () => {}, async (module, recorded, fixture) => { const service = module.executionWorkspaceRepositoryService(recorded); return (service.prepareConfiguredRepositoryCopy ?? service.prepareConfiguredRepositorySource) ({ ...fixture.input, sourceCwd: fixture.cwd }); }); await phase("publication-guard", async (module, recorded, fixture) => { const service = module.executionWorkspaceRepositoryService(recorded); await (service.assertPublicationAllowed ?? service.guardForPublication)(fixture.input); }); await phase("empty-repository-admission", async (module, recorded, fixture) => { assert.deepEqual(await module.executionWorkspaceRepositoryService(recorded).prepareForAdmission(fixture.input), []); }); await phase("recovery-binding", async (module, recorded, fixture) => { const service = module.executionWorkspaceRepositoryService(recorded); await (service.assertRecoveryBinding ?? service.reserveForRecovery)(fixture.input); }); const repositories = []; for (const fixture of fixtures) repositories.push(await addReadyWorkspaceHistoryRepository(db, fixture, { matchingHistoricalReceipts: true })); await db.$client.unsafe("ANALYZE"); await phase("ready-public-repository-admission-and-warm-reuse", async (module, recorded, fixture) => { const rows = await module.executionWorkspaceRepositoryService(recorded).prepareForAdmission(fixture.input); const expected = repositories[fixtures.indexOf(fixture)]; assert.deepEqual(rows, [{ id: expected.row.id, cwd: expected.checkout, repoUrl: expected.row.repoUrl, relativePath: expected.row.relativePath, pinnedCommit: fixture.pin, branchName: "main" }]); assert.equal(await fs.readFile(path.join(rows[0].cwd, "result.txt"), "utf8"), "Synthetic task workspace\n"); }); // Capture actual baseline permission SQL with its original parameters outside timing samples. const captures: Array<{ query: string; params: unknown[] }> = []; const recorded = drizzle(db.$client, { schema: db._.fullSchema, logger: { logQuery: (query: string, params: unknown[]) => captures.push({ query, params }) } }) as Db; await baseline.executionWorkspaceRepositoryService(recorded).guardForPublication(fixtures[0].input); const expensive = captures.find(item => item.query.includes('from "execution_workspaces"') && item.query.includes("jsonb_object_keys")); if (expensive) { const [plan] = await db.$client.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${expensive.query}`, expensive.params as never[]); sqlPlans.push({ size, phase: "baseline-workspace-disclosure-query", sql: expensive.query, plan: plan["QUERY PLAN"] }); } } if (selectedSizes.includes("small") && selectedSizes.includes("stress")) { for (const small of results.filter(result => result.size === "small")) { const stress = results.find(result => result.size === "stress" && result.phase === small.phase)!; assert.deepEqual(stress.treatment.queryCounts, small.treatment.queryCounts, `${small.phase}: SQL count grew with peer history`); assert.deepEqual(stress.treatment.realpathCounts, small.treatment.realpathCounts, `${small.phase}: canonicalization count grew with peer history`); } } const guardPhases = results.filter(result => result.size === "stress" && result.phase !== "ready-public-repository-admission-and-warm-reuse"); const baselineGuardMs = guardPhases.reduce((sum, result) => sum + result.baseline.medianMs, 0); const treatmentGuardMs = guardPhases.reduce((sum, result) => sum + result.treatment.medianMs, 0); const improvementPercent = guardPhases.length ? (1 - treatmentGuardMs / baselineGuardMs) * 100 : null; if (improvementPercent !== null) assert.ok(improvementPercent >= 90, `Guard improvement ${improvementPercent.toFixed(2)}% missed 90% target`); const report = { baselineRevision, treatmentRevision, treatmentWorkingDiff: execFileSync("git", ["diff", "--stat"], { encoding: "utf8" }), node: process.version, platform: process.platform, arch: process.arch, postgres: (await db.$client.unsafe("select version()"))[0].version, databaseSettings: await db.$client.unsafe("select current_setting('shared_buffers') as shared_buffers, current_setting('work_mem') as work_mem"), warmups, samples, fixtureSummary, baselineIsolation: "Complete pinned server/src snapshot; current workspace package dependencies and schema held constant", contract: "Both versions succeed on granted synthetic history; privacy-denial behavior changes are covered by the integration suite", timingScope: "Actual repository service guards/admission with ready local Git receipts; excludes network acquisition, provider startup, and copying", performance: { baselineGuardMs, treatmentGuardMs, improvementPercent }, results, sqlPlans }; const output = JSON.stringify(report, null, 2); if (argument("output")) await fs.writeFile(path.resolve(argument("output")!), `${output}\n`); console.log(output); } finally { try { await db?.$client.end({ timeout: 0 }); await database?.cleanup(); await fs.rm(root, { recursive: true, force: true }); await fs.rm(baselineDirectory, { recursive: true, force: true }); } finally { clearInterval(keepAlive); } }