diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3425955fed7..6fc8fa4f267 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -531,6 +531,35 @@ verifying Vite/HMR behavior. For normal issue work, start with the smallest targeted check that proves the change. Reserve repo-wide typecheck/build/test runs for PR-ready handoff or changes broad enough that narrow checks do not cover the risk. +#### Embedded Postgres template cache + +Suites that call `startEmbeddedPostgresTestDatabase` (from `@paperclipai/db`) +get their own embedded Postgres cluster. The first such suite on a host builds +one fully migrated cluster and publishes its data directory under +`$TMPDIR/paperclip-embedded-postgres-templates-/`. Every later suite +copies that template and starts Postgres on the copy instead of running +`initdb` and all migrations again, and the `getEmbeddedPostgresTestSupport` +probe reuses it instead of starting a throwaway cluster. The key hashes the +embedded-postgres version, the initdb flags, and every migration file, so a +changed migration builds a new template. Each suite still runs +`applyPendingMigrations` on its copy, which is a no-op when the template is +current. Templates for other keys are removed after seven days without use. + +A template is a Postgres data directory, and starting it executes its +configuration, so the cache is private to the current OS user: the root is +created with mode `0700` and is only used when it is a real directory (not a +symlink) owned by the current user with no group or world write bit, and each +template is checked the same way before it is copied. A root that fails those +checks (for example one pre-created by another user in a shared `/tmp`) is left +alone and every suite in that process uses a fresh cluster. + +Set `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` to bypass the cache and give every +suite a fresh `initdb`, for example when debugging cluster startup itself. A +failure on the template path (root checks, build, copy, start, or publish) +falls back to the fresh path automatically, so the cache can make a suite +slower but never fail it or skip it: a migration that cannot apply still fails +the suite on the fresh path. + ### Task search evaluation The task search relevance rubric and regression corpus are documented in diff --git a/packages/db/src/test-embedded-postgres.test.ts b/packages/db/src/test-embedded-postgres.test.ts index 1d4f78c797d..aeec953d73d 100644 --- a/packages/db/src/test-embedded-postgres.test.ts +++ b/packages/db/src/test-embedded-postgres.test.ts @@ -1,9 +1,21 @@ import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import postgres from "postgres"; import { afterEach, describe, expect, it } from "vitest"; +import { inspectMigrations } from "./client.js"; import { __embeddedPostgresStartMaxAttemptsForTests as MAX_ATTEMPTS, + __embeddedPostgresTemplateRootDirForTests as templateRootDir, + __isUsableEmbeddedPostgresTemplateForTests as isUsableTemplate, + __lastEmbeddedPostgresStartModeForTests as lastStartMode, + __resetEmbeddedPostgresTemplateStateForTests as resetTemplateState, __setEmbeddedPostgresCtorProviderForTests, __startEmbeddedPostgresWithRetryForTests as startWithRetry, + computeEmbeddedPostgresTemplateKey, + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, } from "./test-embedded-postgres.js"; // A fake embedded-postgres constructor. It records every constructed instance so @@ -111,3 +123,371 @@ describe("startEmbeddedPostgresWithRetry", () => { expect((error as Error).message).toContain("Address already in use"); }); }); + +describe("computeEmbeddedPostgresTemplateKey", () => { + function writeMigrations(dir: string, files: Record) { + fs.mkdirSync(path.join(dir, "meta"), { recursive: true }); + fs.writeFileSync(path.join(dir, "meta", "_journal.json"), JSON.stringify({ entries: Object.keys(files) })); + for (const [name, content] of Object.entries(files)) { + fs.writeFileSync(path.join(dir, name), content); + } + } + + it("is stable for identical inputs and changes with any migration, flag, or version", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-key-")); + try { + const a = path.join(root, "a"); + const b = path.join(root, "b"); + writeMigrations(a, { "0000_one.sql": "create table one();", "0001_two.sql": "create table two();" }); + writeMigrations(b, { "0000_one.sql": "create table one();", "0001_two.sql": "create table two();" }); + const fixed = { packageVersion: "18.1.0", initdbFlags: ["--locale=C"], user: "paperclip" }; + + const keyA = computeEmbeddedPostgresTemplateKey(a, fixed); + expect(keyA).toMatch(/^[0-9a-f]{24}$/); + expect(computeEmbeddedPostgresTemplateKey(a, fixed)).toBe(keyA); + expect(computeEmbeddedPostgresTemplateKey(b, fixed)).toBe(keyA); + + // Content change, new file, flags, version, and user each change the key. + fs.writeFileSync(path.join(b, "0001_two.sql"), "create table two(id int);"); + const contentChanged = computeEmbeddedPostgresTemplateKey(b, fixed); + expect(contentChanged).not.toBe(keyA); + fs.writeFileSync(path.join(b, "0002_three.sql"), "create table three();"); + expect(computeEmbeddedPostgresTemplateKey(b, fixed)).not.toBe(contentChanged); + expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, initdbFlags: ["--locale=C", "--encoding=UTF8"] })).not.toBe(keyA); + expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, packageVersion: "18.2.0" })).not.toBe(keyA); + expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, user: "other" })).not.toBe(keyA); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("hashes the real migrations directory of this package", () => { + const key = computeEmbeddedPostgresTemplateKey(); + expect(key).toMatch(/^[0-9a-f]{24}$/); + expect(computeEmbeddedPostgresTemplateKey()).toBe(key); + }); +}); + +const describeOnPosix = process.platform === "win32" ? describe.skip : describe; + +describe("isUsableTemplate", () => { + let root: string; + + function writeCluster(dir: string): void { + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(dir, "PG_VERSION"), "18\n"); + } + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + it("accepts a private directory that holds a cleanly stopped cluster", () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-")); + const template = path.join(root, "template"); + writeCluster(template); + expect(isUsableTemplate(template)).toBe(true); + + // A running or crashed cluster leaves `postmaster.pid` behind. + fs.writeFileSync(path.join(template, "postmaster.pid"), "1\n"); + expect(isUsableTemplate(template)).toBe(false); + fs.rmSync(path.join(template, "postmaster.pid")); + expect(isUsableTemplate(template)).toBe(true); + + // A directory without a cluster, or no directory at all. + fs.rmSync(path.join(template, "PG_VERSION")); + expect(isUsableTemplate(template)).toBe(false); + expect(isUsableTemplate(path.join(root, "missing"))).toBe(false); + }); + + it("rejects a symbolic link even when it points at a usable cluster", () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-")); + const template = path.join(root, "template"); + writeCluster(template); + const link = path.join(root, "link"); + fs.symlinkSync(template, link, "dir"); + expect(isUsableTemplate(template)).toBe(true); + expect(isUsableTemplate(link)).toBe(false); + }); + + describeOnPosix("on POSIX", () => { + it("rejects a cluster directory that group or others can write", () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-")); + const template = path.join(root, "template"); + writeCluster(template); + for (const mode of [0o720, 0o702, 0o777]) { + fs.chmodSync(template, mode); + expect(isUsableTemplate(template), `mode ${mode.toString(8)}`).toBe(false); + } + // Read bits for others are not a write path; Postgres narrows the copy + // to 0700 before it starts anyway. + fs.chmodSync(template, 0o755); + expect(isUsableTemplate(template)).toBe(true); + fs.chmodSync(template, 0o700); + expect(isUsableTemplate(template)).toBe(true); + }); + }); +}); + +// Real-cluster coverage of the template path. Skipped on hosts that cannot run +// embedded Postgres, like every other suite that uses the fixture. +const support = await getEmbeddedPostgresTestSupport(); +const describeWithPostgres = support.supported ? describe : describe.skip; + +// Points `os.tmpdir()` at a private scratch directory so a test can drive the +// cache root from an empty state. `os.tmpdir()` reads the environment on every +// call: TMPDIR on POSIX, TEMP and then TMP on Windows, so all three are set. +// Returns the scratch directory; `restore` puts the environment and the +// fixture's cached state back. +const TMPDIR_VARIABLES = ["TMPDIR", "TEMP", "TMP"] as const; + +function useScratchTmpdir(prefix: string): { scratch: string; restore(): void } { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + const previous = new Map(TMPDIR_VARIABLES.map((name) => [name, process.env[name]] as const)); + for (const name of TMPDIR_VARIABLES) process.env[name] = scratch; + return { + scratch, + restore() { + for (const [name, value] of previous) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + resetTemplateState(); + fs.rmSync(scratch, { recursive: true, force: true }); + }, + }; +} + +describeWithPostgres("startEmbeddedPostgresTestDatabase template path", () => { + const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + + afterEach(() => { + if (originalTemplateSetting === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting; + }); + + it("publishes a complete template and starts suites from a copy of it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + // This test covers the template path, so it enables the cache even when + // the suite runs under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0. The first + // start then builds the template if the support probe did not. + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-copy-"); + try { + expect(lastStartMode()).toBe("template"); + // The copy is fully migrated and independent of the template. + const state = await inspectMigrations(database.connectionString); + expect(state.status).toBe("upToDate"); + const sql = postgres(database.connectionString, { max: 1 }); + try { + await sql`create table paperclip_template_copy_probe (id int)`; + } finally { + await sql.end(); + } + } finally { + await database.cleanup(); + } + + const rootDir = templateRootDir(); + const templateDir = path.join(rootDir, computeEmbeddedPostgresTemplateKey()); + expect(fs.existsSync(path.join(templateDir, "PG_VERSION"))).toBe(true); + expect(fs.existsSync(path.join(templateDir, "postmaster.pid"))).toBe(false); + expect(isUsableTemplate(templateDir)).toBe(true); + if (process.platform !== "win32") { + // The root is private to this user, and the template is a data + // directory Postgres accepts: both are 0700. + expect(path.basename(rootDir)).toBe(`paperclip-embedded-postgres-templates-${process.getuid?.()}`); + expect(fs.statSync(rootDir).mode & 0o777).toBe(0o700); + expect(fs.statSync(templateDir).mode & 0o777).toBe(0o700); + } + // Writing to the copy never touches the template, so a second copy does + // not see the table. + const second = await startEmbeddedPostgresTestDatabase("paperclip-template-copy-"); + try { + expect(lastStartMode()).toBe("template"); + const sql = postgres(second.connectionString, { max: 1 }); + try { + const rows = await sql`select to_regclass('paperclip_template_copy_probe') as name`; + expect(rows[0]?.name).toBeNull(); + } finally { + await sql.end(); + } + } finally { + await second.cleanup(); + } + }); + + it("refreshes the template's timestamp on every reuse so idle time drives pruning", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + // Make sure a template exists before ageing it: the support probe does + // not publish one under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0, and a name + // filter can run this test on its own. + resetTemplateState(); + const first = await startEmbeddedPostgresTestDatabase("paperclip-template-touch-"); + await first.cleanup(); + expect(lastStartMode()).toBe("template"); + const templateDir = path.join(templateRootDir(), computeEmbeddedPostgresTemplateKey()); + expect(isUsableTemplate(templateDir)).toBe(true); + // Age the template as if it had not been used for ten days. + const old = new Date(Date.now() - 10 * 24 * 60 * 60 * 1000); + fs.utimesSync(templateDir, old, old); + expect(fs.statSync(templateDir).mtimeMs).toBeLessThan(Date.now() - 9 * 24 * 60 * 60 * 1000); + + resetTemplateState(); + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-touch-"); + try { + expect(lastStartMode()).toBe("template"); + } finally { + await database.cleanup(); + } + expect(fs.statSync(templateDir).mtimeMs).toBeGreaterThan(Date.now() - 60 * 1000); + }); + + it("rebuilds the template when it was pruned after this process resolved it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-pruned-"); + try { + resetTemplateState(); + const first = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-"); + await first.cleanup(); + expect(lastStartMode()).toBe("template"); + const templateDir = path.join(templateRootDir(), computeEmbeddedPostgresTemplateKey()); + expect(isUsableTemplate(templateDir)).toBe(true); + + // Another process prunes the template while this one still holds its path. + fs.rmSync(templateDir, { recursive: true, force: true }); + const second = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-"); + await second.cleanup(); + // This start fell back without retrying the missing path; the fallback + // forgot the stale path, so the next start rebuilt the template. + expect(lastStartMode()).toBe("fresh"); + const third = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-"); + await third.cleanup(); + expect(lastStartMode()).toBe("template"); + expect(isUsableTemplate(templateDir)).toBe(true); + } finally { + scratch.restore(); + } + }); + + it("runs the fresh initdb path when PAPERCLIP_TEST_POSTGRES_TEMPLATE=0", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0"; + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-bypass-"); + try { + expect(lastStartMode()).toBe("fresh"); + const state = await inspectMigrations(database.connectionString); + expect(state.status).toBe("upToDate"); + } finally { + await database.cleanup(); + } + }); + + describeOnPosix("cache root safety", () => { + it("uses fresh clusters when the cache root is not a private directory of this user", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-root-"); + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + // Someone pre-created the predictable root path, world-writable, the + // way a foreign user on a shared host would. + const rootDir = templateRootDir(); + expect(rootDir.startsWith(scratch.scratch)).toBe(true); + fs.mkdirSync(rootDir); + fs.chmodSync(rootDir, 0o777); + + resetTemplateState(); + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-root-copy-"); + try { + expect(lastStartMode()).toBe("fresh"); + } finally { + await database.cleanup(); + } + // Nothing was published into the rejected root, and the reason was logged once. + expect(fs.readdirSync(rootDir)).toEqual([]); + expect(warnings.filter((line) => line.includes("template cache root"))).toHaveLength(1); + expect(warnings[0]).toContain("writable by group or others"); + + // The probe still reports support, from the throwaway cluster. + resetTemplateState(); + expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true }); + } finally { + console.warn = warn; + scratch.restore(); + } + }); + + it("uses fresh clusters when the cache root path is a symbolic link", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-root-"); + const warn = console.warn; + console.warn = () => {}; + try { + const target = path.join(scratch.scratch, "elsewhere"); + fs.mkdirSync(target, { mode: 0o700 }); + fs.symlinkSync(target, templateRootDir(), "dir"); + + resetTemplateState(); + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-root-copy-"); + try { + expect(lastStartMode()).toBe("fresh"); + } finally { + await database.cleanup(); + } + expect(fs.readdirSync(target)).toEqual([]); + } finally { + console.warn = warn; + scratch.restore(); + } + }); + }); +}); + +describe("support probe when the template cannot be built", () => { + afterEach(() => { + __setEmbeddedPostgresCtorProviderForTests(null); + resetTemplateState(); + }); + + it("falls through to the fresh probe instead of reporting the host as unsupported", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-build-fail-"); + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + // This test runs on every platform, so check the redirect took effect + // before asserting on paths under it. + expect(os.tmpdir()).toBe(scratch.scratch); + // The fake cluster "starts" without listening, so the template build + // fails after the start, at the migration step, the way a broken + // migration would. A real broken migration then fails the suite on + // the fresh path instead of being hidden by `describe.skip`. + const { ctor, constructed } = makeFakeCtor(0); + __setEmbeddedPostgresCtorProviderForTests(async () => ctor); + resetTemplateState(); + + expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true }); + // One build attempt, then the throwaway probe, both under the scratch root. + expect(constructed).toHaveLength(2); + for (const options of constructed) expect(options.databaseDir.startsWith(scratch.scratch)).toBe(true); + expect(constructed[0]?.databaseDir).toContain("paperclip-embedded-postgres-template-build-"); + expect(constructed[1]?.databaseDir).toContain("paperclip-embedded-postgres-probe-"); + expect(warnings.filter((line) => line.includes("template build failed"))).toHaveLength(1); + // Nothing was published. + const rootDir = templateRootDir(); + expect(fs.existsSync(rootDir) ? fs.readdirSync(rootDir) : []).toEqual([]); + } finally { + console.warn = warn; + scratch.restore(); + if (originalTemplateSetting === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting; + } + }); +}); diff --git a/packages/db/src/test-embedded-postgres.ts b/packages/db/src/test-embedded-postgres.ts index 67a52cd1b00..862829c8234 100644 --- a/packages/db/src/test-embedded-postgres.ts +++ b/packages/db/src/test-embedded-postgres.ts @@ -1,7 +1,10 @@ +import { createHash } from "node:crypto"; import fs from "node:fs"; +import { createRequire } from "node:module"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import { applyPendingMigrations, closeRegisteredClients, ensurePostgresDatabase } from "./client.js"; import { createEmbeddedPostgresLogBuffer, @@ -113,8 +116,19 @@ async function getAvailablePort(): Promise { ); } +const EMBEDDED_POSTGRES_USER = "paperclip"; +const EMBEDDED_POSTGRES_PASSWORD = "paperclip"; +const EMBEDDED_POSTGRES_INITDB_FLAGS = ["--encoding=UTF8", "--locale=C", "--lc-messages=C"]; + async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix)); + return await createEmbeddedPostgresTestInstanceAt(dataDir); +} + +// Wraps an embedded-postgres instance around `dataDir`. The caller decides +// whether the directory still needs `initialise()` (a fresh cluster) or already +// holds a copied cluster that only needs `start()`. +async function createEmbeddedPostgresTestInstanceAt(dataDir: string) { const port = await getAvailablePort(); const EmbeddedPostgres = await getEmbeddedPostgresCtor(); // Postgres writes the true reason for a failed start to its output, for @@ -124,11 +138,11 @@ async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) { const logBuffer = createEmbeddedPostgresLogBuffer(); const instance = new EmbeddedPostgres({ databaseDir: dataDir, - user: "paperclip", - password: "paperclip", + user: EMBEDDED_POSTGRES_USER, + password: EMBEDDED_POSTGRES_PASSWORD, port, persistent: true, - initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"], + initdbFlags: EMBEDDED_POSTGRES_INITDB_FLAGS, onLog: (message) => logBuffer.append(message), onError: (message) => logBuffer.append(message), }); @@ -136,6 +150,14 @@ async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) { return { dataDir, port, instance, getRecentLogs: () => logBuffer.getRecentLogs() }; } +function adminConnectionStringFor(port: number): string { + return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/postgres`; +} + +function databaseConnectionStringFor(port: number): string { + return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/paperclip`; +} + function cleanupEmbeddedPostgresTestDirs(dataDir: string) { fs.rmSync(dataDir, { recursive: true, force: true }); } @@ -237,13 +259,391 @@ async function startEmbeddedPostgresWithRetry(tempDirPrefix: string): Promise<{ ); } +// --------------------------------------------------------------------------- +// Template cluster cache +// +// Every suite that calls `startEmbeddedPostgresTestDatabase` used to pay for +// `initdb`, a cluster start, and all pending migrations (331 files at the time +// of writing) before its first test ran, and the `getEmbeddedPostgresTestSupport` +// probe started and stopped a second throwaway cluster per test file. On the +// serial CI shards that was 5-6s per suite plus 2.5-3s per probe; measured +// locally it is 3.2s per suite plus 0.75s per probe. +// +// A cluster data directory that was shut down cleanly can be copied and started +// again as an independent cluster. So the first caller on a host builds one +// fully migrated cluster, stops it, and publishes the data directory as a +// template under the OS temp directory. Every later caller copies the template +// (about 90 MB, ~0.2s) and starts Postgres on the copy (~0.05s), then still runs +// `ensurePostgresDatabase` and `applyPendingMigrations` so the returned database +// is exactly what the fresh path produces; both are no-ops on a template copy. +// +// The template key hashes everything that determines the cluster's contents: +// the embedded-postgres package version, the initdb flags, the user, and the +// name and content of every migration file plus the journal. A changed +// migration therefore builds a new template instead of reusing a stale one. +// Publishing is a rename, which is atomic on the same filesystem, so a +// concurrent builder either wins the rename or discards its copy and uses the +// winner's; a template that exists is always complete. +// +// A template is a Postgres data directory, and starting one executes its +// configuration (for example `shared_preload_libraries`). The cache root is +// therefore private to the current OS user: its name carries the user id, it +// is created with mode 0700, and it is only used when it is a real directory +// (not a symlink) owned by this user with no group or world write bit. Each +// template is checked the same way before it is copied. On a host with a +// shared temp directory another user can pre-create the predictable path, and +// those checks reject it. Windows has no uid; its temp directory is already +// per user, so the ownership and mode checks are skipped there. +// +// Set PAPERCLIP_TEST_POSTGRES_TEMPLATE=0 to bypass the cache and run the fresh +// path for every suite. Any failure on the template path (root checks, build, +// copy, start, or publish) falls back to the fresh path, so a broken cache can +// make a suite slower but never fail it or skip it: the fresh path still +// reports a migration that cannot apply or a cluster that cannot start. +// --------------------------------------------------------------------------- + +const TEMPLATE_ROOT_DIR_NAME = "paperclip-embedded-postgres-templates"; +// Templates whose key no longer matches are pruned once they have not been +// used for this long. Every reuse refreshes the template's timestamp. +const TEMPLATE_PRUNE_AGE_MS = 7 * 24 * 60 * 60 * 1000; +// Publishing a template requires the builder cluster to have exited for real, +// so its data files are quiescent before the rename. Give that stop more room +// than the per-suite bounded stop; a template build happens once per host. +const TEMPLATE_BUILD_STOP_TIMEOUT_MS = 30_000; + +export type EmbeddedPostgresStartMode = "template" | "fresh"; + +function isTemplateCacheDisabled(): boolean { + return process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE === "0"; +} + +function migrationsDirectory(): string { + return fileURLToPath(new URL("./migrations/", import.meta.url)); +} + +function embeddedPostgresPackageVersion(): string { + try { + const require = createRequire(import.meta.url); + const manifest = require("embedded-postgres/package.json") as { version?: unknown }; + return typeof manifest.version === "string" ? manifest.version : "unknown"; + } catch { + return "unknown"; + } +} + +// Hash of everything that determines the contents of a migrated cluster. Reads +// every migration file, which costs a few milliseconds per process. +export function computeEmbeddedPostgresTemplateKey( + migrationsDir: string = migrationsDirectory(), + input: { packageVersion?: string; initdbFlags?: string[]; user?: string } = {}, +): string { + const hash = createHash("sha256"); + hash.update(`embedded-postgres=${input.packageVersion ?? embeddedPostgresPackageVersion()}\n`); + hash.update(`initdb=${(input.initdbFlags ?? EMBEDDED_POSTGRES_INITDB_FLAGS).join(" ")}\n`); + hash.update(`user=${input.user ?? EMBEDDED_POSTGRES_USER}\n`); + const journalPath = path.join(migrationsDir, "meta", "_journal.json"); + if (fs.existsSync(journalPath)) { + hash.update(`journal\n`); + hash.update(fs.readFileSync(journalPath)); + hash.update(`\n`); + } + const migrationFiles = fs + .readdirSync(migrationsDir) + .filter((name) => name.endsWith(".sql")) + .sort((a, b) => a.localeCompare(b)); + for (const name of migrationFiles) { + hash.update(`${name}\n`); + hash.update(fs.readFileSync(path.join(migrationsDir, name))); + hash.update(`\n`); + } + return hash.digest("hex").slice(0, 24); +} + +// The uid on POSIX, else the user name. Part of the cache root's name so two +// users on one host never share a root. +function templateOwnerTag(): string { + const uid = process.getuid?.(); + if (uid !== undefined) return String(uid); + let username = "user"; + try { + username = os.userInfo().username; + } catch { + // Fall through to the generic tag. + } + return username.replace(/[^A-Za-z0-9_.-]/g, "_"); +} + +function templateRootDir(): string { + return path.join(os.tmpdir(), `${TEMPLATE_ROOT_DIR_NAME}-${templateOwnerTag()}`); +} + +// Why `dir` cannot be trusted as a private directory of the current user, or +// null when it can. Uses `lstat` so a symlink planted at the path is rejected +// rather than followed. On Windows (no uid) only the directory check applies. +function describePrivateDirectoryProblem(dir: string): string | null { + let stat: fs.Stats; + try { + stat = fs.lstatSync(dir); + } catch (error) { + return `cannot be inspected (${(error as NodeJS.ErrnoException).code ?? "unknown error"})`; + } + if (stat.isSymbolicLink()) return "is a symbolic link"; + if (!stat.isDirectory()) return "is not a directory"; + const uid = process.getuid?.(); + if (uid === undefined) return null; + if (stat.uid !== uid) return `is owned by uid ${stat.uid}, not the current user (uid ${uid})`; + if ((stat.mode & 0o022) !== 0) { + return `is writable by group or others (mode ${(stat.mode & 0o777).toString(8)})`; + } + return null; +} + +// Creates the cache root with mode 0700 and verifies it is private to the +// current user. Returns null, after one warning, when it is not: the cache is +// then unused for this process and every suite takes the fresh path. +function prepareTemplateRootDir(): string | null { + const rootDir = templateRootDir(); + try { + fs.mkdirSync(rootDir, { recursive: true, mode: 0o700 }); + const problem = describePrivateDirectoryProblem(rootDir); + if (problem !== null) { + warnTemplateOnce("root", `template cache root ${rootDir} ${problem}; using fresh clusters`); + return null; + } + // A root created by an earlier version may be wider than 0700. It is + // ours (checked above), so narrow it. + if (process.getuid !== undefined && (fs.lstatSync(rootDir).mode & 0o777) !== 0o700) { + fs.chmodSync(rootDir, 0o700); + } + return rootDir; + } catch (error) { + warnTemplateOnce( + "root", + `template cache root ${rootDir} could not be prepared: ${(error as Error).message}; using fresh clusters`, + ); + return null; + } +} + +// A template is usable when it is a private directory of the current user +// that holds a cleanly stopped cluster: `PG_VERSION` present, no +// `postmaster.pid`. The ownership check repeats the root's check so a +// template is never copied from a directory this user does not own. +function isUsableTemplate(dir: string): boolean { + if (describePrivateDirectoryProblem(dir) !== null) return false; + return fs.existsSync(path.join(dir, "PG_VERSION")) && !fs.existsSync(path.join(dir, "postmaster.pid")); +} + +// Records a use of the template so `pruneStaleTemplates` measures idle time, +// not age. Best effort. +function touchTemplate(dir: string): void { + try { + const now = new Date(); + fs.utimesSync(dir, now, now); + } catch { + // The template still works without the refreshed timestamp. + } +} + +const warnedTemplateTopics = new Set(); + +function warnTemplateOnce(topic: string, message: string): void { + if (warnedTemplateTopics.has(topic)) return; + warnedTemplateTopics.add(topic); + console.warn(`[embedded-postgres] ${message}`); +} + +// Best-effort removal of templates for other keys that have not been used for +// a week, plus abandoned build directories. Every reuse refreshes a template's +// timestamp, so a template that any process resolved in the last week stays. +// A suite never depends on the template after its copy is made, so pruning +// can only affect a process that resolved the path and has not copied yet; +// that process then rebuilds the template instead of failing. Never throws. +function pruneStaleTemplates(rootDir: string, keepDir: string): void { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(rootDir, { withFileTypes: true }); + } catch { + return; + } + const now = Date.now(); + for (const entry of entries) { + const entryPath = path.join(rootDir, entry.name); + if (entryPath === keepDir || !entry.isDirectory()) continue; + try { + const age = now - fs.statSync(entryPath).mtimeMs; + if (age > TEMPLATE_PRUNE_AGE_MS) fs.rmSync(entryPath, { recursive: true, force: true }); + } catch { + // Another process may be using or removing it; leave it alone. + } + } +} + +// Waits for the raw `stop()` so the builder cluster has really exited before +// its data directory is published. Returns false if the stop overran. +async function stopForTemplatePublish(instance: EmbeddedPostgresInstance): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + instance.stop().then(() => true, () => false), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), TEMPLATE_BUILD_STOP_TIMEOUT_MS); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +// Builds a migrated cluster and publishes its data directory at `templateDir`. +// Throws when the cluster cannot start or a migration cannot apply. Returns +// false when the cluster started but the template could not be published; +// callers then use the fresh path for this process. +async function buildEmbeddedPostgresTemplate(templateDir: string): Promise { + const rootDir = path.dirname(templateDir); + const { port, dataDir, instance } = await startEmbeddedPostgresWithRetry( + "paperclip-embedded-postgres-template-build-", + ); + let published = false; + try { + await ensurePostgresDatabase(adminConnectionStringFor(port), "paperclip"); + await applyPendingMigrations(databaseConnectionStringFor(port)); + const stopped = await stopForTemplatePublish(instance); + if (!stopped || !isUsableTemplate(dataDir)) return false; + try { + fs.renameSync(dataDir, templateDir); + published = true; + } catch (error) { + // Another process published the same key first. Use theirs. + const code = (error as NodeJS.ErrnoException).code; + if (code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM") { + return isUsableTemplate(templateDir); + } + throw error; + } + pruneStaleTemplates(rootDir, templateDir); + return true; + } finally { + if (!published) { + await stopEmbeddedPostgresBounded(instance, () => cleanupEmbeddedPostgresTestDirs(dataDir)); + } + } +} + +let templateDirPromise: Promise | null = null; + +// Resolves the template data directory for this host, building it on first +// use. Resolves null when the cache is disabled, the cache root is not private +// to this user, or the template could not be built or published; every caller +// then takes the fresh path, which reports the real error itself (a cluster +// that cannot start, a migration that cannot apply). Never rejects, so a +// broken cache can never turn into `supported: false` and skip a suite. +async function getEmbeddedPostgresTemplateDir(): Promise { + if (isTemplateCacheDisabled()) return null; + if (!templateDirPromise) { + templateDirPromise = (async () => { + const rootDir = prepareTemplateRootDir(); + if (rootDir === null) return null; + const templateDir = path.join(rootDir, computeEmbeddedPostgresTemplateKey()); + if (isUsableTemplate(templateDir)) { + touchTemplate(templateDir); + return templateDir; + } + try { + const built = await buildEmbeddedPostgresTemplate(templateDir); + return built && isUsableTemplate(templateDir) ? templateDir : null; + } catch (error) { + warnTemplateOnce( + "build", + `template build failed, using fresh clusters: ${(error as Error).message}`, + ); + return null; + } + })(); + } + return await templateDirPromise; +} + +// Forgets the resolved template so the next caller in this process checks the +// cache again. Used after a template start failed, for example because the +// template was pruned by another process after this one resolved its path. +function resetTemplateDirCache(): void { + templateDirPromise = null; +} + +// Copies the template and starts a cluster on the copy. Each attempt uses a +// fresh copy and a fresh port, mirroring `startEmbeddedPostgresWithRetry`. A +// template that is no longer usable (pruned, or no longer private) is not +// retried: the caller falls back and the next caller rebuilds it. +async function startEmbeddedPostgresFromTemplateWithRetry( + templateDir: string, + tempDirPrefix: string, +): Promise<{ port: number; dataDir: string; instance: EmbeddedPostgresInstance }> { + let lastError = new Error("embedded Postgres startup from template failed"); + + for (let attempt = 1; attempt <= EMBEDDED_POSTGRES_START_MAX_ATTEMPTS; attempt += 1) { + if (!isUsableTemplate(templateDir)) { + throw new Error(`embedded Postgres template ${templateDir} is no longer usable`); + } + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix)); + const dataDir = path.join(rootDir, "data"); + let created: Awaited> | null = null; + try { + fs.cpSync(templateDir, dataDir, { recursive: true }); + // Postgres refuses a data directory that is group or world accessible. + fs.chmodSync(dataDir, 0o700); + created = await createEmbeddedPostgresTestInstanceAt(dataDir); + await created.instance.start(); + return { port: created.port, dataDir: rootDir, instance: created.instance }; + } catch (error) { + lastError = formatEmbeddedPostgresError(error, { + fallbackMessage: "embedded Postgres startup from template failed", + recentLogs: created?.getRecentLogs(), + }); + await stopEmbeddedPostgresBounded(created?.instance ?? null, () => + cleanupEmbeddedPostgresTestDirs(rootDir), + ); + } + } + + throw new Error( + `Failed to start embedded PostgreSQL from template after ${EMBEDDED_POSTGRES_START_MAX_ATTEMPTS} attempts: ${lastError.message}`, + ); +} + +let templateFallbackWarned = false; +let lastStartMode: EmbeddedPostgresStartMode | null = null; + // Test-only accessors. Production callers use `startEmbeddedPostgresTestDatabase` // or `getEmbeddedPostgresTestSupport`. A test drives the bounded retry directly // so it does not need a real Postgres connection. export const __startEmbeddedPostgresWithRetryForTests = startEmbeddedPostgresWithRetry; export const __embeddedPostgresStartMaxAttemptsForTests = EMBEDDED_POSTGRES_START_MAX_ATTEMPTS; +export const __embeddedPostgresTemplateRootDirForTests = templateRootDir; +export const __isUsableEmbeddedPostgresTemplateForTests = isUsableTemplate; +export function __lastEmbeddedPostgresStartModeForTests(): EmbeddedPostgresStartMode | null { + return lastStartMode; +} +// Forgets the support probe result, the resolved template, and the one-time +// warnings so a test can drive the probe and the cache again in one process. +export function __resetEmbeddedPostgresTemplateStateForTests(): void { + embeddedPostgresSupportPromise = null; + templateDirPromise = null; + templateFallbackWarned = false; + warnedTemplateTopics.clear(); + lastStartMode = null; +} async function probeEmbeddedPostgresSupport(): Promise { + // A published template proves this host can run the cluster: it was built + // here. Reusing it makes the probe free for every file after the first. + // Building it costs one cluster start, the same as the throwaway probe did. + // When the template cannot be built or used, the throwaway probe below + // decides, so only a cluster that cannot start reports `supported: false`; + // a migration that cannot apply fails the suite loudly on the fresh path. + if ((await getEmbeddedPostgresTemplateDir()) !== null) return { supported: true }; + let started: { dataDir: string; instance: EmbeddedPostgresInstance } | null = null; try { @@ -271,17 +671,48 @@ export async function getEmbeddedPostgresTestSupport(): Promise { + const templateDir = await getEmbeddedPostgresTemplateDir(); + + if (templateDir !== null) { + try { + const started = await startEmbeddedPostgresFromTemplateWithRetry(templateDir, tempDirPrefix); + return { ...started, mode: "template" }; + } catch (error) { + // The next caller in this process re-resolves the template: if it was + // pruned it is rebuilt, instead of retrying a path that is gone. + resetTemplateDirCache(); + if (!templateFallbackWarned) { + templateFallbackWarned = true; + console.warn( + `[embedded-postgres] template start failed, using a fresh cluster: ${(error as Error).message}`, + ); + } + } + } + + // The bounded retry hardens the cluster start against the port race. It throws + // with the real Postgres output if every attempt fails. + const started = await startEmbeddedPostgresWithRetry(tempDirPrefix); + return { ...started, mode: "fresh" }; +} + export async function startEmbeddedPostgresTestDatabase( tempDirPrefix: string, ): Promise { - // The bounded retry hardens the cluster start against the port race. It throws - // with the real Postgres output if every attempt fails. - const { port, dataDir, instance } = await startEmbeddedPostgresWithRetry(tempDirPrefix); + const { port, dataDir, instance, mode } = await startEmbeddedPostgresClusterForSuite(tempDirPrefix); + lastStartMode = mode; try { - const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`; - await ensurePostgresDatabase(adminConnectionString, "paperclip"); - const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; + // Both calls are no-ops on a template copy and keep the returned database + // identical to the fresh path even if the template were ever stale. + await ensurePostgresDatabase(adminConnectionStringFor(port), "paperclip"); + const connectionString = databaseConnectionStringFor(port); await applyPendingMigrations(connectionString); return {