From 14d60e1e4f1e2c868110f048873b4fb243a88f0d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 11:27:15 -0500 Subject: [PATCH] Allow owned loopback Postgres databases in recovery qualification Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 9 ++++ .../external-controller-database.test.ts | 52 +++++++++++++++++++ .../external-controller-database.ts | 47 +++++++++++++++++ tests/runner-e2e/harness-env.ts | 2 + tests/runner-e2e/instance-isolation.ts | 20 ++++--- tests/runner-e2e/launch.ts | 13 ++++- 6 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 tests/runner-e2e/external-controller-database.test.ts create mode 100644 tests/runner-e2e/external-controller-database.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index beb3ffe5c3b..61a87d5845c 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -596,6 +596,15 @@ inputs, or a tracked env file. ## Local commands +When the host cannot start embedded Postgres, set +`PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL` to an explicitly supplied loopback +test Postgres administrator. The launcher creates a new `runner_e2e_` +database per attempt; the serving app applies its migrations. This does not +reuse the administrator URL's database. The administrator variable is not +forwarded to the server or browser. The launcher checks the private ownership +receipt and deletes only its database after confirmed process/resource cleanup. +Unknown remote cleanup preserves that database and the private fixture for recovery. + Install dependencies and Chromium once. Native local cells also need the local runner binaries: diff --git a/tests/runner-e2e/external-controller-database.test.ts b/tests/runner-e2e/external-controller-database.test.ts new file mode 100644 index 00000000000..5570b38d6fb --- /dev/null +++ b/tests/runner-e2e/external-controller-database.test.ts @@ -0,0 +1,52 @@ +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { createDb } from "../../packages/db/src/index.js"; +import { createExternalControllerDatabase } from "./external-controller-database.js"; +import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js"; +import { createRunnerE2ETemporaryRoot } from "./server-config.js"; +import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js"; + +it("keeps the database administrator out of both child environments", () => { + const source = { PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL: "postgres://test-secret@localhost/postgres" }; + expect(buildRunnerE2EProcessEnvironment(source, []).PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL).toBeUndefined(); + expect(buildPaperclipServerEnvironment(source).PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL).toBeUndefined(); +}); + +it("rejects a nonlocal administrator before creating a fixture", async () => { + await expect(createExternalControllerDatabase({ adminUrl: "postgres://example.invalid/postgres", + temporaryRoot: "/unused", configPath: "/unused/config.json", serverPort: 3100 })) + .rejects.toThrow("explicit loopback test server"); +}); + +it.skipIf(!process.env.PAPERCLIP_TEST_DATABASE_URL)("owns one empty database, rejects a changed binding, and deletes only that database", async () => { + const root = await createRunnerE2ETemporaryRoot(), configPath = path.join(root, "instance", "config.json"); + const admin = createDb(process.env.PAPERCLIP_TEST_DATABASE_URL!, { maxConnections: 1 }); + let fixture: Awaited> | undefined; + let client: ReturnType | undefined; + try { + fixture = await createExternalControllerDatabase({ adminUrl: process.env.PAPERCLIP_TEST_DATABASE_URL!, + temporaryRoot: root, configPath, serverPort: 3100 }); + const config = JSON.parse(await readFile(configPath, "utf8")); + const name = new URL(config.database.connectionString).pathname.slice(1); + expect(name).toMatch(/^runner_e2e_[a-f0-9]{32}$/); + client = createDb(config.database.connectionString); + expect(await client.$client`select tablename from pg_tables where schemaname = 'public'`).toEqual([]); + await mkdir(path.dirname(config.secrets.localEncrypted.keyFilePath), { recursive: true }); + await writeFile(config.secrets.localEncrypted.keyFilePath, "fixture-key"); + await expect(assertEmbeddedDatabaseIsolation(configPath, root)).resolves.toBeUndefined(); + config.database.connectionString = process.env.PAPERCLIP_TEST_DATABASE_URL; + await writeFile(configPath, JSON.stringify(config)); + await expect(assertEmbeddedDatabaseIsolation(configPath, root)).rejects.toThrow("fixture-owned database"); + await client.$client.end(); client = undefined; + await fixture.cleanup(); + await fixture.cleanup(); + expect(await admin.$client`select datname from pg_database where datname=${name}`).toEqual([]); + expect(await admin.$client`select current_database() as name`).toHaveLength(1); + } finally { + await client?.$client.end({ timeout: 1 }); + await fixture?.cleanup(); + await admin.$client.end({ timeout: 1 }); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/tests/runner-e2e/external-controller-database.ts b/tests/runner-e2e/external-controller-database.ts new file mode 100644 index 00000000000..5a538bd1f80 --- /dev/null +++ b/tests/runner-e2e/external-controller-database.ts @@ -0,0 +1,47 @@ +/** Launcher-owned disposable database. The admin credential never enters the + * server/browser environment. Unknown remote cleanup keeps this database alive. */ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { createDb } from "../../packages/db/src/index.js"; +import { paperclipConfigSchema } from "../../packages/shared/src/config-schema.js"; +import { recordControllerDatabaseOwnership } from "./instance-isolation.js"; +import { prepareRunnerE2EServerConfig } from "./server-config.js"; + +export async function createExternalControllerDatabase(input: { + adminUrl: string; temporaryRoot: string; configPath: string; serverPort: number; +}) { + const url = new URL(input.adminUrl); + if (!["postgres:", "postgresql:"].includes(url.protocol) + || !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname)) { + throw new Error("E2E Postgres administrator must use an explicit loopback test server"); + } + await (await prepareRunnerE2EServerConfig(input))?.close(); + const config = paperclipConfigSchema.parse(JSON.parse(await readFile(input.configPath, "utf8"))); + if (config.database.mode !== "embedded-postgres") throw new Error("E2E database fixture already configured"); + const databaseName = `runner_e2e_${randomUUID().replaceAll("-", "")}`; + const admin = createDb(input.adminUrl, { maxConnections: 1 }); + let created = false; + const cleanup = async () => { + if (!created) return; + const connection = createDb(input.adminUrl, { maxConnections: 1 }); + try { + // No FORCE: a remaining controller keeps its evidence and fails cleanup. + await connection.$client.unsafe(`drop database "${databaseName}"`); + created = false; + } finally { await connection.$client.end({ timeout: 1 }); } + }; + try { + await admin.$client.unsafe(`create database "${databaseName}"`); + created = true; + url.pathname = `/${databaseName}`; + config.database = { ...config.database, mode: "postgres", connectionString: url.toString() }; + await mkdir(config.database.embeddedPostgresDataDir, { recursive: true, mode: 0o700 }); + await recordControllerDatabaseOwnership({ ...input, dataDirectory: config.database.embeddedPostgresDataDir, + connectionString: url.toString(), externalDatabaseName: databaseName }); + await writeFile(input.configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 }); + return { cleanup }; + } catch (error) { + await cleanup(); + throw error; + } finally { await admin.$client.end({ timeout: 1 }); } +} diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index b6814d877f1..853786df670 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -135,6 +135,7 @@ export function buildRunnerE2EProcessEnvironment( const installed = usesInstalledCli(source); if (installed) assertInstalledCliSelection(source, executions); const result = { ...source }; + delete result.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL; // The supervisor's read-only recovery observer must use the same disposable // instance key as the serving child, never a launching shell's vault key. delete result.PAPERCLIP_SECRETS_MASTER_KEY; @@ -204,6 +205,7 @@ export function buildPaperclipServerEnvironment( overrides: NodeJS.ProcessEnv = {}, ): NodeJS.ProcessEnv { const result = { ...source }; + delete result.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL; for (const key of Object.keys(result)) { if (PROVIDER_SECRET_KEY.test(key)) delete result[key]; } diff --git a/tests/runner-e2e/instance-isolation.ts b/tests/runner-e2e/instance-isolation.ts index 9bd98b60a3e..1c2117dbdcd 100644 --- a/tests/runner-e2e/instance-isolation.ts +++ b/tests/runner-e2e/instance-isolation.ts @@ -6,6 +6,7 @@ const CONTROLLER_DATABASE_OWNER_FILE = ".runner-e2e-controller-owner.json"; /** Written only after the supervisor has started this fixture's Postgres. */ export async function recordControllerDatabaseOwnership(input: { configPath: string; temporaryRoot: string; dataDirectory: string; connectionString: string; + externalDatabaseName?: string; }) { const temporaryRoot = await realpath(input.temporaryRoot); const dataDirectory = await realpath(input.dataDirectory); @@ -13,7 +14,8 @@ export async function recordControllerDatabaseOwnership(input: { throw new Error("Controller database path escaped its fixture-owned root"); } await writeFile(path.join(dataDirectory, CONTROLLER_DATABASE_OWNER_FILE), JSON.stringify({ - version: 1, configPath: await realpath(input.configPath), temporaryRoot, + version: input.externalDatabaseName ? 2 : 1, externalDatabaseName: input.externalDatabaseName, + configPath: await realpath(input.configPath), temporaryRoot, dataDirectory, connectionString: input.connectionString, }), { mode: 0o600 }); } @@ -82,12 +84,16 @@ export async function assertEmbeddedDatabaseIsolation( if (!directory.startsWith(`${root}${path.sep}`)) throw new Error("Database escaped fixture"); const owner = JSON.parse(await readFile(path.join(directory, CONTROLLER_DATABASE_OWNER_FILE), "utf8")); const port = config.database.embeddedPostgresPort; - const expectedConnection = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; - if (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65535 - || config.database.connectionString !== expectedConnection - || owner.version !== 1 || owner.temporaryRoot !== root || owner.dataDirectory !== directory - || owner.configPath !== await realpath(configPath) || owner.connectionString !== expectedConnection - || !(await stat(path.join(directory, "PG_VERSION"))).isFile()) { + const connection = new URL(config.database.connectionString!); + const external = owner.version === 2 && /^runner_e2e_[a-f0-9]{32}$/.test(owner.externalDatabaseName) + && ["postgres:", "postgresql:"].includes(connection.protocol) + && ["127.0.0.1", "localhost", "[::1]"].includes(connection.hostname) + && connection.pathname === `/${owner.externalDatabaseName}`; + const embedded = owner.version === 1 && Number.isInteger(port) && Number(port) > 0 && Number(port) <= 65535 + && config.database.connectionString === `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip` + && (await stat(path.join(directory, "PG_VERSION"))).isFile(); + if ((!external && !embedded) || owner.temporaryRoot !== root || owner.dataDirectory !== directory + || owner.configPath !== await realpath(configPath) || owner.connectionString !== config.database.connectionString) { throw new Error("Supervisor database ownership mismatch"); } } catch (cause) { diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index f58a8c89523..36cfa5cd231 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -10,6 +10,7 @@ import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js"; import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js"; import { createRunnerE2ETemporaryRoot } from "./server-config.js"; +import { createExternalControllerDatabase } from "./external-controller-database.js"; import { createHash, randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; import { spawn } from "node:child_process"; @@ -506,6 +507,7 @@ async function runAttempt(input: { let attemptSecrets: string[] = []; let processCleanupFailed = false; let startupReceiptPath: string | undefined; + let externalDatabase: Awaited> | undefined; let gateway: Awaited> | undefined; const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = []; try { @@ -519,6 +521,11 @@ async function runAttempt(input: { "config.json", ); const port = await reserveRunnerE2EServerPort(); + if (process.env.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL) { + externalDatabase = await createExternalControllerDatabase({ + adminUrl: process.env.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL, temporaryRoot, configPath, serverPort: port, + }); + } await Promise.all([ mkdir(paperclipHome, { recursive: true }), mkdir(workspace, { recursive: true }), @@ -886,7 +893,11 @@ async function runAttempt(input: { let cleanupError: unknown; const resourceAdmissionStarted = await access(path.join(privateDir, "resource-admission-started")) .then(() => true).catch((error: NodeJS.ErrnoException) => error.code !== "ENOENT"); - if (mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults })) { + const preserveRecovery = mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults }); + if (!preserveRecovery) { + try { await externalDatabase?.cleanup(); } catch (error) { cleanupError = error; } + } + if (preserveRecovery || cleanupError) { // Remote allocation cleanup is journaled in this instance's database. // Deleting it after the controller exits would strand uncertain creates. await chmod(temporaryRoot, 0o700);