Allow owned loopback Postgres databases in recovery qualification

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -596,6 +596,15 @@ inputs, or a tracked env file.
 
 ## Local commands
 
+When the host cannot start embedded Postgres, set
+`PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL` to an explicitly supplied loopback
+test Postgres administrator. The launcher creates a new `runner_e2e_<random>`
+database per attempt; the serving app applies its migrations. This does not
+reuse the administrator URL's database. The administrator variable is not
+forwarded to the server or browser. The launcher checks the private ownership
+receipt and deletes only its database after confirmed process/resource cleanup.
+Unknown remote cleanup preserves that database and the private fixture for recovery.
+
 Install dependencies and Chromium once. Native local cells also need the local
 runner binaries:
 

tests/runner-e2e/external-controller-database.test.ts

@@ -0,0 +1,52 @@
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { createDb } from "../../packages/db/src/index.js";
+import { createExternalControllerDatabase } from "./external-controller-database.js";
+import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js";
+import { createRunnerE2ETemporaryRoot } from "./server-config.js";
+import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js";
+
+it("keeps the database administrator out of both child environments", () => {
+  const source = { PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL: "postgres://test-secret@localhost/postgres" };
+  expect(buildRunnerE2EProcessEnvironment(source, []).PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL).toBeUndefined();
+  expect(buildPaperclipServerEnvironment(source).PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL).toBeUndefined();
+});
+
+it("rejects a nonlocal administrator before creating a fixture", async () => {
+  await expect(createExternalControllerDatabase({ adminUrl: "postgres://example.invalid/postgres",
+    temporaryRoot: "/unused", configPath: "/unused/config.json", serverPort: 3100 }))
+    .rejects.toThrow("explicit loopback test server");
+});
+
+it.skipIf(!process.env.PAPERCLIP_TEST_DATABASE_URL)("owns one empty database, rejects a changed binding, and deletes only that database", async () => {
+  const root = await createRunnerE2ETemporaryRoot(), configPath = path.join(root, "instance", "config.json");
+  const admin = createDb(process.env.PAPERCLIP_TEST_DATABASE_URL!, { maxConnections: 1 });
+  let fixture: Awaited<ReturnType<typeof createExternalControllerDatabase>> | undefined;
+  let client: ReturnType<typeof createDb> | undefined;
+  try {
+    fixture = await createExternalControllerDatabase({ adminUrl: process.env.PAPERCLIP_TEST_DATABASE_URL!,
+      temporaryRoot: root, configPath, serverPort: 3100 });
+    const config = JSON.parse(await readFile(configPath, "utf8"));
+    const name = new URL(config.database.connectionString).pathname.slice(1);
+    expect(name).toMatch(/^runner_e2e_[a-f0-9]{32}$/);
+    client = createDb(config.database.connectionString);
+    expect(await client.$client`select tablename from pg_tables where schemaname = 'public'`).toEqual([]);
+    await mkdir(path.dirname(config.secrets.localEncrypted.keyFilePath), { recursive: true });
+    await writeFile(config.secrets.localEncrypted.keyFilePath, "fixture-key");
+    await expect(assertEmbeddedDatabaseIsolation(configPath, root)).resolves.toBeUndefined();
+    config.database.connectionString = process.env.PAPERCLIP_TEST_DATABASE_URL;
+    await writeFile(configPath, JSON.stringify(config));
+    await expect(assertEmbeddedDatabaseIsolation(configPath, root)).rejects.toThrow("fixture-owned database");
+    await client.$client.end(); client = undefined;
+    await fixture.cleanup();
+    await fixture.cleanup();
+    expect(await admin.$client`select datname from pg_database where datname=${name}`).toEqual([]);
+    expect(await admin.$client`select current_database() as name`).toHaveLength(1);
+  } finally {
+    await client?.$client.end({ timeout: 1 });
+    await fixture?.cleanup();
+    await admin.$client.end({ timeout: 1 });
+    await rm(root, { recursive: true, force: true });
+  }
+});

tests/runner-e2e/external-controller-database.ts

@@ -0,0 +1,47 @@
+/** Launcher-owned disposable database. The admin credential never enters the
+ * server/browser environment. Unknown remote cleanup keeps this database alive. */
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { createDb } from "../../packages/db/src/index.js";
+import { paperclipConfigSchema } from "../../packages/shared/src/config-schema.js";
+import { recordControllerDatabaseOwnership } from "./instance-isolation.js";
+import { prepareRunnerE2EServerConfig } from "./server-config.js";
+
+export async function createExternalControllerDatabase(input: {
+  adminUrl: string; temporaryRoot: string; configPath: string; serverPort: number;
+}) {
+  const url = new URL(input.adminUrl);
+  if (!["postgres:", "postgresql:"].includes(url.protocol)
+    || !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname)) {
+    throw new Error("E2E Postgres administrator must use an explicit loopback test server");
+  }
+  await (await prepareRunnerE2EServerConfig(input))?.close();
+  const config = paperclipConfigSchema.parse(JSON.parse(await readFile(input.configPath, "utf8")));
+  if (config.database.mode !== "embedded-postgres") throw new Error("E2E database fixture already configured");
+  const databaseName = `runner_e2e_${randomUUID().replaceAll("-", "")}`;
+  const admin = createDb(input.adminUrl, { maxConnections: 1 });
+  let created = false;
+  const cleanup = async () => {
+    if (!created) return;
+    const connection = createDb(input.adminUrl, { maxConnections: 1 });
+    try {
+      // No FORCE: a remaining controller keeps its evidence and fails cleanup.
+      await connection.$client.unsafe(`drop database "${databaseName}"`);
+      created = false;
+    } finally { await connection.$client.end({ timeout: 1 }); }
+  };
+  try {
+    await admin.$client.unsafe(`create database "${databaseName}"`);
+    created = true;
+    url.pathname = `/${databaseName}`;
+    config.database = { ...config.database, mode: "postgres", connectionString: url.toString() };
+    await mkdir(config.database.embeddedPostgresDataDir, { recursive: true, mode: 0o700 });
+    await recordControllerDatabaseOwnership({ ...input, dataDirectory: config.database.embeddedPostgresDataDir,
+      connectionString: url.toString(), externalDatabaseName: databaseName });
+    await writeFile(input.configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
+    return { cleanup };
+  } catch (error) {
+    await cleanup();
+    throw error;
+  } finally { await admin.$client.end({ timeout: 1 }); }
+}

tests/runner-e2e/harness-env.ts

@@ -135,6 +135,7 @@ export function buildRunnerE2EProcessEnvironment(
   const installed = usesInstalledCli(source);
   if (installed) assertInstalledCliSelection(source, executions);
   const result = { ...source };
+  delete result.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL;
   // The supervisor's read-only recovery observer must use the same disposable
   // instance key as the serving child, never a launching shell's vault key.
   delete result.PAPERCLIP_SECRETS_MASTER_KEY;
@@ -204,6 +205,7 @@ export function buildPaperclipServerEnvironment(
   overrides: NodeJS.ProcessEnv = {},
 ): NodeJS.ProcessEnv {
   const result = { ...source };
+  delete result.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL;
   for (const key of Object.keys(result)) {
     if (PROVIDER_SECRET_KEY.test(key)) delete result[key];
   }

tests/runner-e2e/instance-isolation.ts

@@ -6,6 +6,7 @@ const CONTROLLER_DATABASE_OWNER_FILE = ".runner-e2e-controller-owner.json";
 /** Written only after the supervisor has started this fixture's Postgres. */
 export async function recordControllerDatabaseOwnership(input: {
   configPath: string; temporaryRoot: string; dataDirectory: string; connectionString: string;
+  externalDatabaseName?: string;
 }) {
   const temporaryRoot = await realpath(input.temporaryRoot);
   const dataDirectory = await realpath(input.dataDirectory);
@@ -13,7 +14,8 @@ export async function recordControllerDatabaseOwnership(input: {
     throw new Error("Controller database path escaped its fixture-owned root");
   }
   await writeFile(path.join(dataDirectory, CONTROLLER_DATABASE_OWNER_FILE), JSON.stringify({
-    version: 1, configPath: await realpath(input.configPath), temporaryRoot,
+    version: input.externalDatabaseName ? 2 : 1, externalDatabaseName: input.externalDatabaseName,
+    configPath: await realpath(input.configPath), temporaryRoot,
     dataDirectory, connectionString: input.connectionString,
   }), { mode: 0o600 });
 }
@@ -82,12 +84,16 @@ export async function assertEmbeddedDatabaseIsolation(
       if (!directory.startsWith(`${root}${path.sep}`)) throw new Error("Database escaped fixture");
       const owner = JSON.parse(await readFile(path.join(directory, CONTROLLER_DATABASE_OWNER_FILE), "utf8"));
       const port = config.database.embeddedPostgresPort;
-      const expectedConnection = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
-      if (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65535
-        || config.database.connectionString !== expectedConnection
-        || owner.version !== 1 || owner.temporaryRoot !== root || owner.dataDirectory !== directory
-        || owner.configPath !== await realpath(configPath) || owner.connectionString !== expectedConnection
-        || !(await stat(path.join(directory, "PG_VERSION"))).isFile()) {
+      const connection = new URL(config.database.connectionString!);
+      const external = owner.version === 2 && /^runner_e2e_[a-f0-9]{32}$/.test(owner.externalDatabaseName)
+        && ["postgres:", "postgresql:"].includes(connection.protocol)
+        && ["127.0.0.1", "localhost", "[::1]"].includes(connection.hostname)
+        && connection.pathname === `/${owner.externalDatabaseName}`;
+      const embedded = owner.version === 1 && Number.isInteger(port) && Number(port) > 0 && Number(port) <= 65535
+        && config.database.connectionString === `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`
+        && (await stat(path.join(directory, "PG_VERSION"))).isFile();
+      if ((!external && !embedded) || owner.temporaryRoot !== root || owner.dataDirectory !== directory
+        || owner.configPath !== await realpath(configPath) || owner.connectionString !== config.database.connectionString) {
         throw new Error("Supervisor database ownership mismatch");
       }
     } catch (cause) {

tests/runner-e2e/launch.ts

@@ -10,6 +10,7 @@ import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
 import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js";
 import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";
 import { createRunnerE2ETemporaryRoot } from "./server-config.js";
+import { createExternalControllerDatabase } from "./external-controller-database.js";
 import { createHash, randomBytes } from "node:crypto";
 import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
 import { spawn } from "node:child_process";
@@ -506,6 +507,7 @@ async function runAttempt(input: {
   let attemptSecrets: string[] = [];
   let processCleanupFailed = false;
   let startupReceiptPath: string | undefined;
+  let externalDatabase: Awaited<ReturnType<typeof createExternalControllerDatabase>> | undefined;
   let gateway: Awaited<ReturnType<EvalOperations["start"]>> | undefined;
   const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = [];
   try {
@@ -519,6 +521,11 @@ async function runAttempt(input: {
       "config.json",
     );
     const port = await reserveRunnerE2EServerPort();
+    if (process.env.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL) {
+      externalDatabase = await createExternalControllerDatabase({
+        adminUrl: process.env.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL, temporaryRoot, configPath, serverPort: port,
+      });
+    }
     await Promise.all([
       mkdir(paperclipHome, { recursive: true }),
       mkdir(workspace, { recursive: true }),
@@ -886,7 +893,11 @@ async function runAttempt(input: {
     let cleanupError: unknown;
     const resourceAdmissionStarted = await access(path.join(privateDir, "resource-admission-started"))
       .then(() => true).catch((error: NodeJS.ErrnoException) => error.code !== "ENOENT");
-    if (mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults })) {
+    const preserveRecovery = mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults });
+    if (!preserveRecovery) {
+      try { await externalDatabase?.cleanup(); } catch (error) { cleanupError = error; }
+    }
+    if (preserveRecovery || cleanupError) {
       // Remote allocation cleanup is journaled in this instance's database.
       // Deleting it after the controller exits would strand uncertain creates.
       await chmod(temporaryRoot, 0o700);