diff --git a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md index 43898c7bcd8..132461b7423 100644 --- a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md +++ b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md @@ -202,3 +202,20 @@ across the following two qualified hours. Both windows also retained zero new outgoing HTTP requests, non-database TCP connections and non-database bytes. The test requires zero active tracked work throughout the settled boundaries. These are application-level observations, not a host or Neon suspension test. + +## Portable harness and receipt ownership + +When no external database is supplied, the PostgreSQL suites use the repository's +embedded support probe and skip only on an unsupported environment. An explicit +acceptance database bypasses that probe and runs every case. The passive protocol +unit cases still run without PostgreSQL. Dedicated acceptance requires all 34 +cases to execute with no skips. + +The harness owns its temporary directory, proxy, port reservation, log and child +from setup through stop. Stop is idempotent. It finishes child shutdown before +releasing the proxy and files, and it removes owned files on startup failure too. +By default, configuration, logs, session state and measurement receipts are deleted. +Set `PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS=1` for an explicit proof run. Stop then +prints `[maintenance-test] retained artifacts: ` and exposes the same +path as `retainedDirectory` on the harness result. This mode retains receipts on +failure as well. The caller owns removal of explicitly retained proof artifacts. diff --git a/server/src/__tests__/helpers/maintenance/environment.ts b/server/src/__tests__/helpers/maintenance/environment.ts new file mode 100644 index 00000000000..226d9f65915 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/environment.ts @@ -0,0 +1,24 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { getEmbeddedPostgresTestSupport, type EmbeddedPostgresTestSupport } from "../embedded-postgres.js"; + +export async function maintenanceDatabaseSupport(databaseUrl?: string): Promise { + return databaseUrl ? { supported: true } : getEmbeddedPostgresTestSupport(); +} + +export async function createMaintenanceArtifacts(root = tmpdir()) { + const directory = await mkdtemp(path.join(root, "paperclip-maintenance-")); + const retainedDirectory = process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS === "1" ? directory : undefined; + let disposal: Promise | undefined; + return { + directory, + retainedDirectory, + dispose() { + return disposal ??= (async () => { + if (retainedDirectory) process.stdout.write(`[maintenance-test] retained artifacts: ${retainedDirectory}\n`); + else await rm(directory, { recursive: true, force: true }); + })(); + }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/server-harness.ts b/server/src/__tests__/helpers/maintenance/server-harness.ts index 25bd14ef95d..a7197cd50a3 100644 --- a/server/src/__tests__/helpers/maintenance/server-harness.ts +++ b/server/src/__tests__/helpers/maintenance/server-harness.ts @@ -1,13 +1,13 @@ import { fork } from "node:child_process"; import { randomBytes, randomUUID } from "node:crypto"; import { createWriteStream } from "node:fs"; -import { mkdtemp, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { createServer } from "node:net"; -import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { setTimeout as delay } from "node:timers/promises"; import { startMaintenancePostgresProxy } from "./postgres-proxy.js"; +import { createMaintenanceArtifacts } from "./environment.js"; async function withDeadline(promise: Promise, ms: number, message: () => string): Promise { const controller = new AbortController(); @@ -18,102 +18,123 @@ async function withDeadline(promise: Promise, ms: number, message: () => s type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number }; -export async function startMaintenanceServer(databaseUrl: string, options: { enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string }; cloud?: { jwks: string; s3Origin: string; standby: boolean } } = {}) { - const proxy = await startMaintenancePostgresProxy(databaseUrl); - const directory = await mkdtemp(path.join(tmpdir(), "paperclip-maintenance-")); +export async function startMaintenanceServer(databaseUrl: string, options: { artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string }; cloud?: { jwks: string; s3Origin: string; standby: boolean } } = {}) { + const artifacts = await createMaintenanceArtifacts(options.artifactRoot); + const { directory, retainedDirectory } = artifacts; const reservation = createServer(); - await new Promise(resolve => reservation.listen(0, "127.0.0.1", resolve)); - const address = reservation.address(); - if (!address || typeof address === "string") throw new Error("Maintenance HTTP reservation failed"); - const configPath = path.join(directory, "config.json"); - await writeFile(configPath, JSON.stringify({ - $meta: { version: 1, updatedAt: new Date().toISOString(), source: "onboard" }, - database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: false, dir: path.join(directory, "backups") } }, - logging: { mode: "file", logDir: path.join(directory, "logs") }, - server: { deploymentMode: "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: false }, - storage: options.cloud ? { provider: "s3", s3: { bucket: "maintenance-test", region: "us-east-1", endpoint: options.cloud.s3Origin, forcePathStyle: true } } - : { provider: "local_disk", localDisk: { baseDir: path.join(directory, "storage") } }, - secrets: { provider: "local_encrypted", strictMode: false, localEncrypted: { keyFilePath: path.join(directory, "master.key") } }, - telemetry: { enabled: false }, updates: { checkEnabled: false }, - }), { mode: 0o600 }); - await new Promise(resolve => reservation.close(() => resolve())); - const child = fork(fileURLToPath(new URL("./server-child.ts", import.meta.url)), [], { - cwd: fileURLToPath(new URL("../../../../..", import.meta.url)), - execArgv: ["--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], - env: { - PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: "production", - PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test", - PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"), - DATABASE_URL: proxy.url, DATABASE_POOL_MAX: "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", - PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), - PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), - CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false", - PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_UI_DEV_MIDDLEWARE: "false", - HEARTBEAT_SCHEDULER_ENABLED: String(options.enabled ?? true), - PAPERCLIP_MAINTENANCE_FAKE_CLOCK: String(options.clock ?? false), - ...(options.suppressed ? { PAPERCLIP_IN_WORKTREE: "true" } : {}), - ...(options.githubOrigin ? { PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN: options.githubOrigin } : {}), - ...(options.connector ? { - PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", - PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY: options.connector.signPrivateKey, - PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY: options.connector.sealPrivateKey, - PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: options.connector.origin, - } : {}), - ...(options.cloud ? { - PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } }), - PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"), - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "maintenance-tenant-token", PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", - PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.cloud.jwks, PAPERCLIP_CLOUD_WARM_STANDBY: options.cloud.standby ? "1" : "0", - PAPERCLIP_PUBLIC_URL: "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: "pool-maintenance.staging.paperclip.app", - AWS_ACCESS_KEY_ID: "maintenance-fixture", AWS_SECRET_ACCESS_KEY: "maintenance-fixture", AWS_EC2_METADATA_DISABLED: "true", - } : {}), - PAPERCLIP_MIGRATION_PROMPT: "never", PAPERCLIP_MIGRATION_AUTO_APPLY: "true", PAPERCLIP_MANAGED_RUNTIME_HTTPS: "off", - }, - stdio: ["ignore", "pipe", "pipe", "ipc"], - }); - let output = "", requestId = 0; - const log = createWriteStream(path.join(directory, "server.log")); - child.stdout?.on("data", chunk => { output += String(chunk); log.write(chunk); }); - child.stderr?.on("data", chunk => { output += String(chunk); log.write(chunk); }); - const requests = new Map void; reject: (error: Error) => void }>(); - let readyResolve: (port: number) => void, readyReject: (error: Error) => void; - const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; }); - const exited = new Promise(resolve => child.once("exit", () => resolve())); - child.on("message", (message: any) => { - if (message.kind === "ready") readyResolve(message.port); - else if (message.id) { - const request = requests.get(message.id); requests.delete(message.id); - if (message.kind === "error") request?.reject(new Error(message.error)); else request?.resolve(message); - } else if (message.kind === "error") readyReject(new Error(message.error)); - }); - child.once("error", error => readyReject(error)); - child.once("exit", (code, signal) => { - const error = new Error(`Maintenance child exited (${code}/${signal}): ${output.slice(-12_000)}`); - readyReject(error); - for (const request of requests.values()) request.reject(error); - requests.clear(); - }); - async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } } = {}): Promise { - const id = ++requestId; - const result = new Promise((resolve, reject) => requests.set(id, { resolve, reject })); - child.send({ id, command, ...payload }); - try { return await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); } - finally { requests.delete(id); } - } - async function stop() { - try { - if (child.exitCode === null && child.signalCode === null && child.connected) await request("shutdown"); - await withDeadline(exited, 30_000, () => "Maintenance child did not exit after shutdown"); - } finally { - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); - await proxy.stop(); - await new Promise(resolve => log.end(resolve)); - } + let stopProxy = async () => {}; + let stopChild = async () => {}; + let stopLog = async () => {}; + let stopping: Promise | undefined; + function stop() { + return stopping ??= (async () => { + const errors: unknown[] = []; + for (const cleanup of [stopChild, stopProxy, + async () => { if (reservation.listening) await new Promise(resolve => reservation.close(() => resolve())); }, + stopLog, artifacts.dispose]) { + try { await cleanup(); } catch (error) { errors.push(error); } + } + if (errors.length) throw new AggregateError(errors, "Maintenance cleanup failed"); + })(); } try { + const proxy = await startMaintenancePostgresProxy(databaseUrl); + stopProxy = proxy.stop; + await new Promise((resolve, reject) => { reservation.once("error", reject); reservation.listen(0, "127.0.0.1", resolve); }); + const address = reservation.address(); + if (!address || typeof address === "string") throw new Error("Maintenance HTTP reservation failed"); + const configPath = path.join(directory, "config.json"); + await writeFile(configPath, JSON.stringify({ + $meta: { version: 1, updatedAt: new Date().toISOString(), source: "onboard" }, + database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: false, dir: path.join(directory, "backups") } }, + logging: { mode: "file", logDir: path.join(directory, "logs") }, + server: { deploymentMode: "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: false }, + storage: options.cloud ? { provider: "s3", s3: { bucket: "maintenance-test", region: "us-east-1", endpoint: options.cloud.s3Origin, forcePathStyle: true } } + : { provider: "local_disk", localDisk: { baseDir: path.join(directory, "storage") } }, + secrets: { provider: "local_encrypted", strictMode: false, localEncrypted: { keyFilePath: path.join(directory, "master.key") } }, + telemetry: { enabled: false }, updates: { checkEnabled: false }, + }), { mode: 0o600 }); + await new Promise(resolve => reservation.close(() => resolve())); + const child = fork(fileURLToPath(new URL("./server-child.ts", import.meta.url)), [], { + cwd: fileURLToPath(new URL("../../../../..", import.meta.url)), + execArgv: ["--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], + env: { + PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: "production", + PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test", + PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"), + DATABASE_URL: proxy.url, DATABASE_POOL_MAX: "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", + PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), + CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false", + PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_UI_DEV_MIDDLEWARE: "false", + HEARTBEAT_SCHEDULER_ENABLED: String(options.enabled ?? true), + PAPERCLIP_MAINTENANCE_FAKE_CLOCK: String(options.clock ?? false), + ...(options.suppressed ? { PAPERCLIP_IN_WORKTREE: "true" } : {}), + ...(options.githubOrigin ? { PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN: options.githubOrigin } : {}), + ...(options.connector ? { + PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", + PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY: options.connector.signPrivateKey, + PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY: options.connector.sealPrivateKey, + PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: options.connector.origin, + } : {}), + ...(options.cloud ? { + PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } }), + PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"), + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "maintenance-tenant-token", PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.cloud.jwks, PAPERCLIP_CLOUD_WARM_STANDBY: options.cloud.standby ? "1" : "0", + PAPERCLIP_PUBLIC_URL: "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: "pool-maintenance.staging.paperclip.app", + AWS_ACCESS_KEY_ID: "maintenance-fixture", AWS_SECRET_ACCESS_KEY: "maintenance-fixture", AWS_EC2_METADATA_DISABLED: "true", + } : {}), + PAPERCLIP_MIGRATION_PROMPT: "never", PAPERCLIP_MIGRATION_AUTO_APPLY: "true", PAPERCLIP_MANAGED_RUNTIME_HTTPS: "off", + }, + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + let output = "", requestId = 0; + const log = createWriteStream(path.join(directory, "server.log")); + stopLog = () => new Promise(resolve => log.end(resolve)); + let serverReady = false; + child.stdout?.on("data", chunk => { output += String(chunk); log.write(chunk); }); + child.stderr?.on("data", chunk => { output += String(chunk); log.write(chunk); }); + const requests = new Map void; reject: (error: Error) => void }>(); + let readyResolve: (port: number) => void, readyReject: (error: Error) => void; + const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; }); + const exited = new Promise(resolve => child.once("exit", () => resolve())); + child.on("message", (message: any) => { + if (message.kind === "ready") { serverReady = true; readyResolve(message.port); } + else if (message.id) { + const request = requests.get(message.id); requests.delete(message.id); + if (message.kind === "error") request?.reject(new Error(message.error)); else request?.resolve(message); + } else if (message.kind === "error") readyReject(new Error(message.error)); + }); + child.once("error", error => readyReject(error)); + child.once("exit", (code, signal) => { + const error = new Error(`Maintenance child exited (${code}/${signal}): ${output.slice(-12_000)}`); + readyReject(error); + for (const request of requests.values()) request.reject(error); + requests.clear(); + }); + async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } } = {}): Promise { + const id = ++requestId; + const result = new Promise((resolve, reject) => requests.set(id, { resolve, reject })); + child.send({ id, command, ...payload }); + try { return await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); } + finally { requests.delete(id); } + } + stopChild = async () => { + try { + if (!serverReady && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + else if (child.exitCode === null && child.signalCode === null && child.connected) await request("shutdown"); + await withDeadline(exited, 30_000, () => "Maintenance child did not exit after shutdown"); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + await withDeadline(exited, 5_000, () => "Maintenance child did not exit after cancellation"); + } + } + }; const port = await withDeadline(ready, 90_000, () => `Maintenance boot timed out: ${output.slice(-12_000)}`); return { - proxy, directory, port, stop, + proxy, directory, retainedDirectory, port, stop, async api(method: string, route: string, body?: unknown, headers: Record = {}): Promise { if (options.clock) await request("realtime"); try { @@ -153,5 +174,9 @@ export async function startMaintenanceServer(databaseUrl: string, options: { ena throw new Error(`Maintenance did not settle: ${JSON.stringify({ queries: proxy.queries.length, pending: proxy.pending, snapshot: await request("snapshot") })}\n${output.slice(-12_000)}`); }, }; - } catch (error) { await stop().catch(() => {}); throw error; } + } catch (error) { + try { await stop(); } + catch (cleanupError) { throw new AggregateError([error, cleanupError], "Maintenance startup and cleanup failed"); } + throw error; + } } diff --git a/server/src/__tests__/maintenance-harness-resources.test.ts b/server/src/__tests__/maintenance-harness-resources.test.ts new file mode 100644 index 00000000000..e1d8a6af5d9 --- /dev/null +++ b/server/src/__tests__/maintenance-harness-resources.test.ts @@ -0,0 +1,74 @@ +import { access, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createMaintenanceArtifacts, maintenanceDatabaseSupport } from "./helpers/maintenance/environment.js"; +import { startMaintenanceServer } from "./helpers/maintenance/server-harness.js"; + +const embeddedSupport = vi.hoisted(() => vi.fn()); +vi.mock("./helpers/embedded-postgres.js", () => ({ getEmbeddedPostgresTestSupport: embeddedSupport })); + +let root: string | undefined; +beforeEach(() => { vi.clearAllMocks(); vi.stubEnv("PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS", ""); }); +afterEach(async () => { + vi.unstubAllEnvs(); vi.restoreAllMocks(); + if (root) await rm(root, { recursive: true, force: true }); + root = undefined; +}); +async function ownedRoot() { + return root = await mkdtemp(path.join(tmpdir(), "maintenance-environment-test-")); +} + +describe("maintenance database support", () => { + it("does not probe or skip an explicitly configured acceptance database", async () => { + embeddedSupport.mockRejectedValue(new Error("Embedded PostgreSQL must not be launched")); + expect(await maintenanceDatabaseSupport("postgresql://fixture/owned_test")).toEqual({ supported: true }); + expect(embeddedSupport).not.toHaveBeenCalled(); + }); + it("allows a supported default embedded environment", async () => { + embeddedSupport.mockResolvedValue({ supported: true }); + expect(await maintenanceDatabaseSupport()).toEqual({ supported: true }); + expect(embeddedSupport).toHaveBeenCalledOnce(); + }); + it("returns the standard unsupported reason to the suite gate", async () => { + embeddedSupport.mockResolvedValue({ supported: false, reason: "No usable initdb" }); + expect(await maintenanceDatabaseSupport()).toEqual({ supported: false, reason: "No usable initdb" }); + expect(embeddedSupport).toHaveBeenCalledOnce(); + }); +}); + +describe("owned maintenance artifacts", () => { + it("removes owned config and receipt files by default, including repeated disposal", async () => { + const artifacts = await createMaintenanceArtifacts(await ownedRoot()); + await writeFile(path.join(artifacts.directory, "measurement.json"), "{}"); + expect(artifacts.retainedDirectory).toBeUndefined(); + await artifacts.dispose(); await artifacts.dispose(); + await expect(access(artifacts.directory)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(root!)).toEqual([]); + }); + it("retains receipts only with explicit opt-in and exposes their path once", async () => { + vi.stubEnv("PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS", "1"); + const print = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + const artifacts = await createMaintenanceArtifacts(await ownedRoot()); + const receipt = path.join(artifacts.directory, "measurement.json"); + await writeFile(receipt, "{}"); + await artifacts.dispose(); await artifacts.dispose(); + await access(receipt); + expect(artifacts.retainedDirectory).toBe(artifacts.directory); + expect(print).toHaveBeenCalledExactlyOnceWith(`[maintenance-test] retained artifacts: ${artifacts.directory}\n`); + }); + it("cleans the owned directory when setup fails before the child starts", async () => { + const parent = await ownedRoot(); + await expect(startMaintenanceServer("not-a-database-url", { artifactRoot: parent })).rejects.toThrow("Invalid URL"); + expect(await readdir(parent)).toEqual([]); + }); + it("prints retained diagnostics when failed startup was explicitly opted in", async () => { + vi.stubEnv("PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS", "1"); + const print = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + const parent = await ownedRoot(); + await expect(startMaintenanceServer("not-a-database-url", { artifactRoot: parent })).rejects.toThrow("Invalid URL"); + const [name] = await readdir(parent); + expect(name).toMatch(/^paperclip-maintenance-/); + expect(print).toHaveBeenCalledExactlyOnceWith(`[maintenance-test] retained artifacts: ${path.join(parent, name!)}\n`); + }); +}); diff --git a/server/src/__tests__/maintenance-postgres-proxy.test.ts b/server/src/__tests__/maintenance-postgres-proxy.test.ts index ebbd74c7152..caa3fdf0dd9 100644 --- a/server/src/__tests__/maintenance-postgres-proxy.test.ts +++ b/server/src/__tests__/maintenance-postgres-proxy.test.ts @@ -2,6 +2,7 @@ import { createDb } from "@paperclipai/db"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { maintenanceDatabaseSupport } from "./helpers/maintenance/environment.js"; import { createPostgresProtocolObserver, startMaintenancePostgresProxy, type ObservedQuery } from "./helpers/maintenance/postgres-proxy.js"; const startup = Buffer.from([0, 0, 0, 8, 0, 3, 0, 0]); @@ -44,7 +45,10 @@ describe("passive maintenance PostgreSQL observer", () => { let databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_MAINTENANCE_TEST_DATABASE_URL"); let temporary: Awaited> | undefined; -describe("maintenance observer against PostgreSQL", () => { +const databaseSupport = await maintenanceDatabaseSupport(databaseUrl); +const describeDatabase = databaseSupport.supported ? describe : describe.skip; +if (!databaseSupport.supported) console.warn(`Skipping maintenance PostgreSQL tests: ${databaseSupport.reason ?? "unsupported environment"}`); +describeDatabase("maintenance observer against PostgreSQL", () => { beforeAll(async () => { if (!databaseUrl) { temporary = await startEmbeddedPostgresTestDatabase("maintenance-observer-"); databaseUrl = temporary.connectionString; } }, 90_000); diff --git a/server/src/__tests__/maintenance-server-routes.test.ts b/server/src/__tests__/maintenance-server-routes.test.ts index 2d8bdef57e7..a1bf8a2c732 100644 --- a/server/src/__tests__/maintenance-server-routes.test.ts +++ b/server/src/__tests__/maintenance-server-routes.test.ts @@ -7,12 +7,16 @@ import { setTimeout as delay } from "node:timers/promises"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { maintenanceDatabaseSupport } from "./helpers/maintenance/environment.js"; import { CLOUD_RUNTIME_IDENTITY_AUDIENCE, CLOUD_RUNTIME_IDENTITY_ISSUER, CLOUD_RUNTIME_IDENTITY_JWS_TYPE } from "../services/cloud-runtime-identity.js"; import { prepareMaintenanceRuntime } from "./helpers/maintenance/runtime-build.js"; import { startMaintenanceServer } from "./helpers/maintenance/server-harness.js"; let databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_MAINTENANCE_BOOT_TEST_DATABASE_URL"); let temporary: Awaited> | undefined; +const databaseSupport = await maintenanceDatabaseSupport(databaseUrl); +const describeDatabase = databaseSupport.supported ? describe : describe.skip; +if (!databaseSupport.supported) console.warn(`Skipping maintenance PostgreSQL tests: ${databaseSupport.reason ?? "unsupported environment"}`); async function clearDatabase() { if (!temporary && !new URL(databaseUrl!).pathname.endsWith("_test")) throw new Error("Maintenance boot requires an owned disposable _test database"); const fixture = createDb(databaseUrl!, { maxConnections: 1 }); @@ -39,7 +43,7 @@ function receipt(queries: Array<{ protocol: string; sql: string; fingerprint: st } return [...groups.values()]; } -describe("assembled maintenance server production wiring", () => { +describeDatabase("assembled maintenance server production wiring", () => { beforeAll(async () => { await prepareMaintenanceRuntime(); if (!databaseUrl) { temporary = await startEmbeddedPostgresTestDatabase("maintenance-server-"); databaseUrl = temporary.connectionString; } @@ -244,6 +248,8 @@ describe("assembled maintenance server production wiring", () => { await fixture.$client.unsafe("drop function if exists maintenance_age_isolation_attention()").catch(() => {}); await server?.stop(); await fixture.$client.end(); } + if (server!.retainedDirectory) await access(server!.retainedDirectory); + else await expect(access(server!.directory)).rejects.toMatchObject({ code: "ENOENT" }); }, 150_000); it("exposes readiness while real deferred housekeeping is held and settles its work after release", async () => {