diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 90b15c61114..368414a2736 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -250,3 +250,10 @@ complete eligibility receipt. Qualification starts from the saved pristine database, records the separate preparation phase, and then proves the normal bundled offline entry and empty-home restoration. Preparation alone cannot pass enrollment or active-agent release acceptance. + +The online preparation command loads the configured app key before copying. It +requires a valid existing key and retains that key in its process environment +so a missing file cannot create a replacement during publication. Its database +connections have five-second lock waits and 30-second statement and idle +transaction limits. Cancellation closes the command's clients and has a +ten-second process-exit fallback for a dependency that does not settle. diff --git a/server/src/maintenance/native-harness-prestage-runtime.test.ts b/server/src/maintenance/native-harness-prestage-runtime.test.ts new file mode 100644 index 00000000000..dd5873411e0 --- /dev/null +++ b/server/src/maintenance/native-harness-prestage-runtime.test.ts @@ -0,0 +1,80 @@ +import { randomBytes } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { closeRegisteredClients, createDb } from "@paperclipai/db"; +import { sql } from "drizzle-orm"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { createLocalEncryptedCipher } from "../secrets/local-encrypted-provider.js"; +import { initializeNativeHarnessPrestageKey, nativeHarnessPrestageDatabaseUrl } from "./native-harness-prestage-runtime.js"; + +const roots: string[] = []; +afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); }); + +it("writes backups with the configured app key even after its file disappears", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "prestage-key-")); roots.push(root); + const keyFile = path.join(root, "custom-secrets.key"), key = randomBytes(32).toString("hex"); + await fs.writeFile(keyFile, key, { mode: 0o600 }); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", ""); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", ""); + await initializeNativeHarnessPrestageKey(keyFile); + await fs.unlink(keyFile); + const prepared = createLocalEncryptedCipher("write").encrypt("saved harness key"); + // A later app process uses its own original key, not a newly generated file. + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", ""); + await fs.writeFile(keyFile, key, { mode: 0o600 }); + expect(createLocalEncryptedCipher("read").decrypt(prepared)).toBe("saved harness key"); +}); + +it("does not create a replacement when the configured key is missing or invalid", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "prestage-key-missing-")); roots.push(root); + const keyFile = path.join(root, "missing.key"); + await expect(initializeNativeHarnessPrestageKey(keyFile, {})).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.stat(keyFile).catch(() => null)).toBeNull(); + await fs.writeFile(keyFile, "invalid", { mode: 0o600 }); + await expect(initializeNativeHarnessPrestageKey(keyFile, {})).rejects.toThrow("native_harness_prestage_key_invalid"); + expect(await fs.readFile(keyFile, "utf8")).toBe("invalid"); +}); + +it("keeps explicit environment key and file precedence", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "prestage-key-env-")); roots.push(root); + const keyFile = path.join(root, "override.key"), configured = path.join(root, "configured.key"); + const key = randomBytes(32).toString("base64"); + await fs.writeFile(keyFile, key, { mode: 0o600 }); + const env = { PAPERCLIP_SECRETS_MASTER_KEY_FILE: keyFile }; + await initializeNativeHarnessPrestageKey(configured, env); + expect(env).toMatchObject({ PAPERCLIP_SECRETS_MASTER_KEY: key, PAPERCLIP_SECRETS_MASTER_KEY_FILE: keyFile }); + await fs.unlink(keyFile); + await initializeNativeHarnessPrestageKey(configured, env); + expect(await fs.stat(configured).catch(() => null)).toBeNull(); +}); + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("prestage database waits", () => { + it("bounds a real blocked transaction on every connection", async () => { + const database = await startEmbeddedPostgresTestDatabase("prestage-wait-"); + const source = new URL(database.connectionString); source.searchParams.set("application_name", "prestage-test"); + const url = nativeHarnessPrestageDatabaseUrl(source.href), db = createDb(url, { maxConnections: 2 }); + let release!: () => void, held!: () => void; + const ready = new Promise(resolve => { held = resolve; }); + const lock = db.transaction(async tx => { await tx.execute(sql`select pg_advisory_xact_lock(939193)`); held(); + await new Promise(resolve => { release = resolve; }); }); + try { + await ready; + const [settings] = await db.execute(sql`select current_setting('lock_timeout') lock_timeout, + current_setting('statement_timeout') statement_timeout, current_setting('idle_in_transaction_session_timeout') idle_timeout, + current_setting('application_name') application_name`); + expect(settings).toEqual({ lock_timeout: "5s", statement_timeout: "30s", idle_timeout: "30s", application_name: "prestage-test" }); + const started = Date.now(); + await expect(db.transaction(tx => tx.execute(sql`select pg_advisory_xact_lock(939193)`))) + .rejects.toMatchObject({ cause: { code: "55P03" } }); + expect(Date.now() - started).toBeLessThan(10_000); + // Abort cleanup must also interrupt a query whose statement deadline is later. + const waiting = db.execute(sql`select pg_sleep(20)`); + const rejected = expect(waiting).rejects.toBeTruthy(); + await new Promise(resolve => setTimeout(resolve, 100)); + await closeRegisteredClients(url, { scope: "database" }); + await rejected; + } finally { release(); await lock.catch(() => {}); await closeRegisteredClients(url, { scope: "database" }); await database.cleanup(); } + }, 90_000); +}); diff --git a/server/src/maintenance/native-harness-prestage-runtime.ts b/server/src/maintenance/native-harness-prestage-runtime.ts new file mode 100644 index 00000000000..2924b89e9fc --- /dev/null +++ b/server/src/maintenance/native-harness-prestage-runtime.ts @@ -0,0 +1,22 @@ +import fs from "node:fs/promises"; +import { isValidLocalEncryptedMasterKey } from "../secrets/local-encrypted-provider.js"; + +/** Use the same existing vault key as the app for the whole preparation. A + * missing or replaced file must never cause a new key during publication. */ +export async function initializeNativeHarnessPrestageKey(keyFilePath: string, env: NodeJS.ProcessEnv = process.env) { + const path = env.PAPERCLIP_SECRETS_MASTER_KEY_FILE?.trim() || keyFilePath; + const key = env.PAPERCLIP_SECRETS_MASTER_KEY?.trim() || await fs.readFile(path, "utf8"); + if (!isValidLocalEncryptedMasterKey(key)) throw new Error("native_harness_prestage_key_invalid"); + env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = path; + env.PAPERCLIP_SECRETS_MASTER_KEY = key; +} + +/** These settings apply to every connection, including later transactions. + * Existing URL options stay intact; the finite command owns its wait bounds. */ +export function nativeHarnessPrestageDatabaseUrl(source: string) { + const url = new URL(source); + url.searchParams.set("lock_timeout", "5000"); + url.searchParams.set("statement_timeout", "30000"); + url.searchParams.set("idle_in_transaction_session_timeout", "30000"); + return url.href; +} diff --git a/server/src/maintenance/native-harness-prestage.test.ts b/server/src/maintenance/native-harness-prestage.test.ts new file mode 100644 index 00000000000..366123532a0 --- /dev/null +++ b/server/src/maintenance/native-harness-prestage.test.ts @@ -0,0 +1,44 @@ +import { afterEach, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ close: vi.fn(async () => {}), prestage: vi.fn(), key: vi.fn(async () => {}) })); +vi.mock("@paperclipai/db", () => ({ createDb: () => ({}), closeRegisteredClients: mocks.close })); +vi.mock("../config.js", () => ({ loadConfig: () => ({ secretsMasterKeyFilePath: "/fixture/key" }) })); +vi.mock("../services/cloud-runtime-config.js", () => ({ cloudRuntimeConfigService: () => ({}) })); +vi.mock("../services/cloud-runtime-config-snapshot.js", () => ({ setCloudRuntimeConfigSnapshot: () => {} })); +vi.mock("../services/cloud-instance.js", () => ({ isCloudManagedInstance: () => false })); +vi.mock("../storage/index.js", () => ({ getStorageService: () => ({ provider: "s3" }) })); +vi.mock("../services/native-runtime/native-harness-backup-migration.js", () => ({ prestageNativeHarnessBackups: mocks.prestage })); +vi.mock("./native-harness-prestage-runtime.js", async importOriginal => ({ + ...await importOriginal(), initializeNativeHarnessPrestageKey: mocks.key, +})); + +const initialExitCode = process.exitCode; +afterEach(() => { process.exitCode = initialExitCode; vi.useRealTimers(); vi.unstubAllEnvs(); vi.restoreAllMocks(); vi.resetModules(); mocks.close.mockClear(); }); + +it.each(["SIGTERM", "deadline"])("bounds shutdown if a dependency does not settle after %s", async mode => { + vi.useFakeTimers(); + vi.stubEnv("PAPERCLIP_NATIVE_HARNESS_PRESTAGE", "true"); + vi.stubEnv("DATABASE_URL", "postgres://fixture:fixture@localhost/fixture"); + const once = vi.spyOn(process, "once"), exit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + vi.spyOn(console, "error").mockImplementation(() => {}); + const success = vi.spyOn(console, "log").mockImplementation(() => {}); + let settle!: () => void, started!: () => void; + const ready = new Promise(resolve => { started = resolve; }); + mocks.prestage.mockImplementation(async () => { started(); await new Promise(resolve => { settle = resolve; }); return { copied: 1 }; }); + const command = import("./native-harness-prestage.js"); + await ready; + try { + if (mode === "SIGTERM") { + const handler = once.mock.calls.find(([signal]) => signal === "SIGTERM")?.[1]; + expect(handler).toBeTypeOf("function"); handler!(); + } else await vi.advanceTimersByTimeAsync(90 * 60_000); + expect(mocks.key).toHaveBeenCalledWith("/fixture/key"); + expect(mocks.prestage.mock.calls.at(-1)?.[1].signal.aborted).toBe(true); + expect(mocks.close).toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(10_000); + expect(exit).toHaveBeenCalledWith(1); + } finally { settle(); await command; } + expect(success).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/server/src/maintenance/native-harness-prestage.ts b/server/src/maintenance/native-harness-prestage.ts index 11ce3894d98..7e8674ee460 100644 --- a/server/src/maintenance/native-harness-prestage.ts +++ b/server/src/maintenance/native-harness-prestage.ts @@ -1,29 +1,44 @@ import { closeRegisteredClients, createDb } from "@paperclipai/db"; +import { loadConfig } from "../config.js"; import { cloudRuntimeConfigService } from "../services/cloud-runtime-config.js"; import { setCloudRuntimeConfigSnapshot } from "../services/cloud-runtime-config-snapshot.js"; import { isCloudManagedInstance } from "../services/cloud-instance.js"; import { getStorageService } from "../storage/index.js"; import { prestageNativeHarnessBackups } from "../services/native-runtime/native-harness-backup-migration.js"; +import { initializeNativeHarnessPrestageKey, nativeHarnessPrestageDatabaseUrl } from "./native-harness-prestage-runtime.js"; // Operator-only finite preparation. No task drain, scheduler, app routes, // enrollment export, or assertion that other executions have stopped. if (process.env.PAPERCLIP_NATIVE_HARNESS_PRESTAGE !== "true") throw new Error("Explicit harness pre-copy mode is required"); -const url = process.env.DATABASE_URL; -if (!url) throw new Error("Harness pre-copy requires an explicit database"); +const sourceUrl = process.env.DATABASE_URL; +if (!sourceUrl) throw new Error("Harness pre-copy requires an explicit database"); +const url = nativeHarnessPrestageDatabaseUrl(sourceUrl); const cancellation = new AbortController(); -const stop = () => cancellation.abort(new Error("native_harness_prestage_stopped")); -const deadline = setTimeout(() => cancellation.abort(new Error("native_harness_prestage_deadline")), 90 * 60_000); +let shutdown: ReturnType | undefined; +const abort = (reason: string) => { + if (cancellation.signal.aborted) return; + cancellation.abort(new Error(reason)); + // Closing this command's clients interrupts blocked queries. A bounded + // fallback also covers an unresponsive storage or filesystem dependency. + shutdown = setTimeout(() => process.exit(1), 10_000); + shutdown.unref(); + void closeRegisteredClients(url, { scope: "database" }); +}; +const stop = () => abort("native_harness_prestage_stopped"); +const deadline = setTimeout(() => abort("native_harness_prestage_deadline"), 90 * 60_000); process.once("SIGINT", stop); process.once("SIGTERM", stop); -const db = createDb(url, { maxConnections: 2 }); try { + const db = createDb(url, { maxConnections: 2, connectTimeoutSeconds: 10 }); if (isCloudManagedInstance()) { const config = await cloudRuntimeConfigService(db).inspectStored(); if (!config) throw new Error("cloud_runtime_configuration_missing"); setCloudRuntimeConfigSnapshot(config); } + await initializeNativeHarnessPrestageKey(loadConfig().secretsMasterKeyFilePath); const storage = getStorageService(); if (storage.provider !== "s3") throw new Error("shared_object_storage_required"); const counts = await prestageNativeHarnessBackups(db, { storage, signal: cancellation.signal }); + cancellation.signal.throwIfAborted(); console.log(JSON.stringify({ schema: "paperclip.native-harness-prestage.v1", counts, enrollmentReady: false })); } catch (error) { const message = error instanceof Error ? error.message : "native_harness_prestage_failed"; @@ -33,4 +48,5 @@ try { } finally { clearTimeout(deadline); process.off("SIGINT", stop); process.off("SIGTERM", stop); await closeRegisteredClients(url, { scope: "database" }); + clearTimeout(shutdown); }