fix(maintenance): preserve the app key and bound pre-copy shutdown

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md

@@ -250,3 +250,10 @@ complete eligibility receipt. Qualification starts from the saved pristine
 database, records the separate preparation phase, and then proves the normal
 bundled offline entry and empty-home restoration. Preparation alone cannot pass
 enrollment or active-agent release acceptance.
+
+The online preparation command loads the configured app key before copying. It
+requires a valid existing key and retains that key in its process environment
+so a missing file cannot create a replacement during publication. Its database
+connections have five-second lock waits and 30-second statement and idle
+transaction limits. Cancellation closes the command's clients and has a
+ten-second process-exit fallback for a dependency that does not settle.

server/src/maintenance/native-harness-prestage-runtime.test.ts

@@ -0,0 +1,80 @@
+import { randomBytes } from "node:crypto";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { closeRegisteredClients, createDb } from "@paperclipai/db";
+import { sql } from "drizzle-orm";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
+import { createLocalEncryptedCipher } from "../secrets/local-encrypted-provider.js";
+import { initializeNativeHarnessPrestageKey, nativeHarnessPrestageDatabaseUrl } from "./native-harness-prestage-runtime.js";
+
+const roots: string[] = [];
+afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); });
+
+it("writes backups with the configured app key even after its file disappears", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "prestage-key-")); roots.push(root);
+  const keyFile = path.join(root, "custom-secrets.key"), key = randomBytes(32).toString("hex");
+  await fs.writeFile(keyFile, key, { mode: 0o600 });
+  vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", ""); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", "");
+  await initializeNativeHarnessPrestageKey(keyFile);
+  await fs.unlink(keyFile);
+  const prepared = createLocalEncryptedCipher("write").encrypt("saved harness key");
+  // A later app process uses its own original key, not a newly generated file.
+  vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
+  await fs.writeFile(keyFile, key, { mode: 0o600 });
+  expect(createLocalEncryptedCipher("read").decrypt(prepared)).toBe("saved harness key");
+});
+
+it("does not create a replacement when the configured key is missing or invalid", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "prestage-key-missing-")); roots.push(root);
+  const keyFile = path.join(root, "missing.key");
+  await expect(initializeNativeHarnessPrestageKey(keyFile, {})).rejects.toMatchObject({ code: "ENOENT" });
+  expect(await fs.stat(keyFile).catch(() => null)).toBeNull();
+  await fs.writeFile(keyFile, "invalid", { mode: 0o600 });
+  await expect(initializeNativeHarnessPrestageKey(keyFile, {})).rejects.toThrow("native_harness_prestage_key_invalid");
+  expect(await fs.readFile(keyFile, "utf8")).toBe("invalid");
+});
+
+it("keeps explicit environment key and file precedence", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "prestage-key-env-")); roots.push(root);
+  const keyFile = path.join(root, "override.key"), configured = path.join(root, "configured.key");
+  const key = randomBytes(32).toString("base64");
+  await fs.writeFile(keyFile, key, { mode: 0o600 });
+  const env = { PAPERCLIP_SECRETS_MASTER_KEY_FILE: keyFile };
+  await initializeNativeHarnessPrestageKey(configured, env);
+  expect(env).toMatchObject({ PAPERCLIP_SECRETS_MASTER_KEY: key, PAPERCLIP_SECRETS_MASTER_KEY_FILE: keyFile });
+  await fs.unlink(keyFile);
+  await initializeNativeHarnessPrestageKey(configured, env);
+  expect(await fs.stat(configured).catch(() => null)).toBeNull();
+});
+
+const support = await getEmbeddedPostgresTestSupport();
+describe.skipIf(!support.supported)("prestage database waits", () => {
+  it("bounds a real blocked transaction on every connection", async () => {
+    const database = await startEmbeddedPostgresTestDatabase("prestage-wait-");
+    const source = new URL(database.connectionString); source.searchParams.set("application_name", "prestage-test");
+    const url = nativeHarnessPrestageDatabaseUrl(source.href), db = createDb(url, { maxConnections: 2 });
+    let release!: () => void, held!: () => void;
+    const ready = new Promise<void>(resolve => { held = resolve; });
+    const lock = db.transaction(async tx => { await tx.execute(sql`select pg_advisory_xact_lock(939193)`); held();
+      await new Promise<void>(resolve => { release = resolve; }); });
+    try {
+      await ready;
+      const [settings] = await db.execute(sql`select current_setting('lock_timeout') lock_timeout,
+        current_setting('statement_timeout') statement_timeout, current_setting('idle_in_transaction_session_timeout') idle_timeout,
+        current_setting('application_name') application_name`);
+      expect(settings).toEqual({ lock_timeout: "5s", statement_timeout: "30s", idle_timeout: "30s", application_name: "prestage-test" });
+      const started = Date.now();
+      await expect(db.transaction(tx => tx.execute(sql`select pg_advisory_xact_lock(939193)`)))
+        .rejects.toMatchObject({ cause: { code: "55P03" } });
+      expect(Date.now() - started).toBeLessThan(10_000);
+      // Abort cleanup must also interrupt a query whose statement deadline is later.
+      const waiting = db.execute(sql`select pg_sleep(20)`);
+      const rejected = expect(waiting).rejects.toBeTruthy();
+      await new Promise(resolve => setTimeout(resolve, 100));
+      await closeRegisteredClients(url, { scope: "database" });
+      await rejected;
+    } finally { release(); await lock.catch(() => {}); await closeRegisteredClients(url, { scope: "database" }); await database.cleanup(); }
+  }, 90_000);
+});

server/src/maintenance/native-harness-prestage-runtime.ts

@@ -0,0 +1,22 @@
+import fs from "node:fs/promises";
+import { isValidLocalEncryptedMasterKey } from "../secrets/local-encrypted-provider.js";
+
+/** Use the same existing vault key as the app for the whole preparation. A
+ * missing or replaced file must never cause a new key during publication. */
+export async function initializeNativeHarnessPrestageKey(keyFilePath: string, env: NodeJS.ProcessEnv = process.env) {
+  const path = env.PAPERCLIP_SECRETS_MASTER_KEY_FILE?.trim() || keyFilePath;
+  const key = env.PAPERCLIP_SECRETS_MASTER_KEY?.trim() || await fs.readFile(path, "utf8");
+  if (!isValidLocalEncryptedMasterKey(key)) throw new Error("native_harness_prestage_key_invalid");
+  env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = path;
+  env.PAPERCLIP_SECRETS_MASTER_KEY = key;
+}
+
+/** These settings apply to every connection, including later transactions.
+ * Existing URL options stay intact; the finite command owns its wait bounds. */
+export function nativeHarnessPrestageDatabaseUrl(source: string) {
+  const url = new URL(source);
+  url.searchParams.set("lock_timeout", "5000");
+  url.searchParams.set("statement_timeout", "30000");
+  url.searchParams.set("idle_in_transaction_session_timeout", "30000");
+  return url.href;
+}

server/src/maintenance/native-harness-prestage.test.ts

@@ -0,0 +1,44 @@
+import { afterEach, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({ close: vi.fn(async () => {}), prestage: vi.fn(), key: vi.fn(async () => {}) }));
+vi.mock("@paperclipai/db", () => ({ createDb: () => ({}), closeRegisteredClients: mocks.close }));
+vi.mock("../config.js", () => ({ loadConfig: () => ({ secretsMasterKeyFilePath: "/fixture/key" }) }));
+vi.mock("../services/cloud-runtime-config.js", () => ({ cloudRuntimeConfigService: () => ({}) }));
+vi.mock("../services/cloud-runtime-config-snapshot.js", () => ({ setCloudRuntimeConfigSnapshot: () => {} }));
+vi.mock("../services/cloud-instance.js", () => ({ isCloudManagedInstance: () => false }));
+vi.mock("../storage/index.js", () => ({ getStorageService: () => ({ provider: "s3" }) }));
+vi.mock("../services/native-runtime/native-harness-backup-migration.js", () => ({ prestageNativeHarnessBackups: mocks.prestage }));
+vi.mock("./native-harness-prestage-runtime.js", async importOriginal => ({
+  ...await importOriginal<typeof import("./native-harness-prestage-runtime.js")>(), initializeNativeHarnessPrestageKey: mocks.key,
+}));
+
+const initialExitCode = process.exitCode;
+afterEach(() => { process.exitCode = initialExitCode; vi.useRealTimers(); vi.unstubAllEnvs(); vi.restoreAllMocks(); vi.resetModules(); mocks.close.mockClear(); });
+
+it.each(["SIGTERM", "deadline"])("bounds shutdown if a dependency does not settle after %s", async mode => {
+  vi.useFakeTimers();
+  vi.stubEnv("PAPERCLIP_NATIVE_HARNESS_PRESTAGE", "true");
+  vi.stubEnv("DATABASE_URL", "postgres://fixture:fixture@localhost/fixture");
+  const once = vi.spyOn(process, "once"), exit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never);
+  vi.spyOn(console, "error").mockImplementation(() => {});
+  const success = vi.spyOn(console, "log").mockImplementation(() => {});
+  let settle!: () => void, started!: () => void;
+  const ready = new Promise<void>(resolve => { started = resolve; });
+  mocks.prestage.mockImplementation(async () => { started(); await new Promise<void>(resolve => { settle = resolve; }); return { copied: 1 }; });
+  const command = import("./native-harness-prestage.js");
+  await ready;
+  try {
+    if (mode === "SIGTERM") {
+      const handler = once.mock.calls.find(([signal]) => signal === "SIGTERM")?.[1];
+      expect(handler).toBeTypeOf("function"); handler!();
+    } else await vi.advanceTimersByTimeAsync(90 * 60_000);
+    expect(mocks.key).toHaveBeenCalledWith("/fixture/key");
+    expect(mocks.prestage.mock.calls.at(-1)?.[1].signal.aborted).toBe(true);
+    expect(mocks.close).toHaveBeenCalled();
+    await vi.advanceTimersByTimeAsync(10_000);
+    expect(exit).toHaveBeenCalledWith(1);
+  } finally { settle(); await command; }
+  expect(success).not.toHaveBeenCalled();
+  expect(process.exitCode).toBe(1);
+  expect(vi.getTimerCount()).toBe(0);
+});

server/src/maintenance/native-harness-prestage.ts

@@ -1,29 +1,44 @@
 import { closeRegisteredClients, createDb } from "@paperclipai/db";
+import { loadConfig } from "../config.js";
 import { cloudRuntimeConfigService } from "../services/cloud-runtime-config.js";
 import { setCloudRuntimeConfigSnapshot } from "../services/cloud-runtime-config-snapshot.js";
 import { isCloudManagedInstance } from "../services/cloud-instance.js";
 import { getStorageService } from "../storage/index.js";
 import { prestageNativeHarnessBackups } from "../services/native-runtime/native-harness-backup-migration.js";
+import { initializeNativeHarnessPrestageKey, nativeHarnessPrestageDatabaseUrl } from "./native-harness-prestage-runtime.js";
 
 // Operator-only finite preparation. No task drain, scheduler, app routes,
 // enrollment export, or assertion that other executions have stopped.
 if (process.env.PAPERCLIP_NATIVE_HARNESS_PRESTAGE !== "true") throw new Error("Explicit harness pre-copy mode is required");
-const url = process.env.DATABASE_URL;
-if (!url) throw new Error("Harness pre-copy requires an explicit database");
+const sourceUrl = process.env.DATABASE_URL;
+if (!sourceUrl) throw new Error("Harness pre-copy requires an explicit database");
+const url = nativeHarnessPrestageDatabaseUrl(sourceUrl);
 const cancellation = new AbortController();
-const stop = () => cancellation.abort(new Error("native_harness_prestage_stopped"));
-const deadline = setTimeout(() => cancellation.abort(new Error("native_harness_prestage_deadline")), 90 * 60_000);
+let shutdown: ReturnType<typeof setTimeout> | undefined;
+const abort = (reason: string) => {
+  if (cancellation.signal.aborted) return;
+  cancellation.abort(new Error(reason));
+  // Closing this command's clients interrupts blocked queries. A bounded
+  // fallback also covers an unresponsive storage or filesystem dependency.
+  shutdown = setTimeout(() => process.exit(1), 10_000);
+  shutdown.unref();
+  void closeRegisteredClients(url, { scope: "database" });
+};
+const stop = () => abort("native_harness_prestage_stopped");
+const deadline = setTimeout(() => abort("native_harness_prestage_deadline"), 90 * 60_000);
 process.once("SIGINT", stop); process.once("SIGTERM", stop);
-const db = createDb(url, { maxConnections: 2 });
 try {
+  const db = createDb(url, { maxConnections: 2, connectTimeoutSeconds: 10 });
   if (isCloudManagedInstance()) {
     const config = await cloudRuntimeConfigService(db).inspectStored();
     if (!config) throw new Error("cloud_runtime_configuration_missing");
     setCloudRuntimeConfigSnapshot(config);
   }
+  await initializeNativeHarnessPrestageKey(loadConfig().secretsMasterKeyFilePath);
   const storage = getStorageService();
   if (storage.provider !== "s3") throw new Error("shared_object_storage_required");
   const counts = await prestageNativeHarnessBackups(db, { storage, signal: cancellation.signal });
+  cancellation.signal.throwIfAborted();
   console.log(JSON.stringify({ schema: "paperclip.native-harness-prestage.v1", counts, enrollmentReady: false }));
 } catch (error) {
   const message = error instanceof Error ? error.message : "native_harness_prestage_failed";
@@ -33,4 +48,5 @@ try {
 } finally {
   clearTimeout(deadline); process.off("SIGINT", stop); process.off("SIGTERM", stop);
   await closeRegisteredClients(url, { scope: "database" });
+  clearTimeout(shutdown);
 }