PaperClipAI/tests/runner-e2e/server-stop.test.ts

import { spawn, type ChildProcess } from "node:child_process";
import { once } from "node:events";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, expect, it } from "vitest";
import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server-stop.js";
import { readProcessTable } from "./process-tree.js";
import { createProcessTreeOwner, stopOwnedProcessTree, UNCERTAIN_GROUP_IDENTITY } from "./process-tree-owner.js";
import { runnerE2ETypeScriptProcessArgs } from "./web-server-command.js";

const children: ChildProcess[] = [];
const directories: string[] = [];
const worker = `
  process.once('SIGTERM', () => {
    process.send('stopping');
    process.once('message', () => {
      process.send('close-complete', () => process.exit(0));
    });
  });
  process.send('ready');
  setInterval(() => {}, 1000);
`;
async function fixture(source = worker) {
  const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-server-stop-"));
  directories.push(root);
  const entry = path.join(root, "worker.cjs");
  await writeFile(entry, source);
  return { root, entry };
}
function start(entry: string) {
  const child = spawn(process.execPath, [entry], {
    detached: runnerE2EServerDetached,
    stdio: ["ignore", "pipe", "pipe", "ipc"],
  });
  children.push(child);
  return child;
}
function stopper(gracefulTimeoutMs = 2_000) {
  const logs: string[] = [];
  const errors = new WeakSet<ChildProcess>();
  const stop = createRunnerE2EServerStopper({
    gracefulTimeoutMs, forcedTimeoutMs: 2_000,
    hasSpawnError: child => errors.has(child),
    markExpectedStop: () => {}, log: line => logs.push(line),
  });
  return { stop, logs, errors };
}
function expectSuccessfulWrapperReceipt(wrapper: ChildProcess, messages: unknown[], diagnostics = "") {
  expect([wrapper.exitCode, wrapper.signalCode], diagnostics).toEqual([0, null]);
  expect(messages, diagnostics).toContainEqual({ exitCode: 0, signalCode: null });
}
afterEach(async () => {
  for (const child of children.splice(0)) {
    if (child.exitCode !== null || child.signalCode !== null) continue;
    const exited = once(child, "exit").catch(() => {});
    try {
      if (runnerE2EServerDetached && child.pid) process.kill(-child.pid, "SIGKILL");
      else child.kill("SIGKILL");
    } catch { child.kill("SIGKILL"); }
    await exited;
  }
  for (const directory of directories.splice(0)) await rm(directory, { recursive: true, force: true });
});

it.skipIf(process.platform === "win32")("joins signal, final cleanup and repeated calls while a once-handler finishes asynchronous close", async () => {
  const { entry } = await fixture();
  const child = start(entry);
  expect((await once(child, "message"))[0]).toBe("ready");
  const { stop, logs } = stopper();
  const stopping = once(child, "message");
  const first = stop(child);
  expect((await stopping)[0]).toBe("stopping");
  // The server supervisor reaches final cleanup after its 200ms poll.
  await new Promise(resolve => setTimeout(resolve, 250));
  const messages: unknown[] = [];
  child.on("message", message => messages.push(message));
  const finalCleanup = stop(child);
  const repeated = stop(child, "SIGINT");
  // Give an erroneous repeated signal time to interrupt the held close before
  // releasing it. A behavioral failure must not depend on promise identity.
  await new Promise(resolve => setTimeout(resolve, 50));
  if (child.connected) child.send("finish");
  await Promise.all([first, finalCleanup, repeated]);
  expect(messages).toContain("close-complete");
  expect([child.exitCode, child.signalCode]).toEqual([0, null]);
  expect(finalCleanup).toBe(first);
  expect(repeated).toBe(first);
  expect(stop(child)).toBe(first);
  await stop(child);
  expect(logs).toEqual([]);
  expect(() => stop.assertGracefulExit(child)).not.toThrow();
});

it.skipIf(process.platform === "win32")("bounds an unresponsive child and reaps it with one escalation", async () => {
  const { entry } = await fixture(`process.on('SIGTERM', () => {}); process.send('ready'); setInterval(() => {}, 1000);`);
  const child = start(entry);
  await once(child, "message");
  const { stop, logs } = stopper(50);
  await Promise.all([stop(child), stop(child), stop(child)]);
  expect(child.signalCode).toBe("SIGKILL");
  expect(() => stop.assertGracefulExit(child)).toThrow("forced cleanup cannot qualify");
  expect(logs).toHaveLength(1);
  expect(() => process.kill(child.pid!, 0)).toThrow();
});

it("settles an already-exited child without signals or escalation", async () => {
  const { entry } = await fixture("process.exit(0)");
  const child = start(entry);
  await once(child, "exit");
  const { stop, logs } = stopper();
  await stop(child);
  expect([child.exitCode, child.signalCode, logs]).toEqual([0, null, []]);
});

it("settles a failed spawn without attempting escalation", async () => {
  const { root } = await fixture();
  const child = spawn(path.join(root, "missing-server-executable"));
  const { stop, logs, errors } = stopper();
  child.once("error", () => errors.add(child));
  await new Promise<void>(resolve => child.once("error", () => resolve()));
  await stop(child);
  expect(logs).toEqual([]);
});

it.skipIf(process.platform === "win32")("lets launcher cancellation join wrapper shutdown without signaling the server twice", async () => {
  const { root, entry } = await fixture();
  const supervisor = path.join(root, "supervisor.mts");
  const helper = path.join(import.meta.dirname, "server-stop.ts");
  await writeFile(supervisor, `
    import { spawn } from 'node:child_process';
    import { createRunnerE2EServerStopper, runnerE2EServerDetached } from ${JSON.stringify(helper)};
    const child = spawn(process.execPath, [${JSON.stringify(entry)}], {
      detached: runnerE2EServerDetached, stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
    });
    const stop = createRunnerE2EServerStopper({ gracefulTimeoutMs: 2000, forcedTimeoutMs: 2000,
      hasSpawnError: () => false, markExpectedStop: () => {}, log: () => {} });
    const finish = () => { void stop(child).then(() => {
      process.send!({ exitCode: child.exitCode, signalCode: child.signalCode }, () => process.exit(0));
    }); };
    process.on('SIGTERM', finish);
    process.on('message', message => { if (message === 'stop') finish(); else child.send('finish'); });
    child.on('message', message => process.send!({ message, pid: child.pid }));
  `);
  const wrapper = spawn(process.execPath, runnerE2ETypeScriptProcessArgs(path.resolve(import.meta.dirname, "../.."), supervisor), {
    detached: true, stdio: ["ignore", "pipe", "pipe", "ipc"],
  });
  children.push(wrapper);
  let wrapperErrors = "";
  wrapper.stderr?.on("data", chunk => { wrapperErrors += chunk; });
  let workerPid: number | undefined;
  const owner = createProcessTreeOwner(wrapper);
  try {
    workerPid = (await once(wrapper, "message"))[0].pid;
    const stopping = once(wrapper, "message");
    wrapper.send("stop");
    expect((await stopping)[0].message).toBe("stopping");
    // If Paperclip shared the wrapper group, this would kill its once-handler
    // during the async close even though the helper sends only one signal.
    const messages: unknown[] = [];
    wrapper.on("message", message => messages.push(message));
    // Process exit is not IPC completion. Register both barriers before
    // cancellation so the final worker receipt is drained before asserting it.
    const closed = once(wrapper, "close");
    const disconnected = once(wrapper, "disconnect");
    await owner.observe();
    const cancellation = stopOwnedProcessTree(wrapper, owner, 2_000, 2_000);
    await new Promise(resolve => setTimeout(resolve, 250));
    if (wrapper.connected) wrapper.send("finish");
    await Promise.all([closed, disconnected]);
    await cancellation;
    const diagnostics = `${wrapperErrors}\nwrapper exit=${wrapper.exitCode} signal=${wrapper.signalCode} connected=${wrapper.connected}`;
    expect(messages, diagnostics).toContainEqual({ message: "close-complete", pid: workerPid });
    expectSuccessfulWrapperReceipt(wrapper, messages, diagnostics);
    expect(() => process.kill(workerPid!, 0)).toThrow();
  } finally {
    owner.stopObserving();
    if (workerPid) { try { process.kill(-workerPid, "SIGKILL"); } catch { /* reaped */ } }
  }
});

it.each(["failed", "missing"] as const)("rejects a %s worker receipt even when the wrapper closes successfully", async receipt => {
  const { entry } = await fixture(receipt === "failed"
    ? "process.send({ exitCode: 23, signalCode: null }, () => process.exit(0));"
    : "process.exit(0);");
  const wrapper = start(entry);
  const messages: unknown[] = [];
  wrapper.on("message", message => messages.push(message));
  const closed = once(wrapper, "close");
  const disconnected = once(wrapper, "disconnect");
  await Promise.all([closed, disconnected]);
  expect([wrapper.exitCode, wrapper.signalCode]).toEqual([0, null]);
  expect(messages).toEqual(receipt === "failed" ? [{ exitCode: 23, signalCode: null }] : []);
  expect(() => expectSuccessfulWrapperReceipt(wrapper, messages)).toThrow();
});


it.skipIf(process.platform === "win32")("escalates the owned server group so a hung descendant cannot remain", async () => {
  const { entry } = await fixture(`
    const { spawn } = require('node:child_process');
    const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });
    process.on('SIGTERM', () => {});
    process.send({ pid: descendant.pid });
    setInterval(() => {}, 1000);
  `);
  const child = start(entry);
  const pid = (await once(child, "message"))[0].pid as number;
  try {
    await stopper(50).stop(child);
    expect(child.signalCode).toBe("SIGKILL");
    // A stopped orphan may remain a zombie until the host's init reaps it.
    const table = await readProcessTable();
    expect(table).not.toBeNull();
    expect(table!.some(row => row.pid === pid && !row.state?.startsWith("Z"))).toBe(false);
  } finally {
    try { process.kill(pid, "SIGKILL"); } catch { /* reaped */ }
  }
});


it.skipIf(process.platform === "win32")("retires observed descendants after the server leader exits early", async () => {
  const { root, entry } = await fixture();
  const marker = path.join(root, "descendant-closed");
  const descendant = `
    process.once('SIGTERM', () => setTimeout(() => {
      require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'closed'); process.exit(0);
    }, 50));
    process.send('ready'); setInterval(() => {}, 1000);
  `;
  await writeFile(entry, `
    const child = require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(descendant)}], {
      stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
    });
    child.once('message', () => process.send({ pid: child.pid }));
    process.once('message', () => process.exit(1));
  `);
  const child = start(entry);
  const pid = (await once(child, "message"))[0].pid as number;
  const { stop } = stopper();
  try {
    await stop.watch(child);
    const exited = once(child, "exit");
    child.send("crash");
    await exited;
    await stop(child);
    expect(await readFile(marker, "utf8")).toBe("closed");
    const table = await readProcessTable();
    expect(table).not.toBeNull();
    expect(table!.some(row => row.pid === pid && !row.state?.startsWith("Z"))).toBe(false);
  } finally { try { process.kill(pid, "SIGKILL"); } catch { /* stopped */ } }
});

it.skipIf(process.platform === "win32")("recovers a failed cleanup without repeating the graceful signal", async () => {
  const { entry } = await fixture();
  const child = start(entry);
  await once(child, "message");
  let failInspection = false;
  let markInspectionFailed!: () => void;
  const inspectionFailed = new Promise<void>(resolve => { markInspectionFailed = resolve; });
  let owner: ReturnType<typeof createProcessTreeOwner> | undefined;
  const messages: unknown[] = [];
  child.on("message", message => {
    messages.push(message);
    if (message === "stopping") failInspection = true;
  });
  const stop = createRunnerE2EServerStopper({
    gracefulTimeoutMs: 2_000, forcedTimeoutMs: 2_000,
    hasSpawnError: () => false, markExpectedStop: () => {}, log: () => {},
    createOwner: candidate => {
      owner = createProcessTreeOwner(candidate);
      return { ...owner, observe: async () => {
        if (failInspection) {
          failInspection = false;
          markInspectionFailed();
          throw new Error("transient inspection failure");
        }
        await owner!.observe();
      } };
    },
  });
  try {
    const failed = expect(stop(child)).rejects.toThrow("transient inspection failure");
    // Hold the child until the one-shot failure is actually injected. A fixed
    // delay can let it exit during the preceding successful inspection.
    await inspectionFailed;
    if (child.connected) child.send("finish");
    await failed;
    const recovery = stop(child);
    // A second graceful signal would terminate this child's once-handler.
    await new Promise(resolve => setTimeout(resolve, 50));
    if (child.connected) child.send("finish");
    await recovery;
    expect(messages).toEqual(["stopping", "close-complete"]);
    expect([child.exitCode, child.signalCode]).toEqual([0, null]);
  } finally { owner?.stopObserving(); }
});

it.skipIf(process.platform === 'win32').each([['launcher', 'null'], ['server', 'null'], ['launcher', 'throw'], ['server', 'throw']] as const)(
  'bounds the known %s child when inspection fails with %s', async (kind, failure) => {
    const { entry } = await fixture(`process.on('SIGTERM', () => {}); process.send('ready'); setInterval(() => {}, 1000);`);
    const child = start(entry);
    await once(child, 'message');
    const groupSignals: number[] = [];
    const owner = createProcessTreeOwner(child, {
      readTable: async () => { if (failure === "throw") throw new Error("inspection failed"); return null; },
      signalGroup: pid => { groupSignals.push(pid); },
    });
    try {
      const stopping = kind === 'launcher'
        ? stopOwnedProcessTree(child, owner, 50, 500)
        : createRunnerE2EServerStopper({ gracefulTimeoutMs: 50, forcedTimeoutMs: 500,
          hasSpawnError: () => false, markExpectedStop: () => {}, log: () => {}, createOwner: () => owner })(child);
      await expect(stopping).rejects.toThrow(/inspect|incomplete/i);
      expect(child.signalCode).toBe('SIGKILL');
      expect(groupSignals).toEqual([]);
    } finally { owner.stopObserving(); }
  },
);

it.skipIf(process.platform === 'win32')('does not re-signal a server after its graceful wrapper forwards and exits', async () => {
  const { root, entry } = await fixture();
  const marker = path.join(root, 'async-close-finished');
  await writeFile(entry, `
    const child = require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(`
      process.once('SIGTERM', () => setTimeout(() => {
        require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'closed'); process.exit(0);
      }, 300));
      process.send('ready'); setInterval(() => {}, 1000);
    `)}], { detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
    child.once('message', () => process.send({ pid: child.pid }));
    process.once('SIGTERM', () => { child.kill('SIGTERM'); process.exit(0); });
  `);
  const wrapper = start(entry);
  const pid = (await once(wrapper, 'message'))[0].pid as number;
  const owner = createProcessTreeOwner(wrapper);
  try {
    await owner.observe();
    await stopOwnedProcessTree(wrapper, owner, 2_000, 500);
    expect(await readFile(marker, 'utf8')).toBe('closed');
    expect(wrapper.exitCode).toBe(0);
  } finally {
    owner.stopObserving();
    try { process.kill(-pid, 'SIGKILL'); } catch { /* already stopped */ }
  }
});

it.skipIf(process.platform === "win32")("crashes the owned controller and detached descendant without graceful drain or touching a neighboring process", async () => {
  const { root } = await fixture();
  const marker = path.join(root, "graceful-drain.txt"), descendantEntry = path.join(root, "descendant.cjs");
  await writeFile(descendantEntry, `
    process.on('SIGTERM', () => require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'descendant drained'));
    process.send('ready'); setInterval(() => {}, 1000);
  `);
  const { entry } = await fixture(`
    const child = require('node:child_process').fork(${JSON.stringify(descendantEntry)}, [], { detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
    process.on('SIGTERM', () => require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'controller drained'));
    child.once('message', () => process.send({ ready: true, descendantPid: child.pid }));
    setInterval(() => {}, 1000);
  `);
  const controller = start(entry), controllerReady = once(controller, "message");
  const { stop } = stopper();
  await stop.watch(controller);
  try {
    const neighborFixture = await fixture(), neighbor = start(neighborFixture.entry);
    await once(neighbor, "message");
    const [ready] = await controllerReady;
    await stop.crashForRestart(controller);
    expect(controller.signalCode).toBe("SIGKILL");
    await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" });
    const table = await readProcessTable();
    expect(table).not.toBeNull();
    expect(table!.some(row => row.pid === ready.descendantPid && !row.state?.startsWith("Z"))).toBe(false);
    expect(() => process.kill(neighbor.pid!, 0)).not.toThrow();
    await stop.stopAll();
    expect(() => process.kill(neighbor.pid!, 0)).not.toThrow();
  } finally { await stop.stopAll(); }
});

it.skipIf(process.platform === "win32")("waits out a provider group that is briefly unproven during the graceful drain", async () => {
  const { entry } = await fixture();
  const child = start(entry);
  await once(child, "message");
  let uncertainInspections = 0;
  let owner: ReturnType<typeof createProcessTreeOwner> | undefined;
  const messages: unknown[] = [];
  child.on("message", message => {
    messages.push(message);
    // The provider group turns over while the server drains: three inspections cannot prove it.
    if (message === "stopping") uncertainInspections = 3;
  });
  const stop = createRunnerE2EServerStopper({
    gracefulTimeoutMs: 5_000, forcedTimeoutMs: 2_000,
    hasSpawnError: () => false, markExpectedStop: () => {}, log: () => {},
    createOwner: candidate => {
      owner = createProcessTreeOwner(candidate);
      return { ...owner, observe: async () => {
        if (uncertainInspections > 0) {
          uncertainInspections -= 1;
          if (uncertainInspections === 0 && child.connected) child.send("finish");
          throw new Error(`${UNCERTAIN_GROUP_IDENTITY}: 16058 (unowned members: pid=16070 ppid=1 pgid=16058 kind=node started=x)`);
        }
        await owner!.observe();
      } };
    },
  });
  try {
    await stop(child);
    expect(messages).toEqual(["stopping", "close-complete"]);
    expect([child.exitCode, child.signalCode]).toEqual([0, null]);
  } finally { owner?.stopObserving(); }
});

it.skipIf(process.platform === "win32")("still fails safely when a group stays unproven past the drain deadline", async () => {
  const { entry } = await fixture(`process.on('SIGTERM', () => {}); process.send('ready'); setInterval(() => {}, 1000);`);
  const child = start(entry);
  await once(child, "message");
  const groupSignals: number[] = [];
  let stopping = false;
  const owner = createProcessTreeOwner(child, { signalGroup: pid => { groupSignals.push(pid); } });
  const uncertain = { ...owner, observe: async () => {
    if (stopping) throw new Error(`${UNCERTAIN_GROUP_IDENTITY}: 16058 (unowned members: pid=16070 ppid=1 pgid=16058 kind=node started=x)`);
    await owner.observe();
  } };
  try {
    await uncertain.observe();
    stopping = true;
    const stop = createRunnerE2EServerStopper({ gracefulTimeoutMs: 200, forcedTimeoutMs: 200,
      hasSpawnError: () => false, markExpectedStop: () => {}, log: () => {}, createOwner: () => uncertain });
    await expect(stop(child)).rejects.toThrow(/identity became uncertain: 16058 \(unowned members: pid=16070/);
    expect(groupSignals).toEqual([]);
  } finally { owner.stopObserving(); if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); }
});

function eperm() {
  return Object.assign(new Error("kill EPERM"), { code: "EPERM" });
}
function crashStopper(signalError: () => Error, refusedGroup: () => number | undefined = () => undefined) {
  const logs: string[] = [];
  let thrown = false;
  const stop = createRunnerE2EServerStopper({
    gracefulTimeoutMs: 2_000, forcedTimeoutMs: 1_000,
    hasSpawnError: () => false, markExpectedStop: () => {}, log: line => logs.push(line),
    createOwner: candidate => {
      const owner = createProcessTreeOwner(candidate);
      return { ...owner, signal: async (signal, selected) => {
        if (signal === "SIGKILL" && !thrown) {
          thrown = true;
          await owner.observe();
          const refused = refusedGroup();
          if (refused !== undefined) {
            await owner.signal("SIGKILL", new Set(owner.liveGroups().map(group => group.processGroupId).filter(pid => pid !== refused)));
          }
          throw signalError();
        }
        return await owner.signal(signal, selected);
      } };
    },
  });
  return { stop, logs };
}

it.skipIf(process.platform === "win32")("finishes a crash restart with a warning when the group kill reports EPERM but the server is dead", async () => {
  const { entry } = await fixture();
  const child = start(entry);
  await once(child, "message");
  const { stop, logs } = crashStopper(eperm);
  await stop.watch(child);
  try {
    await stop.crashForRestart(child);
    expect(child.signalCode).toBe("SIGKILL");
    expect(logs.join("")).toMatch(new RegExp(`Warning: .*EPERM.*pid=${child.pid}`));
  } finally { await stop.stopAll(); }
});

it.skipIf(process.platform === "win32")("fails a crash restart when the group kill reports EPERM and the server survives", async () => {
  const { entry } = await fixture();
  const child = start(entry);
  await once(child, "message");
  const { stop } = crashStopper(eperm);
  await stop.watch(child);
  const kill = child.kill.bind(child);
  child.kill = () => true;
  try {
    await expect(stop.crashForRestart(child)).rejects.toThrow(/EPERM/);
    expect([child.exitCode, child.signalCode]).toEqual([null, null]);
  } finally {
    child.kill = kill;
    await stop.stopAll().catch(() => {});
  }
});

it.skipIf(process.platform === "win32")("fails a crash restart on a group kill error other than EPERM", async () => {
  const { entry } = await fixture();
  const child = start(entry);
  await once(child, "message");
  const { stop } = crashStopper(() => Object.assign(new Error("kill EINVAL"), { code: "EINVAL" }));
  await stop.watch(child);
  try {
    await expect(stop.crashForRestart(child)).rejects.toThrow(/EINVAL/);
  } finally { await stop.stopAll().catch(() => {}); }
});

it.skipIf(process.platform === "win32")("still SIGKILLs a detached descendant without a graceful drain when its group refuses the bulk kill after the server's", async () => {
  const { root } = await fixture();
  const marker = path.join(root, "graceful-drain.txt"), descendantEntry = path.join(root, "descendant.cjs");
  await writeFile(descendantEntry, `
    process.on('SIGTERM', () => require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'descendant drained'));
    process.send('ready'); setInterval(() => {}, 1000);
  `);
  const { entry } = await fixture(`
    const child = require('node:child_process').fork(${JSON.stringify(descendantEntry)}, [], { detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
    child.once('message', () => process.send({ ready: true, descendantPid: child.pid }));
    setInterval(() => {}, 1000);
  `);
  const controller = start(entry), controllerReady = once(controller, "message");
  let descendantGroup: number | undefined;
  const { stop, logs } = crashStopper(eperm, () => descendantGroup);
  await stop.watch(controller);
  try {
    const [ready] = await controllerReady;
    descendantGroup = ready.descendantPid;
    await stop.crashForRestart(controller);
    expect(controller.signalCode).toBe("SIGKILL");
    await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" });
    const table = await readProcessTable();
    expect(table!.some(row => row.pid === ready.descendantPid && !row.state?.startsWith("Z"))).toBe(false);
    expect(logs.join("")).toMatch(/groups still refusing SIGKILL: none/);
  } finally { await stop.stopAll(); }
});