diff --git a/tests/runner-e2e/server-stop.test.ts b/tests/runner-e2e/server-stop.test.ts index dd5572e49b7..ffeb49872a6 100644 --- a/tests/runner-e2e/server-stop.test.ts +++ b/tests/runner-e2e/server-stop.test.ts @@ -430,3 +430,100 @@ it.skipIf(process.platform === "win32")("still fails safely when a group stays u expect(groupSignals).toEqual([]); } finally { owner.stopObserving(); if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); } }); + +function eperm() { + return Object.assign(new Error("kill EPERM"), { code: "EPERM" }); +} +function crashStopper(signalError: () => Error, refusedGroup: () => number | undefined = () => undefined) { + const logs: string[] = []; + let thrown = false; + const stop = createRunnerE2EServerStopper({ + gracefulTimeoutMs: 2_000, forcedTimeoutMs: 1_000, + hasSpawnError: () => false, markExpectedStop: () => {}, log: line => logs.push(line), + createOwner: candidate => { + const owner = createProcessTreeOwner(candidate); + return { ...owner, signal: async (signal, selected) => { + if (signal === "SIGKILL" && !thrown) { + thrown = true; + await owner.observe(); + const refused = refusedGroup(); + if (refused !== undefined) { + await owner.signal("SIGKILL", new Set(owner.liveGroups().map(group => group.processGroupId).filter(pid => pid !== refused))); + } + throw signalError(); + } + return await owner.signal(signal, selected); + } }; + }, + }); + return { stop, logs }; +} + +it.skipIf(process.platform === "win32")("finishes a crash restart with a warning when the group kill reports EPERM but the server is dead", async () => { + const { entry } = await fixture(); + const child = start(entry); + await once(child, "message"); + const { stop, logs } = crashStopper(eperm); + await stop.watch(child); + try { + await stop.crashForRestart(child); + expect(child.signalCode).toBe("SIGKILL"); + expect(logs.join("")).toMatch(new RegExp(`Warning: .*EPERM.*pid=${child.pid}`)); + } finally { await stop.stopAll(); } +}); + +it.skipIf(process.platform === "win32")("fails a crash restart when the group kill reports EPERM and the server survives", async () => { + const { entry } = await fixture(); + const child = start(entry); + await once(child, "message"); + const { stop } = crashStopper(eperm); + await stop.watch(child); + const kill = child.kill.bind(child); + child.kill = () => true; + try { + await expect(stop.crashForRestart(child)).rejects.toThrow(/EPERM/); + expect([child.exitCode, child.signalCode]).toEqual([null, null]); + } finally { + child.kill = kill; + await stop.stopAll().catch(() => {}); + } +}); + +it.skipIf(process.platform === "win32")("fails a crash restart on a group kill error other than EPERM", async () => { + const { entry } = await fixture(); + const child = start(entry); + await once(child, "message"); + const { stop } = crashStopper(() => Object.assign(new Error("kill EINVAL"), { code: "EINVAL" })); + await stop.watch(child); + try { + await expect(stop.crashForRestart(child)).rejects.toThrow(/EINVAL/); + } finally { await stop.stopAll().catch(() => {}); } +}); + +it.skipIf(process.platform === "win32")("still SIGKILLs a detached descendant without a graceful drain when its group refuses the bulk kill after the server's", async () => { + const { root } = await fixture(); + const marker = path.join(root, "graceful-drain.txt"), descendantEntry = path.join(root, "descendant.cjs"); + await writeFile(descendantEntry, ` + process.on('SIGTERM', () => require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'descendant drained')); + process.send('ready'); setInterval(() => {}, 1000); + `); + const { entry } = await fixture(` + const child = require('node:child_process').fork(${JSON.stringify(descendantEntry)}, [], { detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'] }); + child.once('message', () => process.send({ ready: true, descendantPid: child.pid })); + setInterval(() => {}, 1000); + `); + const controller = start(entry), controllerReady = once(controller, "message"); + let descendantGroup: number | undefined; + const { stop, logs } = crashStopper(eperm, () => descendantGroup); + await stop.watch(controller); + try { + const [ready] = await controllerReady; + descendantGroup = ready.descendantPid; + await stop.crashForRestart(controller); + expect(controller.signalCode).toBe("SIGKILL"); + await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" }); + const table = await readProcessTable(); + expect(table!.some(row => row.pid === ready.descendantPid && !row.state?.startsWith("Z"))).toBe(false); + expect(logs.join("")).toMatch(/groups still refusing SIGKILL: none/); + } finally { await stop.stopAll(); } +}); diff --git a/tests/runner-e2e/server-stop.ts b/tests/runner-e2e/server-stop.ts index 32a523443a7..938b9dcc570 100644 --- a/tests/runner-e2e/server-stop.ts +++ b/tests/runner-e2e/server-stop.ts @@ -124,7 +124,26 @@ export function createRunnerE2EServerStopper(options: { if (state.finalRequested || state.restartPromise) return Promise.reject(new Error("Restart already requested or final cleanup active")); state.restartPromise = (async () => { state.gracefulDeadline = Date.now() + options.forcedTimeoutMs; - await state.owner.signal("SIGKILL"); + try { + await state.owner.signal("SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EPERM") throw error; + if (!exited(child)) child.kill("SIGKILL"); + while (!exited(child) && Date.now() < state.gracefulDeadline) await wait(25); + if (!exited(child)) throw error; + const refused: number[] = []; + await settleUncertainIdentity(() => state.owner.observe(), () => state.gracefulDeadline!); + for (const group of state.owner.liveGroups()) { + try { + await state.owner.signal("SIGKILL", new Set([group.processGroupId])); + } catch (groupError) { + if ((groupError as NodeJS.ErrnoException).code !== "EPERM") throw groupError; + refused.push(group.processGroupId); + } + } + options.log(`Warning: SIGKILL of the Paperclip process groups reported EPERM after server pid=${child.pid} exited; ` + + `groups still refusing SIGKILL: ${refused.join(",") || "none"}; continuing cleanup\n`); + } await stopOnce(child, state, "SIGKILL"); })(); return state.restartPromise;