diff --git a/tests/runner-e2e/process-tree-owner.test.ts b/tests/runner-e2e/process-tree-owner.test.ts index 862ff8ba998..924a3592a8a 100644 --- a/tests/runner-e2e/process-tree-owner.test.ts +++ b/tests/runner-e2e/process-tree-owner.test.ts @@ -280,3 +280,76 @@ it.skipIf(process.platform === "win32")("keeps an uncertainty that starts at SIG expect(signals.filter(([, signal]) => signal === "SIGKILL")).toEqual([]); } finally { owner.stopObserving(); } }); + +it.skipIf(process.platform === "win32")("re-reads a process listing that fails once while the launcher stops", async () => { + const self = row(process.pid, 1, 10); + let table: ProcessObservation[] = [self, row(100, process.pid, 100)]; + let failedReads = 0; + let stopping = false; + const signals: Array<[number, NodeJS.Signals]> = []; + const child = { pid: 100, exitCode: null as number | null, signalCode: null as NodeJS.Signals | null, + kill: () => { throw new Error("must not fall back to the direct child"); } }; + const owner = createProcessTreeOwner(child as unknown as ChildProcess, { + readTable: async () => { + if (stopping && failedReads === 0) { failedReads += 1; return null; } + return table; + }, + signalGroup: (pid, signal) => { + signals.push([pid, signal]); + child.exitCode = 0; table = [self]; + }, + }); + try { + await owner.observe(); + stopping = true; + await stopOwnedProcessTree(child as unknown as ChildProcess, owner, 2_000, 500); + expect(failedReads).toBe(1); + expect(signals).toEqual([[100, "SIGTERM"]]); + } finally { owner.stopObserving(); } +}); + +it.skipIf(process.platform === "win32")("reports why the process listing failed", async () => { + const reasons: string[] = []; + const inspector = spawn(process.execPath, ["-e", "process.stderr.write('listing unavailable'); process.exit(3)"], { + stdio: ["ignore", "pipe", "pipe"], + }); + expect(await readProcessTable(() => inspector, reason => reasons.push(reason))).toBeNull(); + expect(reasons).toEqual(["ps exited code=3 signal=null: listing unavailable"]); +}); + +it.skipIf(process.platform === "win32")("still falls back to the direct child when every process listing fails", async () => { + let reads = 0; + const signals: NodeJS.Signals[] = []; + const child = { pid: 100, exitCode: null as number | null, signalCode: null as NodeJS.Signals | null, + kill: (signal: NodeJS.Signals) => { signals.push(signal); child.signalCode = signal; return true; } }; + const owner = createProcessTreeOwner(child as unknown as ChildProcess, { + readTable: async () => { reads += 1; return null; }, + signalGroup: () => { throw new Error("must not signal an unobserved group"); }, + }); + try { + await expect(owner.observe()).rejects.toThrow("Could not inspect owned process identities"); + reads = 0; + const started = Date.now(); + await expect(stopOwnedProcessTree(child as unknown as ChildProcess, owner, 2_000, 500)) + .rejects.toThrow("Owned tree cleanup incomplete: Error: Could not inspect owned process identities"); + expect(reads).toBeGreaterThanOrEqual(3); + expect(signals).toEqual(["SIGTERM"]); + expect(Date.now() - started).toBeLessThan(1_000); + } finally { owner.stopObserving(); } +}); + +it.skipIf(process.platform === "win32")("does not re-read a failed process listing past the stop deadline", async () => { + let reads = 0; + const child = { pid: 100, exitCode: null as number | null, signalCode: null as NodeJS.Signals | null, + kill: (signal: NodeJS.Signals) => { child.signalCode = signal; return true; } }; + const owner = createProcessTreeOwner(child as unknown as ChildProcess, { + readTable: async () => { reads += 1; return null; }, + signalGroup: () => { throw new Error("must not signal an unobserved group"); }, + }); + try { + await expect(owner.observe()).rejects.toThrow("Could not inspect owned process identities"); + reads = 0; + await expect(stopOwnedProcessTree(child as unknown as ChildProcess, owner, 50, 500)).rejects.toThrow("Owned tree cleanup incomplete"); + expect(reads).toBe(1); + } finally { owner.stopObserving(); } +}); diff --git a/tests/runner-e2e/process-tree-owner.ts b/tests/runner-e2e/process-tree-owner.ts index 8fcbed6aae5..6687b65ffc4 100644 --- a/tests/runner-e2e/process-tree-owner.ts +++ b/tests/runner-e2e/process-tree-owner.ts @@ -29,6 +29,8 @@ export function restartProcessStartMatches(observed: string, expected: string, p } export const UNCERTAIN_GROUP_IDENTITY = "Owned process group identity became uncertain"; +export const INSPECTION_UNAVAILABLE = "Could not inspect owned process identities"; +const INSPECTION_ATTEMPTS = 3; /** * Under load or during a graceful drain a whole group can turn over between @@ -39,10 +41,15 @@ export const UNCERTAIN_GROUP_IDENTITY = "Owned process group identity became unc * meanwhile; a straggler that outlives the deadline keeps the original error. */ export async function settleUncertainIdentity(step: () => Promise, until: () => number, pollMs = 100): Promise { + let inspectionFailures = 0; for (;;) { try { return await step(); } catch (error) { - if (!String(error).includes(UNCERTAIN_GROUP_IDENTITY) || Date.now() >= until()) throw error; + const text = String(error); + const transient = text.includes(UNCERTAIN_GROUP_IDENTITY) + || (text.includes(INSPECTION_UNAVAILABLE) && ++inspectionFailures < INSPECTION_ATTEMPTS); + const remaining = until() - Date.now(); + if (!transient || remaining <= pollMs) throw error; await new Promise(resolve => setTimeout(resolve, pollMs)); } } @@ -59,7 +66,8 @@ export function createProcessTreeOwner(root: ChildProcess, options: { let restartRunner: ProcessObservation | undefined; let observing: Promise | undefined; const covered = new Set(); - const readTable = options.readTable ?? readProcessTable; + let inspectionFailure = ""; + const readTable = options.readTable ?? (() => readProcessTable(undefined, reason => { inspectionFailure = reason; })); const signalGroup = options.signalGroup ?? ((pid, signal) => process.kill(-pid, signal)); const exited = () => root.exitCode !== null || root.signalCode !== null || !root.pid; const running = (candidate: ProcessObservation) => !candidate.state?.startsWith("Z"); @@ -68,8 +76,9 @@ export function createProcessTreeOwner(root: ChildProcess, options: { if (process.platform === "win32") return Promise.resolve(); if (observing) return observing; observing = (async () => { + inspectionFailure = ""; const next = await readTable(); - if (!next) throw new Error("Could not inspect owned process identities"); + if (!next) throw new Error(inspectionFailure ? `${INSPECTION_UNAVAILABLE} (${inspectionFailure})` : INSPECTION_UNAVAILABLE); const callerGroup = next.find(row => row.pid === process.pid)?.processGroupId; if (callerGroup === undefined) throw new Error("Cleanup caller group identity is unavailable"); const rootRow = next.find(row => row.pid === root.pid); diff --git a/tests/runner-e2e/process-tree.ts b/tests/runner-e2e/process-tree.ts index 7a21642d9cd..80e627f4d3c 100644 --- a/tests/runner-e2e/process-tree.ts +++ b/tests/runner-e2e/process-tree.ts @@ -167,7 +167,10 @@ export function diagnosticProcessKind(command: string, platform: NodeJS.Platform return diagnosticProcessKinds.has(name) ? name : "other"; } -export async function readProcessTable(startInspector?: () => ChildProcess): Promise { +export async function readProcessTable( + startInspector?: () => ChildProcess, + onFailure?: (reason: string) => void, +): Promise { if (process.platform === "win32") { return null; } @@ -177,7 +180,7 @@ export async function readProcessTable(startInspector?: () => ChildProcess): Pro ["-e", "-o", "pid=,ppid=,pgid=,lstart=,stat=,comm="], { env: { PATH: "/usr/bin:/bin", LANG: "C", LC_ALL: "C" }, - stdio: ["ignore", "pipe", "ignore"], + stdio: ["ignore", "pipe", "pipe"], // A wrapper-group shutdown must not interrupt its own bounded audit. detached: true, }, @@ -185,29 +188,37 @@ export async function readProcessTable(startInspector?: () => ChildProcess): Pro let output = ""; let settled = false; let truncated = false; - const finish = (value: ProcessObservation[] | null) => { + let errorOutput = ""; + const finish = (value: ProcessObservation[] | null, reason?: string) => { if (settled) return; settled = true; clearTimeout(inspectionTimeout); + if (value === null && reason) onFailure?.(reason); resolve(value); }; const inspectionTimeout = setTimeout(() => { inspector.kill("SIGKILL"); - finish(null); + finish(null, "ps did not finish within 5000ms"); }, 5_000); inspectionTimeout.unref(); inspector.stdout?.setEncoding("utf8").on("data", (chunk: string) => { if (output.length + chunk.length > 1024 * 1024) { truncated = true; inspector.kill("SIGKILL"); - finish(null); + finish(null, "ps output exceeded 1 MiB"); return; } output += chunk; }); - inspector.once("error", () => finish(null)); - inspector.once("close", (code) => { - if (code !== 0 || truncated) { finish(null); return; } + inspector.stderr?.setEncoding("utf8").on("data", (chunk: string) => { + errorOutput = (errorOutput + chunk).slice(-512); + }); + inspector.once("error", (error) => finish(null, `ps could not start: ${error.message}`)); + inspector.once("close", (code, signal) => { + if (code !== 0 || truncated) { + finish(null, `ps exited code=${code} signal=${signal}${errorOutput.trim() ? `: ${errorOutput.trim()}` : ""}`); + return; + } const observations = output .split(/\r?\n/) .map((line) =>