diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 66fdcd6852c..fd7624cb8f9 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -5,6 +5,21 @@ Run-log events write to the `heartbeat_run_events` table Paperclip Telemetry events, and they are not OpenTelemetry exports. A run-log event needs no operator endpoint. +## ACPX sidecar shutdown + +The ACPX sidecar emits `runtime.diagnostic` with code +`sidecar_shutdown_requested` once at the first local shutdown request, before +waiters are removed. Its JSON message records the local reason, pending tool, +input and permission counts, completion-normalization count, and at most 16 +oldest pending tools. Each tool sample contains elapsed milliseconds and a +SHA-256 call ID. Hash the corresponding `runtime.tool_called` call ID to match +it. Arguments, results, operation names and raw call IDs are excluded. +The existing diagnostic path also writes this message to sidecar stderr. +A controller that has already stopped may not ingest the frame. Stdin EOF +identifies a closed controller pipe, not the upstream reason for that closure. +This is local run evidence; no first-party Telemetry event or OpenTelemetry +span is added. Retry, cancellation and authorization behavior stay unchanged. + ## Startup reuse evidence `run.startup.reuse` uses schema `paperclip.startup-reuse.v1`. Each receipt names diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index 6b6f92d2059..b73318e6f5c 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -193,6 +193,18 @@ and terminal settlement at the process boundary. Copilot and Pi remain gated. Verified distributions are build-owned; no provider accepts an arbitrary executable. See [the rich ACP capability report](../../doc/architecture/runner-rich-acp-capabilities.md). +On stdin EOF, SIGTERM, SIGINT, or an internal shutdown request, the sidecar emits +`sidecar_shutdown_requested` once through `runtime.diagnostic` and stderr before +cleanup removes pending calls. The JSON message gives the shutdown reason, +pending tool/input/permission counts, completion-normalization count, and at most +16 oldest tool calls with elapsed milliseconds and SHA-256 call IDs. Hash a +`runtime.tool_called` call ID to correlate it with this snapshot. Arguments, +results, operation names, and raw call IDs are excluded. This diagnostic records +the local trigger; stdin EOF alone does not identify why the controller stopped. +A broken output pipe can prevent event delivery; consult retained stderr too. +It does not change retries, cancellation, or tool authorization. No saved session +shape changes; sessions from the previous release retain their existing behavior. + If the controller sends assigned MCP tools in the PRP catalog, the sidecar uses its authenticated tool bridge. Isolated provider admission accepts this route without a separate native MCP endpoint. diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts index 784b98e8c26..d761f886c2e 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts @@ -1054,6 +1054,172 @@ describe("qualified ACPX runtime sidecar", () => { expect(sidecar.stderr()).not.toContain("ERR_USE_AFTER_CLOSE"); }); + it.each(["stdin", "SIGTERM", "SIGINT"] as const)("records pending calls before %s shutdown rejects them", async trigger => { + const root = mkdtempSync(join(tmpdir(), "acpx-shutdown-")); + const loader = join(root, "loader.mjs"), fixture = join(root, "host.mjs"); + const base = new URL("../test-support/acpx-blocked-output-host.mjs", import.meta.url).href; + writeFileSync(fixture, `import { AcpxRuntimeHost as Base } from ${JSON.stringify(base)}; + import { appendFileSync } from "node:fs"; + import { join } from "node:path"; + export class AcpxRuntimeHost extends Base { + static async open(options) { const host = await Base.open(options); Object.setPrototypeOf(host, this.prototype); return host; } + startTurn({ requestId }) { + for (const callId of ["resolved", "cancelled", "pending-1", "pending-2"]) { + const controller = new AbortController(); + this.options.semanticTools.handler({callId, tool:"call_api", arguments:{secret:"PRIVATE_REQUEST_BODY"}, signal:controller.signal}) + .then(() => appendFileSync(join(process.env.PAPERCLIP_TEST_OUTPUT_ROOT,"outcomes"), callId+":resolved\\n"), + error => appendFileSync(join(process.env.PAPERCLIP_TEST_OUTPUT_ROOT,"outcomes"), callId+":"+error.message+"\\n")); + if (callId === "cancelled") controller.abort(); + } + return {requestId, promptStarted:Promise.resolve(), result:this.result, events:(async function*(){})()}; + } + }`); + writeFileSync(loader, `import { registerHooks } from "node:module"; import { pathToFileURL } from "node:url"; + registerHooks({resolve(specifier,context,next){ + if(context.parentURL?.includes("/acpx-runtime-sidecar.") && specifier.endsWith("/runtime-host.js")) + return {url:pathToFileURL(${JSON.stringify(fixture)}).href,shortCircuit:true}; + return next(specifier,context); + }});`); + const sidecar = new SidecarProcess({loader, environment:{PAPERCLIP_TEST_OUTPUT_ROOT:root}}); + children.add(sidecar); + const send = async (id:number, command:string, params:Record) => { + sidecar.write({protocolVersion:ACPX_SIDECAR_PROTOCOL_VERSION,id,command,params}); + expect(await sidecar.next(frame=>frame.id === id)).toMatchObject({ok:true}); + }; + try { + sidecar.write(initializeRequest(1,"codex")); + expect(await sidecar.next(frame=>frame.id === 1)).toMatchObject({ok:true}); + await send(2,"session.open",{agent:"codex",model:"gpt-5.6-sol",runtimeDirectory:root,workingDirectory:root, + normalizedSessionId:"shutdown-session",permissionMode:"deny-all",systemInstructions:"Test shutdown.",tools:[]}); + await send(3,"run.attach",{runId:"shutdown-run",catalogRevision:1,tools:[]}); + await send(4,"turn.start",{turnId:"shutdown-turn",message:"Test pending calls."}); + await sidecar.next(frame=>frame.eventType === "runtime.tool_called" && (frame.payload as Record).callId === "pending-2"); + await send(5,"tool.resolve",{turnId:"shutdown-turn",callId:"resolved",result:{private:"PRIVATE_RESPONSE_BODY"}}); + await expect(sidecar.shutdown(trigger)).resolves.toEqual({code:0, signal:null}); + expect(sidecar.stderr()).toContain("sidecar_shutdown_requested"); + const observation = await sidecar.next(frame=>frame.eventType === "runtime.diagnostic" && (frame.payload as Record).code === "sidecar_shutdown_requested"); + expect(observation).toMatchObject({runId:"shutdown-run",turnId:"shutdown-turn"}); + const snapshot = JSON.parse((observation.payload as {message:string}).message); + expect(snapshot).toMatchObject({reason:trigger === "stdin" ? "sidecar stdin closed" : `sidecar received ${trigger}`, + pendingTools:2,pendingInputs:0,pendingPermissions:0,normalizingTools:0,truncatedTools:0}); + expect(snapshot.tools.map((tool:{callIdSha256:string})=>tool.callIdSha256)).toEqual( + ["pending-1","pending-2"].map(id=>createHash("sha256").update(id).digest("hex"))); + expect(snapshot.tools.every((tool:{elapsedMs:number})=>Number.isFinite(tool.elapsedMs) && tool.elapsedMs >= 0)).toBe(true); + expect(sidecar.stderr()).not.toContain("PRIVATE_"); + expect(sidecar.pending(frame=>frame.eventType === "runtime.diagnostic" && (frame.payload as Record).code === "sidecar_shutdown_requested")).toBe(0); + const outcomes = readFileSync(join(root,"outcomes"),"utf8"); + expect(outcomes).toContain("resolved:resolved"); + expect(outcomes).toContain("cancelled:ACPX tool call was cancelled"); + expect(outcomes).toContain(`pending-1:${snapshot.reason}`); + expect(outcomes).toContain(`pending-2:${snapshot.reason}`); + expect(JSON.parse(readFileSync(join(root,"provider-cleanup.json"),"utf8"))).toMatchObject({closeCount:1,code:0}); + } finally { await sidecar.close(); rmSync(root,{recursive:true,force:true}); } + }, 15000); + + it("finishes cleanup when the controller closed the output pipe before shutdown", async () => { + const root = mkdtempSync(join(tmpdir(), "acpx-shutdown-")); + const loader = join(root, "loader.mjs"), fixture = join(root, "host.mjs"); + const base = new URL("../test-support/acpx-blocked-output-host.mjs", import.meta.url).href; + writeFileSync(fixture, `import { AcpxRuntimeHost as Base } from ${JSON.stringify(base)}; + import { appendFileSync } from "node:fs"; + import { join } from "node:path"; + export class AcpxRuntimeHost extends Base { + static async open(options) { const host = await Base.open(options); Object.setPrototypeOf(host, this.prototype); return host; } + startTurn({ requestId }) { + for (const callId of ["resolved", "cancelled", "pending-1", "pending-2"]) { + const controller = new AbortController(); + this.options.semanticTools.handler({callId, tool:"call_api", arguments:{secret:"PRIVATE_REQUEST_BODY"}, signal:controller.signal}) + .then(() => appendFileSync(join(process.env.PAPERCLIP_TEST_OUTPUT_ROOT,"outcomes"), callId+":resolved\\n"), + error => appendFileSync(join(process.env.PAPERCLIP_TEST_OUTPUT_ROOT,"outcomes"), callId+":"+error.message+"\\n")); + if (callId === "cancelled") controller.abort(); + } + return {requestId, promptStarted:Promise.resolve(), result:this.result, events:(async function*(){})()}; + } + }`); + writeFileSync(loader, `import { registerHooks } from "node:module"; import { pathToFileURL } from "node:url"; + registerHooks({resolve(specifier,context,next){ + if(context.parentURL?.includes("/acpx-runtime-sidecar.") && specifier.endsWith("/runtime-host.js")) + return {url:pathToFileURL(${JSON.stringify(fixture)}).href,shortCircuit:true}; + return next(specifier,context); + }});`); + const sidecar = new SidecarProcess({loader, environment:{PAPERCLIP_TEST_OUTPUT_ROOT:root}}); + children.add(sidecar); + const send = async (id:number, command:string, params:Record) => { + sidecar.write({protocolVersion:ACPX_SIDECAR_PROTOCOL_VERSION,id,command,params}); + expect(await sidecar.next(frame=>frame.id === id)).toMatchObject({ok:true}); + }; + try { + sidecar.write(initializeRequest(1,"codex")); + expect(await sidecar.next(frame=>frame.id === 1)).toMatchObject({ok:true}); + await send(2,"session.open",{agent:"codex",model:"gpt-5.6-sol",runtimeDirectory:root,workingDirectory:root, + normalizedSessionId:"shutdown-session",permissionMode:"deny-all",systemInstructions:"Test shutdown.",tools:[]}); + await send(3,"run.attach",{runId:"shutdown-run",catalogRevision:1,tools:[]}); + await send(4,"turn.start",{turnId:"shutdown-turn",message:"Test pending calls."}); + await sidecar.next(frame=>frame.eventType === "runtime.tool_called" && (frame.payload as Record).callId === "pending-2"); + sidecar.closeOutput(); + await expect(sidecar.shutdown("stdin")).resolves.toEqual({code:1, signal:null}); + expect(sidecar.stderr()).toContain("sidecar_shutdown_requested"); + expect(sidecar.stderr()).not.toMatch(/Unhandled 'error' event|EPIPE/); + const outcomes = readFileSync(join(root,"outcomes"),"utf8"); + expect(outcomes).toContain("pending-1:sidecar stdin closed"); + expect(outcomes).toContain("pending-2:sidecar stdin closed"); + expect(JSON.parse(readFileSync(join(root,"provider-cleanup.json"),"utf8"))).toMatchObject({closeCount:1,code:0}); + } finally { await sidecar.close(); rmSync(root,{recursive:true,force:true}); } + }, 15000); + + it("shuts down and finishes cleanup when the controller output pipe closes mid-turn", async () => { + const root = mkdtempSync(join(tmpdir(), "acpx-shutdown-")); + const loader = join(root, "loader.mjs"), fixture = join(root, "host.mjs"); + const base = new URL("../test-support/acpx-blocked-output-host.mjs", import.meta.url).href; + writeFileSync(fixture, `import { AcpxRuntimeHost as Base } from ${JSON.stringify(base)}; + import { appendFileSync } from "node:fs"; + import { join } from "node:path"; + export class AcpxRuntimeHost extends Base { + static async open(options) { const host = await Base.open(options); Object.setPrototypeOf(host, this.prototype); return host; } + startTurn({ requestId }) { + for (const callId of ["resolved", "cancelled", "pending-1", "pending-2"]) { + const controller = new AbortController(); + this.options.semanticTools.handler({callId, tool:"call_api", arguments:{secret:"PRIVATE_REQUEST_BODY"}, signal:controller.signal}) + .then(() => appendFileSync(join(process.env.PAPERCLIP_TEST_OUTPUT_ROOT,"outcomes"), callId+":resolved\\n"), + error => appendFileSync(join(process.env.PAPERCLIP_TEST_OUTPUT_ROOT,"outcomes"), callId+":"+error.message+"\\n")); + if (callId === "cancelled") controller.abort(); + } + return {requestId, promptStarted:Promise.resolve(), result:this.result, events:(async function*(){})()}; + } + }`); + writeFileSync(loader, `import { registerHooks } from "node:module"; import { pathToFileURL } from "node:url"; + registerHooks({resolve(specifier,context,next){ + if(context.parentURL?.includes("/acpx-runtime-sidecar.") && specifier.endsWith("/runtime-host.js")) + return {url:pathToFileURL(${JSON.stringify(fixture)}).href,shortCircuit:true}; + return next(specifier,context); + }});`); + const sidecar = new SidecarProcess({loader, environment:{PAPERCLIP_TEST_OUTPUT_ROOT:root}}); + children.add(sidecar); + const send = async (id:number, command:string, params:Record) => { + sidecar.write({protocolVersion:ACPX_SIDECAR_PROTOCOL_VERSION,id,command,params}); + expect(await sidecar.next(frame=>frame.id === id)).toMatchObject({ok:true}); + }; + try { + sidecar.write(initializeRequest(1,"codex")); + expect(await sidecar.next(frame=>frame.id === 1)).toMatchObject({ok:true}); + await send(2,"session.open",{agent:"codex",model:"gpt-5.6-sol",runtimeDirectory:root,workingDirectory:root, + normalizedSessionId:"shutdown-session",permissionMode:"deny-all",systemInstructions:"Test shutdown.",tools:[]}); + await send(3,"run.attach",{runId:"shutdown-run",catalogRevision:1,tools:[]}); + await send(4,"turn.start",{turnId:"shutdown-turn",message:"Test pending calls."}); + await sidecar.next(frame=>frame.eventType === "runtime.tool_called" && (frame.payload as Record).callId === "pending-2"); + sidecar.closeOutput(); + sidecar.write({protocolVersion:ACPX_SIDECAR_PROTOCOL_VERSION,id:5,command:"tool.resolve", + params:{turnId:"shutdown-turn",callId:"resolved",result:{private:"PRIVATE_RESPONSE_BODY"}}}); + await expect(sidecar.exited()).resolves.toEqual({code:1, signal:null}); + expect(sidecar.stderr()).toContain("sidecar_shutdown_requested"); + expect(sidecar.stderr()).not.toMatch(/Unhandled 'error' event|EPIPE/); + const outcomes = readFileSync(join(root,"outcomes"),"utf8"); + expect(outcomes).toContain("pending-1:ACPX sidecar output failed"); + expect(outcomes).toContain("pending-2:ACPX sidecar output failed"); + expect(JSON.parse(readFileSync(join(root,"provider-cleanup.json"),"utf8"))).toMatchObject({closeCount:1,code:0}); + } finally { await sidecar.close(); rmSync(root,{recursive:true,force:true}); } + }, 15000); + it("keeps session admission closed while any cleanup owner remains", () => { const cleanup = Promise.resolve(); @@ -1729,6 +1895,7 @@ class SidecarProcess { readonly #signals: Array<() => void> = []; #stderr = ""; #closed = false; + readonly #exit: Promise<{code:number|null;signal:NodeJS.Signals|null}>; constructor(options: {loader?:string;environment?:Record} = {}) { this.#child = spawn( @@ -1736,6 +1903,7 @@ class SidecarProcess { [...(options.loader ? ["--import","tsx","--import",options.loader] : []), fileURLToPath(new URL("./acpx-runtime-sidecar.ts", import.meta.url))], { stdio: ["pipe", "pipe", "pipe"], env: { PATH: process.env.PATH, LANG: "C.UTF-8", ...options.environment } }, ); + this.#exit = new Promise(resolve => this.#child.once("exit", (code, signal) => resolve({code,signal}))); let stdout = ""; this.#child.stdout.setEncoding("utf8"); this.#child.stdout.on("data", (chunk: string) => { @@ -1760,6 +1928,28 @@ class SidecarProcess { this.#child.stdin.write(`${JSON.stringify(value)}\n`); } + closeOutput(): void { + this.#child.stdout.destroy(); + } + + async shutdown(trigger: "stdin" | "SIGTERM" | "SIGINT") { + if (trigger === "stdin") this.#child.stdin.end(); + else this.#child.kill(trigger); + return this.exited(); + } + + async exited() { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + this.#exit, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("Sidecar did not exit without forced cleanup")), 5000); + }), + ]); + } finally { clearTimeout(timer); } + } + stderr(): string { return this.#stderr; } @@ -1797,7 +1987,7 @@ class SidecarProcess { } async close(): Promise { - if (this.#closed) return; + if (this.#closed || this.#child.exitCode !== null || this.#child.signalCode !== null) return; this.#closed = true; this.#child.stdin.end(); const exit = new Promise((resolve) => { diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts index 941ae867f17..75d63241aa1 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { recordSidecarShutdownThenQueue, sidecarShutdownSnapshot } from "./acpx-sidecar-shutdown.js"; import { classifyClaudeStatusBanner } from "../drivers/acpx/claude-status-banners.js"; import { resolveGrokDisableWebSearch } from "../drivers/acpx/grok-web-search.js"; import { toolFileChangesFromAcpContent, workspaceRootsFor, type ToolFileChange } from "../tool-file-changes.js"; @@ -120,6 +121,11 @@ const MAX_PENDING_INPUTS = 16; const MAX_BUFFERED_OUTPUT_BYTES = 4 * ACPX_SIDECAR_MAX_FRAME_BYTES; const MAX_EVENT_PAYLOAD_BYTES = 256 * 1024; let outputFailed = false; +process.stdout.on("error", () => { + outputFailed = true; + requestShutdown("ACPX sidecar output failed"); +}); +process.stderr.on("error", () => undefined); let goalSourceRevision = 0; function observedGoalProjection(...args: Parameters) { return { ...acpxGoalProjection(...args), providerRevision: ++goalSourceRevision }; @@ -145,6 +151,7 @@ function reportRetainedAcpxCleanupFailure( interface PendingTool { turnId: string; + startedAt: number; settle(value: unknown): void; reject(error: Error): void; cleanup(): void; @@ -225,12 +232,19 @@ process.once("SIGINT", () => { function requestShutdown(reason: string): void { if (shutdownRequested) return; shutdownRequested = true; - if (!inputClosed) lines.pause(); - pendingInput = enqueueAcpxSidecarInput( - pendingInput, - () => shutdown(reason), - (error) => diagnostic("sidecar_shutdown_failed", safeMessage(error)), - ); + if (!recordSidecarShutdownThenQueue( + () => diagnostic("sidecar_shutdown_requested", JSON.stringify(sidecarShutdownSnapshot( + reason, tools, inputs.size, permissions.size, normalizingTools.size, + ))), + () => { + if (!inputClosed) lines.pause(); + pendingInput = enqueueAcpxSidecarInput( + pendingInput, + () => shutdown(reason), + (error) => diagnostic("sidecar_shutdown_failed", safeMessage(error)), + ); + }, + )) outputFailed = true; } async function receiveLine(line: string): Promise { @@ -907,6 +921,7 @@ async function waitForTool(call: RunnerToolCall): Promise { call.signal.addEventListener("abort", abort, { once: true }); tools.set(callId, { turnId: activeTurnId, + startedAt: performance.now(), settle: (result) => { // A pipe write alone does not prove receiver admission. Only this // call's turn-bound tool.resolve success confirms runnerd accepted @@ -939,6 +954,7 @@ async function waitForTool(call: RunnerToolCall): Promise { call.signal.addEventListener("abort", abort, { once: true }); tools.set(callId, { turnId: activeTurnId, + startedAt: performance.now(), settle, reject, cleanup: () => call.signal.removeEventListener("abort", abort), diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-shutdown.test.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-shutdown.test.ts new file mode 100644 index 00000000000..6ceec2fef73 --- /dev/null +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-shutdown.test.ts @@ -0,0 +1,46 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { recordSidecarShutdownThenQueue, sidecarShutdownSnapshot } from "./acpx-sidecar-shutdown.js"; + +describe("sidecar shutdown snapshot", () => { + it("bounds samples, keeps the oldest calls and omits private data", () => { + const tools = new Map(Array.from({ length: 512 }, (_, index) => [ + `private-call-${index}`, + { startedAt: 1000 - index, arguments: "PRIVATE_INPUT", result: "PRIVATE_OUTPUT" }, + ])); + const snapshot = sidecarShutdownSnapshot("sidecar stdin closed", tools, 3, 2, 1, 2000); + expect(snapshot).toMatchObject({ pendingTools: 512, pendingInputs: 3, pendingPermissions: 2, + normalizingTools: 1, truncatedTools: 496 }); + expect(snapshot.tools).toHaveLength(16); + expect(snapshot.tools[0]).toEqual({ + callIdSha256: createHash("sha256").update("private-call-511").digest("hex"), elapsedMs: 1511, + }); + expect(snapshot.tools[15]?.elapsedMs).toBe(1496); + const encoded = JSON.stringify(snapshot); + expect(Buffer.byteLength(encoded)).toBeLessThan(4096); + expect(encoded).not.toMatch(/PRIVATE|private-call/); + expect(tools.size).toBe(512); + }); + + it("records an idle shutdown without retaining completed calls", () => { + expect(sidecarShutdownSnapshot("sidecar received SIGTERM", new Map(), 0, 0, 0)).toEqual({ + reason: "sidecar received SIGTERM", pendingTools: 0, pendingInputs: 0, + pendingPermissions: 0, normalizingTools: 0, tools: [], truncatedTools: 0, + }); + }); + + it("records the snapshot before queueing shutdown", () => { + const order: string[] = []; + expect(recordSidecarShutdownThenQueue(() => order.push("record"), () => order.push("queue"))).toBe(true); + expect(order).toEqual(["record", "queue"]); + }); + + it("still queues shutdown when the snapshot write throws", () => { + const order: string[] = []; + expect(recordSidecarShutdownThenQueue(() => { + order.push("record"); + throw new Error("EPIPE: stdout closed"); + }, () => order.push("queue"))).toBe(false); + expect(order).toEqual(["record", "queue"]); + }); +}); diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-shutdown.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-shutdown.ts new file mode 100644 index 00000000000..28102af10cf --- /dev/null +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-shutdown.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; + +const MAX_SHUTDOWN_TOOL_SAMPLES = 16; + +export function sidecarShutdownSnapshot( + reason: string, + tools: ReadonlyMap, + pendingInputs: number, + pendingPermissions: number, + normalizingTools: number, + now = performance.now(), +) { + const samples = [...tools] + .sort((left, right) => left[1].startedAt - right[1].startedAt) + .slice(0, MAX_SHUTDOWN_TOOL_SAMPLES) + .map(([callId, pending]) => ({ + callIdSha256: createHash("sha256").update(callId).digest("hex"), + elapsedMs: Math.max(0, Math.floor(now - pending.startedAt)), + })); + return { + reason, + pendingTools: tools.size, + pendingInputs, + pendingPermissions, + normalizingTools, + tools: samples, + truncatedTools: tools.size - samples.length, + }; +} + +export function recordSidecarShutdownThenQueue(record: () => void, queueShutdown: () => void): boolean { + let recorded = true; + try { + record(); + } catch { + recorded = false; + } + queueShutdown(); + return recorded; +}