diff --git a/doc/observability.md b/doc/observability.md index a2c5a8bdc83..d503db0f7dc 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1386,6 +1386,18 @@ different tools with different permission rules. Read them this way: Both fields hold only counts. +A sandbox ACP run also records how its agent process session ended, when the +host bridge saw it end before finalization. Startup failures (`create_runtime`, +`ensure_session`, `configure_session`) carry the same fields, so an agent process +that dies during the handshake is reported too. `acpProcessTerminalSource` is one of +`remote_event`, `input_delivery`, `output_poll`, `output_stream`, or +`proxy_close`, and `acpProcessTerminalOutcome` is `exit` or `error`. +`acpProcessTerminalExitCode` (0–255) and `acpProcessTerminalSignal` (a closed +POSIX signal name) are present only when known. These are the same values as the +run log's `ACP process session terminal:` line. They separate a killed or exited +agent process from a host proxy close after a generic ACP connection failure. The +server drops all of them if the source is not a closed label. + ACP and legacy OpenCode results retain the adapter's resolved wall-clock timeout as `adapterExecutionTimeout` in the instance run result. Finalization uses it for `effectiveTimeoutSec`, `timeoutSource`, and `timeoutConfigured`. Sources are diff --git a/packages/adapter-utils/src/acpx-engine/constants.ts b/packages/adapter-utils/src/acpx-engine/constants.ts index 47a67eb143f..de93353ce09 100644 --- a/packages/adapter-utils/src/acpx-engine/constants.ts +++ b/packages/adapter-utils/src/acpx-engine/constants.ts @@ -57,6 +57,17 @@ export const ACP_TOOL_KINDS = [ "read", "edit", "delete", "move", "search", "execute", "think", "fetch", "switch_mode", "other", ] as const; +// Closed labels for how a sandbox agent process session ended. Run failure +// reports copy only these values. +export const ACP_PROCESS_TERMINAL_SOURCES = [ + "remote_event", "input_delivery", "output_poll", "output_stream", "proxy_close", +] as const; + +export const ACP_PROCESS_TERMINAL_SIGNALS = [ + "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGABRT", "SIGFPE", "SIGKILL", + "SIGSEGV", "SIGPIPE", "SIGALRM", "SIGTERM", "SIGBUS", "SIGXCPU", "SIGXFSZ", +] as const; + export const ACPX_ADAPTER_AGENT_IDS = { claude_local: "claude", codex_local: "codex", diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index c480b9fa002..048440df37e 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -36,6 +36,7 @@ import { referencedSourceContentSignature, resolveStalledToolTimeoutMs, rewriteGeminiAcpFlagForVersion, + summarizeAcpProcessTerminal, summarizeAcpStopActivity, summarizeAcpxTurnUsage, type AcpxEngineExecutorOptions, @@ -4034,6 +4035,39 @@ describe("summarizeAcpStopActivity", () => { }); }); +describe("summarizeAcpProcessTerminal", () => { + it("reports nothing before the process session records terminal evidence", () => { + expect(summarizeAcpProcessTerminal(null)).toEqual({}); + expect(summarizeAcpProcessTerminal(undefined)).toEqual({}); + }); + + it("reports a signal-killed remote process", () => { + expect(summarizeAcpProcessTerminal({ + source: "remote_event", outcome: "exit", exitCode: "unknown", signal: "SIGKILL", + })).toEqual({ + acpProcessTerminalSource: "remote_event", + acpProcessTerminalOutcome: "exit", + acpProcessTerminalSignal: "SIGKILL", + }); + }); + + it("reports an exit code and omits an unknown signal", () => { + expect(summarizeAcpProcessTerminal({ + source: "remote_event", outcome: "exit", exitCode: 137, signal: "unknown", + })).toEqual({ + acpProcessTerminalSource: "remote_event", + acpProcessTerminalOutcome: "exit", + acpProcessTerminalExitCode: 137, + }); + }); + + it("reports a host proxy close with no exit details", () => { + expect(summarizeAcpProcessTerminal({ + source: "proxy_close", outcome: "error", exitCode: "unknown", signal: "unknown", + })).toEqual({ acpProcessTerminalSource: "proxy_close", acpProcessTerminalOutcome: "error" }); + }); +}); + describe("summarizeAcpxTurnUsage", () => { it("uses the post-turn amount alone when the cumulative cost counter reset", () => { const summary = summarizeAcpxTurnUsage({ @@ -8130,12 +8164,13 @@ describe("ACPX engine sandbox bridge run-disposition seam (fail-closed)", () => runtime: unknown, sandbox: Awaited>, deps: Partial = {}, + processSessionBridge: Record = {}, ) { vi.mocked(startAdapterExecutionTargetPaperclipBridge).mockImplementationOnce( async () => handle as never, ); vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mockImplementationOnce( - async () => ({ agentCommand: null, stop: async () => {} }) as never, + async () => ({ agentCommand: null, stop: async () => {}, ...processSessionBridge }) as never, ); const execute = createAcpxEngineExecutor({ createRuntime: () => runtime as never, @@ -8350,6 +8385,36 @@ describe("ACPX engine sandbox bridge run-disposition seam (fail-closed)", () => expect(result.errorCode).not.toBe("acpx_handshake_timeout"); }, 10000); + it("reports how the agent process session ended when the process exits during the handshake", async () => { + const sandbox = await setupRemoteSandbox(); + const fake = createFakeBridgeHandle(); + const runtime = { + ensureSession: async () => { + throw new Error("agent process exited during initialize"); + }, + startTurn: () => { + throw new Error("startTurn must not run after a failed handshake"); + }, + setConfigOption: async () => {}, + close: async () => {}, + }; + + const result = await runRemote(fake.handle, runtime, sandbox, {}, { + readTerminalEvidence: () => ({ + source: "remote_event", outcome: "exit", exitCode: 137, signal: "SIGKILL", + }), + }); + + expect(result.exitCode).not.toBe(0); + expect(result.resultJson).toMatchObject({ + phase: "ensure_session", + acpProcessTerminalSource: "remote_event", + acpProcessTerminalOutcome: "exit", + acpProcessTerminalExitCode: 137, + acpProcessTerminalSignal: "SIGKILL", + }); + }, 10000); + it("aborts an in-flight turn and fails the run when the duplex channel latches a loss mid-turn", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeBridgeHandle(); diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index 9fe1f8184a8..3db5489d12b 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -37,6 +37,7 @@ import { type AdapterExecutionTarget, type AdapterExecutionTargetPaperclipBridgeHandle, type AdapterExecutionTargetProcessSessionBridgeHandle, + type AdapterExecutionTargetProcessSessionTerminalEvidence, type AdapterExecutionTargetTimeoutResolution, type AdapterManagedRuntimeAsset, type PreparedAdapterExecutionTargetRuntime, @@ -3318,6 +3319,28 @@ function isPendingAcpTool(tool: { status?: string }): boolean { return tool.status !== "completed" && tool.status !== "failed" && tool.status !== "cancelled"; } +/** + * How the sandbox agent process session ended, when the bridge saw it end. + * Separates a process that exited or was killed from a host proxy that closed + * by itself. Unknown exit codes and signals are left out. + */ +export function summarizeAcpProcessTerminal( + evidence: AdapterExecutionTargetProcessSessionTerminalEvidence | null | undefined, +): { + acpProcessTerminalSource?: AdapterExecutionTargetProcessSessionTerminalEvidence["source"]; + acpProcessTerminalOutcome?: AdapterExecutionTargetProcessSessionTerminalEvidence["outcome"]; + acpProcessTerminalExitCode?: number; + acpProcessTerminalSignal?: string; +} { + if (!evidence) return {}; + return { + acpProcessTerminalSource: evidence.source, + acpProcessTerminalOutcome: evidence.outcome, + ...(typeof evidence.exitCode === "number" ? { acpProcessTerminalExitCode: evidence.exitCode } : {}), + ...(evidence.signal !== "unknown" ? { acpProcessTerminalSignal: evidence.signal } : {}), + }; +} + /** * Activity as of the moment the turn's stop fired, for a stopped turn only. * Separates a turn that went silent long before its deadline from one that @@ -4783,7 +4806,10 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession, - resultJson: { phase: "ensure_session" }, + resultJson: { + phase: "ensure_session", + ...summarizeAcpProcessTerminal(prepared.processSessionBridge?.readTerminalEvidence?.()), + }, summary: message, }; return settleFor("handshake", err); @@ -4813,7 +4839,10 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { ...billingFields, ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, - resultJson: { phase: "ensure_session" }, + resultJson: { + phase: "ensure_session", + ...summarizeAcpProcessTerminal(prepared.processSessionBridge?.readTerminalEvidence?.()), + }, summary: "ACPX did not return a runtime session handle.", }; return settleFor( @@ -4853,7 +4882,10 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession, - resultJson: { phase: "create_runtime" }, + resultJson: { + phase: "create_runtime", + ...summarizeAcpProcessTerminal(prepared.processSessionBridge?.readTerminalEvidence?.()), + }, summary: message, }; return settleFor("runtime_create", err); @@ -4910,6 +4942,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { requestedModel: prepared.requestedModel || null, requestedThinkingEffort: prepared.requestedThinkingEffort || null, fastMode: prepared.fastMode, + ...summarizeAcpProcessTerminal(prepared.processSessionBridge?.readTerminalEvidence?.()), }, summary: message, }; @@ -5282,6 +5315,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { acpPendingToolCount: pendingTools.length, acpToolInventoryComplete: !incompleteToolInventory, ...summarizeAcpStopActivity(turnStopRequestedAt, lastRuntimeEventBeforeStopAt, pendingToolsAtStop ?? pendingTools), + ...summarizeAcpProcessTerminal(prepared.processSessionBridge?.readTerminalEvidence?.()), }; if (input.kind === "terminal") { const terminal = input.terminal; diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 377be29a853..1299a653968 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -1126,6 +1126,12 @@ describe("sandbox adapter execution targets", () => { expect(terminal).toEqual([ `[paperclip] ACP process session terminal: source=remote_event outcome=${exitCode === null ? "error" : "exit"} exitCode=${exitCode ?? "unknown"} signal=unknown.\n`, ]); + expect(bridge!.readTerminalEvidence?.()).toEqual({ + source: "remote_event", + outcome: exitCode === null ? "error" : "exit", + exitCode: exitCode ?? "unknown", + signal: "unknown", + }); if (exitCode === null) { expect(result.stdout).toBe(""); expect(result.stderr).toContain("ENOENT"); @@ -1313,6 +1319,9 @@ describe("sandbox adapter execution targets", () => { if (!authenticated) expect(logs).toEqual([]); } await waitForCondition(() => logs.length === 1, "Missing proxy-close diagnostic."); + expect(bridge!.readTerminalEvidence?.()).toEqual({ + source: "proxy_close", outcome: "error", exitCode: "unknown", signal: "unknown", + }); await bridge!.stop(); expect(logs.filter((line) => line.includes("ACP process session terminal:"))).toEqual([ "[paperclip] ACP process session terminal: source=proxy_close outcome=error exitCode=unknown signal=unknown.\n", diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index d0b66bf7fc1..3abae87bf63 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -7,6 +7,7 @@ import os from "node:os"; import path from "node:path"; import { randomBytes, randomUUID } from "node:crypto"; import { githubLauncherSource } from "./github-launcher.js"; +import { ACP_PROCESS_TERMINAL_SIGNALS, type ACP_PROCESS_TERMINAL_SOURCES } from "./acpx-engine/constants.js"; import type { SshRemoteExecutionSpec } from "./ssh.js"; import { prepareCommandManagedRuntime, @@ -377,8 +378,19 @@ export interface AdapterExecutionTargetPaperclipBridgeHandle { stop(): Promise; } +// First terminal evidence a sandbox process session observed. Every field is a +// closed value: no message, command, path, payload, or provider error text. +export interface AdapterExecutionTargetProcessSessionTerminalEvidence { + source: (typeof ACP_PROCESS_TERMINAL_SOURCES)[number]; + outcome: "exit" | "error"; + exitCode: number | "unknown"; + signal: (typeof ACP_PROCESS_TERMINAL_SIGNALS)[number] | "unknown"; +} + export interface AdapterExecutionTargetProcessSessionBridgeHandle { agentCommand: string; + // Returns null until the session records terminal evidence. + readTerminalEvidence?(): AdapterExecutionTargetProcessSessionTerminalEvidence | null; stop(): Promise; } @@ -2237,22 +2249,19 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { } }; let pollTimer: NodeJS.Timeout | null = null; - let terminalEvidenceRecorded = false; + let terminalEvidence: AdapterExecutionTargetProcessSessionTerminalEvidence | null = null; const recordTerminalEvidence = ( - source: "remote_event" | "input_delivery" | "output_poll" | "output_stream" | "proxy_close", + source: AdapterExecutionTargetProcessSessionTerminalEvidence["source"], event: { type?: string; code?: number | null; signal?: string | null }, ) => { - if (terminalEvidenceRecorded) return; - terminalEvidenceRecorded = true; + if (terminalEvidence) return; // Remote frames are untrusted. Keep the retained diagnostic bounded and // independent of messages, commands, paths, payloads, and provider errors. const outcome = event.type === "exit" ? "exit" : "error"; const code = typeof event.code === "number" && Number.isInteger(event.code) && event.code >= 0 && event.code <= 255 ? event.code : "unknown"; - const signal = typeof event.signal === "string" && [ - "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGABRT", "SIGFPE", "SIGKILL", - "SIGSEGV", "SIGPIPE", "SIGALRM", "SIGTERM", "SIGBUS", "SIGXCPU", "SIGXFSZ", - ].includes(event.signal) ? event.signal : "unknown"; + const signal = ACP_PROCESS_TERMINAL_SIGNALS.find((name) => name === event.signal) ?? "unknown"; + terminalEvidence = { source, outcome, exitCode: code, signal }; logFailureWithoutWaiting( `[paperclip] ACP process session terminal: source=${source} outcome=${outcome} exitCode=${code} signal=${signal}.\n`, ); @@ -2631,6 +2640,7 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { return { agentCommand, + readTerminalEvidence: () => terminalEvidence, stop: async () => { stopping = true; // End the `sandbox.agentProcess` span now, before the caller ends the run diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts index f9228e50ea3..5c6734a4003 100644 --- a/server/src/services/__tests__/run-failure-diagnostics.test.ts +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -481,6 +481,37 @@ describe("run failure diagnostics", () => { }, ); + it("includes how the sandbox agent process session ended", () => { + const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { + acpProcessTerminalSource: "remote_event", acpProcessTerminalOutcome: "exit", + acpProcessTerminalExitCode: 137, acpProcessTerminalSignal: "SIGKILL", + } }), {})); + expect(result.execution).toEqual({ + acpProcessTerminalSource: "remote_event", acpProcessTerminalOutcome: "exit", + acpProcessTerminalExitCode: 137, acpProcessTerminalSignal: "SIGKILL", + }); + }); + + it.each(["private source", "", "PROXY_CLOSE", ["proxy_close"], 1, null])( + "omits every process terminal field when the source is not a closed label (%j)", (value) => { + const result = collectRunFailureDiagnostics(run({ resultJson: { + acpProcessTerminalSource: value, acpProcessTerminalOutcome: "exit", + acpProcessTerminalExitCode: 1, acpProcessTerminalSignal: "SIGTERM", + } }), {}); + expect(result.execution).toEqual({}); + }, + ); + + it.each([-1, 256, 1.5, NaN, "1", "private", "SIGPRIVATE", "sigkill", null])( + "omits invalid process terminal details (%j)", (value) => { + const result = collectRunFailureDiagnostics(run({ resultJson: { + acpProcessTerminalSource: "proxy_close", acpProcessTerminalOutcome: value, + acpProcessTerminalExitCode: value, acpProcessTerminalSignal: value, + } }), {}); + expect(result.execution).toEqual({ acpProcessTerminalSource: "proxy_close" }); + }, + ); + it.each(["execute,private command", "", "execute,", "Execute", ["execute"], 1])( "omits pending tool kinds that are not closed labels (%j)", (value) => { const result = collectRunFailureDiagnostics(run({ resultJson: { acpPendingToolKinds: value } }), {}); diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index d87f62e21f8..a7864795c94 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -6,7 +6,11 @@ import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js"; import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared"; import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction"; import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics"; -import { ACP_TOOL_KINDS } from "@paperclipai/adapter-utils/acpx-engine/constants"; +import { + ACP_PROCESS_TERMINAL_SIGNALS, + ACP_PROCESS_TERMINAL_SOURCES, + ACP_TOOL_KINDS, +} from "@paperclipai/adapter-utils/acpx-engine/constants"; import { redactCurrentUserText } from "../log-redaction.js"; import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js"; import { readNativeModelRejectionDiagnostic, readNativeTurnFailureDiagnostic } from "./native-runtime/native-provider-failure.js"; @@ -62,6 +66,22 @@ function readAcpToolKind(value: unknown): string | null { return typeof value === "string" && ACP_PENDING_TOOL_KIND_LABELS.has(value) ? value : null; } +/** How the sandbox agent process session ended, as closed labels and a bounded exit code. */ +function readAcpProcessTerminal(result: unknown): Context { + const source = ACP_PROCESS_TERMINAL_SOURCES.find((label) => label === read(result, "acpProcessTerminalSource")); + if (!source) return {}; + const fields: Context = { acpProcessTerminalSource: source }; + const outcome = read(result, "acpProcessTerminalOutcome"); + if (outcome === "exit" || outcome === "error") fields.acpProcessTerminalOutcome = outcome; + const exitCode = read(result, "acpProcessTerminalExitCode"); + if (typeof exitCode === "number" && Number.isInteger(exitCode) && exitCode >= 0 && exitCode <= 255) { + fields.acpProcessTerminalExitCode = exitCode; + } + const signal = ACP_PROCESS_TERMINAL_SIGNALS.find((label) => label === read(result, "acpProcessTerminalSignal")); + if (signal) fields.acpProcessTerminalSignal = signal; + return fields; +} + function read(value: unknown, key: string): unknown { if (!value || typeof value !== "object") return undefined; try { @@ -322,6 +342,7 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport } const closeSettlement = readNativeCloseSettlement(options.nativeCloseFailure); if (closeSettlement) Object.assign(execution, closeSettlement); + Object.assign(execution, readAcpProcessTerminal(result)); const pendingToolKinds = readAcpPendingToolKinds(read(result, "acpPendingToolKinds")); if (pendingToolKinds) execution.acpPendingToolKinds = pendingToolKinds; const stalledToolKind = readAcpToolKind(read(result, "acpStalledToolKind"));