diff --git a/doc/observability.md b/doc/observability.md index 7703888f8fa..1052b718785 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1008,6 +1008,25 @@ before the request reached Paperclip. A call whose request arrived stalled later: on the reply's return trip, or after the agent received the decision. Both fields hold only a boolean and a count. +A missing request is informative only if calls like it normally send one. So a +stalled run also records `acpSettledSameKindToolCount` (how many finished calls +of the stalled call's kind the run saw) and +`acpSettledSameKindToolPermissionRequestedCount` (how many of those sent a +permission request). These counts are supporting evidence, not proof of where +the call stalled. The ACP kind is coarse, so peers of the same kind can be +different tools with different permission rules. Read them this way: + +- All or nearly all finished peers sent a request, and the stalled call did not: + this supports a stall before the request reached Paperclip. +- No finished peer sent a request: calls of that kind may skip the request, so + its absence on the stalled call says nothing about where it stalled. +- Some finished peers sent a request and others did not: the counts do not show + whether the stalled call should have sent one. Compare the specific tools in + the run log before drawing a conclusion. +- No finished peers (a count of zero): there is no comparison. + +Both fields hold only counts. + ACP and legacy OpenCode results retain the adapter's resolved wall-clock timeout as `adapterExecutionTimeout` in the instance run result. Finalization uses it for `effectiveTimeoutSec`, `timeoutSource`, and `timeoutConfigured`. Sources are diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 3d58d77bd4a..7ef0e510688 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -3539,6 +3539,16 @@ describe("ACP activity diagnostics", () => { inferredKind: "think", } as never, { signal: new AbortController().signal })); yield { type: "tool_call", toolCallId: "private-task", title: "private task", kind: "think", status: "in_progress" }; + for (const toolCallId of ["private-peer-asked", "private-peer-shell"]) { + await options.onPermissionRequest?.({ + sessionId: "session-1", + raw: { sessionId: "session-1", toolCall: { toolCallId }, options: [] }, + inferredKind: "other", + } as never, { signal: new AbortController().signal }); + } + yield { type: "tool_call", toolCallId: "private-peer-asked", kind: "other", status: "completed" }; + yield { type: "tool_call", toolCallId: "private-peer-silent", kind: "other", status: "failed" }; + yield { type: "tool_call", toolCallId: "private-peer-shell", kind: "execute", status: "completed" }; yield { type: "tool_call", toolCallId: "private-call", title: "private lookup", kind: "other", status: "pending" }; currentNow = 7000; yield { type: "tool_call", tag: "tool_call_update", toolCallId: "private-task", status: "in_progress" }; @@ -3572,6 +3582,8 @@ describe("ACP activity diagnostics", () => { acpPendingToolKinds: "other,think", acpStalledToolPermissionRequested: false, acpPendingToolPermissionRequestedCount: 1, + acpSettledSameKindToolCount: 2, + acpSettledSameKindToolPermissionRequestedCount: 1, }); // The host only observes the request; ACPX's permission mode decides it. expect(permissionDecisions).toEqual([undefined]); @@ -3669,6 +3681,8 @@ describe("ACP activity diagnostics", () => { acpStalledToolKind: "other", acpStalledToolPermissionRequested: true, acpPendingToolPermissionRequestedCount: 1, + acpSettledSameKindToolCount: 0, + acpSettledSameKindToolPermissionRequestedCount: 0, }); expect(logs.join("")).toContain("Its permission request reached Paperclip."); }, 15_000); diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index 82c232930dc..425f802e153 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -4137,6 +4137,8 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { silentMs: number; permissionRequested: boolean; pendingPermissionRequested: number; + settledSameKind: number; + settledSameKindPermissionRequested: number; } | null = null; let stalledToolTimer: ReturnType | undefined; let forcedStop = false; @@ -5055,12 +5057,21 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { } if (!oldest) return; clearInterval(stalledToolTimer); + let settledSameKind = 0; + let settledSameKindPermissionRequested = 0; + for (const [toolCallId, tool] of interruptionTools) { + if (isPendingAcpTool(tool) || tool.kind !== oldest.kind) continue; + settledSameKind += 1; + if (permissionRequestedToolCallIds.has(toolCallId)) settledSameKindPermissionRequested += 1; + } const knownKinds: readonly string[] = ACP_TOOL_KINDS; stalledTool = { kind: oldest.kind && knownKinds.includes(oldest.kind) ? oldest.kind : "unknown", silentMs: oldest.silentMs, permissionRequested: permissionRequestedToolCallIds.has(oldest.toolCallId), pendingPermissionRequested, + settledSameKind, + settledSameKindPermissionRequested, }; turnStopRequestedAt = checkedAt; pendingToolsAtStop = [...interruptionTools.values()].filter(isPendingAcpTool); @@ -5352,6 +5363,8 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { acpStalledToolSilentMs: stalledTool.silentMs, acpStalledToolPermissionRequested: stalledTool.permissionRequested, acpPendingToolPermissionRequestedCount: stalledTool.pendingPermissionRequested, + acpSettledSameKindToolCount: stalledTool.settledSameKind, + acpSettledSameKindToolPermissionRequestedCount: stalledTool.settledSameKindPermissionRequested, } : {}), ...(failureDiagnostic ? { terminalSessionFailure: failureDiagnostic } : {}), diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts index d60f455bd7b..9ffec4f367d 100644 --- a/server/src/services/__tests__/run-failure-diagnostics.test.ts +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -462,10 +462,12 @@ describe("run failure diagnostics", () => { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { acpStalledToolKind: "other", acpStalledToolSilentMs: 1_800_000, acpStalledToolTitle: "private lookup", acpStalledToolPermissionRequested: false, acpPendingToolPermissionRequestedCount: 3, + acpSettledSameKindToolCount: 30, acpSettledSameKindToolPermissionRequestedCount: 29, } }), {})); expect(result.execution).toEqual({ acpStalledToolKind: "other", acpStalledToolSilentMs: 1_800_000, acpStalledToolPermissionRequested: false, acpPendingToolPermissionRequestedCount: 3, + acpSettledSameKindToolCount: 30, acpSettledSameKindToolPermissionRequestedCount: 29, }); expect(JSON.stringify(result)).not.toContain("private"); }); @@ -490,7 +492,8 @@ describe("run failure diagnostics", () => { acpLastEventAgeMs: value, acpObservedEventCount: value, acpPendingToolCount: value, acpToolInventoryComplete: value, acpLastEventAgeAtStopMs: value, acpOldestPendingToolAgeAtStopMs: value, acpStalledToolSilentMs: value, acpStalledToolPermissionRequested: value, - acpPendingToolPermissionRequestedCount: value, + acpPendingToolPermissionRequestedCount: value, acpSettledSameKindToolCount: value, + acpSettledSameKindToolPermissionRequestedCount: value, } }), {}); expect(result.execution).toEqual({}); }, diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index e2fcffa0932..d87f62e21f8 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -314,7 +314,8 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport for (const field of [ "acpLastEventAgeMs", "acpObservedEventCount", "acpPendingToolCount", "acpLastEventAgeAtStopMs", "acpOldestPendingToolAgeAtStopMs", "acpStalledToolSilentMs", - "acpPendingToolPermissionRequestedCount", + "acpPendingToolPermissionRequestedCount", "acpSettledSameKindToolCount", + "acpSettledSameKindToolPermissionRequestedCount", ]) { const value = read(result, field); if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) execution[field] = value;