mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-19 07:42:38 +01:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Before an agent starts on a sandbox environment, the host gets an environment lease. On Daytona, this creates a sandbox. > - When the create does not finish in time, the Daytona plugin records the failed phase, the elapsed time, the budget, and the Daytona sandbox state. #15736 and #15946 added these fields to the run failure report, so that operators can see where the acquisition stopped. > - #20857 changed the error code of these failures from `setup_failed` to `environment_provider_unavailable`, so that the host retries them. > - The run failure diagnostics read the acquisition fields only when the error code is `setup_failed`. Thus every Daytona create timeout now reaches the error report without its phase or provider state. > - On staging, more than 270 Daytona lease timeouts occurred in three days, and none of them shows the provider state. > - This pull request also reads the acquisition fields when the error code is `environment_provider_unavailable`. > - The benefit is that the error report shows again whether the sandbox was stuck in `creating`, in a snapshot state, or absent. This evidence is necessary to find the cause of the timeouts. ## Linked Issues or Issue Description Refs #20857, Refs #20828, Refs #15946, Refs #15736 **What happened?** A Daytona lease acquisition timeout is now classified as `environment_provider_unavailable` (#20857). The run failure report for this failure does not contain `environmentAcquisitionPhase`, `environmentAcquisitionElapsedMs`, `environmentAcquisitionBudgetMs` or `environmentAcquisitionProviderState`. **What did you expect?** The report contains the acquisition phase and the provider state, as it did for `setup_failed` runs before #20857. **Steps to reproduce** 1. Let a Daytona sandbox create pass its acquisition deadline. 2. The plugin returns "Daytona lease acquisition timed out; allocation cleanup is pending" with an acquisition diagnostic. 3. The host fails the run with `environment_provider_unavailable`. 4. Look at the `run_execution` context of the error report. The acquisition fields are not there. ## What Changed - `collectRunFailureDiagnostics` reads the private acquisition diagnostic for a setup failure with the error code `setup_failed` or `environment_provider_unavailable`. The other conditions do not change: the phase must be `setup`, and the diagnostic must belong to the same run and company. - A new test builds the real wrapper chain (lease failure, then provider failure, then the RPC error that holds the diagnostic). It checks that the fields are reported for the failed run, and that they are not reported for another run or for the execute phase. ## Verification - `cd server && npx vitest run src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts` fails on master (the report has only `failurePhase`). It passes with this change. - `cd server && npx vitest run src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts src/services/__tests__/run-failure-diagnostics.test.ts src/__tests__/environment-acquisition-diagnostics.test.ts`: 3 files, 139 tests pass. ## Risks Low risk. The change adds four bounded scalar fields to an error report. These fields already exist for `setup_failed` runs. The change does not alter retries, the error grouping, or the run result. ## Model Used Claude Opus 5.5 (`claude-opus-5-5`), with tool use and code execution in an agent harness. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code)
465 lines
25 KiB
TypeScript
465 lines
25 KiB
TypeScript
import { readWorkspaceBaseRefDiagnostic } from "./workspace-base-ref-diagnostics.js";
|
|
import { getEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js";
|
|
import type { heartbeatRuns } from "@paperclipai/db";
|
|
import { readControllerLeaseLoss, readControllerLeaseLossCause, readRunCancellation } from "./run-cancellation.js";
|
|
import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js";
|
|
import { readReviewedChatBindingDenialReason } from "./reviewed-chat-binding-diagnostics.js";
|
|
import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared";
|
|
import {
|
|
collectDiagnosticEnvironmentSecrets,
|
|
collectHostEnvironmentTokenSecrets,
|
|
redactDiagnosticText,
|
|
} from "@paperclipai/adapter-utils/command-redaction";
|
|
import { directoryMergeLockTimeoutDiagnosticsFor } from "@paperclipai/adapter-utils/directory-merge-lock-diagnostics";
|
|
import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics";
|
|
import {
|
|
ACP_PROCESS_TERMINAL_SIGNALS,
|
|
ACP_PROCESS_TERMINAL_SOURCES,
|
|
ACP_TOOL_KINDS,
|
|
} from "@paperclipai/adapter-utils/acpx-engine/constants";
|
|
import { redactCurrentUserText } from "../log-redaction.js";
|
|
import { storageFailureDiagnosticsFor } from "../storage/failure-diagnostics.js";
|
|
import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";
|
|
import { readNativeModelRejectionDiagnostic, readNativeTurnFailureDiagnostic } from "./native-runtime/native-provider-failure.js";
|
|
import { readRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError } from "./native-runtime/remote-runner-recovery-diagnostics.js";
|
|
import { MANAGED_GIT_WORKTREE_REASON_CODES, PERSISTED_WORKSPACE_SOURCE_REASON_CODES, readManagedGitInspectionDiagnostic } from "./workspace-validation-diagnostics.js";
|
|
|
|
type Run = typeof heartbeatRuns.$inferSelect;
|
|
type Context = Record<string, string | number | boolean>;
|
|
|
|
export interface RunFailureReportOptions {
|
|
/** The caught exception, before callers flatten it to a message. */
|
|
error?: unknown;
|
|
/** Structured adapter error metadata; only known diagnostic keys are read. */
|
|
adapterErrorMeta?: unknown;
|
|
phase?: "setup" | "execute";
|
|
/** Runtime-only values to redact; never included in the captured event. */
|
|
secretValues?: readonly string[];
|
|
/**
|
|
* The caught native session error. Only an unrecoverable-close settlement is
|
|
* read, as closed labels and counts; its message and stack are never copied.
|
|
*/
|
|
nativeCloseFailure?: unknown;
|
|
/**
|
|
* The bounded retry decision for a reviewed chat setup wait. The setup path
|
|
* writes the failed status while `pending`, then reports again with the
|
|
* decision; a scheduled retry owns recovery and reports its own outcome.
|
|
*/
|
|
reviewedChatRetry?: "pending" | "scheduled" | "not_scheduled";
|
|
}
|
|
|
|
export interface RunFailureException {
|
|
name?: string;
|
|
message?: string;
|
|
stack?: string;
|
|
code?: string;
|
|
status?: number;
|
|
requestId?: string;
|
|
}
|
|
|
|
export interface RunFailureDiagnostics {
|
|
execution: Context;
|
|
adapter: Context;
|
|
provider: Context;
|
|
exceptions: RunFailureException[];
|
|
truncatedFields: string[];
|
|
}
|
|
|
|
const ACP_PENDING_TOOL_KIND_LABELS: ReadonlySet<string> = new Set([...ACP_TOOL_KINDS, "unknown"]);
|
|
|
|
/** Closed ACP tool-kind labels only; any other content drops the whole field. */
|
|
function readAcpPendingToolKinds(value: unknown): string | null {
|
|
if (typeof value !== "string" || value.length === 0 || value.length > 200) return null;
|
|
const kinds = value.split(",");
|
|
return kinds.every((kind) => ACP_PENDING_TOOL_KIND_LABELS.has(kind)) ? value : null;
|
|
}
|
|
|
|
/** One closed ACP tool-kind label, or nothing. */
|
|
function readAcpToolKind(value: unknown): string | null {
|
|
return typeof value === "string" && ACP_PENDING_TOOL_KIND_LABELS.has(value) ? value : null;
|
|
}
|
|
|
|
/** How the sandbox agent process session ended, as closed labels and a bounded exit code. */
|
|
function readAcpProcessTerminal(result: unknown): Context {
|
|
const source = ACP_PROCESS_TERMINAL_SOURCES.find((label) => label === read(result, "acpProcessTerminalSource"));
|
|
if (!source) return {};
|
|
const fields: Context = { acpProcessTerminalSource: source };
|
|
const outcome = read(result, "acpProcessTerminalOutcome");
|
|
if (outcome === "exit" || outcome === "error") fields.acpProcessTerminalOutcome = outcome;
|
|
const exitCode = read(result, "acpProcessTerminalExitCode");
|
|
if (typeof exitCode === "number" && Number.isInteger(exitCode) && exitCode >= 0 && exitCode <= 255) {
|
|
fields.acpProcessTerminalExitCode = exitCode;
|
|
}
|
|
const signal = ACP_PROCESS_TERMINAL_SIGNALS.find((label) => label === read(result, "acpProcessTerminalSignal"));
|
|
if (signal) fields.acpProcessTerminalSignal = signal;
|
|
return fields;
|
|
}
|
|
|
|
function read(value: unknown, key: string): unknown {
|
|
if (!value || typeof value !== "object") return undefined;
|
|
try {
|
|
return (value as Record<string, unknown>)[key];
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
const NATIVE_CLOSE_COMMAND_STATUSES: ReadonlySet<unknown> = new Set(["pending", "completed", "failed", "rejected", "indeterminate"]);
|
|
const NATIVE_CLOSE_RUNNER_LIFECYCLES: ReadonlySet<unknown> = new Set(["ready", "suspended", "closed", "recoverable_failure"]);
|
|
|
|
function boundedCount(value: unknown): number | undefined {
|
|
if (Array.isArray(value)) return value.length;
|
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
}
|
|
|
|
/**
|
|
* Which close-barrier condition failed. Tool identities, digests and provider
|
|
* turn ids stay in the run log; only booleans, fixed labels and counts leave.
|
|
*/
|
|
function readNativeCloseSettlement(error: unknown): Context | null {
|
|
let candidate = error;
|
|
for (let depth = 0; depth < 4 && candidate && typeof candidate === "object"; depth++) {
|
|
if (read(candidate, "code") === "native_session_close_unrecoverable") break;
|
|
candidate = read(candidate, "cause");
|
|
}
|
|
if (read(candidate, "code") !== "native_session_close_unrecoverable") return null;
|
|
const settlement = read(candidate, "settlement");
|
|
const result: Context = {};
|
|
for (const [field, key] of [["runnerSuspended", "closeRunnerSuspended"], ["providerDrained", "closeProviderDrained"]] as const) {
|
|
const value = read(settlement, field);
|
|
if (typeof value === "boolean") result[key] = value;
|
|
}
|
|
const suspension = read(settlement, "suspensionState");
|
|
const commandStatus = read(suspension, "commandStatus");
|
|
if (NATIVE_CLOSE_COMMAND_STATUSES.has(commandStatus)) result.closeSuspendCommandStatus = commandStatus as string;
|
|
else if (suspension && commandStatus === null) result.closeSuspendCommandStatus = "not_queued";
|
|
const lifecycle = read(suspension, "runnerLifecycle");
|
|
if (NATIVE_CLOSE_RUNNER_LIFECYCLES.has(lifecycle)) result.closeRunnerLifecycle = lifecycle as string;
|
|
else if (suspension && lifecycle === null) result.closeRunnerLifecycle = "unobserved";
|
|
const identityMatches = read(suspension, "runnerIdentityMatches");
|
|
if (typeof identityMatches === "boolean") result.closeRunnerIdentityMatches = identityMatches;
|
|
const provider = read(settlement, "finalProviderState");
|
|
if (provider === null) result.closeProviderState = "not_local";
|
|
else if (provider === "unreadable") result.closeProviderState = "unreadable";
|
|
else if (provider && typeof provider === "object") {
|
|
const settled = read(provider, "providerSettled");
|
|
if (typeof settled === "boolean") result.closeProviderState = settled ? "settled" : "unsettled";
|
|
const pending = boundedCount(read(provider, "pendingEventCount"));
|
|
if (pending !== undefined) result.closeProviderPendingEventCount = pending;
|
|
const activeTurn = read(provider, "activeProviderTurnId");
|
|
if (activeTurn === null || typeof activeTurn === "string") result.closeProviderTurnActive = activeTurn !== null;
|
|
}
|
|
const tools = read(settlement, "semanticTools");
|
|
const persistenceFailed = read(tools, "persistenceFailed");
|
|
if (typeof persistenceFailed === "boolean") result.closeSemanticResultPersistenceFailed = persistenceFailed;
|
|
for (const [field, key] of [
|
|
["pending", "closeSemanticToolsPending"],
|
|
["deliveries", "closeSemanticToolDeliveriesOpen"],
|
|
["failures", "closeSemanticResultFailures"],
|
|
] as const) {
|
|
const count = boundedCount(read(tools, field));
|
|
if (count !== undefined) result[key] = count;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function scalars(value: unknown, fields: readonly string[]): Context {
|
|
const result: Context = {};
|
|
for (const field of fields) {
|
|
const entry = read(value, field);
|
|
if (
|
|
typeof entry === "string" || typeof entry === "boolean" ||
|
|
(typeof entry === "number" && Number.isFinite(entry))
|
|
) result[field] = entry;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/** Select credential fields, credential-shaped values, and declared bindings. */
|
|
export const collectRunFailureSecretValues = collectDiagnosticEnvironmentSecrets;
|
|
|
|
/** Short host values that stay private as whole tokens only. */
|
|
export const collectRunFailureHostTokenSecrets = collectHostEnvironmentTokenSecrets;
|
|
|
|
/** Redact literal, JSON-escaped and URI-encoded forms without rewriting markers. */
|
|
export function redactRunFailureSecretValues<T>(input: T, values: readonly string[],
|
|
tokenValues: readonly string[] = []): T {
|
|
const variants = (list: readonly string[]) => [...new Set(list.filter(Boolean).flatMap((value) => {
|
|
const forms = [value, JSON.stringify(value).slice(1, -1)];
|
|
try { forms.push(encodeURIComponent(value)); } catch { /* malformed Unicode */ }
|
|
return forms;
|
|
}))].sort((a, b) => b.length - a.length);
|
|
const escape = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
const forms = variants(values);
|
|
// Short host values are private only as whole tokens: "vi" is redacted in
|
|
// "editor vi", never inside "service". One alternation ordered by length
|
|
// across both lists, so a longer complete match always wins over a shorter
|
|
// prefix that starts at the same position.
|
|
const tokenForms = variants(tokenValues).filter((value) => !forms.includes(value));
|
|
const alternatives = [
|
|
...forms.map((value) => ({ value, pattern: escape(value) })),
|
|
...tokenForms.map((value) => ({ value,
|
|
pattern: `(?<![\\p{L}\\p{N}_])${escape(value)}(?![\\p{L}\\p{N}_])` })),
|
|
].sort((a, b) => b.value.length - a.value.length);
|
|
if (alternatives.length === 0) return input;
|
|
const pattern = new RegExp(alternatives.map((entry) => entry.pattern).join("|"), "gu");
|
|
const redact = (value: unknown): unknown => {
|
|
if (typeof value === "string") return value.replace(pattern, () => REDACTED_EVENT_VALUE);
|
|
if (Array.isArray(value)) return value.map(redact);
|
|
if (value && typeof value === "object") {
|
|
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redact(entry)]));
|
|
}
|
|
return value;
|
|
};
|
|
return redact(input) as T;
|
|
}
|
|
|
|
/** Snapshot only diagnostics. Never walk a request, response, config or prompt. */
|
|
export function collectRunFailureDiagnostics(run: Run, options: RunFailureReportOptions): RunFailureDiagnostics {
|
|
const execution = scalars(run, [
|
|
"runtimeMode", "executionStage", "nativePhase", "driverKind", "driverVersion",
|
|
]);
|
|
if (options.phase) execution.failurePhase = options.phase;
|
|
const providerConfig = read(read(run.runnerProfileJson, "nativeExecutionInput"), "provider");
|
|
const providerKind = read(providerConfig, "kind");
|
|
const model = read(providerConfig, "model");
|
|
if (typeof providerKind === "string") execution.providerKind = providerKind;
|
|
if (typeof model === "string") execution.model = model;
|
|
for (const field of ["startedAt", "finishedAt"] as const) {
|
|
const value = run[field];
|
|
if (value instanceof Date && Number.isFinite(value.getTime())) execution[field] = value.toISOString();
|
|
}
|
|
const originalFailureCode = read(run.resultJson, "originalFailureCode");
|
|
if (typeof originalFailureCode === "string" && /^[a-z][a-z0-9_]{1,99}$/.test(originalFailureCode)) {
|
|
execution.originalFailureCode = originalFailureCode;
|
|
}
|
|
if (run.startedAt && run.finishedAt) {
|
|
const durationMs = run.finishedAt.getTime() - run.startedAt.getTime();
|
|
if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;
|
|
}
|
|
const result = run.resultJson;
|
|
// Restart reattachment can fail before the native coordinator saves its
|
|
// result. The thrown error then reaches the generic run failure path, so
|
|
// read the same closed labels from the error.
|
|
const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")) ??
|
|
readRemoteRunnerRecoveryError(options.error);
|
|
if (remoteRunnerRecovery) {
|
|
execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage;
|
|
execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason;
|
|
if (remoteRunnerRecovery.runnerLifecycle) execution.remoteRunnerLifecycle = remoteRunnerRecovery.runnerLifecycle;
|
|
if (remoteRunnerRecovery.runnerSettled !== undefined) execution.remoteRunnerSettled = remoteRunnerRecovery.runnerSettled;
|
|
}
|
|
const configuration = read(result, "configurationIncomplete");
|
|
if (run.errorCode === "configuration_incomplete" && read(configuration, "reason") === "workspace_base_ref_unresolved") {
|
|
const diagnostic = readWorkspaceBaseRefDiagnostic(read(configuration, "baseRefDiagnostic"));
|
|
if (diagnostic) {
|
|
for (const [key, value] of Object.entries(diagnostic)) {
|
|
if (key !== "schemaVersion") execution[`workspaceBaseRef${key[0]!.toUpperCase()}${key.slice(1)}`] = value;
|
|
}
|
|
}
|
|
}
|
|
const workspaceValidation = read(result, "workspaceValidation");
|
|
if (run.errorCode === "workspace_validation_failed" && read(workspaceValidation, "reason") === "git_worktree_not_reusable") {
|
|
execution.workspaceValidationReason = "git_worktree_not_reusable";
|
|
const reasonCode = MANAGED_GIT_WORKTREE_REASON_CODES.find(code => code === read(workspaceValidation, "reasonCode"));
|
|
if (reasonCode) execution.workspaceValidationReasonCode = reasonCode;
|
|
const diagnostic = reasonCode === "git_inspection_failed"
|
|
? readManagedGitInspectionDiagnostic(read(workspaceValidation, "inspectionDiagnostic")) : null;
|
|
if (diagnostic) {
|
|
execution.workspaceValidationInspectionCommand = diagnostic.command;
|
|
execution.workspaceValidationInspectionFailure = diagnostic.failure;
|
|
if (diagnostic.errorCode) execution.workspaceValidationInspectionErrorCode = diagnostic.errorCode;
|
|
if (diagnostic.exitCode !== undefined) execution.workspaceValidationInspectionExitCode = diagnostic.exitCode;
|
|
}
|
|
}
|
|
if (run.errorCode === "workspace_validation_failed" && read(workspaceValidation, "reason") === "persisted_workspace_source_conflict") {
|
|
execution.workspaceValidationReason = "persisted_workspace_source_conflict";
|
|
const reasonCode = PERSISTED_WORKSPACE_SOURCE_REASON_CODES.find(code => code === read(workspaceValidation, "reasonCode"));
|
|
if (reasonCode) execution.workspaceValidationReasonCode = reasonCode;
|
|
}
|
|
if (run.errorCode === "process_lost") {
|
|
const diagnostic = readProcessLossDiagnostic(read(result, "processLossDiagnostic"));
|
|
for (const [field, value] of Object.entries(diagnostic)) {
|
|
execution[`processLoss${field[0]!.toUpperCase()}${field.slice(1)}`] = value;
|
|
}
|
|
}
|
|
const cancellation = readRunCancellation(result);
|
|
if (cancellation) {
|
|
execution.cancellationSource = cancellation.source;
|
|
execution.cancellationExpected = cancellation.expected;
|
|
execution.cancellationInitiatorType = cancellation.initiator.type;
|
|
const leaseLossCause = readControllerLeaseLossCause(cancellation);
|
|
if (leaseLossCause) execution.controllerLeaseLossCause = leaseLossCause;
|
|
} else if (run.status === "cancelled") {
|
|
execution.cancellationSource = "unknown";
|
|
execution.cancellationExpected = false;
|
|
}
|
|
const leaseLoss = readControllerLeaseLoss(read(result, "controllerLeaseLoss"));
|
|
if (leaseLoss) {
|
|
execution.controllerLeaseLossCause = leaseLoss.cause;
|
|
if (leaseLoss.sinceRenewalMs !== undefined) execution.controllerLeaseSinceRenewalMs = leaseLoss.sinceRenewalMs;
|
|
if (leaseLoss.renewalPendingMs !== undefined) execution.controllerLeaseRenewalPendingMs = leaseLoss.renewalPendingMs;
|
|
if (leaseLoss.deadlineLateMs !== undefined) execution.controllerLeaseDeadlineLateMs = leaseLoss.deadlineLateMs;
|
|
if (leaseLoss.renewalCount !== undefined) execution.controllerLeaseRenewalCount = leaseLoss.renewalCount;
|
|
}
|
|
const reviewedChatDenial = run.errorCode === "setup_failed"
|
|
? readReviewedChatBindingDenialReason(read(result, "reviewedChatBinding")) : null;
|
|
if (reviewedChatDenial) execution.reviewedChatBindingDenialReason = reviewedChatDenial;
|
|
Object.assign(execution, scalars(result, [
|
|
"mode", "stopReason", "timeoutFired", "timeoutSource", "timeoutConfigured",
|
|
"effectiveTimeoutSec", "errorFamily",
|
|
]));
|
|
for (const field of [
|
|
"acpLastEventAgeMs", "acpObservedEventCount", "acpPendingToolCount",
|
|
"acpLastEventAgeAtStopMs", "acpOldestPendingToolAgeAtStopMs", "acpStalledToolSilentMs",
|
|
"acpPendingToolPermissionRequestedCount", "acpSettledSameKindToolCount",
|
|
"acpSettledSameKindToolPermissionRequestedCount",
|
|
]) {
|
|
const value = read(result, field);
|
|
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) execution[field] = value;
|
|
}
|
|
const closeSettlement = readNativeCloseSettlement(options.nativeCloseFailure);
|
|
if (closeSettlement) Object.assign(execution, closeSettlement);
|
|
Object.assign(execution, readAcpProcessTerminal(result));
|
|
const pendingToolKinds = readAcpPendingToolKinds(read(result, "acpPendingToolKinds"));
|
|
if (pendingToolKinds) execution.acpPendingToolKinds = pendingToolKinds;
|
|
const stalledToolKind = readAcpToolKind(read(result, "acpStalledToolKind"));
|
|
if (stalledToolKind) execution.acpStalledToolKind = stalledToolKind;
|
|
const stalledToolPermissionRequested = read(result, "acpStalledToolPermissionRequested");
|
|
if (typeof stalledToolPermissionRequested === "boolean") {
|
|
execution.acpStalledToolPermissionRequested = stalledToolPermissionRequested;
|
|
}
|
|
const toolInventoryComplete = read(result, "acpToolInventoryComplete");
|
|
if (typeof toolInventoryComplete === "boolean") execution.acpToolInventoryComplete = toolInventoryComplete;
|
|
const restoreFailure = read(result, "workspaceRestoreFailure");
|
|
const restoreCode = WORKSPACE_RESTORE_FAILURE_CODES.find((code) => code === restoreFailure);
|
|
if (restoreCode) {
|
|
execution.workspaceRestoreFailure = restoreCode;
|
|
const diagnostic = sanitizeWorkspaceRestoreDiagnostic(read(result, "workspaceRestoreDiagnostic"));
|
|
if (diagnostic) {
|
|
execution.workspaceRestorePhase = diagnostic.phase;
|
|
execution.workspaceRestoreErrorCode = diagnostic.errorCode;
|
|
if (diagnostic.step) execution.workspaceRestoreStep = diagnostic.step;
|
|
if (diagnostic.httpStatus !== undefined) execution.workspaceRestoreHttpStatus = diagnostic.httpStatus;
|
|
if (diagnostic.exitCode !== undefined) execution.workspaceRestoreExitCode = diagnostic.exitCode;
|
|
if (diagnostic.transferStep) execution.workspaceRestoreTransferStep = diagnostic.transferStep;
|
|
if (diagnostic.transferFailureKind) execution.workspaceRestoreTransferFailureKind = diagnostic.transferFailureKind;
|
|
if (diagnostic.rpcCode !== undefined) execution.workspaceRestoreRpcCode = diagnostic.rpcCode;
|
|
if (diagnostic.gitCommand) execution.workspaceRestoreGitCommand = diagnostic.gitCommand;
|
|
if (diagnostic.gitFailureKind) execution.workspaceRestoreGitFailureKind = diagnostic.gitFailureKind;
|
|
}
|
|
}
|
|
const adapter = scalars(options.adapterErrorMeta, [
|
|
"category", "phase", "errorName", "acpCode", "causeMessage", "retryable",
|
|
"stackPreview", "status", "statusCode", "requestId",
|
|
]);
|
|
const provider = scalars(read(result, "terminalSessionFailure"), ["category", "title", "details"]);
|
|
const nativeProviderFailure = readNativeModelRejectionDiagnostic(read(result, "nativeProviderFailure"));
|
|
if (nativeProviderFailure) Object.assign(provider, nativeProviderFailure);
|
|
const nativeTurnFailure = readNativeTurnFailureDiagnostic(read(result, "nativeTurnFailure"));
|
|
if (nativeTurnFailure) Object.assign(provider, nativeTurnFailure);
|
|
const truncatedFields: string[] = [];
|
|
const providerTruncation = read(read(result, "terminalSessionFailure"), "truncatedFields");
|
|
if (Array.isArray(providerTruncation)) {
|
|
for (const field of ["title", "details"]) {
|
|
if (providerTruncation.includes(field)) truncatedFields.push(`provider.${field}`);
|
|
}
|
|
}
|
|
const exceptions: RunFailureException[] = [];
|
|
const seen = new Set<unknown>();
|
|
let error = options.error;
|
|
while (error !== undefined && error !== null && exceptions.length < 4) {
|
|
if (seen.has(error)) {
|
|
truncatedFields.push("exceptions.cycle");
|
|
break;
|
|
}
|
|
seen.add(error);
|
|
if (typeof error === "string") {
|
|
exceptions.push({ message: error });
|
|
error = undefined;
|
|
break;
|
|
}
|
|
if (typeof error !== "object") break;
|
|
// A lease acquisition timeout is retried as a transient provider failure;
|
|
// it needs the same phase and provider state as a plain setup failure.
|
|
const acquisition = options.phase === "setup" && (run.errorCode === "setup_failed"
|
|
|| run.errorCode === "environment_provider_unavailable")
|
|
? getEnvironmentAcquisitionDiagnostic(error, run) : null;
|
|
if (acquisition && execution.environmentAcquisitionPhase === undefined) {
|
|
execution.environmentAcquisitionPhase = acquisition.phase;
|
|
execution.environmentAcquisitionElapsedMs = acquisition.elapsedMs;
|
|
execution.environmentAcquisitionBudgetMs = acquisition.budgetMs;
|
|
if (acquisition.providerState) execution.environmentAcquisitionProviderState = acquisition.providerState;
|
|
}
|
|
const lock = execution.restoreLockOwnerState === undefined ? directoryMergeLockTimeoutDiagnosticsFor(error) : null;
|
|
for (const [field, value] of Object.entries(lock ?? {})) {
|
|
execution[`restoreLock${field[0]!.toUpperCase()}${field.slice(1)}`] = value;
|
|
}
|
|
// A bare socket error ("aborted", ECONNRESET) cannot say which storage
|
|
// read it interrupted; the provider's fixed-shape record can.
|
|
const storage = storageFailureDiagnosticsFor(error);
|
|
if (storage && execution.storageOperation === undefined) {
|
|
execution.storageProvider = storage.provider;
|
|
execution.storageOperation = storage.operation;
|
|
execution.storageErrorCode = storage.errorCode;
|
|
if (storage.httpStatus !== null) execution.storageHttpStatus = storage.httpStatus;
|
|
if (storage.requestId !== null) execution.storageRequestId = storage.requestId;
|
|
if (storage.attempts !== null) execution.storageAttempts = storage.attempts;
|
|
}
|
|
const entry: RunFailureException = {};
|
|
for (const field of ["name", "message", "stack", "code"] as const) {
|
|
const value = read(error, field);
|
|
if (typeof value === "string") entry[field] = value;
|
|
else if (field === "code" && typeof value === "number" && Number.isFinite(value)) entry.code = String(value);
|
|
}
|
|
if (entry.code === undefined) {
|
|
const reasonCode = read(error, "reasonCode");
|
|
if (typeof reasonCode === "string") entry.code = reasonCode;
|
|
}
|
|
const status = read(error, "status") ?? read(error, "statusCode");
|
|
if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) entry.status = status;
|
|
const requestId = read(error, "requestId") ?? read(error, "request_id");
|
|
if (typeof requestId === "string") entry.requestId = requestId;
|
|
if (Object.keys(entry).length > 0) exceptions.push(entry);
|
|
else break;
|
|
error = read(error, "cause");
|
|
}
|
|
if (error !== undefined && error !== null && exceptions.length === 4) truncatedFields.push("exceptions.depth");
|
|
return { execution, adapter, provider, exceptions, truncatedFields };
|
|
}
|
|
|
|
/** Redact complete values before cutting, including credentials across a cut. */
|
|
export function sanitizeRunFailureText(input: string, limit: number): string {
|
|
const normalized = input
|
|
.replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "")
|
|
.replace(/[\ud800-\udfff]/gu, "\ufffd");
|
|
const clean = redactSensitiveText(redactDiagnosticText(redactCurrentUserText(normalized)));
|
|
if (clean.length <= limit) return clean;
|
|
const suffix = "\n[truncated]";
|
|
let end = limit - suffix.length;
|
|
if ((clean.codePointAt(end - 1) ?? 0) > 0xffff) end--;
|
|
return clean.slice(0, end) + suffix;
|
|
}
|
|
|
|
/** Called after the run's registered secret values have also been removed. */
|
|
export function sanitizeRunFailureDiagnostics(raw: RunFailureDiagnostics): RunFailureDiagnostics {
|
|
const truncatedFields = [...raw.truncatedFields];
|
|
const text = (value: string, path: string, limit: number) => {
|
|
const sanitized = sanitizeRunFailureText(value, limit);
|
|
if (sanitized.endsWith("\n[truncated]")) truncatedFields.push(path);
|
|
return sanitized;
|
|
};
|
|
const context = (value: Context, prefix: string, limits: Record<string, number> = {}): Context =>
|
|
Object.fromEntries(Object.entries(value).map(([key, value]) => [key,
|
|
typeof value === "string" ? text(value, `${prefix}.${key}`, limits[key] ?? 200) : value,
|
|
]));
|
|
return {
|
|
execution: context(raw.execution, "execution"),
|
|
adapter: context(raw.adapter, "adapter", { causeMessage: 2048, stackPreview: 8192 }),
|
|
provider: context(raw.provider, "provider", { title: 4096, details: 12288 }),
|
|
exceptions: raw.exceptions.map((entry, i) => context(entry as Context, `exceptions.${i}`, { message: 2048, stack: 8192 })),
|
|
truncatedFields: [...new Set(truncatedFields)],
|
|
};
|
|
}
|