mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-01 06:42:06 +01:00
Devin FoleyandClaude Opus 5.5
11d69d04c7
Restart reattachment can fail in the executor before the native coordinator saves its result. The run failure report now reads the closed remote recovery stage and reason from the caught error when the saved result lacks them. The saved result keeps priority and the same closed-vocabulary validation applies. Add a regression for thrown, wrapped-transport, saved-priority, and out-of-vocabulary cases. Full current-head CI and Greptile 5/5 passed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
456 lines
25 KiB
TypeScript
456 lines
25 KiB
TypeScript
import { readWorkspaceBaseRefDiagnostic } from "./workspace-base-ref-diagnostics.js";
|
|
import { getEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js";
|
|
import type { heartbeatRuns } from "@paperclipai/db";
|
|
import { readControllerLeaseLoss, readControllerLeaseLossCause, readRunCancellation } from "./run-cancellation.js";
|
|
import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js";
|
|
import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared";
|
|
import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction";
|
|
import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics";
|
|
import { ACP_TOOL_KINDS } from "@paperclipai/adapter-utils/acpx-engine/constants";
|
|
import { redactCurrentUserText } from "../log-redaction.js";
|
|
import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";
|
|
import { readNativeModelRejectionDiagnostic, readNativeTurnFailureDiagnostic } from "./native-runtime/native-provider-failure.js";
|
|
import { readRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError } from "./native-runtime/remote-runner-recovery-diagnostics.js";
|
|
import { MANAGED_GIT_WORKTREE_REASON_CODES, PERSISTED_WORKSPACE_SOURCE_REASON_CODES, readManagedGitInspectionDiagnostic } from "./workspace-validation-diagnostics.js";
|
|
|
|
type Run = typeof heartbeatRuns.$inferSelect;
|
|
type Context = Record<string, string | number | boolean>;
|
|
|
|
export interface RunFailureReportOptions {
|
|
/** The caught exception, before callers flatten it to a message. */
|
|
error?: unknown;
|
|
/** Structured adapter error metadata; only known diagnostic keys are read. */
|
|
adapterErrorMeta?: unknown;
|
|
phase?: "setup" | "execute";
|
|
/** Runtime-only values to redact; never included in the captured event. */
|
|
secretValues?: readonly string[];
|
|
/**
|
|
* The caught native session error. Only an unrecoverable-close settlement is
|
|
* read, as closed labels and counts; its message and stack are never copied.
|
|
*/
|
|
nativeCloseFailure?: unknown;
|
|
}
|
|
|
|
export interface RunFailureException {
|
|
name?: string;
|
|
message?: string;
|
|
stack?: string;
|
|
code?: string;
|
|
status?: number;
|
|
requestId?: string;
|
|
}
|
|
|
|
export interface RunFailureDiagnostics {
|
|
execution: Context;
|
|
adapter: Context;
|
|
provider: Context;
|
|
exceptions: RunFailureException[];
|
|
truncatedFields: string[];
|
|
}
|
|
|
|
const ACP_PENDING_TOOL_KIND_LABELS: ReadonlySet<string> = new Set([...ACP_TOOL_KINDS, "unknown"]);
|
|
|
|
/** Closed ACP tool-kind labels only; any other content drops the whole field. */
|
|
function readAcpPendingToolKinds(value: unknown): string | null {
|
|
if (typeof value !== "string" || value.length === 0 || value.length > 200) return null;
|
|
const kinds = value.split(",");
|
|
return kinds.every((kind) => ACP_PENDING_TOOL_KIND_LABELS.has(kind)) ? value : null;
|
|
}
|
|
|
|
function read(value: unknown, key: string): unknown {
|
|
if (!value || typeof value !== "object") return undefined;
|
|
try {
|
|
return (value as Record<string, unknown>)[key];
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
const NATIVE_CLOSE_COMMAND_STATUSES: ReadonlySet<unknown> = new Set(["pending", "completed", "failed", "rejected", "indeterminate"]);
|
|
const NATIVE_CLOSE_RUNNER_LIFECYCLES: ReadonlySet<unknown> = new Set(["ready", "suspended", "closed", "recoverable_failure"]);
|
|
|
|
function boundedCount(value: unknown): number | undefined {
|
|
if (Array.isArray(value)) return value.length;
|
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
}
|
|
|
|
/**
|
|
* Which close-barrier condition failed. Tool identities, digests and provider
|
|
* turn ids stay in the run log; only booleans, fixed labels and counts leave.
|
|
*/
|
|
function readNativeCloseSettlement(error: unknown): Context | null {
|
|
let candidate = error;
|
|
for (let depth = 0; depth < 4 && candidate && typeof candidate === "object"; depth++) {
|
|
if (read(candidate, "code") === "native_session_close_unrecoverable") break;
|
|
candidate = read(candidate, "cause");
|
|
}
|
|
if (read(candidate, "code") !== "native_session_close_unrecoverable") return null;
|
|
const settlement = read(candidate, "settlement");
|
|
const result: Context = {};
|
|
for (const [field, key] of [["runnerSuspended", "closeRunnerSuspended"], ["providerDrained", "closeProviderDrained"]] as const) {
|
|
const value = read(settlement, field);
|
|
if (typeof value === "boolean") result[key] = value;
|
|
}
|
|
const suspension = read(settlement, "suspensionState");
|
|
const commandStatus = read(suspension, "commandStatus");
|
|
if (NATIVE_CLOSE_COMMAND_STATUSES.has(commandStatus)) result.closeSuspendCommandStatus = commandStatus as string;
|
|
else if (suspension && commandStatus === null) result.closeSuspendCommandStatus = "not_queued";
|
|
const lifecycle = read(suspension, "runnerLifecycle");
|
|
if (NATIVE_CLOSE_RUNNER_LIFECYCLES.has(lifecycle)) result.closeRunnerLifecycle = lifecycle as string;
|
|
else if (suspension && lifecycle === null) result.closeRunnerLifecycle = "unobserved";
|
|
const identityMatches = read(suspension, "runnerIdentityMatches");
|
|
if (typeof identityMatches === "boolean") result.closeRunnerIdentityMatches = identityMatches;
|
|
const provider = read(settlement, "finalProviderState");
|
|
if (provider === null) result.closeProviderState = "not_local";
|
|
else if (provider === "unreadable") result.closeProviderState = "unreadable";
|
|
else if (provider && typeof provider === "object") {
|
|
const settled = read(provider, "providerSettled");
|
|
if (typeof settled === "boolean") result.closeProviderState = settled ? "settled" : "unsettled";
|
|
const pending = boundedCount(read(provider, "pendingEventCount"));
|
|
if (pending !== undefined) result.closeProviderPendingEventCount = pending;
|
|
const activeTurn = read(provider, "activeProviderTurnId");
|
|
if (activeTurn === null || typeof activeTurn === "string") result.closeProviderTurnActive = activeTurn !== null;
|
|
}
|
|
const tools = read(settlement, "semanticTools");
|
|
const persistenceFailed = read(tools, "persistenceFailed");
|
|
if (typeof persistenceFailed === "boolean") result.closeSemanticResultPersistenceFailed = persistenceFailed;
|
|
for (const [field, key] of [
|
|
["pending", "closeSemanticToolsPending"],
|
|
["deliveries", "closeSemanticToolDeliveriesOpen"],
|
|
["failures", "closeSemanticResultFailures"],
|
|
] as const) {
|
|
const count = boundedCount(read(tools, field));
|
|
if (count !== undefined) result[key] = count;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function scalars(value: unknown, fields: readonly string[]): Context {
|
|
const result: Context = {};
|
|
for (const field of fields) {
|
|
const entry = read(value, field);
|
|
if (
|
|
typeof entry === "string" || typeof entry === "boolean" ||
|
|
(typeof entry === "number" && Number.isFinite(entry))
|
|
) result[field] = entry;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Unknown inherited values may contain credentials under arbitrary names.
|
|
// Explicitly configured values remain private, including overrides of these keys.
|
|
const PUBLIC_ENV_KEYS = new Set([
|
|
"PATH", "PATHEXT", "SYSTEMROOT", "WINDIR", "COMSPEC", "HOME", "USERPROFILE",
|
|
"HOMEDRIVE", "HOMEPATH", "USER", "USERNAME", "LOGNAME", "SHELL", "LANG",
|
|
"LANGUAGE", "LC_ALL", "LC_CTYPE", "TZ", "TMPDIR", "TEMP", "TMP", "NODE_ENV",
|
|
"XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME",
|
|
"PWD", "OLDPWD", "INIT_CWD", "SHLVL", "_", "TERM", "TERM_PROGRAM", "TERM_PROGRAM_VERSION", "COLORTERM",
|
|
"NPM_LIFECYCLE_EVENT", "NPM_COMMAND", "NPM_EXEC_PATH", "NPM_EXECPATH", "NPM_NODE_EXECPATH",
|
|
"NPM_PACKAGE_NAME", "NPM_PACKAGE_VERSION", "NPM_PACKAGE_JSON", "NPM_CONFIG_USER_AGENT",
|
|
"XPC_SERVICE_NAME", "VITEST_POOL_ID", "VITEST_WORKER_ID",
|
|
"PAPERCLIP_AGENT_ID", "PAPERCLIP_COMPANY_ID", "PAPERCLIP_RUN_ID", "PAPERCLIP_TASK_ID",
|
|
]);
|
|
const PUBLIC_BOOLEAN_ENV_KEYS = new Set([
|
|
"OPENCODE_ALLOW_ALL_MODELS", "CLAUDE_CODE_USE_BEDROCK", "GOOGLE_GENAI_USE_GCA",
|
|
"CI", "NO_COLOR", "FORCE_COLOR",
|
|
"VITEST", "MALLOCNANOZONE", "CODEX_CI", "CODEX_SHELL", "CODEX_SAGE_BACKFILL_TRACKER_TAB_REUSE",
|
|
"DEV", "PROD", "SSR", "PAPERCLIP_REQUIRE_SENTRY_TEST_SDK",
|
|
"PAPERCLIP_DB_BACKUP_ENABLED", "PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", "PAPERCLIP_AUTH_DISABLE_SIGN_UP", "PAPERCLIP_ENABLE_COMPANY_DELETION",
|
|
"PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "PAPERCLIP_SECRETS_STRICT_MODE",
|
|
"SERVE_UI", "PAPERCLIP_UI_DEV_MIDDLEWARE", "HEARTBEAT_SCHEDULER_ENABLED", "PAPERCLIP_ANNOUNCEMENTS_ENABLED",
|
|
"GITHUB_ACTIONS", "GITHUB_REF_PROTECTED", "RUNNER_DEBUG", "ACTIONS_RUNNER_DEBUG", "ACTIONS_STEP_DEBUG",
|
|
]);
|
|
const PUBLIC_NUMBER_ENV_KEYS = new Set([
|
|
"PORT", "PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES", "PAPERCLIP_DB_BACKUP_RETENTION_DAYS",
|
|
"PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS",
|
|
"HEARTBEAT_SCHEDULER_INTERVAL_MS",
|
|
"GITHUB_RUN_ATTEMPT", "GITHUB_RUN_NUMBER", "GITHUB_RUN_ID", "GITHUB_RETENTION_DAYS",
|
|
]);
|
|
|
|
/** Include declared bindings and unknown values, not just credential-like keys. */
|
|
export function collectRunFailureSecretValues(
|
|
env: unknown, secretKeys: Iterable<string> = [], inherited = false,
|
|
): string[] {
|
|
if (!env || typeof env !== "object") return [];
|
|
const strings = Object.fromEntries(Object.entries(env).filter(
|
|
(entry): entry is [string, string] => typeof entry[1] === "string",
|
|
));
|
|
const declared = new Set(secretKeys);
|
|
const values: string[] = [];
|
|
for (const [key, value] of Object.entries(strings)) {
|
|
if (!value) continue;
|
|
const publicSetting = (inherited && (
|
|
PUBLIC_ENV_KEYS.has(key.toUpperCase()) ||
|
|
(PUBLIC_NUMBER_ENV_KEYS.has(key.toUpperCase()) && /^\d+$/.test(value)) ||
|
|
(key === "BASE_URL" && value === "/") ||
|
|
(key === "MODE" && /^(?:test|development|production)$/.test(value)) ||
|
|
(key === "PAPERCLIP_SENTRY_RUN_FAILURES" && /^(?:off|unexpected|all)$/.test(value))
|
|
)) ||
|
|
(PUBLIC_BOOLEAN_ENV_KEYS.has(key.toUpperCase()) && /^(?:0|1|true|false)$/i.test(value));
|
|
if (declared.has(key) || !publicSetting) values.push(value);
|
|
try {
|
|
const password = new URL(value).password;
|
|
if (password) values.push(value, password, decodeURIComponent(password));
|
|
} catch { /* ordinary environment values are not URLs */ }
|
|
}
|
|
return [...new Set(values)].sort((a, b) => b.length - a.length);
|
|
}
|
|
|
|
/** Redact literal, JSON-escaped and URI-encoded forms without rewriting markers. */
|
|
export function redactRunFailureSecretValues<T>(input: T, values: readonly string[]): T {
|
|
const forms = [...new Set(values.filter(Boolean).flatMap((value) => {
|
|
const forms = [value, JSON.stringify(value).slice(1, -1)];
|
|
try { forms.push(encodeURIComponent(value)); } catch { /* malformed Unicode */ }
|
|
return forms;
|
|
}))].sort((a, b) => b.length - a.length);
|
|
if (forms.length === 0) return input;
|
|
const pattern = new RegExp(forms.map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"), "gu");
|
|
const redact = (value: unknown): unknown => {
|
|
if (typeof value === "string") return value.replace(pattern, () => REDACTED_EVENT_VALUE);
|
|
if (Array.isArray(value)) return value.map(redact);
|
|
if (value && typeof value === "object") {
|
|
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redact(entry)]));
|
|
}
|
|
return value;
|
|
};
|
|
return redact(input) as T;
|
|
}
|
|
|
|
/** Snapshot only diagnostics. Never walk a request, response, config or prompt. */
|
|
export function collectRunFailureDiagnostics(run: Run, options: RunFailureReportOptions): RunFailureDiagnostics {
|
|
const execution = scalars(run, [
|
|
"runtimeMode", "executionStage", "nativePhase", "driverKind", "driverVersion",
|
|
]);
|
|
if (options.phase) execution.failurePhase = options.phase;
|
|
const providerConfig = read(read(run.runnerProfileJson, "nativeExecutionInput"), "provider");
|
|
const providerKind = read(providerConfig, "kind");
|
|
const model = read(providerConfig, "model");
|
|
if (typeof providerKind === "string") execution.providerKind = providerKind;
|
|
if (typeof model === "string") execution.model = model;
|
|
for (const field of ["startedAt", "finishedAt"] as const) {
|
|
const value = run[field];
|
|
if (value instanceof Date && Number.isFinite(value.getTime())) execution[field] = value.toISOString();
|
|
}
|
|
const originalFailureCode = read(run.resultJson, "originalFailureCode");
|
|
if (typeof originalFailureCode === "string" && /^[a-z][a-z0-9_]{1,99}$/.test(originalFailureCode)) {
|
|
execution.originalFailureCode = originalFailureCode;
|
|
}
|
|
if (run.startedAt && run.finishedAt) {
|
|
const durationMs = run.finishedAt.getTime() - run.startedAt.getTime();
|
|
if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;
|
|
}
|
|
const result = run.resultJson;
|
|
// Restart reattachment can fail before the native coordinator saves its
|
|
// result. The thrown error then reaches the generic run failure path, so
|
|
// read the same closed labels from the error.
|
|
const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")) ??
|
|
readRemoteRunnerRecoveryError(options.error);
|
|
if (remoteRunnerRecovery) {
|
|
execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage;
|
|
execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason;
|
|
}
|
|
const configuration = read(result, "configurationIncomplete");
|
|
if (run.errorCode === "configuration_incomplete" && read(configuration, "reason") === "workspace_base_ref_unresolved") {
|
|
const diagnostic = readWorkspaceBaseRefDiagnostic(read(configuration, "baseRefDiagnostic"));
|
|
if (diagnostic) {
|
|
for (const [key, value] of Object.entries(diagnostic)) {
|
|
if (key !== "schemaVersion") execution[`workspaceBaseRef${key[0]!.toUpperCase()}${key.slice(1)}`] = value;
|
|
}
|
|
}
|
|
}
|
|
const workspaceValidation = read(result, "workspaceValidation");
|
|
if (run.errorCode === "workspace_validation_failed" && read(workspaceValidation, "reason") === "git_worktree_not_reusable") {
|
|
execution.workspaceValidationReason = "git_worktree_not_reusable";
|
|
const reasonCode = MANAGED_GIT_WORKTREE_REASON_CODES.find(code => code === read(workspaceValidation, "reasonCode"));
|
|
if (reasonCode) execution.workspaceValidationReasonCode = reasonCode;
|
|
const diagnostic = reasonCode === "git_inspection_failed"
|
|
? readManagedGitInspectionDiagnostic(read(workspaceValidation, "inspectionDiagnostic")) : null;
|
|
if (diagnostic) {
|
|
execution.workspaceValidationInspectionCommand = diagnostic.command;
|
|
execution.workspaceValidationInspectionFailure = diagnostic.failure;
|
|
if (diagnostic.errorCode) execution.workspaceValidationInspectionErrorCode = diagnostic.errorCode;
|
|
if (diagnostic.exitCode !== undefined) execution.workspaceValidationInspectionExitCode = diagnostic.exitCode;
|
|
}
|
|
}
|
|
if (run.errorCode === "workspace_validation_failed" && read(workspaceValidation, "reason") === "persisted_workspace_source_conflict") {
|
|
execution.workspaceValidationReason = "persisted_workspace_source_conflict";
|
|
const reasonCode = PERSISTED_WORKSPACE_SOURCE_REASON_CODES.find(code => code === read(workspaceValidation, "reasonCode"));
|
|
if (reasonCode) execution.workspaceValidationReasonCode = reasonCode;
|
|
}
|
|
if (run.errorCode === "process_lost") {
|
|
const diagnostic = readProcessLossDiagnostic(read(result, "processLossDiagnostic"));
|
|
for (const [field, value] of Object.entries(diagnostic)) {
|
|
execution[`processLoss${field[0]!.toUpperCase()}${field.slice(1)}`] = value;
|
|
}
|
|
}
|
|
const cancellation = readRunCancellation(result);
|
|
if (cancellation) {
|
|
execution.cancellationSource = cancellation.source;
|
|
execution.cancellationExpected = cancellation.expected;
|
|
execution.cancellationInitiatorType = cancellation.initiator.type;
|
|
const leaseLossCause = readControllerLeaseLossCause(cancellation);
|
|
if (leaseLossCause) execution.controllerLeaseLossCause = leaseLossCause;
|
|
} else if (run.status === "cancelled") {
|
|
execution.cancellationSource = "unknown";
|
|
execution.cancellationExpected = false;
|
|
}
|
|
const leaseLoss = readControllerLeaseLoss(read(result, "controllerLeaseLoss"));
|
|
if (leaseLoss) {
|
|
execution.controllerLeaseLossCause = leaseLoss.cause;
|
|
if (leaseLoss.sinceRenewalMs !== undefined) execution.controllerLeaseSinceRenewalMs = leaseLoss.sinceRenewalMs;
|
|
if (leaseLoss.renewalPendingMs !== undefined) execution.controllerLeaseRenewalPendingMs = leaseLoss.renewalPendingMs;
|
|
if (leaseLoss.deadlineLateMs !== undefined) execution.controllerLeaseDeadlineLateMs = leaseLoss.deadlineLateMs;
|
|
if (leaseLoss.renewalCount !== undefined) execution.controllerLeaseRenewalCount = leaseLoss.renewalCount;
|
|
}
|
|
Object.assign(execution, scalars(result, [
|
|
"mode", "stopReason", "timeoutFired", "timeoutSource", "timeoutConfigured",
|
|
"effectiveTimeoutSec", "errorFamily",
|
|
]));
|
|
for (const field of [
|
|
"acpLastEventAgeMs", "acpObservedEventCount", "acpPendingToolCount",
|
|
"acpLastEventAgeAtStopMs", "acpOldestPendingToolAgeAtStopMs",
|
|
]) {
|
|
const value = read(result, field);
|
|
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) execution[field] = value;
|
|
}
|
|
const closeSettlement = readNativeCloseSettlement(options.nativeCloseFailure);
|
|
if (closeSettlement) Object.assign(execution, closeSettlement);
|
|
const pendingToolKinds = readAcpPendingToolKinds(read(result, "acpPendingToolKinds"));
|
|
if (pendingToolKinds) execution.acpPendingToolKinds = pendingToolKinds;
|
|
const toolInventoryComplete = read(result, "acpToolInventoryComplete");
|
|
if (typeof toolInventoryComplete === "boolean") execution.acpToolInventoryComplete = toolInventoryComplete;
|
|
const restoreFailure = read(result, "workspaceRestoreFailure");
|
|
const restoreCode = WORKSPACE_RESTORE_FAILURE_CODES.find((code) => code === restoreFailure);
|
|
if (restoreCode) {
|
|
execution.workspaceRestoreFailure = restoreCode;
|
|
const diagnostic = sanitizeWorkspaceRestoreDiagnostic(read(result, "workspaceRestoreDiagnostic"));
|
|
if (diagnostic) {
|
|
execution.workspaceRestorePhase = diagnostic.phase;
|
|
execution.workspaceRestoreErrorCode = diagnostic.errorCode;
|
|
if (diagnostic.step) execution.workspaceRestoreStep = diagnostic.step;
|
|
if (diagnostic.httpStatus !== undefined) execution.workspaceRestoreHttpStatus = diagnostic.httpStatus;
|
|
if (diagnostic.exitCode !== undefined) execution.workspaceRestoreExitCode = diagnostic.exitCode;
|
|
if (diagnostic.transferStep) execution.workspaceRestoreTransferStep = diagnostic.transferStep;
|
|
if (diagnostic.transferFailureKind) execution.workspaceRestoreTransferFailureKind = diagnostic.transferFailureKind;
|
|
if (diagnostic.rpcCode !== undefined) execution.workspaceRestoreRpcCode = diagnostic.rpcCode;
|
|
if (diagnostic.gitCommand) execution.workspaceRestoreGitCommand = diagnostic.gitCommand;
|
|
if (diagnostic.gitFailureKind) execution.workspaceRestoreGitFailureKind = diagnostic.gitFailureKind;
|
|
}
|
|
}
|
|
const adapter = scalars(options.adapterErrorMeta, [
|
|
"category", "phase", "errorName", "acpCode", "causeMessage", "retryable",
|
|
"stackPreview", "status", "statusCode", "requestId",
|
|
]);
|
|
const provider = scalars(read(result, "terminalSessionFailure"), ["category", "title", "details"]);
|
|
const nativeProviderFailure = readNativeModelRejectionDiagnostic(read(result, "nativeProviderFailure"));
|
|
if (nativeProviderFailure) Object.assign(provider, nativeProviderFailure);
|
|
const nativeTurnFailure = readNativeTurnFailureDiagnostic(read(result, "nativeTurnFailure"));
|
|
if (nativeTurnFailure) Object.assign(provider, nativeTurnFailure);
|
|
const truncatedFields: string[] = [];
|
|
const providerTruncation = read(read(result, "terminalSessionFailure"), "truncatedFields");
|
|
if (Array.isArray(providerTruncation)) {
|
|
for (const field of ["title", "details"]) {
|
|
if (providerTruncation.includes(field)) truncatedFields.push(`provider.${field}`);
|
|
}
|
|
}
|
|
const exceptions: RunFailureException[] = [];
|
|
const seen = new Set<unknown>();
|
|
let error = options.error;
|
|
while (error !== undefined && error !== null && exceptions.length < 4) {
|
|
if (seen.has(error)) {
|
|
truncatedFields.push("exceptions.cycle");
|
|
break;
|
|
}
|
|
seen.add(error);
|
|
if (typeof error === "string") {
|
|
exceptions.push({ message: error });
|
|
error = undefined;
|
|
break;
|
|
}
|
|
if (typeof error !== "object") break;
|
|
const acquisition = options.phase === "setup" && run.errorCode === "setup_failed"
|
|
? getEnvironmentAcquisitionDiagnostic(error, run) : null;
|
|
if (acquisition && execution.environmentAcquisitionPhase === undefined) {
|
|
execution.environmentAcquisitionPhase = acquisition.phase;
|
|
execution.environmentAcquisitionElapsedMs = acquisition.elapsedMs;
|
|
execution.environmentAcquisitionBudgetMs = acquisition.budgetMs;
|
|
if (acquisition.providerState) execution.environmentAcquisitionProviderState = acquisition.providerState;
|
|
}
|
|
if (execution.restoreLockOwnerState === undefined && read(error, "code") === "ERR_WORKSPACE_RESTORE_LOCK_TIMEOUT") {
|
|
const lock = read(error, "workspaceRestoreLock");
|
|
const operation = read(lock, "operation");
|
|
if (["agent_directory_prepare", "agent_directory_release", "agent_directory_collect", "agent_directory_checkpoint", "agent_directory_handoff"].some(value => value === operation)) {
|
|
execution.restoreLockOperation = operation as string;
|
|
}
|
|
const ownerState = read(lock, "ownerState");
|
|
if (["alive", "dead", "unknown", "missing", "invalid"].some(value => value === ownerState)) {
|
|
execution.restoreLockOwnerState = ownerState as string;
|
|
}
|
|
for (const field of ["knownLocalHolder", "ownerSameProcess", "ownerPredatesProcess"]) {
|
|
const value = read(lock, field);
|
|
if (typeof value === "boolean") execution[`restoreLock${field[0]!.toUpperCase()}${field.slice(1)}`] = value;
|
|
}
|
|
for (const field of ["ownerAgeMs", "waitMs"]) {
|
|
const value = read(lock, field);
|
|
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= 7 * 24 * 60 * 60 * 1000) {
|
|
execution[`restoreLock${field[0]!.toUpperCase()}${field.slice(1)}`] = value;
|
|
}
|
|
}
|
|
}
|
|
const entry: RunFailureException = {};
|
|
for (const field of ["name", "message", "stack", "code"] as const) {
|
|
const value = read(error, field);
|
|
if (typeof value === "string") entry[field] = value;
|
|
else if (field === "code" && typeof value === "number" && Number.isFinite(value)) entry.code = String(value);
|
|
}
|
|
if (entry.code === undefined) {
|
|
const reasonCode = read(error, "reasonCode");
|
|
if (typeof reasonCode === "string") entry.code = reasonCode;
|
|
}
|
|
const status = read(error, "status") ?? read(error, "statusCode");
|
|
if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) entry.status = status;
|
|
const requestId = read(error, "requestId") ?? read(error, "request_id");
|
|
if (typeof requestId === "string") entry.requestId = requestId;
|
|
if (Object.keys(entry).length > 0) exceptions.push(entry);
|
|
else break;
|
|
error = read(error, "cause");
|
|
}
|
|
if (error !== undefined && error !== null && exceptions.length === 4) truncatedFields.push("exceptions.depth");
|
|
return { execution, adapter, provider, exceptions, truncatedFields };
|
|
}
|
|
|
|
/** Redact complete values before cutting, including credentials across a cut. */
|
|
export function sanitizeRunFailureText(input: string, limit: number): string {
|
|
const normalized = input
|
|
.replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "")
|
|
.replace(/[\ud800-\udfff]/gu, "\ufffd");
|
|
const clean = redactSensitiveText(redactDiagnosticText(redactCurrentUserText(normalized)));
|
|
if (clean.length <= limit) return clean;
|
|
const suffix = "\n[truncated]";
|
|
let end = limit - suffix.length;
|
|
if ((clean.codePointAt(end - 1) ?? 0) > 0xffff) end--;
|
|
return clean.slice(0, end) + suffix;
|
|
}
|
|
|
|
/** Called after the run's registered secret values have also been removed. */
|
|
export function sanitizeRunFailureDiagnostics(raw: RunFailureDiagnostics): RunFailureDiagnostics {
|
|
const truncatedFields = [...raw.truncatedFields];
|
|
const text = (value: string, path: string, limit: number) => {
|
|
const sanitized = sanitizeRunFailureText(value, limit);
|
|
if (sanitized.endsWith("\n[truncated]")) truncatedFields.push(path);
|
|
return sanitized;
|
|
};
|
|
const context = (value: Context, prefix: string, limits: Record<string, number> = {}): Context =>
|
|
Object.fromEntries(Object.entries(value).map(([key, value]) => [key,
|
|
typeof value === "string" ? text(value, `${prefix}.${key}`, limits[key] ?? 200) : value,
|
|
]));
|
|
return {
|
|
execution: context(raw.execution, "execution"),
|
|
adapter: context(raw.adapter, "adapter", { causeMessage: 2048, stackPreview: 8192 }),
|
|
provider: context(raw.provider, "provider", { title: 4096, details: 12288 }),
|
|
exceptions: raw.exceptions.map((entry, i) => context(entry as Context, `exceptions.${i}`, { message: 2048, stack: 8192 })),
|
|
truncatedFields: [...new Set(truncatedFields)],
|
|
};
|
|
}
|