import { captureEnvironmentAcquisitionDiagnostic } from "../environment-acquisition-diagnostics.js"; import { describe, expect, it } from "vitest"; import type { heartbeatRuns } from "@paperclipai/db"; import { attachRemoteRunnerRecoveryDiagnostic, RemoteRunnerRecoveryError } from "../native-runtime/remote-runner-recovery-diagnostics.js"; import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js"; type Run = typeof heartbeatRuns.$inferSelect; const run = (overrides: Partial = {}) => ({ resultJson: null, ...overrides }) as Run; const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error }); describe("run failure diagnostics", () => { it("selects the saved provider configuration and gateway error code without reading credentials or tool schemas", () => { const failure = Object.assign(new Error('Tool collision: "calendar.search"'), { reasonCode: "tool_name_collision", details: { token: "private-token" }, }); const diagnostic = collectRunFailureDiagnostics(run({ startedAt: new Date("2026-10-14T00:00:00Z"), finishedAt: new Date("2026-10-14T00:00:01Z"), runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex", model: "saved-model", env: { TOKEN: "private-token" } } } }, resultJson: { originalFailureCode: "tool_name_collision" }, }), { error: failure }); expect(diagnostic.execution).toMatchObject({ providerKind: "codex", model: "saved-model", durationMs: 1000, originalFailureCode: "tool_name_collision", startedAt: "2026-10-14T00:00:00.000Z", finishedAt: "2026-10-14T00:00:01.000Z" }); expect(diagnostic.exceptions[0]).toMatchObject({ code: "tool_name_collision" }); expect(JSON.stringify(diagnostic)).not.toContain("private-token"); }); it("reports only private acquisition observations for the exact failed setup run", () => { const scope = { companyId: "company-fixture", environmentId: "environment-fixture", runId: "run-fixture" }; const diagnostic = { phase: "workspace", elapsedMs: 100, budgetMs: 2_000 } as const; const error = Object.freeze(new Error("Acquisition failed")); const properties = Object.getOwnPropertyDescriptors(error); captureEnvironmentAcquisitionDiagnostic(error, scope, diagnostic); const fixture = run({ id: scope.runId, companyId: scope.companyId, errorCode: "setup_failed" }); const collectSetup = (failure: unknown, record = fixture) => collectRunFailureDiagnostics(record, { error: failure, phase: "setup" }).execution; expect(collectSetup(new Error("Cleanup confirmed", { cause: error }))).toMatchObject({ environmentAcquisitionPhase: "workspace", environmentAcquisitionElapsedMs: 100, environmentAcquisitionBudgetMs: 2_000, }); expect(collectSetup(error, run({ ...fixture, id: "another-run" }))).not.toHaveProperty("environmentAcquisitionPhase"); expect(collectSetup(error, run({ ...fixture, companyId: "another-company" }))).not.toHaveProperty("environmentAcquisitionPhase"); expect(collectSetup(error, run({ ...fixture, errorCode: "adapter_failed" }))).not.toHaveProperty("environmentAcquisitionPhase"); expect(collectRunFailureDiagnostics(fixture, { error, phase: "execute" }).execution).not.toHaveProperty("environmentAcquisitionPhase"); expect(Object.getOwnPropertyDescriptors(error)).toEqual(properties); expect(collectSetup(error)).not.toHaveProperty("environmentAcquisitionProviderState"); captureEnvironmentAcquisitionDiagnostic(error, scope, { phase: "create", elapsedMs: 2_000, budgetMs: 2_000, providerState: "pulling_snapshot" }); expect(collectSetup(error)).toMatchObject({ environmentAcquisitionPhase: "create", environmentAcquisitionProviderState: "pulling_snapshot" }); captureEnvironmentAcquisitionDiagnostic(error, scope, null); expect(collectSetup(error)).not.toHaveProperty("environmentAcquisitionPhase"); }); it("ignores forged acquisition fields and persisted JSON without private producer evidence", () => { const acquisitionDiagnostic = { phase: "shell", elapsedMs: 2_000, budgetMs: 2_000 }; const error = Object.assign(new Error("Acquisition failed"), { acquisitionDiagnostic, data: { schema: "paperclip/environment-creation-cleanup/v1", acquisitionDiagnostic }, }); expect(collectRunFailureDiagnostics(run({ errorCode: "setup_failed", resultJson: { acquisitionDiagnostic, environmentAcquisitionPhase: "shell", environmentAcquisitionElapsedMs: 2_000, } }), { error, phase: "setup" }).execution).toEqual({ failurePhase: "setup" }); }); it("exports only bounded workspace base-ref observations without treating them as proof", () => { const resultJson = { configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "private-ref", fetchError: "private-stderr", baseRefDiagnostic: { schemaVersion: 1, remoteLookup: "resolved", authLookup: "failed", fetch: "failed", fetchExitCode: 128, fetchFailureKind: "authentication_failed", refResolution: "failed", refExitCode: 128, repoUrl: "https://private.example.invalid", stderr: "private-token", credential: "private-credential", } } }; const execution = collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson }), {}).execution; expect(execution).toEqual({ workspaceBaseRefRemoteLookup: "resolved", workspaceBaseRefAuthLookup: "failed", workspaceBaseRefFetch: "failed", workspaceBaseRefFetchExitCode: 128, workspaceBaseRefFetchFailureKind: "authentication_failed", workspaceBaseRefRefResolution: "failed", workspaceBaseRefRefExitCode: 128 }); expect(JSON.stringify(execution)).not.toContain("private"); expect(collectRunFailureDiagnostics(run({ errorCode: "setup_failed", resultJson }), {}).execution).toEqual({}); expect(collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: { configurationIncomplete: { ...resultJson.configurationIncomplete, reason: "other" }, } }), {}).execution).toEqual({}); }); it.each([null, -1, 256, Infinity, NaN, 1.5, "private-value", {}])("omits invalid base-ref process codes (%j)", code => { const execution = collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: { configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic: { schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed", fetchExitCode: code, refResolution: "failed", refExitCode: code, fetchFailureKind: "private-value", } }, } }), {}).execution; expect(execution).not.toHaveProperty("workspaceBaseRefFetchExitCode"); expect(execution).not.toHaveProperty("workspaceBaseRefRefExitCode"); expect(execution).not.toHaveProperty("workspaceBaseRefFetchFailureKind"); }); it("rejects forged or unknown base-ref shapes without reading accessor payloads", () => { const base = { schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed" }; let reads = 0; const accessor = Object.defineProperty({}, "schemaVersion", { get() { reads++; throw new Error("private-accessor"); } }); for (const baseRefDiagnostic of [null, {}, [], accessor, Object.create(base), { ...base, schemaVersion: 2 }, { ...base, remoteLookup: "private-url" }, { ...base, authLookup: "private-token" }, { ...base, fetch: "private-command" }]) { expect(collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: { configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic }, } }), {}).execution).toEqual({}); } expect(reads).toBe(0); }); it("exports only fixed workspace validation codes and bounded inspection evidence", () => { const resultJson = { workspaceValidation: { reason: "git_worktree_not_reusable", reasonCode: "git_inspection_failed", worktreePath: "/private/path", executionWorkspaceId: "private-id", repository: "private-url", message: "private-message", inspectionDiagnostic: { command: "worktree_list", failure: "nonzero_exit", exitCode: 128, stderr: "private-token" } } }; const execution = collectRunFailureDiagnostics(run({ errorCode: "workspace_validation_failed", resultJson }), {}).execution; expect(execution).toEqual({ workspaceValidationReason: "git_worktree_not_reusable", workspaceValidationReasonCode: "git_inspection_failed", workspaceValidationInspectionCommand: "worktree_list", workspaceValidationInspectionFailure: "nonzero_exit", workspaceValidationInspectionExitCode: 128 }); expect(JSON.stringify(execution)).not.toContain("private"); expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed", resultJson }), {}).execution).toEqual({}); }); it.each(["missing_worktree", "not_a_git_checkout", "not_registered", "wrong_repository_root", "branch_mismatch"])( "exports %s without attaching unrelated probe evidence", reasonCode => { expect(collectRunFailureDiagnostics(run({ errorCode: "workspace_validation_failed", resultJson: { workspaceValidation: { reason: "git_worktree_not_reusable", reasonCode, inspectionDiagnostic: { command: "worktree_list", failure: "nonzero_exit", exitCode: 128 }, } } }), {}).execution).toEqual({ workspaceValidationReason: "git_worktree_not_reusable", workspaceValidationReasonCode: reasonCode }); }, ); it.each([null, -1, 0, 256, Infinity, NaN, 1.5, "private", {}])("omits invalid Git inspection exit codes (%j)", exitCode => { const execution = collectRunFailureDiagnostics(run({ errorCode: "workspace_validation_failed", resultJson: { workspaceValidation: { reason: "git_worktree_not_reusable", reasonCode: "git_inspection_failed", inspectionDiagnostic: { command: "worktree_list", failure: "nonzero_exit", exitCode }, } } }), {}).execution; expect(execution).not.toHaveProperty("workspaceValidationInspectionExitCode"); expect(JSON.stringify(execution)).not.toContain("private"); }); it("omits unknown validation values and contains hostile diagnostic getters", () => { const inspect = (workspaceValidation: unknown) => collectRunFailureDiagnostics(run({ errorCode: "workspace_validation_failed", resultJson: { workspaceValidation } }), {}).execution; expect(inspect({ reason: "private" })).toEqual({}); expect(inspect({ reason: "git_worktree_not_reusable", reasonCode: "private" })).toEqual({ workspaceValidationReason: "git_worktree_not_reusable" }); const base = { reason: "git_worktree_not_reusable", reasonCode: "git_inspection_failed" }; for (const inspectionDiagnostic of [null, {}, { command: "private", failure: "nonzero_exit" }, { command: "worktree_list", failure: "private" }, Object.defineProperty({}, "command", { get() { throw new Error("private"); } })]) { expect(inspect({ ...base, inspectionDiagnostic })).toEqual({ workspaceValidationReason: base.reason, workspaceValidationReasonCode: base.reasonCode }); } const error = Object.defineProperty({ command: "worktree_list", failure: "spawn_failed" }, "errorCode", { get() { throw new Error("private"); } }); expect(inspect({ ...base, inspectionDiagnostic: error })).toMatchObject({ workspaceValidationInspectionErrorCode: "unknown" }); expect(inspect({ ...base, inspectionDiagnostic: { command: "worktree_list", failure: "spawn_failed", errorCode: "EACCES", exitCode: 128 } })).toMatchObject({ workspaceValidationInspectionErrorCode: "EACCES" }); }); it.each(["source_scope_mismatch", "explicit_project_workspace_conflict", "source_path_unproven", "source_registration_unproven", "source_repository_mismatch", "source_repository_unavailable"])( "exports only the fixed retained-source reason %s", reasonCode => { const workspaceValidation = { reason: "persisted_workspace_source_conflict", reasonCode, executionWorkspaceId: "private-id", repoUrl: "private-url", cwd: "/private-path", message: "private-message" }; expect(collectRunFailureDiagnostics(run({ errorCode: "workspace_validation_failed", resultJson: { workspaceValidation } }), {}).execution) .toEqual({ workspaceValidationReason: "persisted_workspace_source_conflict", workspaceValidationReasonCode: reasonCode }); expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed", resultJson: { workspaceValidation } }), {}).execution).toEqual({}); }, ); it("omits arbitrary retained-source reason strings and hostile getters", () => { for (const reasonCode of ["private-url", Object.defineProperty({}, "value", { get() { throw new Error("private"); } })]) { expect(collectRunFailureDiagnostics(run({ errorCode: "workspace_validation_failed", resultJson: { workspaceValidation: { reason: "persisted_workspace_source_conflict", reasonCode, } } }), {}).execution).toEqual({ workspaceValidationReason: "persisted_workspace_source_conflict" }); } const workspaceValidation = Object.defineProperty({ reason: "persisted_workspace_source_conflict" }, "reasonCode", { get() { throw new Error("private"); }, }); expect(collectRunFailureDiagnostics(run({ errorCode: "workspace_validation_failed", resultJson: { workspaceValidation } }), {}).execution) .toEqual({ workspaceValidationReason: "persisted_workspace_source_conflict" }); }); it("exports bounded orphan evidence only for a process-loss failure", () => { const processLossDiagnostic = { pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers", retryEligible: false, runPredatesObserver: true, observerUptimeMs: 30_000, lastOutputAgeMs: 120_000, pid: 123, path: "/private-path", prompt: "private-prompt" }; const resultJson = { processLossDiagnostic }; expect(collectRunFailureDiagnostics(run({ errorCode: "process_lost", resultJson }), {}).execution).toEqual({ processLossPidRecorded: false, processLossGroupRecorded: false, processLossLocalCheck: "no_identifiers", processLossRetryEligible: false, processLossRunPredatesObserver: true, processLossObserverUptimeMs: 30_000, processLossLastOutputAgeMs: 120_000, }); expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed", resultJson }), {}).execution).toEqual({}); }); it("exports only the closed native model/auth rejection vocabulary", () => { const diagnostic = { provider: "codex", category: "model_auth_incompatible", status: 400, authMode: "chatgpt" }; const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { nativeProviderFailure: { ...diagnostic, model: "private-model", response: "private-provider-response", prompt: "private-prompt" }, } }), {})); expect(result.provider).toEqual(diagnostic); expect(JSON.stringify(result)).not.toContain("private"); for (const value of [null, [], { ...diagnostic, category: "private-category" }, Object.defineProperty({}, "provider", { get() { throw new Error("private"); } })]) { expect(collectRunFailureDiagnostics(run({ resultJson: { nativeProviderFailure: value } }), {}).provider).toEqual({}); } }); it("exports only the closed vocabulary of an unclassified native turn failure", () => { const diagnostic = { provider: "codex", category: "turn_failed", codexErrorInfo: "responseStreamDisconnected", httpStatus: 502 }; const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { nativeTurnFailure: { ...diagnostic, runnerFailureCode: "private_prompt_slug", message: "private-provider-response" }, } }), {})); expect(result.provider).toEqual({ ...diagnostic, runnerFailureCode: "unrecognized" }); expect(JSON.stringify(result)).not.toContain("rivate"); for (const value of [null, [], { ...diagnostic, codexErrorInfo: "private-kind" }, Object.defineProperty({}, "provider", { get() { throw new Error("private"); } })]) { expect(collectRunFailureDiagnostics(run({ resultJson: { nativeTurnFailure: value } }), {}).provider).toEqual({}); } }); it("exports only the saved remote recovery stage and reason", () => { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable", resultJson: { remoteRunnerRecovery: { stage: "identity_marker", reason: "fingerprint_missing", path: "private-path", pid: 123, nonce: "private-nonce", stderr: "private-stderr" }, }, }), {})); expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "fingerprint_missing" }); expect(result.exceptions).toEqual([]); expect(JSON.stringify(result)).not.toContain("private"); for (const value of [null, { stage: "private-stage", reason: "marker_missing" }, { stage: "identity_marker", reason: "private-reason" }, Object.defineProperty({}, "stage", { get() { throw new Error("private"); } })]) { expect(collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable", resultJson: { remoteRunnerRecovery: value } }), {}).execution).toEqual({}); } expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {}).execution).toEqual({}); }); it("exports the remote recovery stage and reason from an error thrown before the result was saved", () => { const thrown = Object.assign(new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }), { path: "private-path", pid: 123 }); const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown })); expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" }); expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]); expect(JSON.stringify(result)).not.toContain("private-path"); expect(result.exceptions[0]).not.toHaveProperty("path"); expect(result.exceptions[0]).not.toHaveProperty("pid"); const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"), { stage: "identity_marker", reason: "probe_transport_failed" }); expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: new Error("wrapped", { cause: transport }) }).execution) .toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "probe_transport_failed" }); const saved = collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable", resultJson: { remoteRunnerRecovery: { stage: "identity_marker", reason: "marker_missing" } } }), { error: thrown }); expect(saved.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "marker_missing" }); expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: Object.assign(new Error("runner_remote_process_identity_unavailable"), { code: "runner_remote_process_identity_unavailable", remoteRunnerRecovery: { stage: "private-stage", reason: "marker_missing" } }), }).execution).toEqual({}); }); it("exports which native close barrier failed without tool identities or provider text", () => { const closeFailure = (settlement: unknown) => Object.assign( new Error("provider_transport_failed: runner did not durably suspend before checkpoint"), { code: "native_session_close_unrecoverable", settlement }, ); const settlement = { runnerSuspended: false, providerDrained: true, suspensionState: { commandStatus: "pending", runnerLifecycle: "ready", runnerIdentityMatches: true }, semanticTools: { persistenceFailed: false, failures: [{ callId: "private-call", operationId: "private-operation", stage: "persist", code: "private-code" }], pending: [{ sourceEventId: "private-event", callId: "private-call", inputDigest: "private-digest" }], deliveries: [{ commandId: "private-command", status: "pending" }, { commandId: "private-command-2", status: "indeterminate" }], }, finalProviderState: { pendingEventCount: 128, activeProviderTurnId: "private-turn", providerSettled: false }, }; const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run(), { nativeCloseFailure: closeFailure(settlement) })); expect(result.execution).toEqual({ closeRunnerSuspended: false, closeProviderDrained: true, closeSuspendCommandStatus: "pending", closeRunnerLifecycle: "ready", closeRunnerIdentityMatches: true, closeProviderState: "unsettled", closeProviderPendingEventCount: 128, closeProviderTurnActive: true, closeSemanticResultPersistenceFailed: false, closeSemanticToolsPending: 1, closeSemanticToolDeliveriesOpen: 2, closeSemanticResultFailures: 1, }); expect(result.exceptions).toEqual([]); expect(JSON.stringify(result)).not.toContain("private"); expect(JSON.stringify(result)).not.toContain("durably suspend"); const unobserved = collectRunFailureDiagnostics(run(), { nativeCloseFailure: new Error("wrapped", { cause: closeFailure({ runnerSuspended: true, providerDrained: false, suspensionState: null, finalProviderState: null, }) }) }).execution; expect(unobserved).toEqual({ closeRunnerSuspended: true, closeProviderDrained: false, closeProviderState: "not_local" }); expect(collectRunFailureDiagnostics(run(), { nativeCloseFailure: closeFailure({ suspensionState: { commandStatus: null, runnerLifecycle: null, runnerIdentityMatches: null }, finalProviderState: "unreadable", }) }).execution).toEqual({ closeSuspendCommandStatus: "not_queued", closeRunnerLifecycle: "unobserved", closeProviderState: "unreadable" }); }); it("ignores unrelated errors and unknown or hostile native close settlement values", () => { expect(collectRunFailureDiagnostics(run(), { nativeCloseFailure: Object.assign(new Error("x"), { code: "provider_transport_failed", settlement: { runnerSuspended: false }, }) }).execution).toEqual({}); const hostile = Object.assign(new Error("x"), { code: "native_session_close_unrecoverable", settlement: { runnerSuspended: "private", providerDrained: 1, suspensionState: { commandStatus: "private-status", runnerLifecycle: "private-lifecycle", runnerIdentityMatches: "yes" }, finalProviderState: { pendingEventCount: -1, activeProviderTurnId: 7, providerSettled: "private" }, semanticTools: Object.defineProperty({ pending: Number.POSITIVE_INFINITY }, "persistenceFailed", { get() { throw new Error("private"); } }), } }); expect(collectRunFailureDiagnostics(run(), { nativeCloseFailure: hostile }).execution).toEqual({}); expect(collectRunFailureDiagnostics(run(), { nativeCloseFailure: Object.assign(new Error("x"), { code: "native_session_close_unrecoverable", }) }).execution).toEqual({}); }); it("selects bounded lock-owner evidence from a caught timeout cause", () => { const error = new Error("outer", { cause: Object.assign(new Error("lock timeout"), { code: "ERR_WORKSPACE_RESTORE_LOCK_TIMEOUT", workspaceRestoreLock: { operation: "agent_directory_release", ownerState: "alive", ownerSameProcess: true, ownerPredatesProcess: true, knownLocalHolder: false, ownerAgeMs: 120_000, waitMs: 30_001, ownerPid: 123, path: "/sentinel-lock-path", owner: { payload: "sentinel-owner-payload" } }, }) }); const result = sanitizeRunFailureDiagnostics(collect(error)); expect(result.execution).toEqual({ restoreLockOperation: "agent_directory_release", restoreLockOwnerState: "alive", restoreLockOwnerSameProcess: true, restoreLockOwnerPredatesProcess: true, restoreLockKnownLocalHolder: false, restoreLockOwnerAgeMs: 120_000, restoreLockWaitMs: 30_001 }); expect(JSON.stringify(result)).not.toContain("sentinel-"); expect(result.execution).not.toHaveProperty("ownerPid"); }); it.each([null, -1, Infinity, NaN, 1.5, 604_800_001, "private", {}])("omits invalid lock diagnostic values (%j)", value => { const error = Object.assign(new Error("lock timeout"), { code: "ERR_WORKSPACE_RESTORE_LOCK_TIMEOUT", workspaceRestoreLock: { operation: value, ownerState: value, ownerAgeMs: value, waitMs: value, ownerSameProcess: value, ownerPredatesProcess: value, knownLocalHolder: value } }); expect(collect(error).execution).toEqual({}); }); it("does not attach lock evidence to unrelated errors or invoke hostile getters", () => { expect(collect({ code: "OTHER", workspaceRestoreLock: { ownerState: "alive" } }).execution).toEqual({}); const error = { code: "ERR_WORKSPACE_RESTORE_LOCK_TIMEOUT", workspaceRestoreLock: Object.defineProperty({}, "ownerState", { get() { throw new Error("private"); } }) }; expect(collect(error).execution).toEqual({}); }); it.each(["restore_permission_denied", "restore_lock_timeout", "restore_unsafe_archive", "restore_failed"])( "includes the saved %s classification without copying workspace paths or results", (code) => { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: code, workspaceRestorePath: "/private/workspace", executionBeforeRestore: { errorMessage: "private provider response" }, } }), {})); expect(result.execution).toEqual({ workspaceRestoreFailure: code }); expect(JSON.stringify(result)).not.toContain("private"); }, ); it.each([null, false, 1, "private arbitrary code", { error: "private" }])( "omits unknown workspace restore classifications (%j)", (value) => { const result = collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: value } }), {}); expect(result.execution).not.toHaveProperty("workspaceRestoreFailure"); expect(JSON.stringify(result)).not.toContain("private"); }, ); it("copies only classified restore diagnostics from the saved result", () => { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: { phase: "workspace", step: "git_integration", errorCode: "unknown", httpStatus: 503, exitCode: 1, gitCommand: "merge_tree", gitFailureKind: "merge_conflict", message: "private command failed", path: "/private/workspace", stdout: "private file contents", cause: { code: "EIO", message: "private nested cause" }, }, } }), {})); expect(result.execution).toEqual({ workspaceRestoreFailure: "restore_failed", workspaceRestorePhase: "workspace", workspaceRestoreStep: "git_integration", workspaceRestoreErrorCode: "unknown", workspaceRestoreHttpStatus: 503, workspaceRestoreExitCode: 1, workspaceRestoreGitCommand: "merge_tree", workspaceRestoreGitFailureKind: "merge_conflict", }); expect(JSON.stringify(result)).not.toContain("private"); expect(result.exceptions).toEqual([]); }); it.each([ { phase: "asset", step: "git_integration", gitCommand: "merge_tree", gitFailureKind: "merge_conflict" }, { phase: "workspace", step: "git_import", gitCommand: "merge_tree", gitFailureKind: "merge_conflict" }, { phase: "workspace", step: "git_integration", gitCommand: "private-command", gitFailureKind: "private-output" }, ])("omits unrelated or unrecognized persisted Git labels (%j)", (diagnostic) => { const result = collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: diagnostic, } }), {}); expect(result.execution).not.toHaveProperty("workspaceRestoreGitCommand"); expect(result.execution).not.toHaveProperty("workspaceRestoreGitFailureKind"); expect(JSON.stringify(result)).not.toContain("private-"); }); it("requires a known restore failure before reading its diagnostic", () => { const diagnostic = { phase: "workspace", step: "git_import", errorCode: "EIO", exitCode: 1 }; for (const code of [undefined, null, "private unknown classification"]) { const result = collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: code, workspaceRestoreDiagnostic: diagnostic, } }), {}); expect(result.execution).toEqual({}); } const resultJson = Object.defineProperty({}, "workspaceRestoreDiagnostic", { get() { throw new Error("must not read unrelated diagnostic"); }, }); expect(collectRunFailureDiagnostics(run({ resultJson }), {}).execution).toEqual({}); }); it.each([null, false, 1, "private payload", [], { phase: "private phase", errorCode: "EIO" }])( "omits malformed workspace restore diagnostics (%j)", (value) => { const result = collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: value, } }), {}); expect(result.execution).toEqual({ workspaceRestoreFailure: "restore_failed" }); expect(JSON.stringify(result)).not.toContain("private"); }, ); it.each([null, -1, Infinity, NaN, 1.5, 600, "private status", {}])( "omits invalid restore status and exit values (%j)", (value) => { const result = collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: { phase: "asset", step: "private step", errorCode: "private code", httpStatus: value, exitCode: value, }, } }), {}); expect(result.execution).toEqual({ workspaceRestoreFailure: "restore_failed", workspaceRestorePhase: "asset", workspaceRestoreErrorCode: "unknown", }); }, ); it("does not walk restore causes or expose errors from diagnostic getters", () => { const diagnostic = Object.defineProperties({ phase: "workspace", errorCode: "EIO" }, { step: { get() { throw new Error("private getter"); } }, httpStatus: { get() { throw new Error("private getter"); } }, exitCode: { get() { throw new Error("private getter"); } }, cause: { get() { throw new Error("must not walk cause"); } }, }); const result = collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: diagnostic, } }), {}); expect(result.execution).toEqual({ workspaceRestoreFailure: "restore_failed", workspaceRestorePhase: "workspace", workspaceRestoreErrorCode: "EIO", }); expect(result.exceptions).toEqual([]); }); it("includes only bounded ACP activity fields, not tool names, identities, or output", () => { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { acpLastEventAgeMs: 14_000_000, acpObservedEventCount: 9, acpPendingToolCount: 2, acpToolInventoryComplete: false, acpToolNames: ["private command"], lastEvent: "private output", } }), {})); expect(result.execution).toEqual({ acpLastEventAgeMs: 14_000_000, acpObservedEventCount: 9, acpPendingToolCount: 2, acpToolInventoryComplete: false, }); expect(JSON.stringify(result)).not.toContain("private"); }); it("includes stop-time ACP activity and closed pending tool-kind labels", () => { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { acpLastEventAgeMs: 700, acpLastEventAgeAtStopMs: 14_000_000, acpOldestPendingToolAgeAtStopMs: 14_100_000, acpPendingToolKinds: "execute,unknown", } }), {})); expect(result.execution).toEqual({ acpLastEventAgeMs: 700, acpLastEventAgeAtStopMs: 14_000_000, acpOldestPendingToolAgeAtStopMs: 14_100_000, acpPendingToolKinds: "execute,unknown", }); }); it("includes the stalled tool's closed kind label, silent time and permission evidence", () => { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { acpStalledToolKind: "other", acpStalledToolSilentMs: 1_800_000, acpStalledToolTitle: "private lookup", acpStalledToolPermissionRequested: false, acpPendingToolPermissionRequestedCount: 3, acpSettledSameKindToolCount: 30, acpSettledSameKindToolPermissionRequestedCount: 29, } }), {})); expect(result.execution).toEqual({ acpStalledToolKind: "other", acpStalledToolSilentMs: 1_800_000, acpStalledToolPermissionRequested: false, acpPendingToolPermissionRequestedCount: 3, acpSettledSameKindToolCount: 30, acpSettledSameKindToolPermissionRequestedCount: 29, }); expect(JSON.stringify(result)).not.toContain("private"); }); it.each(["other,execute", "private lookup", "", "Other", ["other"], 1])( "omits a stalled tool kind that is not one closed label (%j)", (value) => { const result = collectRunFailureDiagnostics(run({ resultJson: { acpStalledToolKind: value } }), {}); expect(result.execution).toEqual({}); }, ); it.each(["execute,private command", "", "execute,", "Execute", ["execute"], 1])( "omits pending tool kinds that are not closed labels (%j)", (value) => { const result = collectRunFailureDiagnostics(run({ resultJson: { acpPendingToolKinds: value } }), {}); expect(result.execution).toEqual({}); }, ); it.each([null, -1, Infinity, NaN, 1.5, Number.MAX_SAFE_INTEGER + 1, "private", {}])( "omits invalid ACP activity fields (%j)", (value) => { const result = collectRunFailureDiagnostics(run({ resultJson: { acpLastEventAgeMs: value, acpObservedEventCount: value, acpPendingToolCount: value, acpToolInventoryComplete: value, acpLastEventAgeAtStopMs: value, acpOldestPendingToolAgeAtStopMs: value, acpStalledToolSilentMs: value, acpStalledToolPermissionRequested: value, acpPendingToolPermissionRequestedCount: value, acpSettledSameKindToolCount: value, acpSettledSameKindToolPermissionRequestedCount: value, } }), {}); expect(result.execution).toEqual({}); }, ); it("selects declared environment secrets under opaque names and common credential keys", () => { expect(collectRunFailureSecretValues({ CUSTOM_BINDING: "bound-opaque-value", ACCESS_TOKEN: "plain-opaque-value", REGION: "us-east-1", EMPTY_KEY: "", }, ["CUSTOM_BINDING"])).toEqual(["bound-opaque-value", "plain-opaque-value", "us-east-1"]); expect(collectRunFailureSecretValues({ PATH: "/usr/bin", CUSTOM_BINDING: "opaque" }, [], true)).toEqual(["opaque"]); expect(collectRunFailureSecretValues({ PATH: "secret-override" }, ["PATH"], true)).toEqual(["secret-override"]); }); it("redacts encoded credentials and URL passwords without expanding replacement markers", () => { const secret = 'opaque/"credential'; const values = collectRunFailureSecretValues({ DATABASE_URL: `postgres://user:${encodeURIComponent(secret)}@example/db` }); const result = redactRunFailureSecretValues({ text: `${secret} ${encodeURIComponent(secret)} ${JSON.stringify(secret)}`, status: 503 }, [...values, "REDACTED"]); expect(result.text).not.toContain("opaque"); expect(result.status).toBe(503); expect(result.text).not.toContain("***[REDACTED]"); }); it("preserves known boolean and numeric settings without exempting unknown or declared secrets", () => { const env = { PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_DB_BACKUP_RETENTION_DAYS: "1" }; expect(collectRunFailureSecretValues(env, [], true)).toEqual([]); expect(collectRunFailureSecretValues(env, ["PAPERCLIP_DB_BACKUP_ENABLED"], true)).toEqual(["false"]); expect(collectRunFailureSecretValues({ CUSTOM_BINDING: "1" }, [], true)).toEqual(["1"]); expect(collectRunFailureSecretValues({ PAPERCLIP_DB_BACKUP_ENABLED: "opaque" }, [], true)).toEqual(["opaque"]); }); it("preserves generic exceptions, numeric codes and nested network causes", () => { const root = Object.assign(new Error("network failed"), { code: "ECONNRESET", statusCode: 502, request_id: "req-123" }); const outer = Object.assign(new TypeError("adapter failed", { cause: root }), { code: -32000 }); const result = sanitizeRunFailureDiagnostics(collect(outer)); expect(result.exceptions).toMatchObject([ { name: "TypeError", message: "adapter failed", code: "-32000" }, { name: "Error", message: "network failed", code: "ECONNRESET", status: 502, requestId: "req-123" }, ]); expect(result.exceptions[0].stack).toContain("run-failure-diagnostics.test.ts"); }); it("bounds cyclic, deep, and string causes without inspecting arbitrary error objects", () => { const cyclic = new Error("cycle"); cyclic.cause = cyclic; expect(collect(cyclic)).toMatchObject({ exceptions: [{ message: "cycle" }], truncatedFields: ["exceptions.cycle"] }); let error = new Error("leaf"); for (let i = 0; i < 10; i++) error = new Error(`level-${i}`, { cause: error }); expect(collect(error).exceptions).toHaveLength(4); expect(collect(error).truncatedFields).toContain("exceptions.depth"); expect(collect(new Error("outer", { cause: "string cause" })).exceptions).toHaveLength(2); const getter = Object.defineProperty({ message: "valid message" }, "response", { get: () => { throw new Error("must not read"); } }); expect(collect(getter).exceptions).toEqual([{ message: "valid message" }]); expect(collect({ arbitrary: "private payload" }).exceptions).toEqual([]); expect(collect(Object.defineProperty({}, "message", { get: () => { throw new Error("bad getter"); } })).exceptions).toEqual([]); }); it("redacts credential forms before truncation in all diagnostic text", () => { const failure = collectRunFailureDiagnostics(run({ resultJson: { terminalSessionFailure: { category: "service", title: 'password="secret-password"', details: 'Authorization: Bearer secret-bearer' }, } }), { error: Object.assign(new Error('apiKey="secret-api-key"'), { cause: new Error('token="secret-token"') }), adapterErrorMeta: { causeMessage: 'password="secret-password"' } }); const result = sanitizeRunFailureDiagnostics(failure); expect(JSON.stringify(result)).not.toContain("secret-"); expect(sanitizeRunFailureText('x'.repeat(90) + ' password="secret-at-cut-boundary"', 110)).not.toContain("secret-at"); expect(sanitizeRunFailureText('pass\x1b[31mword="secret-colored"', 200)).not.toContain("secret-colored"); }); it("reports truncation, preserves UTF-16 pairs, and keeps the total payload bounded", () => { const huge = "😀".repeat(100_000); let error = Object.assign(new Error(huge), { stack: huge }); for (let i = 0; i < 5; i++) error = Object.assign(new Error(huge, { cause: error }), { stack: huge }); const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { terminalSessionFailure: { category: "service", title: huge, details: huge, truncatedFields: ["details", "private-field"] }, } }), { error })); expect(String(result.provider.title).length).toBeLessThanOrEqual(4096); expect(String(result.provider.details).length).toBeLessThanOrEqual(12288); expect(JSON.stringify(result)).not.toMatch(/[\ud800-\udfff]/u); expect(result.truncatedFields).toContain("provider.details"); expect(result.truncatedFields).toContain("exceptions.0.stack"); expect(result.truncatedFields).not.toContain("provider.private-field"); expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan(150_000); }); it("keeps scalar native and legacy execution evidence and rejects non-finite durations", () => { const result = collectRunFailureDiagnostics(run({ runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3", startedAt: new Date(1000), finishedAt: new Date(8500), resultJson: { stopReason: "failed", timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream", summary: "private prose" }, }), { phase: "execute" }); expect(result.execution).toMatchObject({ runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3", durationMs: 7500, timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream" }); expect(JSON.stringify(result)).not.toContain("private prose"); expect(collectRunFailureDiagnostics(run({ startedAt: new Date(NaN), finishedAt: new Date() }), {}).execution).not.toHaveProperty("durationMs"); }); }); it("revalidates persisted transfer evidence and excludes arbitrary nested payloads", () => { for (const known of [true, false]) { const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: { phase: "workspace", step: "workspace_transfer", errorCode: "unknown", transferStep: known ? "archive_create" : "private-command", transferFailureKind: known ? "command_failed" : "private-output", rpcCode: known ? -32003 : -12345, command: "private-command", cause: { message: "private-nested-message" }, }, } }), {})); expect(result.execution.workspaceRestoreTransferStep).toBe(known ? "archive_create" : undefined); expect(result.execution.workspaceRestoreTransferFailureKind).toBe(known ? "command_failed" : undefined); expect(result.execution.workspaceRestoreRpcCode).toBe(known ? -32003 : undefined); expect(JSON.stringify(result)).not.toContain("private-"); } }); it("does not project transfer fields into an unrelated restore step", () => { const result = collectRunFailureDiagnostics(run({ resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: { phase: "workspace", step: "git_import", errorCode: "unknown", transferStep: "archive_create", transferFailureKind: "command_failed", rpcCode: -32003, }, } }), {}); expect(result.execution).not.toHaveProperty("workspaceRestoreTransferStep"); expect(result.execution).not.toHaveProperty("workspaceRestoreTransferFailureKind"); expect(result.execution).not.toHaveProperty("workspaceRestoreRpcCode"); });