From 128c95837b941bfe38f4f77a3d10776ed4c4f21b Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Wed, 7 Oct 2026 12:05:21 -0700 Subject: [PATCH] Retain bounded orphan process-loss diagnostics (#15475) Record bounded observer uptime, run and output ages, process-check observations and retry eligibility before orphan cleanup changes the evidence. Preserve existing recovery and reporting behavior and omit process IDs, raw paths and credentials. Verified focused helper, actual reaper and real SDK regressions, full typecheck/build, exact-head CI and independent review. An unchanged Cursor timeout passed its isolated retry; unrelated local Slack timing failure is documented separately. Greptile 5/5 with no unresolved comments. Co-Authored-By: Paperclip --- doc/observability.md | 13 ++++ .../heartbeat-process-recovery.test.ts | 8 +++ .../run-failure-sentry-real-sdk.test.ts | 43 +++++++++++ .../process-loss-diagnostics.test.ts | 56 +++++++++++++++ .../__tests__/run-failure-diagnostics.test.ts | 13 ++++ server/src/services/heartbeat.ts | 11 ++- .../src/services/process-loss-diagnostics.ts | 71 +++++++++++++++++++ .../src/services/run-failure-diagnostics.ts | 7 ++ 8 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 server/src/services/__tests__/process-loss-diagnostics.test.ts create mode 100644 server/src/services/process-loss-diagnostics.ts diff --git a/doc/observability.md b/doc/observability.md index e8d1262a0bb..b9d19e6f212 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -524,6 +524,19 @@ and native runs: - `run_exception_0` through `run_exception_3`: exception names, codes, HTTP statuses, and request IDs for a caught exception and up to three causes. +The orphan reaper records a bounded `processLossDiagnostic` before status writes +or cleanup. For `process_lost` failures, `run_execution` includes +`processLossPidRecorded`, `processLossGroupRecorded`, and `processLossLocalCheck` +(`not_observed_alive`, `not_checked`, or `no_identifiers`). It also includes +`processLossRunPredatesObserver`, `processLossObserverUptimeMs`, and +`processLossLastOutputAgeMs` when the timestamps are available and valid. Ages +above seven days are omitted. These are observations, not proof of an OOM, +provider failure, or deployment. The last-output age can include system output. +`processLossRetryEligible` records eligibility for the existing process-loss retry +at detection; it does not claim that a retry was queued or succeeded. This data +contains no process IDs, task text, paths, or credentials and changes no recovery +or ownership decisions. + ACP turns record `acpLastEventAgeMs`, `acpObservedEventCount`, `acpPendingToolCount`, and `acpToolInventoryComplete` at finalization, before usage reads, error logging, and cleanup. The age measures time since the last diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 86b3867ae50..325ffbc0a9b 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -2717,6 +2717,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { stopReason: "process_lost", timeoutConfigured: false, timeoutFired: false, + processLossDiagnostic: { + pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true, + observerUptimeMs: expect.any(Number), runPredatesObserver: true, + }, }); // The legacy engine writes this terminal status through the same // guarded emitter that reports a genuine failed transition to Sentry. @@ -2726,6 +2730,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { runId, errorCode: "process_lost", runStatus: "failed", + diagnostics: expect.objectContaining({ execution: expect.objectContaining({ + processLossPidRecorded: true, processLossGroupRecorded: false, processLossLocalCheck: "not_observed_alive", + processLossRetryEligible: true, processLossRunPredatesObserver: true, + }) }), }), ); const [action] = await db diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts index 5cac0081e31..f92e0a42bc0 100644 --- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts +++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts @@ -42,6 +42,49 @@ afterEach(async () => { }); describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", () => { + it("sends only bounded process-loss evidence without leaking it into another capture", async () => { + const Sentry = sentryPackage!; + const events: Array> = []; + vi.stubEnv("SENTRY_DSN_BACKEND", "https://public@example.invalid/1"); + vi.doMock("../peer-version-check.js", () => ({ checkExactPeerVersions: () => ({ ok: true }) })); + vi.doMock("@sentry/node", () => ({ + ...Sentry, + init: (options: Record) => Sentry.init({ + ...options, + transport: () => ({ send: async () => ({}), flush: async () => true }), + beforeSend: (event: Record) => { events.push(event); return event; }, + }), + })); + vi.resetModules(); + const { sentryReady, captureRunFailure, captureException } = await import("../sentry.js"); + await sentryReady; + const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({ + errorCode: "process_lost", resultJson: { processLossDiagnostic: { + pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true, + runPredatesObserver: true, observerUptimeMs: 30_000, lastOutputAgeMs: 120_000, + pid: "private-process-identity", path: "/private-process-path", prompt: "private-process-prompt", + } }, + } as unknown as typeof heartbeatRuns.$inferSelect, {})); + captureRunFailure({ + taskId: "11111111-1111-4111-8111-111111111111", runId: "22222222-2222-4222-8222-222222222222", + errorMessage: "Process lost -- server may have restarted", errorCode: "process_lost", + agentAdapter: "fixture-adapter", runStatus: "failed", exitCode: null, signal: null, diagnostics, + }); + captureException(new Error("unrelated process-loss fixture")); + await Sentry.flush(2000); + expect(events).toHaveLength(2); + const failure = events.find(event => (event.tags as Record)?.error_code === "process_lost"); + expect(failure).toMatchObject({ contexts: { run_execution: { + processLossPidRecorded: true, processLossGroupRecorded: false, processLossLocalCheck: "not_observed_alive", + processLossRetryEligible: true, processLossRunPredatesObserver: true, + processLossObserverUptimeMs: 30_000, processLossLastOutputAgeMs: 120_000, + } } }); + expect(JSON.stringify(events)).not.toContain("private-process-"); + const unrelated = events.find(event => event !== failure); + expect(unrelated).not.toHaveProperty("contexts.run_execution"); + expect(unrelated).not.toHaveProperty("contexts.run_failure"); + }); + it("keeps portfolio diagnostics bounded and isolated from unrelated captures", async () => { const Sentry = sentryPackage!; const events: Array> = []; diff --git a/server/src/services/__tests__/process-loss-diagnostics.test.ts b/server/src/services/__tests__/process-loss-diagnostics.test.ts new file mode 100644 index 00000000000..f8efe72f582 --- /dev/null +++ b/server/src/services/__tests__/process-loss-diagnostics.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { buildProcessLossDiagnostic, readProcessLossDiagnostic } from "../process-loss-diagnostics.js"; + +const nowMs = Date.UTC(2026, 0, 1, 12); +const run = { + startedAt: new Date(nowMs - 10 * 60_000), lastOutputAt: new Date(nowMs - 2 * 60_000), + processPid: 123, processGroupId: null, +}; +const input = { run, nowMs, observerStartedAtMs: nowMs - 30_000, + checksPersistedChildLiveness: true, retryEligible: true }; + +describe("orphan process-loss evidence", () => { + it("distinguishes a run from the previous observer without inventing a restart cause", () => { + expect(buildProcessLossDiagnostic(input)).toEqual({ + pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true, + runPredatesObserver: true, lastOutputAgeMs: 120_000, observerUptimeMs: 30_000, + }); + const currentObserverRun = buildProcessLossDiagnostic({ ...input, observerStartedAtMs: nowMs - 20 * 60_000 }); + expect(currentObserverRun.runPredatesObserver).toBe(false); + expect(currentObserverRun).not.toHaveProperty("restartCause"); + }); + + it("does not describe missing or unchecked identifiers as confirmed child death", () => { + expect(buildProcessLossDiagnostic({ ...input, checksPersistedChildLiveness: false }).localCheck).toBe("not_checked"); + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, processPid: null }, retryEligible: false })) + .toMatchObject({ pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers", retryEligible: false }); + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, processPid: null, processGroupId: 456 } })) + .toMatchObject({ pidRecorded: false, groupRecorded: true, localCheck: "not_observed_alive" }); + }); + + it("omits unavailable, future and unbounded ages", () => { + for (const lastOutputAt of [null, new Date(NaN), new Date(nowMs + 1), new Date(nowMs - 604_800_001)]) { + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, lastOutputAt } })).not.toHaveProperty("lastOutputAgeMs"); + } + for (const observerStartedAtMs of [NaN, Infinity, nowMs + 1]) { + const diagnostic = buildProcessLossDiagnostic({ ...input, observerStartedAtMs }); + expect(diagnostic).not.toHaveProperty("observerUptimeMs"); + expect(diagnostic).not.toHaveProperty("runPredatesObserver"); + } + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, startedAt: new Date(nowMs + 1) } })) + .not.toHaveProperty("runPredatesObserver"); + expect(buildProcessLossDiagnostic({ ...input, observerStartedAtMs: nowMs - 1.5 }).observerUptimeMs).toBe(2); + }); + + it("reads only bounded scalar evidence, including hostile persisted input", () => { + const diagnostic = buildProcessLossDiagnostic(input); + expect(readProcessLossDiagnostic({ ...diagnostic, pid: 123, prompt: "private-prompt", path: "/private/path" })) + .toEqual(diagnostic); + for (const value of [null, [], "private", { localCheck: "private", pidRecorded: 1, + groupRecorded: "false", retryEligible: "private", runPredatesObserver: null, + observerUptimeMs: 604_800_001, lastOutputAgeMs: -1 }, + Object.defineProperty({}, "localCheck", { get() { throw new Error("private"); } })]) { + expect(readProcessLossDiagnostic(value)).toEqual({}); + } + }); +}); diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts index 6ee721b7a37..0cc4d104b7f 100644 --- a/server/src/services/__tests__/run-failure-diagnostics.test.ts +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -7,6 +7,19 @@ const run = (overrides: Partial = {}) => ({ resultJson: null, ...overrides const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error }); describe("run failure diagnostics", () => { + it("exports bounded orphan evidence only for a process-loss failure", () => { + const processLossDiagnostic = { pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers", + retryEligible: false, runPredatesObserver: true, observerUptimeMs: 30_000, lastOutputAgeMs: 120_000, + pid: 123, path: "/private-path", prompt: "private-prompt" }; + const resultJson = { processLossDiagnostic }; + expect(collectRunFailureDiagnostics(run({ errorCode: "process_lost", resultJson }), {}).execution).toEqual({ + processLossPidRecorded: false, processLossGroupRecorded: false, processLossLocalCheck: "no_identifiers", + processLossRetryEligible: false, processLossRunPredatesObserver: true, + processLossObserverUptimeMs: 30_000, processLossLastOutputAgeMs: 120_000, + }); + expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed", resultJson }), {}).execution).toEqual({}); + }); + it("exports only the closed native model/auth rejection vocabulary", () => { const diagnostic = { provider: "codex", category: "model_auth_incompatible", status: 400, authMode: "chatgpt" }; const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 22895243838..b475c28fd52 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -351,6 +351,8 @@ import { emitAgentTaskRunById, } from "./agent-task-run-telemetry.js"; import { reportRunFailure } from "./run-failure-report.js"; +import { performance } from "node:perf_hooks"; +import { buildProcessLossDiagnostic } from "./process-loss-diagnostics.js"; import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js"; import { companySkillService } from "./company-skills.js"; import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js"; @@ -19617,6 +19619,13 @@ export function heartbeatService( monitorDispatchLostWithoutFutureWake); if (!(await revokeExpiredLegacyController(db, run))) continue; const baseMessage = buildProcessLossMessage(run); + const processLossDiagnostic = buildProcessLossDiagnostic({ + run, + nowMs: Date.now(), + observerStartedAtMs: performance.timeOrigin, + checksPersistedChildLiveness, + retryEligible: shouldRetry, + }); const conversationContinuationEligible = await runUsedConversationAdapter(db, run); const failureWrite = await setRunStatusFromLive( @@ -19633,7 +19642,7 @@ export function heartbeatService( "failed", { conversationContinuationEligible, - resultJson: parseObject(run.resultJson), + resultJson: { ...parseObject(run.resultJson), processLossDiagnostic }, errorCode: "process_lost", errorMessage: shouldRetry ? `${baseMessage}; retrying once` diff --git a/server/src/services/process-loss-diagnostics.ts b/server/src/services/process-loss-diagnostics.ts new file mode 100644 index 00000000000..408c89d0970 --- /dev/null +++ b/server/src/services/process-loss-diagnostics.ts @@ -0,0 +1,71 @@ +import type { heartbeatRuns } from "@paperclipai/db"; + +type Run = Pick; +const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1_000; +const LOCAL_CHECKS = ["not_observed_alive", "not_checked", "no_identifiers"] as const; + +export interface ProcessLossDiagnostic { + pidRecorded: boolean; + groupRecorded: boolean; + localCheck: typeof LOCAL_CHECKS[number]; + retryEligible: boolean; + runPredatesObserver?: boolean; + lastOutputAgeMs?: number; + observerUptimeMs?: number; +} + +function age(now: number, then: number | undefined): number | undefined { + if (then === undefined) return undefined; + const elapsed = now - then; + return Number.isSafeInteger(elapsed) && elapsed >= 0 && elapsed <= MAX_AGE_MS + ? elapsed : undefined; +} + +/** Snapshot before status writes and cleanup. Never infer why ownership was lost. */ +export function buildProcessLossDiagnostic(input: { + run: Run; + nowMs: number; + observerStartedAtMs: number; + checksPersistedChildLiveness: boolean; + retryEligible: boolean; +}): ProcessLossDiagnostic { + const { run, nowMs, observerStartedAtMs } = input; + const pidRecorded = !!run.processPid; + const groupRecorded = !!run.processGroupId; + const startedAt = run.startedAt?.getTime(); + const validObserverStart = Number.isFinite(observerStartedAtMs) && observerStartedAtMs <= nowMs; + const lastOutputAgeMs = age(nowMs, run.lastOutputAt?.getTime()); + const observerUptimeMs = validObserverStart ? age(nowMs, Math.floor(observerStartedAtMs)) : undefined; + return { + pidRecorded, + groupRecorded, + localCheck: !pidRecorded && !groupRecorded ? "no_identifiers" + : input.checksPersistedChildLiveness ? "not_observed_alive" : "not_checked", + retryEligible: input.retryEligible, + ...(validObserverStart && startedAt !== undefined && Number.isFinite(startedAt) && startedAt <= nowMs + ? { runPredatesObserver: startedAt < observerStartedAtMs } : {}), + ...(lastOutputAgeMs !== undefined ? { lastOutputAgeMs } : {}), + ...(observerUptimeMs !== undefined ? { observerUptimeMs } : {}), + }; +} + +/** Persisted adapter data is untrusted. Only closed labels, booleans and ages leave the instance. */ +export function readProcessLossDiagnostic(value: unknown): Partial { + const result: Partial = {}; + if (!value || typeof value !== "object" || Array.isArray(value)) return result; + const read = (key: string): unknown => { + try { return (value as Record)[key]; } catch { return undefined; } + }; + for (const key of ["pidRecorded", "groupRecorded", "retryEligible", "runPredatesObserver"] as const) { + const entry = read(key); + if (typeof entry === "boolean") result[key] = entry; + } + const localCheck = read("localCheck"); + if (LOCAL_CHECKS.some(check => check === localCheck)) result.localCheck = localCheck as ProcessLossDiagnostic["localCheck"]; + for (const key of ["lastOutputAgeMs", "observerUptimeMs"] as const) { + const entry = read(key); + if (typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0 && entry <= MAX_AGE_MS) result[key] = entry; + } + return result; +} diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index c5c9d7e820b..8dc0d78a289 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -1,5 +1,6 @@ import type { heartbeatRuns } from "@paperclipai/db"; import { readRunCancellation } from "./run-cancellation.js"; +import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js"; import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared"; import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction"; import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics"; @@ -146,6 +147,12 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs; } const result = run.resultJson; + if (run.errorCode === "process_lost") { + const diagnostic = readProcessLossDiagnostic(read(result, "processLossDiagnostic")); + for (const [field, value] of Object.entries(diagnostic)) { + execution[`processLoss${field[0]!.toUpperCase()}${field.slice(1)}`] = value; + } + } const cancellation = readRunCancellation(result); if (cancellation) { execution.cancellationSource = cancellation.source;