diff --git a/doc/observability.md b/doc/observability.md index e8d1262a0bb..b9d19e6f212 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -524,6 +524,19 @@ and native runs: - `run_exception_0` through `run_exception_3`: exception names, codes, HTTP statuses, and request IDs for a caught exception and up to three causes. +The orphan reaper records a bounded `processLossDiagnostic` before status writes +or cleanup. For `process_lost` failures, `run_execution` includes +`processLossPidRecorded`, `processLossGroupRecorded`, and `processLossLocalCheck` +(`not_observed_alive`, `not_checked`, or `no_identifiers`). It also includes +`processLossRunPredatesObserver`, `processLossObserverUptimeMs`, and +`processLossLastOutputAgeMs` when the timestamps are available and valid. Ages +above seven days are omitted. These are observations, not proof of an OOM, +provider failure, or deployment. The last-output age can include system output. +`processLossRetryEligible` records eligibility for the existing process-loss retry +at detection; it does not claim that a retry was queued or succeeded. This data +contains no process IDs, task text, paths, or credentials and changes no recovery +or ownership decisions. + ACP turns record `acpLastEventAgeMs`, `acpObservedEventCount`, `acpPendingToolCount`, and `acpToolInventoryComplete` at finalization, before usage reads, error logging, and cleanup. The age measures time since the last diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 86b3867ae50..325ffbc0a9b 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -2717,6 +2717,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { stopReason: "process_lost", timeoutConfigured: false, timeoutFired: false, + processLossDiagnostic: { + pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true, + observerUptimeMs: expect.any(Number), runPredatesObserver: true, + }, }); // The legacy engine writes this terminal status through the same // guarded emitter that reports a genuine failed transition to Sentry. @@ -2726,6 +2730,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { runId, errorCode: "process_lost", runStatus: "failed", + diagnostics: expect.objectContaining({ execution: expect.objectContaining({ + processLossPidRecorded: true, processLossGroupRecorded: false, processLossLocalCheck: "not_observed_alive", + processLossRetryEligible: true, processLossRunPredatesObserver: true, + }) }), }), ); const [action] = await db diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts index 5cac0081e31..f92e0a42bc0 100644 --- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts +++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts @@ -42,6 +42,49 @@ afterEach(async () => { }); describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", () => { + it("sends only bounded process-loss evidence without leaking it into another capture", async () => { + const Sentry = sentryPackage!; + const events: Array> = []; + vi.stubEnv("SENTRY_DSN_BACKEND", "https://public@example.invalid/1"); + vi.doMock("../peer-version-check.js", () => ({ checkExactPeerVersions: () => ({ ok: true }) })); + vi.doMock("@sentry/node", () => ({ + ...Sentry, + init: (options: Record) => Sentry.init({ + ...options, + transport: () => ({ send: async () => ({}), flush: async () => true }), + beforeSend: (event: Record) => { events.push(event); return event; }, + }), + })); + vi.resetModules(); + const { sentryReady, captureRunFailure, captureException } = await import("../sentry.js"); + await sentryReady; + const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({ + errorCode: "process_lost", resultJson: { processLossDiagnostic: { + pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true, + runPredatesObserver: true, observerUptimeMs: 30_000, lastOutputAgeMs: 120_000, + pid: "private-process-identity", path: "/private-process-path", prompt: "private-process-prompt", + } }, + } as unknown as typeof heartbeatRuns.$inferSelect, {})); + captureRunFailure({ + taskId: "11111111-1111-4111-8111-111111111111", runId: "22222222-2222-4222-8222-222222222222", + errorMessage: "Process lost -- server may have restarted", errorCode: "process_lost", + agentAdapter: "fixture-adapter", runStatus: "failed", exitCode: null, signal: null, diagnostics, + }); + captureException(new Error("unrelated process-loss fixture")); + await Sentry.flush(2000); + expect(events).toHaveLength(2); + const failure = events.find(event => (event.tags as Record)?.error_code === "process_lost"); + expect(failure).toMatchObject({ contexts: { run_execution: { + processLossPidRecorded: true, processLossGroupRecorded: false, processLossLocalCheck: "not_observed_alive", + processLossRetryEligible: true, processLossRunPredatesObserver: true, + processLossObserverUptimeMs: 30_000, processLossLastOutputAgeMs: 120_000, + } } }); + expect(JSON.stringify(events)).not.toContain("private-process-"); + const unrelated = events.find(event => event !== failure); + expect(unrelated).not.toHaveProperty("contexts.run_execution"); + expect(unrelated).not.toHaveProperty("contexts.run_failure"); + }); + it("keeps portfolio diagnostics bounded and isolated from unrelated captures", async () => { const Sentry = sentryPackage!; const events: Array> = []; diff --git a/server/src/services/__tests__/process-loss-diagnostics.test.ts b/server/src/services/__tests__/process-loss-diagnostics.test.ts new file mode 100644 index 00000000000..f8efe72f582 --- /dev/null +++ b/server/src/services/__tests__/process-loss-diagnostics.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { buildProcessLossDiagnostic, readProcessLossDiagnostic } from "../process-loss-diagnostics.js"; + +const nowMs = Date.UTC(2026, 0, 1, 12); +const run = { + startedAt: new Date(nowMs - 10 * 60_000), lastOutputAt: new Date(nowMs - 2 * 60_000), + processPid: 123, processGroupId: null, +}; +const input = { run, nowMs, observerStartedAtMs: nowMs - 30_000, + checksPersistedChildLiveness: true, retryEligible: true }; + +describe("orphan process-loss evidence", () => { + it("distinguishes a run from the previous observer without inventing a restart cause", () => { + expect(buildProcessLossDiagnostic(input)).toEqual({ + pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true, + runPredatesObserver: true, lastOutputAgeMs: 120_000, observerUptimeMs: 30_000, + }); + const currentObserverRun = buildProcessLossDiagnostic({ ...input, observerStartedAtMs: nowMs - 20 * 60_000 }); + expect(currentObserverRun.runPredatesObserver).toBe(false); + expect(currentObserverRun).not.toHaveProperty("restartCause"); + }); + + it("does not describe missing or unchecked identifiers as confirmed child death", () => { + expect(buildProcessLossDiagnostic({ ...input, checksPersistedChildLiveness: false }).localCheck).toBe("not_checked"); + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, processPid: null }, retryEligible: false })) + .toMatchObject({ pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers", retryEligible: false }); + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, processPid: null, processGroupId: 456 } })) + .toMatchObject({ pidRecorded: false, groupRecorded: true, localCheck: "not_observed_alive" }); + }); + + it("omits unavailable, future and unbounded ages", () => { + for (const lastOutputAt of [null, new Date(NaN), new Date(nowMs + 1), new Date(nowMs - 604_800_001)]) { + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, lastOutputAt } })).not.toHaveProperty("lastOutputAgeMs"); + } + for (const observerStartedAtMs of [NaN, Infinity, nowMs + 1]) { + const diagnostic = buildProcessLossDiagnostic({ ...input, observerStartedAtMs }); + expect(diagnostic).not.toHaveProperty("observerUptimeMs"); + expect(diagnostic).not.toHaveProperty("runPredatesObserver"); + } + expect(buildProcessLossDiagnostic({ ...input, run: { ...run, startedAt: new Date(nowMs + 1) } })) + .not.toHaveProperty("runPredatesObserver"); + expect(buildProcessLossDiagnostic({ ...input, observerStartedAtMs: nowMs - 1.5 }).observerUptimeMs).toBe(2); + }); + + it("reads only bounded scalar evidence, including hostile persisted input", () => { + const diagnostic = buildProcessLossDiagnostic(input); + expect(readProcessLossDiagnostic({ ...diagnostic, pid: 123, prompt: "private-prompt", path: "/private/path" })) + .toEqual(diagnostic); + for (const value of [null, [], "private", { localCheck: "private", pidRecorded: 1, + groupRecorded: "false", retryEligible: "private", runPredatesObserver: null, + observerUptimeMs: 604_800_001, lastOutputAgeMs: -1 }, + Object.defineProperty({}, "localCheck", { get() { throw new Error("private"); } })]) { + expect(readProcessLossDiagnostic(value)).toEqual({}); + } + }); +}); diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts index 6ee721b7a37..0cc4d104b7f 100644 --- a/server/src/services/__tests__/run-failure-diagnostics.test.ts +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -7,6 +7,19 @@ const run = (overrides: Partial = {}) => ({ resultJson: null, ...overrides const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error }); describe("run failure diagnostics", () => { + it("exports bounded orphan evidence only for a process-loss failure", () => { + const processLossDiagnostic = { pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers", + retryEligible: false, runPredatesObserver: true, observerUptimeMs: 30_000, lastOutputAgeMs: 120_000, + pid: 123, path: "/private-path", prompt: "private-prompt" }; + const resultJson = { processLossDiagnostic }; + expect(collectRunFailureDiagnostics(run({ errorCode: "process_lost", resultJson }), {}).execution).toEqual({ + processLossPidRecorded: false, processLossGroupRecorded: false, processLossLocalCheck: "no_identifiers", + processLossRetryEligible: false, processLossRunPredatesObserver: true, + processLossObserverUptimeMs: 30_000, processLossLastOutputAgeMs: 120_000, + }); + expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed", resultJson }), {}).execution).toEqual({}); + }); + it("exports only the closed native model/auth rejection vocabulary", () => { const diagnostic = { provider: "codex", category: "model_auth_incompatible", status: 400, authMode: "chatgpt" }; const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 22895243838..b475c28fd52 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -351,6 +351,8 @@ import { emitAgentTaskRunById, } from "./agent-task-run-telemetry.js"; import { reportRunFailure } from "./run-failure-report.js"; +import { performance } from "node:perf_hooks"; +import { buildProcessLossDiagnostic } from "./process-loss-diagnostics.js"; import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js"; import { companySkillService } from "./company-skills.js"; import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js"; @@ -19617,6 +19619,13 @@ export function heartbeatService( monitorDispatchLostWithoutFutureWake); if (!(await revokeExpiredLegacyController(db, run))) continue; const baseMessage = buildProcessLossMessage(run); + const processLossDiagnostic = buildProcessLossDiagnostic({ + run, + nowMs: Date.now(), + observerStartedAtMs: performance.timeOrigin, + checksPersistedChildLiveness, + retryEligible: shouldRetry, + }); const conversationContinuationEligible = await runUsedConversationAdapter(db, run); const failureWrite = await setRunStatusFromLive( @@ -19633,7 +19642,7 @@ export function heartbeatService( "failed", { conversationContinuationEligible, - resultJson: parseObject(run.resultJson), + resultJson: { ...parseObject(run.resultJson), processLossDiagnostic }, errorCode: "process_lost", errorMessage: shouldRetry ? `${baseMessage}; retrying once` diff --git a/server/src/services/process-loss-diagnostics.ts b/server/src/services/process-loss-diagnostics.ts new file mode 100644 index 00000000000..408c89d0970 --- /dev/null +++ b/server/src/services/process-loss-diagnostics.ts @@ -0,0 +1,71 @@ +import type { heartbeatRuns } from "@paperclipai/db"; + +type Run = Pick; +const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1_000; +const LOCAL_CHECKS = ["not_observed_alive", "not_checked", "no_identifiers"] as const; + +export interface ProcessLossDiagnostic { + pidRecorded: boolean; + groupRecorded: boolean; + localCheck: typeof LOCAL_CHECKS[number]; + retryEligible: boolean; + runPredatesObserver?: boolean; + lastOutputAgeMs?: number; + observerUptimeMs?: number; +} + +function age(now: number, then: number | undefined): number | undefined { + if (then === undefined) return undefined; + const elapsed = now - then; + return Number.isSafeInteger(elapsed) && elapsed >= 0 && elapsed <= MAX_AGE_MS + ? elapsed : undefined; +} + +/** Snapshot before status writes and cleanup. Never infer why ownership was lost. */ +export function buildProcessLossDiagnostic(input: { + run: Run; + nowMs: number; + observerStartedAtMs: number; + checksPersistedChildLiveness: boolean; + retryEligible: boolean; +}): ProcessLossDiagnostic { + const { run, nowMs, observerStartedAtMs } = input; + const pidRecorded = !!run.processPid; + const groupRecorded = !!run.processGroupId; + const startedAt = run.startedAt?.getTime(); + const validObserverStart = Number.isFinite(observerStartedAtMs) && observerStartedAtMs <= nowMs; + const lastOutputAgeMs = age(nowMs, run.lastOutputAt?.getTime()); + const observerUptimeMs = validObserverStart ? age(nowMs, Math.floor(observerStartedAtMs)) : undefined; + return { + pidRecorded, + groupRecorded, + localCheck: !pidRecorded && !groupRecorded ? "no_identifiers" + : input.checksPersistedChildLiveness ? "not_observed_alive" : "not_checked", + retryEligible: input.retryEligible, + ...(validObserverStart && startedAt !== undefined && Number.isFinite(startedAt) && startedAt <= nowMs + ? { runPredatesObserver: startedAt < observerStartedAtMs } : {}), + ...(lastOutputAgeMs !== undefined ? { lastOutputAgeMs } : {}), + ...(observerUptimeMs !== undefined ? { observerUptimeMs } : {}), + }; +} + +/** Persisted adapter data is untrusted. Only closed labels, booleans and ages leave the instance. */ +export function readProcessLossDiagnostic(value: unknown): Partial { + const result: Partial = {}; + if (!value || typeof value !== "object" || Array.isArray(value)) return result; + const read = (key: string): unknown => { + try { return (value as Record)[key]; } catch { return undefined; } + }; + for (const key of ["pidRecorded", "groupRecorded", "retryEligible", "runPredatesObserver"] as const) { + const entry = read(key); + if (typeof entry === "boolean") result[key] = entry; + } + const localCheck = read("localCheck"); + if (LOCAL_CHECKS.some(check => check === localCheck)) result.localCheck = localCheck as ProcessLossDiagnostic["localCheck"]; + for (const key of ["lastOutputAgeMs", "observerUptimeMs"] as const) { + const entry = read(key); + if (typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0 && entry <= MAX_AGE_MS) result[key] = entry; + } + return result; +} diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index c5c9d7e820b..8dc0d78a289 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -1,5 +1,6 @@ import type { heartbeatRuns } from "@paperclipai/db"; import { readRunCancellation } from "./run-cancellation.js"; +import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js"; import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared"; import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction"; import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics"; @@ -146,6 +147,12 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs; } const result = run.resultJson; + if (run.errorCode === "process_lost") { + const diagnostic = readProcessLossDiagnostic(read(result, "processLossDiagnostic")); + for (const [field, value] of Object.entries(diagnostic)) { + execution[`processLoss${field[0]!.toUpperCase()}${field.slice(1)}`] = value; + } + } const cancellation = readRunCancellation(result); if (cancellation) { execution.cancellationSource = cancellation.source;