Retain bounded orphan process-loss diagnostics (#15475)

Record bounded observer uptime, run and output ages, process-check observations and retry eligibility before orphan cleanup changes the evidence. Preserve existing recovery and reporting behavior and omit process IDs, raw paths and credentials.

Verified focused helper, actual reaper and real SDK regressions, full typecheck/build, exact-head CI and independent review. An unchanged Cursor timeout passed its isolated retry; unrelated local Slack timing failure is documented separately. Greptile 5/5 with no unresolved comments.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-10-07 12:05:21 -07:00
1 parent 43b0aff54b
commit 128c95837b
8 files changed
+221 -1

No files matched your search

+13
View File
@@ -524,6 +524,19 @@ and native runs:
- `run_exception_0` through `run_exception_3`: exception names, codes, HTTP
statuses, and request IDs for a caught exception and up to three causes.
The orphan reaper records a bounded `processLossDiagnostic` before status writes
or cleanup. For `process_lost` failures, `run_execution` includes
`processLossPidRecorded`, `processLossGroupRecorded`, and `processLossLocalCheck`
(`not_observed_alive`, `not_checked`, or `no_identifiers`). It also includes
`processLossRunPredatesObserver`, `processLossObserverUptimeMs`, and
`processLossLastOutputAgeMs` when the timestamps are available and valid. Ages
above seven days are omitted. These are observations, not proof of an OOM,
provider failure, or deployment. The last-output age can include system output.
`processLossRetryEligible` records eligibility for the existing process-loss retry
at detection; it does not claim that a retry was queued or succeeded. This data
contains no process IDs, task text, paths, or credentials and changes no recovery
or ownership decisions.
ACP turns record `acpLastEventAgeMs`, `acpObservedEventCount`,
`acpPendingToolCount`, and `acpToolInventoryComplete` at finalization, before
usage reads, error logging, and cleanup. The age measures time since the last
@@ -2717,6 +2717,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
stopReason: "process_lost",
timeoutConfigured: false,
timeoutFired: false,
processLossDiagnostic: {
pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true,
observerUptimeMs: expect.any(Number), runPredatesObserver: true,
},
});
// The legacy engine writes this terminal status through the same
// guarded emitter that reports a genuine failed transition to Sentry.
@@ -2726,6 +2730,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
runId,
errorCode: "process_lost",
runStatus: "failed",
diagnostics: expect.objectContaining({ execution: expect.objectContaining({
processLossPidRecorded: true, processLossGroupRecorded: false, processLossLocalCheck: "not_observed_alive",
processLossRetryEligible: true, processLossRunPredatesObserver: true,
}) }),
}),
);
const [action] = await db
@@ -42,6 +42,49 @@ afterEach(async () => {
});
describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", () => {
it("sends only bounded process-loss evidence without leaking it into another capture", async () => {
const Sentry = sentryPackage!;
const events: Array<Record<string, unknown>> = [];
vi.stubEnv("SENTRY_DSN_BACKEND", "https://public@example.invalid/1");
vi.doMock("../peer-version-check.js", () => ({ checkExactPeerVersions: () => ({ ok: true }) }));
vi.doMock("@sentry/node", () => ({
...Sentry,
init: (options: Record<string, unknown>) => Sentry.init({
...options,
transport: () => ({ send: async () => ({}), flush: async () => true }),
beforeSend: (event: Record<string, unknown>) => { events.push(event); return event; },
}),
}));
vi.resetModules();
const { sentryReady, captureRunFailure, captureException } = await import("../sentry.js");
await sentryReady;
const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({
errorCode: "process_lost", resultJson: { processLossDiagnostic: {
pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true,
runPredatesObserver: true, observerUptimeMs: 30_000, lastOutputAgeMs: 120_000,
pid: "private-process-identity", path: "/private-process-path", prompt: "private-process-prompt",
} },
} as unknown as typeof heartbeatRuns.$inferSelect, {}));
captureRunFailure({
taskId: "11111111-1111-4111-8111-111111111111", runId: "22222222-2222-4222-8222-222222222222",
errorMessage: "Process lost -- server may have restarted", errorCode: "process_lost",
agentAdapter: "fixture-adapter", runStatus: "failed", exitCode: null, signal: null, diagnostics,
});
captureException(new Error("unrelated process-loss fixture"));
await Sentry.flush(2000);
expect(events).toHaveLength(2);
const failure = events.find(event => (event.tags as Record<string, unknown>)?.error_code === "process_lost");
expect(failure).toMatchObject({ contexts: { run_execution: {
processLossPidRecorded: true, processLossGroupRecorded: false, processLossLocalCheck: "not_observed_alive",
processLossRetryEligible: true, processLossRunPredatesObserver: true,
processLossObserverUptimeMs: 30_000, processLossLastOutputAgeMs: 120_000,
} } });
expect(JSON.stringify(events)).not.toContain("private-process-");
const unrelated = events.find(event => event !== failure);
expect(unrelated).not.toHaveProperty("contexts.run_execution");
expect(unrelated).not.toHaveProperty("contexts.run_failure");
});
it("keeps portfolio diagnostics bounded and isolated from unrelated captures", async () => {
const Sentry = sentryPackage!;
const events: Array<Record<string, unknown>> = [];
@@ -0,0 +1,56 @@
import { describe, expect, it } from "vitest";
import { buildProcessLossDiagnostic, readProcessLossDiagnostic } from "../process-loss-diagnostics.js";
const nowMs = Date.UTC(2026, 0, 1, 12);
const run = {
startedAt: new Date(nowMs - 10 * 60_000), lastOutputAt: new Date(nowMs - 2 * 60_000),
processPid: 123, processGroupId: null,
};
const input = { run, nowMs, observerStartedAtMs: nowMs - 30_000,
checksPersistedChildLiveness: true, retryEligible: true };
describe("orphan process-loss evidence", () => {
it("distinguishes a run from the previous observer without inventing a restart cause", () => {
expect(buildProcessLossDiagnostic(input)).toEqual({
pidRecorded: true, groupRecorded: false, localCheck: "not_observed_alive", retryEligible: true,
runPredatesObserver: true, lastOutputAgeMs: 120_000, observerUptimeMs: 30_000,
});
const currentObserverRun = buildProcessLossDiagnostic({ ...input, observerStartedAtMs: nowMs - 20 * 60_000 });
expect(currentObserverRun.runPredatesObserver).toBe(false);
expect(currentObserverRun).not.toHaveProperty("restartCause");
});
it("does not describe missing or unchecked identifiers as confirmed child death", () => {
expect(buildProcessLossDiagnostic({ ...input, checksPersistedChildLiveness: false }).localCheck).toBe("not_checked");
expect(buildProcessLossDiagnostic({ ...input, run: { ...run, processPid: null }, retryEligible: false }))
.toMatchObject({ pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers", retryEligible: false });
expect(buildProcessLossDiagnostic({ ...input, run: { ...run, processPid: null, processGroupId: 456 } }))
.toMatchObject({ pidRecorded: false, groupRecorded: true, localCheck: "not_observed_alive" });
});
it("omits unavailable, future and unbounded ages", () => {
for (const lastOutputAt of [null, new Date(NaN), new Date(nowMs + 1), new Date(nowMs - 604_800_001)]) {
expect(buildProcessLossDiagnostic({ ...input, run: { ...run, lastOutputAt } })).not.toHaveProperty("lastOutputAgeMs");
}
for (const observerStartedAtMs of [NaN, Infinity, nowMs + 1]) {
const diagnostic = buildProcessLossDiagnostic({ ...input, observerStartedAtMs });
expect(diagnostic).not.toHaveProperty("observerUptimeMs");
expect(diagnostic).not.toHaveProperty("runPredatesObserver");
}
expect(buildProcessLossDiagnostic({ ...input, run: { ...run, startedAt: new Date(nowMs + 1) } }))
.not.toHaveProperty("runPredatesObserver");
expect(buildProcessLossDiagnostic({ ...input, observerStartedAtMs: nowMs - 1.5 }).observerUptimeMs).toBe(2);
});
it("reads only bounded scalar evidence, including hostile persisted input", () => {
const diagnostic = buildProcessLossDiagnostic(input);
expect(readProcessLossDiagnostic({ ...diagnostic, pid: 123, prompt: "private-prompt", path: "/private/path" }))
.toEqual(diagnostic);
for (const value of [null, [], "private", { localCheck: "private", pidRecorded: 1,
groupRecorded: "false", retryEligible: "private", runPredatesObserver: null,
observerUptimeMs: 604_800_001, lastOutputAgeMs: -1 },
Object.defineProperty({}, "localCheck", { get() { throw new Error("private"); } })]) {
expect(readProcessLossDiagnostic(value)).toEqual({});
}
});
});
@@ -7,6 +7,19 @@ const run = (overrides: Partial<Run> = {}) => ({ resultJson: null, ...overrides
const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error });
describe("run failure diagnostics", () => {
it("exports bounded orphan evidence only for a process-loss failure", () => {
const processLossDiagnostic = { pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers",
retryEligible: false, runPredatesObserver: true, observerUptimeMs: 30_000, lastOutputAgeMs: 120_000,
pid: 123, path: "/private-path", prompt: "private-prompt" };
const resultJson = { processLossDiagnostic };
expect(collectRunFailureDiagnostics(run({ errorCode: "process_lost", resultJson }), {}).execution).toEqual({
processLossPidRecorded: false, processLossGroupRecorded: false, processLossLocalCheck: "no_identifiers",
processLossRetryEligible: false, processLossRunPredatesObserver: true,
processLossObserverUptimeMs: 30_000, processLossLastOutputAgeMs: 120_000,
});
expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed", resultJson }), {}).execution).toEqual({});
});
it("exports only the closed native model/auth rejection vocabulary", () => {
const diagnostic = { provider: "codex", category: "model_auth_incompatible", status: 400, authMode: "chatgpt" };
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: {
+10 -1
View File
@@ -351,6 +351,8 @@ import {
emitAgentTaskRunById,
} from "./agent-task-run-telemetry.js";
import { reportRunFailure } from "./run-failure-report.js";
import { performance } from "node:perf_hooks";
import { buildProcessLossDiagnostic } from "./process-loss-diagnostics.js";
import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";
import { companySkillService } from "./company-skills.js";
import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js";
@@ -19617,6 +19619,13 @@ export function heartbeatService(
monitorDispatchLostWithoutFutureWake);
if (!(await revokeExpiredLegacyController(db, run))) continue;
const baseMessage = buildProcessLossMessage(run);
const processLossDiagnostic = buildProcessLossDiagnostic({
run,
nowMs: Date.now(),
observerStartedAtMs: performance.timeOrigin,
checksPersistedChildLiveness,
retryEligible: shouldRetry,
});
const conversationContinuationEligible = await runUsedConversationAdapter(db, run);
const failureWrite = await setRunStatusFromLive(
@@ -19633,7 +19642,7 @@ export function heartbeatService(
"failed",
{
conversationContinuationEligible,
resultJson: parseObject(run.resultJson),
resultJson: { ...parseObject(run.resultJson), processLossDiagnostic },
errorCode: "process_lost",
errorMessage: shouldRetry
? `${baseMessage}; retrying once`
@@ -0,0 +1,71 @@
import type { heartbeatRuns } from "@paperclipai/db";
type Run = Pick<typeof heartbeatRuns.$inferSelect,
"startedAt" | "lastOutputAt" | "processPid" | "processGroupId">;
const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1_000;
const LOCAL_CHECKS = ["not_observed_alive", "not_checked", "no_identifiers"] as const;
export interface ProcessLossDiagnostic {
pidRecorded: boolean;
groupRecorded: boolean;
localCheck: typeof LOCAL_CHECKS[number];
retryEligible: boolean;
runPredatesObserver?: boolean;
lastOutputAgeMs?: number;
observerUptimeMs?: number;
}
function age(now: number, then: number | undefined): number | undefined {
if (then === undefined) return undefined;
const elapsed = now - then;
return Number.isSafeInteger(elapsed) && elapsed >= 0 && elapsed <= MAX_AGE_MS
? elapsed : undefined;
}
/** Snapshot before status writes and cleanup. Never infer why ownership was lost. */
export function buildProcessLossDiagnostic(input: {
run: Run;
nowMs: number;
observerStartedAtMs: number;
checksPersistedChildLiveness: boolean;
retryEligible: boolean;
}): ProcessLossDiagnostic {
const { run, nowMs, observerStartedAtMs } = input;
const pidRecorded = !!run.processPid;
const groupRecorded = !!run.processGroupId;
const startedAt = run.startedAt?.getTime();
const validObserverStart = Number.isFinite(observerStartedAtMs) && observerStartedAtMs <= nowMs;
const lastOutputAgeMs = age(nowMs, run.lastOutputAt?.getTime());
const observerUptimeMs = validObserverStart ? age(nowMs, Math.floor(observerStartedAtMs)) : undefined;
return {
pidRecorded,
groupRecorded,
localCheck: !pidRecorded && !groupRecorded ? "no_identifiers"
: input.checksPersistedChildLiveness ? "not_observed_alive" : "not_checked",
retryEligible: input.retryEligible,
...(validObserverStart && startedAt !== undefined && Number.isFinite(startedAt) && startedAt <= nowMs
? { runPredatesObserver: startedAt < observerStartedAtMs } : {}),
...(lastOutputAgeMs !== undefined ? { lastOutputAgeMs } : {}),
...(observerUptimeMs !== undefined ? { observerUptimeMs } : {}),
};
}
/** Persisted adapter data is untrusted. Only closed labels, booleans and ages leave the instance. */
export function readProcessLossDiagnostic(value: unknown): Partial<ProcessLossDiagnostic> {
const result: Partial<ProcessLossDiagnostic> = {};
if (!value || typeof value !== "object" || Array.isArray(value)) return result;
const read = (key: string): unknown => {
try { return (value as Record<string, unknown>)[key]; } catch { return undefined; }
};
for (const key of ["pidRecorded", "groupRecorded", "retryEligible", "runPredatesObserver"] as const) {
const entry = read(key);
if (typeof entry === "boolean") result[key] = entry;
}
const localCheck = read("localCheck");
if (LOCAL_CHECKS.some(check => check === localCheck)) result.localCheck = localCheck as ProcessLossDiagnostic["localCheck"];
for (const key of ["lastOutputAgeMs", "observerUptimeMs"] as const) {
const entry = read(key);
if (typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0 && entry <= MAX_AGE_MS) result[key] = entry;
}
return result;
}
@@ -1,5 +1,6 @@
import type { heartbeatRuns } from "@paperclipai/db";
import { readRunCancellation } from "./run-cancellation.js";
import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js";
import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared";
import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction";
import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics";
@@ -146,6 +147,12 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;
}
const result = run.resultJson;
if (run.errorCode === "process_lost") {
const diagnostic = readProcessLossDiagnostic(read(result, "processLossDiagnostic"));
for (const [field, value] of Object.entries(diagnostic)) {
execution[`processLoss${field[0]!.toUpperCase()}${field.slice(1)}`] = value;
}
}
const cancellation = readRunCancellation(result);
if (cancellation) {
execution.cancellationSource = cancellation.source;