PaperClipAI/server/src/sentry.ts

// Optional Sentry error monitoring for the server process.
//
// Activated only when the backend DSN resolves to a value — see
// `resolveSentryDsns` in `sentry-dsn.ts` for the precedence between
// `SENTRY_DSN_BACKEND` and the legacy `SENTRY_DSN` fallback. When it
// resolves to `null`, no Sentry package is loaded at all.
//
// The import is dynamic and the package is an optional runtime dependency —
// operators who want server-side error monitoring install `@sentry/node`
// themselves. That keeps Sentry off the default dependency graph and avoids
// forcing a lockfile bump for an opt-in feature. This gate mirrors the
// OpenTelemetry gate in `instrumentation.ts`.
//
// OpenTelemetry keeps ownership of trace setup: the initializer passes
// `skipOpenTelemetrySetup: true` and `tracesSampleRate: 0`, so this module
// adds error monitoring only and starts no span or trace behavior of its
// own.
//
// `serverName`: the initializer sets this to the host name of the process,
// with `os.hostname()`. The `@sentry/node` client already falls back to the
// same host name when the caller omits this option, so this line makes an
// existing default explicit instead of changing captured event content. An
// explicit value stays correct if a future SDK version changes its default.
//
// An operator can still send a different value in place of the host name.
// When the environment variable `SENTRY_NAME` holds a non-empty string, the
// initializer uses that value instead. This keeps the same order the
// `@sentry/node` client itself uses when the caller omits `serverName`.
//
// Default-integration privacy note: `sendDefaultPii: false` filters values
// by name, inside the `RequestData` integration only. Three other default
// integrations copy raw values past that filter, so the initializer removes
// or narrows them with built-in Sentry options — no custom filter code:
//   - `Console` turns a `console.*` call into a breadcrumb with the raw
//     arguments. The initializer drops it.
//   - `ContextLines` reads local source lines around each stack frame off
//     the host disk. The initializer drops it.
//   - `Http` records a breadcrumb for each outbound request, with its URL
//     and query string. The initializer keeps the integration (`RequestData`
//     and request isolation need it) and turns the breadcrumb off with the
//     integration's own `breadcrumbs` option.
//
// A failed secret-sensitive request (chat setup, provider webhooks, MCP,
// company observability settings, and similar routes) reports only the
// generic `Secret-sensitive request failed` error, with its own stack and
// default grouping. `SecretSensitiveRequestError` adds `secret_route_family`,
// `http_method`, `http_status` and `error_kind` tags, optional `errno_code`
// and `db_error_code` tags (a Node errno name or a SQLSTATE), and a
// `secret_sensitive_request` context. The context holds the matched Express
// route definition and up to eight complete V8 frames from the original
// stack. Every value is a closed label, a bounded number, or a code-shaped
// string; anything else is dropped. The event never carries the request
// URL, query, body, headers, or the original error name or message.
//
// `onUnhandledRejectionIntegration` defaults to `mode: "warn"`, which
// registers a `process.on("unhandledRejection")` listener. Node cancels its
// own crash-on-unhandled-rejection behavior when any listener is registered.
// The server relies on that crash today, so the initializer passes
// `mode: "strict"`: Sentry still captures the event, then exits the process,
// so the existing crash-and-restart behavior stays.
//
// `beforeSend` adds run navigation links, then bounded provenance for the
// dynamic-import-callback error — see `sentry-dynamic-import-provenance.ts`.
//
// Before it imports the package, the bootstrap checks the installed
// `@sentry/node` version against the exact version this manifest's
// `peerDependencies` declares — the same audited version documented in
// `doc/observability.md`. A missing or a mismatched version logs one
// diagnostic and leaves the server running without error monitoring; it
// never throws. This gate mirrors the OpenTelemetry gate in
// `instrumentation.ts`.

import os from "node:os";
import type { HeartbeatRunStatus } from "@paperclipai/shared";
import { AdapterStopTimeoutError } from "./services/adapter-stop-timeout.js";
import { CloudPortfolioError } from "./services/cloud-portfolio-error.js";
import { SecretSensitiveRequestError } from "./middleware/secret-sensitive-request-error.js";
import { storageFailureDiagnosticsFor } from "./storage/failure-diagnostics.js";
import { directoryMergeLockTimeoutDiagnosticsFor } from "@paperclipai/adapter-utils/directory-merge-lock-diagnostics";
import type { RunFailureDiagnostics } from "./services/run-failure-diagnostics.js";
import { readBuildCommit } from "./build-commit.js";
import { checkExactPeerVersions } from "./peer-version-check.js";
import { resolveSentryDsns } from "./sentry-dsn.js";
import { readRunFailureReportingMode } from "./sentry-run-failure-policy.js";
import { addSentryRunLinks, type SentryRunReference } from "./sentry-links.js";
import { addDynamicImportProvenance, type ProvenanceHint } from "./sentry-dynamic-import-provenance.js";

const { backend: dsn, legacyFallbackUsed } = resolveSentryDsns();

if (legacyFallbackUsed) {
  // eslint-disable-next-line no-console
  console.warn(
    "[paperclip] SENTRY_DSN_FRONTEND or SENTRY_DSN_BACKEND is not set. " +
      "The server uses the legacy SENTRY_DSN value for the affected " +
      "component. Set SENTRY_DSN_FRONTEND and SENTRY_DSN_BACKEND to send " +
      "each component to its own Sentry project.",
  );
}

/** Event-local context accepted by the optional Sentry package. */
interface SentryCaptureContext {
  tags: Record<string, string>;
  contexts: Record<string, Record<string, unknown>>;
  fingerprint: string[];
}

/** The subset of the `@sentry/node` client surface this gate calls. */
interface SentryHandle {
  captureException(error: unknown, context?: SentryCaptureContext): string;
  close(timeout?: number): Promise<boolean>;
}

let sentryHandle: SentryHandle | null = null;
let shutdownPromise: Promise<void> | null = null;

/**
 * Resolves once the Sentry SDK has started, or once bootstrap has failed and
 * logged, or at once when the backend DSN resolves to `null`. No caller
 * needs to await this before calling `captureException` — it is a no-op
 * until ready — but `index.ts` awaits it at startup so the first real error
 * has a live client.
 */
export const sentryReady: Promise<void> = dsn ? bootstrapSentry(dsn) : Promise.resolve();

/**
 * Report an error to Sentry. A no-op before the gate opens, when the gate
 * never opens (the backend DSN resolves to `null`), or when bootstrap
 * failed. Never throws — observability must not change control flow.
 */
export function captureException(error: unknown, reference?: SentryRunReference): void {
  if (!sentryHandle) return;
  try {
    const tags = reference ? {
      run_id: reference.runId,
      ...(reference.companyId ? { company_id: reference.companyId } : {}),
      ...(reference.agentId ? { agent_id: reference.agentId } : {}),
    } : {};
    const storageFailure = storageFailureDiagnosticsFor(error);
    const lockTimeout = directoryMergeLockTimeoutDiagnosticsFor(error);
    if (error instanceof AdapterStopTimeoutError) {
      // Event-local, fixed-shape context: no ambient scope or raw error fields.
      const exception = new Error(error.message);
      exception.stack = error.stack;
      sentryHandle.captureException(exception, {
        tags: { ...tags, error_code: "adapter_stop_unconfirmed" },
        contexts: { adapter_stop: { ...error.diagnostics } },
        fingerprint: ["{{ default }}"],
      });
    } else if (error instanceof CloudPortfolioError) {
      const exception = new Error(error.message);
      exception.stack = error.stack;
      sentryHandle.captureException(exception, {
        tags: { ...tags, error_code: "cloud_portfolio_failure" },
        contexts: { cloud_portfolio: { ...CloudPortfolioError.diagnosticsFor(error) } },
        fingerprint: ["{{ default }}"],
      });
    } else if (error instanceof SecretSensitiveRequestError) {
      // The sanitized error keeps its generic message, stack and default
      // grouping. Only its fixed-shape diagnostics are added.
      const diagnostics = error.diagnostics;
      sentryHandle.captureException(error, {
        tags: {
          ...tags,
          secret_route_family: diagnostics.routeFamily,
          http_method: diagnostics.method,
          http_status: String(diagnostics.status),
          error_kind: diagnostics.errorKind,
          ...(diagnostics.errnoCode ? { errno_code: diagnostics.errnoCode } : {}),
          ...(diagnostics.databaseErrorCode ? { db_error_code: diagnostics.databaseErrorCode } : {}),
        },
        contexts: { secret_sensitive_request: { ...diagnostics, originFrames: [...diagnostics.originFrames] } },
        fingerprint: ["{{ default }}"],
      });
    } else if (storageFailure) {
      // The original error keeps its default grouping; only the normalized
      // record is added, never the provider's request or response.
      sentryHandle.captureException(error, {
        tags: { ...tags, storage_operation: storageFailure.operation, storage_error_code: storageFailure.errorCode },
        contexts: { storage_failure: { ...storageFailure } },
        fingerprint: ["{{ default }}"],
      });
    } else if (lockTimeout) {
      // Holder evidence distinguishes contention from a stuck owner. The
      // allowlisted record omits the owner's process id.
      sentryHandle.captureException(error, {
        tags: { ...tags, error_code: "workspace_restore_lock_timeout" },
        contexts: { directory_merge_lock: { ...lockTimeout } },
        fingerprint: ["{{ default }}"],
      });
    } else if (reference) {
      sentryHandle.captureException(error, { tags, contexts: {}, fingerprint: ["{{ default }}"] });
    } else {
      sentryHandle.captureException(error);
    }
  } catch (err) {
    // eslint-disable-next-line no-console
    console.error("[paperclip] Sentry captureException failed", err);
  }
}

/** The run status values that mark a run as a genuine terminal failure. */
export type RunFailureStatus = "failed" | "timed_out" | "cancelled";

/**
 * The diagnostic values `captureRunFailure` sends with a terminal-failure
 * event. `errorCode` is `null` when the run holds no error code.
 */
export interface RunFailureEvent {
  taskId: string | null;
  companyId?: string;
  agentId?: string;
  /** The `heartbeat_runs` row id. */
  runId: string;
  /** The redacted error message. */
  errorMessage: string;
  /** The run's error code, or `null` when the run holds none. */
  errorCode: string | null;
  /** The agent's adapter type, or `"unknown"` when the agent row is absent. */
  agentAdapter: string;
  /** The run status that triggered this report. */
  runStatus: HeartbeatRunStatus;
  failureSource?: "task_interruption";
  /** Bounded, redacted diagnostics selected by the run failure reporter. */
  diagnostics?: RunFailureDiagnostics;
  /** Recorded process exit evidence, when available. Validated before capture. */
  exitCode?: number | null;
  signal?: string | null;
}

function normalizeRunExitCode(value: unknown): number | null {
  // Match the persisted PostgreSQL integer. Do not coerce adapter-supplied text.
  return typeof value === "number" &&
    Number.isInteger(value) &&
    value >= -2147483648 &&
    value <= 2147483647
    ? value
    : null;
}

function normalizeRunSignal(value: unknown): string | null {
  if (value == null) return null;
  // Only host signal constants may leave the process; arbitrary adapter text
  // can contain output or credentials even when stored in the signal column.
  return typeof value === "string" && Object.hasOwn(os.constants.signals, value)
    ? value
    : "unknown";
}

/**
 * Report one terminal run failure to Sentry. A no-op before the gate opens
 * or when the gate never opens. Never throws — observability must not
 * change run control flow.
 *
 */
export function captureRunFailure(event: RunFailureEvent): boolean {
  return captureRunFailureEvent(event) !== null;
}

function captureRunFailureEvent(event: RunFailureEvent): string | null {
  if (!sentryHandle || readRunFailureReportingMode() === "off") return null;
  const handle = sentryHandle;
  try {
    const errorCode = event.errorCode ?? "unknown";
    // Sentry's async scope isolation is absent when OTel setup is skipped.
    // A withScope mutation can then persist into unrelated later captures.
    // Pass these fields on this event only; do not mutate the ambient scope.
    const diagnostics = event.diagnostics;
    const rootCodeCandidate = diagnostics?.execution.originalFailureCode ??
      diagnostics?.exceptions.at(-1)?.code;
    const rootErrorCode = typeof rootCodeCandidate === "string" && /^[a-zA-Z][a-zA-Z0-9_]{1,99}$/.test(rootCodeCandidate)
      ? rootCodeCandidate : undefined;
    // Without a usable root code, unrelated failures that share the error code,
    // adapter and phase would collapse into one group. Fall back to the root
    // exception's class name plus its HTTP status or numeric code. These are
    // bounded values; the message stays out of the fingerprint.
    const rootFallback: string[] = [];
    const rootException = diagnostics?.exceptions.at(-1);
    if (!rootErrorCode && rootException) {
      if (typeof rootException.name === "string" && /^[A-Za-z_$][A-Za-z0-9_$]{0,99}$/.test(rootException.name)) {
        rootFallback.push(`exception:${rootException.name}`);
      }
      if (rootException.status !== undefined) rootFallback.push(`status:${rootException.status}`);
      else if (typeof rootException.code === "string" && /^-?[0-9]{1,10}$/.test(rootException.code)) {
        rootFallback.push(`code:${rootException.code}`);
      }
    }
    const phase = diagnostics?.execution.failurePhase;
    const failurePhase = phase === "setup" || phase === "execute" ? phase : undefined;
    const contexts: Record<string, Record<string, unknown>> = {};
    if (event.companyId && event.agentId) {
      const runPath = `/api/heartbeat-runs/${encodeURIComponent(event.runId)}`;
      contexts.paperclip = {
        schemaVersion: 1,
        companyId: event.companyId,
        agentId: event.agentId,
        runPath,
        eventsPath: `${runPath}/events`,
        logPath: `${runPath}/log`,
        ...(event.taskId ? { taskPath: `/api/issues/${encodeURIComponent(event.taskId)}` } : {}),
      };
      try {
        const base = new URL(process.env.PAPERCLIP_PUBLIC_URL ?? "");
        if ((base.protocol === "https:" || base.protocol === "http:") &&
            !base.username && !base.password && !base.search && !base.hash && base.pathname === "/") {
          contexts.paperclip.runUrl = new URL(runPath, base).href;
          contexts.paperclip.eventsUrl = new URL(`${runPath}/events`, base).href;
          contexts.paperclip.logUrl = new URL(`${runPath}/log`, base).href;
        }
      } catch {}
    }
    if (diagnostics) {
      contexts.run_execution = { ...diagnostics.execution, truncatedFields: diagnostics.truncatedFields };
      if (Object.keys(diagnostics.adapter).length) contexts.adapter_failure = diagnostics.adapter;
      if (Object.keys(diagnostics.provider).length) contexts.provider_failure = diagnostics.provider;
      diagnostics.exceptions.forEach(({ name, code, status, requestId }, index) => {
        contexts[`run_exception_${index}`] = { name, code, status, requestId };
      });
    }
    // Rebuild only sanitized exception fields. Passing a raw SDK Error can
    // serialize its request/response, headers, or other enumerable properties.
    let cause: Error | undefined;
    for (const entry of [...(diagnostics?.exceptions ?? [])].reverse()) {
      const error: Error = new Error(entry.message ?? event.errorMessage, cause ? { cause } : undefined);
      error.name = entry.name ?? "Error";
      error.stack = entry.stack;
      cause = error;
    }
    const exception = cause ?? new Error(event.errorMessage);
    if (!cause) {
      // A saved adapter result is not a thrown Error. Do not pretend that the
      // reporter's own stack is the failure location.
      exception.stack = typeof diagnostics?.adapter.stackPreview === "string"
        ? diagnostics.adapter.stackPreview : undefined;
    }
    return handle.captureException(exception, {
      tags: {
        event_kind: "run_failure",
        ...(event.failureSource ? { failure_source: event.failureSource } : {}),
        ...(rootErrorCode ? { root_error_code: rootErrorCode } : {}),
        ...(failurePhase ? { failure_phase: failurePhase } : {}),
        run_id: event.runId,
        ...(event.taskId ? { task_id: event.taskId } : {}),
        ...(event.companyId ? { company_id: event.companyId } : {}),
        ...(event.agentId ? { agent_id: event.agentId } : {}),
        error_code: errorCode,
        agent_adapter: event.agentAdapter,
        run_status: event.runStatus,
      },
      contexts: {
        ...contexts,
        run_failure: {
          taskId: event.taskId,
          runId: event.runId,
          errorMessage: event.errorMessage,
          errorCode,
          agentAdapter: event.agentAdapter,
          exitCode: normalizeRunExitCode(event.exitCode),
          signal: normalizeRunSignal(event.signal),
        },
      },
      fingerprint: [errorCode, event.agentAdapter, ...(failurePhase ? [failurePhase] : []), ...(rootErrorCode ? [rootErrorCode] : rootFallback)],
    });
  } catch (err) {
    // eslint-disable-next-line no-console
    console.error("[paperclip] Sentry captureRunFailure failed", err);
    return null;
  }
}

const reportedTaskInterruptions = new Set<string>();
const MAX_REPORTED_TASK_INTERRUPTIONS = 1_000;
const TASK_INTERRUPTION_DELIVERY_RETRY_MS = 60_000;
const pendingTaskInterruptions = new Map<string, { eventId: string; retryAt: number }>();
const pendingTaskInterruptionEvents = new Map<string, string>();

function forgetPendingTaskInterruption(key: string) {
  const pending = pendingTaskInterruptions.get(key);
  if (pending && pendingTaskInterruptionEvents.get(pending.eventId) === key) {
    pendingTaskInterruptionEvents.delete(pending.eventId);
  }
  pendingTaskInterruptions.delete(key);
}

function confirmTaskInterruptionDelivery(eventId: string | undefined, statusCode: number | undefined) {
  if (!eventId || statusCode === undefined || statusCode < 200 || statusCode >= 300) return;
  const key = pendingTaskInterruptionEvents.get(eventId);
  if (!key) return;
  forgetPendingTaskInterruption(key);
  reportedTaskInterruptions.add(key);
  if (reportedTaskInterruptions.size > MAX_REPORTED_TASK_INTERRUPTIONS) {
    reportedTaskInterruptions.delete(reportedTaskInterruptions.values().next().value!);
  }
}

export function captureTaskInterruptionFailure(event: Pick<RunFailureEvent,
  "companyId" | "agentId" | "taskId" | "runId" | "runStatus" | "agentAdapter"> & { intentId: string | null }): void {
  const key = JSON.stringify([event.companyId, event.taskId, event.runId, event.intentId]);
  if (reportedTaskInterruptions.has(key)) return;
  const now = Date.now();
  if ((pendingTaskInterruptions.get(key)?.retryAt ?? 0) > now) return;
  for (const [pendingKey, pending] of pendingTaskInterruptions) {
    if (pending.retryAt <= now) forgetPendingTaskInterruption(pendingKey);
  }
  const eventId = captureRunFailureEvent({
    companyId: event.companyId,
    agentId: event.agentId,
    taskId: event.taskId,
    runId: event.runId,
    runStatus: event.runStatus,
    agentAdapter: event.agentAdapter,
    errorMessage: "Couldn’t finish this attempt. Your message is saved.",
    errorCode: "task_attempt_failed",
    failureSource: "task_interruption",
  });
  if (!eventId) return;
  pendingTaskInterruptions.set(key, { eventId, retryAt: now + TASK_INTERRUPTION_DELIVERY_RETRY_MS });
  pendingTaskInterruptionEvents.set(eventId, key);
  if (pendingTaskInterruptions.size > MAX_REPORTED_TASK_INTERRUPTIONS) {
    forgetPendingTaskInterruption(pendingTaskInterruptions.keys().next().value!);
  }
}

/**
 * Flush buffered events and close the Sentry client. Idempotent — concurrent
 * callers share one shutdown. A no-op when monitoring is off or bootstrap
 * failed.
 */
export function shutdownSentry(): Promise<void> {
  shutdownPromise ??= (async () => {
    await sentryReady;
    if (!sentryHandle) return;
    try {
      // Awaiting matters: the client flushes buffered events to Sentry
      // during close; exiting before it settles silently drops them.
      await sentryHandle.close(5_000);
    } catch (err) {
      // eslint-disable-next-line no-console
      console.error("[paperclip] Sentry shutdown failed", err);
    }
  })();
  return shutdownPromise;
}

/**
 * The subset of the `@sentry/node` module surface the initializer needs to
 * build its options object. A structural type, not the real Sentry type —
 * the real type is unavailable at compile time because the package is an
 * optional runtime dependency (see the module comment above).
 */
interface SentryModuleLike {
  httpIntegration(options: { breadcrumbs: boolean }): { name: string };
  onUnhandledRejectionIntegration(options: { mode: "strict" }): { name: string };
}

/** The `Sentry.init` options this gate builds. */
export interface SentryInitOptions {
  dsn: string;
  release?: string;
  skipOpenTelemetrySetup: boolean;
  tracesSampleRate: number;
  sendDefaultPii: boolean;
  serverName: string;
  integrations: (defaults: Array<{ name: string }>) => Array<{ name: string }>;
  beforeSend: typeof enrichServerEvent;
}

type ServerEvent = Parameters<typeof addSentryRunLinks>[0] & Parameters<typeof addDynamicImportProvenance>[0];

/** The server's `beforeSend`: run navigation links, then dynamic-import provenance. */
export async function enrichServerEvent<T extends ServerEvent>(event: T, hint?: ProvenanceHint): Promise<T> {
  return addDynamicImportProvenance(await addSentryRunLinks(event), hint);
}

/**
 * Build the `Sentry.init` options object. A pure function, split out from
 * `bootstrapSentry` so a test can call it with a real `@sentry/node` module
 * and assert the resolved integration list and the captured-event shape
 * against the true SDK, not a stand-in.
 */
export function buildSentryInitOptions(
  dsn: string,
  Sentry: SentryModuleLike,
): SentryInitOptions {
  return {
    dsn,
    release: process.env.SENTRY_RELEASE?.trim() || readBuildCommit() || undefined,
    skipOpenTelemetrySetup: true,
    tracesSampleRate: 0,
    sendDefaultPii: false,
    beforeSend: enrichServerEvent,
    serverName: process.env.SENTRY_NAME || os.hostname(),
    integrations: (defaults: Array<{ name: string }>) => {
      const kept = defaults.filter(
        (integration) =>
          integration.name !== "Console" &&
          integration.name !== "ContextLines" &&
          integration.name !== "Http" &&
          integration.name !== "OnUnhandledRejection",
      );
      return [
        ...kept,
        // Keep the rest of the Http integration — RequestData and request
        // isolation need it — but turn the outbound breadcrumb off.
        Sentry.httpIntegration({ breadcrumbs: false }),
        // Keep today's crash-on-unhandled-rejection behavior. See the
        // module comment above for why the default mode cannot stay.
        Sentry.onUnhandledRejectionIntegration({ mode: "strict" }),
      ];
    },
  };
}

async function bootstrapSentry(dsn: string): Promise<void> {
  // Gate on the exact peer version before touching the dynamic import: a
  // package installed at the wrong version can still load and start, which
  // would silently invalidate the privacy audit `doc/observability.md`
  // records against one exact version. Checking first turns that into one
  // precise, fail-open diagnostic.
  const versionCheck = checkExactPeerVersions(["@sentry/node"]);
  if (!versionCheck.ok) {
    // eslint-disable-next-line no-console
    console.warn(
      "[paperclip] The backend Sentry DSN is set, but the @sentry/node " +
        "package is not installed, or is installed at an unsupported " +
        "version. Install the declared version of @sentry/node to enable " +
        "server error monitoring. Continuing without it.",
      versionCheck.detail,
    );
    return;
  }

  try {
    // Dynamic import so type-resolution doesn't require the package to be
    // installed unless the operator actually opts in.
    // @ts-ignore optional peer dep
    const Sentry = await import("@sentry/node");

    Sentry.init(buildSentryInitOptions(dsn, Sentry));

    Sentry.getClient?.()?.on("afterSendEvent", (event: { event_id?: string }, response?: { statusCode?: number }) => {
      confirmTaskInterruptionDelivery(event.event_id, response?.statusCode);
    });

    sentryHandle = {
      captureException: (...args) => Sentry.captureException(...args),
      close: (timeout) => Sentry.close(timeout),
    };
  } catch (err) {
    // The exact-version gate above already confirmed @sentry/node is
    // installed at the declared version, so only a load or init failure
    // after that point reaches this block.
    // eslint-disable-next-line no-console
    console.warn(
      "[paperclip] The backend Sentry DSN is set, and @sentry/node passed " +
        "the version check, but it failed to load or initialize. " +
        "Continuing without error monitoring.",
      err,
    );
  }
}