PaperClipAI/server/src/__tests__/run-failure-sentry.test.ts

import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { RunFailureEvent } from "../sentry.js";

/**
 * Tests for `captureRunFailure`, the Sentry report for a terminal run
 * failure. `@sentry/node` is an optional runtime dependency and is not
 * installed in this test environment, so each test mocks a fake copy of the
 * package the same way `sentry.test.ts` does, then imports a fresh copy of
 * `../sentry.js` with a backend DSN set. That opens the gate and lets the
 * fake package's `captureException` spy records the real call
 * shape.
 */

const DSN_ENV = "SENTRY_DSN";
const FRONTEND_DSN_ENV = "SENTRY_DSN_FRONTEND";
const BACKEND_DSN_ENV = "SENTRY_DSN_BACKEND";
const originalDsn = process.env[DSN_ENV];
const originalFrontendDsn = process.env[FRONTEND_DSN_ENV];
const originalBackendDsn = process.env[BACKEND_DSN_ENV];

function baseEvent(overrides: Partial<RunFailureEvent> = {}): RunFailureEvent {
  return {
    taskId: "11111111-1111-1111-1111-111111111111",
    runId: "22222222-2222-2222-2222-222222222222",
    errorMessage: "the provider process exited with code 1",
    errorCode: "process_lost",
    agentAdapter: "claude-code",
    runStatus: "failed",
    ...overrides,
  };
}

interface CaptureContext {
  tags: Record<string, string>;
  contexts: Record<string, Record<string, unknown>>;
  fingerprint: string[];
}

/** Mirrors `sentry.test.ts`'s `mockSentryPackage`. */
function mockSentryPackage() {
  const init = vi.fn();
  const captureException = vi.fn((_error: unknown, _context?: CaptureContext) => "event-id");
  const close = vi.fn(async () => true);
  const httpIntegration = vi.fn((options: unknown) => ({ name: "Http", ...(options as object) }));
  const onUnhandledRejectionIntegration = vi.fn((options: unknown) => ({
    name: "OnUnhandledRejection",
    ...(options as object),
  }));

  vi.doMock("@sentry/node", () => ({
    init,
    getClient: () => ({ on: vi.fn() }),
    captureException,
    close,
    httpIntegration,
    onUnhandledRejectionIntegration,
  }));
  vi.doMock("../peer-version-check.js", () => ({
    checkExactPeerVersions: () => ({ ok: true }),
  }));

  return { init, captureException, close };
}

async function importFreshSentryWithGateOpen() {
  process.env[BACKEND_DSN_ENV] = "https://public@sentry.example.com/1";
  const mocks = mockSentryPackage();
  vi.resetModules();
  const sentryModule = await import("../sentry.js");
  await sentryModule.sentryReady;
  return { sentryModule, ...mocks };
}

beforeEach(() => {
  vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "unexpected");
  delete process.env[DSN_ENV];
  delete process.env[FRONTEND_DSN_ENV];
  delete process.env[BACKEND_DSN_ENV];
});

afterEach(() => {
  vi.unstubAllEnvs();
  if (originalDsn === undefined) delete process.env[DSN_ENV];
  else process.env[DSN_ENV] = originalDsn;
  if (originalFrontendDsn === undefined) delete process.env[FRONTEND_DSN_ENV];
  else process.env[FRONTEND_DSN_ENV] = originalFrontendDsn;
  if (originalBackendDsn === undefined) delete process.env[BACKEND_DSN_ENV];
  else process.env[BACKEND_DSN_ENV] = originalBackendDsn;
  vi.restoreAllMocks();
  vi.doUnmock("@sentry/node");
  vi.doUnmock("../peer-version-check.js");
});

describe("captureRunFailure", () => {
  it("reports a saved-message failure with the run-failure tag and the actual run status", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const event = { companyId: "company", agentId: "agent", taskId: "task", runId: "run",
      agentAdapter: "process", runStatus: "succeeded" as const, intentId: "intent" };
    sentryModule.captureTaskInterruptionFailure(event);
    sentryModule.captureTaskInterruptionFailure(event);
    expect(captureException).toHaveBeenCalledTimes(1);
    expect(captureException.mock.calls[0]![0]).toMatchObject({
      message: "Couldn’t finish this attempt. Your message is saved.",
    });
    expect(captureException.mock.calls[0]![1]).toMatchObject({
      tags: { event_kind: "run_failure", failure_source: "task_interruption", run_id: "run", task_id: "task",
        company_id: "company", agent_id: "agent", run_status: "succeeded", error_code: "task_attempt_failed" },
      contexts: { paperclip: { runPath: "/api/heartbeat-runs/run", taskPath: "/api/issues/task" } },
      fingerprint: ["task_attempt_failed", "process"],
    });
    sentryModule.captureTaskInterruptionFailure({ ...event, intentId: "next-intent" });
    sentryModule.captureTaskInterruptionFailure({ ...event, runId: "next-run" });
    expect(captureException).toHaveBeenCalledTimes(3);
    expect(captureException.mock.calls[2]![1]?.tags).not.toHaveProperty("run_error_code");
  });

  it("tags a saved-message report with a valid run error code without splitting its group", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const event = { companyId: "company", agentId: "agent", taskId: "task", agentAdapter: "process",
      runStatus: "failed" as const, intentId: null };
    sentryModule.captureTaskInterruptionFailure({ ...event, runId: "run", runErrorCode: "native_workspace_sync_out_failed" });
    sentryModule.captureTaskInterruptionFailure({ ...event, runId: "other", runErrorCode: "token=abc secret" });
    expect(captureException.mock.calls[0]![1]).toMatchObject({
      tags: { run_error_code: "native_workspace_sync_out_failed" }, fingerprint: ["task_attempt_failed", "process"] });
    expect(captureException.mock.calls[1]![1]?.tags).not.toHaveProperty("run_error_code");
    expect(JSON.stringify(captureException.mock.calls[1])).not.toContain("secret");
  });

  it("does not consume a saved-message report while reporting is off or capture fails", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const event = { companyId: "company", agentId: "agent", taskId: "task", runId: "run",
      agentAdapter: "process", runStatus: "failed" as const, intentId: null };
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "off");
    sentryModule.captureTaskInterruptionFailure(event);
    expect(captureException).not.toHaveBeenCalled();
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "unexpected");
    captureException.mockImplementationOnce(() => { throw new Error("Sentry unavailable"); });
    expect(() => sentryModule.captureTaskInterruptionFailure(event)).not.toThrow();
    sentryModule.captureTaskInterruptionFailure(event);
    sentryModule.captureTaskInterruptionFailure(event);
    expect(captureException).toHaveBeenCalledTimes(2);
  });

  it.each([undefined, "off", "true", "invalid"])("keeps ordinary server errors enabled while run reporting is disabled: %s", async mode => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", mode);
    if (mode === undefined) delete process.env.PAPERCLIP_SENTRY_RUN_FAILURES;
    sentryModule.captureRunFailure(baseEvent());
    const error = new Error("ordinary server failure");
    sentryModule.captureException(error);
    expect(captureException).toHaveBeenCalledExactlyOnceWith(error);
  });

  it("provides authenticated run references and groups repeated causes without leaking scope", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://paperclip.example.com");
    const event = baseEvent({ companyId: "company", agentId: "agent", taskId: null,
      diagnostics: { execution: { failurePhase: "setup", originalFailureCode: "tool_name_collision" }, adapter: {}, provider: {}, exceptions: [], truncatedFields: [] } });
    sentryModule.captureRunFailure(event);
    sentryModule.captureRunFailure({ ...event, runId: "another-run", errorMessage: "Another tool failed" });
    sentryModule.captureException(new Error("unrelated"));
    const first = captureException.mock.calls[0]![1]!;
    expect(first.tags).toMatchObject({ event_kind: "run_failure", company_id: "company", agent_id: "agent", root_error_code: "tool_name_collision" });
    expect(first.tags).not.toHaveProperty("task_id");
    expect(first.contexts.paperclip).toEqual({ schemaVersion: 1, companyId: "company", agentId: "agent",
      runPath: `/api/heartbeat-runs/${event.runId}`, eventsPath: `/api/heartbeat-runs/${event.runId}/events`, logPath: `/api/heartbeat-runs/${event.runId}/log`,
      runUrl: `https://paperclip.example.com/api/heartbeat-runs/${event.runId}`, eventsUrl: `https://paperclip.example.com/api/heartbeat-runs/${event.runId}/events`,
      logUrl: `https://paperclip.example.com/api/heartbeat-runs/${event.runId}/log` });
    expect(first.fingerprint).toEqual(["process_lost", "claude-code", "setup", "tool_name_collision"]);
    expect(captureException.mock.calls[1]![1]?.fingerprint).toEqual(first.fingerprint);
    expect(captureException.mock.calls[2]![1]).toBeUndefined();
  });

  it.each([undefined, "invalid", "ftp://example.com", "https://user:password@example.com", "https://example.com?token=private", "https://example.com/#private", "https://example.com/private"])("omits unsafe or absent instance URLs: %s", async value => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    vi.stubEnv("PAPERCLIP_PUBLIC_URL", value);
    sentryModule.captureRunFailure(baseEvent({ companyId: "company", agentId: "agent" }));
    const context = captureException.mock.calls[0]![1]!.contexts.paperclip;
    expect(context).toHaveProperty("runPath");
    expect(context).not.toHaveProperty("runUrl");
    expect(JSON.stringify(context)).not.toMatch(/password|private/);
  });

  it("sets the fingerprint [errorCode, agentAdapter] in that order", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();

    sentryModule.captureRunFailure(baseEvent({ errorCode: "timeout", agentAdapter: "codex" }));

    expect(captureException.mock.calls[0]![1]?.fingerprint).toEqual(["timeout", "codex"]);
  });

  it("keeps the error message out of the fingerprint for two runs with the same code and adapter but different messages", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();

    sentryModule.captureRunFailure(
      baseEvent({ errorMessage: "message one", errorCode: "timeout", agentAdapter: "codex" }),
    );
    sentryModule.captureRunFailure(
      baseEvent({ errorMessage: "message two", errorCode: "timeout", agentAdapter: "codex" }),
    );

    expect(captureException.mock.calls[0]![1]?.fingerprint).toEqual(["timeout", "codex"]);
    expect(captureException.mock.calls[1]![1]?.fingerprint).toEqual(["timeout", "codex"]);
  });

  it("separates setup failures without a usable root code by the root exception", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const setupFailure = (exceptions: NonNullable<RunFailureEvent["diagnostics"]>["exceptions"]) => baseEvent({
      errorCode: "setup_failed", agentAdapter: "paperclip_runner",
      diagnostics: { execution: { failurePhase: "setup" }, adapter: {}, provider: {}, exceptions, truncatedFields: [] },
    });

    sentryModule.captureRunFailure(setupFailure([{ name: "LegacyControllerLeaseLostError", message: "Legacy controller lease lost" }]));
    sentryModule.captureRunFailure(setupFailure([{ name: "Error", message: "The workspace still has an unfinished writer", status: 409 }]));
    sentryModule.captureRunFailure(setupFailure([{ name: "Error", message: "Workspace ownership changed", status: 409 }]));
    sentryModule.captureRunFailure(setupFailure([
      { name: "EnvironmentRunError", message: "Failed to acquire lease", code: "lease_acquire_failed" },
      { name: "JsonRpcCallError", message: "Daytona lease acquisition timed out", code: "-32002" },
    ]));
    sentryModule.captureRunFailure(setupFailure([{ name: "JsonRpcCallError", message: "RPC call timed out", code: "-32003" }]));
    sentryModule.captureRunFailure(setupFailure([{ name: "Error", message: "plain failure" }]));

    const fingerprints = captureException.mock.calls.map(([, context]) => context?.fingerprint);
    const prefix = ["setup_failed", "paperclip_runner", "setup"];
    expect(fingerprints).toEqual([
      [...prefix, "exception:LegacyControllerLeaseLostError"],
      [...prefix, "exception:Error", "status:409"],
      [...prefix, "exception:Error", "status:409"],
      [...prefix, "exception:JsonRpcCallError", "code:-32002"],
      [...prefix, "exception:JsonRpcCallError", "code:-32003"],
      [...prefix, "exception:Error"],
    ]);
    expect(JSON.stringify(fingerprints)).not.toMatch(/workspace|timed out|plain failure/i);
  });

  it("keeps an identifier root code as the only root grouping element", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();

    sentryModule.captureRunFailure(baseEvent({
      errorCode: "setup_failed", agentAdapter: "paperclip_runner",
      diagnostics: { execution: { failurePhase: "setup" }, adapter: {}, provider: {},
        exceptions: [{ name: "Error", code: "ECONNRESET", status: 503 }], truncatedFields: [] },
    }));

    expect(captureException.mock.calls[0]![1]?.fingerprint).toEqual(["setup_failed", "paperclip_runner", "setup", "ECONNRESET"]);
  });

  it("drops unbounded root exception names and codes from the fallback", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();

    sentryModule.captureRunFailure(baseEvent({
      errorCode: "setup_failed", agentAdapter: "paperclip_runner",
      diagnostics: { execution: { failurePhase: "setup" }, adapter: {}, provider: {},
        exceptions: [{ name: "Error: token=secret value", code: "not a code 123" }], truncatedFields: [] },
    }));

    expect(captureException.mock.calls[0]![1]?.fingerprint).toEqual(["setup_failed", "paperclip_runner", "setup"]);
  });

  it("sets the fingerprint element \"unknown\" when the error code is absent", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();

    sentryModule.captureRunFailure(baseEvent({ errorCode: null }));

    expect(captureException.mock.calls[0]![1]?.fingerprint).toEqual(["unknown", "claude-code"]);
  });

  it("keeps missing process exit evidence null on the run_failure context", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const event = baseEvent();

    sentryModule.captureRunFailure(event);

    expect(captureException.mock.calls[0]![1]?.contexts.run_failure).toEqual({
      taskId: event.taskId,
      runId: event.runId,
      errorMessage: event.errorMessage,
      errorCode: event.errorCode,
      agentAdapter: event.agentAdapter,
      exitCode: null,
      signal: null,
    });
  });

  it.each([
    { exitCode: 1, signal: null },
    { exitCode: 0, signal: null },
    { exitCode: -1, signal: null },
    { exitCode: -2147483648, signal: null },
    { exitCode: 2147483647, signal: null },
    { exitCode: null, signal: "SIGTERM" },
    { exitCode: null, signal: "SIGKILL" },
  ])("retains process exit evidence without changing grouping: %j", async (processExit) => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const event = baseEvent({ ...processExit, errorMessage: "Adapter failed", errorCode: "adapter_failed" });

    sentryModule.captureRunFailure(event);

    const [error, context] = captureException.mock.calls[0]!;
    expect((error as Error).message).toBe("Adapter failed");
    expect(context?.contexts.run_failure).toMatchObject(processExit);
    expect(context?.fingerprint).toEqual(["adapter_failed", event.agentAdapter]);
    expect(context?.tags).not.toHaveProperty("exitCode");
    expect(context?.tags).not.toHaveProperty("signal");
  });

  it.each([
    ["private-exit-payload", "SIGTERM private-signal-payload"],
    ["1", "constructor"],
    [true, "__proto__"],
    [1.5, "SIGCUSTOM"],
    [NaN, ""],
    [Infinity, 9],
    [2147483648, { private: "signal-payload" }],
    [-2147483649, ["SIGTERM"]],
  ])("rejects malformed process exit fields (%j, %j)", async (exitCode, signal) => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();

    sentryModule.captureRunFailure({ ...baseEvent(), exitCode, signal } as RunFailureEvent);

    expect(captureException.mock.calls[0]![1]?.contexts.run_failure).toMatchObject({
      exitCode: null,
      signal: "unknown",
    });
    expect(JSON.stringify(captureException.mock.calls)).not.toContain("private-");
  });

  it("does not set an instance key on the run_failure context", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();

    sentryModule.captureRunFailure(baseEvent());

    const context = captureException.mock.calls[0]![1]?.contexts.run_failure;
    expect(context).not.toHaveProperty("instance");
  });

  it("sets run_id, task_id, error_code, agent_adapter, and run_status as tags", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const event = baseEvent();

    sentryModule.captureRunFailure(event);

    expect(captureException.mock.calls[0]![1]?.tags).toEqual({
      event_kind: "run_failure",
      run_id: event.runId,
      task_id: event.taskId,
      error_code: event.errorCode,
      agent_adapter: event.agentAdapter,
      run_status: event.runStatus,
    });
  });

  it("captures the redacted error message as the exception message", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    const event = baseEvent({ errorMessage: "boom" });

    sentryModule.captureRunFailure(event);

    expect(captureException).toHaveBeenCalledTimes(1);
    const [received] = captureException.mock.calls[0]!;
    expect(received).toBeInstanceOf(Error);
    expect((received as Error).message).toBe("boom");
  });

  it("uses sanitized original stacks and causes instead of the reporting stack", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    sentryModule.captureRunFailure(baseEvent({ diagnostics: {
      execution: { runtimeMode: "native", failurePhase: "setup" },
      adapter: {}, provider: {}, truncatedFields: [],
      exceptions: [
        { name: "TypeError", message: "setup failed", stack: "TypeError: setup failed\n    at originalSetup (/app/setup.js:42:7)" },
        { name: "Error", message: "connection reset", stack: "Error: connection reset\n    at socketRead (/app/network.js:9:4)", code: "ECONNRESET", requestId: "req-123" },
      ],
    } }));
    const [error, context] = captureException.mock.calls[0]!;
    expect(error).toBeInstanceOf(Error);
    expect(error).toMatchObject({ name: "TypeError", message: "setup failed", cause: { message: "connection reset" } });
    expect((error as Error).stack).toContain("originalSetup");
    expect((error as Error).stack).not.toContain("captureRunFailure");
    expect(context?.contexts.run_exception_1).toMatchObject({ code: "ECONNRESET", requestId: "req-123" });
    expect(context?.contexts.run_execution).toMatchObject({ failurePhase: "setup" });
  });

  it("does not invent a reporter stack for a saved result with no original exception", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    sentryModule.captureRunFailure(baseEvent());
    expect((captureException.mock.calls[0]![0] as Error).stack).toBeUndefined();
  });

  it("does not throw and captures nothing when the gate is closed", async () => {
    vi.resetModules();
    const sentryModule = await import("../sentry.js");
    await sentryModule.sentryReady;

    expect(() => sentryModule.captureRunFailure(baseEvent())).not.toThrow();
  });

  it("does not throw when the client throws", async () => {
    const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
    captureException.mockImplementation(() => {
      throw new Error("sentry client is down");
    });

    expect(() => sentryModule.captureRunFailure(baseEvent())).not.toThrow();
  });
});