Fix DB backup health alerts

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/DEVELOPING.md

@@ -633,6 +633,10 @@ Environment overrides:
 - `PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES=<minutes>`
 - `PAPERCLIP_DB_BACKUP_RETENTION_DAYS=<days>`
 - `PAPERCLIP_DB_BACKUP_DIR=/absolute/or/~/path`
+- `PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS=<hours>` controls the `/api/health`
+  stale-backup warning threshold
+- `PAPERCLIP_DB_BACKUP_ALERT_FILE=/path/to/failure-marker` lets external cron
+  wrappers surface the last failed backup in `/api/health`
 
 DB backups are not full instance filesystem backups. For full local disaster
 recovery, also back up local storage files and the local encrypted secrets key if

server/src/__tests__/health.test.ts

@@ -1,3 +1,6 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import express from "express";
 import request from "supertest";
@@ -25,12 +28,22 @@ const testServerInfo = {
   },
 } as const;
 
+function createHealthyDb(): Db {
+  return {
+    execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
+  } as unknown as Db;
+}
+
 vi.mock("../dev-server-status.js", () => ({
   readPersistedDevServerStatus: mockReadPersistedDevServerStatus,
   toDevServerHealthStatus: vi.fn(),
 }));
 
-function createApp(db?: Db, serverInfo = testServerInfo) {
+function createApp(
+  db?: Db,
+  serverInfo = testServerInfo,
+  databaseBackupHealth?: Parameters<typeof healthRoutes>[1]["databaseBackupHealth"],
+) {
   const app = express();
   app.use(
     "/health",
@@ -40,6 +53,7 @@ function createApp(db?: Db, serverInfo = testServerInfo) {
       authReady: true,
       companyDeletionEnabled: true,
       serverInfo,
+      databaseBackupHealth,
     }),
   );
   return app;
@@ -116,6 +130,74 @@ describe("GET /health", () => {
     });
   });
 
+  it("surfaces a stale database backup warning in full health details", async () => {
+    const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
+    const backupFile = path.join(backupDir, "paperclip-20260705-031702.sql.gz");
+    fs.writeFileSync(backupFile, "backup");
+    fs.utimesSync(
+      backupFile,
+      new Date("2026-07-05T03:17:02.000Z"),
+      new Date("2026-07-05T03:17:02.000Z"),
+    );
+    const app = createApp(createHealthyDb(), testServerInfo, {
+      enabled: true,
+      backupDir,
+      maxAgeHours: 26,
+      now: new Date("2026-07-06T13:00:00.000Z"),
+    });
+
+    const res = await request(app).get("/health");
+
+    expect(res.status).toBe(200);
+    expect(res.body.databaseBackup).toMatchObject({
+      status: "warning",
+      backupDir,
+      maxAgeHours: 26,
+      latestBackup: {
+        name: "paperclip-20260705-031702.sql.gz",
+        ageHours: 33.7,
+      },
+      warnings: [
+        {
+          code: "database_backup_stale",
+        },
+      ],
+    });
+    expect(res.body.warnings).toEqual(res.body.databaseBackup.warnings);
+  });
+
+  it("surfaces database backup failure markers in full health details", async () => {
+    const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
+    const backupFile = path.join(backupDir, "paperclip-20260706-031702.sql.gz");
+    const alertFile = path.join(backupDir, "db-backup-to-s3.failure");
+    fs.writeFileSync(backupFile, "backup");
+    fs.writeFileSync(alertFile, "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1\n");
+    const app = createApp(createHealthyDb(), testServerInfo, {
+      enabled: true,
+      backupDir,
+      maxAgeHours: 26,
+      alertFile,
+      now: new Date("2026-07-06T04:00:00.000Z"),
+    });
+
+    const res = await request(app).get("/health");
+
+    expect(res.status).toBe(200);
+    expect(res.body.databaseBackup).toMatchObject({
+      status: "warning",
+      lastFailure: {
+        path: alertFile,
+        message: "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1",
+      },
+      warnings: [
+        {
+          code: "database_backup_last_failure",
+          message: "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1",
+        },
+      ],
+    });
+  });
+
   it("redacts detailed metadata for anonymous requests in authenticated mode", async () => {
     const devServerStatus = await import("../dev-server-status.js");
     vi.spyOn(devServerStatus, "readPersistedDevServerStatus").mockReturnValue(undefined);

server/src/app.ts

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import { fileURLToPath } from "node:url";
 import type { Db } from "@paperclipai/db";
 import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";
+import type { InspectDatabaseBackupHealthOptions } from "./services/database-backup-health.js";
 import type { StorageService } from "./storage/types.js";
 import { httpLogger, errorHandler } from "./middleware/index.js";
 import { actorMiddleware } from "./middleware/auth.js";
@@ -142,6 +143,7 @@ export async function createApp(
       }): Promise<unknown>;
     };
     databaseBackupService?: InstanceDatabaseBackupService;
+    databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
     deploymentMode: DeploymentMode;
     deploymentExposure: DeploymentExposure;
     allowedHostnames: string[];
@@ -217,6 +219,7 @@ export async function createApp(
       deploymentExposure: opts.deploymentExposure,
       authReady: opts.authReady,
       companyDeletionEnabled: opts.companyDeletionEnabled,
+      databaseBackupHealth: opts.databaseBackupHealth,
     }),
   );
   api.use(openApiRoutes());

server/src/index.ts

@@ -583,6 +583,14 @@ export async function startServer(): Promise<StartedServer> {
     shareClient: createFeedbackTraceShareClientFromConfig(config),
   });
   const backupSettingsSvc = instanceSettingsService(db);
+  const databaseBackupMaxAgeHours = Math.max(
+    1,
+    Number(process.env.PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS) ||
+      Math.max(26, Math.ceil((config.databaseBackupIntervalMinutes / 60) * 2)),
+  );
+  const databaseBackupAlertFile =
+    process.env.PAPERCLIP_DB_BACKUP_ALERT_FILE ||
+    resolve(config.databaseBackupDir, "..", "health", "db-backup-to-s3.failure");
   let databaseBackupInFlight = false;
   const runServerDatabaseBackup = async (
     trigger: InstanceDatabaseBackupTrigger,
@@ -657,6 +665,14 @@ export async function startServer(): Promise<StartedServer> {
         return result;
       },
     },
+    databaseBackupHealth: config.databaseBackupEnabled
+      ? {
+          enabled: config.databaseBackupEnabled,
+          backupDir: config.databaseBackupDir,
+          maxAgeHours: databaseBackupMaxAgeHours,
+          alertFile: databaseBackupAlertFile,
+        }
+      : undefined,
     deploymentMode: config.deploymentMode,
     deploymentExposure: config.deploymentExposure,
     allowedHostnames: config.allowedHostnames,

server/src/routes/health.ts

@@ -7,6 +7,10 @@ import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";
 import { readPersistedDevServerStatus, toDevServerHealthStatus, writeDevServerRestartRequest } from "../dev-server-status.js";
 import { logger } from "../middleware/logger.js";
 import { getServerInfoSnapshot, type ServerInfoSnapshot } from "../server-info.js";
+import {
+  inspectDatabaseBackupHealth,
+  type InspectDatabaseBackupHealthOptions,
+} from "../services/database-backup-health.js";
 import { instanceSettingsService } from "../services/instance-settings.js";
 import { serverVersion } from "../version.js";
 
@@ -37,6 +41,7 @@ export function healthRoutes(
     authReady: boolean;
     companyDeletionEnabled: boolean;
     serverInfo?: ServerInfoSnapshot;
+    databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
   } = {
     deploymentMode: "local_trusted",
     deploymentExposure: "private",
@@ -162,6 +167,11 @@ export function healthRoutes(
       });
     }
 
+    const databaseBackup = exposeFullDetails && opts.databaseBackupHealth
+      ? inspectDatabaseBackupHealth(opts.databaseBackupHealth)
+      : undefined;
+    const warnings = databaseBackup?.warnings.length ? databaseBackup.warnings : undefined;
+
     if (!exposeFullDetails) {
       res.json({
         status: "ok",
@@ -186,6 +196,8 @@ export function healthRoutes(
         companyDeletionEnabled: opts.companyDeletionEnabled,
       },
       serverInfo,
+      ...(databaseBackup ? { databaseBackup } : {}),
+      ...(warnings ? { warnings } : {}),
       ...(devServer ? { devServer } : {}),
     });
   });

server/src/routes/openapi.ts

@@ -880,6 +880,37 @@ registry.registerPath({
       deploymentMode: z.string().optional(),
       bootstrapStatus: z.enum(["ready", "bootstrap_pending"]).optional(),
       bootstrapInviteActive: z.boolean().optional(),
+      databaseBackup: z.object({
+        enabled: z.boolean(),
+        status: z.enum(["ok", "warning"]),
+        backupDir: z.string(),
+        maxAgeHours: z.number(),
+        latestBackup: z.object({
+          name: z.string(),
+          path: z.string(),
+          mtime: z.string().datetime(),
+          ageHours: z.number(),
+          sizeBytes: z.number(),
+        }).nullable(),
+        lastFailure: z.object({
+          path: z.string(),
+          mtime: z.string().datetime(),
+          message: z.string(),
+        }).nullable(),
+        warnings: z.array(z.object({
+          code: z.enum([
+            "database_backup_check_failed",
+            "database_backup_last_failure",
+            "database_backup_missing",
+            "database_backup_stale",
+          ]),
+          message: z.string(),
+        })),
+      }).optional(),
+      warnings: z.array(z.object({
+        code: z.string(),
+        message: z.string(),
+      })).optional(),
       serverInfo: z.object({
         processStartedAt: z.string().datetime(),
         git: z.union([

server/src/services/database-backup-health.ts

@@ -0,0 +1,131 @@
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, join } from "node:path";
+
+export type DatabaseBackupHealthWarningCode =
+  | "database_backup_check_failed"
+  | "database_backup_last_failure"
+  | "database_backup_missing"
+  | "database_backup_stale";
+
+export type DatabaseBackupHealthWarning = {
+  code: DatabaseBackupHealthWarningCode;
+  message: string;
+};
+
+export type DatabaseBackupHealthStatus = {
+  enabled: boolean;
+  status: "ok" | "warning";
+  backupDir: string;
+  maxAgeHours: number;
+  latestBackup: {
+    name: string;
+    path: string;
+    mtime: string;
+    ageHours: number;
+    sizeBytes: number;
+  } | null;
+  lastFailure: {
+    path: string;
+    mtime: string;
+    message: string;
+  } | null;
+  warnings: DatabaseBackupHealthWarning[];
+};
+
+export type InspectDatabaseBackupHealthOptions = {
+  enabled: boolean;
+  backupDir: string;
+  maxAgeHours: number;
+  alertFile?: string;
+  now?: Date;
+};
+
+function roundHours(value: number): number {
+  return Math.round(value * 10) / 10;
+}
+
+function readLastFailure(alertFile: string | undefined) {
+  if (!alertFile || !existsSync(alertFile)) return null;
+
+  const stat = statSync(alertFile);
+  const message = readFileSync(alertFile, "utf8").trim().split(/\r?\n/)[0] || "Database backup failure marker is present.";
+  return {
+    path: alertFile,
+    mtime: new Date(stat.mtimeMs).toISOString(),
+    message,
+  };
+}
+
+function findLatestBackup(backupDir: string, nowMs: number) {
+  if (!existsSync(backupDir)) return null;
+
+  const candidates = readdirSync(backupDir)
+    .filter((name) => name.endsWith(".sql.gz"))
+    .map((name) => {
+      const fullPath = join(backupDir, name);
+      const stat = statSync(fullPath);
+      return { fullPath, name, stat };
+    })
+    .sort((a, b) => b.stat.mtimeMs - a.stat.mtimeMs);
+
+  const latest = candidates[0];
+  if (!latest) return null;
+
+  return {
+    name: basename(latest.fullPath),
+    path: latest.fullPath,
+    mtime: new Date(latest.stat.mtimeMs).toISOString(),
+    ageHours: roundHours((nowMs - latest.stat.mtimeMs) / 3_600_000),
+    sizeBytes: latest.stat.size,
+  };
+}
+
+export function inspectDatabaseBackupHealth(
+  opts: InspectDatabaseBackupHealthOptions,
+): DatabaseBackupHealthStatus {
+  const warnings: DatabaseBackupHealthWarning[] = [];
+  const now = opts.now ?? new Date();
+  const maxAgeHours = Math.max(1, opts.maxAgeHours);
+
+  let latestBackup: DatabaseBackupHealthStatus["latestBackup"] = null;
+  let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null;
+
+  try {
+    latestBackup = findLatestBackup(opts.backupDir, now.getTime());
+    lastFailure = readLastFailure(opts.alertFile);
+
+    if (!latestBackup) {
+      warnings.push({
+        code: "database_backup_missing",
+        message: `No .sql.gz database backups found in ${opts.backupDir}.`,
+      });
+    } else if (latestBackup.ageHours > maxAgeHours) {
+      warnings.push({
+        code: "database_backup_stale",
+        message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`,
+      });
+    }
+
+    if (lastFailure) {
+      warnings.push({
+        code: "database_backup_last_failure",
+        message: lastFailure.message,
+      });
+    }
+  } catch (error) {
+    warnings.push({
+      code: "database_backup_check_failed",
+      message: `Database backup health check failed: ${error instanceof Error ? error.message : String(error)}`,
+    });
+  }
+
+  return {
+    enabled: opts.enabled,
+    status: warnings.length > 0 ? "warning" : "ok",
+    backupDir: opts.backupDir,
+    maxAgeHours,
+    latestBackup,
+    lastFailure,
+    warnings,
+  };
+}