mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-31 06:04:11 +01:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Operators depend on `/api/health` and OpenAPI status surfaces to know whether the local control plane is healthy. > - Database backups are a safety-critical background process, but backup failures were not represented in health responses. > - That gap means an instance can look healthy while backup state is stale, failing, or unavailable. > - This pull request adds backup-health evaluation and exposes it through the health route, server startup wiring, and OpenAPI contract. > - The benefit is earlier operator visibility when automatic backups stop protecting instance data. ## Linked Issues or Issue Description No public GitHub issue exists. Inline bug report: **Pre-submission checklist** - [x] I have searched existing open and closed issues and this is not a duplicate. - [x] I am on the latest released version of Paperclip (or can reproduce on `master`). - [x] I have confirmed the error originates in Paperclip itself — not in my agent adapter, API provider, or local configuration. **What happened?** Automatic database backup health was not included in the app health response, so backup failures or stale backups could be missed while `/api/health` still looked otherwise usable. **Expected behavior** The health endpoint should include backup-health details that let operators identify disabled, stale, failing, or healthy backup states. **Steps to reproduce** 1. Configure a Paperclip instance with automatic database backups. 2. Force backup status into a stale or failing state. 3. Call `/api/health` and inspect whether backup state is represented. **Paperclip version or commit** `master` at the PR base. **Deployment mode** Local dev (`pnpm dev`) and self-hosted server deployments. **Installation method** Built from source (`pnpm dev` / `pnpm build`). **Agent adapter(s) involved** - [x] Not adapter-specific (core bug) **Database mode** Embedded development Postgres and external Postgres backup paths. **Access context** Board/operator health checks. **Relevant logs or output** Covered by the added `server/src/__tests__/health.test.ts` cases. **Relevant config (if applicable)** Not applicable. **Additional context** This surfaces backup status only; it does not change backup execution scheduling. **Privacy checklist** - [x] I have reviewed all pasted output for PII (usernames, file paths, API keys, tokens, company names) and redacted where necessary. ## What Changed - Added a database backup health service that classifies backup recency, status, and failure conditions. - Wired backup health into app/server startup and the health route response. - Documented the backup-health behavior in development docs and OpenAPI output. - Added focused health route tests for healthy, stale, disabled, and failing backup states. ## Verification - `/srv/paperclip/home/paperclipai/paperclip/node_modules/.bin/vitest run server/src/__tests__/health.test.ts` ## Risks Low-to-medium risk. This changes health response content and may affect external health consumers that parse fields strictly. It should not alter backup execution itself. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex, GPT-5.5 coding agent with repository tool use and local shell execution. Context window was not surfaced by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
154 lines
4.2 KiB
TypeScript
154 lines
4.2 KiB
TypeScript
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
import { basename, join, resolve } from "node:path";
|
|
|
|
export type DatabaseBackupHealthWarningCode =
|
|
| "database_backup_check_failed"
|
|
| "database_backup_last_failure"
|
|
| "database_backup_missing"
|
|
| "database_backup_stale";
|
|
|
|
export type DatabaseBackupHealthWarning = {
|
|
code: DatabaseBackupHealthWarningCode;
|
|
message: string;
|
|
};
|
|
|
|
export type DatabaseBackupHealthStatus = {
|
|
enabled: boolean;
|
|
status: "ok" | "warning";
|
|
backupDir: string;
|
|
maxAgeHours: number;
|
|
latestBackup: {
|
|
name: string;
|
|
path: string;
|
|
mtime: string;
|
|
ageHours: number;
|
|
sizeBytes: number;
|
|
} | null;
|
|
lastFailure: {
|
|
path: string;
|
|
mtime: string;
|
|
message: string;
|
|
} | null;
|
|
warnings: DatabaseBackupHealthWarning[];
|
|
};
|
|
|
|
export type InspectDatabaseBackupHealthOptions = {
|
|
enabled: boolean;
|
|
backupDir: string;
|
|
maxAgeHours: number;
|
|
alertFile?: string;
|
|
alertFiles?: string[];
|
|
now?: Date;
|
|
};
|
|
|
|
function roundHours(value: number): number {
|
|
return Math.round(value * 10) / 10;
|
|
}
|
|
|
|
function alertFileCandidates(opts: InspectDatabaseBackupHealthOptions) {
|
|
return [...new Set([
|
|
opts.alertFile,
|
|
...(opts.alertFiles ?? []),
|
|
join(opts.backupDir, "db-backup-to-s3.failure"),
|
|
resolve(opts.backupDir, "..", "db-backup-to-s3.failure"),
|
|
].filter((value): value is string => Boolean(value)))];
|
|
}
|
|
|
|
function readLastFailure(alertFiles: string[]) {
|
|
const failures = alertFiles
|
|
.filter((alertFile) => existsSync(alertFile))
|
|
.map((alertFile) => {
|
|
const stat = statSync(alertFile);
|
|
const message = readFileSync(alertFile, "utf8").trim().split(/\r?\n/)[0] ||
|
|
"Database backup failure marker is present.";
|
|
return {
|
|
path: alertFile,
|
|
mtime: new Date(stat.mtimeMs).toISOString(),
|
|
mtimeMs: stat.mtimeMs,
|
|
message,
|
|
};
|
|
})
|
|
.sort((left, right) => right.mtimeMs - left.mtimeMs);
|
|
const latest = failures[0];
|
|
if (!latest) return null;
|
|
return {
|
|
path: latest.path,
|
|
mtime: latest.mtime,
|
|
message: latest.message,
|
|
};
|
|
}
|
|
|
|
function findLatestBackup(backupDir: string, nowMs: number) {
|
|
if (!existsSync(backupDir)) return null;
|
|
|
|
const candidates = readdirSync(backupDir)
|
|
.filter((name) => name.endsWith(".sql.gz"))
|
|
.map((name) => {
|
|
const fullPath = join(backupDir, name);
|
|
const stat = statSync(fullPath);
|
|
return { fullPath, name, stat };
|
|
})
|
|
.sort((a, b) => b.stat.mtimeMs - a.stat.mtimeMs);
|
|
|
|
const latest = candidates[0];
|
|
if (!latest) return null;
|
|
|
|
return {
|
|
name: basename(latest.fullPath),
|
|
path: latest.fullPath,
|
|
mtime: new Date(latest.stat.mtimeMs).toISOString(),
|
|
ageHours: roundHours((nowMs - latest.stat.mtimeMs) / 3_600_000),
|
|
sizeBytes: latest.stat.size,
|
|
};
|
|
}
|
|
|
|
export function inspectDatabaseBackupHealth(
|
|
opts: InspectDatabaseBackupHealthOptions,
|
|
): DatabaseBackupHealthStatus {
|
|
const warnings: DatabaseBackupHealthWarning[] = [];
|
|
const now = opts.now ?? new Date();
|
|
const maxAgeHours = Math.max(1, opts.maxAgeHours);
|
|
|
|
let latestBackup: DatabaseBackupHealthStatus["latestBackup"] = null;
|
|
let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null;
|
|
|
|
try {
|
|
latestBackup = findLatestBackup(opts.backupDir, now.getTime());
|
|
lastFailure = readLastFailure(alertFileCandidates(opts));
|
|
|
|
if (!latestBackup) {
|
|
warnings.push({
|
|
code: "database_backup_missing",
|
|
message: `No .sql.gz database backups found in ${opts.backupDir}.`,
|
|
});
|
|
} else if (latestBackup.ageHours > maxAgeHours) {
|
|
warnings.push({
|
|
code: "database_backup_stale",
|
|
message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`,
|
|
});
|
|
}
|
|
|
|
if (lastFailure) {
|
|
warnings.push({
|
|
code: "database_backup_last_failure",
|
|
message: lastFailure.message,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
warnings.push({
|
|
code: "database_backup_check_failed",
|
|
message: `Database backup health check failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
});
|
|
}
|
|
|
|
return {
|
|
enabled: opts.enabled,
|
|
status: warnings.length > 0 ? "warning" : "ok",
|
|
backupDir: opts.backupDir,
|
|
maxAgeHours,
|
|
latestBackup,
|
|
lastFailure,
|
|
warnings,
|
|
};
|
|
}
|