diff --git a/cli/src/commands/state.ts b/cli/src/commands/state.ts new file mode 100644 index 00000000000..204dd6cba7f --- /dev/null +++ b/cli/src/commands/state.ts @@ -0,0 +1,34 @@ +import * as p from "@clack/prompts"; +import pc from "picocolors"; +import { printPaperclipCliBanner } from "../utils/banner.js"; + +type StateApiOptions = { apiUrl?: string; token?: string; json?: boolean }; + +async function callStateApi(pathname: string, opts: StateApiOptions, body?: unknown) { + const apiUrl = (opts.apiUrl?.trim() || process.env.PAPERCLIP_API_URL?.trim() || "http://127.0.0.1:3100/api").replace(/\/$/, ""); + const token = opts.token?.trim() || process.env.PAPERCLIP_API_KEY?.trim(); + const response = await fetch(`${apiUrl}${pathname}`, { + method: "POST", + headers: { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}) }, + body: JSON.stringify(body ?? {}), + }); + const result = await response.json(); + if (!response.ok) throw new Error(typeof result?.error === "string" ? result.error : `Request failed (${response.status})`); + return result; +} + +export async function stateSnapshotCommand(opts: StateApiOptions) { + printPaperclipCliBanner(); + p.intro(pc.bgCyan(pc.black(" paperclip state snapshot "))); + const result = await callStateApi("/instance/state-snapshots", opts); + if (opts.json) console.log(JSON.stringify(result, null, 2)); + else p.outro(pc.green(`Snapshot stored: ${result.objectKey}`)); +} + +export async function stateRestoreCommand(objectKey: string, opts: StateApiOptions) { + printPaperclipCliBanner(); + p.intro(pc.bgCyan(pc.black(" paperclip state restore "))); + const result = await callStateApi("/instance/state-snapshots/restore", opts, { objectKey }); + if (opts.json) console.log(JSON.stringify(result, null, 2)); + else p.outro(pc.green(`Snapshot restored: ${objectKey}`)); +} diff --git a/cli/src/index.ts b/cli/src/index.ts index b26b5cbe187..a31513ad6e1 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -9,6 +9,7 @@ import { heartbeatRun } from "./commands/heartbeat-run.js"; import { runCommand } from "./commands/run.js"; import { bootstrapCeoInvite } from "./commands/auth-bootstrap-ceo.js"; import { dbBackupCommand } from "./commands/db-backup.js"; +import { stateRestoreCommand, stateSnapshotCommand } from "./commands/state.js"; import { registerEnvLabCommands } from "./commands/env-lab.js"; import { registerContextCommands } from "./commands/client/context.js"; import { registerCompanyCommands } from "./commands/client/company.js"; @@ -160,6 +161,21 @@ program await dbBackupCommand(opts); }); +const state = program.command("state").description("Inspect, snapshot, and restore instance state"); +state.command("snapshot") + .description("Create an encrypted instance-state snapshot") + .option("--api-url ", "Paperclip API base URL") + .option("--token ", "Board/admin bearer token") + .option("--json", "Print result as JSON") + .action(stateSnapshotCommand); +state.command("restore") + .description("Restore an encrypted instance-state snapshot") + .argument("", "Storage-provider object key returned by state snapshot") + .option("--api-url ", "Paperclip API base URL") + .option("--token ", "Board/admin bearer token") + .option("--json", "Print result as JSON") + .action(stateRestoreCommand); + program .command("allowed-hostname") .description("Allow a hostname for authenticated/private mode access") diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index d446e818a81..7b2fa4e586f 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -811,6 +811,22 @@ DB backups are not full instance filesystem backups. For full local disaster recovery, also back up local storage files and the local encrypted secrets key if those providers are enabled. +## Instance-State Snapshots + +Set `PAPERCLIP_STATE_SNAPSHOT_KEY` to a 32-byte key encoded as 64 hex characters +or base64 to enable encrypted instance-state snapshots. The server writes them +through the configured storage provider, runs them daily by default, and reports +success/failure markers in `/api/health`. + +```sh +pnpm paperclipai state snapshot +pnpm paperclipai state restore 'instance-state/default/.tar.gz.enc' +``` + +Use `PAPERCLIP_STATE_SNAPSHOT_INTERVAL_MINUTES` to change the schedule and +`PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS` to change the health warning threshold. +The encryption key must be held separately from the instance being protected. + ## Secrets in Dev Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config. diff --git a/packages/shared/src/home-paths.ts b/packages/shared/src/home-paths.ts index fa4f1c44065..f88dace9cd3 100644 --- a/packages/shared/src/home-paths.ts +++ b/packages/shared/src/home-paths.ts @@ -34,6 +34,24 @@ export function resolvePaperclipInstanceRoot(input: { return path.resolve(resolvePaperclipHomeDir(input.homeDir), "instances", resolvePaperclipInstanceId(input.instanceId)); } +export function resolvePaperclipHomePath( + input: { homeDir?: string } = {}, + ...segments: string[] +): string { + return path.resolve(resolvePaperclipHomeDir(input.homeDir), ...segments); +} + +export function resolvePaperclipInstancePath( + input: { homeDir?: string; instanceId?: string } = {}, + ...segments: string[] +): string { + return path.resolve(resolvePaperclipInstanceRoot(input), ...segments); +} + +export function resolveUserHomePath(...segments: string[]): string { + return path.resolve(os.homedir(), ...segments); +} + export function resolvePaperclipInstanceConfigPath(input: { homeDir?: string; instanceId?: string; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index eb81657f74c..a0bb1114c8f 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,4 +1,6 @@ export { agentAdapterTypeSchema, optionalAgentAdapterTypeSchema } from "./adapter-type.js"; +export * from "./home-paths.js"; +export * from "./state-manifest.js"; export { decisionEffectStalenessSchema, decisionOptionStyleSchema, diff --git a/packages/shared/src/state-manifest.test.ts b/packages/shared/src/state-manifest.test.ts new file mode 100644 index 00000000000..9d9673dcbbe --- /dev/null +++ b/packages/shared/src/state-manifest.test.ts @@ -0,0 +1,26 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { STATE_MANIFEST } from "./state-manifest.js"; + +describe("STATE_MANIFEST", () => { + it("covers every approved inventory class with absolute live-layout paths", () => { + const homeDir = path.resolve("/tmp/paperclip-manifest-home"); + const expected = [ + "cli_auth_context", "plugin_install_set", "adapter_state", "host_secrets", "host_maintenance", + "host_ephemeral", "instance_config", "database", "attachment_storage", "run_logs", + "local_backup_staging", "agent_instructions_bundle", "secrets_master_key", "codex_agent_home", + "runtime_materializations", "skill_bundles", "project_repositories", "execution_workspaces", + "instance_logs", "claude_memory", "claude_transcripts", "claude_runtime_state", "claude_cache", + "external_cli_homes", + ]; + + expect(STATE_MANIFEST.map((entry) => entry.id)).toEqual(expected); + for (const entry of STATE_MANIFEST) { + const paths = entry.resolve({ homeDir, instanceId: "live" }); + expect(paths.length).toBeGreaterThan(0); + for (const resolved of paths) expect(path.isAbsolute(resolved)).toBe(true); + } + expect(STATE_MANIFEST.find((entry) => entry.id === "secrets_master_key")?.resolve({ homeDir, instanceId: "live" })[0]) + .toBe(path.join(homeDir, "instances", "live", "secrets", "master.key")); + }); +}); diff --git a/packages/shared/src/state-manifest.ts b/packages/shared/src/state-manifest.ts new file mode 100644 index 00000000000..171b303b93d --- /dev/null +++ b/packages/shared/src/state-manifest.ts @@ -0,0 +1,57 @@ +import { + resolveDefaultEmbeddedPostgresDir, + resolveDefaultSecretsKeyFilePath, + resolveDefaultStorageDir, + resolvePaperclipHomePath, + resolvePaperclipInstanceConfigPath, + resolvePaperclipInstancePath, + resolveUserHomePath, +} from "./home-paths.js"; + +export type StateDisposition = "git" | "s3_bulk" | "s3_secret" | "db" | "cache" | "ephemeral"; +export type StateRedactionClass = "none" | "secret_refs" | "forbid"; +export type StateCopyConsistency = "plain" | "sqlite_backup"; + +export type StateManifestContext = { homeDir?: string; instanceId?: string }; +export type StateClassEntry = { + id: string; + resolve(ctx: StateManifestContext): string[]; + disposition: StateDisposition; + redact: StateRedactionClass; + consistency: StateCopyConsistency; + retention?: { days?: number }; +}; + +const home = (ctx: StateManifestContext, ...segments: string[]) => resolvePaperclipHomePath(ctx, ...segments); +const instance = (ctx: StateManifestContext, ...segments: string[]) => resolvePaperclipInstancePath(ctx, ...segments); + +export const STATE_MANIFEST: readonly StateClassEntry[] = [ + { id: "cli_auth_context", resolve: (ctx) => [home(ctx, "auth.json"), home(ctx, "context.json")], disposition: "s3_secret", redact: "forbid", consistency: "plain" }, + { id: "plugin_install_set", resolve: (ctx) => [home(ctx, "adapter-plugins.json"), home(ctx, "adapter-plugins"), home(ctx, "plugins")], disposition: "git", redact: "secret_refs", consistency: "plain" }, + { id: "adapter_state", resolve: (ctx) => [home(ctx, "adapter-state")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain" }, + { id: "host_secrets", resolve: (ctx) => [home(ctx, "secrets")], disposition: "s3_secret", redact: "forbid", consistency: "plain" }, + { id: "host_maintenance", resolve: (ctx) => [home(ctx, "maintenance"), home(ctx, "hot-restart-report.json")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 30 } }, + { id: "host_ephemeral", resolve: (ctx) => [home(ctx, "feedback-artifacts"), home(ctx, "scratch"), home(ctx, "tmp"), home(ctx, "worktrees"), home(ctx, "migration-backups")], disposition: "ephemeral", redact: "none", consistency: "plain", retention: { days: 7 } }, + { id: "instance_config", resolve: (ctx) => [resolvePaperclipInstanceConfigPath(ctx), instance(ctx, ".env")], disposition: "git", redact: "secret_refs", consistency: "plain" }, + { id: "database", resolve: (ctx) => [resolveDefaultEmbeddedPostgresDir(ctx)], disposition: "db", redact: "forbid", consistency: "plain" }, + { id: "attachment_storage", resolve: (ctx) => [resolveDefaultStorageDir(ctx)], disposition: "s3_bulk", redact: "none", consistency: "plain" }, + { id: "run_logs", resolve: (ctx) => [instance(ctx, "data", "run-logs")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 90 } }, + { id: "local_backup_staging", resolve: (ctx) => [instance(ctx, "data", "backups"), instance(ctx, "data", "instance-backups")], disposition: "cache", redact: "forbid", consistency: "plain", retention: { days: 30 } }, + { id: "agent_instructions_bundle", resolve: (ctx) => [instance(ctx, "companies", "*", "agents", "*", "instructions")], disposition: "git", redact: "secret_refs", consistency: "plain" }, + { id: "secrets_master_key", resolve: (ctx) => [resolveDefaultSecretsKeyFilePath(ctx)], disposition: "s3_secret", redact: "forbid", consistency: "plain" }, + { id: "codex_agent_home", resolve: (ctx) => [instance(ctx, "companies", "*", "codex-home")], disposition: "s3_bulk", redact: "forbid", consistency: "sqlite_backup" }, + { id: "runtime_materializations", resolve: (ctx) => [instance(ctx, "companies", "*", "acp-engine"), instance(ctx, "companies", "*", "acpx-local"), instance(ctx, "companies", "*", "claude-prompt-cache")], disposition: "cache", redact: "secret_refs", consistency: "plain" }, + { id: "skill_bundles", resolve: (ctx) => [instance(ctx, "skills", "*")], disposition: "git", redact: "secret_refs", consistency: "plain" }, + { id: "project_repositories", resolve: (ctx) => [instance(ctx, "projects", "*", "*", "*")], disposition: "git", redact: "forbid", consistency: "plain" }, + { id: "execution_workspaces", resolve: (ctx) => [instance(ctx, "workspaces", "*")], disposition: "ephemeral", redact: "forbid", consistency: "plain", retention: { days: 7 } }, + { id: "instance_logs", resolve: (ctx) => [instance(ctx, "logs")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 30 } }, + { id: "claude_memory", resolve: () => [resolveUserHomePath(".claude", "projects", "**", "memory", "**")], disposition: "git", redact: "secret_refs", consistency: "plain" }, + { id: "claude_transcripts", resolve: () => [resolveUserHomePath(".claude", "projects", "**", "*.jsonl")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 90 } }, + { id: "claude_runtime_state", resolve: () => [resolveUserHomePath(".claude", "history.jsonl"), resolveUserHomePath(".claude", "todos"), resolveUserHomePath(".claude", "session-env")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 30 } }, + { id: "claude_cache", resolve: () => [resolveUserHomePath(".claude", "cache"), resolveUserHomePath(".claude", "debug"), resolveUserHomePath(".claude", "shell-snapshots"), resolveUserHomePath(".claude", "statsig")], disposition: "cache", redact: "none", consistency: "plain", retention: { days: 7 } }, + { id: "external_cli_homes", resolve: () => [resolveUserHomePath(".codex"), resolveUserHomePath(".gemini")], disposition: "s3_secret", redact: "forbid", consistency: "plain" }, +] as const; + +export function getStateManifestEntry(id: string): StateClassEntry | undefined { + return STATE_MANIFEST.find((entry) => entry.id === id); +} diff --git a/server/src/__tests__/instance-state-snapshot.test.ts b/server/src/__tests__/instance-state-snapshot.test.ts new file mode 100644 index 00000000000..71f674c5220 --- /dev/null +++ b/server/src/__tests__/instance-state-snapshot.test.ts @@ -0,0 +1,44 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { afterEach, describe, expect, it } from "vitest"; +import type { StateClassEntry } from "@paperclipai/shared"; +import type { StorageProvider } from "../storage/types.js"; +import { createAesStateSnapshotEncryptionProvider, createInstanceStateSnapshotService } from "../services/instance-state-snapshot.js"; + +const tempDirs: string[] = []; +afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); + +describe("instance state snapshots", () => { + it("encrypts, stores, and restores manifest state", async () => { + const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-test-")); + tempDirs.push(homeDir); + const source = path.join(homeDir, "instances", "test", "config.json"); + await fs.mkdir(path.dirname(source), { recursive: true }); + await fs.writeFile(source, "before"); + const objects = new Map(); + const provider: StorageProvider = { + id: "local_disk", + async putObject(input) { objects.set(input.objectKey, Buffer.isBuffer(input.body) ? input.body : Buffer.from(await Readable.toArray(input.body))); }, + async getObject(input) { const body = objects.get(input.objectKey); if (!body) throw new Error("missing"); return { stream: Readable.from(body), contentLength: body.length }; }, + async headObject(input) { const body = objects.get(input.objectKey); return { exists: Boolean(body), contentLength: body?.length }; }, + async deleteObject(input) { objects.delete(input.objectKey); }, + }; + const manifest: StateClassEntry[] = [{ id: "config", resolve: () => [source], disposition: "s3_secret", redact: "forbid", consistency: "plain" }]; + const service = createInstanceStateSnapshotService({ + storageProvider: provider, + encryptionProvider: createAesStateSnapshotEncryptionProvider(Buffer.alloc(32, 7)), + context: { homeDir, instanceId: "test" }, + markerDir: path.join(homeDir, "markers"), + manifest, + }); + + const result = await service.runSnapshot(); + expect(objects.get(result.objectKey)?.subarray(0, 8).toString()).toBe("PCSTATE1"); + await fs.writeFile(source, "after"); + await service.restoreSnapshot(result.objectKey); + expect(await fs.readFile(source, "utf8")).toBe("before"); + expect(JSON.parse(await fs.readFile(path.join(homeDir, "markers", "state-snapshot.success.json"), "utf8"))).toMatchObject({ objectKey: result.objectKey }); + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index e9798f191a1..8a22cf4f08a 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -65,6 +65,7 @@ import { instanceDatabaseBackupRoutes, type InstanceDatabaseBackupService, } from "./routes/instance-database-backups.js"; +import { instanceStateSnapshotRoutes, type InstanceStateSnapshotService } from "./routes/instance-state-snapshots.js"; import { llmRoutes } from "./routes/llms.js"; import { authRoutes } from "./routes/auth.js"; import { assetRoutes } from "./routes/assets.js"; @@ -256,7 +257,9 @@ export async function createApp( }): Promise; }; databaseBackupService?: InstanceDatabaseBackupService; + stateSnapshotService?: InstanceStateSnapshotService; databaseBackupHealth?: InspectDatabaseBackupHealthOptions; + stateSnapshotHealth?: { markerDir: string; enabled: boolean; maxAgeHours: number }; deploymentMode: DeploymentMode; deploymentExposure: DeploymentExposure; allowedHostnames: string[]; @@ -373,6 +376,7 @@ export async function createApp( authReady: opts.authReady, companyDeletionEnabled: opts.companyDeletionEnabled, databaseBackupHealth: opts.databaseBackupHealth, + stateSnapshotHealth: opts.stateSnapshotHealth, }), ); api.use(openApiRoutes()); @@ -464,6 +468,9 @@ export async function createApp( if (opts.databaseBackupService) { api.use(instanceDatabaseBackupRoutes(opts.databaseBackupService)); } + if (opts.stateSnapshotService) { + api.use(instanceStateSnapshotRoutes(opts.stateSnapshotService)); + } const pluginRegistry = pluginRegistryService(db); const eventBus = createPluginEventBus(); setPluginEventBus(eventBus); diff --git a/server/src/index.ts b/server/src/index.ts index c338564356d..14fb2027e0c 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -81,6 +81,8 @@ import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runt import { isLoopbackHost, rewriteLoopbackUrlPort } from "./url-utils.js"; import { createPluginWorkerManager } from "./services/plugin-worker-manager.js"; import { createStorageServiceFromConfig } from "./storage/index.js"; +import { createStorageProviderFromConfig } from "./storage/provider-registry.js"; +import { createAesStateSnapshotEncryptionProvider, createInstanceStateSnapshotService } from "./services/instance-state-snapshot.js"; import { printStartupBanner } from "./startup-banner.js"; import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js"; import { maybePersistWorktreeRuntimePorts } from "./worktree-config.js"; @@ -98,6 +100,7 @@ import type { InstanceDatabaseBackupRunResult, InstanceDatabaseBackupTrigger, } from "./routes/instance-database-backups.js"; +import { resolvePaperclipInstanceId, resolvePaperclipInstancePath } from "@paperclipai/shared"; type BetterAuthSessionUser = { id: string; @@ -637,6 +640,28 @@ export async function startServer(): Promise { const uiMode = config.uiDevMiddleware ? "vite-dev" : config.serveUi ? "static" : "none"; const storageService = createStorageServiceFromConfig(config); + const storageProvider = createStorageProviderFromConfig(config); + const stateSnapshotInstanceId = resolvePaperclipInstanceId(); + const stateSnapshotKeyRaw = process.env.PAPERCLIP_STATE_SNAPSHOT_KEY?.trim(); + const stateSnapshotKey = stateSnapshotKeyRaw + ? Buffer.from(stateSnapshotKeyRaw, /^[0-9a-f]{64}$/i.test(stateSnapshotKeyRaw) ? "hex" : "base64") + : null; + const stateSnapshotMarkerDir = resolvePaperclipInstancePath({ instanceId: stateSnapshotInstanceId }, "data", "instance-backups"); + const stateSnapshotService = stateSnapshotKey + ? createInstanceStateSnapshotService({ + storageProvider, + encryptionProvider: createAesStateSnapshotEncryptionProvider(stateSnapshotKey), + context: { instanceId: stateSnapshotInstanceId }, + markerDir: stateSnapshotMarkerDir, + }) + : null; + let stateSnapshotInFlight = false; + const runStateSnapshot = async () => { + if (!stateSnapshotService || stateSnapshotInFlight) return null; + stateSnapshotInFlight = true; + try { return await stateSnapshotService.runSnapshot(); } + finally { stateSnapshotInFlight = false; } + }; const feedback = feedbackService(db as any, { shareClient: createFeedbackTraceShareClientFromConfig(config), }); @@ -738,6 +763,16 @@ export async function startServer(): Promise { return result; }, }, + stateSnapshotService: stateSnapshotService + ? { + runSnapshot: async () => { + const result = await runStateSnapshot(); + if (!result) throw conflict("Instance-state snapshot already in progress"); + return result; + }, + restoreSnapshot: stateSnapshotService.restoreSnapshot, + } + : undefined, databaseBackupHealth: config.databaseBackupEnabled ? { enabled: config.databaseBackupEnabled, @@ -747,6 +782,11 @@ export async function startServer(): Promise { alertFiles: databaseBackupAlertFiles, } : undefined, + stateSnapshotHealth: { + enabled: Boolean(stateSnapshotService), + markerDir: stateSnapshotMarkerDir, + maxAgeHours: Math.max(1, Number(process.env.PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS) || 48), + }, deploymentMode: config.deploymentMode, deploymentExposure: config.deploymentExposure, allowedHostnames: config.allowedHostnames, @@ -1413,6 +1453,14 @@ export async function startServer(): Promise { }); }, backupIntervalMs); } + + if (stateSnapshotService) { + const intervalMinutes = Math.max(15, Number(process.env.PAPERCLIP_STATE_SNAPSHOT_INTERVAL_MINUTES) || 1440); + logger.info({ intervalMinutes, provider: storageProvider.id }, "Automatic encrypted instance-state snapshots enabled"); + setInterval(() => { void runStateSnapshot().catch((err) => logger.error({ err }, "Automatic instance-state snapshot failed")); }, intervalMinutes * 60 * 1000); + } else { + logger.warn("Instance-state snapshots disabled: PAPERCLIP_STATE_SNAPSHOT_KEY is not configured"); + } // Wait for external adapters to finish loading before accepting requests. // Without this, adapter type validation (assertKnownAdapterType) would diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts index f81f9098a88..5b55ccf1fa1 100644 --- a/server/src/routes/health.ts +++ b/server/src/routes/health.ts @@ -19,6 +19,7 @@ import { type InspectDatabaseBackupHealthOptions, } from "../services/database-backup-health.js"; import { instanceSettingsService } from "../services/instance-settings.js"; +import { inspectInstanceStateSnapshotHealth } from "../services/instance-state-snapshot-health.js"; import { serverVersion } from "../version.js"; function shouldExposeFullHealthDetails( @@ -83,6 +84,7 @@ export function healthRoutes( serverInfo?: ServerInfoSnapshot; databaseBackupHealth?: InspectDatabaseBackupHealthOptions; runtimeEnv?: CloudInstanceEnv; + stateSnapshotHealth?: { markerDir: string; enabled: boolean; maxAgeHours: number }; } = { deploymentMode: "local_trusted", deploymentExposure: "private", @@ -239,7 +241,11 @@ export function healthRoutes( const databaseBackup = opts.databaseBackupHealth ? inspectDatabaseBackupHealth(opts.databaseBackupHealth) : undefined; - const warnings = databaseBackup?.warnings.length ? databaseBackup.warnings : undefined; + const stateSnapshot = opts.stateSnapshotHealth + ? inspectInstanceStateSnapshotHealth(opts.stateSnapshotHealth) + : undefined; + const combinedWarnings = [...(databaseBackup?.warnings ?? []), ...(stateSnapshot?.warnings ?? [])]; + const warnings = combinedWarnings.length ? combinedWarnings : undefined; if (!exposeFullDetails) { const redactedDatabaseBackup = databaseBackup ? redactedDatabaseBackupHealth(databaseBackup) : undefined; @@ -252,6 +258,7 @@ export function healthRoutes( bootstrapStatus, bootstrapInviteActive, ...(redactedDatabaseBackup ? { databaseBackup: redactedDatabaseBackup } : {}), + ...(stateSnapshot ? { stateSnapshot: { enabled: stateSnapshot.enabled, status: stateSnapshot.status, warnings: stateSnapshot.warnings } } : {}), ...(redactedWarnings ? { warnings: redactedWarnings } : {}), ...(devServer ? { devServer } : {}), ...(cloud ? { cloud } : {}), @@ -274,6 +281,7 @@ export function healthRoutes( }, serverInfo, ...(databaseBackup ? { databaseBackup } : {}), + ...(stateSnapshot ? { stateSnapshot } : {}), ...(warnings ? { warnings } : {}), ...(devServer ? { devServer } : {}), ...(cloud ? { cloud } : {}), diff --git a/server/src/routes/instance-state-snapshots.ts b/server/src/routes/instance-state-snapshots.ts new file mode 100644 index 00000000000..5e965179ad3 --- /dev/null +++ b/server/src/routes/instance-state-snapshots.ts @@ -0,0 +1,24 @@ +import { Router } from "express"; +import { assertInstanceAdmin } from "./authz.js"; +import type { InstanceStateSnapshotResult } from "../services/instance-state-snapshot.js"; + +export type InstanceStateSnapshotService = { + runSnapshot(): Promise; + restoreSnapshot(objectKey: string): Promise; +}; + +export function instanceStateSnapshotRoutes(service: InstanceStateSnapshotService) { + const router = Router(); + router.post("/instance/state-snapshots", async (req, res) => { + assertInstanceAdmin(req); + res.status(201).json(await service.runSnapshot()); + }); + router.post("/instance/state-snapshots/restore", async (req, res) => { + assertInstanceAdmin(req); + const objectKey = typeof req.body?.objectKey === "string" ? req.body.objectKey.trim() : ""; + if (!objectKey) return res.status(400).json({ error: "objectKey is required" }); + await service.restoreSnapshot(objectKey); + res.status(200).json({ restored: true, objectKey }); + }); + return router; +} diff --git a/server/src/services/instance-state-snapshot-health.ts b/server/src/services/instance-state-snapshot-health.ts new file mode 100644 index 00000000000..78e14712aed --- /dev/null +++ b/server/src/services/instance-state-snapshot-health.ts @@ -0,0 +1,16 @@ +import { existsSync, readFileSync, statSync } from "node:fs"; +import path from "node:path"; + +export function inspectInstanceStateSnapshotHealth(opts: { markerDir: string; enabled: boolean; maxAgeHours: number; now?: Date }) { + const successPath = path.join(opts.markerDir, "state-snapshot.success.json"); + const failurePath = path.join(opts.markerDir, "state-snapshot.failure"); + const warnings: Array<{ code: string; message: string }> = []; + let latestSnapshot: Record | null = null; + if (existsSync(successPath)) { + latestSnapshot = JSON.parse(readFileSync(successPath, "utf8")); + const ageHours = ((opts.now ?? new Date()).getTime() - statSync(successPath).mtimeMs) / 3_600_000; + if (ageHours > opts.maxAgeHours) warnings.push({ code: "instance_state_snapshot_stale", message: `Latest instance-state snapshot is ${Math.round(ageHours * 10) / 10}h old.` }); + } else warnings.push({ code: "instance_state_snapshot_missing", message: "No successful instance-state snapshot marker found." }); + if (existsSync(failurePath)) warnings.push({ code: "instance_state_snapshot_last_failure", message: readFileSync(failurePath, "utf8").trim().split(/\r?\n/)[0] }); + return { enabled: opts.enabled, status: warnings.length ? "warning" : "ok", markerDir: opts.markerDir, latestSnapshot, warnings }; +} diff --git a/server/src/services/instance-state-snapshot.ts b/server/src/services/instance-state-snapshot.ts new file mode 100644 index 00000000000..193674263e6 --- /dev/null +++ b/server/src/services/instance-state-snapshot.ts @@ -0,0 +1,179 @@ +import { createCipheriv, createDecipheriv, createHash } from "node:crypto"; +import { createReadStream, createWriteStream, promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pipeline } from "node:stream/promises"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { + STATE_MANIFEST, + resolvePaperclipHomeDir, + resolvePaperclipInstancePath, + type StateClassEntry, + type StateManifestContext, +} from "@paperclipai/shared"; +import type { StorageProvider } from "../storage/types.js"; + +const execFileAsync = promisify(execFile); +const SNAPSHOT_MAGIC = Buffer.from("PCSTATE1"); + +export interface StateSnapshotEncryptionProvider { + id: string; + encrypt(inputPath: string, outputPath: string): Promise; + decrypt(inputPath: string, outputPath: string): Promise; +} + +export function createAesStateSnapshotEncryptionProvider(key: Buffer): StateSnapshotEncryptionProvider { + if (key.length !== 32) throw new Error("State snapshot encryption key must be exactly 32 bytes."); + return { + id: "aes-256-gcm", + async encrypt(inputPath, outputPath) { + const iv = Buffer.from(cryptoRandom(12)); + const cipher = createCipheriv("aes-256-gcm", key, iv); + await fs.writeFile(outputPath, Buffer.concat([SNAPSHOT_MAGIC, iv])); + await pipeline(createReadStream(inputPath), cipher, createWriteStream(outputPath, { flags: "a" })); + await fs.appendFile(outputPath, cipher.getAuthTag()); + }, + async decrypt(inputPath, outputPath) { + const stat = await fs.stat(inputPath); + const handle = await fs.open(inputPath, "r"); + try { + const header = Buffer.alloc(SNAPSHOT_MAGIC.length + 12); + await handle.read(header, 0, header.length, 0); + if (!header.subarray(0, SNAPSHOT_MAGIC.length).equals(SNAPSHOT_MAGIC)) throw new Error("Invalid state snapshot header."); + const tag = Buffer.alloc(16); + await handle.read(tag, 0, tag.length, stat.size - tag.length); + const decipher = createDecipheriv("aes-256-gcm", key, header.subarray(SNAPSHOT_MAGIC.length)); + decipher.setAuthTag(tag); + await pipeline(createReadStream(inputPath, { start: header.length, end: stat.size - tag.length - 1 }), decipher, createWriteStream(outputPath)); + } finally { + await handle.close(); + } + }, + }; +} + +function cryptoRandom(length: number): Uint8Array { + return globalThis.crypto.getRandomValues(new Uint8Array(length)); +} + +async function exists(target: string): Promise { + try { await fs.access(target); return true; } catch { return false; } +} + +async function expandPattern(pattern: string): Promise { + const parsed = path.parse(pattern); + const segments = pattern.slice(parsed.root.length).split(path.sep); + let current = [parsed.root]; + for (const segment of segments) { + if (!segment.includes("*")) { + current = current.map((base) => path.join(base, segment)); + continue; + } + const next: string[] = []; + for (const base of current) { + if (!(await exists(base))) continue; + const names = await fs.readdir(base); + const regex = new RegExp(`^${segment.replace(/\./g, "\\.").replace(/\*\*/g, ".*").replace(/\*/g, "[^/]*")}$`); + for (const name of names) if (regex.test(name)) next.push(path.join(base, name)); + } + current = next; + } + const found: string[] = []; + for (const candidate of current) if (await exists(candidate)) found.push(candidate); + return found; +} + +async function copyEntry(entry: StateClassEntry, source: string, destination: string): Promise { + const stat = await fs.stat(source); + await fs.mkdir(path.dirname(destination), { recursive: true }); + if (entry.consistency === "sqlite_backup" && stat.isFile() && /\.(sqlite|sqlite3|db)$/i.test(source)) { + await execFileAsync("sqlite3", [source, `.backup '${destination.replace(/'/g, "''")}'`]); + return; + } + await fs.cp(source, destination, { recursive: true, preserveTimestamps: true }); +} + +export type InstanceStateSnapshotResult = { + objectKey: string; + sizeBytes: number; + sha256: string; + entryCount: number; + startedAt: string; + finishedAt: string; +}; + +export function createInstanceStateSnapshotService(opts: { + storageProvider: StorageProvider; + encryptionProvider: StateSnapshotEncryptionProvider; + context?: StateManifestContext; + markerDir?: string; + manifest?: readonly StateClassEntry[]; +}) { + const context = opts.context ?? {}; + const homeRoot = resolvePaperclipHomeDir(context.homeDir); + const userRoot = os.homedir(); + const markerDir = opts.markerDir ?? resolvePaperclipInstancePath(context, "data", "instance-backups"); + const manifest = opts.manifest ?? STATE_MANIFEST; + + async function runSnapshot(): Promise { + const startedAt = new Date(); + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-snapshot-")); + try { + const stageDir = path.join(tempDir, "stage"); + let entryCount = 0; + for (const entry of manifest) { + if (entry.disposition === "db" || entry.disposition === "cache" || entry.disposition === "ephemeral") continue; + for (const pattern of entry.resolve(context)) { + for (const source of await expandPattern(pattern)) { + const root = source === homeRoot || source.startsWith(homeRoot + path.sep) ? homeRoot : userRoot; + const rootName = root === homeRoot ? "paperclip-home" : "user-home"; + await copyEntry(entry, source, path.join(stageDir, rootName, path.relative(root, source))); + entryCount += 1; + } + } + } + await fs.writeFile(path.join(stageDir, "snapshot-manifest.json"), JSON.stringify({ version: 1, createdAt: startedAt.toISOString(), entries: manifest.map(({ resolve: _resolve, ...entry }) => entry) }, null, 2)); + const archivePath = path.join(tempDir, "snapshot.tar.gz"); + await execFileAsync("tar", ["-czf", archivePath, "-C", stageDir, "."]); + const encryptedPath = `${archivePath}.enc`; + await opts.encryptionProvider.encrypt(archivePath, encryptedPath); + const body = await fs.readFile(encryptedPath); + const objectKey = `instance-state/${context.instanceId ?? "default"}/${startedAt.toISOString().replace(/[:.]/g, "-")}.tar.gz.enc`; + await opts.storageProvider.putObject({ objectKey, body, contentType: "application/vnd.paperclip.state-snapshot", contentLength: body.length }); + const result = { objectKey, sizeBytes: body.length, sha256: createHash("sha256").update(body).digest("hex"), entryCount, startedAt: startedAt.toISOString(), finishedAt: new Date().toISOString() }; + await fs.mkdir(markerDir, { recursive: true }); + await fs.writeFile(path.join(markerDir, "state-snapshot.success.json"), JSON.stringify(result)); + await fs.rm(path.join(markerDir, "state-snapshot.failure"), { force: true }); + return result; + } catch (error) { + await fs.mkdir(markerDir, { recursive: true }); + await fs.writeFile(path.join(markerDir, "state-snapshot.failure"), `${new Date().toISOString()} ${error instanceof Error ? error.message : String(error)}\n`); + throw error; + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + } + + async function restoreSnapshot(objectKey: string): Promise { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-restore-")); + try { + const encryptedPath = path.join(tempDir, "snapshot.enc"); + const object = await opts.storageProvider.getObject({ objectKey }); + await pipeline(object.stream, createWriteStream(encryptedPath)); + const archivePath = path.join(tempDir, "snapshot.tar.gz"); + await opts.encryptionProvider.decrypt(encryptedPath, archivePath); + const extractDir = path.join(tempDir, "extract"); + await fs.mkdir(extractDir); + await execFileAsync("tar", ["-xzf", archivePath, "-C", extractDir, "--no-same-owner", "--no-same-permissions"]); + for (const [name, root] of [["paperclip-home", homeRoot], ["user-home", userRoot]] as const) { + const source = path.join(extractDir, name); + if (await exists(source)) await fs.cp(source, root, { recursive: true, force: true }); + } + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + } + + return { runSnapshot, restoreSnapshot }; +}