feat: add encrypted instance state snapshots

cli/src/commands/state.ts

@@ -0,0 +1,34 @@
+import * as p from "@clack/prompts";
+import pc from "picocolors";
+import { printPaperclipCliBanner } from "../utils/banner.js";
+
+type StateApiOptions = { apiUrl?: string; token?: string; json?: boolean };
+
+async function callStateApi(pathname: string, opts: StateApiOptions, body?: unknown) {
+  const apiUrl = (opts.apiUrl?.trim() || process.env.PAPERCLIP_API_URL?.trim() || "http://127.0.0.1:3100/api").replace(/\/$/, "");
+  const token = opts.token?.trim() || process.env.PAPERCLIP_API_KEY?.trim();
+  const response = await fetch(`${apiUrl}${pathname}`, {
+    method: "POST",
+    headers: { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}) },
+    body: JSON.stringify(body ?? {}),
+  });
+  const result = await response.json();
+  if (!response.ok) throw new Error(typeof result?.error === "string" ? result.error : `Request failed (${response.status})`);
+  return result;
+}
+
+export async function stateSnapshotCommand(opts: StateApiOptions) {
+  printPaperclipCliBanner();
+  p.intro(pc.bgCyan(pc.black(" paperclip state snapshot ")));
+  const result = await callStateApi("/instance/state-snapshots", opts);
+  if (opts.json) console.log(JSON.stringify(result, null, 2));
+  else p.outro(pc.green(`Snapshot stored: ${result.objectKey}`));
+}
+
+export async function stateRestoreCommand(objectKey: string, opts: StateApiOptions) {
+  printPaperclipCliBanner();
+  p.intro(pc.bgCyan(pc.black(" paperclip state restore ")));
+  const result = await callStateApi("/instance/state-snapshots/restore", opts, { objectKey });
+  if (opts.json) console.log(JSON.stringify(result, null, 2));
+  else p.outro(pc.green(`Snapshot restored: ${objectKey}`));
+}

cli/src/index.ts

@@ -9,6 +9,7 @@ import { heartbeatRun } from "./commands/heartbeat-run.js";
 import { runCommand } from "./commands/run.js";
 import { bootstrapCeoInvite } from "./commands/auth-bootstrap-ceo.js";
 import { dbBackupCommand } from "./commands/db-backup.js";
+import { stateRestoreCommand, stateSnapshotCommand } from "./commands/state.js";
 import { registerEnvLabCommands } from "./commands/env-lab.js";
 import { registerContextCommands } from "./commands/client/context.js";
 import { registerCompanyCommands } from "./commands/client/company.js";
@@ -160,6 +161,21 @@ program
     await dbBackupCommand(opts);
   });
 
+const state = program.command("state").description("Inspect, snapshot, and restore instance state");
+state.command("snapshot")
+  .description("Create an encrypted instance-state snapshot")
+  .option("--api-url <url>", "Paperclip API base URL")
+  .option("--token <token>", "Board/admin bearer token")
+  .option("--json", "Print result as JSON")
+  .action(stateSnapshotCommand);
+state.command("restore")
+  .description("Restore an encrypted instance-state snapshot")
+  .argument("<object-key>", "Storage-provider object key returned by state snapshot")
+  .option("--api-url <url>", "Paperclip API base URL")
+  .option("--token <token>", "Board/admin bearer token")
+  .option("--json", "Print result as JSON")
+  .action(stateRestoreCommand);
+
 program
   .command("allowed-hostname")
   .description("Allow a hostname for authenticated/private mode access")

doc/DEVELOPING.md

@@ -811,6 +811,22 @@ DB backups are not full instance filesystem backups. For full local disaster
 recovery, also back up local storage files and the local encrypted secrets key if
 those providers are enabled.
 
+## Instance-State Snapshots
+
+Set `PAPERCLIP_STATE_SNAPSHOT_KEY` to a 32-byte key encoded as 64 hex characters
+or base64 to enable encrypted instance-state snapshots. The server writes them
+through the configured storage provider, runs them daily by default, and reports
+success/failure markers in `/api/health`.
+
+```sh
+pnpm paperclipai state snapshot
+pnpm paperclipai state restore 'instance-state/default/<snapshot>.tar.gz.enc'
+```
+
+Use `PAPERCLIP_STATE_SNAPSHOT_INTERVAL_MINUTES` to change the schedule and
+`PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS` to change the health warning threshold.
+The encryption key must be held separately from the instance being protected.
+
 ## Secrets in Dev
 
 Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config.

packages/shared/src/home-paths.ts

@@ -34,6 +34,24 @@ export function resolvePaperclipInstanceRoot(input: {
   return path.resolve(resolvePaperclipHomeDir(input.homeDir), "instances", resolvePaperclipInstanceId(input.instanceId));
 }
 
+export function resolvePaperclipHomePath(
+  input: { homeDir?: string } = {},
+  ...segments: string[]
+): string {
+  return path.resolve(resolvePaperclipHomeDir(input.homeDir), ...segments);
+}
+
+export function resolvePaperclipInstancePath(
+  input: { homeDir?: string; instanceId?: string } = {},
+  ...segments: string[]
+): string {
+  return path.resolve(resolvePaperclipInstanceRoot(input), ...segments);
+}
+
+export function resolveUserHomePath(...segments: string[]): string {
+  return path.resolve(os.homedir(), ...segments);
+}
+
 export function resolvePaperclipInstanceConfigPath(input: {
   homeDir?: string;
   instanceId?: string;

packages/shared/src/index.ts

@@ -1,4 +1,6 @@
 export { agentAdapterTypeSchema, optionalAgentAdapterTypeSchema } from "./adapter-type.js";
+export * from "./home-paths.js";
+export * from "./state-manifest.js";
 export {
   decisionEffectStalenessSchema,
   decisionOptionStyleSchema,

packages/shared/src/state-manifest.test.ts

@@ -0,0 +1,26 @@
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { STATE_MANIFEST } from "./state-manifest.js";
+
+describe("STATE_MANIFEST", () => {
+  it("covers every approved inventory class with absolute live-layout paths", () => {
+    const homeDir = path.resolve("/tmp/paperclip-manifest-home");
+    const expected = [
+      "cli_auth_context", "plugin_install_set", "adapter_state", "host_secrets", "host_maintenance",
+      "host_ephemeral", "instance_config", "database", "attachment_storage", "run_logs",
+      "local_backup_staging", "agent_instructions_bundle", "secrets_master_key", "codex_agent_home",
+      "runtime_materializations", "skill_bundles", "project_repositories", "execution_workspaces",
+      "instance_logs", "claude_memory", "claude_transcripts", "claude_runtime_state", "claude_cache",
+      "external_cli_homes",
+    ];
+
+    expect(STATE_MANIFEST.map((entry) => entry.id)).toEqual(expected);
+    for (const entry of STATE_MANIFEST) {
+      const paths = entry.resolve({ homeDir, instanceId: "live" });
+      expect(paths.length).toBeGreaterThan(0);
+      for (const resolved of paths) expect(path.isAbsolute(resolved)).toBe(true);
+    }
+    expect(STATE_MANIFEST.find((entry) => entry.id === "secrets_master_key")?.resolve({ homeDir, instanceId: "live" })[0])
+      .toBe(path.join(homeDir, "instances", "live", "secrets", "master.key"));
+  });
+});

packages/shared/src/state-manifest.ts

@@ -0,0 +1,57 @@
+import {
+  resolveDefaultEmbeddedPostgresDir,
+  resolveDefaultSecretsKeyFilePath,
+  resolveDefaultStorageDir,
+  resolvePaperclipHomePath,
+  resolvePaperclipInstanceConfigPath,
+  resolvePaperclipInstancePath,
+  resolveUserHomePath,
+} from "./home-paths.js";
+
+export type StateDisposition = "git" | "s3_bulk" | "s3_secret" | "db" | "cache" | "ephemeral";
+export type StateRedactionClass = "none" | "secret_refs" | "forbid";
+export type StateCopyConsistency = "plain" | "sqlite_backup";
+
+export type StateManifestContext = { homeDir?: string; instanceId?: string };
+export type StateClassEntry = {
+  id: string;
+  resolve(ctx: StateManifestContext): string[];
+  disposition: StateDisposition;
+  redact: StateRedactionClass;
+  consistency: StateCopyConsistency;
+  retention?: { days?: number };
+};
+
+const home = (ctx: StateManifestContext, ...segments: string[]) => resolvePaperclipHomePath(ctx, ...segments);
+const instance = (ctx: StateManifestContext, ...segments: string[]) => resolvePaperclipInstancePath(ctx, ...segments);
+
+export const STATE_MANIFEST: readonly StateClassEntry[] = [
+  { id: "cli_auth_context", resolve: (ctx) => [home(ctx, "auth.json"), home(ctx, "context.json")], disposition: "s3_secret", redact: "forbid", consistency: "plain" },
+  { id: "plugin_install_set", resolve: (ctx) => [home(ctx, "adapter-plugins.json"), home(ctx, "adapter-plugins"), home(ctx, "plugins")], disposition: "git", redact: "secret_refs", consistency: "plain" },
+  { id: "adapter_state", resolve: (ctx) => [home(ctx, "adapter-state")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain" },
+  { id: "host_secrets", resolve: (ctx) => [home(ctx, "secrets")], disposition: "s3_secret", redact: "forbid", consistency: "plain" },
+  { id: "host_maintenance", resolve: (ctx) => [home(ctx, "maintenance"), home(ctx, "hot-restart-report.json")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 30 } },
+  { id: "host_ephemeral", resolve: (ctx) => [home(ctx, "feedback-artifacts"), home(ctx, "scratch"), home(ctx, "tmp"), home(ctx, "worktrees"), home(ctx, "migration-backups")], disposition: "ephemeral", redact: "none", consistency: "plain", retention: { days: 7 } },
+  { id: "instance_config", resolve: (ctx) => [resolvePaperclipInstanceConfigPath(ctx), instance(ctx, ".env")], disposition: "git", redact: "secret_refs", consistency: "plain" },
+  { id: "database", resolve: (ctx) => [resolveDefaultEmbeddedPostgresDir(ctx)], disposition: "db", redact: "forbid", consistency: "plain" },
+  { id: "attachment_storage", resolve: (ctx) => [resolveDefaultStorageDir(ctx)], disposition: "s3_bulk", redact: "none", consistency: "plain" },
+  { id: "run_logs", resolve: (ctx) => [instance(ctx, "data", "run-logs")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 90 } },
+  { id: "local_backup_staging", resolve: (ctx) => [instance(ctx, "data", "backups"), instance(ctx, "data", "instance-backups")], disposition: "cache", redact: "forbid", consistency: "plain", retention: { days: 30 } },
+  { id: "agent_instructions_bundle", resolve: (ctx) => [instance(ctx, "companies", "*", "agents", "*", "instructions")], disposition: "git", redact: "secret_refs", consistency: "plain" },
+  { id: "secrets_master_key", resolve: (ctx) => [resolveDefaultSecretsKeyFilePath(ctx)], disposition: "s3_secret", redact: "forbid", consistency: "plain" },
+  { id: "codex_agent_home", resolve: (ctx) => [instance(ctx, "companies", "*", "codex-home")], disposition: "s3_bulk", redact: "forbid", consistency: "sqlite_backup" },
+  { id: "runtime_materializations", resolve: (ctx) => [instance(ctx, "companies", "*", "acp-engine"), instance(ctx, "companies", "*", "acpx-local"), instance(ctx, "companies", "*", "claude-prompt-cache")], disposition: "cache", redact: "secret_refs", consistency: "plain" },
+  { id: "skill_bundles", resolve: (ctx) => [instance(ctx, "skills", "*")], disposition: "git", redact: "secret_refs", consistency: "plain" },
+  { id: "project_repositories", resolve: (ctx) => [instance(ctx, "projects", "*", "*", "*")], disposition: "git", redact: "forbid", consistency: "plain" },
+  { id: "execution_workspaces", resolve: (ctx) => [instance(ctx, "workspaces", "*")], disposition: "ephemeral", redact: "forbid", consistency: "plain", retention: { days: 7 } },
+  { id: "instance_logs", resolve: (ctx) => [instance(ctx, "logs")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 30 } },
+  { id: "claude_memory", resolve: () => [resolveUserHomePath(".claude", "projects", "**", "memory", "**")], disposition: "git", redact: "secret_refs", consistency: "plain" },
+  { id: "claude_transcripts", resolve: () => [resolveUserHomePath(".claude", "projects", "**", "*.jsonl")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 90 } },
+  { id: "claude_runtime_state", resolve: () => [resolveUserHomePath(".claude", "history.jsonl"), resolveUserHomePath(".claude", "todos"), resolveUserHomePath(".claude", "session-env")], disposition: "s3_bulk", redact: "secret_refs", consistency: "plain", retention: { days: 30 } },
+  { id: "claude_cache", resolve: () => [resolveUserHomePath(".claude", "cache"), resolveUserHomePath(".claude", "debug"), resolveUserHomePath(".claude", "shell-snapshots"), resolveUserHomePath(".claude", "statsig")], disposition: "cache", redact: "none", consistency: "plain", retention: { days: 7 } },
+  { id: "external_cli_homes", resolve: () => [resolveUserHomePath(".codex"), resolveUserHomePath(".gemini")], disposition: "s3_secret", redact: "forbid", consistency: "plain" },
+] as const;
+
+export function getStateManifestEntry(id: string): StateClassEntry | undefined {
+  return STATE_MANIFEST.find((entry) => entry.id === id);
+}

server/src/__tests__/instance-state-snapshot.test.ts

@@ -0,0 +1,44 @@
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { Readable } from "node:stream";
+import { afterEach, describe, expect, it } from "vitest";
+import type { StateClassEntry } from "@paperclipai/shared";
+import type { StorageProvider } from "../storage/types.js";
+import { createAesStateSnapshotEncryptionProvider, createInstanceStateSnapshotService } from "../services/instance-state-snapshot.js";
+
+const tempDirs: string[] = [];
+afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); });
+
+describe("instance state snapshots", () => {
+  it("encrypts, stores, and restores manifest state", async () => {
+    const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-test-"));
+    tempDirs.push(homeDir);
+    const source = path.join(homeDir, "instances", "test", "config.json");
+    await fs.mkdir(path.dirname(source), { recursive: true });
+    await fs.writeFile(source, "before");
+    const objects = new Map<string, Buffer>();
+    const provider: StorageProvider = {
+      id: "local_disk",
+      async putObject(input) { objects.set(input.objectKey, Buffer.isBuffer(input.body) ? input.body : Buffer.from(await Readable.toArray(input.body))); },
+      async getObject(input) { const body = objects.get(input.objectKey); if (!body) throw new Error("missing"); return { stream: Readable.from(body), contentLength: body.length }; },
+      async headObject(input) { const body = objects.get(input.objectKey); return { exists: Boolean(body), contentLength: body?.length }; },
+      async deleteObject(input) { objects.delete(input.objectKey); },
+    };
+    const manifest: StateClassEntry[] = [{ id: "config", resolve: () => [source], disposition: "s3_secret", redact: "forbid", consistency: "plain" }];
+    const service = createInstanceStateSnapshotService({
+      storageProvider: provider,
+      encryptionProvider: createAesStateSnapshotEncryptionProvider(Buffer.alloc(32, 7)),
+      context: { homeDir, instanceId: "test" },
+      markerDir: path.join(homeDir, "markers"),
+      manifest,
+    });
+
+    const result = await service.runSnapshot();
+    expect(objects.get(result.objectKey)?.subarray(0, 8).toString()).toBe("PCSTATE1");
+    await fs.writeFile(source, "after");
+    await service.restoreSnapshot(result.objectKey);
+    expect(await fs.readFile(source, "utf8")).toBe("before");
+    expect(JSON.parse(await fs.readFile(path.join(homeDir, "markers", "state-snapshot.success.json"), "utf8"))).toMatchObject({ objectKey: result.objectKey });
+  });
+});

server/src/app.ts

@@ -65,6 +65,7 @@ import {
   instanceDatabaseBackupRoutes,
   type InstanceDatabaseBackupService,
 } from "./routes/instance-database-backups.js";
+import { instanceStateSnapshotRoutes, type InstanceStateSnapshotService } from "./routes/instance-state-snapshots.js";
 import { llmRoutes } from "./routes/llms.js";
 import { authRoutes } from "./routes/auth.js";
 import { assetRoutes } from "./routes/assets.js";
@@ -256,7 +257,9 @@ export async function createApp(
       }): Promise<unknown>;
     };
     databaseBackupService?: InstanceDatabaseBackupService;
+    stateSnapshotService?: InstanceStateSnapshotService;
     databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
+    stateSnapshotHealth?: { markerDir: string; enabled: boolean; maxAgeHours: number };
     deploymentMode: DeploymentMode;
     deploymentExposure: DeploymentExposure;
     allowedHostnames: string[];
@@ -373,6 +376,7 @@ export async function createApp(
       authReady: opts.authReady,
       companyDeletionEnabled: opts.companyDeletionEnabled,
       databaseBackupHealth: opts.databaseBackupHealth,
+      stateSnapshotHealth: opts.stateSnapshotHealth,
     }),
   );
   api.use(openApiRoutes());
@@ -464,6 +468,9 @@ export async function createApp(
   if (opts.databaseBackupService) {
     api.use(instanceDatabaseBackupRoutes(opts.databaseBackupService));
   }
+  if (opts.stateSnapshotService) {
+    api.use(instanceStateSnapshotRoutes(opts.stateSnapshotService));
+  }
   const pluginRegistry = pluginRegistryService(db);
   const eventBus = createPluginEventBus();
   setPluginEventBus(eventBus);

server/src/index.ts

@@ -81,6 +81,8 @@ import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runt
 import { isLoopbackHost, rewriteLoopbackUrlPort } from "./url-utils.js";
 import { createPluginWorkerManager } from "./services/plugin-worker-manager.js";
 import { createStorageServiceFromConfig } from "./storage/index.js";
+import { createStorageProviderFromConfig } from "./storage/provider-registry.js";
+import { createAesStateSnapshotEncryptionProvider, createInstanceStateSnapshotService } from "./services/instance-state-snapshot.js";
 import { printStartupBanner } from "./startup-banner.js";
 import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js";
 import { maybePersistWorktreeRuntimePorts } from "./worktree-config.js";
@@ -98,6 +100,7 @@ import type {
   InstanceDatabaseBackupRunResult,
   InstanceDatabaseBackupTrigger,
 } from "./routes/instance-database-backups.js";
+import { resolvePaperclipInstanceId, resolvePaperclipInstancePath } from "@paperclipai/shared";
 
 type BetterAuthSessionUser = {
   id: string;
@@ -637,6 +640,28 @@ export async function startServer(): Promise<StartedServer> {
 
   const uiMode = config.uiDevMiddleware ? "vite-dev" : config.serveUi ? "static" : "none";
   const storageService = createStorageServiceFromConfig(config);
+  const storageProvider = createStorageProviderFromConfig(config);
+  const stateSnapshotInstanceId = resolvePaperclipInstanceId();
+  const stateSnapshotKeyRaw = process.env.PAPERCLIP_STATE_SNAPSHOT_KEY?.trim();
+  const stateSnapshotKey = stateSnapshotKeyRaw
+    ? Buffer.from(stateSnapshotKeyRaw, /^[0-9a-f]{64}$/i.test(stateSnapshotKeyRaw) ? "hex" : "base64")
+    : null;
+  const stateSnapshotMarkerDir = resolvePaperclipInstancePath({ instanceId: stateSnapshotInstanceId }, "data", "instance-backups");
+  const stateSnapshotService = stateSnapshotKey
+    ? createInstanceStateSnapshotService({
+        storageProvider,
+        encryptionProvider: createAesStateSnapshotEncryptionProvider(stateSnapshotKey),
+        context: { instanceId: stateSnapshotInstanceId },
+        markerDir: stateSnapshotMarkerDir,
+      })
+    : null;
+  let stateSnapshotInFlight = false;
+  const runStateSnapshot = async () => {
+    if (!stateSnapshotService || stateSnapshotInFlight) return null;
+    stateSnapshotInFlight = true;
+    try { return await stateSnapshotService.runSnapshot(); }
+    finally { stateSnapshotInFlight = false; }
+  };
   const feedback = feedbackService(db as any, {
     shareClient: createFeedbackTraceShareClientFromConfig(config),
   });
@@ -738,6 +763,16 @@ export async function startServer(): Promise<StartedServer> {
         return result;
       },
     },
+    stateSnapshotService: stateSnapshotService
+      ? {
+          runSnapshot: async () => {
+            const result = await runStateSnapshot();
+            if (!result) throw conflict("Instance-state snapshot already in progress");
+            return result;
+          },
+          restoreSnapshot: stateSnapshotService.restoreSnapshot,
+        }
+      : undefined,
     databaseBackupHealth: config.databaseBackupEnabled
       ? {
           enabled: config.databaseBackupEnabled,
@@ -747,6 +782,11 @@ export async function startServer(): Promise<StartedServer> {
           alertFiles: databaseBackupAlertFiles,
         }
       : undefined,
+    stateSnapshotHealth: {
+      enabled: Boolean(stateSnapshotService),
+      markerDir: stateSnapshotMarkerDir,
+      maxAgeHours: Math.max(1, Number(process.env.PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS) || 48),
+    },
     deploymentMode: config.deploymentMode,
     deploymentExposure: config.deploymentExposure,
     allowedHostnames: config.allowedHostnames,
@@ -1413,6 +1453,14 @@ export async function startServer(): Promise<StartedServer> {
       });
     }, backupIntervalMs);
   }
+
+  if (stateSnapshotService) {
+    const intervalMinutes = Math.max(15, Number(process.env.PAPERCLIP_STATE_SNAPSHOT_INTERVAL_MINUTES) || 1440);
+    logger.info({ intervalMinutes, provider: storageProvider.id }, "Automatic encrypted instance-state snapshots enabled");
+    setInterval(() => { void runStateSnapshot().catch((err) => logger.error({ err }, "Automatic instance-state snapshot failed")); }, intervalMinutes * 60 * 1000);
+  } else {
+    logger.warn("Instance-state snapshots disabled: PAPERCLIP_STATE_SNAPSHOT_KEY is not configured");
+  }
   
   // Wait for external adapters to finish loading before accepting requests.
   // Without this, adapter type validation (assertKnownAdapterType) would

server/src/routes/health.ts

@@ -19,6 +19,7 @@ import {
   type InspectDatabaseBackupHealthOptions,
 } from "../services/database-backup-health.js";
 import { instanceSettingsService } from "../services/instance-settings.js";
+import { inspectInstanceStateSnapshotHealth } from "../services/instance-state-snapshot-health.js";
 import { serverVersion } from "../version.js";
 
 function shouldExposeFullHealthDetails(
@@ -83,6 +84,7 @@ export function healthRoutes(
     serverInfo?: ServerInfoSnapshot;
     databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
     runtimeEnv?: CloudInstanceEnv;
+    stateSnapshotHealth?: { markerDir: string; enabled: boolean; maxAgeHours: number };
   } = {
     deploymentMode: "local_trusted",
     deploymentExposure: "private",
@@ -239,7 +241,11 @@ export function healthRoutes(
     const databaseBackup = opts.databaseBackupHealth
       ? inspectDatabaseBackupHealth(opts.databaseBackupHealth)
       : undefined;
-    const warnings = databaseBackup?.warnings.length ? databaseBackup.warnings : undefined;
+    const stateSnapshot = opts.stateSnapshotHealth
+      ? inspectInstanceStateSnapshotHealth(opts.stateSnapshotHealth)
+      : undefined;
+    const combinedWarnings = [...(databaseBackup?.warnings ?? []), ...(stateSnapshot?.warnings ?? [])];
+    const warnings = combinedWarnings.length ? combinedWarnings : undefined;
 
     if (!exposeFullDetails) {
       const redactedDatabaseBackup = databaseBackup ? redactedDatabaseBackupHealth(databaseBackup) : undefined;
@@ -252,6 +258,7 @@ export function healthRoutes(
         bootstrapStatus,
         bootstrapInviteActive,
         ...(redactedDatabaseBackup ? { databaseBackup: redactedDatabaseBackup } : {}),
+        ...(stateSnapshot ? { stateSnapshot: { enabled: stateSnapshot.enabled, status: stateSnapshot.status, warnings: stateSnapshot.warnings } } : {}),
         ...(redactedWarnings ? { warnings: redactedWarnings } : {}),
         ...(devServer ? { devServer } : {}),
         ...(cloud ? { cloud } : {}),
@@ -274,6 +281,7 @@ export function healthRoutes(
       },
       serverInfo,
       ...(databaseBackup ? { databaseBackup } : {}),
+      ...(stateSnapshot ? { stateSnapshot } : {}),
       ...(warnings ? { warnings } : {}),
       ...(devServer ? { devServer } : {}),
       ...(cloud ? { cloud } : {}),

server/src/routes/instance-state-snapshots.ts

@@ -0,0 +1,24 @@
+import { Router } from "express";
+import { assertInstanceAdmin } from "./authz.js";
+import type { InstanceStateSnapshotResult } from "../services/instance-state-snapshot.js";
+
+export type InstanceStateSnapshotService = {
+  runSnapshot(): Promise<InstanceStateSnapshotResult>;
+  restoreSnapshot(objectKey: string): Promise<void>;
+};
+
+export function instanceStateSnapshotRoutes(service: InstanceStateSnapshotService) {
+  const router = Router();
+  router.post("/instance/state-snapshots", async (req, res) => {
+    assertInstanceAdmin(req);
+    res.status(201).json(await service.runSnapshot());
+  });
+  router.post("/instance/state-snapshots/restore", async (req, res) => {
+    assertInstanceAdmin(req);
+    const objectKey = typeof req.body?.objectKey === "string" ? req.body.objectKey.trim() : "";
+    if (!objectKey) return res.status(400).json({ error: "objectKey is required" });
+    await service.restoreSnapshot(objectKey);
+    res.status(200).json({ restored: true, objectKey });
+  });
+  return router;
+}

server/src/services/instance-state-snapshot-health.ts

@@ -0,0 +1,16 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import path from "node:path";
+
+export function inspectInstanceStateSnapshotHealth(opts: { markerDir: string; enabled: boolean; maxAgeHours: number; now?: Date }) {
+  const successPath = path.join(opts.markerDir, "state-snapshot.success.json");
+  const failurePath = path.join(opts.markerDir, "state-snapshot.failure");
+  const warnings: Array<{ code: string; message: string }> = [];
+  let latestSnapshot: Record<string, unknown> | null = null;
+  if (existsSync(successPath)) {
+    latestSnapshot = JSON.parse(readFileSync(successPath, "utf8"));
+    const ageHours = ((opts.now ?? new Date()).getTime() - statSync(successPath).mtimeMs) / 3_600_000;
+    if (ageHours > opts.maxAgeHours) warnings.push({ code: "instance_state_snapshot_stale", message: `Latest instance-state snapshot is ${Math.round(ageHours * 10) / 10}h old.` });
+  } else warnings.push({ code: "instance_state_snapshot_missing", message: "No successful instance-state snapshot marker found." });
+  if (existsSync(failurePath)) warnings.push({ code: "instance_state_snapshot_last_failure", message: readFileSync(failurePath, "utf8").trim().split(/\r?\n/)[0] });
+  return { enabled: opts.enabled, status: warnings.length ? "warning" : "ok", markerDir: opts.markerDir, latestSnapshot, warnings };
+}

server/src/services/instance-state-snapshot.ts

@@ -0,0 +1,179 @@
+import { createCipheriv, createDecipheriv, createHash } from "node:crypto";
+import { createReadStream, createWriteStream, promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { pipeline } from "node:stream/promises";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import {
+  STATE_MANIFEST,
+  resolvePaperclipHomeDir,
+  resolvePaperclipInstancePath,
+  type StateClassEntry,
+  type StateManifestContext,
+} from "@paperclipai/shared";
+import type { StorageProvider } from "../storage/types.js";
+
+const execFileAsync = promisify(execFile);
+const SNAPSHOT_MAGIC = Buffer.from("PCSTATE1");
+
+export interface StateSnapshotEncryptionProvider {
+  id: string;
+  encrypt(inputPath: string, outputPath: string): Promise<void>;
+  decrypt(inputPath: string, outputPath: string): Promise<void>;
+}
+
+export function createAesStateSnapshotEncryptionProvider(key: Buffer): StateSnapshotEncryptionProvider {
+  if (key.length !== 32) throw new Error("State snapshot encryption key must be exactly 32 bytes.");
+  return {
+    id: "aes-256-gcm",
+    async encrypt(inputPath, outputPath) {
+      const iv = Buffer.from(cryptoRandom(12));
+      const cipher = createCipheriv("aes-256-gcm", key, iv);
+      await fs.writeFile(outputPath, Buffer.concat([SNAPSHOT_MAGIC, iv]));
+      await pipeline(createReadStream(inputPath), cipher, createWriteStream(outputPath, { flags: "a" }));
+      await fs.appendFile(outputPath, cipher.getAuthTag());
+    },
+    async decrypt(inputPath, outputPath) {
+      const stat = await fs.stat(inputPath);
+      const handle = await fs.open(inputPath, "r");
+      try {
+        const header = Buffer.alloc(SNAPSHOT_MAGIC.length + 12);
+        await handle.read(header, 0, header.length, 0);
+        if (!header.subarray(0, SNAPSHOT_MAGIC.length).equals(SNAPSHOT_MAGIC)) throw new Error("Invalid state snapshot header.");
+        const tag = Buffer.alloc(16);
+        await handle.read(tag, 0, tag.length, stat.size - tag.length);
+        const decipher = createDecipheriv("aes-256-gcm", key, header.subarray(SNAPSHOT_MAGIC.length));
+        decipher.setAuthTag(tag);
+        await pipeline(createReadStream(inputPath, { start: header.length, end: stat.size - tag.length - 1 }), decipher, createWriteStream(outputPath));
+      } finally {
+        await handle.close();
+      }
+    },
+  };
+}
+
+function cryptoRandom(length: number): Uint8Array {
+  return globalThis.crypto.getRandomValues(new Uint8Array(length));
+}
+
+async function exists(target: string): Promise<boolean> {
+  try { await fs.access(target); return true; } catch { return false; }
+}
+
+async function expandPattern(pattern: string): Promise<string[]> {
+  const parsed = path.parse(pattern);
+  const segments = pattern.slice(parsed.root.length).split(path.sep);
+  let current = [parsed.root];
+  for (const segment of segments) {
+    if (!segment.includes("*")) {
+      current = current.map((base) => path.join(base, segment));
+      continue;
+    }
+    const next: string[] = [];
+    for (const base of current) {
+      if (!(await exists(base))) continue;
+      const names = await fs.readdir(base);
+      const regex = new RegExp(`^${segment.replace(/\./g, "\\.").replace(/\*\*/g, ".*").replace(/\*/g, "[^/]*")}$`);
+      for (const name of names) if (regex.test(name)) next.push(path.join(base, name));
+    }
+    current = next;
+  }
+  const found: string[] = [];
+  for (const candidate of current) if (await exists(candidate)) found.push(candidate);
+  return found;
+}
+
+async function copyEntry(entry: StateClassEntry, source: string, destination: string): Promise<void> {
+  const stat = await fs.stat(source);
+  await fs.mkdir(path.dirname(destination), { recursive: true });
+  if (entry.consistency === "sqlite_backup" && stat.isFile() && /\.(sqlite|sqlite3|db)$/i.test(source)) {
+    await execFileAsync("sqlite3", [source, `.backup '${destination.replace(/'/g, "''")}'`]);
+    return;
+  }
+  await fs.cp(source, destination, { recursive: true, preserveTimestamps: true });
+}
+
+export type InstanceStateSnapshotResult = {
+  objectKey: string;
+  sizeBytes: number;
+  sha256: string;
+  entryCount: number;
+  startedAt: string;
+  finishedAt: string;
+};
+
+export function createInstanceStateSnapshotService(opts: {
+  storageProvider: StorageProvider;
+  encryptionProvider: StateSnapshotEncryptionProvider;
+  context?: StateManifestContext;
+  markerDir?: string;
+  manifest?: readonly StateClassEntry[];
+}) {
+  const context = opts.context ?? {};
+  const homeRoot = resolvePaperclipHomeDir(context.homeDir);
+  const userRoot = os.homedir();
+  const markerDir = opts.markerDir ?? resolvePaperclipInstancePath(context, "data", "instance-backups");
+  const manifest = opts.manifest ?? STATE_MANIFEST;
+
+  async function runSnapshot(): Promise<InstanceStateSnapshotResult> {
+    const startedAt = new Date();
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-snapshot-"));
+    try {
+      const stageDir = path.join(tempDir, "stage");
+      let entryCount = 0;
+      for (const entry of manifest) {
+        if (entry.disposition === "db" || entry.disposition === "cache" || entry.disposition === "ephemeral") continue;
+        for (const pattern of entry.resolve(context)) {
+          for (const source of await expandPattern(pattern)) {
+            const root = source === homeRoot || source.startsWith(homeRoot + path.sep) ? homeRoot : userRoot;
+            const rootName = root === homeRoot ? "paperclip-home" : "user-home";
+            await copyEntry(entry, source, path.join(stageDir, rootName, path.relative(root, source)));
+            entryCount += 1;
+          }
+        }
+      }
+      await fs.writeFile(path.join(stageDir, "snapshot-manifest.json"), JSON.stringify({ version: 1, createdAt: startedAt.toISOString(), entries: manifest.map(({ resolve: _resolve, ...entry }) => entry) }, null, 2));
+      const archivePath = path.join(tempDir, "snapshot.tar.gz");
+      await execFileAsync("tar", ["-czf", archivePath, "-C", stageDir, "."]);
+      const encryptedPath = `${archivePath}.enc`;
+      await opts.encryptionProvider.encrypt(archivePath, encryptedPath);
+      const body = await fs.readFile(encryptedPath);
+      const objectKey = `instance-state/${context.instanceId ?? "default"}/${startedAt.toISOString().replace(/[:.]/g, "-")}.tar.gz.enc`;
+      await opts.storageProvider.putObject({ objectKey, body, contentType: "application/vnd.paperclip.state-snapshot", contentLength: body.length });
+      const result = { objectKey, sizeBytes: body.length, sha256: createHash("sha256").update(body).digest("hex"), entryCount, startedAt: startedAt.toISOString(), finishedAt: new Date().toISOString() };
+      await fs.mkdir(markerDir, { recursive: true });
+      await fs.writeFile(path.join(markerDir, "state-snapshot.success.json"), JSON.stringify(result));
+      await fs.rm(path.join(markerDir, "state-snapshot.failure"), { force: true });
+      return result;
+    } catch (error) {
+      await fs.mkdir(markerDir, { recursive: true });
+      await fs.writeFile(path.join(markerDir, "state-snapshot.failure"), `${new Date().toISOString()} ${error instanceof Error ? error.message : String(error)}\n`);
+      throw error;
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  }
+
+  async function restoreSnapshot(objectKey: string): Promise<void> {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-restore-"));
+    try {
+      const encryptedPath = path.join(tempDir, "snapshot.enc");
+      const object = await opts.storageProvider.getObject({ objectKey });
+      await pipeline(object.stream, createWriteStream(encryptedPath));
+      const archivePath = path.join(tempDir, "snapshot.tar.gz");
+      await opts.encryptionProvider.decrypt(encryptedPath, archivePath);
+      const extractDir = path.join(tempDir, "extract");
+      await fs.mkdir(extractDir);
+      await execFileAsync("tar", ["-xzf", archivePath, "-C", extractDir, "--no-same-owner", "--no-same-permissions"]);
+      for (const [name, root] of [["paperclip-home", homeRoot], ["user-home", userRoot]] as const) {
+        const source = path.join(extractDir, name);
+        if (await exists(source)) await fs.cp(source, root, { recursive: true, force: true });
+      }
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  }
+
+  return { runSnapshot, restoreSnapshot };
+}