From 09696ad5d7acb9fb88361ea74e99bcc00cc5c3c2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 21 Jul 2026 03:06:33 +0000 Subject: [PATCH] feat: integrate managed state repo --- cli/src/__tests__/state.test.ts | 50 +++++++ cli/src/commands/state.ts | 15 +- cli/src/index.ts | 8 +- doc/DEVELOPING.md | 25 ++++ .../src/__tests__/state-repo-routes.test.ts | 53 ++++++++ server/src/__tests__/state-repo.test.ts | 35 ++++- server/src/app.ts | 10 +- server/src/index.ts | 50 ++++++- server/src/routes/agents.ts | 18 +++ server/src/routes/company-skills.ts | 28 +++- server/src/routes/state-repo.ts | 64 +++++++++ server/src/services/state-repo.ts | 128 +++++++++++++++--- 12 files changed, 454 insertions(+), 30 deletions(-) create mode 100644 cli/src/__tests__/state.test.ts create mode 100644 server/src/__tests__/state-repo-routes.test.ts create mode 100644 server/src/routes/state-repo.ts diff --git a/cli/src/__tests__/state.test.ts b/cli/src/__tests__/state.test.ts new file mode 100644 index 00000000000..afe026647e6 --- /dev/null +++ b/cli/src/__tests__/state.test.ts @@ -0,0 +1,50 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../utils/banner.js", () => ({ printPaperclipCliBanner: vi.fn() })); +vi.mock("@clack/prompts", () => ({ intro: vi.fn(), outro: vi.fn() })); + +import { stateRestoreCommand } from "../commands/state.js"; + +const originalEnv = process.env; + +describe("state restore command", () => { + beforeEach(() => { + process.env = { ...originalEnv }; + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ restored: ["agents/a/AGENTS.md"], dryRun: true }), + })); + vi.spyOn(console, "log").mockImplementation(() => undefined); + }); + + afterEach(() => { + process.env = originalEnv; + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("calls the company restore API for --from-git", async () => { + await stateRestoreCommand(undefined, { + apiUrl: "http://paperclip.test/api", + token: "token", + json: true, + fromGit: "/backup/state.git", + companyId: "company-1", + ref: "refs/heads/main", + dryRun: true, + }); + + expect(fetch).toHaveBeenCalledWith( + "http://paperclip.test/api/companies/company-1/state-repo/restore", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ source: "/backup/state.git", ref: "refs/heads/main", dryRun: true }), + }), + ); + }); + + it("requires a company for git restores", async () => { + delete process.env.PAPERCLIP_COMPANY_ID; + await expect(stateRestoreCommand(undefined, { fromGit: "/backup/state.git" })).rejects.toThrow("--company-id"); + }); +}); diff --git a/cli/src/commands/state.ts b/cli/src/commands/state.ts index 204dd6cba7f..e8dc42cf21d 100644 --- a/cli/src/commands/state.ts +++ b/cli/src/commands/state.ts @@ -25,10 +25,19 @@ export async function stateSnapshotCommand(opts: StateApiOptions) { else p.outro(pc.green(`Snapshot stored: ${result.objectKey}`)); } -export async function stateRestoreCommand(objectKey: string, opts: StateApiOptions) { +export async function stateRestoreCommand(objectKey: string | undefined, opts: StateApiOptions & { fromGit?: string; companyId?: string; ref?: string; dryRun?: boolean }) { printPaperclipCliBanner(); p.intro(pc.bgCyan(pc.black(" paperclip state restore "))); - const result = await callStateApi("/instance/state-snapshots/restore", opts, { objectKey }); + const companyId = opts.companyId || process.env.PAPERCLIP_COMPANY_ID; + if (opts.fromGit && !companyId) throw new Error("--company-id or PAPERCLIP_COMPANY_ID is required with --from-git"); + if (!opts.fromGit && !objectKey) throw new Error("object-key is required unless --from-git is used"); + const result = opts.fromGit + ? await callStateApi(`/companies/${companyId}/state-repo/restore`, opts, { + source: opts.fromGit, + ref: opts.ref, + dryRun: opts.dryRun, + }) + : await callStateApi("/instance/state-snapshots/restore", opts, { objectKey }); if (opts.json) console.log(JSON.stringify(result, null, 2)); - else p.outro(pc.green(`Snapshot restored: ${objectKey}`)); + else p.outro(pc.green(opts.fromGit ? `Git state restored: ${opts.fromGit}` : `Snapshot restored: ${objectKey}`)); } diff --git a/cli/src/index.ts b/cli/src/index.ts index a31513ad6e1..800d32cc469 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -169,8 +169,12 @@ state.command("snapshot") .option("--json", "Print result as JSON") .action(stateSnapshotCommand); state.command("restore") - .description("Restore an encrypted instance-state snapshot") - .argument("", "Storage-provider object key returned by state snapshot") + .description("Restore an encrypted snapshot or company state from git") + .argument("[object-key]", "Storage-provider object key returned by state snapshot") + .option("--from-git ", "Restore company state from a bare repo or git bundle") + .option("--company-id ", "Company to restore (defaults to PAPERCLIP_COMPANY_ID)") + .option("--ref ", "Git ref to restore", "main") + .option("--dry-run", "List files without writing them") .option("--api-url ", "Paperclip API base URL") .option("--token ", "Board/admin bearer token") .option("--json", "Print result as JSON") diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7b2fa4e586f..f3010115465 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -827,6 +827,31 @@ Use `PAPERCLIP_STATE_SNAPSHOT_INTERVAL_MINUTES` to change the schedule and `PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS` to change the health warning threshold. The encryption key must be held separately from the instance being protected. +## Managed State Repositories + +Paperclip keeps a bare git repository per company under the instance +`state-repos/` directory. Agent-instruction and company-skill edits commit +immediately with actor attribution; Claude memory markdown is detected with a +30-second debounce and a daily sweep captures any remaining drift. + +Company mirror remotes are optional and configured with +`PAPERCLIP_STATE_REPO_MIRRORS_JSON`. Tokens are resolved from company secrets, +not stored inline: + +```json +{ + "": { + "url": "https://github.com/example/company-state.git", + "secretRef": { "secretId": "", "version": "latest" } + } +} +``` + +Use `GET /api/companies/:companyId/state-repo/bundle` for a company-scoped git +bundle, `POST .../mirror/test` and `GET .../health` for mirror diagnostics, and +`paperclipai state restore --from-git --company-id ` to +restore tracked files. Add `--dry-run` to inspect the restore first. + ## Secrets in Dev Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config. diff --git a/server/src/__tests__/state-repo-routes.test.ts b/server/src/__tests__/state-repo-routes.test.ts new file mode 100644 index 00000000000..26de2c89539 --- /dev/null +++ b/server/src/__tests__/state-repo-routes.test.ts @@ -0,0 +1,53 @@ +import express from "express"; +import request from "supertest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { stateRepoRoutes } from "../routes/state-repo.js"; + +const mockLogActivity = vi.hoisted(() => vi.fn()); +vi.mock("../services/activity-log.js", () => ({ logActivity: mockLogActivity })); + +function createApp(service: Record>) { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { type: "board", userId: "board-user", source: "local_implicit" }; + next(); + }); + app.use("/api", stateRepoRoutes({} as never, service as never, "/tmp")); + return app; +} + +describe("state repo routes", () => { + beforeEach(() => mockLogActivity.mockReset()); + + it("returns company-scoped mirror health", async () => { + const service = { health: vi.fn().mockResolvedValue({ configured: true, healthy: true }) }; + const response = await request(createApp(service)).get("/api/companies/company-1/state-repo/health"); + expect(response.status).toBe(200); + expect(response.body).toEqual({ configured: true, healthy: true }); + expect(service.health).toHaveBeenCalledWith("company-1"); + }); + + it("tests the configured mirror", async () => { + const service = { + testMirror: vi.fn().mockResolvedValue(undefined), + health: vi.fn().mockResolvedValue({ configured: true, healthy: true }), + }; + const response = await request(createApp(service)).post("/api/companies/company-1/state-repo/mirror/test"); + expect(response.status).toBe(200); + expect(service.testMirror).toHaveBeenCalledWith("company-1"); + }); + + it("restores state and records activity", async () => { + const service = { restore: vi.fn().mockResolvedValue({ restored: ["agents/a/AGENTS.md"], dryRun: false }) }; + const response = await request(createApp(service)) + .post("/api/companies/company-1/state-repo/restore") + .send({ source: "/backup/state.git", ref: "main" }); + expect(response.status).toBe(200); + expect(service.restore).toHaveBeenCalledWith("company-1", "/backup/state.git", "main", false); + expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + companyId: "company-1", + action: "company.state_repo_restored", + })); + }); +}); diff --git a/server/src/__tests__/state-repo.test.ts b/server/src/__tests__/state-repo.test.ts index e496153e0a9..4144ff1f286 100644 --- a/server/src/__tests__/state-repo.test.ts +++ b/server/src/__tests__/state-repo.test.ts @@ -37,16 +37,43 @@ describe("state repo service", () => { expect((await exec("git", ["--git-dir", remote, "rev-parse", "main"])).stdout.trim()).toMatch(/^[0-9a-f]{40}$/); const bundle = path.join(homeDir, "state.bundle"); await service.exportBundle("company-1", bundle); - const restoredRepo = path.join(homeDir, "restored.git"); - await exec("git", ["init", "--bare", restoredRepo]); - await exec("git", ["--git-dir", restoredRepo, "fetch", bundle, "main:main"]); await fs.rm(path.join(instance, "companies", "company-1"), { recursive: true, force: true }); await fs.rm(path.join(instance, "skills", "company-1"), { recursive: true, force: true }); - const restored = await service.restore("company-1", restoredRepo); + const restored = await service.restore("company-1", bundle); expect(restored.restored).toContain("agents/agent-1/AGENTS.md"); expect(await fs.readFile(path.join(instructions, "AGENTS.md"), "utf8")).toBe("# Initial\n"); expect(await fs.readFile(path.join(skill, "SKILL.md"), "utf8")).toBe("# Updated skill\n"); await fs.writeFile(path.join(instructions, "AGENTS.md"), "ghp_abcdefghijklmnopqrstuvwxyz123456\n"); await expect(service.commit({ companyId: "company-1", actor: { name: "Fable", email: "agent+fable@paperclip.invalid" }, message: "bad" })).rejects.toThrow("secret scan blocked"); }); + + it("debounces Claude memory changes into an automatic commit", async () => { + const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-memory-")); + const instance = path.join(homeDir, "instances", "test"); + const memory = path.join(homeDir, ".claude", "projects", "workspace", "memory"); + await fs.mkdir(memory, { recursive: true }); + await fs.writeFile(path.join(memory, "MEMORY.md"), "# Initial\n"); + const service = createStateRepoService({ + homeDir, + instanceId: "test", + markerDir: path.join(instance, "health"), + resolveMemorySources: async () => [{ agentId: "agent-1", root: memory }], + }); + await service.commit({ companyId: "company-1", actor: { name: "Setup", email: "setup@paperclip.invalid" }, message: "initial" }); + const stop = service.startWatcher({ listCompanyIds: async () => ["company-1"], debounceMs: 20, sweepMs: 60_000 }); + await new Promise((resolve) => setTimeout(resolve, 30)); + await fs.writeFile(path.join(memory, "MEMORY.md"), "# Updated\n"); + await new Promise((resolve) => setTimeout(resolve, 90)); + stop(); + const repo = service.repoPathFor("company-1"); + const log = (await exec("git", ["--git-dir", repo, "log", "--format=%s"])).stdout; + expect(log).toContain("claude-memory: capture external changes"); + const content = (await exec("git", ["--git-dir", repo, "show", "main:companies/company-1/agents/agent-1/memory/MEMORY.md"])).stdout; + expect(content).toBe("# Updated\n"); + const bundle = path.join(homeDir, "memory.bundle"); + await service.exportBundle("company-1", bundle); + await fs.rm(path.join(memory, "MEMORY.md")); + await service.restore("company-1", bundle); + expect(await fs.readFile(path.join(memory, "MEMORY.md"), "utf8")).toBe("# Updated\n"); + }); }); diff --git a/server/src/app.ts b/server/src/app.ts index 8a22cf4f08a..a9bc87631db 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -21,6 +21,8 @@ import { healthRoutes } from "./routes/health.js"; import { cloudRoutes } from "./routes/cloud.js"; import { companyRoutes } from "./routes/companies.js"; import { companySkillRoutes } from "./routes/company-skills.js"; +import { stateRepoRoutes } from "./routes/state-repo.js"; +import type { StateRepoService } from "./services/state-repo.js"; import { companySkillPolicyRoutes } from "./routes/company-skill-policy.js"; import { inboxAgentPolicyRoutes } from "./routes/inbox-agent-policy.js"; import { builtInAgentRoutes } from "./routes/built-in-agents.js"; @@ -260,6 +262,8 @@ export async function createApp( stateSnapshotService?: InstanceStateSnapshotService; databaseBackupHealth?: InspectDatabaseBackupHealthOptions; stateSnapshotHealth?: { markerDir: string; enabled: boolean; maxAgeHours: number }; + stateRepoService?: StateRepoService; + stateRepoMarkerDir?: string; deploymentMode: DeploymentMode; deploymentExposure: DeploymentExposure; allowedHostnames: string[]; @@ -384,7 +388,7 @@ export async function createApp( api.use("/companies", companyRoutes(db, opts.storageService)); api.use(llmRoutes(db)); api.use(folderRoutes(db)); - api.use(companySkillRoutes(db)); + api.use(companySkillRoutes(db, { stateRepo: opts.stateRepoService })); api.use(companySkillPolicyRoutes(db)); api.use(inboxAgentPolicyRoutes(db)); api.use(builtInAgentRoutes(db)); @@ -414,6 +418,7 @@ export async function createApp( api.use( agentRoutes(db, { pluginWorkerManager: workerManager, + stateRepo: opts.stateRepoService, deploymentMode: opts.deploymentMode, confidentialProxyAllowlist: setupTokenLoginProxyAllowlist, onSetupTokenLoginService: (service) => { @@ -426,6 +431,9 @@ export async function createApp( }, }), ); + if (opts.stateRepoService && opts.stateRepoMarkerDir) { + api.use(stateRepoRoutes(db, opts.stateRepoService, opts.stateRepoMarkerDir)); + } api.use(assetRoutes(db, opts.storageService)); api.use(projectRoutes(db)); api.use(caseRoutes(db, opts.storageService)); diff --git a/server/src/index.ts b/server/src/index.ts index 4278b353685..099b9226268 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -26,6 +26,7 @@ import { runDatabaseBackup, authUsers, companies, + agents, companyMemberships, instanceUserRoles, } from "@paperclipai/db"; @@ -62,6 +63,7 @@ import { routineService, statusCardService, toolAccessService, + secretService, } from "./services/index.js"; import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js"; import { createSecretProposalsService } from "./services/secret-proposals.js"; @@ -71,6 +73,7 @@ import { createCodexDeviceLoginReaper, createProductionLoginSessionReaperRuntime, } from "./services/codex-device-login-reaper.js"; +import { createStateRepoService } from "./services/state-repo.js"; import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js"; import { parseAdapterRegistryEnv, @@ -101,7 +104,7 @@ import type { InstanceDatabaseBackupRunResult, InstanceDatabaseBackupTrigger, } from "./routes/instance-database-backups.js"; -import { resolvePaperclipInstanceId, resolvePaperclipInstancePath } from "@paperclipai/shared"; +import { resolvePaperclipInstanceId, resolvePaperclipInstancePath, resolveUserHomePath } from "@paperclipai/shared"; type BetterAuthSessionUser = { id: string; @@ -758,6 +761,45 @@ export async function startServer(): Promise { // document parsed fail-closed above (`plugins.autoInstall`). Absent env means // self-hosted: createApp falls back to its built-in kubernetes-only default. const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null; + const stateRepoMarkerDir = resolvePaperclipInstancePath({ instanceId: stateSnapshotInstanceId }, "health"); + const stateRepoMirrors = (() => { + try { + return JSON.parse(process.env.PAPERCLIP_STATE_REPO_MIRRORS_JSON || "{}") as Record; + } catch { + logger.warn("Ignoring invalid PAPERCLIP_STATE_REPO_MIRRORS_JSON"); + return {}; + } + })(); + const stateRepoService = createStateRepoService({ + instanceId: stateSnapshotInstanceId, + repoPerCompany: true, + markerDir: stateRepoMarkerDir, + resolveMirror: async (companyId) => { + const configured = stateRepoMirrors[companyId]; + if (!configured?.url) return null; + const ref = configured.secretRef; + const token = ref?.secretId + ? await secretService(db as any).resolveSecretValue(companyId, ref.secretId, ref.version ?? "latest", { + consumerType: "system", + consumerId: "state-repo-mirror", + configPath: "stateRepo.remote.secretRef", + }) + : undefined; + return { url: configured.url, token }; + }, + resolveMemorySources: async (companyId) => { + const rows = await db.select({ id: agents.id, adapterType: agents.adapterType, adapterConfig: agents.adapterConfig }) + .from(agents) + .where(eq(agents.companyId, companyId)); + const claudeRoot = process.env.CLAUDE_CONFIG_DIR?.trim() || resolveUserHomePath(".claude"); + return rows.flatMap((agent) => { + if (agent.adapterType !== "claude_local") return []; + const cwd = typeof agent.adapterConfig?.cwd === "string" ? agent.adapterConfig.cwd : null; + if (!cwd) return []; + return [{ agentId: agent.id, root: resolve(claudeRoot, "projects", cwd.replace(/[^a-zA-Z0-9-]/g, "-"), "memory") }]; + }); + }, + }); const app = await createApp(db as any, { uiMode, serverPort: listenPort, @@ -796,6 +838,8 @@ export async function startServer(): Promise { markerDir: stateSnapshotMarkerDir, maxAgeHours: Math.max(1, Number(process.env.PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS) || 48), }, + stateRepoService, + stateRepoMarkerDir, deploymentMode: config.deploymentMode, deploymentExposure: config.deploymentExposure, allowedHostnames: config.allowedHostnames, @@ -811,6 +855,9 @@ export async function startServer(): Promise { managedPluginAutoInstall, }); const server = createServer(app as unknown as Parameters[0]); + const stopStateRepoWatcher = stateRepoService.startWatcher({ + listCompanyIds: async () => (await db.select({ id: companies.id }).from(companies)).map((company) => company.id), + }); // Increase keep-alive timeouts to safely outlive default idle timeouts // of common reverse proxies and load balancers (like AWS ALB, Nginx, or Traefik). @@ -1556,6 +1603,7 @@ export async function startServer(): Promise { { const shutdown = async (signal: "SIGINT" | "SIGTERM") => { await systemdNotify(["--stopping", `--status=Stopping after ${signal}`]); + stopStateRepoWatcher(); heartbeatSchedulerStopped = true; if (heartbeatSchedulerInterval) { clearInterval(heartbeatSchedulerInterval); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 7321c387d2e..bc7a0e4e82c 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -2,6 +2,7 @@ import { Router, type NextFunction, type Request, type Response } from "express" import { generateKeyPairSync, randomUUID } from "node:crypto"; import path from "node:path"; import type { Db } from "@paperclipai/db"; +import type { StateRepoService } from "../services/state-repo.js"; import { agents as agentsTable, companies, heartbeatRuns, issues as issuesTable, projects as projectsTable } from "@paperclipai/db"; import { and, desc, eq, inArray, not, sql } from "drizzle-orm"; import { @@ -206,6 +207,7 @@ export function agentRoutes( db: Db, options: { pluginWorkerManager?: PluginWorkerManager; + stateRepo?: StateRepoService; /** The active deployment mode. The confidential transport guard reads it. */ deploymentMode?: DeploymentMode; /** @@ -3523,6 +3525,14 @@ export function agentRoutes( clearLegacyPromptTemplate: req.body.clearLegacyPromptTemplate === true, }, }); + await options.stateRepo?.commit({ + companyId: existing.companyId, + actor: { + name: actor.actorType === "agent" ? existing.name : "Board User", + email: `${actor.actorType}+${actor.actorId}@paperclip.invalid`, + }, + message: `agent-instructions: update ${existing.name}`, + }); res.json(result.file); }); @@ -3555,6 +3565,14 @@ export function agentRoutes( path: relativePath, }, }); + await options.stateRepo?.commit({ + companyId: existing.companyId, + actor: { + name: actor.actorType === "agent" ? existing.name : "Board User", + email: `${actor.actorType}+${actor.actorId}@paperclip.invalid`, + }, + message: `agent-instructions: delete ${relativePath} for ${existing.name}`, + }); res.json(result.bundle); }); diff --git a/server/src/routes/company-skills.ts b/server/src/routes/company-skills.ts index 1aaba318e31..74dd8619003 100644 --- a/server/src/routes/company-skills.ts +++ b/server/src/routes/company-skills.ts @@ -1,5 +1,6 @@ import { Router, type Request } from "express"; import type { Db } from "@paperclipai/db"; +import type { StateRepoService } from "../services/state-repo.js"; import { catalogSkillListQuerySchema, companySkillCommentCreateSchema, @@ -83,7 +84,7 @@ type SkillPolicyResourceInput = | Promise | (() => SkillPolicyEvaluationResource | Promise); -export function companySkillRoutes(db: Db) { +export function companySkillRoutes(db: Db, options: { stateRepo?: StateRepoService } = {}) { const router = Router(); const access = accessService(db); const svc = companySkillService(db); @@ -1031,6 +1032,11 @@ export function companySkillRoutes(db: Db) { name: result.name, }, }); + await options.stateRepo?.commit({ + companyId, + actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` }, + message: `skill: create ${result.slug}`, + }); res.status(201).json(result); }, @@ -1062,6 +1068,11 @@ export function companySkillRoutes(db: Db) { sharingScope: result.sharingScope, }, }); + await options.stateRepo?.commit({ + companyId, + actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` }, + message: `skill: update ${result.slug}`, + }); res.json(result); }, @@ -1098,6 +1109,11 @@ export function companySkillRoutes(db: Db) { markdown: result.markdown, }, }); + await options.stateRepo?.commit({ + companyId, + actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` }, + message: `skill: update ${result.path}`, + }); res.json(result); }, @@ -1129,6 +1145,11 @@ export function companySkillRoutes(db: Db) { deletedPaths: result.deletedPaths, }, }); + await options.stateRepo?.commit({ + companyId, + actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` }, + message: `skill: delete ${result.path}`, + }); res.json(result); }, @@ -1284,6 +1305,11 @@ export function companySkillRoutes(db: Db) { name: result.name, }, }); + await options.stateRepo?.commit({ + companyId, + actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` }, + message: `skill: delete ${result.slug}`, + }); res.json(result); }); diff --git a/server/src/routes/state-repo.ts b/server/src/routes/state-repo.ts new file mode 100644 index 00000000000..e7a259e14b9 --- /dev/null +++ b/server/src/routes/state-repo.ts @@ -0,0 +1,64 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { Router } from "express"; +import type { Db } from "@paperclipai/db"; +import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js"; +import type { StateRepoService } from "../services/state-repo.js"; +import { logActivity } from "../services/activity-log.js"; + +export function stateRepoRoutes(db: Db, service: StateRepoService, markerDir: string) { + const router = Router(); + + router.get("/companies/:companyId/state-repo/bundle", async (req, res) => { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + await fs.mkdir(markerDir, { recursive: true }); + const outputPath = path.join(markerDir, `state-repo-${companyId}-${Date.now()}.bundle`); + await service.exportBundle(companyId, outputPath); + res.download(outputPath, `paperclip-state-${companyId}.bundle`, () => void fs.rm(outputPath, { force: true })); + }); + + router.post("/companies/:companyId/state-repo/mirror/test", async (req, res) => { + const companyId = req.params.companyId as string; + assertBoard(req); + assertCompanyAccess(req, companyId); + await service.testMirror(companyId); + res.json(await service.health(companyId)); + }); + + router.get("/companies/:companyId/state-repo/health", async (req, res) => { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + res.json(await service.health(companyId)); + }); + + router.post("/companies/:companyId/state-repo/restore", async (req, res) => { + const companyId = req.params.companyId as string; + assertBoard(req); + assertCompanyAccess(req, companyId); + const source = typeof req.body?.source === "string" ? req.body.source.trim() : ""; + if (!source) { + res.status(422).json({ error: "source is required" }); + return; + } + const result = await service.restore(companyId, source, req.body?.ref || "main", req.body?.dryRun === true); + if (!result.dryRun) { + const actor = getActorInfo(req); + await logActivity(db, { + companyId, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: actor.agentId, + runId: actor.runId, + agentApiKeyId: actor.agentApiKeyId, + action: "company.state_repo_restored", + entityType: "company", + entityId: companyId, + details: { source, ref: req.body?.ref || "main", restoredCount: result.restored.length }, + }); + } + res.json(result); + }); + + return router; +} diff --git a/server/src/services/state-repo.ts b/server/src/services/state-repo.ts index ff27d5da446..0ba5dc52d6c 100644 --- a/server/src/services/state-repo.ts +++ b/server/src/services/state-repo.ts @@ -28,6 +28,7 @@ const SECRET_PATTERNS = [ ] as const; export type StateRepoActor = { name: string; email: string }; +export type StateRepoMemorySource = { agentId: string; root: string }; type Entry = { oid: string; mode: string }; async function git(repo: string, args: string[], input?: string | Buffer) { @@ -102,6 +103,7 @@ export function createStateRepoService(options: { repoPerCompany?: boolean; markerDir: string; resolveMirror?: (companyId: string) => Promise<{ url: string; token?: string } | null>; + resolveMemorySources?: (companyId: string) => Promise; knownSecrets?: () => Promise; }) { let queue = Promise.resolve(); @@ -131,6 +133,13 @@ export function createStateRepoService(options: { for (const [relative, content] of await walk(path.join(instance, "skills", companyId))) { files.set(`companies/${companyId}/skills/${relative}`, content); } + for (const source of await options.resolveMemorySources?.(companyId) ?? []) { + for (const [relative, content] of await walk(source.root)) { + if (relative.endsWith(".md")) { + files.set(`companies/${companyId}/agents/${source.agentId}/memory/${relative}`, content); + } + } + } return files; } async function commitNow(input: { companyId: string; actor: StateRepoActor; message: string }) { @@ -166,12 +175,12 @@ export function createStateRepoService(options: { }); const commit = stdout.trim(); await git(repo, ["update-ref", "refs/heads/main", commit, ...(parent ? [parent] : [])]); - mirrorQueue = mirrorQueue.then(() => pushMirror(input.companyId)).catch(() => undefined); + mirrorQueue = mirrorQueue.then(async () => { await pushMirror(input.companyId); }).catch(() => undefined); return commit; } async function pushMirror(companyId: string) { const mirror = await options.resolveMirror?.(companyId); - if (!mirror) return; + if (!mirror) return false; const repo = repoPathFor(companyId); try { const args = ["--git-dir", repo]; @@ -181,6 +190,7 @@ export function createStateRepoService(options: { await fs.mkdir(options.markerDir, { recursive: true }); await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.success.json`), JSON.stringify({ pushedAt: new Date().toISOString() })); await fs.rm(path.join(options.markerDir, `state-repo-${companyId}.failure`), { force: true }); + return true; } catch (error) { await fs.mkdir(options.markerDir, { recursive: true }); await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.failure`), error instanceof Error ? error.message : String(error)); @@ -194,7 +204,17 @@ export function createStateRepoService(options: { queue = result.then(() => undefined, () => undefined); return result; }, - async testMirror(companyId: string) { await mirrorQueue; await pushMirror(companyId); }, + async testMirror(companyId: string) { + await mirrorQueue; + if (!await pushMirror(companyId)) throw new Error("State repo mirror is not configured"); + }, + async health(companyId: string) { + const successPath = path.join(options.markerDir, `state-repo-${companyId}.success.json`); + const failurePath = path.join(options.markerDir, `state-repo-${companyId}.failure`); + const success = await fs.readFile(successPath, "utf8").then(JSON.parse).catch(() => null); + const failure = await fs.readFile(failurePath, "utf8").catch(() => null); + return { configured: Boolean(await options.resolveMirror?.(companyId)), healthy: !failure, success, failure }; + }, async exportBundle(companyId: string, outputPath: string) { const repo = repoPathFor(companyId); await ensureRepo(repo); @@ -203,22 +223,94 @@ export function createStateRepoService(options: { async restore(companyId: string, source: string, ref = "main", dryRun = false) { const instance = resolvePaperclipInstancePath({ homeDir: options.homeDir, instanceId: options.instanceId }); const companyRoot = path.join(instance, "companies", companyId); - const { stdout } = await run("git", ["--git-dir", source, "ls-tree", "-r", ref, `companies/${companyId}`]); - const restored: string[] = []; - for (const line of stdout.trim().split("\n").filter(Boolean)) { - const match = line.match(/^\d+ blob ([0-9a-f]+)\tcompanies\/[^/]+\/(agents\/([^/]+)\/(.+)|skills\/(.+))$/); - if (!match) continue; - const relative = match[2]!; - restored.push(relative); - if (dryRun) continue; - const destination = match[3] - ? path.join(companyRoot, "agents", match[3], "instructions", match[4]!) - : path.join(instance, "skills", companyId, match[5]!); - const content = (await run("git", ["--git-dir", source, "cat-file", "blob", match[1]!])).stdout; - await fs.mkdir(path.dirname(destination), { recursive: true }); - await fs.writeFile(destination, content); + let gitDir = source; + let temporaryRepo: string | null = null; + try { + await fs.access(path.join(source, "HEAD")); + } catch { + await fs.mkdir(options.markerDir, { recursive: true }); + temporaryRepo = await fs.mkdtemp(path.join(options.markerDir, "state-restore-")); + await run("git", ["init", "--bare", temporaryRepo]); + await run("git", ["--git-dir", temporaryRepo, "fetch", source, `${ref}:${ref}`]); + gitDir = temporaryRepo; } - return { restored, dryRun }; + try { + const { stdout } = await run("git", ["--git-dir", gitDir, "ls-tree", "-r", ref, `companies/${companyId}`]); + const restored: string[] = []; + const memorySources = new Map((await options.resolveMemorySources?.(companyId) ?? []).map((source) => [source.agentId, source.root])); + for (const line of stdout.trim().split("\n").filter(Boolean)) { + const match = line.match(/^\d+ blob ([0-9a-f]+)\tcompanies\/[^/]+\/(agents\/([^/]+)\/(.+)|skills\/(.+))$/); + if (!match) continue; + const relative = match[2]!; + restored.push(relative); + if (dryRun) continue; + const agentId = match[3]; + const agentRelative = match[4]; + const memoryRelative = agentRelative?.startsWith("memory/") ? agentRelative.slice("memory/".length) : null; + const memoryRoot = agentId && memoryRelative ? memorySources.get(agentId) : null; + const destination = agentId && agentRelative + ? memoryRoot + ? path.join(memoryRoot, memoryRelative!) + : path.join(companyRoot, "agents", agentId, "instructions", agentRelative) + : path.join(instance, "skills", companyId, match[5]!); + const content = (await run("git", ["--git-dir", gitDir, "cat-file", "blob", match[1]!])).stdout; + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.writeFile(destination, content); + } + return { restored, dryRun }; + } finally { + if (temporaryRepo) await fs.rm(temporaryRepo, { recursive: true, force: true }); + } + }, + startWatcher(input: { listCompanyIds: () => Promise; debounceMs?: number; sweepMs?: number }) { + const debounceMs = input.debounceMs ?? 30_000; + const sweepMs = input.sweepMs ?? 24 * 60 * 60 * 1_000; + const pending = new Map(); + const fingerprints = new Map(); + let stopped = false; + const sweep = async (message: string) => { + for (const companyId of await input.listCompanyIds()) { + await this.commit({ + companyId, + actor: { name: "paperclip-state-bot", email: "state-bot@paperclip.invalid" }, + message, + }).catch(() => undefined); + } + }; + const poll = async () => { + if (stopped) return; + for (const companyId of await input.listCompanyIds()) { + const files = await collect(companyId); + const memory = [...files].filter(([filePath]) => filePath.includes("/memory/")); + const fingerprint = createHash("sha256") + .update(memory.map(([filePath, content]) => `${filePath}\0${createHash("sha256").update(content).digest("hex")}`).join("\n")) + .digest("hex"); + const previous = fingerprints.get(companyId); + fingerprints.set(companyId, fingerprint); + if (previous && previous !== fingerprint) { + clearTimeout(pending.get(companyId)); + pending.set(companyId, setTimeout(() => { + pending.delete(companyId); + void this.commit({ + companyId, + actor: { name: "paperclip-state-bot", email: "state-bot@paperclip.invalid" }, + message: "claude-memory: capture external changes", + }); + }, debounceMs)); + } + } + }; + const pollTimer = setInterval(() => void poll(), Math.min(5_000, debounceMs)); + const sweepTimer = setInterval(() => void sweep("state: daily drift sweep"), sweepMs); + void poll(); + return () => { + stopped = true; + clearInterval(pollTimer); + clearInterval(sweepTimer); + for (const timer of pending.values()) clearTimeout(timer); + }; }, }; } + +export type StateRepoService = ReturnType;