diff --git a/packages/shared/src/home-paths.ts b/packages/shared/src/home-paths.ts index f88dace9cd3..229fff81709 100644 --- a/packages/shared/src/home-paths.ts +++ b/packages/shared/src/home-paths.ts @@ -105,6 +105,16 @@ export function resolveDefaultBackupDir(input: { return path.resolve(resolvePaperclipInstanceRoot(input), "data", "backups"); } +export function resolveStateRepoPath(input: { + homeDir?: string; + instanceId?: string; + companyId?: string; +} = {}): string { + return input.companyId + ? path.resolve(resolvePaperclipInstanceRoot(input), "state-repos", `${input.companyId}.git`) + : path.resolve(resolvePaperclipInstanceRoot(input), "state-repo.git"); +} + export function resolveHomeAwarePath(value: string): string { return path.resolve(expandHomePrefix(value)); } diff --git a/server/src/__tests__/state-repo.test.ts b/server/src/__tests__/state-repo.test.ts new file mode 100644 index 00000000000..e496153e0a9 --- /dev/null +++ b/server/src/__tests__/state-repo.test.ts @@ -0,0 +1,52 @@ +import { execFile } from "node:child_process"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; +import { createStateRepoService } from "../services/state-repo.js"; + +const exec = promisify(execFile); + +describe("state repo service", () => { + it("commits attributed state, mirrors, bundles, restores, and blocks secrets", async () => { + const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-repo-")); + const instance = path.join(homeDir, "instances", "test"); + const instructions = path.join(instance, "companies", "company-1", "agents", "agent-1", "instructions"); + const skill = path.join(instance, "skills", "company-1", "skill-1"); + await fs.mkdir(instructions, { recursive: true }); + await fs.mkdir(skill, { recursive: true }); + await fs.writeFile(path.join(instructions, "AGENTS.md"), "# Initial\n"); + await fs.writeFile(path.join(skill, "SKILL.md"), "# Skill\n"); + const remote = path.join(homeDir, "remote.git"); + await exec("git", ["init", "--bare", remote]); + const service = createStateRepoService({ + homeDir, + instanceId: "test", + markerDir: path.join(instance, "health"), + resolveMirror: async () => ({ url: remote }), + }); + await service.commit({ companyId: "company-1", actor: { name: "Fable", email: "agent+fable@paperclip.invalid" }, message: "agent-instructions: update Fable" }); + await fs.writeFile(path.join(skill, "SKILL.md"), "# Updated skill\n"); + await service.commit({ companyId: "company-1", actor: { name: "Board User", email: "user+board@paperclip.invalid" }, message: "skill: update skill-1" }); + await service.testMirror("company-1"); + const repo = service.repoPathFor("company-1"); + const log = (await exec("git", ["--git-dir", repo, "log", "--format=%an|%ae|%cn|%s"])).stdout; + expect(log).toContain("Board User|user+board@paperclip.invalid|paperclip-state-bot|skill: update skill-1"); + expect(log).toContain("Fable|agent+fable@paperclip.invalid|paperclip-state-bot|agent-instructions: update Fable"); + expect((await exec("git", ["--git-dir", remote, "rev-parse", "main"])).stdout.trim()).toMatch(/^[0-9a-f]{40}$/); + const bundle = path.join(homeDir, "state.bundle"); + await service.exportBundle("company-1", bundle); + const restoredRepo = path.join(homeDir, "restored.git"); + await exec("git", ["init", "--bare", restoredRepo]); + await exec("git", ["--git-dir", restoredRepo, "fetch", bundle, "main:main"]); + await fs.rm(path.join(instance, "companies", "company-1"), { recursive: true, force: true }); + await fs.rm(path.join(instance, "skills", "company-1"), { recursive: true, force: true }); + const restored = await service.restore("company-1", restoredRepo); + expect(restored.restored).toContain("agents/agent-1/AGENTS.md"); + expect(await fs.readFile(path.join(instructions, "AGENTS.md"), "utf8")).toBe("# Initial\n"); + expect(await fs.readFile(path.join(skill, "SKILL.md"), "utf8")).toBe("# Updated skill\n"); + await fs.writeFile(path.join(instructions, "AGENTS.md"), "ghp_abcdefghijklmnopqrstuvwxyz123456\n"); + await expect(service.commit({ companyId: "company-1", actor: { name: "Fable", email: "agent+fable@paperclip.invalid" }, message: "bad" })).rejects.toThrow("secret scan blocked"); + }); +}); diff --git a/server/src/services/state-repo.ts b/server/src/services/state-repo.ts new file mode 100644 index 00000000000..ff27d5da446 --- /dev/null +++ b/server/src/services/state-repo.ts @@ -0,0 +1,224 @@ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { resolvePaperclipInstancePath, resolveStateRepoPath } from "@paperclipai/shared"; + +async function run(file: string, args: string[], options: { input?: string | Buffer; env?: NodeJS.ProcessEnv } = {}) { + return await new Promise<{ stdout: string; stderr: string }>((resolve, reject) => { + const child = spawn(file, args, { env: options.env ?? process.env, stdio: ["pipe", "pipe", "pipe"] }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); + child.on("error", reject); + child.on("close", (code) => { + const result = { stdout: Buffer.concat(stdout).toString("utf8"), stderr: Buffer.concat(stderr).toString("utf8") }; + if (code === 0) resolve(result); + else reject(Object.assign(new Error(result.stderr || `${file} exited ${code}`), result)); + }); + child.stdin.end(options.input); + }); +} +const SECRET_PATTERNS = [ + /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/, + /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/, + /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i, + /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/, +] as const; + +export type StateRepoActor = { name: string; email: string }; +type Entry = { oid: string; mode: string }; + +async function git(repo: string, args: string[], input?: string | Buffer) { + return run("git", ["--git-dir", repo, ...args], { + input, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, + }); +} + +async function walk(root: string): Promise> { + const files: Array<[string, Buffer]> = []; + async function visit(current: string, prefix = "") { + let entries; + try { entries = await fs.readdir(current, { withFileTypes: true }); } catch { return; } + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (entry.isSymbolicLink()) continue; + const absolute = path.join(current, entry.name); + const relative = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) await visit(absolute, relative); + else if (entry.isFile()) files.push([relative, await fs.readFile(absolute)]); + } + } + await visit(root); + return files; +} + +function scan(files: Map, knownSecrets: string[]) { + for (const [filePath, content] of files) { + const text = content.toString("utf8"); + for (const pattern of SECRET_PATTERNS) { + if (pattern.test(text)) throw new Error(`State repo secret scan blocked ${filePath}`); + } + for (const secret of knownSecrets) { + if (secret.length >= 8 && text.includes(secret)) { + const fingerprint = createHash("sha256").update(secret).digest("hex").slice(0, 12); + throw new Error(`State repo secret scan blocked ${filePath} (known-secret:${fingerprint})`); + } + } + } +} + +async function writeTree(repo: string, entries: Map) { + type Node = { files: Map; dirs: Map }; + const root: Node = { files: new Map(), dirs: new Map() }; + for (const [filePath, entry] of entries) { + const parts = filePath.split("/"); + const name = parts.pop()!; + let node = root; + for (const part of parts) { + let child = node.dirs.get(part); + if (!child) { child = { files: new Map(), dirs: new Map() }; node.dirs.set(part, child); } + node = child; + } + node.files.set(name, entry); + } + async function persist(node: Node): Promise { + const rows: string[] = []; + for (const [name, child] of [...node.dirs].sort(([a], [b]) => a.localeCompare(b))) { + rows.push(`040000 tree ${await persist(child)}\t${name}\n`); + } + for (const [name, entry] of [...node.files].sort(([a], [b]) => a.localeCompare(b))) { + rows.push(`${entry.mode} blob ${entry.oid}\t${name}\n`); + } + return (await git(repo, ["mktree"], rows.join(""))).stdout.trim(); + } + return persist(root); +} + +export function createStateRepoService(options: { + homeDir?: string; + instanceId?: string; + repoPerCompany?: boolean; + markerDir: string; + resolveMirror?: (companyId: string) => Promise<{ url: string; token?: string } | null>; + knownSecrets?: () => Promise; +}) { + let queue = Promise.resolve(); + let mirrorQueue = Promise.resolve(); + const repoPathFor = (companyId: string) => resolveStateRepoPath({ + homeDir: options.homeDir, + instanceId: options.instanceId, + companyId: options.repoPerCompany ? companyId : undefined, + }); + async function ensureRepo(repo: string) { + try { await fs.access(path.join(repo, "HEAD")); } catch { + await fs.mkdir(path.dirname(repo), { recursive: true }); + await run("git", ["init", "--bare", "--initial-branch=main", repo]); + } + } + async function collect(companyId: string) { + const files = new Map(); + const instance = resolvePaperclipInstancePath({ homeDir: options.homeDir, instanceId: options.instanceId }); + const agentsRoot = path.join(instance, "companies", companyId, "agents"); + let agents: string[] = []; + try { agents = await fs.readdir(agentsRoot); } catch { /* absent */ } + for (const agentId of agents) { + for (const [relative, content] of await walk(path.join(agentsRoot, agentId, "instructions"))) { + files.set(`companies/${companyId}/agents/${agentId}/${relative}`, content); + } + } + for (const [relative, content] of await walk(path.join(instance, "skills", companyId))) { + files.set(`companies/${companyId}/skills/${relative}`, content); + } + return files; + } + async function commitNow(input: { companyId: string; actor: StateRepoActor; message: string }) { + const repo = repoPathFor(input.companyId); + await ensureRepo(repo); + const files = await collect(input.companyId); + scan(files, await options.knownSecrets?.() ?? []); + const entries = new Map(); + try { + for (const line of (await git(repo, ["ls-tree", "-r", "main"])).stdout.trim().split("\n")) { + const match = line.match(/^(\d+) blob ([0-9a-f]+)\t(.+)$/); + if (match) entries.set(match[3]!, { mode: match[1]!, oid: match[2]! }); + } + } catch { /* first commit */ } + const prefix = `companies/${input.companyId}/`; + for (const key of [...entries.keys()]) if (key.startsWith(prefix)) entries.delete(key); + for (const [filePath, content] of files) { + entries.set(filePath, { mode: "100644", oid: (await git(repo, ["hash-object", "-w", "--stdin"], content)).stdout.trim() }); + } + const tree = await writeTree(repo, entries); + let parent: string | null = null; + try { parent = (await git(repo, ["rev-parse", "main"])).stdout.trim(); } catch { /* first */ } + if (parent && (await git(repo, ["show", "-s", "--format=%T", parent])).stdout.trim() === tree) return parent; + const { stdout } = await run("git", ["--git-dir", repo, "commit-tree", tree, ...(parent ? ["-p", parent] : [])], { + input: `${input.message}\n`, + env: { + ...process.env, + GIT_AUTHOR_NAME: input.actor.name, + GIT_AUTHOR_EMAIL: input.actor.email, + GIT_COMMITTER_NAME: "paperclip-state-bot", + GIT_COMMITTER_EMAIL: "state-bot@paperclip.invalid", + }, + }); + const commit = stdout.trim(); + await git(repo, ["update-ref", "refs/heads/main", commit, ...(parent ? [parent] : [])]); + mirrorQueue = mirrorQueue.then(() => pushMirror(input.companyId)).catch(() => undefined); + return commit; + } + async function pushMirror(companyId: string) { + const mirror = await options.resolveMirror?.(companyId); + if (!mirror) return; + const repo = repoPathFor(companyId); + try { + const args = ["--git-dir", repo]; + if (mirror.token) args.push("-c", `http.extraHeader=Authorization: Bearer ${mirror.token}`); + args.push("push", mirror.url, "refs/heads/main:refs/heads/main"); + await run("git", args, { env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); + await fs.mkdir(options.markerDir, { recursive: true }); + await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.success.json`), JSON.stringify({ pushedAt: new Date().toISOString() })); + await fs.rm(path.join(options.markerDir, `state-repo-${companyId}.failure`), { force: true }); + } catch (error) { + await fs.mkdir(options.markerDir, { recursive: true }); + await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.failure`), error instanceof Error ? error.message : String(error)); + throw error; + } + } + return { + repoPathFor, + commit(input: { companyId: string; actor: StateRepoActor; message: string }) { + const result = queue.then(() => commitNow(input)); + queue = result.then(() => undefined, () => undefined); + return result; + }, + async testMirror(companyId: string) { await mirrorQueue; await pushMirror(companyId); }, + async exportBundle(companyId: string, outputPath: string) { + const repo = repoPathFor(companyId); + await ensureRepo(repo); + await git(repo, ["bundle", "create", outputPath, "--all"]); + }, + async restore(companyId: string, source: string, ref = "main", dryRun = false) { + const instance = resolvePaperclipInstancePath({ homeDir: options.homeDir, instanceId: options.instanceId }); + const companyRoot = path.join(instance, "companies", companyId); + const { stdout } = await run("git", ["--git-dir", source, "ls-tree", "-r", ref, `companies/${companyId}`]); + const restored: string[] = []; + for (const line of stdout.trim().split("\n").filter(Boolean)) { + const match = line.match(/^\d+ blob ([0-9a-f]+)\tcompanies\/[^/]+\/(agents\/([^/]+)\/(.+)|skills\/(.+))$/); + if (!match) continue; + const relative = match[2]!; + restored.push(relative); + if (dryRun) continue; + const destination = match[3] + ? path.join(companyRoot, "agents", match[3], "instructions", match[4]!) + : path.join(instance, "skills", companyId, match[5]!); + const content = (await run("git", ["--git-dir", source, "cat-file", "blob", match[1]!])).stdout; + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.writeFile(destination, content); + } + return { restored, dryRun }; + }, + }; +}