feat: integrate managed state repo

This commit is contained in:
Dotta committed 2026-08-14 07:18:50 +00:00
1 parent bcf6c0ad65
commit 09696ad5d7
12 files changed
+454 -30

No files matched your search

+50
View File
@@ -0,0 +1,50 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("../utils/banner.js", () => ({ printPaperclipCliBanner: vi.fn() }));
vi.mock("@clack/prompts", () => ({ intro: vi.fn(), outro: vi.fn() }));
import { stateRestoreCommand } from "../commands/state.js";
const originalEnv = process.env;
describe("state restore command", () => {
beforeEach(() => {
process.env = { ...originalEnv };
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ restored: ["agents/a/AGENTS.md"], dryRun: true }),
}));
vi.spyOn(console, "log").mockImplementation(() => undefined);
});
afterEach(() => {
process.env = originalEnv;
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
it("calls the company restore API for --from-git", async () => {
await stateRestoreCommand(undefined, {
apiUrl: "http://paperclip.test/api",
token: "token",
json: true,
fromGit: "/backup/state.git",
companyId: "company-1",
ref: "refs/heads/main",
dryRun: true,
});
expect(fetch).toHaveBeenCalledWith(
"http://paperclip.test/api/companies/company-1/state-repo/restore",
expect.objectContaining({
method: "POST",
body: JSON.stringify({ source: "/backup/state.git", ref: "refs/heads/main", dryRun: true }),
}),
);
});
it("requires a company for git restores", async () => {
delete process.env.PAPERCLIP_COMPANY_ID;
await expect(stateRestoreCommand(undefined, { fromGit: "/backup/state.git" })).rejects.toThrow("--company-id");
});
});
+12 -3
View File
@@ -25,10 +25,19 @@ export async function stateSnapshotCommand(opts: StateApiOptions) {
else p.outro(pc.green(`Snapshot stored: ${result.objectKey}`));
}
export async function stateRestoreCommand(objectKey: string, opts: StateApiOptions) {
export async function stateRestoreCommand(objectKey: string | undefined, opts: StateApiOptions & { fromGit?: string; companyId?: string; ref?: string; dryRun?: boolean }) {
printPaperclipCliBanner();
p.intro(pc.bgCyan(pc.black(" paperclip state restore ")));
const result = await callStateApi("/instance/state-snapshots/restore", opts, { objectKey });
const companyId = opts.companyId || process.env.PAPERCLIP_COMPANY_ID;
if (opts.fromGit && !companyId) throw new Error("--company-id or PAPERCLIP_COMPANY_ID is required with --from-git");
if (!opts.fromGit && !objectKey) throw new Error("object-key is required unless --from-git is used");
const result = opts.fromGit
? await callStateApi(`/companies/${companyId}/state-repo/restore`, opts, {
source: opts.fromGit,
ref: opts.ref,
dryRun: opts.dryRun,
})
: await callStateApi("/instance/state-snapshots/restore", opts, { objectKey });
if (opts.json) console.log(JSON.stringify(result, null, 2));
else p.outro(pc.green(`Snapshot restored: ${objectKey}`));
else p.outro(pc.green(opts.fromGit ? `Git state restored: ${opts.fromGit}` : `Snapshot restored: ${objectKey}`));
}
+6 -2
View File
@@ -169,8 +169,12 @@ state.command("snapshot")
.option("--json", "Print result as JSON")
.action(stateSnapshotCommand);
state.command("restore")
.description("Restore an encrypted instance-state snapshot")
.argument("<object-key>", "Storage-provider object key returned by state snapshot")
.description("Restore an encrypted snapshot or company state from git")
.argument("[object-key]", "Storage-provider object key returned by state snapshot")
.option("--from-git <path>", "Restore company state from a bare repo or git bundle")
.option("--company-id <id>", "Company to restore (defaults to PAPERCLIP_COMPANY_ID)")
.option("--ref <ref>", "Git ref to restore", "main")
.option("--dry-run", "List files without writing them")
.option("--api-url <url>", "Paperclip API base URL")
.option("--token <token>", "Board/admin bearer token")
.option("--json", "Print result as JSON")
+25
View File
@@ -827,6 +827,31 @@ Use `PAPERCLIP_STATE_SNAPSHOT_INTERVAL_MINUTES` to change the schedule and
`PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS` to change the health warning threshold.
The encryption key must be held separately from the instance being protected.
## Managed State Repositories
Paperclip keeps a bare git repository per company under the instance
`state-repos/` directory. Agent-instruction and company-skill edits commit
immediately with actor attribution; Claude memory markdown is detected with a
30-second debounce and a daily sweep captures any remaining drift.
Company mirror remotes are optional and configured with
`PAPERCLIP_STATE_REPO_MIRRORS_JSON`. Tokens are resolved from company secrets,
not stored inline:
```json
{
"<company-id>": {
"url": "https://github.com/example/company-state.git",
"secretRef": { "secretId": "<company-secret-id>", "version": "latest" }
}
}
```
Use `GET /api/companies/:companyId/state-repo/bundle` for a company-scoped git
bundle, `POST .../mirror/test` and `GET .../health` for mirror diagnostics, and
`paperclipai state restore --from-git <bare-repo> --company-id <company-id>` to
restore tracked files. Add `--dry-run` to inspect the restore first.
## Secrets in Dev
Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config.
@@ -0,0 +1,53 @@
import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { stateRepoRoutes } from "../routes/state-repo.js";
const mockLogActivity = vi.hoisted(() => vi.fn());
vi.mock("../services/activity-log.js", () => ({ logActivity: mockLogActivity }));
function createApp(service: Record<string, ReturnType<typeof vi.fn>>) {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
req.actor = { type: "board", userId: "board-user", source: "local_implicit" };
next();
});
app.use("/api", stateRepoRoutes({} as never, service as never, "/tmp"));
return app;
}
describe("state repo routes", () => {
beforeEach(() => mockLogActivity.mockReset());
it("returns company-scoped mirror health", async () => {
const service = { health: vi.fn().mockResolvedValue({ configured: true, healthy: true }) };
const response = await request(createApp(service)).get("/api/companies/company-1/state-repo/health");
expect(response.status).toBe(200);
expect(response.body).toEqual({ configured: true, healthy: true });
expect(service.health).toHaveBeenCalledWith("company-1");
});
it("tests the configured mirror", async () => {
const service = {
testMirror: vi.fn().mockResolvedValue(undefined),
health: vi.fn().mockResolvedValue({ configured: true, healthy: true }),
};
const response = await request(createApp(service)).post("/api/companies/company-1/state-repo/mirror/test");
expect(response.status).toBe(200);
expect(service.testMirror).toHaveBeenCalledWith("company-1");
});
it("restores state and records activity", async () => {
const service = { restore: vi.fn().mockResolvedValue({ restored: ["agents/a/AGENTS.md"], dryRun: false }) };
const response = await request(createApp(service))
.post("/api/companies/company-1/state-repo/restore")
.send({ source: "/backup/state.git", ref: "main" });
expect(response.status).toBe(200);
expect(service.restore).toHaveBeenCalledWith("company-1", "/backup/state.git", "main", false);
expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
companyId: "company-1",
action: "company.state_repo_restored",
}));
});
});
+31 -4
View File
@@ -37,16 +37,43 @@ describe("state repo service", () => {
expect((await exec("git", ["--git-dir", remote, "rev-parse", "main"])).stdout.trim()).toMatch(/^[0-9a-f]{40}$/);
const bundle = path.join(homeDir, "state.bundle");
await service.exportBundle("company-1", bundle);
const restoredRepo = path.join(homeDir, "restored.git");
await exec("git", ["init", "--bare", restoredRepo]);
await exec("git", ["--git-dir", restoredRepo, "fetch", bundle, "main:main"]);
await fs.rm(path.join(instance, "companies", "company-1"), { recursive: true, force: true });
await fs.rm(path.join(instance, "skills", "company-1"), { recursive: true, force: true });
const restored = await service.restore("company-1", restoredRepo);
const restored = await service.restore("company-1", bundle);
expect(restored.restored).toContain("agents/agent-1/AGENTS.md");
expect(await fs.readFile(path.join(instructions, "AGENTS.md"), "utf8")).toBe("# Initial\n");
expect(await fs.readFile(path.join(skill, "SKILL.md"), "utf8")).toBe("# Updated skill\n");
await fs.writeFile(path.join(instructions, "AGENTS.md"), "ghp_abcdefghijklmnopqrstuvwxyz123456\n");
await expect(service.commit({ companyId: "company-1", actor: { name: "Fable", email: "agent+fable@paperclip.invalid" }, message: "bad" })).rejects.toThrow("secret scan blocked");
});
it("debounces Claude memory changes into an automatic commit", async () => {
const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-memory-"));
const instance = path.join(homeDir, "instances", "test");
const memory = path.join(homeDir, ".claude", "projects", "workspace", "memory");
await fs.mkdir(memory, { recursive: true });
await fs.writeFile(path.join(memory, "MEMORY.md"), "# Initial\n");
const service = createStateRepoService({
homeDir,
instanceId: "test",
markerDir: path.join(instance, "health"),
resolveMemorySources: async () => [{ agentId: "agent-1", root: memory }],
});
await service.commit({ companyId: "company-1", actor: { name: "Setup", email: "setup@paperclip.invalid" }, message: "initial" });
const stop = service.startWatcher({ listCompanyIds: async () => ["company-1"], debounceMs: 20, sweepMs: 60_000 });
await new Promise((resolve) => setTimeout(resolve, 30));
await fs.writeFile(path.join(memory, "MEMORY.md"), "# Updated\n");
await new Promise((resolve) => setTimeout(resolve, 90));
stop();
const repo = service.repoPathFor("company-1");
const log = (await exec("git", ["--git-dir", repo, "log", "--format=%s"])).stdout;
expect(log).toContain("claude-memory: capture external changes");
const content = (await exec("git", ["--git-dir", repo, "show", "main:companies/company-1/agents/agent-1/memory/MEMORY.md"])).stdout;
expect(content).toBe("# Updated\n");
const bundle = path.join(homeDir, "memory.bundle");
await service.exportBundle("company-1", bundle);
await fs.rm(path.join(memory, "MEMORY.md"));
await service.restore("company-1", bundle);
expect(await fs.readFile(path.join(memory, "MEMORY.md"), "utf8")).toBe("# Updated\n");
});
});
+9 -1
View File
@@ -21,6 +21,8 @@ import { healthRoutes } from "./routes/health.js";
import { cloudRoutes } from "./routes/cloud.js";
import { companyRoutes } from "./routes/companies.js";
import { companySkillRoutes } from "./routes/company-skills.js";
import { stateRepoRoutes } from "./routes/state-repo.js";
import type { StateRepoService } from "./services/state-repo.js";
import { companySkillPolicyRoutes } from "./routes/company-skill-policy.js";
import { inboxAgentPolicyRoutes } from "./routes/inbox-agent-policy.js";
import { builtInAgentRoutes } from "./routes/built-in-agents.js";
@@ -260,6 +262,8 @@ export async function createApp(
stateSnapshotService?: InstanceStateSnapshotService;
databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
stateSnapshotHealth?: { markerDir: string; enabled: boolean; maxAgeHours: number };
stateRepoService?: StateRepoService;
stateRepoMarkerDir?: string;
deploymentMode: DeploymentMode;
deploymentExposure: DeploymentExposure;
allowedHostnames: string[];
@@ -384,7 +388,7 @@ export async function createApp(
api.use("/companies", companyRoutes(db, opts.storageService));
api.use(llmRoutes(db));
api.use(folderRoutes(db));
api.use(companySkillRoutes(db));
api.use(companySkillRoutes(db, { stateRepo: opts.stateRepoService }));
api.use(companySkillPolicyRoutes(db));
api.use(inboxAgentPolicyRoutes(db));
api.use(builtInAgentRoutes(db));
@@ -414,6 +418,7 @@ export async function createApp(
api.use(
agentRoutes(db, {
pluginWorkerManager: workerManager,
stateRepo: opts.stateRepoService,
deploymentMode: opts.deploymentMode,
confidentialProxyAllowlist: setupTokenLoginProxyAllowlist,
onSetupTokenLoginService: (service) => {
@@ -426,6 +431,9 @@ export async function createApp(
},
}),
);
if (opts.stateRepoService && opts.stateRepoMarkerDir) {
api.use(stateRepoRoutes(db, opts.stateRepoService, opts.stateRepoMarkerDir));
}
api.use(assetRoutes(db, opts.storageService));
api.use(projectRoutes(db));
api.use(caseRoutes(db, opts.storageService));
+49 -1
View File
@@ -26,6 +26,7 @@ import {
runDatabaseBackup,
authUsers,
companies,
agents,
companyMemberships,
instanceUserRoles,
} from "@paperclipai/db";
@@ -62,6 +63,7 @@ import {
routineService,
statusCardService,
toolAccessService,
secretService,
} from "./services/index.js";
import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js";
import { createSecretProposalsService } from "./services/secret-proposals.js";
@@ -71,6 +73,7 @@ import {
createCodexDeviceLoginReaper,
createProductionLoginSessionReaperRuntime,
} from "./services/codex-device-login-reaper.js";
import { createStateRepoService } from "./services/state-repo.js";
import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js";
import {
parseAdapterRegistryEnv,
@@ -101,7 +104,7 @@ import type {
InstanceDatabaseBackupRunResult,
InstanceDatabaseBackupTrigger,
} from "./routes/instance-database-backups.js";
import { resolvePaperclipInstanceId, resolvePaperclipInstancePath } from "@paperclipai/shared";
import { resolvePaperclipInstanceId, resolvePaperclipInstancePath, resolveUserHomePath } from "@paperclipai/shared";
type BetterAuthSessionUser = {
id: string;
@@ -758,6 +761,45 @@ export async function startServer(): Promise<StartedServer> {
// document parsed fail-closed above (`plugins.autoInstall`). Absent env means
// self-hosted: createApp falls back to its built-in kubernetes-only default.
const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null;
const stateRepoMarkerDir = resolvePaperclipInstancePath({ instanceId: stateSnapshotInstanceId }, "health");
const stateRepoMirrors = (() => {
try {
return JSON.parse(process.env.PAPERCLIP_STATE_REPO_MIRRORS_JSON || "{}") as Record<string, { url?: string; secretRef?: { secretId?: string; version?: number | "latest" } }>;
} catch {
logger.warn("Ignoring invalid PAPERCLIP_STATE_REPO_MIRRORS_JSON");
return {};
}
})();
const stateRepoService = createStateRepoService({
instanceId: stateSnapshotInstanceId,
repoPerCompany: true,
markerDir: stateRepoMarkerDir,
resolveMirror: async (companyId) => {
const configured = stateRepoMirrors[companyId];
if (!configured?.url) return null;
const ref = configured.secretRef;
const token = ref?.secretId
? await secretService(db as any).resolveSecretValue(companyId, ref.secretId, ref.version ?? "latest", {
consumerType: "system",
consumerId: "state-repo-mirror",
configPath: "stateRepo.remote.secretRef",
})
: undefined;
return { url: configured.url, token };
},
resolveMemorySources: async (companyId) => {
const rows = await db.select({ id: agents.id, adapterType: agents.adapterType, adapterConfig: agents.adapterConfig })
.from(agents)
.where(eq(agents.companyId, companyId));
const claudeRoot = process.env.CLAUDE_CONFIG_DIR?.trim() || resolveUserHomePath(".claude");
return rows.flatMap((agent) => {
if (agent.adapterType !== "claude_local") return [];
const cwd = typeof agent.adapterConfig?.cwd === "string" ? agent.adapterConfig.cwd : null;
if (!cwd) return [];
return [{ agentId: agent.id, root: resolve(claudeRoot, "projects", cwd.replace(/[^a-zA-Z0-9-]/g, "-"), "memory") }];
});
},
});
const app = await createApp(db as any, {
uiMode,
serverPort: listenPort,
@@ -796,6 +838,8 @@ export async function startServer(): Promise<StartedServer> {
markerDir: stateSnapshotMarkerDir,
maxAgeHours: Math.max(1, Number(process.env.PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS) || 48),
},
stateRepoService,
stateRepoMarkerDir,
deploymentMode: config.deploymentMode,
deploymentExposure: config.deploymentExposure,
allowedHostnames: config.allowedHostnames,
@@ -811,6 +855,9 @@ export async function startServer(): Promise<StartedServer> {
managedPluginAutoInstall,
});
const server = createServer(app as unknown as Parameters<typeof createServer>[0]);
const stopStateRepoWatcher = stateRepoService.startWatcher({
listCompanyIds: async () => (await db.select({ id: companies.id }).from(companies)).map((company) => company.id),
});
// Increase keep-alive timeouts to safely outlive default idle timeouts
// of common reverse proxies and load balancers (like AWS ALB, Nginx, or Traefik).
@@ -1556,6 +1603,7 @@ export async function startServer(): Promise<StartedServer> {
{
const shutdown = async (signal: "SIGINT" | "SIGTERM") => {
await systemdNotify(["--stopping", `--status=Stopping after ${signal}`]);
stopStateRepoWatcher();
heartbeatSchedulerStopped = true;
if (heartbeatSchedulerInterval) {
clearInterval(heartbeatSchedulerInterval);
+18
View File
@@ -2,6 +2,7 @@ import { Router, type NextFunction, type Request, type Response } from "express"
import { generateKeyPairSync, randomUUID } from "node:crypto";
import path from "node:path";
import type { Db } from "@paperclipai/db";
import type { StateRepoService } from "../services/state-repo.js";
import { agents as agentsTable, companies, heartbeatRuns, issues as issuesTable, projects as projectsTable } from "@paperclipai/db";
import { and, desc, eq, inArray, not, sql } from "drizzle-orm";
import {
@@ -206,6 +207,7 @@ export function agentRoutes(
db: Db,
options: {
pluginWorkerManager?: PluginWorkerManager;
stateRepo?: StateRepoService;
/** The active deployment mode. The confidential transport guard reads it. */
deploymentMode?: DeploymentMode;
/**
@@ -3523,6 +3525,14 @@ export function agentRoutes(
clearLegacyPromptTemplate: req.body.clearLegacyPromptTemplate === true,
},
});
await options.stateRepo?.commit({
companyId: existing.companyId,
actor: {
name: actor.actorType === "agent" ? existing.name : "Board User",
email: `${actor.actorType}+${actor.actorId}@paperclip.invalid`,
},
message: `agent-instructions: update ${existing.name}`,
});
res.json(result.file);
});
@@ -3555,6 +3565,14 @@ export function agentRoutes(
path: relativePath,
},
});
await options.stateRepo?.commit({
companyId: existing.companyId,
actor: {
name: actor.actorType === "agent" ? existing.name : "Board User",
email: `${actor.actorType}+${actor.actorId}@paperclip.invalid`,
},
message: `agent-instructions: delete ${relativePath} for ${existing.name}`,
});
res.json(result.bundle);
});
+27 -1
View File
@@ -1,5 +1,6 @@
import { Router, type Request } from "express";
import type { Db } from "@paperclipai/db";
import type { StateRepoService } from "../services/state-repo.js";
import {
catalogSkillListQuerySchema,
companySkillCommentCreateSchema,
@@ -83,7 +84,7 @@ type SkillPolicyResourceInput =
| Promise<SkillPolicyEvaluationResource>
| (() => SkillPolicyEvaluationResource | Promise<SkillPolicyEvaluationResource>);
export function companySkillRoutes(db: Db) {
export function companySkillRoutes(db: Db, options: { stateRepo?: StateRepoService } = {}) {
const router = Router();
const access = accessService(db);
const svc = companySkillService(db);
@@ -1031,6 +1032,11 @@ export function companySkillRoutes(db: Db) {
name: result.name,
},
});
await options.stateRepo?.commit({
companyId,
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
message: `skill: create ${result.slug}`,
});
res.status(201).json(result);
},
@@ -1062,6 +1068,11 @@ export function companySkillRoutes(db: Db) {
sharingScope: result.sharingScope,
},
});
await options.stateRepo?.commit({
companyId,
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
message: `skill: update ${result.slug}`,
});
res.json(result);
},
@@ -1098,6 +1109,11 @@ export function companySkillRoutes(db: Db) {
markdown: result.markdown,
},
});
await options.stateRepo?.commit({
companyId,
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
message: `skill: update ${result.path}`,
});
res.json(result);
},
@@ -1129,6 +1145,11 @@ export function companySkillRoutes(db: Db) {
deletedPaths: result.deletedPaths,
},
});
await options.stateRepo?.commit({
companyId,
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
message: `skill: delete ${result.path}`,
});
res.json(result);
},
@@ -1284,6 +1305,11 @@ export function companySkillRoutes(db: Db) {
name: result.name,
},
});
await options.stateRepo?.commit({
companyId,
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
message: `skill: delete ${result.slug}`,
});
res.json(result);
});
+64
View File
@@ -0,0 +1,64 @@
import { promises as fs } from "node:fs";
import path from "node:path";
import { Router } from "express";
import type { Db } from "@paperclipai/db";
import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js";
import type { StateRepoService } from "../services/state-repo.js";
import { logActivity } from "../services/activity-log.js";
export function stateRepoRoutes(db: Db, service: StateRepoService, markerDir: string) {
const router = Router();
router.get("/companies/:companyId/state-repo/bundle", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
await fs.mkdir(markerDir, { recursive: true });
const outputPath = path.join(markerDir, `state-repo-${companyId}-${Date.now()}.bundle`);
await service.exportBundle(companyId, outputPath);
res.download(outputPath, `paperclip-state-${companyId}.bundle`, () => void fs.rm(outputPath, { force: true }));
});
router.post("/companies/:companyId/state-repo/mirror/test", async (req, res) => {
const companyId = req.params.companyId as string;
assertBoard(req);
assertCompanyAccess(req, companyId);
await service.testMirror(companyId);
res.json(await service.health(companyId));
});
router.get("/companies/:companyId/state-repo/health", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
res.json(await service.health(companyId));
});
router.post("/companies/:companyId/state-repo/restore", async (req, res) => {
const companyId = req.params.companyId as string;
assertBoard(req);
assertCompanyAccess(req, companyId);
const source = typeof req.body?.source === "string" ? req.body.source.trim() : "";
if (!source) {
res.status(422).json({ error: "source is required" });
return;
}
const result = await service.restore(companyId, source, req.body?.ref || "main", req.body?.dryRun === true);
if (!result.dryRun) {
const actor = getActorInfo(req);
await logActivity(db, {
companyId,
actorType: actor.actorType,
actorId: actor.actorId,
agentId: actor.agentId,
runId: actor.runId,
agentApiKeyId: actor.agentApiKeyId,
action: "company.state_repo_restored",
entityType: "company",
entityId: companyId,
details: { source, ref: req.body?.ref || "main", restoredCount: result.restored.length },
});
}
res.json(result);
});
return router;
}
+110 -18
View File
@@ -28,6 +28,7 @@ const SECRET_PATTERNS = [
] as const;
export type StateRepoActor = { name: string; email: string };
export type StateRepoMemorySource = { agentId: string; root: string };
type Entry = { oid: string; mode: string };
async function git(repo: string, args: string[], input?: string | Buffer) {
@@ -102,6 +103,7 @@ export function createStateRepoService(options: {
repoPerCompany?: boolean;
markerDir: string;
resolveMirror?: (companyId: string) => Promise<{ url: string; token?: string } | null>;
resolveMemorySources?: (companyId: string) => Promise<StateRepoMemorySource[]>;
knownSecrets?: () => Promise<string[]>;
}) {
let queue = Promise.resolve();
@@ -131,6 +133,13 @@ export function createStateRepoService(options: {
for (const [relative, content] of await walk(path.join(instance, "skills", companyId))) {
files.set(`companies/${companyId}/skills/${relative}`, content);
}
for (const source of await options.resolveMemorySources?.(companyId) ?? []) {
for (const [relative, content] of await walk(source.root)) {
if (relative.endsWith(".md")) {
files.set(`companies/${companyId}/agents/${source.agentId}/memory/${relative}`, content);
}
}
}
return files;
}
async function commitNow(input: { companyId: string; actor: StateRepoActor; message: string }) {
@@ -166,12 +175,12 @@ export function createStateRepoService(options: {
});
const commit = stdout.trim();
await git(repo, ["update-ref", "refs/heads/main", commit, ...(parent ? [parent] : [])]);
mirrorQueue = mirrorQueue.then(() => pushMirror(input.companyId)).catch(() => undefined);
mirrorQueue = mirrorQueue.then(async () => { await pushMirror(input.companyId); }).catch(() => undefined);
return commit;
}
async function pushMirror(companyId: string) {
const mirror = await options.resolveMirror?.(companyId);
if (!mirror) return;
if (!mirror) return false;
const repo = repoPathFor(companyId);
try {
const args = ["--git-dir", repo];
@@ -181,6 +190,7 @@ export function createStateRepoService(options: {
await fs.mkdir(options.markerDir, { recursive: true });
await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.success.json`), JSON.stringify({ pushedAt: new Date().toISOString() }));
await fs.rm(path.join(options.markerDir, `state-repo-${companyId}.failure`), { force: true });
return true;
} catch (error) {
await fs.mkdir(options.markerDir, { recursive: true });
await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.failure`), error instanceof Error ? error.message : String(error));
@@ -194,7 +204,17 @@ export function createStateRepoService(options: {
queue = result.then(() => undefined, () => undefined);
return result;
},
async testMirror(companyId: string) { await mirrorQueue; await pushMirror(companyId); },
async testMirror(companyId: string) {
await mirrorQueue;
if (!await pushMirror(companyId)) throw new Error("State repo mirror is not configured");
},
async health(companyId: string) {
const successPath = path.join(options.markerDir, `state-repo-${companyId}.success.json`);
const failurePath = path.join(options.markerDir, `state-repo-${companyId}.failure`);
const success = await fs.readFile(successPath, "utf8").then(JSON.parse).catch(() => null);
const failure = await fs.readFile(failurePath, "utf8").catch(() => null);
return { configured: Boolean(await options.resolveMirror?.(companyId)), healthy: !failure, success, failure };
},
async exportBundle(companyId: string, outputPath: string) {
const repo = repoPathFor(companyId);
await ensureRepo(repo);
@@ -203,22 +223,94 @@ export function createStateRepoService(options: {
async restore(companyId: string, source: string, ref = "main", dryRun = false) {
const instance = resolvePaperclipInstancePath({ homeDir: options.homeDir, instanceId: options.instanceId });
const companyRoot = path.join(instance, "companies", companyId);
const { stdout } = await run("git", ["--git-dir", source, "ls-tree", "-r", ref, `companies/${companyId}`]);
const restored: string[] = [];
for (const line of stdout.trim().split("\n").filter(Boolean)) {
const match = line.match(/^\d+ blob ([0-9a-f]+)\tcompanies\/[^/]+\/(agents\/([^/]+)\/(.+)|skills\/(.+))$/);
if (!match) continue;
const relative = match[2]!;
restored.push(relative);
if (dryRun) continue;
const destination = match[3]
? path.join(companyRoot, "agents", match[3], "instructions", match[4]!)
: path.join(instance, "skills", companyId, match[5]!);
const content = (await run("git", ["--git-dir", source, "cat-file", "blob", match[1]!])).stdout;
await fs.mkdir(path.dirname(destination), { recursive: true });
await fs.writeFile(destination, content);
let gitDir = source;
let temporaryRepo: string | null = null;
try {
await fs.access(path.join(source, "HEAD"));
} catch {
await fs.mkdir(options.markerDir, { recursive: true });
temporaryRepo = await fs.mkdtemp(path.join(options.markerDir, "state-restore-"));
await run("git", ["init", "--bare", temporaryRepo]);
await run("git", ["--git-dir", temporaryRepo, "fetch", source, `${ref}:${ref}`]);
gitDir = temporaryRepo;
}
return { restored, dryRun };
try {
const { stdout } = await run("git", ["--git-dir", gitDir, "ls-tree", "-r", ref, `companies/${companyId}`]);
const restored: string[] = [];
const memorySources = new Map((await options.resolveMemorySources?.(companyId) ?? []).map((source) => [source.agentId, source.root]));
for (const line of stdout.trim().split("\n").filter(Boolean)) {
const match = line.match(/^\d+ blob ([0-9a-f]+)\tcompanies\/[^/]+\/(agents\/([^/]+)\/(.+)|skills\/(.+))$/);
if (!match) continue;
const relative = match[2]!;
restored.push(relative);
if (dryRun) continue;
const agentId = match[3];
const agentRelative = match[4];
const memoryRelative = agentRelative?.startsWith("memory/") ? agentRelative.slice("memory/".length) : null;
const memoryRoot = agentId && memoryRelative ? memorySources.get(agentId) : null;
const destination = agentId && agentRelative
? memoryRoot
? path.join(memoryRoot, memoryRelative!)
: path.join(companyRoot, "agents", agentId, "instructions", agentRelative)
: path.join(instance, "skills", companyId, match[5]!);
const content = (await run("git", ["--git-dir", gitDir, "cat-file", "blob", match[1]!])).stdout;
await fs.mkdir(path.dirname(destination), { recursive: true });
await fs.writeFile(destination, content);
}
return { restored, dryRun };
} finally {
if (temporaryRepo) await fs.rm(temporaryRepo, { recursive: true, force: true });
}
},
startWatcher(input: { listCompanyIds: () => Promise<string[]>; debounceMs?: number; sweepMs?: number }) {
const debounceMs = input.debounceMs ?? 30_000;
const sweepMs = input.sweepMs ?? 24 * 60 * 60 * 1_000;
const pending = new Map<string, NodeJS.Timeout>();
const fingerprints = new Map<string, string>();
let stopped = false;
const sweep = async (message: string) => {
for (const companyId of await input.listCompanyIds()) {
await this.commit({
companyId,
actor: { name: "paperclip-state-bot", email: "state-bot@paperclip.invalid" },
message,
}).catch(() => undefined);
}
};
const poll = async () => {
if (stopped) return;
for (const companyId of await input.listCompanyIds()) {
const files = await collect(companyId);
const memory = [...files].filter(([filePath]) => filePath.includes("/memory/"));
const fingerprint = createHash("sha256")
.update(memory.map(([filePath, content]) => `${filePath}\0${createHash("sha256").update(content).digest("hex")}`).join("\n"))
.digest("hex");
const previous = fingerprints.get(companyId);
fingerprints.set(companyId, fingerprint);
if (previous && previous !== fingerprint) {
clearTimeout(pending.get(companyId));
pending.set(companyId, setTimeout(() => {
pending.delete(companyId);
void this.commit({
companyId,
actor: { name: "paperclip-state-bot", email: "state-bot@paperclip.invalid" },
message: "claude-memory: capture external changes",
});
}, debounceMs));
}
}
};
const pollTimer = setInterval(() => void poll(), Math.min(5_000, debounceMs));
const sweepTimer = setInterval(() => void sweep("state: daily drift sweep"), sweepMs);
void poll();
return () => {
stopped = true;
clearInterval(pollTimer);
clearInterval(sweepTimer);
for (const timer of pending.values()) clearTimeout(timer);
};
},
};
}
export type StateRepoService = ReturnType<typeof createStateRepoService>;