mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 02:44:50 +01:00
feat: integrate managed state repo
This commit is contained in:
1 parent
bcf6c0ad65
commit
09696ad5d7
12 files changed
+454
-30
No files matched your search
@@ -0,0 +1,50 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../utils/banner.js", () => ({ printPaperclipCliBanner: vi.fn() }));
|
||||
vi.mock("@clack/prompts", () => ({ intro: vi.fn(), outro: vi.fn() }));
|
||||
|
||||
import { stateRestoreCommand } from "../commands/state.js";
|
||||
|
||||
const originalEnv = process.env;
|
||||
|
||||
describe("state restore command", () => {
|
||||
beforeEach(() => {
|
||||
process.env = { ...originalEnv };
|
||||
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ restored: ["agents/a/AGENTS.md"], dryRun: true }),
|
||||
}));
|
||||
vi.spyOn(console, "log").mockImplementation(() => undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("calls the company restore API for --from-git", async () => {
|
||||
await stateRestoreCommand(undefined, {
|
||||
apiUrl: "http://paperclip.test/api",
|
||||
token: "token",
|
||||
json: true,
|
||||
fromGit: "/backup/state.git",
|
||||
companyId: "company-1",
|
||||
ref: "refs/heads/main",
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith(
|
||||
"http://paperclip.test/api/companies/company-1/state-repo/restore",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: JSON.stringify({ source: "/backup/state.git", ref: "refs/heads/main", dryRun: true }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("requires a company for git restores", async () => {
|
||||
delete process.env.PAPERCLIP_COMPANY_ID;
|
||||
await expect(stateRestoreCommand(undefined, { fromGit: "/backup/state.git" })).rejects.toThrow("--company-id");
|
||||
});
|
||||
});
|
||||
@@ -25,10 +25,19 @@ export async function stateSnapshotCommand(opts: StateApiOptions) {
|
||||
else p.outro(pc.green(`Snapshot stored: ${result.objectKey}`));
|
||||
}
|
||||
|
||||
export async function stateRestoreCommand(objectKey: string, opts: StateApiOptions) {
|
||||
export async function stateRestoreCommand(objectKey: string | undefined, opts: StateApiOptions & { fromGit?: string; companyId?: string; ref?: string; dryRun?: boolean }) {
|
||||
printPaperclipCliBanner();
|
||||
p.intro(pc.bgCyan(pc.black(" paperclip state restore ")));
|
||||
const result = await callStateApi("/instance/state-snapshots/restore", opts, { objectKey });
|
||||
const companyId = opts.companyId || process.env.PAPERCLIP_COMPANY_ID;
|
||||
if (opts.fromGit && !companyId) throw new Error("--company-id or PAPERCLIP_COMPANY_ID is required with --from-git");
|
||||
if (!opts.fromGit && !objectKey) throw new Error("object-key is required unless --from-git is used");
|
||||
const result = opts.fromGit
|
||||
? await callStateApi(`/companies/${companyId}/state-repo/restore`, opts, {
|
||||
source: opts.fromGit,
|
||||
ref: opts.ref,
|
||||
dryRun: opts.dryRun,
|
||||
})
|
||||
: await callStateApi("/instance/state-snapshots/restore", opts, { objectKey });
|
||||
if (opts.json) console.log(JSON.stringify(result, null, 2));
|
||||
else p.outro(pc.green(`Snapshot restored: ${objectKey}`));
|
||||
else p.outro(pc.green(opts.fromGit ? `Git state restored: ${opts.fromGit}` : `Snapshot restored: ${objectKey}`));
|
||||
}
|
||||
+6
-2
@@ -169,8 +169,12 @@ state.command("snapshot")
|
||||
.option("--json", "Print result as JSON")
|
||||
.action(stateSnapshotCommand);
|
||||
state.command("restore")
|
||||
.description("Restore an encrypted instance-state snapshot")
|
||||
.argument("<object-key>", "Storage-provider object key returned by state snapshot")
|
||||
.description("Restore an encrypted snapshot or company state from git")
|
||||
.argument("[object-key]", "Storage-provider object key returned by state snapshot")
|
||||
.option("--from-git <path>", "Restore company state from a bare repo or git bundle")
|
||||
.option("--company-id <id>", "Company to restore (defaults to PAPERCLIP_COMPANY_ID)")
|
||||
.option("--ref <ref>", "Git ref to restore", "main")
|
||||
.option("--dry-run", "List files without writing them")
|
||||
.option("--api-url <url>", "Paperclip API base URL")
|
||||
.option("--token <token>", "Board/admin bearer token")
|
||||
.option("--json", "Print result as JSON")
|
||||
|
||||
@@ -827,6 +827,31 @@ Use `PAPERCLIP_STATE_SNAPSHOT_INTERVAL_MINUTES` to change the schedule and
|
||||
`PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS` to change the health warning threshold.
|
||||
The encryption key must be held separately from the instance being protected.
|
||||
|
||||
## Managed State Repositories
|
||||
|
||||
Paperclip keeps a bare git repository per company under the instance
|
||||
`state-repos/` directory. Agent-instruction and company-skill edits commit
|
||||
immediately with actor attribution; Claude memory markdown is detected with a
|
||||
30-second debounce and a daily sweep captures any remaining drift.
|
||||
|
||||
Company mirror remotes are optional and configured with
|
||||
`PAPERCLIP_STATE_REPO_MIRRORS_JSON`. Tokens are resolved from company secrets,
|
||||
not stored inline:
|
||||
|
||||
```json
|
||||
{
|
||||
"<company-id>": {
|
||||
"url": "https://github.com/example/company-state.git",
|
||||
"secretRef": { "secretId": "<company-secret-id>", "version": "latest" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use `GET /api/companies/:companyId/state-repo/bundle` for a company-scoped git
|
||||
bundle, `POST .../mirror/test` and `GET .../health` for mirror diagnostics, and
|
||||
`paperclipai state restore --from-git <bare-repo> --company-id <company-id>` to
|
||||
restore tracked files. Add `--dry-run` to inspect the restore first.
|
||||
|
||||
## Secrets in Dev
|
||||
|
||||
Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config.
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { stateRepoRoutes } from "../routes/state-repo.js";
|
||||
|
||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../services/activity-log.js", () => ({ logActivity: mockLogActivity }));
|
||||
|
||||
function createApp(service: Record<string, ReturnType<typeof vi.fn>>) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", userId: "board-user", source: "local_implicit" };
|
||||
next();
|
||||
});
|
||||
app.use("/api", stateRepoRoutes({} as never, service as never, "/tmp"));
|
||||
return app;
|
||||
}
|
||||
|
||||
describe("state repo routes", () => {
|
||||
beforeEach(() => mockLogActivity.mockReset());
|
||||
|
||||
it("returns company-scoped mirror health", async () => {
|
||||
const service = { health: vi.fn().mockResolvedValue({ configured: true, healthy: true }) };
|
||||
const response = await request(createApp(service)).get("/api/companies/company-1/state-repo/health");
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body).toEqual({ configured: true, healthy: true });
|
||||
expect(service.health).toHaveBeenCalledWith("company-1");
|
||||
});
|
||||
|
||||
it("tests the configured mirror", async () => {
|
||||
const service = {
|
||||
testMirror: vi.fn().mockResolvedValue(undefined),
|
||||
health: vi.fn().mockResolvedValue({ configured: true, healthy: true }),
|
||||
};
|
||||
const response = await request(createApp(service)).post("/api/companies/company-1/state-repo/mirror/test");
|
||||
expect(response.status).toBe(200);
|
||||
expect(service.testMirror).toHaveBeenCalledWith("company-1");
|
||||
});
|
||||
|
||||
it("restores state and records activity", async () => {
|
||||
const service = { restore: vi.fn().mockResolvedValue({ restored: ["agents/a/AGENTS.md"], dryRun: false }) };
|
||||
const response = await request(createApp(service))
|
||||
.post("/api/companies/company-1/state-repo/restore")
|
||||
.send({ source: "/backup/state.git", ref: "main" });
|
||||
expect(response.status).toBe(200);
|
||||
expect(service.restore).toHaveBeenCalledWith("company-1", "/backup/state.git", "main", false);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
|
||||
companyId: "company-1",
|
||||
action: "company.state_repo_restored",
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -37,16 +37,43 @@ describe("state repo service", () => {
|
||||
expect((await exec("git", ["--git-dir", remote, "rev-parse", "main"])).stdout.trim()).toMatch(/^[0-9a-f]{40}$/);
|
||||
const bundle = path.join(homeDir, "state.bundle");
|
||||
await service.exportBundle("company-1", bundle);
|
||||
const restoredRepo = path.join(homeDir, "restored.git");
|
||||
await exec("git", ["init", "--bare", restoredRepo]);
|
||||
await exec("git", ["--git-dir", restoredRepo, "fetch", bundle, "main:main"]);
|
||||
await fs.rm(path.join(instance, "companies", "company-1"), { recursive: true, force: true });
|
||||
await fs.rm(path.join(instance, "skills", "company-1"), { recursive: true, force: true });
|
||||
const restored = await service.restore("company-1", restoredRepo);
|
||||
const restored = await service.restore("company-1", bundle);
|
||||
expect(restored.restored).toContain("agents/agent-1/AGENTS.md");
|
||||
expect(await fs.readFile(path.join(instructions, "AGENTS.md"), "utf8")).toBe("# Initial\n");
|
||||
expect(await fs.readFile(path.join(skill, "SKILL.md"), "utf8")).toBe("# Updated skill\n");
|
||||
await fs.writeFile(path.join(instructions, "AGENTS.md"), "ghp_abcdefghijklmnopqrstuvwxyz123456\n");
|
||||
await expect(service.commit({ companyId: "company-1", actor: { name: "Fable", email: "agent+fable@paperclip.invalid" }, message: "bad" })).rejects.toThrow("secret scan blocked");
|
||||
});
|
||||
|
||||
it("debounces Claude memory changes into an automatic commit", async () => {
|
||||
const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-state-memory-"));
|
||||
const instance = path.join(homeDir, "instances", "test");
|
||||
const memory = path.join(homeDir, ".claude", "projects", "workspace", "memory");
|
||||
await fs.mkdir(memory, { recursive: true });
|
||||
await fs.writeFile(path.join(memory, "MEMORY.md"), "# Initial\n");
|
||||
const service = createStateRepoService({
|
||||
homeDir,
|
||||
instanceId: "test",
|
||||
markerDir: path.join(instance, "health"),
|
||||
resolveMemorySources: async () => [{ agentId: "agent-1", root: memory }],
|
||||
});
|
||||
await service.commit({ companyId: "company-1", actor: { name: "Setup", email: "setup@paperclip.invalid" }, message: "initial" });
|
||||
const stop = service.startWatcher({ listCompanyIds: async () => ["company-1"], debounceMs: 20, sweepMs: 60_000 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
await fs.writeFile(path.join(memory, "MEMORY.md"), "# Updated\n");
|
||||
await new Promise((resolve) => setTimeout(resolve, 90));
|
||||
stop();
|
||||
const repo = service.repoPathFor("company-1");
|
||||
const log = (await exec("git", ["--git-dir", repo, "log", "--format=%s"])).stdout;
|
||||
expect(log).toContain("claude-memory: capture external changes");
|
||||
const content = (await exec("git", ["--git-dir", repo, "show", "main:companies/company-1/agents/agent-1/memory/MEMORY.md"])).stdout;
|
||||
expect(content).toBe("# Updated\n");
|
||||
const bundle = path.join(homeDir, "memory.bundle");
|
||||
await service.exportBundle("company-1", bundle);
|
||||
await fs.rm(path.join(memory, "MEMORY.md"));
|
||||
await service.restore("company-1", bundle);
|
||||
expect(await fs.readFile(path.join(memory, "MEMORY.md"), "utf8")).toBe("# Updated\n");
|
||||
});
|
||||
});
|
||||
+9
-1
@@ -21,6 +21,8 @@ import { healthRoutes } from "./routes/health.js";
|
||||
import { cloudRoutes } from "./routes/cloud.js";
|
||||
import { companyRoutes } from "./routes/companies.js";
|
||||
import { companySkillRoutes } from "./routes/company-skills.js";
|
||||
import { stateRepoRoutes } from "./routes/state-repo.js";
|
||||
import type { StateRepoService } from "./services/state-repo.js";
|
||||
import { companySkillPolicyRoutes } from "./routes/company-skill-policy.js";
|
||||
import { inboxAgentPolicyRoutes } from "./routes/inbox-agent-policy.js";
|
||||
import { builtInAgentRoutes } from "./routes/built-in-agents.js";
|
||||
@@ -260,6 +262,8 @@ export async function createApp(
|
||||
stateSnapshotService?: InstanceStateSnapshotService;
|
||||
databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
|
||||
stateSnapshotHealth?: { markerDir: string; enabled: boolean; maxAgeHours: number };
|
||||
stateRepoService?: StateRepoService;
|
||||
stateRepoMarkerDir?: string;
|
||||
deploymentMode: DeploymentMode;
|
||||
deploymentExposure: DeploymentExposure;
|
||||
allowedHostnames: string[];
|
||||
@@ -384,7 +388,7 @@ export async function createApp(
|
||||
api.use("/companies", companyRoutes(db, opts.storageService));
|
||||
api.use(llmRoutes(db));
|
||||
api.use(folderRoutes(db));
|
||||
api.use(companySkillRoutes(db));
|
||||
api.use(companySkillRoutes(db, { stateRepo: opts.stateRepoService }));
|
||||
api.use(companySkillPolicyRoutes(db));
|
||||
api.use(inboxAgentPolicyRoutes(db));
|
||||
api.use(builtInAgentRoutes(db));
|
||||
@@ -414,6 +418,7 @@ export async function createApp(
|
||||
api.use(
|
||||
agentRoutes(db, {
|
||||
pluginWorkerManager: workerManager,
|
||||
stateRepo: opts.stateRepoService,
|
||||
deploymentMode: opts.deploymentMode,
|
||||
confidentialProxyAllowlist: setupTokenLoginProxyAllowlist,
|
||||
onSetupTokenLoginService: (service) => {
|
||||
@@ -426,6 +431,9 @@ export async function createApp(
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (opts.stateRepoService && opts.stateRepoMarkerDir) {
|
||||
api.use(stateRepoRoutes(db, opts.stateRepoService, opts.stateRepoMarkerDir));
|
||||
}
|
||||
api.use(assetRoutes(db, opts.storageService));
|
||||
api.use(projectRoutes(db));
|
||||
api.use(caseRoutes(db, opts.storageService));
|
||||
|
||||
+49
-1
@@ -26,6 +26,7 @@ import {
|
||||
runDatabaseBackup,
|
||||
authUsers,
|
||||
companies,
|
||||
agents,
|
||||
companyMemberships,
|
||||
instanceUserRoles,
|
||||
} from "@paperclipai/db";
|
||||
@@ -62,6 +63,7 @@ import {
|
||||
routineService,
|
||||
statusCardService,
|
||||
toolAccessService,
|
||||
secretService,
|
||||
} from "./services/index.js";
|
||||
import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js";
|
||||
import { createSecretProposalsService } from "./services/secret-proposals.js";
|
||||
@@ -71,6 +73,7 @@ import {
|
||||
createCodexDeviceLoginReaper,
|
||||
createProductionLoginSessionReaperRuntime,
|
||||
} from "./services/codex-device-login-reaper.js";
|
||||
import { createStateRepoService } from "./services/state-repo.js";
|
||||
import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js";
|
||||
import {
|
||||
parseAdapterRegistryEnv,
|
||||
@@ -101,7 +104,7 @@ import type {
|
||||
InstanceDatabaseBackupRunResult,
|
||||
InstanceDatabaseBackupTrigger,
|
||||
} from "./routes/instance-database-backups.js";
|
||||
import { resolvePaperclipInstanceId, resolvePaperclipInstancePath } from "@paperclipai/shared";
|
||||
import { resolvePaperclipInstanceId, resolvePaperclipInstancePath, resolveUserHomePath } from "@paperclipai/shared";
|
||||
|
||||
type BetterAuthSessionUser = {
|
||||
id: string;
|
||||
@@ -758,6 +761,45 @@ export async function startServer(): Promise<StartedServer> {
|
||||
// document parsed fail-closed above (`plugins.autoInstall`). Absent env means
|
||||
// self-hosted: createApp falls back to its built-in kubernetes-only default.
|
||||
const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null;
|
||||
const stateRepoMarkerDir = resolvePaperclipInstancePath({ instanceId: stateSnapshotInstanceId }, "health");
|
||||
const stateRepoMirrors = (() => {
|
||||
try {
|
||||
return JSON.parse(process.env.PAPERCLIP_STATE_REPO_MIRRORS_JSON || "{}") as Record<string, { url?: string; secretRef?: { secretId?: string; version?: number | "latest" } }>;
|
||||
} catch {
|
||||
logger.warn("Ignoring invalid PAPERCLIP_STATE_REPO_MIRRORS_JSON");
|
||||
return {};
|
||||
}
|
||||
})();
|
||||
const stateRepoService = createStateRepoService({
|
||||
instanceId: stateSnapshotInstanceId,
|
||||
repoPerCompany: true,
|
||||
markerDir: stateRepoMarkerDir,
|
||||
resolveMirror: async (companyId) => {
|
||||
const configured = stateRepoMirrors[companyId];
|
||||
if (!configured?.url) return null;
|
||||
const ref = configured.secretRef;
|
||||
const token = ref?.secretId
|
||||
? await secretService(db as any).resolveSecretValue(companyId, ref.secretId, ref.version ?? "latest", {
|
||||
consumerType: "system",
|
||||
consumerId: "state-repo-mirror",
|
||||
configPath: "stateRepo.remote.secretRef",
|
||||
})
|
||||
: undefined;
|
||||
return { url: configured.url, token };
|
||||
},
|
||||
resolveMemorySources: async (companyId) => {
|
||||
const rows = await db.select({ id: agents.id, adapterType: agents.adapterType, adapterConfig: agents.adapterConfig })
|
||||
.from(agents)
|
||||
.where(eq(agents.companyId, companyId));
|
||||
const claudeRoot = process.env.CLAUDE_CONFIG_DIR?.trim() || resolveUserHomePath(".claude");
|
||||
return rows.flatMap((agent) => {
|
||||
if (agent.adapterType !== "claude_local") return [];
|
||||
const cwd = typeof agent.adapterConfig?.cwd === "string" ? agent.adapterConfig.cwd : null;
|
||||
if (!cwd) return [];
|
||||
return [{ agentId: agent.id, root: resolve(claudeRoot, "projects", cwd.replace(/[^a-zA-Z0-9-]/g, "-"), "memory") }];
|
||||
});
|
||||
},
|
||||
});
|
||||
const app = await createApp(db as any, {
|
||||
uiMode,
|
||||
serverPort: listenPort,
|
||||
@@ -796,6 +838,8 @@ export async function startServer(): Promise<StartedServer> {
|
||||
markerDir: stateSnapshotMarkerDir,
|
||||
maxAgeHours: Math.max(1, Number(process.env.PAPERCLIP_STATE_SNAPSHOT_MAX_AGE_HOURS) || 48),
|
||||
},
|
||||
stateRepoService,
|
||||
stateRepoMarkerDir,
|
||||
deploymentMode: config.deploymentMode,
|
||||
deploymentExposure: config.deploymentExposure,
|
||||
allowedHostnames: config.allowedHostnames,
|
||||
@@ -811,6 +855,9 @@ export async function startServer(): Promise<StartedServer> {
|
||||
managedPluginAutoInstall,
|
||||
});
|
||||
const server = createServer(app as unknown as Parameters<typeof createServer>[0]);
|
||||
const stopStateRepoWatcher = stateRepoService.startWatcher({
|
||||
listCompanyIds: async () => (await db.select({ id: companies.id }).from(companies)).map((company) => company.id),
|
||||
});
|
||||
|
||||
// Increase keep-alive timeouts to safely outlive default idle timeouts
|
||||
// of common reverse proxies and load balancers (like AWS ALB, Nginx, or Traefik).
|
||||
@@ -1556,6 +1603,7 @@ export async function startServer(): Promise<StartedServer> {
|
||||
{
|
||||
const shutdown = async (signal: "SIGINT" | "SIGTERM") => {
|
||||
await systemdNotify(["--stopping", `--status=Stopping after ${signal}`]);
|
||||
stopStateRepoWatcher();
|
||||
heartbeatSchedulerStopped = true;
|
||||
if (heartbeatSchedulerInterval) {
|
||||
clearInterval(heartbeatSchedulerInterval);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Router, type NextFunction, type Request, type Response } from "express"
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { StateRepoService } from "../services/state-repo.js";
|
||||
import { agents as agentsTable, companies, heartbeatRuns, issues as issuesTable, projects as projectsTable } from "@paperclipai/db";
|
||||
import { and, desc, eq, inArray, not, sql } from "drizzle-orm";
|
||||
import {
|
||||
@@ -206,6 +207,7 @@ export function agentRoutes(
|
||||
db: Db,
|
||||
options: {
|
||||
pluginWorkerManager?: PluginWorkerManager;
|
||||
stateRepo?: StateRepoService;
|
||||
/** The active deployment mode. The confidential transport guard reads it. */
|
||||
deploymentMode?: DeploymentMode;
|
||||
/**
|
||||
@@ -3523,6 +3525,14 @@ export function agentRoutes(
|
||||
clearLegacyPromptTemplate: req.body.clearLegacyPromptTemplate === true,
|
||||
},
|
||||
});
|
||||
await options.stateRepo?.commit({
|
||||
companyId: existing.companyId,
|
||||
actor: {
|
||||
name: actor.actorType === "agent" ? existing.name : "Board User",
|
||||
email: `${actor.actorType}+${actor.actorId}@paperclip.invalid`,
|
||||
},
|
||||
message: `agent-instructions: update ${existing.name}`,
|
||||
});
|
||||
|
||||
res.json(result.file);
|
||||
});
|
||||
@@ -3555,6 +3565,14 @@ export function agentRoutes(
|
||||
path: relativePath,
|
||||
},
|
||||
});
|
||||
await options.stateRepo?.commit({
|
||||
companyId: existing.companyId,
|
||||
actor: {
|
||||
name: actor.actorType === "agent" ? existing.name : "Board User",
|
||||
email: `${actor.actorType}+${actor.actorId}@paperclip.invalid`,
|
||||
},
|
||||
message: `agent-instructions: delete ${relativePath} for ${existing.name}`,
|
||||
});
|
||||
|
||||
res.json(result.bundle);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Router, type Request } from "express";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { StateRepoService } from "../services/state-repo.js";
|
||||
import {
|
||||
catalogSkillListQuerySchema,
|
||||
companySkillCommentCreateSchema,
|
||||
@@ -83,7 +84,7 @@ type SkillPolicyResourceInput =
|
||||
| Promise<SkillPolicyEvaluationResource>
|
||||
| (() => SkillPolicyEvaluationResource | Promise<SkillPolicyEvaluationResource>);
|
||||
|
||||
export function companySkillRoutes(db: Db) {
|
||||
export function companySkillRoutes(db: Db, options: { stateRepo?: StateRepoService } = {}) {
|
||||
const router = Router();
|
||||
const access = accessService(db);
|
||||
const svc = companySkillService(db);
|
||||
@@ -1031,6 +1032,11 @@ export function companySkillRoutes(db: Db) {
|
||||
name: result.name,
|
||||
},
|
||||
});
|
||||
await options.stateRepo?.commit({
|
||||
companyId,
|
||||
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
|
||||
message: `skill: create ${result.slug}`,
|
||||
});
|
||||
|
||||
res.status(201).json(result);
|
||||
},
|
||||
@@ -1062,6 +1068,11 @@ export function companySkillRoutes(db: Db) {
|
||||
sharingScope: result.sharingScope,
|
||||
},
|
||||
});
|
||||
await options.stateRepo?.commit({
|
||||
companyId,
|
||||
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
|
||||
message: `skill: update ${result.slug}`,
|
||||
});
|
||||
|
||||
res.json(result);
|
||||
},
|
||||
@@ -1098,6 +1109,11 @@ export function companySkillRoutes(db: Db) {
|
||||
markdown: result.markdown,
|
||||
},
|
||||
});
|
||||
await options.stateRepo?.commit({
|
||||
companyId,
|
||||
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
|
||||
message: `skill: update ${result.path}`,
|
||||
});
|
||||
|
||||
res.json(result);
|
||||
},
|
||||
@@ -1129,6 +1145,11 @@ export function companySkillRoutes(db: Db) {
|
||||
deletedPaths: result.deletedPaths,
|
||||
},
|
||||
});
|
||||
await options.stateRepo?.commit({
|
||||
companyId,
|
||||
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
|
||||
message: `skill: delete ${result.path}`,
|
||||
});
|
||||
|
||||
res.json(result);
|
||||
},
|
||||
@@ -1284,6 +1305,11 @@ export function companySkillRoutes(db: Db) {
|
||||
name: result.name,
|
||||
},
|
||||
});
|
||||
await options.stateRepo?.commit({
|
||||
companyId,
|
||||
actor: { name: actor.actorType === "agent" ? `Agent ${actor.actorId}` : "Board User", email: `${actor.actorType}+${actor.actorId}@paperclip.invalid` },
|
||||
message: `skill: delete ${result.slug}`,
|
||||
});
|
||||
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { Router } from "express";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||
import type { StateRepoService } from "../services/state-repo.js";
|
||||
import { logActivity } from "../services/activity-log.js";
|
||||
|
||||
export function stateRepoRoutes(db: Db, service: StateRepoService, markerDir: string) {
|
||||
const router = Router();
|
||||
|
||||
router.get("/companies/:companyId/state-repo/bundle", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
await fs.mkdir(markerDir, { recursive: true });
|
||||
const outputPath = path.join(markerDir, `state-repo-${companyId}-${Date.now()}.bundle`);
|
||||
await service.exportBundle(companyId, outputPath);
|
||||
res.download(outputPath, `paperclip-state-${companyId}.bundle`, () => void fs.rm(outputPath, { force: true }));
|
||||
});
|
||||
|
||||
router.post("/companies/:companyId/state-repo/mirror/test", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertBoard(req);
|
||||
assertCompanyAccess(req, companyId);
|
||||
await service.testMirror(companyId);
|
||||
res.json(await service.health(companyId));
|
||||
});
|
||||
|
||||
router.get("/companies/:companyId/state-repo/health", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
res.json(await service.health(companyId));
|
||||
});
|
||||
|
||||
router.post("/companies/:companyId/state-repo/restore", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertBoard(req);
|
||||
assertCompanyAccess(req, companyId);
|
||||
const source = typeof req.body?.source === "string" ? req.body.source.trim() : "";
|
||||
if (!source) {
|
||||
res.status(422).json({ error: "source is required" });
|
||||
return;
|
||||
}
|
||||
const result = await service.restore(companyId, source, req.body?.ref || "main", req.body?.dryRun === true);
|
||||
if (!result.dryRun) {
|
||||
const actor = getActorInfo(req);
|
||||
await logActivity(db, {
|
||||
companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
agentApiKeyId: actor.agentApiKeyId,
|
||||
action: "company.state_repo_restored",
|
||||
entityType: "company",
|
||||
entityId: companyId,
|
||||
details: { source, ref: req.body?.ref || "main", restoredCount: result.restored.length },
|
||||
});
|
||||
}
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -28,6 +28,7 @@ const SECRET_PATTERNS = [
|
||||
] as const;
|
||||
|
||||
export type StateRepoActor = { name: string; email: string };
|
||||
export type StateRepoMemorySource = { agentId: string; root: string };
|
||||
type Entry = { oid: string; mode: string };
|
||||
|
||||
async function git(repo: string, args: string[], input?: string | Buffer) {
|
||||
@@ -102,6 +103,7 @@ export function createStateRepoService(options: {
|
||||
repoPerCompany?: boolean;
|
||||
markerDir: string;
|
||||
resolveMirror?: (companyId: string) => Promise<{ url: string; token?: string } | null>;
|
||||
resolveMemorySources?: (companyId: string) => Promise<StateRepoMemorySource[]>;
|
||||
knownSecrets?: () => Promise<string[]>;
|
||||
}) {
|
||||
let queue = Promise.resolve();
|
||||
@@ -131,6 +133,13 @@ export function createStateRepoService(options: {
|
||||
for (const [relative, content] of await walk(path.join(instance, "skills", companyId))) {
|
||||
files.set(`companies/${companyId}/skills/${relative}`, content);
|
||||
}
|
||||
for (const source of await options.resolveMemorySources?.(companyId) ?? []) {
|
||||
for (const [relative, content] of await walk(source.root)) {
|
||||
if (relative.endsWith(".md")) {
|
||||
files.set(`companies/${companyId}/agents/${source.agentId}/memory/${relative}`, content);
|
||||
}
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
async function commitNow(input: { companyId: string; actor: StateRepoActor; message: string }) {
|
||||
@@ -166,12 +175,12 @@ export function createStateRepoService(options: {
|
||||
});
|
||||
const commit = stdout.trim();
|
||||
await git(repo, ["update-ref", "refs/heads/main", commit, ...(parent ? [parent] : [])]);
|
||||
mirrorQueue = mirrorQueue.then(() => pushMirror(input.companyId)).catch(() => undefined);
|
||||
mirrorQueue = mirrorQueue.then(async () => { await pushMirror(input.companyId); }).catch(() => undefined);
|
||||
return commit;
|
||||
}
|
||||
async function pushMirror(companyId: string) {
|
||||
const mirror = await options.resolveMirror?.(companyId);
|
||||
if (!mirror) return;
|
||||
if (!mirror) return false;
|
||||
const repo = repoPathFor(companyId);
|
||||
try {
|
||||
const args = ["--git-dir", repo];
|
||||
@@ -181,6 +190,7 @@ export function createStateRepoService(options: {
|
||||
await fs.mkdir(options.markerDir, { recursive: true });
|
||||
await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.success.json`), JSON.stringify({ pushedAt: new Date().toISOString() }));
|
||||
await fs.rm(path.join(options.markerDir, `state-repo-${companyId}.failure`), { force: true });
|
||||
return true;
|
||||
} catch (error) {
|
||||
await fs.mkdir(options.markerDir, { recursive: true });
|
||||
await fs.writeFile(path.join(options.markerDir, `state-repo-${companyId}.failure`), error instanceof Error ? error.message : String(error));
|
||||
@@ -194,7 +204,17 @@ export function createStateRepoService(options: {
|
||||
queue = result.then(() => undefined, () => undefined);
|
||||
return result;
|
||||
},
|
||||
async testMirror(companyId: string) { await mirrorQueue; await pushMirror(companyId); },
|
||||
async testMirror(companyId: string) {
|
||||
await mirrorQueue;
|
||||
if (!await pushMirror(companyId)) throw new Error("State repo mirror is not configured");
|
||||
},
|
||||
async health(companyId: string) {
|
||||
const successPath = path.join(options.markerDir, `state-repo-${companyId}.success.json`);
|
||||
const failurePath = path.join(options.markerDir, `state-repo-${companyId}.failure`);
|
||||
const success = await fs.readFile(successPath, "utf8").then(JSON.parse).catch(() => null);
|
||||
const failure = await fs.readFile(failurePath, "utf8").catch(() => null);
|
||||
return { configured: Boolean(await options.resolveMirror?.(companyId)), healthy: !failure, success, failure };
|
||||
},
|
||||
async exportBundle(companyId: string, outputPath: string) {
|
||||
const repo = repoPathFor(companyId);
|
||||
await ensureRepo(repo);
|
||||
@@ -203,22 +223,94 @@ export function createStateRepoService(options: {
|
||||
async restore(companyId: string, source: string, ref = "main", dryRun = false) {
|
||||
const instance = resolvePaperclipInstancePath({ homeDir: options.homeDir, instanceId: options.instanceId });
|
||||
const companyRoot = path.join(instance, "companies", companyId);
|
||||
const { stdout } = await run("git", ["--git-dir", source, "ls-tree", "-r", ref, `companies/${companyId}`]);
|
||||
const restored: string[] = [];
|
||||
for (const line of stdout.trim().split("\n").filter(Boolean)) {
|
||||
const match = line.match(/^\d+ blob ([0-9a-f]+)\tcompanies\/[^/]+\/(agents\/([^/]+)\/(.+)|skills\/(.+))$/);
|
||||
if (!match) continue;
|
||||
const relative = match[2]!;
|
||||
restored.push(relative);
|
||||
if (dryRun) continue;
|
||||
const destination = match[3]
|
||||
? path.join(companyRoot, "agents", match[3], "instructions", match[4]!)
|
||||
: path.join(instance, "skills", companyId, match[5]!);
|
||||
const content = (await run("git", ["--git-dir", source, "cat-file", "blob", match[1]!])).stdout;
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true });
|
||||
await fs.writeFile(destination, content);
|
||||
let gitDir = source;
|
||||
let temporaryRepo: string | null = null;
|
||||
try {
|
||||
await fs.access(path.join(source, "HEAD"));
|
||||
} catch {
|
||||
await fs.mkdir(options.markerDir, { recursive: true });
|
||||
temporaryRepo = await fs.mkdtemp(path.join(options.markerDir, "state-restore-"));
|
||||
await run("git", ["init", "--bare", temporaryRepo]);
|
||||
await run("git", ["--git-dir", temporaryRepo, "fetch", source, `${ref}:${ref}`]);
|
||||
gitDir = temporaryRepo;
|
||||
}
|
||||
return { restored, dryRun };
|
||||
try {
|
||||
const { stdout } = await run("git", ["--git-dir", gitDir, "ls-tree", "-r", ref, `companies/${companyId}`]);
|
||||
const restored: string[] = [];
|
||||
const memorySources = new Map((await options.resolveMemorySources?.(companyId) ?? []).map((source) => [source.agentId, source.root]));
|
||||
for (const line of stdout.trim().split("\n").filter(Boolean)) {
|
||||
const match = line.match(/^\d+ blob ([0-9a-f]+)\tcompanies\/[^/]+\/(agents\/([^/]+)\/(.+)|skills\/(.+))$/);
|
||||
if (!match) continue;
|
||||
const relative = match[2]!;
|
||||
restored.push(relative);
|
||||
if (dryRun) continue;
|
||||
const agentId = match[3];
|
||||
const agentRelative = match[4];
|
||||
const memoryRelative = agentRelative?.startsWith("memory/") ? agentRelative.slice("memory/".length) : null;
|
||||
const memoryRoot = agentId && memoryRelative ? memorySources.get(agentId) : null;
|
||||
const destination = agentId && agentRelative
|
||||
? memoryRoot
|
||||
? path.join(memoryRoot, memoryRelative!)
|
||||
: path.join(companyRoot, "agents", agentId, "instructions", agentRelative)
|
||||
: path.join(instance, "skills", companyId, match[5]!);
|
||||
const content = (await run("git", ["--git-dir", gitDir, "cat-file", "blob", match[1]!])).stdout;
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true });
|
||||
await fs.writeFile(destination, content);
|
||||
}
|
||||
return { restored, dryRun };
|
||||
} finally {
|
||||
if (temporaryRepo) await fs.rm(temporaryRepo, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
startWatcher(input: { listCompanyIds: () => Promise<string[]>; debounceMs?: number; sweepMs?: number }) {
|
||||
const debounceMs = input.debounceMs ?? 30_000;
|
||||
const sweepMs = input.sweepMs ?? 24 * 60 * 60 * 1_000;
|
||||
const pending = new Map<string, NodeJS.Timeout>();
|
||||
const fingerprints = new Map<string, string>();
|
||||
let stopped = false;
|
||||
const sweep = async (message: string) => {
|
||||
for (const companyId of await input.listCompanyIds()) {
|
||||
await this.commit({
|
||||
companyId,
|
||||
actor: { name: "paperclip-state-bot", email: "state-bot@paperclip.invalid" },
|
||||
message,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
};
|
||||
const poll = async () => {
|
||||
if (stopped) return;
|
||||
for (const companyId of await input.listCompanyIds()) {
|
||||
const files = await collect(companyId);
|
||||
const memory = [...files].filter(([filePath]) => filePath.includes("/memory/"));
|
||||
const fingerprint = createHash("sha256")
|
||||
.update(memory.map(([filePath, content]) => `${filePath}\0${createHash("sha256").update(content).digest("hex")}`).join("\n"))
|
||||
.digest("hex");
|
||||
const previous = fingerprints.get(companyId);
|
||||
fingerprints.set(companyId, fingerprint);
|
||||
if (previous && previous !== fingerprint) {
|
||||
clearTimeout(pending.get(companyId));
|
||||
pending.set(companyId, setTimeout(() => {
|
||||
pending.delete(companyId);
|
||||
void this.commit({
|
||||
companyId,
|
||||
actor: { name: "paperclip-state-bot", email: "state-bot@paperclip.invalid" },
|
||||
message: "claude-memory: capture external changes",
|
||||
});
|
||||
}, debounceMs));
|
||||
}
|
||||
}
|
||||
};
|
||||
const pollTimer = setInterval(() => void poll(), Math.min(5_000, debounceMs));
|
||||
const sweepTimer = setInterval(() => void sweep("state: daily drift sweep"), sweepMs);
|
||||
void poll();
|
||||
return () => {
|
||||
stopped = true;
|
||||
clearInterval(pollTimer);
|
||||
clearInterval(sweepTimer);
|
||||
for (const timer of pending.values()) clearTimeout(timer);
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type StateRepoService = ReturnType<typeof createStateRepoService>;
|
||||
Reference in new issue
Block a user