From 90062af85f8ae600bb24179d8655129c0eeb9f88 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 9 May 2026 21:35:17 -0500 Subject: [PATCH] Add LLM Wiki host support Co-Authored-By: Paperclip --- Dockerfile | 1 + cli/src/__tests__/home-paths.test.ts | 11 +- cli/src/__tests__/onboard.test.ts | 39 ++ cli/src/__tests__/spaces.test.ts | 218 +++++++++ cli/src/commands/configure.ts | 4 + cli/src/commands/onboard.ts | 7 +- cli/src/commands/spaces.ts | 269 +++++++++++ cli/src/config/home.ts | 73 +-- cli/src/index.ts | 2 + doc/CLI.md | 78 +++- doc/DATABASE.md | 10 +- doc/DEVELOPING.md | 55 ++- doc/SPEC-implementation.md | 6 +- ...-llm-wiki-paperclip-asset-security-gate.md | 135 ++++++ packages/adapter-utils/src/server-utils.ts | 118 ++++- .../adapters/acpx-local/src/server/execute.ts | 7 +- .../claude-local/src/server/claude-config.ts | 14 +- .../claude-local/src/server/prompt-cache.ts | 21 +- .../codex-local/src/server/codex-home.ts | 14 +- packages/db/src/backup.ts | 36 +- packages/db/src/runtime-config.test.ts | 20 + packages/db/src/runtime-config.ts | 47 +- .../plugin-bookmarks-example/.gitignore | 3 + .../plugin-bookmarks-example/README.md | 53 +++ .../esbuild.config.mjs | 17 + .../migrations/001_bookmarks.sql | 16 + .../plugin-bookmarks-example/package.json | 48 ++ .../rollup.config.mjs | 28 ++ .../plugin-bookmarks-example/src/manifest.ts | 102 +++++ .../plugin-bookmarks-example/src/ui/index.tsx | 359 +++++++++++++++ .../plugin-bookmarks-example/src/worker.ts | 421 ++++++++++++++++++ .../tests/plugin.spec.ts | 370 +++++++++++++++ .../plugin-bookmarks-example/tsconfig.json | 27 ++ .../plugin-bookmarks-example/vitest.config.ts | 8 + .../plugins/sdk/src/host-client-factory.ts | 26 ++ packages/plugins/sdk/src/index.ts | 5 + packages/plugins/sdk/src/protocol.ts | 20 +- packages/plugins/sdk/src/testing.ts | 142 +++++- packages/plugins/sdk/src/types.ts | 28 +- packages/plugins/sdk/src/worker-rpc-host.ts | 20 +- packages/shared/src/config-schema.ts | 20 +- packages/shared/src/constants.ts | 1 + packages/shared/src/index.ts | 3 + packages/shared/src/space-paths.test.ts | 74 +++ packages/shared/src/space-paths.ts | 247 ++++++++++ packages/shared/src/types/index.ts | 3 + packages/shared/src/types/plugin.ts | 49 +- packages/shared/src/validators/index.ts | 4 + packages/shared/src/validators/plugin.test.ts | 37 +- packages/shared/src/validators/plugin.ts | 61 ++- pnpm-workspace.yaml | 3 + .../__tests__/plugin-managed-skills.test.ts | 262 +++++++++++ server/src/home-paths.ts | 69 ++- server/src/routes/plugins.ts | 8 + server/src/services/agent-instructions.ts | 4 +- server/src/services/company-skills.ts | 4 +- server/src/services/feedback.ts | 4 +- server/src/services/plugin-host-services.ts | 36 ++ server/src/services/plugin-local-folders.ts | 48 ++ server/src/services/plugin-managed-agents.ts | 74 ++- server/src/services/plugin-managed-skills.ts | 342 ++++++++++++++ server/src/services/run-log-store.ts | 4 +- .../services/workspace-operation-log-store.ts | 4 +- 63 files changed, 4003 insertions(+), 236 deletions(-) create mode 100644 cli/src/__tests__/spaces.test.ts create mode 100644 cli/src/commands/spaces.ts create mode 100644 doc/plans/2026-05-06-llm-wiki-paperclip-asset-security-gate.md create mode 100644 packages/plugins/examples/plugin-bookmarks-example/.gitignore create mode 100644 packages/plugins/examples/plugin-bookmarks-example/README.md create mode 100644 packages/plugins/examples/plugin-bookmarks-example/esbuild.config.mjs create mode 100644 packages/plugins/examples/plugin-bookmarks-example/migrations/001_bookmarks.sql create mode 100644 packages/plugins/examples/plugin-bookmarks-example/package.json create mode 100644 packages/plugins/examples/plugin-bookmarks-example/rollup.config.mjs create mode 100644 packages/plugins/examples/plugin-bookmarks-example/src/manifest.ts create mode 100644 packages/plugins/examples/plugin-bookmarks-example/src/ui/index.tsx create mode 100644 packages/plugins/examples/plugin-bookmarks-example/src/worker.ts create mode 100644 packages/plugins/examples/plugin-bookmarks-example/tests/plugin.spec.ts create mode 100644 packages/plugins/examples/plugin-bookmarks-example/tsconfig.json create mode 100644 packages/plugins/examples/plugin-bookmarks-example/vitest.config.ts create mode 100644 packages/shared/src/space-paths.test.ts create mode 100644 packages/shared/src/space-paths.ts create mode 100644 server/src/__tests__/plugin-managed-skills.test.ts create mode 100644 server/src/services/plugin-managed-skills.ts diff --git a/Dockerfile b/Dockerfile index 6fa685d935f..ec82a186f03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ COPY packages/adapters/openclaw-gateway/package.json packages/adapters/openclaw- COPY packages/adapters/opencode-local/package.json packages/adapters/opencode-local/ COPY packages/adapters/pi-local/package.json packages/adapters/pi-local/ COPY packages/plugins/sdk/package.json packages/plugins/sdk/ +COPY packages/plugins/plugin-llm-wiki/package.json packages/plugins/plugin-llm-wiki/ COPY --parents packages/plugins/sandbox-providers/./*/package.json packages/plugins/sandbox-providers/ COPY packages/plugins/paperclip-plugin-fake-sandbox/package.json packages/plugins/paperclip-plugin-fake-sandbox/ COPY patches/ patches/ diff --git a/cli/src/__tests__/home-paths.test.ts b/cli/src/__tests__/home-paths.test.ts index 1d9c654eeef..53ae028a4aa 100644 --- a/cli/src/__tests__/home-paths.test.ts +++ b/cli/src/__tests__/home-paths.test.ts @@ -1,3 +1,4 @@ +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -16,13 +17,15 @@ describe("home path resolution", () => { }); it("defaults to ~/.paperclip and default instance", () => { - delete process.env.PAPERCLIP_HOME; + const home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-home-paths-")); + process.env.PAPERCLIP_HOME = home; delete process.env.PAPERCLIP_INSTANCE_ID; const paths = describeLocalInstancePaths(); - expect(paths.homeDir).toBe(path.resolve(os.homedir(), ".paperclip")); + expect(paths.homeDir).toBe(home); expect(paths.instanceId).toBe("default"); - expect(paths.configPath).toBe(path.resolve(os.homedir(), ".paperclip", "instances", "default", "config.json")); + expect(paths.activeSpaceId).toBe("default"); + expect(paths.configPath).toBe(path.resolve(home, "instances", "default", "spaces", "default", "config.json")); }); it("supports PAPERCLIP_HOME and explicit instance ids", () => { @@ -34,7 +37,7 @@ describe("home path resolution", () => { }); it("rejects invalid instance ids", () => { - expect(() => resolvePaperclipInstanceId("bad/id")).toThrow(/Invalid instance id/); + expect(() => resolvePaperclipInstanceId("bad/id")).toThrow(/Invalid PAPERCLIP_INSTANCE_ID/); }); it("expands ~ prefixes", () => { diff --git a/cli/src/__tests__/onboard.test.ts b/cli/src/__tests__/onboard.test.ts index b66c20fa8c1..851d476e17c 100644 --- a/cli/src/__tests__/onboard.test.ts +++ b/cli/src/__tests__/onboard.test.ts @@ -6,6 +6,7 @@ import { onboard } from "../commands/onboard.js"; import type { PaperclipConfig } from "../config/schema.js"; const ORIGINAL_ENV = { ...process.env }; +const ORIGINAL_CWD = process.cwd(); function createExistingConfigFixture() { const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-onboard-")); @@ -85,10 +86,19 @@ describe("onboard", () => { delete process.env.PAPERCLIP_AGENT_JWT_SECRET; delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; delete process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; + delete process.env.PAPERCLIP_HOME; + delete process.env.PAPERCLIP_CONFIG; + delete process.env.PAPERCLIP_INSTANCE_ID; + delete process.env.PAPERCLIP_SPACE_ID; + delete process.env.PAPERCLIP_BIND; + delete process.env.PAPERCLIP_BIND_HOST; + delete process.env.PAPERCLIP_TAILNET_BIND_HOST; + delete process.env.HOST; }); afterEach(() => { process.env = { ...ORIGINAL_ENV }; + process.chdir(ORIGINAL_CWD); }); it("preserves an existing config when rerun without flags", async () => { @@ -125,6 +135,35 @@ describe("onboard", () => { expect(raw.server.host).toBe("127.0.0.1"); }); + it("creates default-space config and data paths for a fresh PAPERCLIP_HOME", async () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-onboard-space-")); + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-onboard-cwd-")); + process.chdir(cwd); + process.env.PAPERCLIP_HOME = home; + + await onboard({ yes: true, invokedByRun: true }); + + const instanceRoot = path.join(home, "instances", "default"); + const spaceRoot = path.join(instanceRoot, "spaces", "default"); + const configPath = path.join(spaceRoot, "config.json"); + const registryPath = path.join(instanceRoot, "config.json"); + const raw = JSON.parse(fs.readFileSync(configPath, "utf8")) as PaperclipConfig; + const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")) as { activeSpaceId?: string }; + + expect(registry.activeSpaceId).toBe("default"); + expect(raw.database.embeddedPostgresDataDir).toBe(path.join(spaceRoot, "db")); + expect(raw.database.backup.dir).toBe(path.join(spaceRoot, "data", "backups")); + expect(raw.logging.logDir).toBe(path.join(spaceRoot, "logs")); + expect(raw.storage.localDisk.baseDir).toBe(path.join(spaceRoot, "data", "storage")); + expect(raw.secrets.localEncrypted.keyFilePath).toBe(path.join(spaceRoot, "secrets", "master.key")); + expect(fs.existsSync(path.join(spaceRoot, ".env"))).toBe(true); + expect(fs.existsSync(path.join(spaceRoot, "secrets", "master.key"))).toBe(true); + + for (const rootName of ["db", "data", "secrets", "logs", "workspaces", "projects", "companies"]) { + expect(fs.existsSync(path.join(instanceRoot, rootName))).toBe(false); + } + }); + it("supports authenticated/private quickstart bind presets", async () => { const configPath = createFreshConfigPath(); process.env.PAPERCLIP_TAILNET_BIND_HOST = "100.64.0.8"; diff --git a/cli/src/__tests__/spaces.test.ts b/cli/src/__tests__/spaces.test.ts new file mode 100644 index 00000000000..7cb200aed4a --- /dev/null +++ b/cli/src/__tests__/spaces.test.ts @@ -0,0 +1,218 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { defaultServerRunningCheck, migrateDefaultSpaceInstall } from "../commands/spaces.js"; +import { readConfig, resolveConfigPath } from "../config/store.js"; +import type { PaperclipConfig } from "../config/schema.js"; + +const ORIGINAL_ENV = { ...process.env }; +const ORIGINAL_CWD = process.cwd(); + +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); +} + +function writeText(filePath: string, value: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, value); +} + +function createLegacyInstallFixture() { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-space-migrate-home-")); + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-space-migrate-cwd-")); + process.chdir(cwd); + process.env.PAPERCLIP_HOME = home; + delete process.env.PAPERCLIP_CONFIG; + delete process.env.PAPERCLIP_INSTANCE_ID; + delete process.env.PAPERCLIP_SPACE_ID; + + const instanceRoot = path.join(home, "instances", "default"); + const spaceRoot = path.join(instanceRoot, "spaces", "default"); + const config: PaperclipConfig = { + $meta: { + version: 1, + updatedAt: "2026-05-09T00:00:00.000Z", + source: "onboard", + }, + database: { + mode: "embedded-postgres", + embeddedPostgresDataDir: path.join(instanceRoot, "db"), + embeddedPostgresPort: 54329, + backup: { + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: path.join(instanceRoot, "data", "backups"), + }, + }, + logging: { + mode: "file", + logDir: path.join(instanceRoot, "logs"), + }, + server: { + deploymentMode: "local_trusted", + exposure: "private", + host: "127.0.0.1", + port: 3100, + allowedHostnames: [], + serveUi: true, + }, + auth: { + baseUrlMode: "auto", + disableSignUp: false, + }, + telemetry: { + enabled: true, + }, + storage: { + provider: "local_disk", + localDisk: { + baseDir: path.join(instanceRoot, "data", "storage"), + }, + s3: { + bucket: "paperclip", + region: "us-east-1", + prefix: "", + forcePathStyle: false, + }, + }, + secrets: { + provider: "local_encrypted", + strictMode: false, + localEncrypted: { + keyFilePath: path.join(instanceRoot, "secrets", "master.key"), + }, + }, + }; + + writeJson(path.join(instanceRoot, "config.json"), config); + writeText(path.join(instanceRoot, ".env"), "PAPERCLIP_AGENT_JWT_SECRET=test-secret\n"); + writeText(path.join(instanceRoot, "db", "PG_VERSION"), "17\n"); + writeText(path.join(instanceRoot, "data", "storage", "hello.txt"), "hello\n"); + writeText(path.join(instanceRoot, "logs", "server.log"), "log\n"); + writeText(path.join(instanceRoot, "secrets", "master.key"), "01234567890123456789012345678901"); + writeText(path.join(instanceRoot, "workspaces", "agent-1", "README.md"), "workspace\n"); + writeText(path.join(instanceRoot, "projects", "company", "project", "repo", "README.md"), "project\n"); + writeText(path.join(instanceRoot, "companies", "company-1", "codex-home", "config.toml"), "model=\"x\"\n"); + writeText(path.join(instanceRoot, "codex-home", "config.toml"), "model=\"shared\"\n"); + + return { instanceRoot, spaceRoot }; +} + +beforeEach(() => { + process.env = { ...ORIGINAL_ENV }; +}); + +afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + process.chdir(ORIGINAL_CWD); + vi.unstubAllGlobals(); +}); + +describe("spaces migrate-default", () => { + it("moves a legacy root-shaped install into spaces/default and rewrites config paths", async () => { + const fixture = createLegacyInstallFixture(); + + const result = await migrateDefaultSpaceInstall({ + serverRunningCheck: async () => false, + }); + + expect(result.status).toBe("migrated"); + expect(result.movedPaths).toEqual([ + "config.json", + ".env", + "db", + "data", + "logs", + "secrets", + "workspaces", + "projects", + "companies", + "codex-home", + ]); + + expect(fs.existsSync(path.join(fixture.instanceRoot, "db"))).toBe(false); + expect(fs.readFileSync(path.join(fixture.spaceRoot, ".env"), "utf8")).toContain("PAPERCLIP_AGENT_JWT_SECRET"); + expect(fs.readFileSync(path.join(fixture.spaceRoot, "db", "PG_VERSION"), "utf8")).toBe("17\n"); + expect(fs.readFileSync(path.join(fixture.spaceRoot, "data", "storage", "hello.txt"), "utf8")).toBe("hello\n"); + expect(fs.readFileSync(path.join(fixture.spaceRoot, "companies", "company-1", "codex-home", "config.toml"), "utf8")).toContain("model"); + expect(fs.readFileSync(path.join(fixture.spaceRoot, "codex-home", "config.toml"), "utf8")).toContain("shared"); + + const migratedConfig = JSON.parse(fs.readFileSync(path.join(fixture.spaceRoot, "config.json"), "utf8")) as PaperclipConfig; + expect(migratedConfig.database.embeddedPostgresDataDir).toBe(path.join(fixture.spaceRoot, "db")); + expect(migratedConfig.database.backup.dir).toBe(path.join(fixture.spaceRoot, "data", "backups")); + expect(migratedConfig.logging.logDir).toBe(path.join(fixture.spaceRoot, "logs")); + expect(migratedConfig.storage.localDisk.baseDir).toBe(path.join(fixture.spaceRoot, "data", "storage")); + expect(migratedConfig.secrets.localEncrypted.keyFilePath).toBe(path.join(fixture.spaceRoot, "secrets", "master.key")); + + const marker = JSON.parse(fs.readFileSync(path.join(fixture.instanceRoot, "config.json"), "utf8")) as { + activeSpaceId?: string; + defaultSpaceMigration?: { sourceRoot?: string; destinationRoot?: string; movedPaths?: string[] }; + }; + expect(marker.activeSpaceId).toBe("default"); + expect(marker.defaultSpaceMigration).toMatchObject({ + sourceRoot: fixture.instanceRoot, + destinationRoot: fixture.spaceRoot, + movedPaths: result.movedPaths, + }); + + expect(resolveConfigPath()).toBe(path.join(fixture.spaceRoot, "config.json")); + expect(readConfig()?.database.embeddedPostgresDataDir).toBe(path.join(fixture.spaceRoot, "db")); + }); + + it("refuses to merge when destination data already exists", async () => { + const fixture = createLegacyInstallFixture(); + writeText(path.join(fixture.spaceRoot, "db", "PG_VERSION"), "existing\n"); + + await expect(migrateDefaultSpaceInstall({ + serverRunningCheck: async () => false, + })).rejects.toThrow(/destination paths already exist:[\s\S]*db/); + + expect(fs.readFileSync(path.join(fixture.instanceRoot, "db", "PG_VERSION"), "utf8")).toBe("17\n"); + expect(fs.readFileSync(path.join(fixture.spaceRoot, "db", "PG_VERSION"), "utf8")).toBe("existing\n"); + }); + + it("refuses to migrate when the server appears to be running", async () => { + const fixture = createLegacyInstallFixture(); + + await expect(migrateDefaultSpaceInstall({ + serverRunningCheck: async () => true, + })).rejects.toThrow(/server appears to be running/); + + expect(fs.existsSync(path.join(fixture.instanceRoot, "config.json"))).toBe(true); + expect(fs.existsSync(path.join(fixture.spaceRoot, "config.json"))).toBe(false); + }); + + it("checks the destination space config when the instance root only has a space registry", async () => { + const fixture = createLegacyInstallFixture(); + const destinationConfig = JSON.parse(fs.readFileSync(path.join(fixture.instanceRoot, "config.json"), "utf8")) as PaperclipConfig; + fs.rmSync(path.join(fixture.instanceRoot, "config.json")); + writeJson(path.join(fixture.instanceRoot, "config.json"), { + $meta: { version: 1, updatedAt: "2026-05-10T00:00:00.000Z", source: "system" }, + activeSpaceId: "default", + spaces: [{ id: "default", root: "spaces/default", createdAt: "2026-05-10T00:00:00.000Z" }], + }); + writeJson(path.join(fixture.spaceRoot, "config.json"), { + ...destinationConfig, + server: { + ...destinationConfig.server, + port: 43210, + }, + }); + const fetchMock = vi.fn(async () => ({ status: 204 })); + vi.stubGlobal("fetch", fetchMock); + + await expect(defaultServerRunningCheck({ + instanceId: "default", + sourceRoot: fixture.instanceRoot, + destinationRoot: fixture.spaceRoot, + legacyRuntimeConfig: false, + sourcePathNames: [], + conflicts: [], + })).resolves.toBe(true); + + expect(fetchMock).toHaveBeenCalledWith("http://127.0.0.1:43210/api/health", expect.any(Object)); + }); +}); diff --git a/cli/src/commands/configure.ts b/cli/src/commands/configure.ts index b7d2dcbc07d..4b96d10a3f0 100644 --- a/cli/src/commands/configure.ts +++ b/cli/src/commands/configure.ts @@ -15,6 +15,7 @@ import { resolveDefaultLogsDir, resolvePaperclipInstanceId, } from "../config/home.js"; +import { ensureDefaultSpaceRegistry } from "@paperclipai/shared/space-paths"; import { printPaperclipCliBanner } from "../utils/banner.js"; type Section = "llm" | "database" | "logging" | "server" | "storage" | "secrets"; @@ -177,6 +178,9 @@ export async function configure(opts: { config.$meta.updatedAt = new Date().toISOString(); config.$meta.source = "configure"; + if (configPath === resolveConfigPath()) { + ensureDefaultSpaceRegistry({ source: "configure" }); + } writeConfig(config, opts.config); p.log.success(`${SECTION_LABELS[section]} configuration updated.`); diff --git a/cli/src/commands/onboard.ts b/cli/src/commands/onboard.ts index 62158e05bbf..c0f409092ee 100644 --- a/cli/src/commands/onboard.ts +++ b/cli/src/commands/onboard.ts @@ -36,6 +36,7 @@ import { resolveDefaultLogsDir, resolvePaperclipInstanceId, } from "../config/home.js"; +import { ensureDefaultSpaceRegistry } from "@paperclipai/shared/space-paths"; import { bootstrapCeoInvite } from "./auth-bootstrap-ceo.js"; import { printPaperclipCliBanner } from "../utils/banner.js"; import { @@ -333,7 +334,7 @@ export async function onboard(opts: OnboardOptions): Promise { const instance = describeLocalInstancePaths(resolvePaperclipInstanceId()); p.log.message( pc.dim( - `Local home: ${instance.homeDir} | instance: ${instance.instanceId} | config: ${configPath}`, + `Local home: ${instance.homeDir} | instance: ${instance.instanceId} | space: ${instance.activeSpaceId} | config: ${configPath}`, ), ); @@ -625,6 +626,10 @@ export async function onboard(opts: OnboardOptions): Promise { p.log.message(pc.dim(`Using existing local secrets key file at ${keyResult.path}`)); } + if (configPath === resolveConfigPath()) { + ensureDefaultSpaceRegistry({ source: "onboard" }); + } + writeConfig(config, opts.config); if (tc) trackInstallCompleted(tc, { diff --git a/cli/src/commands/spaces.ts b/cli/src/commands/spaces.ts new file mode 100644 index 00000000000..d443f650532 --- /dev/null +++ b/cli/src/commands/spaces.ts @@ -0,0 +1,269 @@ +import fs from "node:fs"; +import path from "node:path"; +import { Command } from "commander"; +import pc from "picocolors"; +import { + DEFAULT_PAPERCLIP_SPACE_ID, + DEFAULT_SPACE_ADAPTER_LOCAL_PATH_NAMES, + DEFAULT_SPACE_OWNED_PATH_NAMES, + createDefaultSpaceRegistry, + expandHomePrefix, + isPaperclipRuntimeConfig, + resolvePaperclipInstanceConfigPath, + resolvePaperclipInstanceId, + resolvePaperclipInstanceRoot, + resolvePaperclipSpacesRoot, +} from "@paperclipai/shared/space-paths"; + +type JsonObject = Record; + +export interface DefaultSpaceMigrationPlan { + instanceId: string; + sourceRoot: string; + destinationRoot: string; + legacyRuntimeConfig: boolean; + sourcePathNames: string[]; + conflicts: Array<{ pathName: string; sourcePath: string; destinationPath: string }>; +} + +export interface DefaultSpaceMigrationResult { + status: "migrated" | "noop" | "dry_run"; + plan: DefaultSpaceMigrationPlan; + movedPaths: string[]; + markerPath: string | null; +} + +export interface MigrateDefaultSpaceOptions { + instanceId?: string; + dryRun?: boolean; + skipServerCheck?: boolean; + serverRunningCheck?: (plan: DefaultSpaceMigrationPlan) => Promise; +} + +const MIGRATION_PATH_NAMES = [ + ...DEFAULT_SPACE_OWNED_PATH_NAMES, + ...DEFAULT_SPACE_ADAPTER_LOCAL_PATH_NAMES, +] as const; + +function readJsonIfPresent(filePath: string): unknown | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + return null; + } +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isPathWithin(rootPath: string, candidatePath: string): boolean { + const root = path.resolve(rootPath); + const candidate = path.resolve(candidatePath); + const relative = path.relative(root, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function rewriteSpaceRootString(value: string, sourceRoot: string, destinationRoot: string): string { + const resolved = path.resolve(expandHomePrefix(value)); + if (!isPathWithin(sourceRoot, resolved)) return value; + const relative = path.relative(path.resolve(sourceRoot), resolved); + return path.resolve(destinationRoot, relative); +} + +function rewriteConfigPaths(value: unknown, sourceRoot: string, destinationRoot: string): unknown { + if (typeof value === "string") return rewriteSpaceRootString(value, sourceRoot, destinationRoot); + if (Array.isArray(value)) return value.map((entry) => rewriteConfigPaths(entry, sourceRoot, destinationRoot)); + if (!isObject(value)) return value; + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [key, rewriteConfigPaths(entry, sourceRoot, destinationRoot)]), + ); +} + +function writeJson(filePath: string, value: unknown, mode = 0o600): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, { mode }); +} + +function atomicWriteJson(filePath: string, value: unknown, mode = 0o600): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const tmpPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.tmp`); + writeJson(tmpPath, value, mode); + fs.renameSync(tmpPath, filePath); +} + +export function buildDefaultSpaceMigrationPlan(options: Pick = {}): DefaultSpaceMigrationPlan { + const instanceId = resolvePaperclipInstanceId(options.instanceId); + const sourceRoot = resolvePaperclipInstanceRoot({ instanceId }); + const destinationRoot = path.resolve(resolvePaperclipSpacesRoot({ instanceId }), DEFAULT_PAPERCLIP_SPACE_ID); + const instanceConfigPath = resolvePaperclipInstanceConfigPath({ instanceId }); + const legacyRuntimeConfig = isPaperclipRuntimeConfig(readJsonIfPresent(instanceConfigPath)); + + const sourcePathNames = MIGRATION_PATH_NAMES.filter((pathName) => { + if (pathName === "config.json" && !legacyRuntimeConfig) return false; + return fs.existsSync(path.join(sourceRoot, pathName)); + }); + + const conflicts = sourcePathNames + .map((pathName) => ({ + pathName, + sourcePath: path.join(sourceRoot, pathName), + destinationPath: path.join(destinationRoot, pathName), + })) + .filter((entry) => fs.existsSync(entry.destinationPath)); + + return { + instanceId, + sourceRoot, + destinationRoot, + legacyRuntimeConfig, + sourcePathNames, + conflicts, + }; +} + +function readServerEndpointFromConfig(configPath: string): { host: string; port: number } | null { + const parsed = readJsonIfPresent(configPath); + if (!isObject(parsed)) return null; + const server = parsed.server; + if (!isObject(server)) return null; + const rawHost = typeof server.host === "string" && server.host.trim().length > 0 + ? server.host.trim() + : "127.0.0.1"; + const host = rawHost === "0.0.0.0" || rawHost === "::" ? "127.0.0.1" : rawHost; + const port = typeof server.port === "number" && Number.isInteger(server.port) ? server.port : 3100; + if (port <= 0 || port > 65535) return null; + return { host, port }; +} + +export async function defaultServerRunningCheck(plan: DefaultSpaceMigrationPlan): Promise { + const endpoint = + readServerEndpointFromConfig(path.join(plan.sourceRoot, "config.json")) ?? + readServerEndpointFromConfig(path.join(plan.destinationRoot, "config.json")); + if (!endpoint) return false; + + const host = endpoint.host.includes(":") ? `[${endpoint.host}]` : endpoint.host; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 500); + try { + const response = await fetch(`http://${host}:${endpoint.port}/api/health`, { + signal: controller.signal, + }); + return response.status >= 200 && response.status < 500; + } catch { + return false; + } finally { + clearTimeout(timeout); + } +} + +function assertMigrationPreflight(plan: DefaultSpaceMigrationPlan): void { + if (plan.conflicts.length === 0) return; + const details = plan.conflicts + .map((conflict) => `- ${conflict.pathName}: ${conflict.sourcePath} -> ${conflict.destinationPath}`) + .join("\n"); + throw new Error(`Cannot migrate default space because destination paths already exist:\n${details}`); +} + +function moveConfigWithRewrittenPaths(sourcePath: string, destinationPath: string, sourceRoot: string, destinationRoot: string): void { + const parsed = JSON.parse(fs.readFileSync(sourcePath, "utf8")) as unknown; + const rewritten = rewriteConfigPaths(parsed, sourceRoot, destinationRoot); + atomicWriteJson(destinationPath, rewritten); + fs.unlinkSync(sourcePath); +} + +function writeMigrationMarker(plan: DefaultSpaceMigrationPlan, movedPaths: string[]): string { + const migratedAt = new Date().toISOString(); + const registry = createDefaultSpaceRegistry("system"); + registry.$meta.updatedAt = migratedAt; + registry.defaultSpaceMigration = { + migratedAt, + sourceRoot: plan.sourceRoot, + destinationRoot: plan.destinationRoot, + movedPaths, + }; + + const markerPath = path.join(plan.sourceRoot, "config.json"); + writeJson(markerPath, registry); + return markerPath; +} + +export async function migrateDefaultSpaceInstall( + options: MigrateDefaultSpaceOptions = {}, +): Promise { + const plan = buildDefaultSpaceMigrationPlan(options); + + if (plan.sourcePathNames.length === 0) { + return { status: "noop", plan, movedPaths: [], markerPath: null }; + } + + assertMigrationPreflight(plan); + + const serverRunningCheck = options.serverRunningCheck ?? defaultServerRunningCheck; + if (!options.skipServerCheck && await serverRunningCheck(plan)) { + throw new Error( + `Cannot migrate default space while a Paperclip server appears to be running for ${plan.sourceRoot}. Stop Paperclip and retry.`, + ); + } + + if (options.dryRun) { + return { status: "dry_run", plan, movedPaths: plan.sourcePathNames, markerPath: null }; + } + + fs.mkdirSync(plan.destinationRoot, { recursive: true }); + const movedPaths: string[] = []; + + for (const pathName of plan.sourcePathNames) { + const sourcePath = path.join(plan.sourceRoot, pathName); + const destinationPath = path.join(plan.destinationRoot, pathName); + fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); + + if (pathName === "config.json") { + moveConfigWithRewrittenPaths(sourcePath, destinationPath, plan.sourceRoot, plan.destinationRoot); + } else { + fs.renameSync(sourcePath, destinationPath); + } + movedPaths.push(pathName); + } + + const markerPath = writeMigrationMarker(plan, movedPaths); + return { status: "migrated", plan, movedPaths, markerPath }; +} + +function printMigrationResult(result: DefaultSpaceMigrationResult): void { + if (result.status === "noop") { + console.log(pc.dim(`No legacy default-space data found under ${result.plan.sourceRoot}.`)); + return; + } + + const verb = result.status === "dry_run" ? "Would migrate" : "Migrated"; + console.log(`${pc.green(verb)} default space:`); + console.log(` ${pc.dim("from")} ${result.plan.sourceRoot}`); + console.log(` ${pc.dim("to")} ${result.plan.destinationRoot}`); + for (const pathName of result.movedPaths) { + console.log(` ${pc.dim("-")} ${pathName}`); + } + if (result.markerPath) { + console.log(` ${pc.dim("marker")} ${result.markerPath}`); + } +} + +export function registerSpacesCommands(program: Command): void { + const spaces = program.command("spaces").description("Manage local Paperclip spaces"); + + spaces + .command("migrate-default") + .description("Offline migration from legacy root-shaped instance data into spaces/default") + .option("-i, --instance ", "Local instance id (default: default)") + .option("--dry-run", "Show the migration plan without moving files", false) + .option("--skip-server-check", "Skip the local /api/health preflight", false) + .action(async (opts: { instance?: string; dryRun?: boolean; skipServerCheck?: boolean }) => { + const result = await migrateDefaultSpaceInstall({ + instanceId: opts.instance, + dryRun: opts.dryRun, + skipServerCheck: opts.skipServerCheck, + }); + printMigrationResult(result); + }); +} diff --git a/cli/src/config/home.ts b/cli/src/config/home.ts index ef4d8e09386..0b93a70ed26 100644 --- a/cli/src/config/home.ts +++ b/cli/src/config/home.ts @@ -1,32 +1,38 @@ -import os from "node:os"; import path from "node:path"; +import { + expandHomePrefix, + resolveDefaultBackupDir as resolveSharedDefaultBackupDir, + resolveDefaultEmbeddedPostgresDir as resolveSharedDefaultEmbeddedPostgresDir, + resolveDefaultLogsDir as resolveSharedDefaultLogsDir, + resolveDefaultSecretsKeyFilePath as resolveSharedDefaultSecretsKeyFilePath, + resolveDefaultStorageDir as resolveSharedDefaultStorageDir, + resolveHomeAwarePath, + resolvePaperclipConfigPathForInstance, + resolvePaperclipHomeDir, + resolvePaperclipInstanceId, + resolvePaperclipInstanceRoot as resolveSharedPaperclipInstanceRoot, + resolvePaperclipSpaceId, + resolvePaperclipSpaceRoot as resolveSharedPaperclipSpaceRoot, +} from "@paperclipai/shared/space-paths"; -const DEFAULT_INSTANCE_ID = "default"; -const INSTANCE_ID_RE = /^[a-zA-Z0-9_-]+$/; - -export function resolvePaperclipHomeDir(): string { - const envHome = process.env.PAPERCLIP_HOME?.trim(); - if (envHome) return path.resolve(expandHomePrefix(envHome)); - return path.resolve(os.homedir(), ".paperclip"); -} - -export function resolvePaperclipInstanceId(override?: string): string { - const raw = override?.trim() || process.env.PAPERCLIP_INSTANCE_ID?.trim() || DEFAULT_INSTANCE_ID; - if (!INSTANCE_ID_RE.test(raw)) { - throw new Error( - `Invalid instance id '${raw}'. Allowed characters: letters, numbers, '_' and '-'.`, - ); - } - return raw; -} +export { + expandHomePrefix, + resolveHomeAwarePath, + resolvePaperclipHomeDir, + resolvePaperclipInstanceId, + resolvePaperclipSpaceId, +}; export function resolvePaperclipInstanceRoot(instanceId?: string): string { - const id = resolvePaperclipInstanceId(instanceId); - return path.resolve(resolvePaperclipHomeDir(), "instances", id); + return resolveSharedPaperclipInstanceRoot({ instanceId }); +} + +export function resolvePaperclipSpaceRoot(instanceId?: string): string { + return resolveSharedPaperclipSpaceRoot({ instanceId }); } export function resolveDefaultConfigPath(instanceId?: string): string { - return path.resolve(resolvePaperclipInstanceRoot(instanceId), "config.json"); + return resolvePaperclipConfigPathForInstance({ instanceId }); } export function resolveDefaultContextPath(): string { @@ -38,38 +44,39 @@ export function resolveDefaultCliAuthPath(): string { } export function resolveDefaultEmbeddedPostgresDir(instanceId?: string): string { - return path.resolve(resolvePaperclipInstanceRoot(instanceId), "db"); + return resolveSharedDefaultEmbeddedPostgresDir({ instanceId }); } export function resolveDefaultLogsDir(instanceId?: string): string { - return path.resolve(resolvePaperclipInstanceRoot(instanceId), "logs"); + return resolveSharedDefaultLogsDir({ instanceId }); } export function resolveDefaultSecretsKeyFilePath(instanceId?: string): string { - return path.resolve(resolvePaperclipInstanceRoot(instanceId), "secrets", "master.key"); + return resolveSharedDefaultSecretsKeyFilePath({ instanceId }); } export function resolveDefaultStorageDir(instanceId?: string): string { - return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage"); + return resolveSharedDefaultStorageDir({ instanceId }); } export function resolveDefaultBackupDir(instanceId?: string): string { - return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "backups"); -} - -export function expandHomePrefix(value: string): string { - if (value === "~") return os.homedir(); - if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); - return value; + return resolveSharedDefaultBackupDir({ instanceId }); } export function describeLocalInstancePaths(instanceId?: string) { const resolvedInstanceId = resolvePaperclipInstanceId(instanceId); const instanceRoot = resolvePaperclipInstanceRoot(resolvedInstanceId); + const activeSpaceId = resolvePaperclipSpaceId({ instanceId: resolvedInstanceId }); + const activeSpaceRoot = resolveSharedPaperclipSpaceRoot({ + instanceId: resolvedInstanceId, + spaceId: activeSpaceId, + }); return { homeDir: resolvePaperclipHomeDir(), instanceId: resolvedInstanceId, instanceRoot, + activeSpaceId, + activeSpaceRoot, configPath: resolveDefaultConfigPath(resolvedInstanceId), embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId), backupDir: resolveDefaultBackupDir(resolvedInstanceId), diff --git a/cli/src/index.ts b/cli/src/index.ts index f1a2084a727..a718f02e409 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -25,6 +25,7 @@ import { initTelemetryFromConfigFile, flushTelemetry } from "./telemetry.js"; import { registerWorktreeCommands } from "./commands/worktree.js"; import { registerPluginCommands } from "./commands/client/plugin.js"; import { registerClientAuthCommands } from "./commands/client/auth.js"; +import { registerSpacesCommands } from "./commands/spaces.js"; import { cliVersion } from "./version.js"; const program = new Command(); @@ -152,6 +153,7 @@ registerSecretCommands(program); registerWorktreeCommands(program); registerEnvLabCommands(program); registerPluginCommands(program); +registerSpacesCommands(program); const auth = program.command("auth").description("Authentication and bootstrap utilities"); diff --git a/doc/CLI.md b/doc/CLI.md index 29c494e3bed..bbc4bb871f4 100644 --- a/doc/CLI.md +++ b/doc/CLI.md @@ -204,20 +204,84 @@ pnpm paperclipai heartbeat run --agent-id [--api-base http://localhos ## Local Storage Defaults -Default local instance root is `~/.paperclip/instances/default`: +Local Paperclip data is split between an instance root and one or more spaces inside it. The default install creates a single space called `default` under `spaces/default`. -- config: `~/.paperclip/instances/default/config.json` -- embedded db: `~/.paperclip/instances/default/db` -- logs: `~/.paperclip/instances/default/logs` -- storage: `~/.paperclip/instances/default/data/storage` -- secrets key: `~/.paperclip/instances/default/secrets/master.key` +```text +~/.paperclip/ # PAPERCLIP_HOME +└── instances/ + └── default/ # instance root (PAPERCLIP_INSTANCE_ID) + ├── config.json # space registry (activeSpaceId, known spaces) + └── spaces/ + └── default/ # active space root (PAPERCLIP_SPACE_ID) + ├── config.json # runtime config for this space + ├── .env # space-scoped env file + ├── db/ # embedded PostgreSQL data + ├── data/ + │ ├── storage/ # local_disk uploads + │ └── backups/ # automatic DB backups + ├── logs/ + ├── secrets/ + │ └── master.key # local_encrypted master key + ├── workspaces/ # default agent workspaces + ├── projects/ # project execution workspaces + ├── companies/ # per-company adapter homes (e.g. codex-home) + └── codex-home/ # per-instance codex home (when not company-scoped) +``` -Override base home or instance with env vars: +Default paths for the canonical install: + +- instance registry: `~/.paperclip/instances/default/config.json` +- space config: `~/.paperclip/instances/default/spaces/default/config.json` +- embedded db: `~/.paperclip/instances/default/spaces/default/db` +- logs: `~/.paperclip/instances/default/spaces/default/logs` +- storage: `~/.paperclip/instances/default/spaces/default/data/storage` +- secrets key: `~/.paperclip/instances/default/spaces/default/secrets/master.key` + +The instance root holds only cross-space metadata. All runtime state — database, storage, secrets, logs, agent workspaces — lives inside the active space. + +Override base home, instance, or active space with env vars: ```sh PAPERCLIP_HOME=/custom/home PAPERCLIP_INSTANCE_ID=dev pnpm paperclipai run +PAPERCLIP_SPACE_ID=staging pnpm paperclipai run # use an alternate space inside the same instance ``` +`PAPERCLIP_SPACE_ID` overrides whatever `activeSpaceId` is recorded in the instance registry. When unset, Paperclip resolves the active space from `instances//config.json` and falls back to `default`. + +## Migrating a Legacy Default-Space Install + +Earlier Paperclip versions stored default-space data directly at the instance root (for example `~/.paperclip/instances/default/db`). Those installs continue to start through a compatibility resolver, but new code paths assume `spaces/default`. Migrate explicitly with: + +```sh +# Stop Paperclip first; the migration refuses to run while a local server is up. +pnpm paperclipai spaces migrate-default + +# Preview the move without touching files. +pnpm paperclipai spaces migrate-default --dry-run + +# Operate on a non-default instance. +pnpm paperclipai spaces migrate-default --instance dev +``` + +The command: + +- detects a legacy root-shaped install by inspecting `instances//config.json` +- preflight-checks `http://:/api/health` and refuses to run when the server responds +- refuses to merge if the destination already contains conflicting paths under `spaces/default/` +- moves only known space-owned paths: `config.json`, `.env`, `db`, `data`, `logs`, `secrets`, `workspaces`, `projects`, `companies`, and the legacy top-level `codex-home` +- rewrites absolute paths inside the migrated `config.json` from the instance root to the new space root (`embeddedPostgresDataDir`, backup `dir`, `logging.logDir`, `storage.localDisk.baseDir`, `secrets.localEncrypted.keyFilePath`) +- writes the registry/marker back at `instances//config.json` recording `activeSpaceId`, the migration timestamp, and the moved paths + +Available flags: + +| Option | Description | +|---|---| +| `-i, --instance ` | Local instance id (default: `default`) | +| `--dry-run` | Show the migration plan without moving files | +| `--skip-server-check` | Skip the `/api/health` preflight (only for offline installs) | + +Restart Paperclip after a successful migration. Subsequent commands resolve the active space root from the new registry automatically. + ## Storage Configuration Configure storage provider and settings: diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 2e0ad661358..65886a62e7e 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -12,12 +12,12 @@ pnpm dev That's it. On first start the server: -1. Creates a `~/.paperclip/instances/default/db/` directory for storage +1. Creates a `~/.paperclip/instances/default/spaces/default/db/` directory for storage 2. Ensures the `paperclip` database exists 3. Runs migrations automatically for empty databases 4. Starts serving requests -Data persists across restarts in `~/.paperclip/instances/default/db/`. To reset local dev data, delete that directory. +Data persists across restarts in `~/.paperclip/instances/default/spaces/default/db/`. To reset local dev data, delete that directory. Older installs that still keep data at `~/.paperclip/instances/default/db/` continue to start through the legacy resolver — see [Migrating a Legacy Default-Space Install](CLI.md#migrating-a-legacy-default-space-install). If you need to apply pending migrations manually, run: @@ -137,7 +137,7 @@ The database mode is controlled by `DATABASE_URL`: | `DATABASE_URL` | Mode | |---|---| -| Not set | Embedded PostgreSQL (`~/.paperclip/instances/default/db/`) | +| Not set | Embedded PostgreSQL (`~/.paperclip/instances/default/spaces/default/db/`) | | `postgres://...localhost...` | Local Docker PostgreSQL | | `postgres://...supabase.com...` | Hosted Supabase | @@ -169,8 +169,8 @@ Paperclip stores secret metadata and versions in: For local/default installs, the active provider is `local_encrypted`: - Secret material is encrypted at rest with a local master key. -- Default key file: `~/.paperclip/instances/default/secrets/master.key` (auto-created if missing). -- CLI config location: `~/.paperclip/instances/default/config.json` under `secrets.localEncrypted.keyFilePath`. +- Default key file: `~/.paperclip/instances/default/spaces/default/secrets/master.key` (auto-created if missing). +- CLI config location: `~/.paperclip/instances/default/spaces/default/config.json` under `secrets.localEncrypted.keyFilePath`. (The instance-level `~/.paperclip/instances/default/config.json` is the cross-space registry that records which space is active.) - Backup/restore requires both the database metadata and the local master key file; either artifact alone is insufficient. - The server best-effort enforces `0600` key file permissions and provider health reports permission warnings. diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index c9a2a19414c..eff89e061ae 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -157,17 +157,44 @@ See `doc/DOCKER.md` for API key wiring (`OPENAI_API_KEY` / `ANTHROPIC_API_KEY`) For a separate review-oriented container that keeps `codex`/`claude` login state in Docker volumes and checks out PRs into an isolated scratch workspace, see `doc/UNTRUSTED-PR-REVIEW.md`. +## Local Instance Layout + +Every local install nests instance metadata, the active space, and runtime state under `~/.paperclip/`: + +```text +~/.paperclip/instances/default/ # instance root (cross-space metadata only) + config.json # space registry (activeSpaceId, known spaces) + spaces/ + default/ # active space root + config.json # space runtime config + .env # space-scoped env file + db/ # embedded PostgreSQL data + data/ + storage/ # local_disk uploads + backups/ # automatic DB backups + logs/ + secrets/master.key # local_encrypted master key + workspaces// # default agent workspaces + projects/ # project execution workspaces + companies//codex-home/ # per-company codex_local home +``` + +The instance root no longer doubles as the default space. New installs created by `paperclipai onboard`, `paperclipai run`, and `paperclipai configure` always write the registry at `instances//config.json` and put runtime data under `spaces/default/`. To migrate an older root-shaped install in place, see [Migrating a Legacy Default-Space Install](CLI.md#migrating-a-legacy-default-space-install). + +`PAPERCLIP_HOME`, `PAPERCLIP_INSTANCE_ID`, and `PAPERCLIP_SPACE_ID` override the home root, instance id, and active space respectively. `paperclipai onboard` echoes the resolved values in its banner (`Local home: | instance: | space: | config: `) so you can confirm where state will land before continuing. + ## Database in Dev (Auto-Handled) For local development, leave `DATABASE_URL` unset. The server will automatically use embedded PostgreSQL and persist data at: -- `~/.paperclip/instances/default/db` +- `~/.paperclip/instances/default/spaces/default/db` -Override home and instance: +Override home, instance, or space: ```sh PAPERCLIP_HOME=/custom/path PAPERCLIP_INSTANCE_ID=dev pnpm paperclipai run +PAPERCLIP_SPACE_ID=staging pnpm paperclipai run ``` No Docker or external database is required for this mode. @@ -176,7 +203,7 @@ No Docker or external database is required for this mode. For local development, the default storage provider is `local_disk`, which persists uploaded images/attachments at: -- `~/.paperclip/instances/default/data/storage` +- `~/.paperclip/instances/default/spaces/default/data/storage` Configure storage provider/settings: @@ -186,15 +213,15 @@ pnpm paperclipai configure --section storage ## Default Agent Workspaces -When a local agent run has no resolved project/session workspace, Paperclip falls back to an agent home workspace under the instance root: +When a local agent run has no resolved project/session workspace, Paperclip falls back to an agent home workspace under the active space: -- `~/.paperclip/instances/default/workspaces/` +- `~/.paperclip/instances/default/spaces/default/workspaces/` -This path honors `PAPERCLIP_HOME` and `PAPERCLIP_INSTANCE_ID` in non-default setups. +This path honors `PAPERCLIP_HOME`, `PAPERCLIP_INSTANCE_ID`, and `PAPERCLIP_SPACE_ID` in non-default setups. -For `codex_local`, Paperclip also manages a per-company Codex home under the instance root and seeds it from the shared Codex login/config home (`$CODEX_HOME` or `~/.codex`): +For `codex_local`, Paperclip also manages a per-company Codex home under the active space and seeds it from the shared Codex login/config home (`$CODEX_HOME` or `~/.codex`): -- `~/.paperclip/instances/default/companies//codex-home` +- `~/.paperclip/instances/default/spaces/default/companies//codex-home` If the `codex` CLI is not installed or not on `PATH`, `codex_local` agent runs fail at execution time with a clear adapter error. Quota polling uses a short-lived `codex app-server` subprocess: when `codex` cannot be spawned, that provider reports `ok: false` in aggregated quota results and the API server keeps running (it must not exit on a missing binary). @@ -280,13 +307,13 @@ paperclipai worktree init --from-data-dir ~/.paperclip paperclipai worktree init --force ``` -Repair an already-created repo-managed worktree and reseed its isolated instance from the main default install: +Repair an already-created repo-managed worktree and reseed its isolated instance from the main default install. Point `--from-config` at the active space config (`spaces/default/config.json` for post-migration installs; the legacy `~/.paperclip/instances/default/config.json` still works on root-shaped installs that haven't run `paperclipai spaces migrate-default` yet): ```sh cd /path/to/paperclip/.paperclip/worktrees/PAP-884-ai-commits-component pnpm paperclipai worktree init --force --seed-mode minimal \ --name PAP-884-ai-commits-component \ - --from-config ~/.paperclip/instances/default/config.json + --from-config ~/.paperclip/instances/default/spaces/default/config.json ``` That rewrites the worktree-local `.paperclip/config.json` + `.paperclip/.env`, recreates the isolated instance under `~/.paperclip-worktrees/instances//`, and preserves the git worktree contents themselves. @@ -411,10 +438,12 @@ Expected: To wipe local dev data and start fresh: ```sh -rm -rf ~/.paperclip/instances/default/db +rm -rf ~/.paperclip/instances/default/spaces/default/db pnpm dev ``` +If you migrated from a legacy root-shaped install and still have a stale `~/.paperclip/instances/default/db`, remove that too — but only after confirming `paperclipai spaces migrate-default` ran and the active database lives under `spaces/default/`. + ## Optional: Use External Postgres If you set `DATABASE_URL`, the server will use that instead of embedded PostgreSQL. @@ -428,7 +457,7 @@ schemas. Defaults: - enabled - every 60 minutes - retain 30 days -- backup dir: `~/.paperclip/instances/default/data/backups` +- backup dir: `~/.paperclip/instances/default/spaces/default/data/backups` Configure these in: @@ -459,7 +488,7 @@ those providers are enabled. Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config. -- Default local key path: `~/.paperclip/instances/default/secrets/master.key` +- Default local key path: `~/.paperclip/instances/default/spaces/default/secrets/master.key` - Override key material directly: `PAPERCLIP_SECRETS_MASTER_KEY` - Override key file path: `PAPERCLIP_SECRETS_MASTER_KEY_FILE` - Back up the key file and database together; either one alone is not enough to restore local encrypted secrets. diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 22075e0453a..8ae2df78d66 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -93,11 +93,11 @@ V1 implementation extends this baseline into a company-centric, governance-aware ## 6.2 Data Stores - Primary: PostgreSQL -- Local default: embedded PostgreSQL at `~/.paperclip/instances/default/db` +- Local default: embedded PostgreSQL at `~/.paperclip/instances/default/spaces/default/db` - Optional local prod-like: Docker Postgres - Optional hosted: Supabase/Postgres-compatible - File/object storage: - - local default: `~/.paperclip/instances/default/data/storage` (`local_disk`) + - local default: `~/.paperclip/instances/default/spaces/default/data/storage` (`local_disk`) - cloud: S3-compatible object storage (`s3`) ## 6.3 Background Processing @@ -785,7 +785,7 @@ Required UX behaviors: - Node 20+ - `DATABASE_URL` optional -- if unset, auto-use embedded PostgreSQL under `~/.paperclip/instances/default/db` +- if unset, auto-use embedded PostgreSQL under `~/.paperclip/instances/default/spaces/default/db` ## 15.2 Migrations diff --git a/doc/plans/2026-05-06-llm-wiki-paperclip-asset-security-gate.md b/doc/plans/2026-05-06-llm-wiki-paperclip-asset-security-gate.md new file mode 100644 index 00000000000..cb527ae7ad3 --- /dev/null +++ b/doc/plans/2026-05-06-llm-wiki-paperclip-asset-security-gate.md @@ -0,0 +1,135 @@ +# LLM Wiki Paperclip Asset And Work-Product Security Gate + +Status: accepted Phase 5 policy +Date: 2026-05-06 +Owner: Security engineering +Scope: Paperclip-derived ingestion into the LLM Wiki before any asset or work-product content indexing ships + +## Decision + +Phase 5 remains **fail-closed** for Paperclip assets and work products. + +- Paperclip-derived **text extraction is allowed only** for issue titles/descriptions, issue comments, and issue documents. +- Paperclip **assets/attachments** and **issue work products** are **metadata-only** in Phase 5. +- **Linked summaries** and **content extraction** for assets/work products are **not approved** in Phase 5. +- No implementation may fetch `/api/assets/:id/content`, dereference a work-product `url`, scrape preview pages, or embed binary/blob content into source bundles or source snapshots. + +This keeps the secure path easier than the insecure one and avoids broadening the wiki into a second content-distribution channel. + +## Allowed Source Kinds + +These source kinds may contribute body text to Paperclip-derived source bundles: + +| Source kind | Allowed body fields | Reason | +| --- | --- | --- | +| Issue | `title`, `description`, identifier/status metadata | First-party Paperclip text under company ACL | +| Comment | `body` | First-party Paperclip text under company ACL | +| Document | `body`, `title`, `key`, revision metadata | First-party Paperclip text under company ACL | + +## Assets And Work Products + +### Assets / attachments + +Allowed in Phase 5: + +- metadata-only references built from allowlisted structured fields already stored in Paperclip +- recommended fields: `issueId`, `issueCommentId`, `attachmentId`, `assetId`, `originalFilename`, `contentType`, `byteSize`, `sha256`, `createdAt`, `createdByAgentId`, `createdByUserId` + +Disallowed in Phase 5: + +- fetching asset bytes from `/api/assets/:id/content` +- parsing any blob body, including `text/plain`, `text/markdown`, `application/json`, images, SVG, PDFs, archives, or office formats +- storing `contentPath` in wiki source bundles or source snapshots +- model summarization of attachment bodies + +### Work products + +Allowed in Phase 5: + +- metadata-only references built from allowlisted structured fields already stored in Paperclip +- recommended fields: `issueId`, `workProductId`, `type`, `provider`, `title`, `status`, `reviewState`, `healthStatus`, `externalId`, `isPrimary`, `createdAt`, `updatedAt` +- optional boolean/derived metadata such as `hasUrl: true` + +Disallowed in Phase 5: + +- fetching or crawling the work-product `url` +- scraping preview pages, artifacts, pull requests, branches, commits, or custom provider targets through the wiki ingestion path +- storing raw `url` values in wiki source bundles or source snapshots +- model-authored linked summaries derived from off-record content + +## MIME Allowlists And Size Caps + +No MIME allowlist is approved for asset content extraction in Phase 5 because **no asset body extraction is approved at all**. + +- Every asset MIME type is treated as opaque for Paperclip-derived indexing. +- Existing upload limits remain storage concerns, not ingestion approvals. +- Work-product destinations are also opaque regardless of MIME type or size. + +Any future issue that wants blob parsing must define: + +- a positive MIME allowlist +- per-type parser strategy +- per-source size caps +- sandbox/isolation requirements +- prompt-injection handling +- regression tests for refusal paths + +## Redaction Rules + +Metadata-only means **structured facts only**, not capability-bearing links. + +- Do not persist `contentPath` for assets. +- Do not persist raw work-product `url` values. +- Do not persist query strings, fragments, signed URL tokens, or userinfo. +- Prefer stable identifiers (`assetId`, `workProductId`, `externalId`) over links. + +This addresses Sensitive Information Disclosure, Unsafe Consumption of APIs, and Insecure Output Handling risks. + +## Provenance Rules + +Every metadata-only reference must preserve enough provenance to explain where it came from without reading the underlying content: + +- `companyId` +- `issueId` +- attachment/work-product id +- producer identity when available +- timestamps +- an explicit `metadata_only` marker in any future reference/snapshot schema + +## Review-Required Behavior + +Human review is **not** required for plain metadata-only references that stay inside the allowlisted fields above. + +Human review **is required**, with a separate security sign-off issue, before enabling any of the following: + +- asset body extraction +- work-product URL fetching +- linked summaries generated from asset/work-product content +- storing raw blob links or raw remote URLs in wiki source material +- non-default-space routing for Paperclip-derived asset/work-product references + +## Security Rationale + +This gate exists because the current host surfaces have different trust properties: + +- issue/comment/document text is first-party Paperclip content already exposed through company-scoped issue/document APIs +- asset content is a blob download surface (`/api/assets/:id/content`) and can carry prompt-injection or parser-risk payloads +- work products can point at arbitrary destinations through `url`, which reintroduces SSRF, token leakage, and prompt-injection risk if dereferenced automatically + +Relevant threat classes: + +- OWASP LLM Top 10: Prompt Injection, Sensitive Information Disclosure, Insecure Output Handling, Excessive Agency +- OWASP API Top 10: SSRF, Unsafe Consumption of APIs, Broken Object Property Level Authorization +- Saltzer & Schroeder: Least Privilege, Fail Securely, Complete Mediation, Secure Defaults + +## Follow-Up Implementation Scope + +A follow-up implementation issue is justified only for **metadata-only references**. + +That implementation must: + +- keep assets/work products out of source-bundle body text +- never fetch blob bytes or remote URLs +- redact capability-bearing link fields +- mark references as `metadata_only` +- ship tests proving source bundles/snapshots never contain `contentPath` or raw work-product `url` fields diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index bc3bc4a4cea..ee172e3122f 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -1,6 +1,7 @@ import { spawn, type ChildProcess } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { constants as fsConstants, promises as fs, type Dirent } from "node:fs"; +import { constants as fsConstants, existsSync, promises as fs, readFileSync, type Dirent } from "node:fs"; +import os from "node:os"; import path from "node:path"; import { sanitizeRemoteExecutionEnv } from "./remote-execution-env.js"; import { buildSshSpawnTarget, type SshRemoteExecutionSpec } from "./ssh.js"; @@ -78,6 +79,121 @@ export const runningProcesses = new Map(); export const MAX_CAPTURE_BYTES = 4 * 1024 * 1024; export const MAX_EXCERPT_BYTES = 32 * 1024; const TERMINAL_RESULT_SCAN_OVERLAP_CHARS = 64 * 1024; +const DEFAULT_PAPERCLIP_INSTANCE_ID = "default"; +const DEFAULT_PAPERCLIP_SPACE_ID = "default"; +const PAPERCLIP_CONFIG_BASENAME = "config.json"; +const PATH_SEGMENT_RE = /^[a-zA-Z0-9_-]+$/; + +function expandHomePrefix(value: string): string { + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); + return value; +} + +function resolvePaperclipHomeDirForAdapter(homeOverride?: string, env: NodeJS.ProcessEnv = process.env): string { + const raw = homeOverride?.trim() || env.PAPERCLIP_HOME?.trim(); + if (raw) return path.resolve(expandHomePrefix(raw)); + return path.resolve(os.homedir(), ".paperclip"); +} + +function resolvePaperclipInstanceIdForAdapter(instanceIdOverride?: string, env: NodeJS.ProcessEnv = process.env): string { + const raw = instanceIdOverride?.trim() || env.PAPERCLIP_INSTANCE_ID?.trim() || DEFAULT_PAPERCLIP_INSTANCE_ID; + if (!PATH_SEGMENT_RE.test(raw)) { + throw new Error(`Invalid PAPERCLIP_INSTANCE_ID '${raw}'.`); + } + return raw; +} + +function resolvePaperclipInstanceRootForAdapter(input: { + homeDir?: string; + instanceId?: string; + env?: NodeJS.ProcessEnv; +} = {}): string { + return path.resolve( + resolvePaperclipHomeDirForAdapter(input.homeDir, input.env), + "instances", + resolvePaperclipInstanceIdForAdapter(input.instanceId, input.env), + ); +} + +function resolvePaperclipInstanceConfigPathForAdapter(input: { + homeDir?: string; + instanceId?: string; + env?: NodeJS.ProcessEnv; +} = {}): string { + return path.resolve(resolvePaperclipInstanceRootForAdapter(input), PAPERCLIP_CONFIG_BASENAME); +} + +function readJsonIfPresent(filePath: string): unknown | null { + if (!existsSync(filePath)) return null; + try { + return JSON.parse(readFileSync(filePath, "utf8")); + } catch { + return null; + } +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isPaperclipRuntimeConfig(value: unknown): boolean { + return isObject(value) && isObject(value.database) && isObject(value.server); +} + +function readActiveSpaceIdFromRegistry(instanceConfigPath: string): string | null { + const parsed = readJsonIfPresent(instanceConfigPath); + if (!isObject(parsed) || isPaperclipRuntimeConfig(parsed)) return null; + const activeSpaceId = parsed.activeSpaceId; + return typeof activeSpaceId === "string" && PATH_SEGMENT_RE.test(activeSpaceId) ? activeSpaceId : null; +} + +function hasLegacyDefaultSpaceInstall(input: { + homeDir?: string; + instanceId?: string; + env?: NodeJS.ProcessEnv; +} = {}): boolean { + return isPaperclipRuntimeConfig(readJsonIfPresent(resolvePaperclipInstanceConfigPathForAdapter(input))); +} + +function resolvePaperclipSpaceIdForAdapter(input: { + homeDir?: string; + instanceId?: string; + spaceId?: string; + env?: NodeJS.ProcessEnv; +} = {}): string { + const raw = + input.spaceId?.trim() || + input.env?.PAPERCLIP_SPACE_ID?.trim() || + process.env.PAPERCLIP_SPACE_ID?.trim() || + readActiveSpaceIdFromRegistry(resolvePaperclipInstanceConfigPathForAdapter(input)) || + DEFAULT_PAPERCLIP_SPACE_ID; + if (!PATH_SEGMENT_RE.test(raw)) { + throw new Error(`Invalid PAPERCLIP_SPACE_ID '${raw}'.`); + } + return raw; +} + +function resolvePaperclipSpacesRootForAdapter(input: { + homeDir?: string; + instanceId?: string; + env?: NodeJS.ProcessEnv; +} = {}): string { + return path.resolve(resolvePaperclipInstanceRootForAdapter(input), "spaces"); +} + +export function resolvePaperclipSpaceRootForAdapter(input: { + homeDir?: string; + instanceId?: string; + spaceId?: string; + env?: NodeJS.ProcessEnv; +} = {}): string { + const spaceId = resolvePaperclipSpaceIdForAdapter(input); + if (spaceId === DEFAULT_PAPERCLIP_SPACE_ID && hasLegacyDefaultSpaceInstall(input)) { + return resolvePaperclipInstanceRootForAdapter(input); + } + return path.resolve(resolvePaperclipSpacesRootForAdapter(input), spaceId); +} const SENSITIVE_ENV_KEY = /(key|token|secret|password|passwd|authorization|cookie)/i; const REDACTED_LOG_VALUE = "***REDACTED***"; const PAPERCLIP_SKILL_ROOT_RELATIVE_CANDIDATES = [ diff --git a/packages/adapters/acpx-local/src/server/execute.ts b/packages/adapters/acpx-local/src/server/execute.ts index 3450cb9e586..f1d0d637bfe 100644 --- a/packages/adapters/acpx-local/src/server/execute.ts +++ b/packages/adapters/acpx-local/src/server/execute.ts @@ -22,6 +22,7 @@ import { renderPaperclipWakePrompt, renderTemplate, resolvePaperclipDesiredSkillNames, + resolvePaperclipSpaceRootForAdapter, rewriteWorkspaceCwdEnvVarsForExecution, shapePaperclipWorkspaceEnvForExecution, stringifyPaperclipWakePayload, @@ -115,7 +116,11 @@ function shortHash(value: unknown): string { function defaultPaperclipInstanceDir(): string { const home = process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip"); const instanceId = process.env.PAPERCLIP_INSTANCE_ID?.trim() || "default"; - return path.join(home, "instances", instanceId); + return resolvePaperclipSpaceRootForAdapter({ + homeDir: home, + instanceId, + spaceId: process.env.PAPERCLIP_SPACE_ID?.trim() || undefined, + }); } function defaultStateDir(companyId: string, agentId: string): string { diff --git a/packages/adapters/claude-local/src/server/claude-config.ts b/packages/adapters/claude-local/src/server/claude-config.ts index 5e4f22788a0..b323f6cc073 100644 --- a/packages/adapters/claude-local/src/server/claude-config.ts +++ b/packages/adapters/claude-local/src/server/claude-config.ts @@ -3,8 +3,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; +import { resolvePaperclipSpaceRootForAdapter } from "@paperclipai/adapter-utils/server-utils"; -const DEFAULT_PAPERCLIP_INSTANCE_ID = "default"; const SEEDED_SHARED_FILES = [ ".credentials.json", "credentials.json", @@ -92,11 +92,15 @@ export function resolveManagedClaudeConfigSeedDir( env: NodeJS.ProcessEnv, companyId?: string, ): string { - const paperclipHome = nonEmpty(env.PAPERCLIP_HOME) ?? path.resolve(os.homedir(), ".paperclip"); - const instanceId = nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? DEFAULT_PAPERCLIP_INSTANCE_ID; + const spaceRoot = resolvePaperclipSpaceRootForAdapter({ + homeDir: nonEmpty(env.PAPERCLIP_HOME) ?? undefined, + instanceId: nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? undefined, + spaceId: nonEmpty(env.PAPERCLIP_SPACE_ID) ?? undefined, + env, + }); return companyId - ? path.resolve(paperclipHome, "instances", instanceId, "companies", companyId, "claude-config-seed") - : path.resolve(paperclipHome, "instances", instanceId, "claude-config-seed"); + ? path.resolve(spaceRoot, "companies", companyId, "claude-config-seed") + : path.resolve(spaceRoot, "claude-config-seed"); } export async function prepareClaudeConfigSeed( diff --git a/packages/adapters/claude-local/src/server/prompt-cache.ts b/packages/adapters/claude-local/src/server/prompt-cache.ts index ce719799979..91a1ad6f515 100644 --- a/packages/adapters/claude-local/src/server/prompt-cache.ts +++ b/packages/adapters/claude-local/src/server/prompt-cache.ts @@ -1,12 +1,13 @@ import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; -import os from "node:os"; import path from "node:path"; import { createHash, type Hash } from "node:crypto"; import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; -import { ensurePaperclipSkillSymlink, type PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils"; - -const DEFAULT_PAPERCLIP_INSTANCE_ID = "default"; +import { + ensurePaperclipSkillSymlink, + resolvePaperclipSpaceRootForAdapter, + type PaperclipSkillEntry, +} from "@paperclipai/adapter-utils/server-utils"; type SkillEntry = PaperclipSkillEntry; @@ -25,12 +26,14 @@ function resolveManagedClaudePromptCacheRoot( env: NodeJS.ProcessEnv, companyId: string, ): string { - const paperclipHome = nonEmpty(env.PAPERCLIP_HOME) ?? path.resolve(os.homedir(), ".paperclip"); - const instanceId = nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? DEFAULT_PAPERCLIP_INSTANCE_ID; + const spaceRoot = resolvePaperclipSpaceRootForAdapter({ + homeDir: nonEmpty(env.PAPERCLIP_HOME) ?? undefined, + instanceId: nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? undefined, + spaceId: nonEmpty(env.PAPERCLIP_SPACE_ID) ?? undefined, + env, + }); return path.resolve( - paperclipHome, - "instances", - instanceId, + spaceRoot, "companies", companyId, "claude-prompt-cache", diff --git a/packages/adapters/codex-local/src/server/codex-home.ts b/packages/adapters/codex-local/src/server/codex-home.ts index 10b3ae96b4b..be9213e3326 100644 --- a/packages/adapters/codex-local/src/server/codex-home.ts +++ b/packages/adapters/codex-local/src/server/codex-home.ts @@ -2,11 +2,11 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; +import { resolvePaperclipSpaceRootForAdapter } from "@paperclipai/adapter-utils/server-utils"; const TRUTHY_ENV_RE = /^(1|true|yes|on)$/i; const COPIED_SHARED_FILES = ["config.json", "config.toml", "instructions.md"] as const; const SYMLINKED_SHARED_FILES = ["auth.json"] as const; -const DEFAULT_PAPERCLIP_INSTANCE_ID = "default"; function nonEmpty(value: string | undefined): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; @@ -31,11 +31,15 @@ export function resolveManagedCodexHomeDir( env: NodeJS.ProcessEnv, companyId?: string, ): string { - const paperclipHome = nonEmpty(env.PAPERCLIP_HOME) ?? path.resolve(os.homedir(), ".paperclip"); - const instanceId = nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? DEFAULT_PAPERCLIP_INSTANCE_ID; + const spaceRoot = resolvePaperclipSpaceRootForAdapter({ + homeDir: nonEmpty(env.PAPERCLIP_HOME) ?? undefined, + instanceId: nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? undefined, + spaceId: nonEmpty(env.PAPERCLIP_SPACE_ID) ?? undefined, + env, + }); return companyId - ? path.resolve(paperclipHome, "instances", instanceId, "companies", companyId, "codex-home") - : path.resolve(paperclipHome, "instances", instanceId, "codex-home"); + ? path.resolve(spaceRoot, "companies", companyId, "codex-home") + : path.resolve(spaceRoot, "codex-home"); } async function ensureParentDir(target: string): Promise { diff --git a/packages/db/src/backup.ts b/packages/db/src/backup.ts index c11822c9503..40ab3e134e4 100644 --- a/packages/db/src/backup.ts +++ b/packages/db/src/backup.ts @@ -1,7 +1,11 @@ import { existsSync, readFileSync } from "node:fs"; -import os from "node:os"; import path from "node:path"; import { formatDatabaseBackupResult, runDatabaseBackup } from "./backup-lib.js"; +import { + expandHomePrefix, + resolveDefaultBackupDir, + resolvePaperclipConfigPathForInstance, +} from "@paperclipai/shared/space-paths"; type PartialConfig = { database?: { @@ -15,30 +19,6 @@ type PartialConfig = { }; }; -function expandHomePrefix(value: string): string { - if (value === "~") return os.homedir(); - if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); - return value; -} - -function resolvePaperclipHomeDir(): string { - const envHome = process.env.PAPERCLIP_HOME?.trim(); - if (envHome) return path.resolve(expandHomePrefix(envHome)); - return path.resolve(os.homedir(), ".paperclip"); -} - -function resolvePaperclipInstanceId(): string { - const raw = process.env.PAPERCLIP_INSTANCE_ID?.trim() || "default"; - if (!/^[a-zA-Z0-9_-]+$/.test(raw)) { - throw new Error(`Invalid PAPERCLIP_INSTANCE_ID '${raw}'.`); - } - return raw; -} - -function resolveDefaultConfigPath(): string { - return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "config.json"); -} - function readConfig(configPath: string): PartialConfig | null { if (!existsSync(configPath)) return null; try { @@ -72,10 +52,6 @@ function resolveConnectionString(config: PartialConfig | null): string { return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; } -function resolveDefaultBackupDir(): string { - return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "data", "backups"); -} - function resolveBackupDir(config: PartialConfig | null): string { const raw = config?.database?.backup?.dir; if (typeof raw === "string" && raw.trim().length > 0) { @@ -89,7 +65,7 @@ function resolveRetentionDays(config: PartialConfig | null): number { } async function main() { - const configPath = resolveDefaultConfigPath(); + const configPath = resolvePaperclipConfigPathForInstance(); const config = readConfig(configPath); const connectionString = resolveConnectionString(config); const backupDir = resolveBackupDir(config); diff --git a/packages/db/src/runtime-config.test.ts b/packages/db/src/runtime-config.test.ts index 4627e6918d0..8e853d17709 100644 --- a/packages/db/src/runtime-config.test.ts +++ b/packages/db/src/runtime-config.test.ts @@ -105,4 +105,24 @@ describe("resolveDatabaseTarget", () => { source: "embedded-postgres@55444", }); }); + + it("uses spaces/default for a fresh default embedded postgres target", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-db-space-")); + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-db-cwd-")); + process.chdir(cwd); + process.env.PAPERCLIP_HOME = home; + delete process.env.PAPERCLIP_CONFIG; + delete process.env.DATABASE_URL; + + const target = resolveDatabaseTarget(); + + expect(target).toMatchObject({ + mode: "embedded-postgres", + dataDir: path.join(home, "instances", "default", "spaces", "default", "db"), + port: 54329, + source: "embedded-postgres@54329", + configPath: path.join(home, "instances", "default", "spaces", "default", "config.json"), + envPath: path.join(home, "instances", "default", "spaces", "default", ".env"), + }); + }); }); diff --git a/packages/db/src/runtime-config.ts b/packages/db/src/runtime-config.ts index c6c64a38fa0..6e8740ef78d 100644 --- a/packages/db/src/runtime-config.ts +++ b/packages/db/src/runtime-config.ts @@ -1,11 +1,13 @@ import { existsSync, readFileSync } from "node:fs"; -import os from "node:os"; import path from "node:path"; +import { + expandHomePrefix, + resolveDefaultEmbeddedPostgresDir, + resolvePaperclipConfigPathForInstance, + resolvePaperclipEnvPathForConfig, +} from "@paperclipai/shared/space-paths"; -const DEFAULT_INSTANCE_ID = "default"; const CONFIG_BASENAME = "config.json"; -const ENV_BASENAME = ".env"; -const INSTANCE_ID_RE = /^[a-zA-Z0-9_-]+$/; type PartialConfig = { database?: { @@ -35,39 +37,6 @@ export type ResolvedDatabaseTarget = envPath: string; }; -function expandHomePrefix(value: string): string { - if (value === "~") return os.homedir(); - if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); - return value; -} - -function resolvePaperclipHomeDir(): string { - const envHome = process.env.PAPERCLIP_HOME?.trim(); - if (envHome) return path.resolve(expandHomePrefix(envHome)); - return path.resolve(os.homedir(), ".paperclip"); -} - -function resolvePaperclipInstanceId(): string { - const raw = process.env.PAPERCLIP_INSTANCE_ID?.trim() || DEFAULT_INSTANCE_ID; - if (!INSTANCE_ID_RE.test(raw)) { - throw new Error(`Invalid PAPERCLIP_INSTANCE_ID '${raw}'.`); - } - return raw; -} - -function resolveDefaultConfigPath(): string { - return path.resolve( - resolvePaperclipHomeDir(), - "instances", - resolvePaperclipInstanceId(), - CONFIG_BASENAME, - ); -} - -function resolveDefaultEmbeddedPostgresDir(): string { - return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "db"); -} - function resolveHomeAwarePath(value: string): string { return path.resolve(expandHomePrefix(value)); } @@ -89,11 +58,11 @@ function resolvePaperclipConfigPath(): string { if (process.env.PAPERCLIP_CONFIG?.trim()) { return path.resolve(process.env.PAPERCLIP_CONFIG.trim()); } - return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath(); + return findConfigFileFromAncestors(process.cwd()) ?? resolvePaperclipConfigPathForInstance(); } function resolvePaperclipEnvPath(configPath: string): string { - return path.resolve(path.dirname(configPath), ENV_BASENAME); + return resolvePaperclipEnvPathForConfig(configPath); } function parseEnvFile(contents: string): Record { diff --git a/packages/plugins/examples/plugin-bookmarks-example/.gitignore b/packages/plugins/examples/plugin-bookmarks-example/.gitignore new file mode 100644 index 00000000000..0430cc35608 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/.gitignore @@ -0,0 +1,3 @@ +dist +node_modules +.paperclip-sdk diff --git a/packages/plugins/examples/plugin-bookmarks-example/README.md b/packages/plugins/examples/plugin-bookmarks-example/README.md new file mode 100644 index 00000000000..17c48182f45 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/README.md @@ -0,0 +1,53 @@ +# @paperclipai/plugin-bookmarks-example + +A first-party example plugin that adds a small but real product surface to Paperclip: a +company-scoped bookmark library backed by a plugin database namespace and a local markdown +folder. + +The package is a follow-up to the core plugin host surface work and is intended as a +focused reference for plugin authors. The orchestration smoke example remains the +acceptance fixture for the orchestration APIs; this plugin is the user-facing +counterpart that exercises the broader UI + storage surface. + +## Surfaces exercised + +- **Plugin database namespace.** A migration creates a `bookmarks` table inside the + plugin's restricted Postgres schema. The worker reads and writes only that + namespace; the restricted SQL helpers prevent cross-namespace access. +- **Local folders.** A `bookmarks-root` folder declaration is configured by the + operator and atomically receives one markdown file per bookmark via + `ctx.localFolders.writeTextAtomic`. The DB row remains the source of truth so + list and delete operations stay correct even when the folder is unconfigured + or the disk write fails. +- **Scoped API routes.** Three plugin-owned routes (`list`, `create`, `delete`) + declare board-or-agent auth and resolve the company from query/body + parameters before the worker sees them. +- **UI extension slots.** A full plugin page, a sidebar entry, a dashboard + widget, and a settings page are all wired up to the same worker handlers via + `usePluginData` / `usePluginAction`. + +## Install + +```sh +pnpm --filter @paperclipai/plugin-bookmarks-example build +pnpm paperclipai plugin install ./packages/plugins/examples/plugin-bookmarks-example +``` + +After install, configure the plugin's `bookmarks-root` local folder under the +plugin's Folders settings to enable the markdown sidecar files. The plugin works +without it (rows live in the database namespace), but operators that want +shareable, file-backed snapshots should point the folder at a checked-in +directory. + +## Development + +```sh +pnpm --filter @paperclipai/plugin-bookmarks-example test +pnpm --filter @paperclipai/plugin-bookmarks-example typecheck +pnpm --filter @paperclipai/plugin-bookmarks-example build +``` + +The worker code uses module-scoped handler closures populated in `setup()`, +mirroring the orchestration smoke example, so the handlers can be invoked from +both `ctx.data` / `ctx.actions` (UI) and `onApiRequest` (HTTP) without +duplicating logic. diff --git a/packages/plugins/examples/plugin-bookmarks-example/esbuild.config.mjs b/packages/plugins/examples/plugin-bookmarks-example/esbuild.config.mjs new file mode 100644 index 00000000000..b5cfd36ed95 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/esbuild.config.mjs @@ -0,0 +1,17 @@ +import esbuild from "esbuild"; +import { createPluginBundlerPresets } from "@paperclipai/plugin-sdk/bundlers"; + +const presets = createPluginBundlerPresets({ uiEntry: "src/ui/index.tsx" }); +const watch = process.argv.includes("--watch"); + +const workerCtx = await esbuild.context(presets.esbuild.worker); +const manifestCtx = await esbuild.context(presets.esbuild.manifest); +const uiCtx = await esbuild.context(presets.esbuild.ui); + +if (watch) { + await Promise.all([workerCtx.watch(), manifestCtx.watch(), uiCtx.watch()]); + console.log("esbuild watch mode enabled for worker, manifest, and ui"); +} else { + await Promise.all([workerCtx.rebuild(), manifestCtx.rebuild(), uiCtx.rebuild()]); + await Promise.all([workerCtx.dispose(), manifestCtx.dispose(), uiCtx.dispose()]); +} diff --git a/packages/plugins/examples/plugin-bookmarks-example/migrations/001_bookmarks.sql b/packages/plugins/examples/plugin-bookmarks-example/migrations/001_bookmarks.sql new file mode 100644 index 00000000000..c326e11dab6 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/migrations/001_bookmarks.sql @@ -0,0 +1,16 @@ +CREATE TABLE plugin_bookmarks_b34a9f8617.bookmarks ( + id uuid PRIMARY KEY, + company_id uuid NOT NULL, + slug text NOT NULL, + url text NOT NULL, + title text NOT NULL, + notes text NOT NULL DEFAULT '', + tags text[] NOT NULL DEFAULT '{}', + file_path text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT bookmarks_company_slug_uq UNIQUE (company_id, slug) +); + +CREATE INDEX bookmarks_company_idx + ON plugin_bookmarks_b34a9f8617.bookmarks (company_id, created_at DESC); diff --git a/packages/plugins/examples/plugin-bookmarks-example/package.json b/packages/plugins/examples/plugin-bookmarks-example/package.json new file mode 100644 index 00000000000..be68a4b2e03 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/package.json @@ -0,0 +1,48 @@ +{ + "name": "@paperclipai/plugin-bookmarks-example", + "version": "0.1.0", + "type": "module", + "private": true, + "description": "First-party example plugin: company-scoped bookmark library backed by a plugin database namespace and a local markdown folder.", + "scripts": { + "prebuild": "pnpm --filter @paperclipai/plugin-sdk ensure-build-deps", + "build": "node ./esbuild.config.mjs", + "build:rollup": "rollup -c", + "dev": "node ./esbuild.config.mjs --watch", + "dev:ui": "paperclip-plugin-dev-server --root . --ui-dir dist/ui --port 4178", + "test": "vitest run --config ./vitest.config.ts", + "typecheck": "pnpm --filter @paperclipai/plugin-sdk ensure-build-deps && tsc --noEmit" + }, + "paperclipPlugin": { + "manifest": "./dist/manifest.js", + "worker": "./dist/worker.js", + "ui": "./dist/ui/" + }, + "keywords": [ + "paperclip", + "plugin", + "example", + "bookmarks", + "knowledge" + ], + "author": "Paperclip", + "license": "MIT", + "dependencies": { + "@paperclipai/plugin-sdk": "workspace:*" + }, + "devDependencies": { + "@paperclipai/shared": "workspace:*", + "@rollup/plugin-node-resolve": "^16.0.1", + "@rollup/plugin-typescript": "^12.1.2", + "@types/node": "^24.6.0", + "@types/react": "^19.0.8", + "esbuild": "^0.27.3", + "rollup": "^4.38.0", + "tslib": "^2.8.1", + "typescript": "^5.7.3", + "vitest": "^3.0.5" + }, + "peerDependencies": { + "react": ">=18" + } +} diff --git a/packages/plugins/examples/plugin-bookmarks-example/rollup.config.mjs b/packages/plugins/examples/plugin-bookmarks-example/rollup.config.mjs new file mode 100644 index 00000000000..ccee40a7443 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/rollup.config.mjs @@ -0,0 +1,28 @@ +import { nodeResolve } from "@rollup/plugin-node-resolve"; +import typescript from "@rollup/plugin-typescript"; +import { createPluginBundlerPresets } from "@paperclipai/plugin-sdk/bundlers"; + +const presets = createPluginBundlerPresets({ uiEntry: "src/ui/index.tsx" }); + +function withPlugins(config) { + if (!config) return null; + return { + ...config, + plugins: [ + nodeResolve({ + extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs"], + }), + typescript({ + tsconfig: "./tsconfig.json", + declaration: false, + declarationMap: false, + }), + ], + }; +} + +export default [ + withPlugins(presets.rollup.manifest), + withPlugins(presets.rollup.worker), + withPlugins(presets.rollup.ui), +].filter(Boolean); diff --git a/packages/plugins/examples/plugin-bookmarks-example/src/manifest.ts b/packages/plugins/examples/plugin-bookmarks-example/src/manifest.ts new file mode 100644 index 00000000000..402efb51369 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/src/manifest.ts @@ -0,0 +1,102 @@ +import type { PaperclipPluginManifestV1 } from "@paperclipai/plugin-sdk"; + +export const PLUGIN_ID = "paperclipai.plugin-bookmarks-example"; +export const BOOKMARKS_FOLDER_KEY = "bookmarks-root"; + +const manifest: PaperclipPluginManifestV1 = { + id: PLUGIN_ID, + apiVersion: 1, + version: "0.1.0", + displayName: "Bookmarks (Example)", + description: + "First-party example plugin: company-scoped bookmark library backed by a plugin database namespace and a local markdown folder. Demonstrates scoped API routes, local folders, dashboard widget, and a plugin page.", + author: "Paperclip", + categories: ["workspace", "ui"], + capabilities: [ + "api.routes.register", + "database.namespace.migrate", + "database.namespace.read", + "database.namespace.write", + "local.folders", + "companies.read", + "ui.dashboardWidget.register", + "ui.page.register", + "ui.sidebar.register", + "instance.settings.register", + ], + entrypoints: { + worker: "./dist/worker.js", + ui: "./dist/ui", + }, + database: { + namespaceSlug: "bookmarks", + migrationsDir: "migrations", + }, + localFolders: [ + { + folderKey: BOOKMARKS_FOLDER_KEY, + displayName: "Bookmarks root", + description: + "Company-scoped local folder that stores one markdown file per bookmark. Each file uses YAML frontmatter for url, title, tags, and notes.", + access: "readWrite", + requiredDirectories: ["bookmarks"], + }, + ], + apiRoutes: [ + { + routeKey: "list", + method: "GET", + path: "/bookmarks", + auth: "board-or-agent", + capability: "api.routes.register", + companyResolution: { from: "query", key: "companyId" }, + }, + { + routeKey: "create", + method: "POST", + path: "/bookmarks", + auth: "board-or-agent", + capability: "api.routes.register", + companyResolution: { from: "body", key: "companyId" }, + }, + { + routeKey: "delete", + method: "DELETE", + path: "/bookmarks/:slug", + auth: "board-or-agent", + capability: "api.routes.register", + companyResolution: { from: "query", key: "companyId" }, + }, + ], + ui: { + slots: [ + { + type: "page", + id: "bookmarks-page", + displayName: "Bookmarks", + exportName: "BookmarksPage", + routePath: "bookmarks", + }, + { + type: "sidebar", + id: "bookmarks-sidebar-link", + displayName: "Bookmarks", + exportName: "BookmarksSidebarLink", + }, + { + type: "dashboardWidget", + id: "bookmarks-widget", + displayName: "Bookmarks", + exportName: "BookmarksDashboardWidget", + }, + { + type: "settingsPage", + id: "bookmarks-settings", + displayName: "Bookmarks", + exportName: "BookmarksSettingsPage", + }, + ], + }, +}; + +export default manifest; diff --git a/packages/plugins/examples/plugin-bookmarks-example/src/ui/index.tsx b/packages/plugins/examples/plugin-bookmarks-example/src/ui/index.tsx new file mode 100644 index 00000000000..b832006df25 --- /dev/null +++ b/packages/plugins/examples/plugin-bookmarks-example/src/ui/index.tsx @@ -0,0 +1,359 @@ +import { + usePluginAction, + usePluginData, + type PluginPageProps, + type PluginSettingsPageProps, + type PluginSidebarProps, + type PluginWidgetProps, +} from "@paperclipai/plugin-sdk/ui"; +import { useMemo, useState, type CSSProperties, type FormEvent } from "react"; + +interface BookmarkRecord { + id: string; + companyId: string; + slug: string; + url: string; + title: string; + notes: string; + tags: string[]; + filePath: string; + createdAt: string; + updatedAt: string; +} + +interface BookmarkListResult { + databaseNamespace: string; + bookmarks: BookmarkRecord[]; +} + +const containerStyle: CSSProperties = { + display: "grid", + gap: 16, + font: "13px system-ui, sans-serif", + color: "#111827", + padding: 16, + maxWidth: 760, +}; + +const cardStyle: CSSProperties = { + border: "1px solid #e5e7eb", + borderRadius: 8, + padding: 12, + background: "#fff", +}; + +const inputStyle: CSSProperties = { + border: "1px solid #d1d5db", + borderRadius: 6, + padding: "6px 8px", + font: "inherit", + width: "100%", +}; + +const buttonStyle: CSSProperties = { + border: "1px solid #1f2937", + background: "#111827", + color: "#fff", + borderRadius: 6, + padding: "6px 10px", + font: "inherit", + cursor: "pointer", +}; + +const subtleButtonStyle: CSSProperties = { + border: "1px solid #d1d5db", + background: "#fff", + color: "#111827", + borderRadius: 6, + padding: "4px 8px", + font: "inherit", + cursor: "pointer", +}; + +const tagStyle: CSSProperties = { + display: "inline-block", + border: "1px solid #cbd5f5", + background: "#eef2ff", + color: "#3730a3", + borderRadius: 999, + padding: "1px 8px", + marginRight: 4, + fontSize: 11, +}; + +function parseTagInput(raw: string): string[] { + return raw + .split(/[\s,]+/) + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function formatDate(value: string): string { + const date = new Date(value); + if (Number.isNaN(date.getTime())) return value; + return date.toLocaleString(); +} + +function BookmarksList({ + result, + onDelete, + loading, +}: { + result: BookmarkListResult | null; + onDelete: (slug: string) => Promise; + loading: boolean; +}) { + if (loading) return
Loading bookmarks…
; + if (!result) return null; + if (result.bookmarks.length === 0) { + return
No bookmarks yet.
; + } + return ( +
+ {result.bookmarks.map((bookmark) => ( +
+
+ + {bookmark.title} + + +
+
{bookmark.url}
+ {bookmark.tags.length > 0 ? ( +
+ {bookmark.tags.map((tag) => ( + + {tag} + + ))} +
+ ) : null} + {bookmark.notes ? ( +
{bookmark.notes}
+ ) : null} +
+ Added {formatDate(bookmark.createdAt)} · {bookmark.filePath} +
+
+ ))} +
+ ); +} + +function BookmarksSurface({ companyId }: { companyId: string }) { + const [search, setSearch] = useState(""); + const [tag, setTag] = useState(""); + const [url, setUrl] = useState(""); + const [title, setTitle] = useState(""); + const [tagsRaw, setTagsRaw] = useState(""); + const [notes, setNotes] = useState(""); + const [errorMessage, setErrorMessage] = useState(null); + + const listParams = useMemo( + () => ({ companyId, search: search.trim() || null, tag: tag.trim() || null }), + [companyId, search, tag], + ); + + const { data, loading, error, refresh } = usePluginData("list", listParams); + const create = usePluginAction("create"); + const remove = usePluginAction("delete"); + + async function handleCreate(event: FormEvent) { + event.preventDefault(); + setErrorMessage(null); + try { + await create({ + companyId, + url, + title: title || null, + notes: notes || null, + tags: parseTagInput(tagsRaw), + }); + setUrl(""); + setTitle(""); + setNotes(""); + setTagsRaw(""); + refresh(); + } catch (cause) { + setErrorMessage(cause instanceof Error ? cause.message : "Failed to add bookmark"); + } + } + + async function handleDelete(slug: string) { + setErrorMessage(null); + try { + await remove({ companyId, slug }); + refresh(); + } catch (cause) { + setErrorMessage(cause instanceof Error ? cause.message : "Failed to delete bookmark"); + } + } + + return ( +
+
+

Bookmarks

+
+ Company-scoped bookmark library backed by a plugin database namespace and a local markdown + folder. +
+
+ +
+ Add bookmark + setUrl(event.target.value)} + /> + setTitle(event.target.value)} + /> + setTagsRaw(event.target.value)} + /> +