diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 18a7208f208..e4407d2e17d 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -202,6 +202,17 @@ Triage writes serialize on the company and attention-source identity so concurre The plugin runtime tracks plugin-owned database namespaces and migrations in `plugin_database_namespaces` and `plugin_migrations`. Hosted deployments that separate runtime and migration connections should set `DATABASE_MIGRATION_URL`; plugin namespace migration work uses the migration connection when present. +## Agent instruction bundle durability + +Managed agent instruction bundles are stored in Postgres through `built_in_managed_resources` instruction bindings; the instance `companies//agents//instructions/` directory is a rebuildable materialization cache. Existing installations can import disk-backed managed bundles idempotently with: + +```sh +pnpm instructions:import-bindings # report only +pnpm instructions:import-bindings --apply # persist missing bindings +``` + +Bundles explicitly configured with `instructionsBundleMode: "external"` remain pointers by design. Their roots genuinely live outside Paperclip's managed instance tree, are not copied into Postgres, and must be backed up by the external owner. + ## Backups Paperclip supports automatic and manual logical database backups. These dumps include diff --git a/package.json b/package.json index 7ef3cad63f1..0919049f07a 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "test:release-smoke:headed": "npx playwright test --config tests/release-smoke/playwright.config.ts --headed", "metrics:paperclip-commits": "tsx scripts/paperclip-commit-metrics.ts", "perf:issue-chat-long-thread": "node scripts/measure-issue-chat-long-thread.mjs", - "connections:ingest-app-definitions": "node scripts/ingest-app-definitions.mjs" + "connections:ingest-app-definitions": "node scripts/ingest-app-definitions.mjs", + "instructions:import-bindings": "pnpm --filter @paperclipai/server exec tsx scripts/import-agent-instruction-bindings.ts" }, "devDependencies": { "@playwright/test": "^1.61.1", diff --git a/server/scripts/import-agent-instruction-bindings.ts b/server/scripts/import-agent-instruction-bindings.ts new file mode 100644 index 00000000000..f62541449ee --- /dev/null +++ b/server/scripts/import-agent-instruction-bindings.ts @@ -0,0 +1,54 @@ +import { and, eq } from "drizzle-orm"; +import { agents, builtInManagedResources, createDb } from "@paperclipai/db"; +import { resolveDatabaseTarget } from "../../packages/db/src/runtime-config.js"; +import { agentInstructionsService } from "../src/services/agent-instructions.js"; + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +async function main() { + const target = resolveDatabaseTarget(); + const dbUrl = target.mode === "postgres" + ? target.connectionString + : `postgres://paperclip:paperclip@127.0.0.1:${target.port}/paperclip`; + const apply = process.argv.includes("--apply"); + const db = createDb(dbUrl); + const instructions = agentInstructionsService(db); + let imported = 0; + let skippedExternal = 0; + let skippedEmpty = 0; + let alreadyBound = 0; + + for (const agent of await db.select().from(agents)) { + const config = asRecord(agent.adapterConfig); + if (config.instructionsBundleMode === "external") { + skippedExternal += 1; + continue; + } + const existing = await db.select({ id: builtInManagedResources.id }).from(builtInManagedResources).where(and( + eq(builtInManagedResources.companyId, agent.companyId), + eq(builtInManagedResources.resourceKind, "instructions"), + eq(builtInManagedResources.resourceId, agent.id), + )).then((rows) => rows[0] ?? null); + if (existing) { + alreadyBound += 1; + continue; + } + const bundle = await instructions.exportFiles(agent); + if (Object.keys(bundle.files).length === 0) { + skippedEmpty += 1; + continue; + } + if (apply) await instructions.persistManagedBundle(agent, bundle.entryFile, bundle.files); + imported += 1; + } + + console.log(JSON.stringify({ mode: apply ? "apply" : "dry-run", imported, alreadyBound, skippedExternal, skippedEmpty }, null, 2)); + process.exit(0); +} + +void main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 7ff546cb9ff..7321c387d2e 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -386,7 +386,7 @@ export function agentRoutes( const recovery = recoveryService(db, { enqueueWakeup: heartbeat.wakeup }); const issueApprovalsSvc = issueApprovalService(db); const secretsSvc = secretService(db); - const instructions = agentInstructionsService(); + const instructions = agentInstructionsService(db); const companySkills = companySkillService(db); const workspaceOperations = workspaceOperationService(db); const instanceSettings = instanceSettingsService(db); diff --git a/server/src/services/agent-instructions.ts b/server/src/services/agent-instructions.ts index 6dcbb38f047..81b9a80abf5 100644 --- a/server/src/services/agent-instructions.ts +++ b/server/src/services/agent-instructions.ts @@ -1,5 +1,9 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { createHash } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { builtInManagedResources } from "@paperclipai/db"; import { notFound, unprocessable } from "../errors.js"; import { resolveHomeAwarePath, resolvePaperclipInstanceRoot } from "../home-paths.js"; @@ -451,9 +455,69 @@ export function syncInstructionsBundleConfigFromFilePath( return applyBundleConfig(next, { mode, rootPath, entryFile }); } -export function agentInstructionsService() { +export function agentInstructionsService(db?: Db) { + async function getStoredBundle(agent: AgentLike) { + if (!db) return null; + const rows = await db.select().from(builtInManagedResources).where(and( + eq(builtInManagedResources.companyId, agent.companyId), + eq(builtInManagedResources.resourceKind, "instructions"), + eq(builtInManagedResources.resourceId, agent.id), + )); + for (const row of rows) { + const defaults = asRecord(row.defaultsJson); + const files = Object.fromEntries(Object.entries(asRecord(defaults.files)).filter((entry): entry is [string, string] => typeof entry[1] === "string")); + if (Object.keys(files).length > 0) return { row, entryFile: asString(defaults.entryFile) ?? ENTRY_FILE_DEFAULT, files }; + } + return null; + } + + async function persistManagedBundle(agent: AgentLike, entryFile: string, files: Record) { + if (!db || Object.keys(files).length === 0) return; + const existing = await getStoredBundle(agent); + const stockHash = `sha256:${createHash("sha256").update(JSON.stringify({ entryFile, files })).digest("hex")}`; + const values = { + companyId: agent.companyId, + bundleKey: existing?.row.bundleKey ?? `custom-agent:${agent.id}`, + resourceKind: "instructions", + resourceKey: existing?.row.resourceKey ?? "bundle", + resourceId: agent.id, + stockVersion: existing?.row.stockVersion ?? "custom-v1", + stockHash, + defaultsJson: { entryFile, files }, + }; + await db.insert(builtInManagedResources).values(values).onConflictDoUpdate({ + target: [builtInManagedResources.companyId, builtInManagedResources.bundleKey, builtInManagedResources.resourceKind, builtInManagedResources.resourceKey], + set: { resourceId: agent.id, stockHash, defaultsJson: values.defaultsJson, updatedAt: new Date() }, + }); + } + + async function resolveState(agent: AgentLike) { + const derived = deriveBundleState(agent); + if (derived.mode !== "external") { + const stored = await getStoredBundle(agent); + if (stored) { + const rootPath = resolveManagedInstructionsRoot(agent); + const stat = await statIfExists(rootPath); + const files = stat?.isDirectory() ? await listFilesRecursive(rootPath) : []; + if (files.length === 0) { + await fs.mkdir(rootPath, { recursive: true }); + await writeBundleFiles(rootPath, stored.files, { overwriteExisting: true }); + } + return recoverManagedBundleState(agent, { ...derived, mode: "managed", rootPath, entryFile: stored.entryFile, resolvedEntryPath: path.resolve(rootPath, stored.entryFile) }); + } + } + return recoverManagedBundleState(agent, derived); + } + + async function persistState(agent: AgentLike, state: BundleState) { + if (state.mode !== "managed" || !state.rootPath) return; + const relativePaths = await listFilesRecursive(state.rootPath); + const files = Object.fromEntries(await Promise.all(relativePaths.map(async (relativePath) => [relativePath, await fs.readFile(resolvePathWithinRoot(state.rootPath!, relativePath), "utf8")] as const))); + await persistManagedBundle(agent, state.entryFile, files); + } + async function getBundle(agent: AgentLike): Promise { - const state = await recoverManagedBundleState(agent, deriveBundleState(agent)); + const state = await resolveState(agent); if (!state.rootPath) return toBundle(agent, state, []); const stat = await statIfExists(state.rootPath); if (!stat?.isDirectory()) { @@ -468,7 +532,7 @@ export function agentInstructionsService() { } async function readFile(agent: AgentLike, relativePath: string): Promise { - const state = await recoverManagedBundleState(agent, deriveBundleState(agent)); + const state = await resolveState(agent); if (relativePath === LEGACY_PROMPT_TEMPLATE_PATH) { const content = asString(state.config[PROMPT_KEY]); if (content === null) throw notFound("Instructions file not found"); @@ -510,7 +574,7 @@ export function agentInstructionsService() { options?: { clearLegacyPromptTemplate?: boolean }, ): Promise<{ adapterConfig: Record; state: BundleState }> { const derived = deriveBundleState(agent); - const current = await recoverManagedBundleState(agent, derived); + const current = await resolveState(agent); if (current.rootPath && current.mode) { const adapterConfig = buildPersistedBundleConfig(derived, current, options); return { @@ -554,7 +618,7 @@ export function agentInstructionsService() { clearLegacyPromptTemplate?: boolean; }, ): Promise<{ bundle: AgentInstructionsBundle; adapterConfig: Record }> { - const state = await recoverManagedBundleState(agent, deriveBundleState(agent)); + const state = await resolveState(agent); const nextMode = input.mode ?? state.mode ?? "managed"; const nextEntryFile = input.entryFile ? normalizeRelativeFilePath(input.entryFile) : state.entryFile; let nextRootPath: string; @@ -592,7 +656,9 @@ export function agentInstructionsService() { entryFile: nextEntryFile, clearLegacyPromptTemplate: input.clearLegacyPromptTemplate, }); - const nextBundle = await getBundle({ ...agent, adapterConfig: nextConfig }); + const nextAgent = { ...agent, adapterConfig: nextConfig }; + if (nextMode === "managed") await persistState(nextAgent, await resolveState(nextAgent)); + const nextBundle = await getBundle(nextAgent); return { bundle: nextBundle, adapterConfig: nextConfig }; } @@ -625,6 +691,7 @@ export function agentInstructionsService() { await fs.mkdir(path.dirname(absolutePath), { recursive: true }); await fs.writeFile(absolutePath, content, "utf8"); const nextAgent = { ...agent, adapterConfig: prepared.adapterConfig }; + await persistState(nextAgent, prepared.state); const [bundle, file] = await Promise.all([ getBundle(nextAgent), readFile(nextAgent, relativePath), @@ -637,7 +704,7 @@ export function agentInstructionsService() { adapterConfig: Record; }> { const derived = deriveBundleState(agent); - const state = await recoverManagedBundleState(agent, derived); + const state = await resolveState(agent); if (relativePath === LEGACY_PROMPT_TEMPLATE_PATH) { throw unprocessable("Cannot delete the legacy promptTemplate pseudo-file"); } @@ -649,6 +716,7 @@ export function agentInstructionsService() { const absolutePath = resolvePathWithinRoot(state.rootPath, normalizedPath); await fs.rm(absolutePath, { force: true }); const adapterConfig = buildPersistedBundleConfig(derived, state); + await persistState({ ...agent, adapterConfig }, state); const bundle = await getBundle({ ...agent, adapterConfig }); return { bundle, adapterConfig }; } @@ -658,7 +726,7 @@ export function agentInstructionsService() { entryFile: string; warnings: string[]; }> { - const state = await recoverManagedBundleState(agent, deriveBundleState(agent)); + const state = await resolveState(agent); if (state.rootPath) { const stat = await statIfExists(state.rootPath); if (stat?.isDirectory()) { @@ -718,7 +786,11 @@ export function agentInstructionsService() { entryFile, clearLegacyPromptTemplate: options?.clearLegacyPromptTemplate, }); - const bundle = await getBundle({ ...agent, adapterConfig }); + const nextAgent = { ...agent, adapterConfig }; + const persistedFiles = Object.fromEntries(normalizedEntries); + if (!(entryFile in persistedFiles)) persistedFiles[entryFile] = ""; + await persistManagedBundle(nextAgent, entryFile, persistedFiles); + const bundle = await getBundle(nextAgent); return { bundle, adapterConfig }; } @@ -731,5 +803,6 @@ export function agentInstructionsService() { exportFiles, ensureManagedBundle: ensureWritableBundle, materializeManagedBundle, + persistManagedBundle, }; } diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index c86c2bbbefd..8d8dcb8a8ce 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -3476,7 +3476,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const companies = companyService(db); const agents = agentService(db); const assetRecords = assetService(db); - const instructions = agentInstructionsService(); + const instructions = agentInstructionsService(db); const access = accessService(db); const projects = projectService(db); const issues = issueService(db); diff --git a/server/src/services/plugin-managed-agents.ts b/server/src/services/plugin-managed-agents.ts index ea95b59ce8c..07f292aa3c2 100644 --- a/server/src/services/plugin-managed-agents.ts +++ b/server/src/services/plugin-managed-agents.ts @@ -176,7 +176,7 @@ export function pluginManagedAgentService( ) { const agentSvc = agentService(db); const approvalSvc = approvalService(db); - const instructions = agentInstructionsService(); + const instructions = agentInstructionsService(db); function declarationFor(agentKey: string) { const declaration = options.manifest?.agents?.find((agent) => agent.agentKey === agentKey);