diff --git a/server/src/__tests__/agent-instructions-service.test.ts b/server/src/__tests__/agent-instructions-service.test.ts index 67eea3ca85d..52c2142ade3 100644 --- a/server/src/__tests__/agent-instructions-service.test.ts +++ b/server/src/__tests__/agent-instructions-service.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { agentInstructionsService } from "../services/agent-instructions.js"; type TestAgent = { @@ -358,4 +358,164 @@ describe("agent instructions service", () => { ]); expect(exported.files).toEqual({ "AGENTS.md": "# Managed Agent\n" }); }); + + it("preserves the whole operator-modified tree during routine materialization", async () => { + const paperclipHome = await makeTempDir("paperclip-agent-instructions-drift-"); + cleanupDirs.add(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "test-instance"; + const svc = agentInstructionsService(); + const agent = makeAgent({}); + + const initial = await svc.materializeManagedBundle(agent, { + "AGENTS.md": "# Stock v1\n", + "docs/STOCK.md": "Stock docs\n", + }); + expect(initial.materialization).toMatchObject({ stockStatus: "missing", action: "added" }); + const managedRoot = initial.bundle.managedRootPath; + await fs.writeFile(path.join(managedRoot, "AGENTS.md"), "# Operator edit\n", "utf8"); + await fs.writeFile(path.join(managedRoot, "OPERATOR.md"), "Operator-added file\n", "utf8"); + + const rebuilt = await svc.materializeManagedBundle( + { ...agent, adapterConfig: initial.adapterConfig }, + { + "AGENTS.md": "# Stock v2\n", + "docs/STOCK.md": "Updated stock docs\n", + }, + ); + + expect(rebuilt.materialization).toMatchObject({ + stockStatus: "operator_modified", + action: "skipped", + skipped: ["AGENTS.md", "OPERATOR.md", "docs/STOCK.md"], + backupPath: null, + }); + expect(rebuilt.adapterConfig.instructionsStockHash).toBe(initial.adapterConfig.instructionsStockHash); + await expect(fs.readFile(path.join(managedRoot, "AGENTS.md"), "utf8")).resolves.toBe("# Operator edit\n"); + await expect(fs.readFile(path.join(managedRoot, "OPERATOR.md"), "utf8")).resolves.toBe("Operator-added file\n"); + await expect(fs.readFile(path.join(managedRoot, "docs", "STOCK.md"), "utf8")).resolves.toBe("Stock docs\n"); + }); + + it("atomically advances a clean stock tree and its recorded hash", async () => { + const paperclipHome = await makeTempDir("paperclip-agent-instructions-update-"); + cleanupDirs.add(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "test-instance"; + const svc = agentInstructionsService(); + const agent = makeAgent({}); + const initial = await svc.materializeManagedBundle(agent, { + "AGENTS.md": "# Stock v1\n", + "docs/OLD.md": "Old\n", + }); + + const updated = await svc.materializeManagedBundle( + { ...agent, adapterConfig: initial.adapterConfig }, + { + "AGENTS.md": "# Stock v2\n", + "docs/NEW.md": "New\n", + }, + ); + + expect(updated.materialization).toMatchObject({ + stockStatus: "stock_update_available", + action: "updated", + added: ["docs/NEW.md"], + removed: ["docs/OLD.md"], + updated: ["AGENTS.md"], + backupPath: null, + }); + expect(updated.adapterConfig.instructionsStockHash).toBe(updated.materialization.stockHash); + expect(updated.adapterConfig.instructionsStockHash).not.toBe(initial.adapterConfig.instructionsStockHash); + await expect(fs.readFile(path.join(updated.bundle.managedRootPath, "AGENTS.md"), "utf8")).resolves.toBe("# Stock v2\n"); + await expect(fs.readFile(path.join(updated.bundle.managedRootPath, "docs", "NEW.md"), "utf8")).resolves.toBe("New\n"); + await expect(fs.stat(path.join(updated.bundle.managedRootPath, "docs", "OLD.md"))).rejects.toThrow(); + }); + + it("canonicalizes line endings and ignores transient files when classifying drift", async () => { + const paperclipHome = await makeTempDir("paperclip-agent-instructions-canonical-"); + cleanupDirs.add(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "test-instance"; + const svc = agentInstructionsService(); + const agent = makeAgent({}); + const initial = await svc.materializeManagedBundle(agent, { "AGENTS.md": "# Stock\nLine\n" }); + const managedRoot = initial.bundle.managedRootPath; + await fs.writeFile(path.join(managedRoot, "AGENTS.md"), "# Stock\r\nLine\r\n", "utf8"); + await fs.writeFile(path.join(managedRoot, ".DS_Store"), "transient", "utf8"); + + const unchanged = await svc.materializeManagedBundle( + { ...agent, adapterConfig: initial.adapterConfig }, + { "AGENTS.md": "# Stock\nLine\n" }, + ); + + expect(unchanged.materialization).toMatchObject({ stockStatus: "stock_current", action: "unchanged" }); + await expect(fs.readFile(path.join(managedRoot, "AGENTS.md"), "utf8")).resolves.toBe("# Stock\r\nLine\r\n"); + await expect(fs.readFile(path.join(managedRoot, ".DS_Store"), "utf8")).resolves.toBe("transient"); + }); + + it("rolls the live tree back when the staged atomic swap fails", async () => { + const paperclipHome = await makeTempDir("paperclip-agent-instructions-rollback-"); + cleanupDirs.add(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "test-instance"; + const svc = agentInstructionsService(); + const agent = makeAgent({}); + const initial = await svc.materializeManagedBundle(agent, { "AGENTS.md": "# Stock v1\n" }); + const originalRename = fs.rename.bind(fs); + const renameSpy = vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + if (path.basename(String(source)) === "staged") throw new Error("simulated swap failure"); + return originalRename(source, destination); + }); + + try { + await expect(svc.materializeManagedBundle( + { ...agent, adapterConfig: initial.adapterConfig }, + { "AGENTS.md": "# Stock v2\n" }, + )).rejects.toThrow("simulated swap failure"); + } finally { + renameSpy.mockRestore(); + } + + await expect(fs.readFile(path.join(initial.bundle.managedRootPath, "AGENTS.md"), "utf8")).resolves.toBe("# Stock v1\n"); + }); + + it("backs up operator state before an explicitly requested force reset", async () => { + const paperclipHome = await makeTempDir("paperclip-agent-instructions-reset-"); + cleanupDirs.add(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "test-instance"; + const svc = agentInstructionsService(); + const agent = makeAgent({}); + const initial = await svc.materializeManagedBundle(agent, { "AGENTS.md": "# Stock v1\n" }); + const managedRoot = initial.bundle.managedRootPath; + await fs.writeFile(path.join(managedRoot, "AGENTS.md"), "# Operator edit\n", "utf8"); + await fs.writeFile(path.join(managedRoot, "OPERATOR.md"), "Keep me\n", "utf8"); + + const reset = await svc.forceResetManagedBundle( + { ...agent, adapterConfig: initial.adapterConfig }, + { "AGENTS.md": "# Stock v2\n" }, + ); + + expect(reset.materialization).toMatchObject({ + stockStatus: "operator_modified", + action: "reset", + backupPath: `${managedRoot}.backup-1`, + backedUp: ["AGENTS.md", "OPERATOR.md"], + }); + await expect(fs.readFile(path.join(`${managedRoot}.backup-1`, "AGENTS.md"), "utf8")).resolves.toBe("# Operator edit\n"); + await expect(fs.readFile(path.join(`${managedRoot}.backup-1`, "OPERATOR.md"), "utf8")).resolves.toBe("Keep me\n"); + await expect(fs.readFile(path.join(managedRoot, "AGENTS.md"), "utf8")).resolves.toBe("# Stock v2\n"); + await expect(fs.stat(path.join(managedRoot, "OPERATOR.md"))).rejects.toThrow(); + + await fs.writeFile(path.join(managedRoot, "AGENTS.md"), "# Second operator edit\n", "utf8"); + const secondReset = await svc.forceResetManagedBundle( + { ...agent, adapterConfig: reset.adapterConfig }, + { "AGENTS.md": "# Stock v3\n" }, + ); + expect(secondReset.materialization.backupPath).toBe(`${managedRoot}.backup-2`); + await expect(fs.readFile(path.join(`${managedRoot}.backup-1`, "AGENTS.md"), "utf8")).resolves.toBe("# Operator edit\n"); + await expect(fs.readFile(path.join(`${managedRoot}.backup-2`, "AGENTS.md"), "utf8")).resolves.toBe( + "# Second operator edit\n", + ); + }); }); diff --git a/server/src/__tests__/agent-skills-routes.test.ts b/server/src/__tests__/agent-skills-routes.test.ts index 6dd77801ed1..ae3f98797dd 100644 --- a/server/src/__tests__/agent-skills-routes.test.ts +++ b/server/src/__tests__/agent-skills-routes.test.ts @@ -1084,7 +1084,7 @@ describe.sequential("agent skill routes", () => { "SOUL.md": expect.stringContaining("CEO Persona"), "TOOLS.md": expect.stringContaining("# Tools"), }), - { entryFile: "AGENTS.md", replaceExisting: false }, + { entryFile: "AGENTS.md" }, ); }); @@ -1110,7 +1110,7 @@ describe.sequential("agent skill routes", () => { expect.objectContaining({ "AGENTS.md": expect.stringMatching(/Start actionable work in the same heartbeat\.[\s\S]*Keep the work moving until it is done\./), }), - { entryFile: "AGENTS.md", replaceExisting: false }, + { entryFile: "AGENTS.md" }, ); expect(mockAgentInstructionsService.materializeManagedBundle).toHaveBeenCalledWith( expect.any(Object), diff --git a/server/src/__tests__/built-in-agents.test.ts b/server/src/__tests__/built-in-agents.test.ts index 5ab8b6dde07..e7e24b38350 100644 --- a/server/src/__tests__/built-in-agents.test.ts +++ b/server/src/__tests__/built-in-agents.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import fs from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -797,6 +798,7 @@ describeEmbeddedPostgres("built-in agents", () => { const instructions = agentInstructionsService(); await instructions.writeFile(created.agent!, "AGENTS.md", "# Custom Reflection Coach\n\nOperator edit.\n"); + await instructions.writeFile(created.agent!, "OPERATOR.md", "Operator-added file.\n"); const reconciled = await builtIns.ensure(companyId, "reflection-coach"); const drift = reconciled.resources.find((resource) => resource.resourceKind === "instructions"); @@ -804,12 +806,16 @@ describeEmbeddedPostgres("built-in agents", () => { stockStatus: "operator_modified", updateAvailable: true, resetAvailable: true, - changedFiles: ["AGENTS.md"], + changedFiles: ["AGENTS.md", "OPERATOR.md"], }); await expect(instructions.readFile(reconciled.agent!, "AGENTS.md")).resolves.toMatchObject({ content: "# Custom Reflection Coach\n\nOperator edit.\n", }); + await expect(instructions.readFile(reconciled.agent!, "OPERATOR.md")).resolves.toMatchObject({ + content: "Operator-added file.\n", + }); + const managedRoot = String(reconciled.agent?.adapterConfig.instructionsRootPath); const reset = await builtIns.reset(companyId, "reflection-coach"); expect(reset.resources.find((resource) => resource.resourceKind === "instructions")).toMatchObject({ stockStatus: "stock_current", @@ -818,6 +824,12 @@ describeEmbeddedPostgres("built-in agents", () => { const resetFile = await instructions.readFile(reset.agent!, "AGENTS.md"); expect(resetFile.content).toContain("Reflection Coach"); expect(resetFile.content).not.toContain("Operator edit."); + await expect(fs.readFile(path.join(`${managedRoot}.backup-1`, "AGENTS.md"), "utf8")).resolves.toBe( + "# Custom Reflection Coach\n\nOperator edit.\n", + ); + await expect(fs.readFile(path.join(`${managedRoot}.backup-1`, "OPERATOR.md"), "utf8")).resolves.toBe( + "Operator-added file.\n", + ); }); it("blocks deleting a built-in agent", async () => { diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index 9631ddd3b7b..c648caa2eb1 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -454,6 +454,19 @@ describe("company portability", () => { instructionsEntryFile: "AGENTS.md", instructionsFilePath: `/tmp/${agent.id}/AGENTS.md`, }, + materialization: { + stockStatus: "missing", + action: "added", + stockHash: "sha256:stock", + previousHash: null, + recordedStockHash: null, + added: ["AGENTS.md"], + removed: [], + updated: [], + skipped: [], + backedUp: [], + backupPath: null, + }, })); }); @@ -3428,7 +3441,6 @@ describe("company portability", () => { }), expect.objectContaining({ clearLegacyPromptTemplate: true, - replaceExisting: true, }), ); const materializedFiles = agentInstructionsSvc.materializeManagedBundle.mock.calls[0]?.[1] as Record; @@ -5213,6 +5225,15 @@ describe("company portability", () => { }, }); + agentSvc.list.mockResolvedValue([{ + id: "agent-1", + name: "ClaudeCoder", + status: "idle", + adapterConfig: { + instructionsStockHash: "sha256:previous-stock", + }, + }]); + secretSvc.normalizeAdapterConfigForPersistence.mockResolvedValueOnce({ normalized: "updated", }); @@ -5222,8 +5243,32 @@ describe("company portability", () => { adapterType: patch.adapterType, adapterConfig: patch.adapterConfig, })); + agentInstructionsSvc.materializeManagedBundle.mockImplementationOnce(async (agent: { id: string; adapterConfig: Record }) => ({ + bundle: null, + adapterConfig: { + ...agent.adapterConfig, + instructionsBundleMode: "managed", + instructionsRootPath: `/tmp/${agent.id}`, + instructionsEntryFile: "AGENTS.md", + instructionsFilePath: `/tmp/${agent.id}/AGENTS.md`, + instructionsStockHash: "sha256:previous-stock", + }, + materialization: { + stockStatus: "operator_modified", + action: "skipped", + stockHash: "sha256:imported-stock", + previousHash: "sha256:operator-tree", + recordedStockHash: "sha256:previous-stock", + added: [], + removed: ["OPERATOR.md"], + updated: ["AGENTS.md"], + skipped: ["AGENTS.md", "OPERATOR.md"], + backedUp: [], + backupPath: null, + }, + })); - await portability.importBundle({ + const result = await portability.importBundle({ source: { type: "inline", rootPath: exported.rootPath, @@ -5263,8 +5308,22 @@ describe("company portability", () => { adapterType: "codex_local", adapterConfig: { normalized: "updated", + instructionsStockHash: "sha256:previous-stock", }, })); + expect(agentInstructionsSvc.materializeManagedBundle).toHaveBeenCalledWith( + expect.objectContaining({ + id: "agent-1", + adapterConfig: expect.objectContaining({ + instructionsStockHash: "sha256:previous-stock", + }), + }), + expect.any(Object), + { clearLegacyPromptTemplate: true }, + ); + expect(result.warnings).toContain( + "Preserved operator-modified instructions bundle for claudecoder; imported stock files were skipped.", + ); }); it("nameOverrides applied after collision detection do not re-validate uniqueness", async () => { diff --git a/server/src/__tests__/plugin-managed-agents.test.ts b/server/src/__tests__/plugin-managed-agents.test.ts index ed512a8d65a..0b0ece4b084 100644 --- a/server/src/__tests__/plugin-managed-agents.test.ts +++ b/server/src/__tests__/plugin-managed-agents.test.ts @@ -301,6 +301,21 @@ describeEmbeddedPostgres("plugin-managed agents", () => { expect(content).toContain("You are the LLM Wiki Maintainer."); expect(content).toContain(`Wiki root: \`${wikiRoot}\``); expect(content).toContain(`Wiki schema: \`${path.join(wikiRoot, "AGENTS.md")}\``); + + const instructionsRoot = path.dirname(instructionsFilePath as string); + await fs.writeFile(instructionsFilePath as string, "# Operator plugin edit\n", "utf8"); + await fs.writeFile(path.join(instructionsRoot, "OPERATOR.md"), "Operator-added plugin file\n", "utf8"); + await services.agents.managedReset({ companyId, agentKey: "wiki-maintainer" }); + + await expect(fs.readFile(path.join(`${instructionsRoot}.backup-1`, "AGENTS.md"), "utf8")).resolves.toBe( + "# Operator plugin edit\n", + ); + await expect(fs.readFile(path.join(`${instructionsRoot}.backup-1`, "OPERATOR.md"), "utf8")).resolves.toBe( + "Operator-added plugin file\n", + ); + await expect(fs.readFile(instructionsFilePath as string, "utf8")).resolves.toContain( + "You are the LLM Wiki Maintainer.", + ); } finally { if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 97754e352d8..c25abd85609 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -1438,7 +1438,7 @@ export function agentRoutes( const materialized = await instructions.materializeManagedBundle( agent, files, - { entryFile: input?.entryFile ?? "AGENTS.md", replaceExisting: false }, + { entryFile: input?.entryFile ?? "AGENTS.md" }, ); const nextAdapterConfig = { ...materialized.adapterConfig }; delete nextAdapterConfig.promptTemplate; diff --git a/server/src/services/agent-instructions.ts b/server/src/services/agent-instructions.ts index 6dcbb38f047..ae6964133b9 100644 --- a/server/src/services/agent-instructions.ts +++ b/server/src/services/agent-instructions.ts @@ -1,7 +1,10 @@ +import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { notFound, unprocessable } from "../errors.js"; import { resolveHomeAwarePath, resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { resourceStatus } from "./managed-resource-drift.js"; +import type { ManagedResourceStockStatus } from "./managed-resource-drift.js"; const ENTRY_FILE_DEFAULT = "AGENTS.md"; const MODE_KEY = "instructionsBundleMode"; @@ -9,6 +12,7 @@ const ROOT_KEY = "instructionsRootPath"; const ENTRY_KEY = "instructionsEntryFile"; const FILE_KEY = "instructionsFilePath"; const PROMPT_KEY = "promptTemplate"; +const STOCK_HASH_KEY = "instructionsStockHash"; /** @deprecated Use the managed instructions bundle system instead. */ const BOOTSTRAP_PROMPT_KEY = "bootstrapPromptTemplate"; const LEGACY_PROMPT_TEMPLATE_PATH = "promptTemplate.legacy.md"; @@ -65,6 +69,20 @@ type AgentInstructionsBundle = { files: AgentInstructionsFileSummary[]; }; +export type ManagedBundleMaterialization = { + stockStatus: ManagedResourceStockStatus; + action: "added" | "updated" | "unchanged" | "skipped" | "reset"; + stockHash: string; + previousHash: string | null; + recordedStockHash: string | null; + added: string[]; + removed: string[]; + updated: string[]; + skipped: string[]; + backedUp: string[]; + backupPath: string | null; +}; + type BundleState = { config: Record; mode: BundleMode | null; @@ -191,7 +209,80 @@ async function listFilesRecursive(rootPath: string): Promise { } await walk(rootPath, ""); - return output.sort((left, right) => left.localeCompare(right)); + return output.sort(compareInstructionPaths); +} + +function canonicalizeInstructionsContent(content: string) { + return content.replace(/\r\n?/g, "\n"); +} + +function compareInstructionPaths(left: string, right: string) { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function managedInstructionsTreeHash(files: Record) { + const canonicalEntries = Object.entries(files) + .map(([relativePath, content]) => [ + normalizeRelativeFilePath(relativePath), + canonicalizeInstructionsContent(content), + ] as const) + .sort(([left], [right]) => compareInstructionPaths(left, right)); + const canonicalJson = `{${canonicalEntries + .map(([relativePath, content]) => `${JSON.stringify(relativePath)}:${JSON.stringify(content)}`) + .join(",")}}`; + return `sha256:${createHash("sha256").update(canonicalJson).digest("hex")}`; +} + +async function readInstructionsTree(rootPath: string): Promise | null> { + const rootStat = await statIfExists(rootPath); + if (!rootStat) return null; + if (!rootStat.isDirectory()) return {}; + const relativePaths = await listFilesRecursive(rootPath); + return Object.fromEntries(await Promise.all(relativePaths.map(async (relativePath) => [ + relativePath, + await fs.readFile(resolvePathWithinRoot(rootPath, relativePath), "utf8"), + ] as const))); +} + +function normalizeMaterializedFiles(files: Record, entryFile: string) { + const normalized = new Map(); + for (const [relativePath, content] of Object.entries(files)) { + const normalizedPath = normalizeRelativeFilePath(relativePath); + if (normalized.has(normalizedPath)) { + throw unprocessable(`Duplicate instructions file path after normalization: ${normalizedPath}`); + } + normalized.set(normalizedPath, content); + } + if (!normalized.has(entryFile)) normalized.set(entryFile, ""); + return Object.fromEntries( + [...normalized.entries()].sort(([left], [right]) => compareInstructionPaths(left, right)), + ); +} + +function changedInstructionFiles(current: Record, prospective: Record) { + const added: string[] = []; + const removed: string[] = []; + const updated: string[] = []; + const paths = [...new Set([...Object.keys(current), ...Object.keys(prospective)])] + .sort(compareInstructionPaths); + for (const relativePath of paths) { + const hasCurrent = Object.prototype.hasOwnProperty.call(current, relativePath); + const hasProspective = Object.prototype.hasOwnProperty.call(prospective, relativePath); + if (!hasCurrent && hasProspective) added.push(relativePath); + else if (hasCurrent && !hasProspective) removed.push(relativePath); + else if ( + canonicalizeInstructionsContent(current[relativePath]!) + !== canonicalizeInstructionsContent(prospective[relativePath]!) + ) updated.push(relativePath); + } + return { added, removed, updated }; +} + +async function nextInstructionsBackupPath(rootPath: string) { + for (let index = 1; ; index += 1) { + const candidate = `${rootPath}.backup-${index}`; + if (!(await statIfExists(candidate))) return candidate; + } } async function readFileSummary(rootPath: string, relativePath: string, entryFile: string): Promise { @@ -344,7 +435,7 @@ function toBundle(agent: AgentLike, state: BundleState, files: AgentInstructions virtual: true, }); } - nextFiles.sort((left, right) => left.path.localeCompare(right.path)); + nextFiles.sort((left, right) => compareInstructionPaths(left.path, right.path)); return { agentId: agent.id, companyId: agent.companyId, @@ -381,6 +472,7 @@ function applyBundleConfig( delete next[PROMPT_KEY]; delete next[BOOTSTRAP_PROMPT_KEY]; } + if (input.mode === "external") delete next[STOCK_HASH_KEY]; return next; } @@ -439,6 +531,7 @@ export function syncInstructionsBundleConfigFromFilePath( delete next[MODE_KEY]; delete next[ROOT_KEY]; delete next[ENTRY_KEY]; + delete next[STOCK_HASH_KEY]; return next; } const resolvedPath = resolveLegacyInstructionsPath(instructionsFilePath, adapterConfig); @@ -682,44 +775,181 @@ export function agentInstructionsService() { }; } + async function materializeManagedBundleInternal( + agent: AgentLike, + files: Record, + options?: { + clearLegacyPromptTemplate?: boolean; + entryFile?: string; + recordedStockHash?: string | null; + forceReset?: boolean; + }, + ): Promise<{ + bundle: AgentInstructionsBundle; + adapterConfig: Record; + materialization: ManagedBundleMaterialization; + }> { + const rootPath = resolveManagedInstructionsRoot(agent); + const entryFile = options?.entryFile ? normalizeRelativeFilePath(options.entryFile) : ENTRY_FILE_DEFAULT; + await fs.mkdir(path.dirname(rootPath), { recursive: true }); + + const prospectiveFiles = normalizeMaterializedFiles(files, entryFile); + const stockHash = managedInstructionsTreeHash(prospectiveFiles); + const configuredStockHash = asString(asRecord(agent.adapterConfig)[STOCK_HASH_KEY]); + const recordedStockHash = options?.recordedStockHash ?? configuredStockHash; + const initialFiles = await readInstructionsTree(rootPath); + const initialHash = initialFiles === null ? null : managedInstructionsTreeHash(initialFiles); + const initialStatus = resourceStatus({ + resourceId: initialFiles === null ? null : agent.id, + currentHash: initialHash, + bindingStockHash: recordedStockHash, + latestStockHash: stockHash, + }); + const configuredEntryFile = deriveBundleState(agent).entryFile; + const preservedEntryFile = (currentFiles: Record) => { + if (Object.prototype.hasOwnProperty.call(currentFiles, entryFile)) return entryFile; + if (Object.prototype.hasOwnProperty.call(currentFiles, configuredEntryFile)) return configuredEntryFile; + if (Object.prototype.hasOwnProperty.call(currentFiles, ENTRY_FILE_DEFAULT)) return ENTRY_FILE_DEFAULT; + return Object.keys(currentFiles).sort(compareInstructionPaths)[0] ?? entryFile; + }; + const configFor = (advanceStockHash: boolean, currentFiles: Record) => { + const next = applyBundleConfig(asRecord(agent.adapterConfig), { + mode: "managed", + rootPath, + entryFile: advanceStockHash ? entryFile : preservedEntryFile(currentFiles), + clearLegacyPromptTemplate: options?.clearLegacyPromptTemplate, + }); + if (advanceStockHash) next[STOCK_HASH_KEY] = stockHash; + return next; + }; + + const finish = async ( + action: ManagedBundleMaterialization["action"], + stockStatus: ManagedResourceStockStatus, + currentFiles: Record, + currentHash: string | null, + input: { advanceStockHash: boolean; backupPath?: string | null }, + ) => { + const changes = changedInstructionFiles(currentFiles, prospectiveFiles); + const backupPath = input.backupPath ?? null; + const adapterConfig = configFor(input.advanceStockHash, currentFiles); + const bundle = await getBundle({ ...agent, adapterConfig }); + const changedPaths = [...changes.added, ...changes.removed, ...changes.updated] + .sort(compareInstructionPaths); + return { + bundle, + adapterConfig, + materialization: { + stockStatus, + action, + stockHash, + previousHash: currentHash, + recordedStockHash, + ...changes, + skipped: action === "skipped" ? changedPaths : [], + backedUp: backupPath ? Object.keys(currentFiles).sort(compareInstructionPaths) : [], + backupPath, + }, + }; + }; + + if (!options?.forceReset && initialStatus === "operator_modified") { + return finish("skipped", initialStatus, initialFiles ?? {}, initialHash, { advanceStockHash: false }); + } + if (!options?.forceReset && initialStatus === "stock_current") { + return finish("unchanged", initialStatus, initialFiles ?? {}, initialHash, { advanceStockHash: true }); + } + + const stagingRoot = await fs.mkdtemp(path.join(path.dirname(rootPath), ".instructions-materialize-")); + const stagedPath = path.join(stagingRoot, "staged"); + await fs.mkdir(stagedPath, { recursive: true }); + try { + await writeBundleFiles(stagedPath, prospectiveFiles, { overwriteExisting: true }); + const stagedFiles = await readInstructionsTree(stagedPath); + if (!stagedFiles || managedInstructionsTreeHash(stagedFiles) !== stockHash) { + throw new Error("Staged instructions bundle hash did not match the prospective stock hash"); + } + + const recheckedFiles = await readInstructionsTree(rootPath); + const recheckedHash = recheckedFiles === null ? null : managedInstructionsTreeHash(recheckedFiles); + const recheckedStatus = resourceStatus({ + resourceId: recheckedFiles === null ? null : agent.id, + currentHash: recheckedHash, + bindingStockHash: recordedStockHash, + latestStockHash: stockHash, + }); + if (!options?.forceReset && recheckedStatus === "operator_modified") { + return finish("skipped", recheckedStatus, recheckedFiles ?? {}, recheckedHash, { advanceStockHash: false }); + } + if (!options?.forceReset && recheckedStatus === "stock_current") { + return finish("unchanged", recheckedStatus, recheckedFiles ?? {}, recheckedHash, { advanceStockHash: true }); + } + + const backupPath = options?.forceReset && recheckedFiles !== null + ? await nextInstructionsBackupPath(rootPath) + : null; + const displacedPath = recheckedFiles === null + ? null + : backupPath ?? path.join(stagingRoot, "previous"); + if (displacedPath) { + await fs.rename(rootPath, displacedPath); + if (!options?.forceReset) { + const displacedFiles = await readInstructionsTree(displacedPath); + const displacedHash = displacedFiles === null ? null : managedInstructionsTreeHash(displacedFiles); + if (displacedHash !== recheckedHash) { + await fs.rename(displacedPath, rootPath); + return finish("skipped", "operator_modified", displacedFiles ?? {}, displacedHash, { + advanceStockHash: false, + }); + } + } + } + + try { + await fs.rename(stagedPath, rootPath); + } catch (error) { + if (displacedPath && !(await statIfExists(rootPath))) { + await fs.rename(displacedPath, rootPath).catch(() => undefined); + } + throw error; + } + + const action: ManagedBundleMaterialization["action"] = options?.forceReset + ? "reset" + : recheckedStatus === "missing" + ? "added" + : "updated"; + return finish(action, recheckedStatus, recheckedFiles ?? {}, recheckedHash, { + advanceStockHash: true, + backupPath, + }); + } finally { + await fs.rm(stagingRoot, { recursive: true, force: true }); + } + } + async function materializeManagedBundle( agent: AgentLike, files: Record, options?: { clearLegacyPromptTemplate?: boolean; - replaceExisting?: boolean; entryFile?: string; + recordedStockHash?: string | null; }, - ): Promise<{ bundle: AgentInstructionsBundle; adapterConfig: Record }> { - const rootPath = resolveManagedInstructionsRoot(agent); - const entryFile = options?.entryFile ? normalizeRelativeFilePath(options.entryFile) : ENTRY_FILE_DEFAULT; + ) { + return materializeManagedBundleInternal(agent, files, options); + } - if (options?.replaceExisting) { - await fs.rm(rootPath, { recursive: true, force: true }); - } - await fs.mkdir(rootPath, { recursive: true }); - - const normalizedEntries = Object.entries(files).map(([relativePath, content]) => [ - normalizeRelativeFilePath(relativePath), - content, - ] as const); - for (const [relativePath, content] of normalizedEntries) { - const absolutePath = resolvePathWithinRoot(rootPath, relativePath); - await fs.mkdir(path.dirname(absolutePath), { recursive: true }); - await fs.writeFile(absolutePath, content, "utf8"); - } - if (!normalizedEntries.some(([relativePath]) => relativePath === entryFile)) { - await fs.writeFile(resolvePathWithinRoot(rootPath, entryFile), "", "utf8"); - } - - const adapterConfig = applyBundleConfig(asRecord(agent.adapterConfig), { - mode: "managed", - rootPath, - entryFile, - clearLegacyPromptTemplate: options?.clearLegacyPromptTemplate, - }); - const bundle = await getBundle({ ...agent, adapterConfig }); - return { bundle, adapterConfig }; + async function forceResetManagedBundle( + agent: AgentLike, + files: Record, + options?: { + clearLegacyPromptTemplate?: boolean; + entryFile?: string; + recordedStockHash?: string | null; + }, + ) { + return materializeManagedBundleInternal(agent, files, { ...options, forceReset: true }); } return { @@ -731,5 +961,6 @@ export function agentInstructionsService() { exportFiles, ensureManagedBundle: ensureWritableBundle, materializeManagedBundle, + forceResetManagedBundle, }; } diff --git a/server/src/services/built-in-agents.ts b/server/src/services/built-in-agents.ts index b8a25460c72..902468d95ac 100644 --- a/server/src/services/built-in-agents.ts +++ b/server/src/services/built-in-agents.ts @@ -10,7 +10,7 @@ import { syncRoutineVariablesWithTemplate } from "@paperclipai/shared"; import type { Agent, Approval, CompanySkill, PermissionKey, Routine, RoutineTrigger, RoutineVariable } from "@paperclipai/shared"; import { conflict, HttpError, notFound, unprocessable } from "../errors.js"; import { logActivity } from "./activity-log.js"; -import { agentInstructionsService } from "./agent-instructions.js"; +import { agentInstructionsService, managedInstructionsTreeHash } from "./agent-instructions.js"; import { agentService } from "./agents.js"; import { approvalService } from "./approvals.js"; import { @@ -936,11 +936,17 @@ export function builtInAgentService(db: Db) { async function currentInstructionFiles(agent: Agent, bundle: BuiltInAgentBundleDefinition) { const currentFiles: Record = {}; - for (const filePath of Object.keys(bundle.instructions.files)) { + const currentBundle = await instructionsSvc.getBundle(agent); + const diskFiles = currentBundle.files.filter((file) => !file.virtual); + if (diskFiles.length === 0) { + for (const filePath of Object.keys(bundle.instructions.files)) currentFiles[filePath] = null; + return currentFiles; + } + for (const file of diskFiles) { try { - currentFiles[filePath] = (await instructionsSvc.readFile(agent, filePath)).content; + currentFiles[file.path] = (await instructionsSvc.readFile(agent, file.path)).content; } catch { - currentFiles[filePath] = null; + currentFiles[file.path] = null; } } return currentFiles; @@ -948,10 +954,12 @@ export function builtInAgentService(db: Db) { async function materializeInstructions(agent: Agent, definition: BuiltInAgentDefinition, mode: "reconcile" | "reset") { const bundle = definition.bundle!; - const stock = stockHash(bundle.instructions.files); + const stock = managedInstructionsTreeHash(bundle.instructions.files); const binding = await getManagedResourceBinding(agent.companyId, definition.key, "instructions", "AGENTS.md"); const currentFiles = await currentInstructionFiles(agent, bundle); - const currentHash = Object.values(currentFiles).some((value) => value === null) ? null : stockHash(currentFiles); + const currentHash = Object.values(currentFiles).some((value) => value === null) + ? null + : managedInstructionsTreeHash(currentFiles as Record); const currentState = stockState({ resourceKind: "instructions", resourceKey: "AGENTS.md", @@ -963,54 +971,55 @@ export function builtInAgentService(db: Db) { changedFiles: changedFileList(currentFiles, bundle.instructions.files), }); - const shouldWrite = - mode === "reset" - || currentState.stockStatus === "missing" - || currentState.stockStatus === "stock_update_available"; - if (!shouldWrite) { - if (!binding && currentHash === stock) { - await upsertManagedResourceBinding({ - companyId: agent.companyId, - bundleKey: definition.key, - resourceKind: "instructions", - resourceKey: "AGENTS.md", - resourceId: agent.id, - stockVersion: bundle.stockVersion, - stockHash: stock, - defaultsJson: { - entryFile: bundle.instructions.entryFile, - files: Object.keys(bundle.instructions.files), - }, - }); - } - return currentState; - } - - const materialized = await instructionsSvc.materializeManagedBundle(agent, bundle.instructions.files, { + const materializeOptions = { entryFile: bundle.instructions.entryFile, - replaceExisting: true, clearLegacyPromptTemplate: true, - }); - const updated = await agentSvc.update(agent.id, { - adapterConfig: materialized.adapterConfig, - }, { - allowBuiltInAgentMetadata: true, - recordRevision: { source: `built-in-bundle:${mode}:instructions` }, - }); - if (!updated) throw notFound("Built-in agent not found"); - await upsertManagedResourceBinding({ - companyId: agent.companyId, - bundleKey: definition.key, - resourceKind: "instructions", - resourceKey: "AGENTS.md", - resourceId: agent.id, - stockVersion: bundle.stockVersion, - stockHash: stock, - defaultsJson: { - entryFile: bundle.instructions.entryFile, - files: Object.keys(bundle.instructions.files), - }, - }); + recordedStockHash: binding?.stockHash ?? null, + }; + const materialized = mode === "reset" + ? await instructionsSvc.forceResetManagedBundle(agent, bundle.instructions.files, materializeOptions) + : await instructionsSvc.materializeManagedBundle(agent, bundle.instructions.files, materializeOptions); + if (stableJson(materialized.adapterConfig) !== stableJson(agent.adapterConfig)) { + const updated = await agentSvc.update(agent.id, { + adapterConfig: materialized.adapterConfig, + }, { + allowBuiltInAgentMetadata: true, + recordRevision: { source: `built-in-bundle:${mode}:instructions` }, + }); + if (!updated) throw notFound("Built-in agent not found"); + } + const stockControlled = materialized.materialization.action !== "skipped"; + if (stockControlled) { + await upsertManagedResourceBinding({ + companyId: agent.companyId, + bundleKey: definition.key, + resourceKind: "instructions", + resourceKey: "AGENTS.md", + resourceId: agent.id, + stockVersion: bundle.stockVersion, + stockHash: stock, + defaultsJson: { + entryFile: bundle.instructions.entryFile, + files: Object.keys(bundle.instructions.files), + }, + }); + } + if (materialized.materialization.action !== "unchanged") { + await logActivity(db, { + companyId: agent.companyId, + actorType: "system", + actorId: "built-in-agents", + action: "built_in_agent.instructions_materialized", + entityType: "agent", + entityId: agent.id, + details: { + key: definition.key, + mode, + ...materialized.materialization, + }, + }); + } + if (!stockControlled) return currentState; return stockState({ resourceKind: "instructions", resourceKey: "AGENTS.md", @@ -1423,7 +1432,7 @@ export function builtInAgentService(db: Db) { const { routine, triggers } = await getRoutineByBinding(companyId, definition); const proposal = await pendingUpdateProposal(agent); const instructionHash = Object.values(instructionFiles).every((value) => value !== null) - ? stockHash(instructionFiles) + ? managedInstructionsTreeHash(instructionFiles as Record) : null; const skillHash = skill && Object.values(skillFiles).every((value) => value !== null) ? stockHash(skillFiles) @@ -1442,7 +1451,7 @@ export function builtInAgentService(db: Db) { resourceKey: "AGENTS.md", resourceId: agent.id, stockVersion: bundle.stockVersion, - latestStockHash: stockHash(bundle.instructions.files), + latestStockHash: managedInstructionsTreeHash(bundle.instructions.files), currentHash: instructionHash, bindingStockHash: instructionBinding?.stockHash ?? null, changedFiles: changedFileList(instructionFiles, bundle.instructions.files), diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index cdf4cb56303..01d1b5ab92f 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -2154,6 +2154,7 @@ function normalizePortableConfig( key === "instructionsBundleMode" || key === "instructionsRootPath" || key === "instructionsEntryFile" || + key === "instructionsStockHash" || key === "promptTemplate" || key === "bootstrapPromptTemplate" || // deprecated — kept for backward compat key === "paperclipSkillSync" @@ -3510,6 +3511,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { delete nextAdapterConfig.instructionsBundleMode; delete nextAdapterConfig.instructionsRootPath; delete nextAdapterConfig.instructionsEntryFile; + delete nextAdapterConfig.instructionsStockHash; const normalizedAdapterConfig = await secrets.normalizeAdapterConfigForPersistence( companyId, nextAdapterConfig, @@ -5303,6 +5305,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const preImportExistingAgentIds = new Set(); const agentStatusById = new Map(); const existingAgents = await agents.list(targetCompany.id); + const existingAgentById = new Map(existingAgents.map((agent) => [agent.id, agent])); for (const existing of existingAgents) { const slug = normalizeAgentUrlKey(existing.name) ?? existing.id; existingSlugToAgentId.set(slug, existing.id); @@ -5418,8 +5421,18 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { : {}; if (planAgent.action === "update" && planAgent.existingAgentId) { + const existingTarget = existingAgentById.get(planAgent.existingAgentId); + const existingTargetConfig = isPlainRecord(existingTarget?.adapterConfig) + ? existingTarget.adapterConfig + : {}; + const targetInstructionsStockHash = asString(existingTargetConfig.instructionsStockHash); + const updateAdapterConfig = { ...patch.adapterConfig }; + if (targetInstructionsStockHash) { + updateAdapterConfig.instructionsStockHash = targetInstructionsStockHash; + } let updated = await agents.update(planAgent.existingAgentId, { ...patch, + adapterConfig: updateAdapterConfig, ...automationPausePatch, }); if (!updated) { @@ -5436,8 +5449,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { try { const materialized = await instructions.materializeManagedBundle(updated, bundleFiles, { clearLegacyPromptTemplate: true, - replaceExisting: true, }); + if (materialized.materialization.action === "skipped") { + warnings.push(`Preserved operator-modified instructions bundle for ${manifestAgent.slug}; imported stock files were skipped.`); + } updated = await agents.update(updated.id, { adapterConfig: materialized.adapterConfig }) ?? updated; } catch (err) { warnings.push(`Failed to materialize instructions bundle for ${manifestAgent.slug}: ${err instanceof Error ? err.message : String(err)}`); @@ -5483,8 +5498,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { try { const materialized = await instructions.materializeManagedBundle(created, bundleFiles, { clearLegacyPromptTemplate: true, - replaceExisting: true, }); + if (materialized.materialization.action === "skipped") { + warnings.push(`Preserved operator-modified instructions bundle for ${manifestAgent.slug}; imported stock files were skipped.`); + } created = await agents.update(created.id, { adapterConfig: materialized.adapterConfig }) ?? created; } catch (err) { warnings.push(`Failed to materialize instructions bundle for ${manifestAgent.slug}: ${err instanceof Error ? err.message : String(err)}`); diff --git a/server/src/services/managed-resource-drift.ts b/server/src/services/managed-resource-drift.ts index addce116bc5..908d4d81264 100644 --- a/server/src/services/managed-resource-drift.ts +++ b/server/src/services/managed-resource-drift.ts @@ -1,5 +1,4 @@ import { createHash } from "node:crypto"; - export type ManagedResourceStockStatus = | "missing" | "stock_current" @@ -20,7 +19,6 @@ export function stableJson(value: unknown): string { export function stockHash(value: unknown) { return `sha256:${createHash("sha256").update(stableJson(value)).digest("hex")}`; } - export function resourceStatus(input: { resourceId: string | null; currentHash: string | null; diff --git a/server/src/services/plugin-managed-agents.ts b/server/src/services/plugin-managed-agents.ts index ea95b59ce8c..2b77c40997d 100644 --- a/server/src/services/plugin-managed-agents.ts +++ b/server/src/services/plugin-managed-agents.ts @@ -326,21 +326,19 @@ export function pluginManagedAgentService( companyId: string, agent: Agent, declaration: PluginManagedAgentDeclaration, - materializeOptions: { replaceExisting: boolean }, + materializeOptions: { forceReset?: boolean } = {}, ): Promise { const variables = await optionsForInstructionVariables(companyId); const declared = declaredInstructionFiles(declaration, variables); if (!declared) return agent; - const materialized = await instructions.materializeManagedBundle( - agent, - declared.files, - { - entryFile: declared.entryFile, - replaceExisting: materializeOptions.replaceExisting, - clearLegacyPromptTemplate: true, - }, - ); + const instructionOptions = { + entryFile: declared.entryFile, + clearLegacyPromptTemplate: true, + }; + const materialized = materializeOptions.forceReset + ? await instructions.forceResetManagedBundle(agent, declared.files, instructionOptions) + : await instructions.materializeManagedBundle(agent, declared.files, instructionOptions); const updated = await agentSvc.update(agent.id, { adapterConfig: materialized.adapterConfig, }, { @@ -348,6 +346,19 @@ export function pluginManagedAgentService( source: `plugin:${optionsForRevisionSource()}:managed-agent-instructions`, }, }); + await logActivity(db, { + companyId, + actorType: "plugin", + actorId: options.pluginId, + action: "plugin.managed_agent.instructions_materialized", + entityType: "agent", + entityId: agent.id, + details: { + sourcePluginKey: options.pluginKey, + managedResourceKey: declaration.agentKey, + ...materialized.materialization, + }, + }); return (updated as Agent | null) ?? { ...agent, adapterConfig: materialized.adapterConfig }; } @@ -423,7 +434,7 @@ export function pluginManagedAgentService( spentMonthlyCents: 0, lastHeartbeatAt: null, }) as Agent; - created = await materializeDeclaredInstructions(companyId, created, declaration, { replaceExisting: true }); + created = await materializeDeclaredInstructions(companyId, created, declaration); let approvalId: string | null = null; if (requiresApproval) { @@ -537,7 +548,7 @@ export function pluginManagedAgentService( }, }); if (!updated) throw notFound("Managed agent not found"); - const updatedAgent = await materializeDeclaredInstructions(companyId, updated as Agent, declaration, { replaceExisting: true }); + const updatedAgent = await materializeDeclaredInstructions(companyId, updated as Agent, declaration, { forceReset: true }); await upsertBinding(companyId, declaration, updatedAgent.id, {}, adapterType); await logActivity(db, { companyId,