diff --git a/doc/plugins/DISTRIBUTION-PLUGINS.md b/doc/plugins/DISTRIBUTION-PLUGINS.md index 4e58c36326c..6ce76d248c1 100644 --- a/doc/plugins/DISTRIBUTION-PLUGINS.md +++ b/doc/plugins/DISTRIBUTION-PLUGINS.md @@ -48,6 +48,41 @@ Managed instances select a distribution key through the existing compatibility, worker and health mechanisms apply. A worker or install failure is recorded as a plugin error without taking down the application. +A managed instance can also give a selected plugin process variables through +`plugins.workerEnv`. The document holds variable names, not values. The deployment +sets the values in the server's process environment: + +```json +{ "plugins": { + "autoInstall": ["example-extension"], + "workerEnv": { "example-extension": ["EXAMPLE_ORIGIN", "EXAMPLE_CAPABILITY"] } +} } +``` + +Each key must be in `autoInstall`. Startup fails if a name looks like a host +credential, such as one with `KEY`, `TOKEN`, `SECRET`, `PASSWORD`, `DATABASE` or +`JWT` as a word, or if it controls how the worker loads code, such as +`NODE_OPTIONS`. Give a granted variable a name for its own use, not a reused host +variable. Only the plugin with that catalog key receives the variables. A managed +instance runs a selected plugin only from the current image, so an install from +another source does not start. + +A grant does not give access to any company. To let a plugin serve a company with +no saved configuration, name the company in `plugins.enabledCompanies`: + +```json +{ "plugins": { + "autoInstall": ["example-extension"], + "enabledCompanies": { "example-extension": [""] } +} } +``` + +The plugin then receives resource lifecycle events for that company and can use +project instance secrets there. A company can still disable the plugin. This does +not give access for background worker calls. Those still need a saved company +configuration. Core reads the list from the document on each use and does not save +it, so a database restore cannot bring back access that the document removed. + The catalog alone does not auto-enable plugins on self-hosted instances. Operators can explicitly install catalog entries through the normal plugin CLI. The package's manifest ID and version must match the catalog. diff --git a/server/src/__tests__/managed-config.test.ts b/server/src/__tests__/managed-config.test.ts index 624947234e7..226ef9bf401 100644 --- a/server/src/__tests__/managed-config.test.ts +++ b/server/src/__tests__/managed-config.test.ts @@ -21,6 +21,63 @@ function validDoc(overrides: Record = {}) { }); } +describe("plugins.workerEnv", () => { + const plugins = (workerEnv: unknown) => ({ plugins: { autoInstall: ["daytona"], workerEnv } }); + + it("is absent from documents that do not declare it", () => { + expect(parseManagedConfigEnv(envWith(validDoc()))?.plugins).toEqual({ autoInstall: ["daytona", "kubernetes"] }); + }); + + it("accepts variable names for an auto-installed plugin", () => { + const parsed = parseManagedConfigEnv(envWith(validDoc(plugins({ daytona: ["PROVIDER_ORIGIN", "PROVIDER_CAPABILITY_2"] })))); + expect(parsed?.plugins.workerEnv).toEqual({ daytona: ["PROVIDER_ORIGIN", "PROVIDER_CAPABILITY_2"] }); + }); + + it.each([ + "ANTHROPIC_API_KEY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "BETTER_AUTH_SECRET", "DATABASE_MIGRATION_URL", "DATABASE_URL", "DB_PASSWORD", + "GH_TOKEN", "GITHUB_TOKEN", "LD_PRELOAD", "NODE_OPTIONS", "PAPERCLIP_AGENT_JWT_SECRET", "PAPERCLIP_AUTH_SECRET", "PAPERCLIP_BACKUP_WORKER_DATABASE_URL", + "PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_DECISION_SIGNING_SECRET", "PAPERCLIP_MANAGED_CONFIG", + "PAPERCLIP_SECRETS_MASTER_KEY", "PAPERCLIP_SECRETS_MASTER_KEY_FILE", "PAPERCLIP_TEST_POSTGRES_ADMIN_URL", "PATH", "SENTRY_DSN", + ])("refuses to start with a grant of %s", (name) => { + expect(() => parseManagedConfigEnv(envWith(validDoc(plugins({ daytona: [name] }))))).toThrow(`must not grant host credential or loader variable "${name}"`); + }); + + it.each(["PAPERCLIP_CLOUD_API_ORIGIN", "PAPERCLIP_PRIMARY_HOST", "PROVIDER_CAPABILITY", "MONKEY_ORIGIN"])("accepts a grant of %s", (name) => { + expect(parseManagedConfigEnv(envWith(validDoc(plugins({ daytona: [name] }))))?.plugins.workerEnv).toEqual({ daytona: [name] }); + }); + + it.each([ + ["a plugin that is not auto-installed", { other: ["PROVIDER_ORIGIN"] }, /not in "plugins.autoInstall"/], + ["a non-object section", ["PROVIDER_ORIGIN"], /must be an object/], + ["an empty grant", { daytona: [] }, /non-empty array/], + ["a value instead of a name", { daytona: ["https://example.test"] }, /uppercase variable names/], + ["a lowercase name", { daytona: ["database_url"] }, /uppercase variable names/], + ["a duplicate name", { daytona: ["PROVIDER_ORIGIN", "PROVIDER_ORIGIN"] }, /duplicate entry/], + ])("refuses to start with %s", (_label, workerEnv, message) => { + expect(() => parseManagedConfigEnv(envWith(validDoc(plugins(workerEnv))))).toThrow(message); + }); +}); + +describe("plugins.enabledCompanies", () => { + const companyId = "f0679247-e2be-5a76-adb6-6bd360e5a833"; + const plugins = (enabledCompanies: unknown) => ({ plugins: { autoInstall: ["daytona"], enabledCompanies } }); + + it("accepts company ids for an auto-installed plugin", () => { + expect(parseManagedConfigEnv(envWith(validDoc(plugins({ daytona: [companyId] }))))?.plugins.enabledCompanies).toEqual({ daytona: [companyId] }); + }); + + it.each([ + ["a plugin that is not auto-installed", { other: [companyId] }, /not in "plugins.autoInstall"/], + ["a non-object section", [companyId], /must be an object/], + ["an empty list", { daytona: [] }, /non-empty array/], + ["a value that is not a UUID", { daytona: ["all"] }, /lowercase company UUIDs/], + ["an uppercase UUID", { daytona: [companyId.toUpperCase()] }, /lowercase company UUIDs/], + ["a duplicate id", { daytona: [companyId, companyId] }, /duplicate entry/], + ])("refuses to start with %s", (_label, enabledCompanies, message) => { + expect(() => parseManagedConfigEnv(envWith(validDoc(plugins(enabledCompanies))))).toThrow(message); + }); +}); + describe("managedFeatureKeySet", () => { it("contains exactly the boolean flags of the experimental schema", () => { const keys = managedFeatureKeySet(); diff --git a/server/src/__tests__/managed-plugin-worker-env.test.ts b/server/src/__tests__/managed-plugin-worker-env.test.ts new file mode 100644 index 00000000000..795a18e6579 --- /dev/null +++ b/server/src/__tests__/managed-plugin-worker-env.test.ts @@ -0,0 +1,66 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { BUNDLED_CATALOG_ROOT_ENV_VAR } from "../services/bundled-plugins.js"; +import { distributionBundleDigest } from "../services/distribution-plugin-catalog.js"; +import { MANAGED_CONFIG_ENV_KEY } from "../services/managed-config.js"; +import { managedPluginEnabledCompanies, managedPluginWorkerEnvGrant } from "../services/managed-plugin-worker-env.js"; + +describe("managed plugin grants", () => { + const pluginKey = "paperclip.daytona-sandbox-provider"; + const companyId = "f0679247-e2be-5a76-adb6-6bd360e5a833"; + let root: string; + const env = (plugins: Record) => ({ + [BUNDLED_CATALOG_ROOT_ENV_VAR]: root, + [MANAGED_CONFIG_ENV_KEY]: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "1", features: {}, plugins }), + }); + async function distributionPlugin(key: string, id: string) { + const directory = path.join(root, "distribution", key); + await mkdir(path.join(directory, "dist"), { recursive: true }); + await writeFile(path.join(directory, "package.json"), JSON.stringify({ version: "1.0.0", paperclipPlugin: { manifest: "./dist/manifest.js", worker: "./dist/worker.js" } })); + await writeFile(path.join(directory, "dist/manifest.js"), "export default {};"); + await writeFile(path.join(directory, "dist/worker.js"), "export {};"); + await writeFile(path.join(root, "distribution/catalog.json"), JSON.stringify({ schemaVersion: 1, plugins: [{ + key, pluginKey: id, version: "1.0.0", directory: key, digest: distributionBundleDigest(directory), + }] })); + } + + beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "managed-worker-env-")); + await mkdir(path.join(root, "sandbox-providers/daytona"), { recursive: true }); + }); + afterEach(async () => { await rm(root, { recursive: true, force: true }); }); + + it("reads each section separately", () => { + const granted = env({ autoInstall: ["daytona"], workerEnv: { daytona: ["PROVIDER_ORIGIN"] } }); + expect(managedPluginWorkerEnvGrant(pluginKey, granted)).toEqual(["PROVIDER_ORIGIN"]); + expect(managedPluginEnabledCompanies(pluginKey, granted)).toEqual([]); + const enabled = env({ autoInstall: ["daytona"], enabledCompanies: { daytona: [companyId] } }); + expect(managedPluginEnabledCompanies(pluginKey, enabled)).toEqual([companyId]); + expect(managedPluginWorkerEnvGrant(pluginKey, enabled)).toEqual([]); + }); + + it("gives nothing on a self-hosted instance or to another plugin", () => { + expect(managedPluginWorkerEnvGrant(pluginKey, { [BUNDLED_CATALOG_ROOT_ENV_VAR]: root })).toEqual([]); + const granted = env({ autoInstall: ["daytona"], workerEnv: { daytona: ["PROVIDER_ORIGIN"] }, enabledCompanies: { daytona: [companyId] } }); + expect(managedPluginWorkerEnvGrant("example.other", granted)).toEqual([]); + expect(managedPluginEnabledCompanies("example.other", granted)).toEqual([]); + }); + + it("gives variables to a digest-verified distribution plugin", async () => { + await distributionPlugin("example-extension", "example.extension"); + const granted = env({ autoInstall: ["example-extension"], workerEnv: { "example-extension": ["PROVIDER_ORIGIN"] } }); + expect(managedPluginWorkerEnvGrant("example.extension", granted)).toEqual(["PROVIDER_ORIGIN"]); + }); + + it("does not read inherited properties for a catalog key named after one", async () => { + await distributionPlugin("constructor", "example.extension"); + const granted = env({ autoInstall: ["constructor"], workerEnv: { constructor: ["PROVIDER_ORIGIN"] }, enabledCompanies: {} }); + expect(managedPluginWorkerEnvGrant("example.extension", granted)).toEqual(["PROVIDER_ORIGIN"]); + expect(managedPluginEnabledCompanies("example.extension", granted)).toEqual([]); + const enabled = env({ autoInstall: ["constructor"], workerEnv: {}, enabledCompanies: { constructor: [companyId] } }); + expect(managedPluginEnabledCompanies("example.extension", enabled)).toEqual([companyId]); + expect(managedPluginWorkerEnvGrant("example.extension", enabled)).toEqual([]); + }); +}); diff --git a/server/src/__tests__/plugin-database.test.ts b/server/src/__tests__/plugin-database.test.ts index f844d1fe456..0cb31569788 100644 --- a/server/src/__tests__/plugin-database.test.ts +++ b/server/src/__tests__/plugin-database.test.ts @@ -148,6 +148,20 @@ describe("buildPluginWorkerEnv", () => { deploymentExposure: "public", }; + it("forwards only granted variables to a plugin that registers no environment driver", () => { + const env = buildPluginWorkerEnv({ + manifest: { capabilities: [] }, + instanceInfo, + grantedEnvVars: ["PROVIDER_ORIGIN", "PROVIDER_BLANK", "PROVIDER_MISSING"], + processEnv: { PROVIDER_ORIGIN: "https://provider.test", PROVIDER_BLANK: " ", DATABASE_URL: "postgres://host", OPENAI_API_KEY: "openai-token" }, + }); + expect(env).toEqual({ + PAPERCLIP_DEPLOYMENT_MODE: "authenticated", + PAPERCLIP_DEPLOYMENT_EXPOSURE: "public", + PROVIDER_ORIGIN: "https://provider.test", + }); + }); + it("passes only model provider keys through to environment driver plugins", () => { const env = buildPluginWorkerEnv({ manifest: { capabilities: ["environment.drivers.register"] }, diff --git a/server/src/__tests__/plugin-lifecycle-delivery.test.ts b/server/src/__tests__/plugin-lifecycle-delivery.test.ts index 8e5e5b70bef..e98797b0188 100644 --- a/server/src/__tests__/plugin-lifecycle-delivery.test.ts +++ b/server/src/__tests__/plugin-lifecycle-delivery.test.ts @@ -1,7 +1,10 @@ import { randomBytes, randomUUID } from "node:crypto"; import { EventEmitter } from "node:events"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { eq } from "drizzle-orm"; -import { companies, createDb, companySecrets, projects, pluginConfig, plugins, resourceLifecycleEvents, type Db } from "@paperclipai/db"; +import { companies, createDb, companySecrets, projects, pluginCompanySettings, pluginConfig, plugins, resourceLifecycleEvents, type Db } from "@paperclipai/db"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { logger } from "../middleware/logger.js"; @@ -54,6 +57,76 @@ describePostgres("durable plugin lifecycle delivery", () => { const worker = registerPluginLifecycleDelivery(coordinator as never, db, { call, isRunning: running, onConfigApplied } as unknown as PluginWorkerManager, lifecycle as unknown as PluginLifecycleManager); return { call, running, wake, lifecycle, worker, applied: () => { for (const listener of configApplied) listener(); }, run: () => task.run(new AbortController().signal), pending: () => task.hasPending(), nextRunAt: () => task.nextRunAt?.(), settled: () => task.onSettled?.() }; } + async function withManagedPlugin(enabledCompanies: string[], run: () => Promise) { + const root = await mkdtemp(path.join(tmpdir(), "delivery-managed-")); + await mkdir(path.join(root, "sandbox-providers/daytona"), { recursive: true }); + vi.stubEnv("PAPERCLIP_BUNDLED_PLUGIN_ROOT", root); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "1", features: {}, + plugins: { autoInstall: ["daytona"], workerEnv: { daytona: ["PROVIDER_ORIGIN"] }, enabledCompanies: { daytona: enabledCompanies } } })); + try { await run(); } + finally { vi.unstubAllEnvs(); await rm(root, { recursive: true, force: true }); } + } + it("delivers to a managed plugin with no config row and a stale saved path only in the companies the document enables", async () => { + const unlistedCompanyId = randomUUID(), disabledCompanyId = randomUUID(), absentCompanyId = randomUUID(); + await withManagedPlugin([companyId, disabledCompanyId, absentCompanyId], async () => { + const managedId = randomUUID(), ungrantedId = randomUUID(); + const manifestJson = { resourceLifecycle: ["project"], capabilities: ["events.subscribe"] } as never; + await db.insert(companies).values([ + { id: unlistedCompanyId, name: "Unlisted company", issuePrefix: `O${unlistedCompanyId.slice(0, 6)}` }, + { id: disabledCompanyId, name: "Disabled company", issuePrefix: `D${disabledCompanyId.slice(0, 6)}` }, + ]); + await db.insert(plugins).values([ + { id: managedId, pluginKey: "paperclip.daytona-sandbox-provider", packageName: "managed", version: "1.0.0", status: "ready", packagePath: "/old/image/plugins/daytona", manifestJson }, + { id: ungrantedId, pluginKey: ungrantedId, packageName: "ungranted", version: "1.0.0", status: "ready", manifestJson }, + ]); + await db.insert(pluginCompanySettings).values({ pluginId: managedId, companyId: disabledCompanyId, enabled: false }); + const resourceId = await event(); + await db.transaction(async tx => { + await recordResourceCreationEvent(tx as unknown as Db, unlistedCompanyId, "project", randomUUID()); + await recordResourceCreationEvent(tx as unknown as Db, disabledCompanyId, "project", randomUUID()); + }); + const d = delivery(); + await d.run(); + const delivered = (id: string) => d.call.mock.calls.filter(([plugin]) => plugin === id).map(([, , input]) => input.resourceId); + expect(delivered(managedId)).toEqual([resourceId]); + expect(delivered(ungrantedId)).toEqual([]); + expect(delivered(pluginId)).toEqual([resourceId]); + d.worker.dispose(); + }); + }); + it("listens for a managed plugin with no config row until its last enabled company disables it", async () => { + await withManagedPlugin([companyId], async () => { + await db.delete(plugins); + const managedId = randomUUID(); + await db.insert(plugins).values({ id: managedId, pluginKey: "paperclip.daytona-sandbox-provider", packageName: "managed", version: "1.0.0", status: "ready", + manifestJson: { resourceLifecycle: ["agent_runtime"], capabilities: ["events.subscribe"] } as never }); + const subscription = { unlisten: vi.fn(), state: {} }; + vi.mocked(db.$client.listen).mockResolvedValueOnce(subscription as never); + const d = delivery(); + await d.worker.ready; + expect(db.$client.listen).toHaveBeenCalledWith("paperclip_plugin_lifecycle", expect.any(Function), expect.any(Function)); + await db.insert(pluginCompanySettings).values({ pluginId: managedId, companyId, enabled: false }); + d.applied(); + await vi.waitFor(() => expect(subscription.unlisten).toHaveBeenCalledTimes(1)); + d.worker.dispose(); + }); + }); + it("does not listen for a managed plugin that the document enables for no company", async () => { + const root = await mkdtemp(path.join(tmpdir(), "delivery-managed-")); + await mkdir(path.join(root, "sandbox-providers/daytona"), { recursive: true }); + vi.stubEnv("PAPERCLIP_BUNDLED_PLUGIN_ROOT", root); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "1", features: {}, + plugins: { autoInstall: ["daytona"], workerEnv: { daytona: ["PROVIDER_ORIGIN"] } } })); + try { + await db.delete(plugins); + await db.insert(plugins).values({ id: randomUUID(), pluginKey: "paperclip.daytona-sandbox-provider", packageName: "managed", version: "1.0.0", status: "ready", + manifestJson: { resourceLifecycle: ["agent_runtime"], capabilities: ["events.subscribe"] } as never }); + const d = delivery(); + await d.worker.ready; + expect(db.$client.listen).not.toHaveBeenCalled(); + d.worker.dispose(); + } finally { vi.unstubAllEnvs(); await rm(root, { recursive: true, force: true }); } + }); it("delivers project events to a storage provider only for projects attached to it", async () => { await db.update(plugins).set({ manifestJson: { resourceLifecycle: ["project"], capabilities: ["events.subscribe", "workspaces.manage"], workspaceProvider: { modes: ["isolated_workspace"], strategies: ["git_worktree"] } } as never }).where(eq(plugins.id, pluginId)); const [core] = await db.insert(projects).values({ companyId, name: "Core project" }).returning(); diff --git a/server/src/__tests__/project-instance-secrets.test.ts b/server/src/__tests__/project-instance-secrets.test.ts index 23457832d0c..cc372cfb2ce 100644 --- a/server/src/__tests__/project-instance-secrets.test.ts +++ b/server/src/__tests__/project-instance-secrets.test.ts @@ -1,4 +1,7 @@ import { randomBytes, randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { and, eq } from "drizzle-orm"; import { @@ -450,6 +453,35 @@ const support = await getEmbeddedPostgresTestSupport(); }, ); describe("project instance secret patches", () => { + it.skipIf(!support.supported)("requires a config row unless the managed config enables the plugin for this company", async () => { + const root = await mkdtemp(path.join(tmpdir(), "instance-secrets-managed-")); + await mkdir(path.join(root, "sandbox-providers/daytona"), { recursive: true }); + const database = await startEmbeddedPostgresTestDatabase("paperclip-instance-secrets-managed-"); + const db = createDb(database.connectionString); + try { + const companyId = randomUUID(), pluginId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Managed credentials", issuePrefix: "MCI" }); + await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.daytona-sandbox-provider", packageName: "managed", version: "1.0.0", status: "ready", packagePath: "/old/image/plugins/daytona", + manifestJson: { resourceLifecycle: ["project"], capabilities: ["events.subscribe", "projects.instance-secrets"] } as never }); + const [project] = await db.insert(projects).values({ companyId, name: "Credential owner" }).returning(); + await db.transaction(tx => recordResourceCreationEvent(tx as unknown as Db, companyId, "project", project.id)); + const [event] = await pluginLifecycleInbox(db, pluginId).list(companyId, 100); + await expect(projectInstanceSecretService(db, pluginId).prepare(event)).rejects.toThrow("require plugin configuration"); + vi.stubEnv("PAPERCLIP_BUNDLED_PLUGIN_ROOT", root); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "1", features: {}, + plugins: { autoInstall: ["daytona"], workerEnv: { daytona: ["PROVIDER_ORIGIN"] }, enabledCompanies: { daytona: [randomUUID()] } } })); + await expect(projectInstanceSecretService(db, pluginId).prepare(event)).rejects.toThrow("require plugin configuration"); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "1", features: {}, + plugins: { autoInstall: ["daytona"], enabledCompanies: { daytona: [companyId] } } })); + await expect(projectInstanceSecretService(db, pluginId).prepare(event)).resolves.toMatchObject({ instanceSecrets: expect.any(Object) }); + } finally { + vi.unstubAllEnvs(); + await db.$client.end({ timeout: 0 }); + await database.cleanup(); + await rm(root, { recursive: true, force: true }); + } + }, 90_000); + it("accepts bounded bundles and removal and rejects malformed patches without leaking values", () => { expect(() => validateProjectInstanceSecretChanges({ diff --git a/server/src/services/managed-config.ts b/server/src/services/managed-config.ts index 8757de417da..0bdcb72a304 100644 --- a/server/src/services/managed-config.ts +++ b/server/src/services/managed-config.ts @@ -49,7 +49,11 @@ export interface ManagedInstanceConfig { /** App feature-catalog version the document was validated against. */ catalogVersion: string; features: Readonly>>; - plugins: { readonly autoInstall: readonly string[] }; + plugins: { + readonly autoInstall: readonly string[]; + readonly workerEnv?: Readonly>; + readonly enabledCompanies?: Readonly>; + }; /** Sandbox environments the control plane provisions at boot (empty when the section is absent). */ environments: readonly ManagedEnvironmentSpec[]; } @@ -82,6 +86,12 @@ export interface ManagedEnvironmentSpec { */ export const SECRET_LIKE_CONFIG_KEY_PATTERN = /(api[-_]?key|token|secret|password|credential)/i; +const WORKER_ENV_NAME_PATTERN = /^[A-Z][A-Z0-9_]*$/; +const COMPANY_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +const WORKER_ENV_DENIED_NAME_PATTERN = + /(^|_)(AWS|CREDENTIAL|CREDENTIALS|DATABASE|DSN|JWT|KEY|KEYS|PASSWORD|POSTGRES|PRIVATE|SECRET|SECRETS|SIGNING|TOKEN)(_|$)|^(DB|DYLD|LD|NODE)_|^(PATH|PAPERCLIP_MANAGED_CONFIG)$/; + function findSecretLikeConfigKey(value: Record, path: string): string | null { for (const [key, child] of Object.entries(value)) { const childPath = path.length > 0 ? `${path}.${key}` : key; @@ -224,8 +234,8 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon fail(`"plugins" must be an object (got ${describeJsonValue(doc.plugins)})`); } for (const key of Object.keys(doc.plugins)) { - if (key !== "autoInstall") { - fail(`"plugins" has unknown key "${key}" (allowed: autoInstall)`); + if (key !== "autoInstall" && key !== "workerEnv" && key !== "enabledCompanies") { + fail(`"plugins" has unknown key "${key}" (allowed: autoInstall, workerEnv, enabledCompanies)`); } } const rawAutoInstall = doc.plugins.autoInstall; @@ -247,6 +257,63 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon autoInstall.push(entry); } + let workerEnv: Record | undefined; + if (doc.plugins.workerEnv !== undefined) { + if (!isPlainObject(doc.plugins.workerEnv)) { + fail(`"plugins.workerEnv" must be an object mapping plugin key → variable names (got ${describeJsonValue(doc.plugins.workerEnv)})`); + } + workerEnv = Object.create(null) as Record; + for (const [key, names] of Object.entries(doc.plugins.workerEnv)) { + if (!autoInstall.includes(key)) { + fail(`"plugins.workerEnv" key "${key}" is not in "plugins.autoInstall"; a grant requires its plugin to be provisioned`); + } + if (!Array.isArray(names) || names.length === 0) { + fail(`"plugins.workerEnv.${key}" must be a non-empty array of variable names (got ${describeJsonValue(names)})`); + } + const granted: string[] = []; + for (const name of names) { + if (typeof name !== "string" || !WORKER_ENV_NAME_PATTERN.test(name)) { + fail(`"plugins.workerEnv.${key}" entries must be uppercase variable names (got ${describeJsonValue(name)})`); + } + if (WORKER_ENV_DENIED_NAME_PATTERN.test(name)) { + fail(`"plugins.workerEnv.${key}" must not grant host credential or loader variable "${name}"`); + } + if (granted.includes(name)) { + fail(`"plugins.workerEnv.${key}" has duplicate entry "${name}"`); + } + granted.push(name); + } + workerEnv[key] = Object.freeze(granted); + } + } + + let enabledCompanies: Record | undefined; + if (doc.plugins.enabledCompanies !== undefined) { + if (!isPlainObject(doc.plugins.enabledCompanies)) { + fail(`"plugins.enabledCompanies" must be an object mapping plugin key → company ids (got ${describeJsonValue(doc.plugins.enabledCompanies)})`); + } + enabledCompanies = Object.create(null) as Record; + for (const [key, ids] of Object.entries(doc.plugins.enabledCompanies)) { + if (!autoInstall.includes(key)) { + fail(`"plugins.enabledCompanies" key "${key}" is not in "plugins.autoInstall"; enabling a company requires its plugin to be provisioned`); + } + if (!Array.isArray(ids) || ids.length === 0) { + fail(`"plugins.enabledCompanies.${key}" must be a non-empty array of company ids (got ${describeJsonValue(ids)})`); + } + const companyIds: string[] = []; + for (const id of ids) { + if (typeof id !== "string" || !COMPANY_ID_PATTERN.test(id)) { + fail(`"plugins.enabledCompanies.${key}" entries must be lowercase company UUIDs (got ${describeJsonValue(id)})`); + } + if (companyIds.includes(id)) { + fail(`"plugins.enabledCompanies.${key}" has duplicate entry "${id}"`); + } + companyIds.push(id); + } + enabledCompanies[key] = Object.freeze(companyIds); + } + } + // `environments` is OPTIONAL, unlike `features` and `plugins`: documents // delivered before the section existed must keep booting newer builds (a // fleet image roll cannot be lockstepped with a config re-delivery), and @@ -332,7 +399,11 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon mode: "cloud", catalogVersion: doc.catalogVersion, features: Object.freeze(features), - plugins: Object.freeze({ autoInstall: Object.freeze(autoInstall) }), + plugins: Object.freeze({ + autoInstall: Object.freeze(autoInstall), + ...(workerEnv ? { workerEnv: Object.freeze(workerEnv) } : {}), + ...(enabledCompanies ? { enabledCompanies: Object.freeze(enabledCompanies) } : {}), + }), environments: Object.freeze(environmentSpecs), }) as ManagedInstanceConfig; } diff --git a/server/src/services/managed-plugin-worker-env.ts b/server/src/services/managed-plugin-worker-env.ts new file mode 100644 index 00000000000..a15790f2853 --- /dev/null +++ b/server/src/services/managed-plugin-worker-env.ts @@ -0,0 +1,29 @@ +import { resolveBundledCatalogRoot, resolveBundledPluginInstalls, type ResolvedBundledPlugin } from "./bundled-plugins.js"; +import { getManagedInstanceConfig, type ManagedConfigEnv, type ManagedInstanceConfig } from "./managed-config.js"; + +let cache: { config: ManagedInstanceConfig; catalogRoot: string; installs: readonly ResolvedBundledPlugin[] } | null = null; + +function managedInstalls(config: ManagedInstanceConfig, env: ManagedConfigEnv) { + const catalogRoot = resolveBundledCatalogRoot(env); + if (cache?.config === config && cache.catalogRoot === catalogRoot) return cache.installs; + const keys = new Set([...Object.keys(config.plugins.workerEnv ?? {}), ...Object.keys(config.plugins.enabledCompanies ?? {})]); + const installs = resolveBundledPluginInstalls([...keys], { catalogRoot, env, enforceCatalogRoot: true }); + cache = { config, catalogRoot, installs }; + return installs; +} + +function managedEntry(section: "workerEnv" | "enabledCompanies", pluginKey: string, env: ManagedConfigEnv): readonly string[] { + const config = getManagedInstanceConfig(env); + const entries = config?.plugins[section]; + if (!config || !entries) return []; + const key = managedInstalls(config, env).find(install => install.pluginKey === pluginKey)?.key; + return key !== undefined && Object.hasOwn(entries, key) ? entries[key]! : []; +} + +export function managedPluginWorkerEnvGrant(pluginKey: string, env: ManagedConfigEnv = process.env): readonly string[] { + return managedEntry("workerEnv", pluginKey, env); +} + +export function managedPluginEnabledCompanies(pluginKey: string, env: ManagedConfigEnv = process.env): readonly string[] { + return managedEntry("enabledCompanies", pluginKey, env); +} diff --git a/server/src/services/plugin-lifecycle-delivery.ts b/server/src/services/plugin-lifecycle-delivery.ts index e70c45d0ff2..d801fa3dd3b 100644 --- a/server/src/services/plugin-lifecycle-delivery.ts +++ b/server/src/services/plugin-lifecycle-delivery.ts @@ -1,12 +1,13 @@ import { projectInstanceSecretService } from "./project-instance-secrets.js"; -import { and, asc, eq, isNull, or, sql } from "drizzle-orm"; -import { pluginCompanySettings, pluginConfig, plugins, projects, type Db } from "@paperclipai/db"; +import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { companies, pluginCompanySettings, pluginConfig, plugins, projects, type Db } from "@paperclipai/db"; import { logger } from "../middleware/logger.js"; import { pluginLifecycleInbox } from "./plugin-lifecycle-inbox.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { parseProjectExecutionWorkspacePolicy } from "./execution-workspace-policy.js"; import { isWorkspaceProvider } from "./project-workspace-provider.js"; +import { managedPluginEnabledCompanies } from "./managed-plugin-worker-env.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; import type { PluginLifecycleManager } from "./plugin-lifecycle.js"; @@ -26,6 +27,34 @@ export function registerPluginLifecycleDelivery( const retryAt = new Map(); const refreshedRuntimeCompanies = new Set(); const retryMs = 30_000; + async function managedTargets() { + const ready = await db.select().from(plugins).where(eq(plugins.status, "ready")); + const targets = ready.flatMap(plugin => managedPluginEnabledCompanies(plugin.pluginKey).map(companyId => ({ plugin, companyId }))); + if (!targets.length) return targets; + const existing = await db.select({ id: companies.id }).from(companies) + .where(inArray(companies.id, targets.map(target => target.companyId))); + const existingIds = new Set(existing.map(company => company.id)); + return targets.filter(target => existingIds.has(target.companyId)); + } + async function enabledManagedPlugins() { + const targets = await managedTargets(); + if (!targets.length) return []; + const disabled = await db.select({ pluginId: pluginCompanySettings.pluginId, companyId: pluginCompanySettings.companyId }).from(pluginCompanySettings) + .where(and(inArray(pluginCompanySettings.pluginId, targets.map(target => target.plugin.id)), eq(pluginCompanySettings.enabled, false))); + const disabledKeys = new Set(disabled.map(row => `${row.pluginId}:${row.companyId}`)); + return targets.filter(target => !disabledKeys.has(`${target.plugin.id}:${target.companyId}`)).map(target => target.plugin); + } + async function deliveryTargets() { + const configured = await db.select({ plugin: plugins, companyId: pluginConfig.companyId }) + .from(plugins).innerJoin(pluginConfig, eq(pluginConfig.pluginId, plugins.id)) + .where(eq(plugins.status, "ready")) + .orderBy(asc(plugins.installOrder), asc(pluginConfig.companyId)); + const managed = await managedTargets(); + if (!managed.length) return configured; + const seen = new Set(configured.map(target => `${target.plugin.id}:${target.companyId}`)); + return [...configured, ...managed.filter(target => !seen.has(`${target.plugin.id}:${target.companyId}`))].sort((a, b) => + (a.plugin.installOrder ?? Infinity) - (b.plugin.installOrder ?? Infinity) || a.companyId.localeCompare(b.companyId)); + } const delivery = coordinator.register(DELIVERY_QUEUES.pluginLifecycle, { retryMs, hasPending: async () => pending, @@ -44,11 +73,7 @@ export function registerPluginLifecycleDelivery( try { await ensureListening(); for (const [key, at] of retryAt) if (at <= Date.now()) retryAt.delete(key); - const configured = await db.select({ plugin: plugins, companyId: pluginConfig.companyId }) - .from(plugins).innerJoin(pluginConfig, eq(pluginConfig.pluginId, plugins.id)) - .where(eq(plugins.status, "ready")) - .orderBy(asc(plugins.installOrder), asc(pluginConfig.companyId)); - for (const { plugin, companyId } of configured) { + for (const { plugin, companyId } of await deliveryTargets()) { const types = plugin.manifestJson.resourceLifecycle; if (!types?.length || !plugin.manifestJson.capabilities.includes("events.subscribe")) continue; const [disabled] = await db.select({ id: pluginCompanySettings.id }).from(pluginCompanySettings) @@ -134,10 +159,11 @@ export function registerPluginLifecycleDelivery( function ensureListening() { const update = listenerUpdate.then(async () => { if (disposed || !db.$client?.listen) return; - const installed = await db.select({ manifest: plugins.manifestJson }).from(plugins) + const configured = await db.select({ manifest: plugins.manifestJson }).from(plugins) .innerJoin(pluginConfig, eq(pluginConfig.pluginId, plugins.id)) .leftJoin(pluginCompanySettings, and(eq(pluginCompanySettings.pluginId, plugins.id), eq(pluginCompanySettings.companyId, pluginConfig.companyId))) .where(and(eq(plugins.status, "ready"), or(isNull(pluginCompanySettings.enabled), eq(pluginCompanySettings.enabled, true)))); + const installed = [...configured, ...(await enabledManagedPlugins()).map(plugin => ({ manifest: plugin.manifestJson }))]; if (disposed) return; const wanted = installed.some(plugin => plugin.manifest.resourceLifecycle?.includes("agent_runtime") && plugin.manifest.capabilities?.includes("events.subscribe")); if (wanted) listening ??= await db.$client.listen("paperclip_plugin_lifecycle", wake, wake); diff --git a/server/src/services/plugin-loader.ts b/server/src/services/plugin-loader.ts index 1928ff94c0f..909bece5edd 100644 --- a/server/src/services/plugin-loader.ts +++ b/server/src/services/plugin-loader.ts @@ -53,6 +53,7 @@ import type { PluginToolDispatcher } from "./plugin-tool-dispatcher.js"; import type { PluginLifecycleManager } from "./plugin-lifecycle.js"; import { pluginDatabaseService } from "./plugin-database.js"; import { resolveBundledCatalogRoot } from "./bundled-plugins.js"; +import { managedPluginWorkerEnvGrant } from "./managed-plugin-worker-env.js"; const execFileAsync = promisify(execFile); const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -160,6 +161,7 @@ export function buildPluginWorkerEnv(input: { packagePath?: string | null; /** Test seam; defaults to the repo plugin tree and the bundled catalog root. */ trustedLocalPluginRoots?: readonly string[]; + grantedEnvVars?: readonly string[]; instanceInfo: { deploymentMode?: string | null; deploymentExposure?: string | null }; processEnv?: NodeJS.ProcessEnv; }): Record { @@ -168,6 +170,10 @@ export function buildPluginWorkerEnv(input: { PAPERCLIP_DEPLOYMENT_MODE: input.instanceInfo.deploymentMode ?? "", PAPERCLIP_DEPLOYMENT_EXPOSURE: input.instanceInfo.deploymentExposure ?? "", }; + for (const key of input.grantedEnvVars ?? []) { + const value = processEnv[key]; + if (value && value.trim().length > 0) env[key] = value; + } const canRegisterEnvironmentDrivers = Array.isArray(input.manifest.capabilities) && input.manifest.capabilities.includes("environment.drivers.register"); if (!canRegisterEnvironmentDrivers) return env; @@ -2341,6 +2347,7 @@ export function pluginLoader( packageName: activePlugin.packageName, packagePath: resolvePackageRoot ? packageRoot : activePlugin.packagePath, instanceInfo, + grantedEnvVars: managedPluginWorkerEnvGrant(activePlugin.pluginKey), }), // Authorize the worker to act on each configured company from its // proactive loops/timers (LOOA-629). Seeded here so it is in place diff --git a/server/src/services/project-instance-secrets.ts b/server/src/services/project-instance-secrets.ts index e8b7a70c1da..921bfc03502 100644 --- a/server/src/services/project-instance-secrets.ts +++ b/server/src/services/project-instance-secrets.ts @@ -14,6 +14,7 @@ import type { ResourceLifecycleEvent } from "@paperclipai/plugin-sdk"; import { z } from "zod"; import { secretService } from "./secrets.js"; import { persistActivity, type ActivityPublication } from "./activity-log.js"; +import { managedPluginEnabledCompanies } from "./managed-plugin-worker-env.js"; export type ProjectSecretSnapshot = { revision: string }; export function validateProjectInstanceSecretChanges( @@ -82,7 +83,7 @@ export function projectInstanceSecretService(db: Db, pluginId: string) { ), ); const [config] = await (lock ? configQuery.for("share") : configQuery); - if (!config) + if (!config && !managedPluginEnabledCompanies(plugin.pluginKey).includes(companyId)) throw new Error("Project instance secrets require plugin configuration"); const projectQuery = db .select()