diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 5704355c9fc..5a7659c4ee9 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1919,6 +1919,37 @@ that the whole web server is disposable or that all execution and recovery paths survive web-server replacement. Run-log/trace storage and producer retention are separate changes. +### Managed environments + +The `environments` list in `PAPERCLIP_MANAGED_CONFIG` declares instance-level +environments that boot provisions and tenants cannot edit. It holds at most one +sandbox entry (no `driver`, or `driver: "sandbox"`) and any number of +`driver: "plugin"` entries. A plugin entry names a plugin by its manifest id +(`pluginKey`) and one of its environment drivers: + +```json +{ + "name": "Remote Host", + "description": "Runs the agent on a remote host", + "driver": "plugin", + "pluginKey": "example.remote-host-plugin", + "driverKey": "tasks", + "driverConfig": {} +} +``` + +Names and `pluginKey`–`driverKey` pairs must be unique, and `driverConfig` must +not contain secret-like keys. Any other key refuses startup. `plugins.autoInstall` +holds catalog keys, not manifest ids, so boot also refuses to start unless an +auto-installed bundled or distribution catalog entry installs the entry's +`pluginKey`. + +Boot validates the driver config through the plugin, marks the row +`managedPluginEnvironment: ":"`, and adopts an existing +unmarked row for the same plugin and driver. While the plugin is down, the row is +archived; it comes back when the worker is ready. Removing an entry never archives +its row. Plugin entries do not change the instance default. + ### Experimental native remote workspace storage `experimental.remoteWorkspaceStorageEnabled` is a managed feature, on by default diff --git a/server/src/__tests__/environment-routes.test.ts b/server/src/__tests__/environment-routes.test.ts index 786e97d6ea2..0274007ac2d 100644 --- a/server/src/__tests__/environment-routes.test.ts +++ b/server/src/__tests__/environment-routes.test.ts @@ -1022,6 +1022,20 @@ describe("environment routes", () => { describe("managed plugin environment rows", () => { const PLUGIN_KEY = "example.remote-host-plugin"; + const MANAGED_CONFIG_WITH_PLUGIN_ENTRY = JSON.stringify({ + v: 1, + mode: "cloud", + catalogVersion: "2026.1106.0", + features: {}, + plugins: { autoInstall: ["remote-host"] }, + environments: [{ + name: "Remote Host", + driver: "plugin", + pluginKey: PLUGIN_KEY, + driverKey: "tasks", + driverConfig: {}, + }], + }); const pluginRow = () => ({ ...createPlatformSandboxEnvironment(), id: "env-plugin-1", @@ -1058,20 +1072,46 @@ describe("environment routes", () => { expect(res.body.details).toMatchObject({ code: "environment_platform_managed" }); }); - it("allows the marker clear while no managed-config entry can own the row", async () => { - // Nothing provisions marked plugin rows yet, so a marker is stale and - // stays recoverable. The managed-config plugin entry PR refuses this - // while the document declares the row's entry. - const row = pluginRow(); - mockEnvironmentService.getById.mockResolvedValue(row); - mockEnvironmentService.update.mockResolvedValue({ ...row, metadata: {} }); + it("refuses a marker clear while the document declares the entry", async () => { + process.env.PAPERCLIP_MANAGED_CONFIG = MANAGED_CONFIG_WITH_PLUGIN_ENTRY; + try { + mockEnvironmentService.getById.mockResolvedValue(pluginRow()); + const app = createApp(ownerAdminActor); - const res = await request(createApp(ownerAdminActor)) - .patch("/api/environments/env-plugin-1") - .send({ metadata: { managedByPaperclip: null, managedPluginEnvironment: null } }); + for (const metadata of [ + { managedByPaperclip: false }, + { managedPluginEnvironment: null }, + { managedByPaperclip: null, managedPluginEnvironment: null }, + ]) { + const res = await request(app).patch("/api/environments/env-plugin-1").send({ metadata }); + expect(res.status).toBe(403); + expect(res.body.details).toMatchObject({ code: "environment_platform_managed" }); + } + expect(mockEnvironmentService.update).not.toHaveBeenCalled(); + } finally { + delete process.env.PAPERCLIP_MANAGED_CONFIG; + } + }); - expect(res.status).toBe(200); - expect(mockEnvironmentService.update).toHaveBeenCalled(); + it("allows the marker clear once the entry left the document", async () => { + process.env.PAPERCLIP_MANAGED_CONFIG = JSON.stringify({ + ...JSON.parse(MANAGED_CONFIG_WITH_PLUGIN_ENTRY), + environments: [], + }); + try { + const row = pluginRow(); + mockEnvironmentService.getById.mockResolvedValue(row); + mockEnvironmentService.update.mockResolvedValue({ ...row, metadata: {} }); + + const res = await request(createApp(ownerAdminActor)) + .patch("/api/environments/env-plugin-1") + .send({ metadata: { managedByPaperclip: null, managedPluginEnvironment: null } }); + + expect(res.status).toBe(200); + expect(mockEnvironmentService.update).toHaveBeenCalled(); + } finally { + delete process.env.PAPERCLIP_MANAGED_CONFIG; + } }); it("refuses a client that stamps the plugin marker on its own row", async () => { diff --git a/server/src/__tests__/instance-settings-cloud-defaults.test.ts b/server/src/__tests__/instance-settings-cloud-defaults.test.ts index 76ff39b0074..6a9a550032f 100644 --- a/server/src/__tests__/instance-settings-cloud-defaults.test.ts +++ b/server/src/__tests__/instance-settings-cloud-defaults.test.ts @@ -17,6 +17,7 @@ function managedConfig(features: ManagedInstanceConfig["features"] = {}): Manage features, plugins: { autoInstall: [] }, environments: [], + pluginEnvironments: [], }; } diff --git a/server/src/__tests__/instance-settings-service.test.ts b/server/src/__tests__/instance-settings-service.test.ts index 6c2847b6eb9..4ced4dcc4c1 100644 --- a/server/src/__tests__/instance-settings-service.test.ts +++ b/server/src/__tests__/instance-settings-service.test.ts @@ -26,7 +26,7 @@ describe("instance settings service", () => { } expect(applyExperimentalSettingsPatch({}, { enableAgentChat: false }).enableAgentChat).toBe(true); const config: ManagedInstanceConfig = { - v: 1, mode: "cloud", catalogVersion: "test", plugins: { autoInstall: [] }, environments: [], + v: 1, mode: "cloud", catalogVersion: "test", plugins: { autoInstall: [] }, environments: [], pluginEnvironments: [], features: { enableAgentChat: false }, }; const result = applyManagedExperimentalOverlay(normalizeExperimentalSettings({ enableAgentChat: false }), config); @@ -54,7 +54,7 @@ describe("instance settings service", () => { { stored: { remoteWorkspaceStorageEnabled: true }, managed: false, expected: false }, ])("respects Cloud enrollment defaults and managed choices: $expected", ({ stored, managed, expected }) => { const config: ManagedInstanceConfig = { - v: 1, mode: "cloud", catalogVersion: "test", plugins: { autoInstall: [] }, environments: [], + v: 1, mode: "cloud", catalogVersion: "test", plugins: { autoInstall: [] }, environments: [], pluginEnvironments: [], features: managed === undefined ? {} : { remoteWorkspaceStorageEnabled: managed }, }; const normalized = normalizeExperimentalSettings(stored); diff --git a/server/src/__tests__/managed-config.test.ts b/server/src/__tests__/managed-config.test.ts index 226ef9bf401..e01bdbb31d0 100644 --- a/server/src/__tests__/managed-config.test.ts +++ b/server/src/__tests__/managed-config.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { MANAGED_CONFIG_ENV_KEY, + MANAGED_ENVIRONMENT_DRIVERS, getManagedInstanceConfig, managedFeatureKeySet, parseManagedConfigEnv, @@ -110,6 +111,7 @@ describe("parseManagedConfigEnv", () => { features: { enableApps: false, enableStatusCards: true }, plugins: { autoInstall: ["daytona", "kubernetes"] }, environments: [], + pluginEnvironments: [], }); }); @@ -132,6 +134,7 @@ describe("parseManagedConfigEnv", () => { features: {}, plugins: { autoInstall: [] }, environments: [], + pluginEnvironments: [], }); }); @@ -342,12 +345,17 @@ describe("parseManagedConfigEnv environments section", () => { ).toThrow(/"environments\[0\]" must be an object/); }); - it("rejects more than one entry (single managed sandbox slot)", () => { + it("rejects more than one sandbox entry (single managed sandbox slot)", () => { expect(() => parseManagedConfigEnv( envWith(validDoc({ environments: [entry(), entry({ name: "Other", provider: "kubernetes" })] })), ), - ).toThrow(/at most one entry/); + ).toThrow(/at most one sandbox entry/); + expect(() => + parseManagedConfigEnv( + envWith(validDoc({ environments: [entry(), entry({ name: "Other", driver: "sandbox", provider: "kubernetes" })] })), + ), + ).toThrow(/at most one sandbox entry/); }); it("rejects unknown entry keys", () => { @@ -406,6 +414,94 @@ describe("parseManagedConfigEnv environments section", () => { }); }); +describe("parseManagedConfigEnv plugin environment entries", () => { + const PLUGIN_KEY = "example.remote-host-plugin"; + const remoteHost = (overrides: Record = {}) => ({ + name: "Remote Host", + description: "Runs the agent on a remote host", + driver: "plugin", + pluginKey: PLUGIN_KEY, + driverKey: "tasks", + driverConfig: {}, + ...overrides, + }); + const sandbox = { name: "Managed Sandbox", provider: "daytona", config: { target: "us" } }; + // `plugins.autoInstall` holds catalog keys ("remote-host"); the entry's + // `pluginKey` is the plugin's manifest id, which boot maps through the catalog. + const docWith = (environments: unknown[], autoInstall = ["daytona", "remote-host"]) => + envWith(validDoc({ plugins: { autoInstall }, environments })); + + it("lists the drivers an entry may declare", () => { + expect(MANAGED_ENVIRONMENT_DRIVERS).toEqual(["sandbox", "plugin"]); + }); + + it("parses an old document's sandbox entry unchanged", () => { + const config = parseManagedConfigEnv(docWith([sandbox])); + expect(config?.environments).toEqual([{ name: "Managed Sandbox", provider: "daytona", config: { target: "us" } }]); + expect(config?.pluginEnvironments).toEqual([]); + }); + + it("parses the sandbox entry beside a plugin entry", () => { + const config = parseManagedConfigEnv(docWith([sandbox, remoteHost()])); + expect(config?.environments).toEqual([{ name: "Managed Sandbox", provider: "daytona", config: { target: "us" } }]); + expect(config?.pluginEnvironments).toEqual([{ + driver: "plugin", + name: "Remote Host", + description: "Runs the agent on a remote host", + pluginKey: PLUGIN_KEY, + driverKey: "tasks", + driverConfig: {}, + }]); + expect(Object.isFrozen(config?.pluginEnvironments)).toBe(true); + expect(Object.isFrozen(config?.pluginEnvironments[0])).toBe(true); + }); + + it("parses a plugin entry alone, with optional fields omitted", () => { + const config = parseManagedConfigEnv(docWith([{ name: "Remote", driver: "plugin", pluginKey: PLUGIN_KEY, driverKey: "tasks" }])); + expect(config?.environments).toEqual([]); + expect(config?.pluginEnvironments[0]).toMatchObject({ driverConfig: {} }); + expect(config?.pluginEnvironments[0]?.description).toBeUndefined(); + }); + + it("does not compare the manifest id with autoInstall catalog keys (boot checks the catalog)", () => { + expect(parseManagedConfigEnv(docWith([remoteHost()]))?.pluginEnvironments[0]?.pluginKey).toBe(PLUGIN_KEY); + }); + + it("refuses duplicate names, including across drivers", () => { + expect(() => parseManagedConfigEnv(docWith([sandbox, remoteHost({ name: "Managed Sandbox" })]))) + .toThrow(/"environments\[1\].name" duplicates another entry's name/); + }); + + it("refuses a duplicate pluginKey and driverKey pair", () => { + expect(() => parseManagedConfigEnv(docWith([remoteHost(), remoteHost({ name: "Second" })]))) + .toThrow(/duplicates another plugin entry's pluginKey and driverKey/); + }); + + it("refuses secret-like driverConfig keys at any depth", () => { + expect(() => parseManagedConfigEnv(docWith([remoteHost({ driverConfig: { apiToken: "t" } })]))) + .toThrow(/"environments\[0\].driverConfig" key "apiToken" looks secret-bearing/); + expect(() => parseManagedConfigEnv(docWith([remoteHost({ driverConfig: { auth: [{ password: "p" }] } })]))) + .toThrow(/auth\[0\].password/); + }); + + it("refuses unknown keys, including sandbox keys on a plugin entry and the reverse", () => { + expect(() => parseManagedConfigEnv(docWith([remoteHost({ envVars: {} })]))).toThrow(/unknown key "envVars"/); + expect(() => parseManagedConfigEnv(docWith([remoteHost({ provider: "daytona" })]))).toThrow(/unknown key "provider"/); + expect(() => parseManagedConfigEnv(docWith([{ ...sandbox, pluginKey: PLUGIN_KEY }]))).toThrow(/unknown key "pluginKey"/); + // Not supported by this build on either entry. + expect(() => parseManagedConfigEnv(docWith([remoteHost({ instanceDefault: true })]))).toThrow(/unknown key "instanceDefault"/); + expect(() => parseManagedConfigEnv(docWith([{ ...sandbox, instanceDefault: false }]))).toThrow(/unknown key "instanceDefault"/); + }); + + it("refuses an unknown driver and malformed plugin fields", () => { + expect(() => parseManagedConfigEnv(docWith([remoteHost({ driver: "ssh" })]))).toThrow(/"environments\[0\].driver" must be one of "sandbox", "plugin"/); + expect(() => parseManagedConfigEnv(docWith([{ ...sandbox, driver: null }]))).toThrow(/"environments\[0\].driver" must be one of "sandbox", "plugin" when present \(got null\)/); + expect(() => parseManagedConfigEnv(docWith([remoteHost({ driverKey: "Tasks" })]))).toThrow(/"environments\[0\].driverKey" must start/); + expect(() => parseManagedConfigEnv(docWith([remoteHost({ driverKey: undefined })]))).toThrow(/"environments\[0\].driverKey" must be a non-empty string/); + expect(() => parseManagedConfigEnv(docWith([remoteHost({ driverConfig: [] })]))).toThrow(/"environments\[0\].driverConfig" must be an object/); + }); +}); + describe("getManagedInstanceConfig", () => { it("caches by raw env value and reparses when it changes", () => { const raw = validDoc(); diff --git a/server/src/__tests__/managed-plugin-environments.test.ts b/server/src/__tests__/managed-plugin-environments.test.ts index fc8496b236d..4179d238dbf 100644 --- a/server/src/__tests__/managed-plugin-environments.test.ts +++ b/server/src/__tests__/managed-plugin-environments.test.ts @@ -7,15 +7,42 @@ import { companies, createDb, environments, + instanceSettings, plugins, type Db, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { environmentService } from "../services/environments.js"; +import { MANAGED_CONFIG_ENV_KEY, parseManagedConfigEnv } from "../services/managed-config.js"; +import { applyManagedEnvironments } from "../services/managed-environments.js"; const PLUGIN_KEY = "example.remote-host-plugin"; const MARKER = `${PLUGIN_KEY}:tasks`; +type ReadyListener = (payload: { pluginId: string }) => void; + +function managedConfig(entries: unknown[]) { + return parseManagedConfigEnv({ + [MANAGED_CONFIG_ENV_KEY]: JSON.stringify({ + v: 1, + mode: "cloud", + catalogVersion: "2026.1106.0", + features: {}, + plugins: { autoInstall: ["remote-host"] }, + environments: entries, + }), + }); +} + +const remoteHostEntry = { + name: "Remote Host", + description: "Runs the agent on a remote host", + driver: "plugin", + pluginKey: PLUGIN_KEY, + driverKey: "tasks", + driverConfig: {}, +}; + const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("managed plugin environment rows", () => { let database: Awaited>; @@ -24,7 +51,17 @@ const support = await getEmbeddedPostgresTestSupport(); let running: boolean; const call = vi.fn(async (_id: string, method: string, input: { config: Record }) => method === "environmentValidateConfig" ? { ok: true, normalizedConfig: { ...input.config, normalized: true } } : null); - const workerManager = { isRunning: vi.fn(() => running), call }; + let readyListeners: ReadyListener[] = []; + const workerManager = { + isRunning: vi.fn(() => running), + call, + getWorker: vi.fn(() => ({ + on: (_event: "ready", listener: ReadyListener) => { readyListeners.push(listener); }, + off: (_event: "ready", listener: ReadyListener) => { + readyListeners = readyListeners.filter((candidate) => candidate !== listener); + }, + })), + }; beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-managed-plugin-environment-rows-"); @@ -33,9 +70,11 @@ const support = await getEmbeddedPostgresTestSupport(); afterAll(async () => { await database?.cleanup(); }); beforeEach(async () => { running = true; + readyListeners = []; call.mockClear(); await db.delete(activityLog); await db.delete(agents); + await db.delete(instanceSettings); await db.delete(environments); await db.delete(plugins); await db.delete(companies); @@ -175,4 +214,42 @@ const support = await getEmbeddedPostgresTestSupport(); metadata: { managedPluginEnvironment: MARKER }, })).rejects.toThrow(); }); + + describe("boot (applyManagedEnvironments)", () => { + const boot = (entries: unknown[] = [remoteHostEntry]) => + applyManagedEnvironments(db, managedConfig(entries), { env: {}, workerManager, autoInstalledPluginKeys: [PLUGIN_KEY] }); + const instanceDefault = async () => + (await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "default")))[0]?.defaultEnvironmentId ?? null; + + it("creates the row on the first boot and changes nothing on the second", async () => { + const first = await boot(); + expect(first).toMatchObject({ ensured: 1, added: 1, failed: 0 }); + const [row] = await pluginRows(); + expect(row).toMatchObject({ name: "Remote Host", status: "active", metadata: { managedByPaperclip: true, managedPluginEnvironment: MARKER } }); + + const second = await boot(); + expect(second).toMatchObject({ ensured: 1, added: 0, unchanged: 1 }); + const after = await pluginRows(); + expect(after).toHaveLength(1); + expect(after[0]!.updatedAt.getTime()).toBe(row!.updatedAt.getTime()); + expect(await instanceDefault()).toBeNull(); + }); + + it("archives the row while the plugin is down and reactivates it when the worker recovers", async () => { + await boot(); + running = false; + expect(await boot()).toMatchObject({ ensured: 0, failed: 1 }); + expect((await pluginRows())[0]!.status).toBe("archived"); + + running = true; + for (const listener of [...readyListeners]) listener({ pluginId }); + await vi.waitFor(async () => expect((await pluginRows())[0]?.status).toBe("active")); + }); + + it("never archives a row whose entry left the document", async () => { + await boot(); + expect(await boot([])).toBeNull(); + expect((await pluginRows())[0]!.status).toBe("active"); + }); + }); }); diff --git a/server/src/app.ts b/server/src/app.ts index 75a3ee1c207..5a765ee0c95 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1348,6 +1348,9 @@ export async function createApp( logger.error({ err }, "Failed to load ready plugins on startup"); }); app.locals.bundledPluginsStartup = trackIdleWork(bundledPluginsStartup); + // Manifest ids the managed auto-install list resolves to; the managed + // environments bootstrap checks plugin entries against them. + app.locals.managedBundledPluginKeys = managedBundledPluginKeys; void bundledPluginsStartup.then(() => { lifecyclePluginsReady = true; resourceLifecycleDelivery.wake(); agentLifecycleDelivery.wake(); }).catch(() => logger.warn("Agent lifecycle recovery failed; retrying.")); // The shutdown hook runs at most once. It caches the in-flight promise, so a // second caller (for example the `exit` handler) awaits the same completion diff --git a/server/src/index.ts b/server/src/index.ts index e36818bfdee..13c55b20807 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1175,8 +1175,8 @@ async function startServerWithDatabaseTeardown( throw err; } - // Ensure sandbox environments declared in the managed-config document - // (`environments` section) before the heartbeat resumes queued runs. The + // Ensure the sandbox and plugin environments declared in the managed-config + // document (`environments` section) before the heartbeat resumes queued runs. The // document already parsed fail-closed above; the ensure step itself is // fail-safe per entry (a degraded boot beats a fleet-wide crash loop), but // a contradictory deployment that also forces PAPERCLIP_EXECUTION_MODE @@ -1186,14 +1186,16 @@ async function startServerWithDatabaseTeardown( // additionally gates each entry on a live plugin worker (and archives the // row of a provider that did not come up). try { - const bundledPluginsStartup = (app as { locals?: { bundledPluginsStartup?: Promise } }) - .locals?.bundledPluginsStartup; + const appLocals = (app as { + locals?: { bundledPluginsStartup?: Promise; managedBundledPluginKeys?: readonly string[] }; + }).locals; const managedEnvironmentsResult = await applyManagedEnvironments(db as any, managedConfig, { - pluginsReady: bundledPluginsStartup, + pluginsReady: appLocals?.bundledPluginsStartup, + autoInstalledPluginKeys: appLocals?.managedBundledPluginKeys, workerManager: pluginWorkerManager, }); if (managedEnvironmentsResult) { - logger.warn(managedEnvironmentsResult, "managed sandbox environments ensured from managed config"); + logger.warn(managedEnvironmentsResult, "managed environments ensured from managed config"); } } catch (err) { logger.error({ err }, "failed to apply managed environments from managed config"); diff --git a/server/src/routes/environments.ts b/server/src/routes/environments.ts index 9a33f1b66f9..09b2023b7e7 100644 --- a/server/src/routes/environments.ts +++ b/server/src/routes/environments.ts @@ -19,7 +19,10 @@ import { import { conflict, forbidden, unprocessable } from "../errors.js"; import { isCloudManagedInstance } from "../services/cloud-instance.js"; import { getManagedInstanceConfig, SECRET_LIKE_CONFIG_KEY_PATTERN } from "../services/managed-config.js"; -import { MANAGED_PLUGIN_ENVIRONMENT_MARKER_KEY } from "../services/managed-plugin-environments.js"; +import { + MANAGED_PLUGIN_ENVIRONMENT_MARKER_KEY, + managedPluginEnvironmentMarker, +} from "../services/managed-plugin-environments.js"; import { parseExecutionPolicyBootstrapEnv } from "../services/execution-policy-bootstrap.js"; import { isExecutionForcedToKubernetes } from "../services/execution-allowlist.js"; import { validate } from "../middleware/validate.js"; @@ -182,6 +185,20 @@ function isManagedSandboxProvisioningConfigured(): boolean { } } +/** + * Whether the managed-config document declares the plugin entry this row's + * marker names. Fails closed like `isManagedSandboxProvisioningConfigured`. + */ +function isManagedPluginEnvironmentDeclared(marker: string): boolean { + try { + return (getManagedInstanceConfig()?.pluginEnvironments ?? []).some( + (spec) => managedPluginEnvironmentMarker(spec) === marker, + ); + } catch { + return true; + } +} + /** * Whether this row occupies a provisioner-owned environment slot whose * platform markers are LIVE state — i.e. some platform code path converges @@ -196,6 +213,9 @@ function isManagedSandboxProvisioningConfigured(): boolean { * owned only while a managed-sandbox bootstrap path is configured — * `ensureManagedSandboxEnvironment` then adopts and refreshes whichever * row holds the slot on every boot. + * - The row of a managed-config plugin entry (its + * `managedPluginEnvironment` marker) while the document declares that + * entry: `applyManagedEnvironments` converges the markers on it every boot. * - Any sandbox row bearing the legacy kubernetes marker while the * persisted instance execution policy forces kubernetes: * `findKubernetesEnvironment` selects marked rows (newest first) for @@ -214,6 +234,10 @@ async function isPlatformSlotEnvironment( options: { isForcedKubernetesExecution: () => Promise }, ): Promise { if (environment.driver === "local") return true; + if (environment.driver === "plugin") { + const pluginMarker = environment.metadata?.[MANAGED_PLUGIN_ENVIRONMENT_MARKER_KEY]; + return typeof pluginMarker === "string" && isManagedPluginEnvironmentDeclared(pluginMarker); + } if (environment.driver !== "sandbox") return false; if ( environment.metadata?.managedByPaperclip === true && diff --git a/server/src/services/managed-config.ts b/server/src/services/managed-config.ts index 0bdcb72a304..bea4e28ab6f 100644 --- a/server/src/services/managed-config.ts +++ b/server/src/services/managed-config.ts @@ -9,12 +9,23 @@ * "mode": "cloud", * "catalogVersion": "2026.720.0", * "features": { "": true | false, ... }, - * "plugins": { "autoInstall": ["daytona", "kubernetes"] }, + * "plugins": { "autoInstall": ["daytona", "remote-host"] }, * "environments": [ - * { "name": "Daytona", "provider": "daytona", "config": { "target": "us" } } + * { "name": "Daytona", "provider": "daytona", "config": { "target": "us" } }, + * { + * "name": "Remote Host", + * "driver": "plugin", + * "pluginKey": "example.remote-host-plugin", + * "driverKey": "tasks", + * "driverConfig": {} + * } * ] * } * + * An `environments` entry without `driver` is the single sandbox entry; any + * number of `driver: "plugin"` entries may sit beside it (see + * `MANAGED_ENVIRONMENT_DRIVERS`). + * * Parsing follows the `execution-policy-bootstrap.ts` doctrine: a pure * function over `Record`, strict, and fail * closed — bad JSON, an unsupported `v`, an unknown feature key, a feature @@ -54,8 +65,47 @@ export interface ManagedInstanceConfig { readonly workerEnv?: Readonly>; readonly enabledCompanies?: Readonly>; }; - /** Sandbox environments the control plane provisions at boot (empty when the section is absent). */ + /** + * The sandbox entry of the `environments` section: zero or one, since it + * provisions the single Paperclip-managed sandbox slot (empty when the + * section is absent or declares only plugin entries). + */ environments: readonly ManagedEnvironmentSpec[]; + /** The `driver: "plugin"` entries of the `environments` section (empty when none are declared). */ + pluginEnvironments: readonly ManagedPluginEnvironmentSpec[]; +} + +/** + * The environment drivers a managed-config `environments` entry may declare. + * An entry without `driver` is the sandbox entry, so documents written + * before plugin entries existed parse unchanged. + */ +export const MANAGED_ENVIRONMENT_DRIVERS = ["sandbox", "plugin"] as const; +export type ManagedEnvironmentDriver = (typeof MANAGED_ENVIRONMENT_DRIVERS)[number]; + +/** + * One declared managed plugin environment: an instance-level `plugin`-driver + * environment row served by an auto-installed plugin's environment driver. + * The row is + * identified by its `pluginKey`–`driverKey` pair, which boot stamps on the + * row as `metadata.managedPluginEnvironment`. + */ +export interface ManagedPluginEnvironmentSpec { + driver: "plugin"; + /** Display name of the instance-level environment row (unique per instance). */ + name: string; + description?: string; + /** + * The serving plugin's manifest id (for example + * `example.remote-host-plugin`), not its `plugins.autoInstall` catalog key. + * Boot requires a catalog entry + * whose `key` is in `plugins.autoInstall` to install this id. + */ + pluginKey: string; + /** The plugin's environment driver key. */ + driverKey: string; + /** Driver config, validated by the plugin at ensure time; never carries secrets. */ + driverConfig: Readonly>; } /** @@ -142,6 +192,155 @@ function describeJsonValue(value: unknown): string { return `${JSON.stringify(value)}`; } +const SANDBOX_ENTRY_KEYS = ["name", "description", "driver", "provider", "config"] as const; +const PLUGIN_ENTRY_KEYS = [ + "name", + "description", + "driver", + "pluginKey", + "driverKey", + "driverConfig", +] as const; +/** Same rule as the plugin environment config schema in `environment-config.ts`. */ +const PLUGIN_DRIVER_KEY_PATTERN = /^[a-z0-9][a-z0-9._-]*$/; + +function requireTrimmedString(value: unknown, path: string): string { + if (typeof value !== "string" || value.length === 0 || value.trim() !== value) { + fail(`"${path}" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(value)})`); + } + return value; +} + +function readOptionalDescription(value: unknown, path: string): string | undefined { + if (value === undefined) return undefined; + if (typeof value !== "string" || value.trim().length === 0) { + fail(`"${path}" must be a non-empty string when present (got ${describeJsonValue(value)})`); + } + return value; +} + +function readSecretFreeObject(value: unknown, path: string): Record { + if (value === undefined) return {}; + if (!isPlainObject(value)) { + fail(`"${path}" must be an object (got ${describeJsonValue(value)})`); + } + const secretLikeKey = findSecretLikeConfigKey(value, ""); + if (secretLikeKey) { + fail( + `"${path}" key "${secretLikeKey}" looks secret-bearing; credentials are delivered to managed instances as process environment variables (the provider's documented env fallback), never in the managed-config document`, + ); + } + return { ...value }; +} + +/** + * Parse the `environments` section: at most one sandbox entry (no `driver`, + * or `driver: "sandbox"`) plus any number of `driver: "plugin"` entries. + * Names are unique across the section, and plugin entries are unique by + * their `pluginKey`–`driverKey` pair. + */ +function parseManagedEnvironmentsSection( + raw: unknown, + autoInstall: readonly string[], +): { sandbox: ManagedEnvironmentSpec[]; plugin: ManagedPluginEnvironmentSpec[] } { + const sandbox: ManagedEnvironmentSpec[] = []; + const plugin: ManagedPluginEnvironmentSpec[] = []; + if (raw === undefined) return { sandbox, plugin }; + if (!Array.isArray(raw)) { + fail(`"environments" must be an array of environment objects (got ${describeJsonValue(raw)})`); + } + const names = new Set(); + const pluginDrivers = new Set(); + for (const [index, entry] of raw.entries()) { + const at = `environments[${index}]`; + if (!isPlainObject(entry)) { + fail(`"${at}" must be an object (got ${describeJsonValue(entry)})`); + } + // Only an absent `driver` means the sandbox entry; an explicit `null` is malformed. + const driver = entry.driver === undefined ? "sandbox" : entry.driver; + if (!(MANAGED_ENVIRONMENT_DRIVERS as readonly unknown[]).includes(driver)) { + fail( + `"${at}.driver" must be one of ${MANAGED_ENVIRONMENT_DRIVERS.map((value) => JSON.stringify(value)).join(", ")} when present (got ${describeJsonValue(entry.driver)})`, + ); + } + const allowedKeys: readonly string[] = driver === "plugin" ? PLUGIN_ENTRY_KEYS : SANDBOX_ENTRY_KEYS; + for (const key of Object.keys(entry)) { + if (!allowedKeys.includes(key)) { + fail(`"${at}" has unknown key "${key}" (allowed for a ${driver} entry: ${allowedKeys.join(", ")})`); + } + } + const name = requireTrimmedString(entry.name, `${at}.name`); + if (names.has(name)) { + fail(`"${at}.name" duplicates another entry's name "${name}"; environment names are unique per instance`); + } + names.add(name); + const description = readOptionalDescription(entry.description, `${at}.description`); + + if (driver === "sandbox") { + // The DB enforces at most ONE Paperclip-managed sandbox row per instance + // (partial unique index `environments_managed_sandbox_idx`); every + // sandbox entry provisions that row, so a second one can never be + // satisfied. + if (sandbox.length > 0) { + fail( + `"environments" supports at most one sandbox entry: each sandbox entry provisions the single Paperclip-managed sandbox environment (DB invariant environments_managed_sandbox_idx)`, + ); + } + const provider = requireTrimmedString(entry.provider, `${at}.provider`); + // Coherence: on a managed instance the control plane is the only plugin + // install path, so an environment whose provider plugin is not + // auto-installed could never serve a lease. Catch the skew at parse time. + if (!autoInstall.includes(provider)) { + fail( + `"${at}.provider" is "${provider}", which is not in "plugins.autoInstall"; a managed environment requires its provider plugin to be provisioned`, + ); + } + if (isPlainObject(entry.config) && entry.config.provider !== undefined) { + fail(`"${at}.config" must not set "provider"; it is forced from the entry's provider key`); + } + const config = readSecretFreeObject(entry.config, `${at}.config`); + sandbox.push( + Object.freeze({ + name, + ...(description !== undefined ? { description } : {}), + provider, + config: Object.freeze(config), + }) as ManagedEnvironmentSpec, + ); + continue; + } + + // `pluginKey` is the plugin's manifest id (what the row's config stores), + // not a `plugins.autoInstall` catalog key, so this pure parser cannot map + // it. `applyManagedEnvironments` refuses startup when no auto-installed + // catalog entry installs this manifest id. + const pluginKey = requireTrimmedString(entry.pluginKey, `${at}.pluginKey`); + const driverKey = requireTrimmedString(entry.driverKey, `${at}.driverKey`); + if (!PLUGIN_DRIVER_KEY_PATTERN.test(driverKey)) { + fail( + `"${at}.driverKey" must start with a lowercase letter or digit and contain only lowercase letters, digits, dots, hyphens, or underscores (got ${describeJsonValue(driverKey)})`, + ); + } + const pair = `${pluginKey}:${driverKey}`; + if (pluginDrivers.has(pair)) { + fail(`"${at}" duplicates another plugin entry's pluginKey and driverKey ("${pair}"); each plugin environment driver is declared once`); + } + pluginDrivers.add(pair); + const driverConfig = readSecretFreeObject(entry.driverConfig, `${at}.driverConfig`); + plugin.push( + Object.freeze({ + driver: "plugin", + name, + ...(description !== undefined ? { description } : {}), + pluginKey, + driverKey, + driverConfig: Object.freeze(driverConfig), + }) as ManagedPluginEnvironmentSpec, + ); + } + return { sandbox, plugin }; +} + /** * Parse `PAPERCLIP_MANAGED_CONFIG` from a raw env map. Returns null only when * the variable is absent (self-hosted). Throws with a precise error on a @@ -320,79 +519,10 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon // absence is not a dropped security control — it simply declares no managed // environments. When present, the section is validated fail-closed like // everything else. - const environmentSpecs: ManagedEnvironmentSpec[] = []; - if (doc.environments !== undefined) { - if (!Array.isArray(doc.environments)) { - fail(`"environments" must be an array of environment objects (got ${describeJsonValue(doc.environments)})`); - } - // The DB enforces at most ONE Paperclip-managed sandbox row per instance - // (partial unique index `environments_managed_sandbox_idx`); every entry - // here provisions that row, so a longer list can never be satisfied. - if (doc.environments.length > 1) { - fail( - `"environments" supports at most one entry: each entry provisions the single Paperclip-managed sandbox environment (DB invariant environments_managed_sandbox_idx)`, - ); - } - for (const [index, entry] of doc.environments.entries()) { - if (!isPlainObject(entry)) { - fail(`"environments[${index}]" must be an object (got ${describeJsonValue(entry)})`); - } - const allowedEntryKeys = new Set(["name", "description", "provider", "config"]); - for (const key of Object.keys(entry)) { - if (!allowedEntryKeys.has(key)) { - fail(`"environments[${index}]" has unknown key "${key}" (allowed: name, description, provider, config)`); - } - } - if (typeof entry.name !== "string" || entry.name.length === 0 || entry.name.trim() !== entry.name) { - fail( - `"environments[${index}].name" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.name)})`, - ); - } - if ( - entry.description !== undefined - && (typeof entry.description !== "string" || entry.description.trim().length === 0) - ) { - fail(`"environments[${index}].description" must be a non-empty string when present (got ${describeJsonValue(entry.description)})`); - } - if (typeof entry.provider !== "string" || entry.provider.length === 0 || entry.provider.trim() !== entry.provider) { - fail( - `"environments[${index}].provider" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.provider)})`, - ); - } - // Coherence: on a managed instance the control plane is the only plugin - // install path, so an environment whose provider plugin is not - // auto-installed could never serve a lease. Catch the skew at parse time. - if (!autoInstall.includes(entry.provider)) { - fail( - `"environments[${index}].provider" is "${entry.provider}", which is not in "plugins.autoInstall"; a managed environment requires its provider plugin to be provisioned`, - ); - } - const config: Record = {}; - if (entry.config !== undefined) { - if (!isPlainObject(entry.config)) { - fail(`"environments[${index}].config" must be an object (got ${describeJsonValue(entry.config)})`); - } - if (entry.config.provider !== undefined) { - fail(`"environments[${index}].config" must not set "provider"; it is forced from the entry's provider key`); - } - const secretLikeKey = findSecretLikeConfigKey(entry.config, ""); - if (secretLikeKey) { - fail( - `"environments[${index}].config" key "${secretLikeKey}" looks secret-bearing; credentials are delivered to managed instances as process environment variables (the provider's documented env fallback), never in the managed-config document`, - ); - } - Object.assign(config, entry.config); - } - environmentSpecs.push( - Object.freeze({ - name: entry.name, - ...(entry.description !== undefined ? { description: entry.description } : {}), - provider: entry.provider, - config: Object.freeze(config), - }) as ManagedEnvironmentSpec, - ); - } - } + const { sandbox: environmentSpecs, plugin: pluginEnvironmentSpecs } = parseManagedEnvironmentsSection( + doc.environments, + autoInstall, + ); return Object.freeze({ v: SUPPORTED_MANAGED_CONFIG_VERSION, @@ -405,6 +535,7 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon ...(enabledCompanies ? { enabledCompanies: Object.freeze(enabledCompanies) } : {}), }), environments: Object.freeze(environmentSpecs), + pluginEnvironments: Object.freeze(pluginEnvironmentSpecs), }) as ManagedInstanceConfig; } diff --git a/server/src/services/managed-environments-plugin-entries.test.ts b/server/src/services/managed-environments-plugin-entries.test.ts new file mode 100644 index 00000000000..e9241b6728f --- /dev/null +++ b/server/src/services/managed-environments-plugin-entries.test.ts @@ -0,0 +1,385 @@ +import { describe, expect, it, vi } from "vitest"; +import { unprocessable } from "../errors.js"; +import { applyManagedEnvironments } from "./managed-environments.js"; +import { + PLUGIN_KEY, + environmentsSeam, + instanceSettingsSeam, + noDb, + parsedConfig, + pluginEnvironmentRow, + readyDriverResolver, + reconciliationResult, + recoveryHandle, + tick, + type InstanceSettingsSeam, + type RecoveryHandle, +} from "./managed-environments.test-support.js"; + +describe("applyManagedEnvironments plugin entries", () => { + const pluginEntry = (overrides: Record = {}) => ({ + name: "Remote Host", + description: "Runs the agent on a remote host", + driver: "plugin", + pluginKey: PLUGIN_KEY, + driverKey: "tasks", + driverConfig: {}, + ...overrides, + }); + const configWith = (environments: unknown[], features: Record = {}) => parsedConfig({ + plugins: { autoInstall: ["daytona", "remote-host"] }, + features, + environments, + }); + const readyPlugin = (status = "ready", declaresDriver = true) => vi.fn(async () => ({ + plugin: { id: "remote-host-plugin", pluginKey: PLUGIN_KEY, status }, + declaresDriver, + })); + const workerManager = (running = true, handle?: RecoveryHandle) => ({ + isRunning: vi.fn(() => running), + call: vi.fn(), + getWorker: vi.fn(() => handle), + }); + + it("ensures a plugin entry with the plugin validator's worker manager", async () => { + const environments = environmentsSeam(); + const result = await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: workerManager(), + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + + expect(environments.ensureManagedPluginEnvironment).toHaveBeenCalledWith({ + name: "Remote Host", + description: "Runs the agent on a remote host", + pluginKey: PLUGIN_KEY, + driverKey: "tasks", + driverConfig: {}, + workerManager: { isRunning: expect.any(Function), call: expect.any(Function) }, + }); + expect(result).toMatchObject({ ensured: 1, added: 1, failed: 0 }); + expect(result?.pluginOutcomes).toEqual([{ + environmentId: "plugin-env", + name: "Remote Host", + pluginKey: PLUGIN_KEY, + driverKey: "tasks", + action: "added", + }]); + expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); + }); + + it("runs the sandbox entry first", async () => { + const order: string[] = []; + const environments = environmentsSeam({ + ensureManagedSandboxEnvironment: vi.fn(async () => { + order.push("sandbox"); + return reconciliationResult() as never; + }), + ensureManagedPluginEnvironment: vi.fn(async () => { + order.push("plugin"); + return { environment: pluginEnvironmentRow(), action: "added" as const }; + }), + }); + await applyManagedEnvironments(noDb, configWith([{ name: "Daytona", provider: "daytona" }, pluginEntry()]), { + env: {}, + workerManager: workerManager(), + environments, + instanceSettings: instanceSettingsSeam(), + resolveSandboxProviderDriver: readyDriverResolver(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + expect(order).toEqual(["sandbox", "plugin"]); + }); + + it("refuses startup when no auto-installed catalog entry installs the entry's plugin", async () => { + const environments = environmentsSeam(); + await expect(applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: workerManager(), + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: ["paperclip.daytona-sandbox-provider"], + })).rejects.toThrow(/uses plugin "example.remote-host-plugin", which no "plugins.autoInstall" catalog entry installs/); + expect(environments.ensureManagedPluginEnvironment).not.toHaveBeenCalled(); + }); + + it("resolves the auto-installed manifest ids from the catalog when the caller passes none", async () => { + // "remote-host" is not in this build's bundled catalog and the test root + // has no distribution catalog, so the catalog lookup refuses. + await expect(applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: { PAPERCLIP_BUNDLED_PLUGIN_ROOT: "/nonexistent-paperclip-catalog" }, + workerManager: workerManager(), + environments: environmentsSeam(), + instanceSettings: instanceSettingsSeam(), + })).rejects.toThrow(/auto-install key "remote-host" is not in the bundled catalog/); + }); + + it("does not claim the sandbox slot, so a forced execution mode can coexist with plugin-only entries", async () => { + await expect(applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: { PAPERCLIP_EXECUTION_MODE: "kubernetes" }, + workerManager: workerManager(), + environments: environmentsSeam(), + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + })).resolves.toMatchObject({ ensured: 1 }); + }); + + it("archives the row of a plugin that is down and reactivates it on worker recovery", async () => { + const { handle, fireReady } = recoveryHandle(); + const environments = environmentsSeam({ + archiveManagedPluginEnvironment: vi.fn().mockResolvedValue(pluginEnvironmentRow({ status: "archived" })), + }); + const manager = workerManager(false, handle); + const result = await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: manager, + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + + expect(result).toMatchObject({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedPluginEnvironment).toHaveBeenCalledWith({ pluginKey: PLUGIN_KEY, driverKey: "tasks" }); + expect(environments.ensureManagedPluginEnvironment).not.toHaveBeenCalled(); + + manager.isRunning.mockReturnValue(true); + fireReady(); + fireReady(); + await tick(); + + expect(environments.ensureManagedPluginEnvironment).toHaveBeenCalledTimes(1); + expect(handle.off).toHaveBeenCalledTimes(1); + }); + + it("keeps recovery armed after a worker failure and stops after one success", async () => { + const { handle, fireReady } = recoveryHandle(); + const ensureManagedPluginEnvironment = vi.fn() + .mockRejectedValueOnce(new Error("worker crashed during environmentValidateConfig")) + .mockResolvedValue({ environment: pluginEnvironmentRow(), action: "updated" }); + const environments = environmentsSeam({ ensureManagedPluginEnvironment }); + const manager = workerManager(false, handle); + await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: manager, + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + manager.isRunning.mockReturnValue(true); + + fireReady(); + await tick(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(1); + expect(handle.off).not.toHaveBeenCalled(); + + fireReady(); + await tick(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(2); + expect(handle.off).toHaveBeenCalledTimes(1); + + fireReady(); + await tick(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(2); + }); + + it("queues one more attempt when the worker becomes ready during an attempt", async () => { + const { handle, fireReady } = recoveryHandle(); + let failFirst!: (err: Error) => void; + const ensureManagedPluginEnvironment = vi.fn() + .mockImplementationOnce(() => new Promise((_resolve, reject) => { failFirst = reject; })) + .mockResolvedValue({ environment: pluginEnvironmentRow(), action: "updated" }); + const manager = workerManager(false, handle); + await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: manager, + environments: environmentsSeam({ ensureManagedPluginEnvironment }), + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + manager.isRunning.mockReturnValue(true); + + fireReady(); + fireReady(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(1); + failFirst(new Error("worker crashed")); + await tick(); + await tick(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(2); + expect(handle.off).toHaveBeenCalledTimes(1); + }); + + it("stops recovery when the plugin permanently rejects the config", async () => { + const { handle, fireReady } = recoveryHandle(); + const ensureManagedPluginEnvironment = vi.fn().mockRejectedValue(unprocessable("driverConfig.region is not supported")); + const manager = workerManager(false, handle); + await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: manager, + environments: environmentsSeam({ ensureManagedPluginEnvironment }), + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + manager.isRunning.mockReturnValue(true); + + fireReady(); + await tick(); + fireReady(); + await tick(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(1); + expect(handle.off).toHaveBeenCalledTimes(1); + }); + + it("archives and arms recovery when the worker dies during the boot ensure", async () => { + const { handle, fireReady } = recoveryHandle(); + const manager = workerManager(true, handle); + const ensureManagedPluginEnvironment = vi.fn() + .mockImplementationOnce(async () => { + manager.isRunning.mockReturnValue(false); + throw new Error("worker crashed during environmentValidateConfig"); + }) + .mockResolvedValue({ environment: pluginEnvironmentRow(), action: "updated" }); + const environments = environmentsSeam({ ensureManagedPluginEnvironment }); + const result = await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: manager, + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + await tick(); + + expect(result).toMatchObject({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedPluginEnvironment).toHaveBeenCalledWith({ pluginKey: PLUGIN_KEY, driverKey: "tasks" }); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(1); + + manager.isRunning.mockReturnValue(true); + fireReady(); + await tick(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(2); + expect(handle.off).toHaveBeenCalledTimes(1); + }); + + it("keeps the row and retries when the ensure fails while the worker keeps running", async () => { + const { handle, fireReady } = recoveryHandle(); + const ensureManagedPluginEnvironment = vi.fn() + .mockRejectedValueOnce(new Error("environmentValidateConfig timed out")) + .mockRejectedValueOnce(new Error("environmentValidateConfig timed out")) + .mockResolvedValue({ environment: pluginEnvironmentRow(), action: "unchanged" }); + const environments = environmentsSeam({ ensureManagedPluginEnvironment }); + const result = await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: workerManager(true, handle), + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + await tick(); + + expect(result).toMatchObject({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedPluginEnvironment).not.toHaveBeenCalled(); + // The immediate retry also failed; the next `ready` succeeds. + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(2); + fireReady(); + await tick(); + expect(ensureManagedPluginEnvironment).toHaveBeenCalledTimes(3); + expect(handle.off).toHaveBeenCalledTimes(1); + }); + + it("neither archives nor retries when the boot ensure gets a permanent config rejection", async () => { + const { handle } = recoveryHandle(); + const environments = environmentsSeam({ + ensureManagedPluginEnvironment: vi.fn().mockRejectedValue(unprocessable("driverConfig.region is not supported")), + }); + const manager = workerManager(true, handle); + const result = await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: manager, + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedPluginEnvironment).not.toHaveBeenCalled(); + expect(manager.getWorker).not.toHaveBeenCalled(); + }); + + it("treats a missing plugin, a plugin without the driver, or a worker manager without call as unavailable", async () => { + for (const [resolveEnvironmentPlugin, manager] of [ + [vi.fn(async () => null), workerManager()], + [readyPlugin("ready", false), workerManager()], + [readyPlugin("error"), workerManager()], + [readyPlugin(), { isRunning: vi.fn(() => true), getWorker: vi.fn(() => undefined) }], + ] as const) { + const environments = environmentsSeam(); + const result = await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: manager, + environments, + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin, + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedPluginEnvironment).toHaveBeenCalledTimes(1); + expect(environments.ensureManagedPluginEnvironment).not.toHaveBeenCalled(); + expect(manager.getWorker).not.toHaveBeenCalled(); + } + }); + + it("never archives a row whose entry was removed from the document", async () => { + const environments = environmentsSeam(); + expect(await applyManagedEnvironments(noDb, configWith([]), { + env: {}, + workerManager: workerManager(), + environments, + instanceSettings: instanceSettingsSeam(), + })).toBeNull(); + expect(environments.archiveManagedPluginEnvironment).not.toHaveBeenCalled(); + }); + + it("never changes the instance default, and the sandbox stamp never replaces a plugin default", async () => { + const settings = instanceSettingsSeam({ + get: vi.fn().mockResolvedValue({ defaultEnvironmentId: "plugin-env" }) as InstanceSettingsSeam["get"], + }); + await applyManagedEnvironments(noDb, configWith( + [{ name: "Daytona", provider: "daytona" }, pluginEntry()], + { enableManagedSandboxOnly: true }, + ), { + env: {}, + workerManager: workerManager(), + instanceSettings: settings, + environments: environmentsSeam({ getById: vi.fn().mockResolvedValue(pluginEnvironmentRow()) }), + resolveSandboxProviderDriver: readyDriverResolver(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + expect(settings.update).not.toHaveBeenCalled(); + }); + + it("is fail-safe: an ensure failure is logged and counted, not thrown", async () => { + const result = await applyManagedEnvironments(noDb, configWith([pluginEntry()]), { + env: {}, + workerManager: workerManager(), + environments: environmentsSeam({ + ensureManagedPluginEnvironment: vi.fn().mockRejectedValue(new Error("validator refused")), + }), + instanceSettings: instanceSettingsSeam(), + resolveEnvironmentPlugin: readyPlugin(), + autoInstalledPluginKeys: [PLUGIN_KEY], + }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); + }); +}); diff --git a/server/src/services/managed-environments.test-support.ts b/server/src/services/managed-environments.test-support.ts new file mode 100644 index 00000000000..a3019692692 --- /dev/null +++ b/server/src/services/managed-environments.test-support.ts @@ -0,0 +1,139 @@ +/** + * Shared fixtures and service seams for the `applyManagedEnvironments` tests + * (`managed-environments.test.ts` for the sandbox entry and instance default, + * `managed-environments-plugin-entries.test.ts` for plugin entries). + */ +import { vi } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { MANAGED_CONFIG_ENV_KEY, parseManagedConfigEnv } from "./managed-config.js"; +import type { ApplyManagedEnvironmentsOptions } from "./managed-environments.js"; + +export const noDb = null as unknown as Db; + +export function parsedConfig(overrides: Record = {}) { + const config = parseManagedConfigEnv({ + [MANAGED_CONFIG_ENV_KEY]: JSON.stringify({ + v: 1, + mode: "cloud", + catalogVersion: "2026.720.0", + features: {}, + plugins: { autoInstall: ["daytona"] }, + ...overrides, + }), + }); + if (!config) throw new Error("expected a parsed managed config"); + return config; +} + +export function environmentRow(overrides: Record = {}) { + const now = new Date(); + return { + id: "env-1", + name: "Daytona", + description: null, + driver: "sandbox", + status: "active", + config: {}, + envVars: {}, + metadata: {}, + createdAt: now, + updatedAt: now, + ...overrides, + }; +} + +export function reconciliationResult( + overrides: Record = {}, +) { + return { + environment: environmentRow(), + action: "added" as const, + stockStatus: "missing" as const, + updateAvailable: false, + stockHash: "sha256:stock", + ...overrides, + }; +} + +export function readyDriverResolver(status = "ready") { + return vi.fn(async () => ({ + plugin: { id: "plugin-1", pluginKey: "sandbox-providers/daytona", status }, + })); +} + +export function runningWorkerManager(running = true) { + return { isRunning: vi.fn(() => running), getWorker: vi.fn(() => undefined) }; +} + +export type WorkerManagerSeam = NonNullable; +export type RecoveryHandle = NonNullable>; + +/** A worker handle fake that records `ready` listeners so tests can fire them. */ +export function recoveryHandle() { + const readyListeners: Array<(payload: { pluginId: string }) => void> = []; + const handle: RecoveryHandle = { + on: vi.fn((_event, listener) => { + readyListeners.push(listener); + }), + off: vi.fn((_event, listener) => { + const at = readyListeners.indexOf(listener); + if (at !== -1) readyListeners.splice(at, 1); + }), + }; + return { + handle, + fireReady: () => { + for (const listener of [...readyListeners]) listener({ pluginId: "plugin-1" }); + }, + }; +} + +/** Lets the fire-and-forget recovery ensure settle. */ +export function tick() { + return new Promise((resolve) => setImmediate(resolve)); +} + +export type EnvironmentsSeam = NonNullable; +export type InstanceSettingsSeam = NonNullable; + +export function environmentsSeam(overrides: Partial = {}): EnvironmentsSeam { + return { + ensureManagedSandboxEnvironment: + overrides.ensureManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(reconciliationResult()), + archiveManagedSandboxEnvironment: + overrides.archiveManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null), + getById: overrides.getById ?? vi.fn().mockResolvedValue(null), + update: overrides.update ?? vi.fn().mockResolvedValue(environmentRow()), + findManagedSandboxEnvironment: + overrides.findManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null), + ensureManagedPluginEnvironment: + overrides.ensureManagedPluginEnvironment + ?? vi.fn().mockResolvedValue({ environment: pluginEnvironmentRow(), action: "added" }), + archiveManagedPluginEnvironment: + overrides.archiveManagedPluginEnvironment ?? vi.fn().mockResolvedValue(null), + }; +} + +export const PLUGIN_KEY = "example.remote-host-plugin"; + +export function pluginEnvironmentRow(overrides: Record = {}) { + return environmentRow({ + id: "plugin-env", + name: "Remote Host", + driver: "plugin", + config: { pluginKey: PLUGIN_KEY, driverKey: "tasks", driverConfig: {} }, + metadata: { managedByPaperclip: true, managedPluginEnvironment: `${PLUGIN_KEY}:tasks` }, + ...overrides, + }) as never; +} + +export function instanceSettingsSeam(overrides: Partial = {}): InstanceSettingsSeam { + return { + get: + overrides.get ?? + (vi.fn().mockResolvedValue({ defaultEnvironmentId: null }) as InstanceSettingsSeam["get"]), + update: + overrides.update ?? + (vi.fn().mockResolvedValue({ defaultEnvironmentId: "env-1" }) as InstanceSettingsSeam["update"]), + }; +} diff --git a/server/src/services/managed-environments.test.ts b/server/src/services/managed-environments.test.ts index 2a38e6c076d..bf8ec41e6bc 100644 --- a/server/src/services/managed-environments.test.ts +++ b/server/src/services/managed-environments.test.ts @@ -1,122 +1,18 @@ import { describe, expect, it, vi } from "vitest"; -import type { Db } from "@paperclipai/db"; -import { MANAGED_CONFIG_ENV_KEY, parseManagedConfigEnv } from "./managed-config.js"; +import { applyManagedEnvironments } from "./managed-environments.js"; import { - applyManagedEnvironments, - type ApplyManagedEnvironmentsOptions, -} from "./managed-environments.js"; - -const noDb = null as unknown as Db; - -function parsedConfig(overrides: Record = {}) { - const config = parseManagedConfigEnv({ - [MANAGED_CONFIG_ENV_KEY]: JSON.stringify({ - v: 1, - mode: "cloud", - catalogVersion: "2026.720.0", - features: {}, - plugins: { autoInstall: ["daytona"] }, - ...overrides, - }), - }); - if (!config) throw new Error("expected a parsed managed config"); - return config; -} - -function environmentRow(overrides: Record = {}) { - const now = new Date(); - return { - id: "env-1", - name: "Daytona", - description: null, - driver: "sandbox", - status: "active", - config: {}, - envVars: {}, - metadata: {}, - createdAt: now, - updatedAt: now, - ...overrides, - }; -} - -function reconciliationResult( - overrides: Record = {}, -) { - return { - environment: environmentRow(), - action: "added" as const, - stockStatus: "missing" as const, - updateAvailable: false, - stockHash: "sha256:stock", - ...overrides, - }; -} - -function readyDriverResolver(status = "ready") { - return vi.fn(async () => ({ - plugin: { id: "plugin-1", pluginKey: "sandbox-providers/daytona", status }, - })); -} - -function runningWorkerManager(running = true) { - return { isRunning: vi.fn(() => running), getWorker: vi.fn(() => undefined) }; -} - -type WorkerManagerSeam = NonNullable; -type RecoveryHandle = NonNullable>; - -/** A worker handle fake that records `ready` listeners so tests can fire them. */ -function recoveryHandle() { - const readyListeners: Array<(payload: { pluginId: string }) => void> = []; - const handle: RecoveryHandle = { - on: vi.fn((_event, listener) => { - readyListeners.push(listener); - }), - off: vi.fn((_event, listener) => { - const at = readyListeners.indexOf(listener); - if (at !== -1) readyListeners.splice(at, 1); - }), - }; - return { - handle, - fireReady: () => { - for (const listener of [...readyListeners]) listener({ pluginId: "plugin-1" }); - }, - }; -} - -/** Lets the fire-and-forget recovery ensure settle. */ -function tick() { - return new Promise((resolve) => setImmediate(resolve)); -} - -type EnvironmentsSeam = NonNullable; -type InstanceSettingsSeam = NonNullable; - -function environmentsSeam(overrides: Partial = {}): EnvironmentsSeam { - return { - ensureManagedSandboxEnvironment: - overrides.ensureManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(reconciliationResult()), - archiveManagedSandboxEnvironment: - overrides.archiveManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null), - getById: overrides.getById ?? vi.fn().mockResolvedValue(null), - update: overrides.update ?? vi.fn().mockResolvedValue(environmentRow()), - findManagedSandboxEnvironment: - overrides.findManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null), - }; -} - -function instanceSettingsSeam(overrides: Partial = {}): InstanceSettingsSeam { - return { - get: - overrides.get ?? - (vi.fn().mockResolvedValue({ defaultEnvironmentId: null }) as InstanceSettingsSeam["get"]), - update: - overrides.update ?? - (vi.fn().mockResolvedValue({ defaultEnvironmentId: "env-1" }) as InstanceSettingsSeam["update"]), - }; -} + environmentRow, + environmentsSeam, + instanceSettingsSeam, + noDb, + parsedConfig, + readyDriverResolver, + reconciliationResult, + recoveryHandle, + runningWorkerManager, + tick, + type InstanceSettingsSeam, +} from "./managed-environments.test-support.js"; describe("applyManagedEnvironments", () => { it("no-ops for self-hosted instances and for documents without environments", async () => { diff --git a/server/src/services/managed-environments.ts b/server/src/services/managed-environments.ts index d201728cb7a..36d9c515c3a 100644 --- a/server/src/services/managed-environments.ts +++ b/server/src/services/managed-environments.ts @@ -43,9 +43,17 @@ * sandbox provider falls back to its documented process environment variable * (e.g. `DAYTONA_API_KEY`) when `config` omits the key, so the deployment * delivers secrets as env vars and the managed document stays secret-free. + * + * `driver: "plugin"` entries take the same readiness path after the sandbox + * entry: await `pluginsReady`, require the plugin `ready`, declaring the + * entry's driver, and running a worker; else archive the entry's row and + * re-ensure it on the worker's `ready`. Each entry owns one row (see + * `managed-plugin-environments.ts`). Plugin entries never touch the instance + * default. */ import type { Db } from "@paperclipai/db"; +import { HttpError } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { environmentService } from "./environments.js"; import type { ManagedSandboxEnvironmentReconcileAction } from "./environments.js"; @@ -54,7 +62,10 @@ import type { ManagedInstanceConfig } from "./managed-config.js"; import type { ManagedResourceStockStatus } from "./managed-resource-drift.js"; import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js"; import { resolvePluginSandboxProviderDriverByKey } from "./plugin-environment-driver.js"; +import { pluginRegistryService } from "./plugin-registry.js"; +import { resolveBundledCatalogRoot, resolveBundledPluginInstalls } from "./bundled-plugins.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; +import type { ManagedPluginEnvironmentReconcileAction } from "./managed-plugin-environments.js"; export interface ApplyManagedEnvironmentsOptions { env?: Record; @@ -75,8 +86,11 @@ export interface ApplyManagedEnvironmentsOptions { * subscription used for post-archive recovery: a worker in crash-restart * backoff re-emits `ready` when the manager respawns it, and the listener * re-ensures the archived environment (see `applyManagedEnvironments`). + * + * Plugin entries also need `call`, to run the plugin's own config + * validation; without it every plugin entry is treated as unavailable. */ - workerManager?: Pick & { + workerManager?: Pick & Partial> & { getWorker(pluginId: string): | { on(event: "ready", listener: (payload: { pluginId: string }) => void): void; @@ -92,6 +106,8 @@ export interface ApplyManagedEnvironmentsOptions { | "getById" | "update" | "findManagedSandboxEnvironment" + | "ensureManagedPluginEnvironment" + | "archiveManagedPluginEnvironment" >; /** Test seam: overrides the instance-settings service built from `db`. */ instanceSettings?: Pick, "get" | "update">; @@ -100,6 +116,32 @@ export interface ApplyManagedEnvironmentsOptions { db: Db; driverKey: string; }) => Promise<{ plugin: { id: string; pluginKey: string; status: string } } | null>; + /** + * Test seam: overrides the lookup of a plugin entry's plugin. Returns null + * when the plugin is not installed; `declaresDriver` is false when its + * manifest does not declare the entry's environment driver. + */ + resolveEnvironmentPlugin?: (input: { + db: Db; + pluginKey: string; + driverKey: string; + }) => Promise<{ plugin: { id: string; pluginKey: string; status: string }; declaresDriver: boolean } | null>; + /** + * Manifest ids (`pluginKey`) that the `plugins.autoInstall` catalog keys + * resolve to: `app.locals.managedBundledPluginKeys`, from + * `resolveBundledPluginInstalls` over the bundled and distribution + * catalogs. A plugin entry whose `pluginKey` is not in this list refuses + * startup. When omitted, the list is resolved here the same way. + */ + autoInstalledPluginKeys?: readonly string[]; +} + +export interface ManagedPluginEnvironmentReconciliationOutcome { + environmentId: string; + name: string; + pluginKey: string; + driverKey: string; + action: ManagedPluginEnvironmentReconcileAction; } export interface ManagedEnvironmentReconciliationOutcome { @@ -123,6 +165,28 @@ export interface ApplyManagedEnvironmentsResult { /** Preserve-only reconciliation never creates replacement backups. */ backedUp: number; outcomes: ManagedEnvironmentReconciliationOutcome[]; + pluginOutcomes: ManagedPluginEnvironmentReconciliationOutcome[]; +} + +/** + * The plugin refused the entry's config (`HttpError` 422 from + * `normalizeEnvironmentConfig` or `validatePluginEnvironmentDriverConfig`). + * Retrying cannot help until the config changes. Every other failure, such + * as a worker that crashed mid-call, is treated as transient. + */ +function isPermanentConfigRejection(err: unknown): boolean { + return err instanceof HttpError && err.status === 422; +} + +async function resolveEnvironmentPluginByKey(input: { db: Db; pluginKey: string; driverKey: string }) { + const plugin = await pluginRegistryService(input.db).getByKey(input.pluginKey); + if (!plugin) return null; + return { + plugin: { id: plugin.id, pluginKey: plugin.pluginKey, status: plugin.status }, + declaresDriver: plugin.manifestJson.environmentDrivers?.some( + (driver) => driver.driverKey === input.driverKey && driver.kind !== "sandbox_provider", + ) ?? false, + }; } /** @@ -176,19 +240,38 @@ export async function applyManagedEnvironments( } } - if (managedConfig.environments.length === 0) return null; + const pluginSpecs = managedConfig.pluginEnvironments; + if (managedConfig.environments.length === 0 && pluginSpecs.length === 0) return null; // The forced-execution-mode bootstrap (`PAPERCLIP_EXECUTION_MODE=kubernetes`) // and this one both own the single Paperclip-managed sandbox row // (`environments_managed_sandbox_idx`). Configuring both is contradictory; - // refuse startup rather than let bootstrap ordering pick a winner. + // refuse startup rather than let bootstrap ordering pick a winner. Plugin + // entries do not touch that slot. const env = opts.env ?? process.env; - if (parseExecutionPolicyBootstrapEnv(env)) { + if (managedConfig.environments.length > 0 && parseExecutionPolicyBootstrapEnv(env)) { throw new Error( `PAPERCLIP_EXECUTION_MODE and the PAPERCLIP_MANAGED_CONFIG "environments" section are mutually exclusive: both manage the single instance sandbox environment`, ); } + // Coherence, fail closed like the parser: a plugin entry names its plugin by + // manifest id, and only `plugins.autoInstall` provisions plugins on a + // managed instance, so some auto-installed catalog entry must install it. + if (pluginSpecs.length > 0) { + const autoInstalledPluginKeys = opts.autoInstalledPluginKeys ?? resolveBundledPluginInstalls( + managedConfig.plugins.autoInstall, + { catalogRoot: resolveBundledCatalogRoot(env), env, enforceCatalogRoot: true }, + ).map((install) => install.pluginKey); + for (const spec of pluginSpecs) { + if (!autoInstalledPluginKeys.includes(spec.pluginKey)) { + throw new Error( + `PAPERCLIP_MANAGED_CONFIG "environments" entry "${spec.name}" uses plugin "${spec.pluginKey}", which no "plugins.autoInstall" catalog entry installs; a managed plugin environment requires its plugin to be provisioned`, + ); + } + } + } + // The heartbeat resumes queued runs right after this bootstrap step, and // lease acquisition fails hard on a provider whose plugin is missing or not // ready. Wait for the bundled-plugin startup pass to finish, then refuse to @@ -266,10 +349,7 @@ export async function applyManagedEnvironments( // worker recovered between the gate check and the subscription. No handle // means no respawn is coming (activation never started a worker), so the // degraded-until-next-boot posture stands. - const scheduleRecoveryReactivation = ( - spec: ManagedInstanceConfig["environments"][number], - pluginId: string, - ): void => { + const scheduleRecoveryReactivation = (pluginId: string, reactivateEntry: () => Promise): void => { const manager = opts.workerManager; const handle = manager?.getWorker(pluginId); if (!handle) return; @@ -278,39 +358,41 @@ export async function applyManagedEnvironments( if (reactivated) return; reactivated = true; handle.off("ready", reactivate); - void environments - .ensureManagedSandboxEnvironment({ - name: spec.name, - description: spec.description, - provider: spec.provider, - config: { ...spec.config }, - stockVersion: managedConfig.catalogVersion, - platformFullyManaged: true, - }) - .then((result) => { - logger.info( - { - environmentId: result.environment.id, - name: spec.name, - provider: spec.provider, - action: result.action, - stockStatus: result.stockStatus, - updateAvailable: result.updateAvailable, - }, - "managed sandbox environment reactivated after provider worker recovery", - ); - }) - .catch((err: unknown) => { - logger.error( - { err, name: spec.name, provider: spec.provider }, - "failed to reactivate managed sandbox environment after provider worker recovery (degraded: environment unavailable until next boot)", - ); - }); + void reactivateEntry(); }; handle.on("ready", reactivate); if (manager?.isRunning(pluginId)) reactivate(); }; + const reactivateSandboxEntry = async (spec: ManagedInstanceConfig["environments"][number]): Promise => { + try { + const result = await environments.ensureManagedSandboxEnvironment({ + name: spec.name, + description: spec.description, + provider: spec.provider, + config: { ...spec.config }, + stockVersion: managedConfig.catalogVersion, + platformFullyManaged: true, + }); + logger.info( + { + environmentId: result.environment.id, + name: spec.name, + provider: spec.provider, + action: result.action, + stockStatus: result.stockStatus, + updateAvailable: result.updateAvailable, + }, + "managed sandbox environment reactivated after provider worker recovery", + ); + } catch (err) { + logger.error( + { err, name: spec.name, provider: spec.provider }, + "failed to reactivate managed sandbox environment after provider worker recovery (degraded: environment unavailable until next boot)", + ); + } + }; + let ensured = 0; let failed = 0; let added = 0; @@ -355,7 +437,7 @@ export async function applyManagedEnvironments( // manager restarts crashed workers, but nothing (re)installs a missing // or non-ready plugin at runtime. if (resolved && resolved.plugin.status === "ready") { - scheduleRecoveryReactivation(spec, resolved.plugin.id); + scheduleRecoveryReactivation(resolved.plugin.id, () => reactivateSandboxEntry(spec)); } continue; } @@ -407,6 +489,179 @@ export async function applyManagedEnvironments( ); } } + const resolvePlugin = opts.resolveEnvironmentPlugin ?? resolveEnvironmentPluginByKey; + const pluginOutcomes: ManagedPluginEnvironmentReconciliationOutcome[] = []; + + const ensurePluginEntry = ( + spec: ManagedInstanceConfig["pluginEnvironments"][number], + workerManager: Pick, + ) => + environments.ensureManagedPluginEnvironment({ + name: spec.name, + description: spec.description, + pluginKey: spec.pluginKey, + driverKey: spec.driverKey, + driverConfig: { ...spec.driverConfig }, + workerManager, + }); + + type PluginSpec = ManagedInstanceConfig["pluginEnvironments"][number]; + type PluginWorkerAccess = Pick; + + const archivePluginEntry = (spec: PluginSpec) => + environments + .archiveManagedPluginEnvironment({ pluginKey: spec.pluginKey, driverKey: spec.driverKey }) + .catch((archiveErr: unknown) => { + logger.error( + { err: archiveErr, name: spec.name, pluginKey: spec.pluginKey, driverKey: spec.driverKey }, + "failed to archive the managed plugin environment of an unavailable plugin", + ); + return null; + }); + + /** + * Re-ensure a plugin entry each time its worker reports `ready`, until one + * ensure succeeds or the plugin permanently rejects the entry's config. A + * worker that crashes during an attempt (for example inside + * `environmentValidateConfig`) leaves the listener armed, so its next + * respawn retries. A `ready` that arrives during an attempt queues one more + * attempt. No handle means no respawn is coming (activation never started a + * worker), so the degraded-until-next-boot posture stands. + */ + const schedulePluginRecovery = (spec: PluginSpec, pluginId: string, workerAccess: PluginWorkerAccess): void => { + const manager = opts.workerManager; + const handle = manager?.getWorker(pluginId); + if (!handle) return; + const logBase = { name: spec.name, pluginKey: spec.pluginKey, driverKey: spec.driverKey }; + let settled = false; + let inFlight = false; + let queued = false; + const settle = (): void => { + settled = true; + handle.off("ready", attempt); + }; + function attempt(): void { + if (settled) return; + if (inFlight) { + queued = true; + return; + } + inFlight = true; + void ensurePluginEntry(spec, workerAccess) + .then((result) => { + settle(); + logger.info( + { ...logBase, environmentId: result.environment.id, action: result.action }, + "managed plugin environment reactivated after plugin worker recovery", + ); + }) + .catch((err: unknown) => { + if (isPermanentConfigRejection(err)) { + settle(); + logger.error( + { err, ...logBase }, + "the plugin rejected the managed plugin environment's config; not retrying (degraded: environment unavailable until the config changes)", + ); + } else { + logger.warn( + { err, ...logBase }, + "failed to reactivate managed plugin environment; retrying when the plugin worker is ready again", + ); + } + }) + .finally(() => { + inFlight = false; + if (queued && !settled) { + queued = false; + attempt(); + } + }); + } + handle.on("ready", attempt); + if (manager?.isRunning(pluginId)) attempt(); + }; + + for (const spec of pluginSpecs) { + const logBase = { name: spec.name, pluginKey: spec.pluginKey, driverKey: spec.driverKey }; + let recoverable: { pluginId: string; workerAccess: PluginWorkerAccess } | null = null; + try { + const resolved = await resolvePlugin({ db, pluginKey: spec.pluginKey, driverKey: spec.driverKey }); + const manager = opts.workerManager; + const call = manager?.call; + // Only a `ready` record that declares the driver can recover without + // another boot: the worker manager restarts crashed workers, but nothing + // (re)installs a missing or non-ready plugin at runtime. + if (resolved && resolved.plugin.status === "ready" && resolved.declaresDriver && manager && call) { + recoverable = { + pluginId: resolved.plugin.id, + workerAccess: { isRunning: manager.isRunning.bind(manager), call: call.bind(manager) }, + }; + } + const workerRunning = resolved != null && (manager?.isRunning(resolved.plugin.id) ?? false); + if (!recoverable || !workerRunning) { + failed += 1; + const archived = await archivePluginEntry(spec); + logger.error( + { + ...logBase, + pluginStatus: resolved?.plugin.status ?? null, + declaresDriver: resolved?.declaresDriver ?? null, + workerRunning, + archivedEnvironmentId: archived?.id ?? null, + }, + "managed plugin environment's plugin is not installed, ready, declaring its driver, and running; skipping ensure and archiving any previously provisioned row (degraded: environment unavailable)", + ); + if (recoverable) schedulePluginRecovery(spec, recoverable.pluginId, recoverable.workerAccess); + continue; + } + const result = await ensurePluginEntry(spec, recoverable.workerAccess); + ensured += 1; + if (result.action === "added") added += 1; + else if (result.action === "unchanged") unchanged += 1; + else updated += 1; + pluginOutcomes.push({ + environmentId: result.environment.id, + name: result.environment.name, + pluginKey: spec.pluginKey, + driverKey: spec.driverKey, + action: result.action, + }); + logger.info( + { ...logBase, environmentId: result.environment.id, action: result.action }, + "managed plugin environment reconciled", + ); + } catch (err) { + failed += 1; + if (recoverable && !isPermanentConfigRejection(err)) { + if (!recoverable.workerAccess.isRunning(recoverable.pluginId)) { + // The worker died after the readiness check (for example inside + // `environmentValidateConfig`): treat it like a plugin that is + // down, so the row is not left active, and retry on its `ready`. + const archived = await archivePluginEntry(spec); + logger.error( + { err, ...logBase, archivedEnvironmentId: archived?.id ?? null }, + "managed plugin environment's worker stopped during ensure; archived any previously provisioned row and retrying when the plugin worker is ready again (degraded: environment unavailable)", + ); + } else { + // The worker is still running (for example the call timed out). + // Keep any previously provisioned row as it is, so agents keep a + // working environment; retry once now and on the worker's next + // `ready`, and otherwise at the next boot. + logger.error( + { err, ...logBase }, + "failed to ensure managed plugin environment while its worker is running; keeping any previously provisioned row and retrying (degraded: declaration not applied)", + ); + } + schedulePluginRecovery(spec, recoverable.pluginId, recoverable.workerAccess); + } else { + logger.error( + { err, ...logBase }, + "failed to ensure managed plugin environment; continuing boot (degraded: environment unavailable)", + ); + } + } + } + return { ensured, failed, @@ -417,5 +672,6 @@ export async function applyManagedEnvironments( removed: 0, backedUp: 0, outcomes, + pluginOutcomes, }; }