diff --git a/doc/DATABASE.md b/doc/DATABASE.md index dd8126cbcd1..bf4c88a98ca 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -826,6 +826,19 @@ pnpm secrets:migrate-inline-env --apply Hosted AWS provider notes live in [SECRETS-AWS-PROVIDER.md](./SECRETS-AWS-PROVIDER.md). +Sandbox runtime configuration resolves a provider's secret-reference fields +through its schema-only lookup. Each lookup reads the selected plugin ID +and a SHA-256 hash of its current schema in one database statement. A matching +retained schema is omitted from the query result. Selection order, schema +changes, and deletion remain visible on the next lookup, without a time-based +cache. A database error still fails the lookup. + +The retained schemas are private to one `Db` object, with at most 32 entries of +64 KiB each in serialized form. Larger schemas still work and are read in full. +Callers receive independent schema objects. This reuse stores neither resolved +secrets nor authorization decisions. Default driver lookups still read complete +current plugin records and enforce their existing readiness and worker checks. + ### Persistent agent conversations Migration `0274_agent_chat.sql` adds conversation identity/state and session generation/boundary columns to `issues`, plus idempotent client request IDs and processed session-boundary generations to `issue_comments`. The company/agent/user unique index resolves concurrent first writes to one issue. A check constraint preserves the assigned-agent identity and prevents terminal conversation status. Comment request IDs are unique per issue and user. There is no separate chat/message store. Provider sessions continue to use `agent_task_sessions`; `/new` removes only the matching conversation session, and session writers fence stale generations against the issue row. diff --git a/server/src/__tests__/environment-custom-image-routes.test.ts b/server/src/__tests__/environment-custom-image-routes.test.ts index 82b8eda9d37..43b3ae382e4 100644 --- a/server/src/__tests__/environment-custom-image-routes.test.ts +++ b/server/src/__tests__/environment-custom-image-routes.test.ts @@ -92,6 +92,7 @@ vi.mock("../services/plugin-environment-driver.js", () => ({ REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"], listReadyPluginEnvironmentDrivers: vi.fn(async () => []), resolvePluginSandboxProviderDriverByKey: vi.fn(async () => null), + resolvePluginSandboxProviderConfigSchema: vi.fn(async () => null), validatePluginEnvironmentDriverConfig: vi.fn(async ({ config }) => config), validatePluginSandboxProviderConfig: vi.fn(async ({ provider, config }) => ({ normalizedConfig: config, diff --git a/server/src/__tests__/environment-instance-routes.test.ts b/server/src/__tests__/environment-instance-routes.test.ts index 51664e01d98..293e12b7876 100644 --- a/server/src/__tests__/environment-instance-routes.test.ts +++ b/server/src/__tests__/environment-instance-routes.test.ts @@ -79,6 +79,7 @@ vi.mock("../services/plugin-environment-driver.js", () => ({ REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"], listReadyPluginEnvironmentDrivers: vi.fn(async () => []), resolvePluginSandboxProviderDriverByKey: vi.fn(async () => null), + resolvePluginSandboxProviderConfigSchema: vi.fn(async () => null), validatePluginEnvironmentDriverConfig: vi.fn(async ({ config }) => config), validatePluginSandboxProviderConfig: vi.fn(async ({ provider, config }) => ({ normalizedConfig: config, diff --git a/server/src/__tests__/environment-probe.test.ts b/server/src/__tests__/environment-probe.test.ts index e00c2d3eb6f..9474fd8b5a9 100644 --- a/server/src/__tests__/environment-probe.test.ts +++ b/server/src/__tests__/environment-probe.test.ts @@ -25,6 +25,8 @@ vi.mock("../services/plugin-environment-driver.js", () => ({ probePluginEnvironmentDriver: mockProbePluginEnvironmentDriver, probePluginSandboxProviderDriver: mockProbePluginSandboxProviderDriver, resolvePluginSandboxProviderDriverByKey: mockResolvePluginSandboxProviderDriverByKey, + resolvePluginSandboxProviderConfigSchema: async (input: { db: unknown; driverKey: string }) => + (await mockResolvePluginSandboxProviderDriverByKey(input))?.driver?.configSchema ?? null, })); vi.mock("../services/environment-runtime.js", () => ({ diff --git a/server/src/__tests__/environment-routes.test.ts b/server/src/__tests__/environment-routes.test.ts index 07c90052f37..9ad0bf972c4 100644 --- a/server/src/__tests__/environment-routes.test.ts +++ b/server/src/__tests__/environment-routes.test.ts @@ -129,6 +129,8 @@ vi.mock("../services/plugin-environment-driver.js", () => ({ REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"], listReadyPluginEnvironmentDrivers: mockListReadyPluginEnvironmentDrivers, resolvePluginSandboxProviderDriverByKey: mockResolvePluginSandboxProviderDriverByKey, + resolvePluginSandboxProviderConfigSchema: async (input: { db: unknown; driverKey: string }) => + (await mockResolvePluginSandboxProviderDriverByKey(input))?.driver?.configSchema ?? null, startPluginEnvironmentInteractiveSetup: mockStartPluginEnvironmentInteractiveSetup, getPluginEnvironmentInteractiveSetup: mockGetPluginEnvironmentInteractiveSetup, capturePluginEnvironmentTemplate: mockCapturePluginEnvironmentTemplate, diff --git a/server/src/__tests__/sandbox-provider-schema-reads.test.ts b/server/src/__tests__/sandbox-provider-schema-reads.test.ts new file mode 100644 index 00000000000..55f81f8bfbe --- /dev/null +++ b/server/src/__tests__/sandbox-provider-schema-reads.test.ts @@ -0,0 +1,173 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { eq, sql, type SQL } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { createDb, plugins, type Db } from "@paperclipai/db"; +import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared"; +import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js"; +import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("sandbox provider schema reads", () => { + let temporary: Awaited>; + let db: Db; + const clients: Db[] = []; + const key = "fixture-schema-provider"; + const driver = (driverKey = key, configSchema: unknown = { type: "object" }) => ({ + driverKey, kind: "sandbox_provider", displayName: "Schema fixture", configSchema, + }) as PluginEnvironmentDriverDeclaration; + const manifest = (id: string, declarations: PluginEnvironmentDriverDeclaration[]): PaperclipPluginManifestV1 => ({ + id: `test.schema.${id}`, apiVersion: 1, version: "1.0.0", displayName: "Schema fixture", + description: "Fixture", author: "Test", categories: ["automation"], + capabilities: ["environment.drivers.register"], entrypoints: { worker: "worker.js" }, environmentDrivers: declarations, + }); + async function seed(declarations = [driver()], installOrder = 0) { + const id = randomUUID(); + await db.insert(plugins).values({ + id, pluginKey: `test.schema.${id}`, packageName: "@test/schema", version: "1.0.0", + installOrder, status: "disabled", manifestJson: manifest(id, declarations), + }); + return id; + } + function observer() { + const observed = createDb(temporary.connectionString); + clients.push(observed); + return observed; + } + const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey }); + async function measure(client: Db, operation: () => Promise) { + const events: Array> = [], now = Date.now(); + const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now); + installDatabaseResponseCapture(client, { plugins }, capture); + try { + const value = await operation(); + capture.flush(); + const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0), + calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) }; + } finally { capture.flush(); } + } + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("sandbox-schema-reads-"); + db = createDb(temporary.connectionString); + }, 120_000); + beforeEach(async () => { await db.delete(plugins); }); + afterEach(async () => { await Promise.all(clients.splice(0).map(client => client.$client.end())); }); + afterAll(async () => { await temporary?.cleanup(); }); + + it("keeps runtime secret paths current without retransferring an unchanged schema", async () => { + const schema = { type: "object", description: randomBytes(16 * 1024).toString("hex"), properties: { + credentials: { type: "object", properties: { token: { type: "string", format: "secret-ref" } } }, + } }; + const id = await seed([driver(key, schema)]), observed = observer(); + const before = await db.select().from(plugins).where(eq(plugins.id, id)); + const cold = await measure(observed, () => resolveSandboxProviderSecretRefPaths(observed, key)); + const warm = await measure(observed, () => resolveSandboxProviderSecretRefPaths(observed, key)); + expect(cold.value).toEqual(new Set(["credentials.token"])); + expect(warm.value).toEqual(cold.value); + expect(cold.calls).toBe(1); + expect(warm.calls).toBe(1); + expect(cold.bytes).toBeGreaterThan(32 * 1024); + expect(warm.bytes).toBeLessThan(256); + const changed = { type: "object", allOf: [{ properties: { apiKey: { type: "string", format: "secret-ref" } } }] }; + await db.update(plugins).set({ manifestJson: manifest(id, [driver(key, changed)]) }).where(eq(plugins.id, id)); + expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set(["apiKey"])); + const environment = { id: randomUUID(), driver: "sandbox" as const, config: { provider: key, apiKey: randomUUID() } }; + await expect(resolveEnvironmentDriverConfigForRuntime(observed, null, environment)).rejects.toThrow("companyId"); + await db.update(plugins).set({ manifestJson: before[0].manifestJson }).where(eq(plugins.id, id)); + expect(await db.select().from(plugins).where(eq(plugins.id, id))).toEqual(before); + }); + + it("preserves install order, first matching declarations, and default readiness checks", async () => { + const a = await seed([driver(key, { title: "first" }), driver(key, { title: "second" })], 1); + const b = await seed([driver(key, { title: "other plugin" })], 2), observed = observer(); + expect((await lookup(observed))).toEqual({ title: "first" }); + const full = await resolvePluginSandboxProviderDriverByKey({ db: observed, driverKey: key }); + expect(full?.plugin.id).toBe(a); + expect(full?.plugin.status).toBe("disabled"); + expect(await resolvePluginSandboxProviderDriverByKey({ db: observed, driverKey: key, requireRunning: true })).toBeNull(); + await db.update(plugins).set({ installOrder: 0 }).where(eq(plugins.id, b)); + expect((await lookup(observed))).toEqual({ title: "other plugin" }); + await db.delete(plugins).where(eq(plugins.id, b)); + expect((await lookup(observed))).toEqual({ title: "first" }); + await db.update(plugins).set({ manifestJson: manifest(a, [driver("replacement")]) }).where(eq(plugins.id, a)); + expect(await lookup(observed)).toBeNull(); + }); + + it.each([null, true, "invalid", [], undefined])("preserves non-object schema behavior: %s", async value => { + const first = driver(key, value); + if (value === undefined) delete (first as Partial).configSchema; + await seed([first, driver(key, { properties: { secret: { format: "secret-ref" } } })]); + const observed = observer(); + expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set()); + expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set()); + }); + + it("returns independent schema objects and keeps separate database clients cold", async () => { + await seed([driver(key, { description: "x".repeat(4096), properties: { token: { format: "secret-ref" } } })]); + const a = observer(), b = observer(); + const first = await lookup(a); + first!.properties = {}; + expect(await resolveSandboxProviderSecretRefPaths(a, key)).toEqual(new Set(["token"])); + const warm = await measure(a, () => lookup(a)), independent = await measure(b, () => lookup(b)); + expect(warm.bytes).toBeLessThan(256); + expect(independent.bytes).toBeGreaterThan(4096); + warm.value!.properties = {}; + expect(await resolveSandboxProviderSecretRefPaths(a, key)).toEqual(new Set(["token"])); + }); + + it("bounds retained schemas and does not retain oversized definitions", async () => { + const declarations = Array.from({ length: 33 }, (_, index) => driver(`provider-${index}`, { description: "x".repeat(2048) })); + declarations.push(driver("oversized", { description: "x".repeat(128 * 1024) })); + await seed(declarations); + const observed = observer(); + for (let index = 0; index < 33; index++) await lookup(observed, `provider-${index}`); + expect((await measure(observed, () => lookup(observed, "provider-32"))).bytes).toBeLessThan(256); + expect((await measure(observed, () => lookup(observed, "provider-0"))).bytes).toBeGreaterThan(2048); + await lookup(observed, "oversized"); + expect((await measure(observed, () => lookup(observed, "oversized"))).bytes).toBeGreaterThan(128 * 1024); + }); + + it("binds unusual keys and never serves retained schemas after a lookup error", async () => { + const unusual = "provider-\"quoted' OR true --"; + await seed([driver(unusual, { title: "exact provider" })]); + const observed = observer(); + let fail = false; + const guarded = { execute: async (query: SQL) => { + if (fail) throw new Error("fixture database unavailable"); + return observed.execute(query); + } } as unknown as Db; + expect((await lookup(guarded, unusual))).toEqual({ title: "exact provider" }); + fail = true; + await expect(lookup(guarded, unusual)).rejects.toThrow("fixture database unavailable"); + fail = false; + expect((await lookup(guarded, unusual))).toEqual({ title: "exact provider" }); + expect(await lookup(guarded, "unknown-provider")).toBeNull(); + }); + + it("keeps each concurrent lookup tied to the database revision it observed", async () => { + const id = await seed([driver(key, { title: "old" })]), observed = observer(); + let pause = false, release!: () => void, reached!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const paused = new Promise(resolve => { reached = resolve; }); + const raced = { execute: async (query: SQL) => { + const rows = await observed.execute(query); + if (pause) { pause = false; reached(); await gate; } + return rows; + } } as unknown as Db; + expect((await lookup(raced))).toEqual({ title: "old" }); + pause = true; + const oldRead = lookup(raced); + await paused; + try { + await db.update(plugins).set({ manifestJson: manifest(id, [driver(key, { title: "new" })]) }).where(eq(plugins.id, id)); + expect((await lookup(raced))).toEqual({ title: "new" }); + } finally { release(); } + expect((await oldRead)).toEqual({ title: "old" }); + expect((await lookup(raced))).toEqual({ title: "new" }); + expect((await db.execute<{ title: string }>(sql`select ${plugins.manifestJson}->'environmentDrivers'->0->'configSchema'->>'title' as title from ${plugins} where ${plugins.id}=${id}`))[0].title).toBe("new"); + }); +}); diff --git a/server/src/services/environment-config.ts b/server/src/services/environment-config.ts index b9c041a35d1..6431c4802ff 100644 --- a/server/src/services/environment-config.ts +++ b/server/src/services/environment-config.ts @@ -17,7 +17,7 @@ import { unprocessable } from "../errors.js"; import { parseObject } from "../adapters/utils.js"; import { secretService } from "./secrets.js"; import { - resolvePluginSandboxProviderDriverByKey, + resolvePluginSandboxProviderConfigSchema, validatePluginEnvironmentDriverConfig, validatePluginSandboxProviderConfig, } from "./plugin-environment-driver.js"; @@ -179,14 +179,10 @@ async function getSandboxProviderConfigSchema( db: Db, provider: string, ): Promise | null> { - const resolved = await resolvePluginSandboxProviderDriverByKey({ + return resolvePluginSandboxProviderConfigSchema({ db, driverKey: provider, }); - const schema = resolved?.driver.configSchema; - return schema && typeof schema === "object" && !Array.isArray(schema) - ? schema as Record - : null; } export async function resolveSandboxProviderSecretRefPaths( diff --git a/server/src/services/plugin-environment-driver.ts b/server/src/services/plugin-environment-driver.ts index d7f56f2b197..f26f32483ff 100644 --- a/server/src/services/plugin-environment-driver.ts +++ b/server/src/services/plugin-environment-driver.ts @@ -1,4 +1,5 @@ -import type { Db } from "@paperclipai/db"; +import { plugins, type Db } from "@paperclipai/db"; +import { sql } from "drizzle-orm"; import type { EnvironmentProbeResult, PluginEnvironmentConfig, @@ -162,6 +163,62 @@ export async function resolvePluginEnvironmentDriverByKey(input: { }); } +type CachedProviderSchema = { pluginId: string; revision: string; schema: Record | null }; +const providerSchemas = new WeakMap>(); +const MAX_CACHED_PROVIDER_SCHEMAS = 32; +const MAX_CACHED_PROVIDER_SCHEMA_BYTES = 64 * 1024; + +export async function resolvePluginSandboxProviderConfigSchema(input: { + db: Db; driverKey: string; +}): Promise | null> { + const { db, driverKey } = input; + let cache = providerSchemas.get(db); + const cached = cache?.get(driverKey); + const unchanged = sql`coalesce(plugin_id = ${cached?.pluginId ?? null}::uuid + and revision = ${cached?.revision ?? null}, false)`; + const [row] = await db.execute<{ + pluginId: string; revision: string; unchanged: boolean; configSchema: unknown; + }>(sql`with selected_provider as materialized ( + select ${plugins.id} as plugin_id, ( + select declaration.value->'configSchema' + from jsonb_array_elements(${plugins.manifestJson}->'environmentDrivers') + with ordinality as declaration(value, position) + where declaration.value->'driverKey' = to_jsonb(${driverKey}::text) + and declaration.value->>'kind' = 'sandbox_provider' + order by declaration.position limit 1 + ) as schema_value + from ${plugins} + where ${plugins.manifestJson} @> ${JSON.stringify({ environmentDrivers: [{ driverKey, kind: "sandbox_provider" }] })}::jsonb + order by ${plugins.installOrder} limit 1 + ), versioned_provider as materialized ( + select plugin_id, schema_value, + encode(sha256(convert_to(coalesce(schema_value::text, 'null'), 'UTF8')), 'hex') as revision + from selected_provider + ) + select plugin_id as "pluginId", revision, ${unchanged} as unchanged, + case when ${unchanged} then null else schema_value end as "configSchema" + from versioned_provider`); + if (!row) { + cache?.delete(driverKey); + return null; + } + if (row.unchanged) { + if (!cached || cached.pluginId !== row.pluginId || cached.revision !== row.revision) { + throw new Error("Sandbox provider schema revision did not match the retained definition"); + } + return structuredClone(cached.schema); + } + const schema = row.configSchema && typeof row.configSchema === "object" && !Array.isArray(row.configSchema) + ? row.configSchema as Record : null; + cache?.delete(driverKey); + if (Buffer.byteLength(JSON.stringify(schema)) <= MAX_CACHED_PROVIDER_SCHEMA_BYTES) { + if (!cache) { cache = new Map(); providerSchemas.set(db, cache); } + if (cache.size >= MAX_CACHED_PROVIDER_SCHEMAS) cache.delete(cache.keys().next().value!); + cache.set(driverKey, { pluginId: row.pluginId, revision: row.revision, schema: structuredClone(schema) }); + } + return schema; +} + export async function resolvePluginSandboxProviderDriverByKey(input: { db: Db; driverKey: string;