perf(environments): reuse provider schemas after content validation (#17581)

Add a separate schema-only provider lookup with current plugin identity and SHA-256 content validation in one statement. Retain bounded schema definitions, return independent objects, and preserve current full driver and secret access paths.

Build, repository typecheck, 485 affected tests, the supported-host database tests, and the complete required CI matrix passed. Greptile scored this exact head 5/5 at medium impact with no unresolved findings.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/DATABASE.md

@@ -826,6 +826,19 @@ pnpm secrets:migrate-inline-env --apply
 
 Hosted AWS provider notes live in [SECRETS-AWS-PROVIDER.md](./SECRETS-AWS-PROVIDER.md).
 
+Sandbox runtime configuration resolves a provider's secret-reference fields
+through its schema-only lookup. Each lookup reads the selected plugin ID
+and a SHA-256 hash of its current schema in one database statement. A matching
+retained schema is omitted from the query result. Selection order, schema
+changes, and deletion remain visible on the next lookup, without a time-based
+cache. A database error still fails the lookup.
+
+The retained schemas are private to one `Db` object, with at most 32 entries of
+64 KiB each in serialized form. Larger schemas still work and are read in full.
+Callers receive independent schema objects. This reuse stores neither resolved
+secrets nor authorization decisions. Default driver lookups still read complete
+current plugin records and enforce their existing readiness and worker checks.
+
 ### Persistent agent conversations
 
 Migration `0274_agent_chat.sql` adds conversation identity/state and session generation/boundary columns to `issues`, plus idempotent client request IDs and processed session-boundary generations to `issue_comments`. The company/agent/user unique index resolves concurrent first writes to one issue. A check constraint preserves the assigned-agent identity and prevents terminal conversation status. Comment request IDs are unique per issue and user. There is no separate chat/message store. Provider sessions continue to use `agent_task_sessions`; `/new` removes only the matching conversation session, and session writers fence stale generations against the issue row.

server/src/__tests__/environment-custom-image-routes.test.ts

@@ -92,6 +92,7 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
   REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
   listReadyPluginEnvironmentDrivers: vi.fn(async () => []),
   resolvePluginSandboxProviderDriverByKey: vi.fn(async () => null),
+  resolvePluginSandboxProviderConfigSchema: vi.fn(async () => null),
   validatePluginEnvironmentDriverConfig: vi.fn(async ({ config }) => config),
   validatePluginSandboxProviderConfig: vi.fn(async ({ provider, config }) => ({
     normalizedConfig: config,

server/src/__tests__/environment-instance-routes.test.ts

@@ -79,6 +79,7 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
   REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
   listReadyPluginEnvironmentDrivers: vi.fn(async () => []),
   resolvePluginSandboxProviderDriverByKey: vi.fn(async () => null),
+  resolvePluginSandboxProviderConfigSchema: vi.fn(async () => null),
   validatePluginEnvironmentDriverConfig: vi.fn(async ({ config }) => config),
   validatePluginSandboxProviderConfig: vi.fn(async ({ provider, config }) => ({
     normalizedConfig: config,

server/src/__tests__/environment-probe.test.ts

@@ -25,6 +25,8 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
   probePluginEnvironmentDriver: mockProbePluginEnvironmentDriver,
   probePluginSandboxProviderDriver: mockProbePluginSandboxProviderDriver,
   resolvePluginSandboxProviderDriverByKey: mockResolvePluginSandboxProviderDriverByKey,
+  resolvePluginSandboxProviderConfigSchema: async (input: { db: unknown; driverKey: string }) =>
+    (await mockResolvePluginSandboxProviderDriverByKey(input))?.driver?.configSchema ?? null,
 }));
 
 vi.mock("../services/environment-runtime.js", () => ({

server/src/__tests__/environment-routes.test.ts

@@ -129,6 +129,8 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
   REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
   listReadyPluginEnvironmentDrivers: mockListReadyPluginEnvironmentDrivers,
   resolvePluginSandboxProviderDriverByKey: mockResolvePluginSandboxProviderDriverByKey,
+  resolvePluginSandboxProviderConfigSchema: async (input: { db: unknown; driverKey: string }) =>
+    (await mockResolvePluginSandboxProviderDriverByKey(input))?.driver?.configSchema ?? null,
   startPluginEnvironmentInteractiveSetup: mockStartPluginEnvironmentInteractiveSetup,
   getPluginEnvironmentInteractiveSetup: mockGetPluginEnvironmentInteractiveSetup,
   capturePluginEnvironmentTemplate: mockCapturePluginEnvironmentTemplate,

server/src/__tests__/sandbox-provider-schema-reads.test.ts

@@ -0,0 +1,173 @@
+import { randomBytes, randomUUID } from "node:crypto";
+import { eq, sql, type SQL } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
+import { createDb, plugins, type Db } from "@paperclipai/db";
+import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
+import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared";
+import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js";
+import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+
+const support = await getEmbeddedPostgresTestSupport();
+
+describe.skipIf(!support.supported)("sandbox provider schema reads", () => {
+  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let db: Db;
+  const clients: Db[] = [];
+  const key = "fixture-schema-provider";
+  const driver = (driverKey = key, configSchema: unknown = { type: "object" }) => ({
+    driverKey, kind: "sandbox_provider", displayName: "Schema fixture", configSchema,
+  }) as PluginEnvironmentDriverDeclaration;
+  const manifest = (id: string, declarations: PluginEnvironmentDriverDeclaration[]): PaperclipPluginManifestV1 => ({
+    id: `test.schema.${id}`, apiVersion: 1, version: "1.0.0", displayName: "Schema fixture",
+    description: "Fixture", author: "Test", categories: ["automation"],
+    capabilities: ["environment.drivers.register"], entrypoints: { worker: "worker.js" }, environmentDrivers: declarations,
+  });
+  async function seed(declarations = [driver()], installOrder = 0) {
+    const id = randomUUID();
+    await db.insert(plugins).values({
+      id, pluginKey: `test.schema.${id}`, packageName: "@test/schema", version: "1.0.0",
+      installOrder, status: "disabled", manifestJson: manifest(id, declarations),
+    });
+    return id;
+  }
+  function observer() {
+    const observed = createDb(temporary.connectionString);
+    clients.push(observed);
+    return observed;
+  }
+  const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey });
+  async function measure<T>(client: Db, operation: () => Promise<T>) {
+    const events: Array<Record<string, unknown>> = [], now = Date.now();
+    const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
+    installDatabaseResponseCapture(client, { plugins }, capture);
+    try {
+      const value = await operation();
+      capture.flush();
+      const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
+      return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0),
+        calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) };
+    } finally { capture.flush(); }
+  }
+
+  beforeAll(async () => {
+    temporary = await startEmbeddedPostgresTestDatabase("sandbox-schema-reads-");
+    db = createDb(temporary.connectionString);
+  }, 120_000);
+  beforeEach(async () => { await db.delete(plugins); });
+  afterEach(async () => { await Promise.all(clients.splice(0).map(client => client.$client.end())); });
+  afterAll(async () => { await temporary?.cleanup(); });
+
+  it("keeps runtime secret paths current without retransferring an unchanged schema", async () => {
+    const schema = { type: "object", description: randomBytes(16 * 1024).toString("hex"), properties: {
+      credentials: { type: "object", properties: { token: { type: "string", format: "secret-ref" } } },
+    } };
+    const id = await seed([driver(key, schema)]), observed = observer();
+    const before = await db.select().from(plugins).where(eq(plugins.id, id));
+    const cold = await measure(observed, () => resolveSandboxProviderSecretRefPaths(observed, key));
+    const warm = await measure(observed, () => resolveSandboxProviderSecretRefPaths(observed, key));
+    expect(cold.value).toEqual(new Set(["credentials.token"]));
+    expect(warm.value).toEqual(cold.value);
+    expect(cold.calls).toBe(1);
+    expect(warm.calls).toBe(1);
+    expect(cold.bytes).toBeGreaterThan(32 * 1024);
+    expect(warm.bytes).toBeLessThan(256);
+    const changed = { type: "object", allOf: [{ properties: { apiKey: { type: "string", format: "secret-ref" } } }] };
+    await db.update(plugins).set({ manifestJson: manifest(id, [driver(key, changed)]) }).where(eq(plugins.id, id));
+    expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set(["apiKey"]));
+    const environment = { id: randomUUID(), driver: "sandbox" as const, config: { provider: key, apiKey: randomUUID() } };
+    await expect(resolveEnvironmentDriverConfigForRuntime(observed, null, environment)).rejects.toThrow("companyId");
+    await db.update(plugins).set({ manifestJson: before[0].manifestJson }).where(eq(plugins.id, id));
+    expect(await db.select().from(plugins).where(eq(plugins.id, id))).toEqual(before);
+  });
+
+  it("preserves install order, first matching declarations, and default readiness checks", async () => {
+    const a = await seed([driver(key, { title: "first" }), driver(key, { title: "second" })], 1);
+    const b = await seed([driver(key, { title: "other plugin" })], 2), observed = observer();
+    expect((await lookup(observed))).toEqual({ title: "first" });
+    const full = await resolvePluginSandboxProviderDriverByKey({ db: observed, driverKey: key });
+    expect(full?.plugin.id).toBe(a);
+    expect(full?.plugin.status).toBe("disabled");
+    expect(await resolvePluginSandboxProviderDriverByKey({ db: observed, driverKey: key, requireRunning: true })).toBeNull();
+    await db.update(plugins).set({ installOrder: 0 }).where(eq(plugins.id, b));
+    expect((await lookup(observed))).toEqual({ title: "other plugin" });
+    await db.delete(plugins).where(eq(plugins.id, b));
+    expect((await lookup(observed))).toEqual({ title: "first" });
+    await db.update(plugins).set({ manifestJson: manifest(a, [driver("replacement")]) }).where(eq(plugins.id, a));
+    expect(await lookup(observed)).toBeNull();
+  });
+
+  it.each([null, true, "invalid", [], undefined])("preserves non-object schema behavior: %s", async value => {
+    const first = driver(key, value);
+    if (value === undefined) delete (first as Partial<PluginEnvironmentDriverDeclaration>).configSchema;
+    await seed([first, driver(key, { properties: { secret: { format: "secret-ref" } } })]);
+    const observed = observer();
+    expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set());
+    expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set());
+  });
+
+  it("returns independent schema objects and keeps separate database clients cold", async () => {
+    await seed([driver(key, { description: "x".repeat(4096), properties: { token: { format: "secret-ref" } } })]);
+    const a = observer(), b = observer();
+    const first = await lookup(a);
+    first!.properties = {};
+    expect(await resolveSandboxProviderSecretRefPaths(a, key)).toEqual(new Set(["token"]));
+    const warm = await measure(a, () => lookup(a)), independent = await measure(b, () => lookup(b));
+    expect(warm.bytes).toBeLessThan(256);
+    expect(independent.bytes).toBeGreaterThan(4096);
+    warm.value!.properties = {};
+    expect(await resolveSandboxProviderSecretRefPaths(a, key)).toEqual(new Set(["token"]));
+  });
+
+  it("bounds retained schemas and does not retain oversized definitions", async () => {
+    const declarations = Array.from({ length: 33 }, (_, index) => driver(`provider-${index}`, { description: "x".repeat(2048) }));
+    declarations.push(driver("oversized", { description: "x".repeat(128 * 1024) }));
+    await seed(declarations);
+    const observed = observer();
+    for (let index = 0; index < 33; index++) await lookup(observed, `provider-${index}`);
+    expect((await measure(observed, () => lookup(observed, "provider-32"))).bytes).toBeLessThan(256);
+    expect((await measure(observed, () => lookup(observed, "provider-0"))).bytes).toBeGreaterThan(2048);
+    await lookup(observed, "oversized");
+    expect((await measure(observed, () => lookup(observed, "oversized"))).bytes).toBeGreaterThan(128 * 1024);
+  });
+
+  it("binds unusual keys and never serves retained schemas after a lookup error", async () => {
+    const unusual = "provider-\"quoted' OR true --";
+    await seed([driver(unusual, { title: "exact provider" })]);
+    const observed = observer();
+    let fail = false;
+    const guarded = { execute: async (query: SQL) => {
+      if (fail) throw new Error("fixture database unavailable");
+      return observed.execute(query);
+    } } as unknown as Db;
+    expect((await lookup(guarded, unusual))).toEqual({ title: "exact provider" });
+    fail = true;
+    await expect(lookup(guarded, unusual)).rejects.toThrow("fixture database unavailable");
+    fail = false;
+    expect((await lookup(guarded, unusual))).toEqual({ title: "exact provider" });
+    expect(await lookup(guarded, "unknown-provider")).toBeNull();
+  });
+
+  it("keeps each concurrent lookup tied to the database revision it observed", async () => {
+    const id = await seed([driver(key, { title: "old" })]), observed = observer();
+    let pause = false, release!: () => void, reached!: () => void;
+    const gate = new Promise<void>(resolve => { release = resolve; });
+    const paused = new Promise<void>(resolve => { reached = resolve; });
+    const raced = { execute: async (query: SQL) => {
+      const rows = await observed.execute(query);
+      if (pause) { pause = false; reached(); await gate; }
+      return rows;
+    } } as unknown as Db;
+    expect((await lookup(raced))).toEqual({ title: "old" });
+    pause = true;
+    const oldRead = lookup(raced);
+    await paused;
+    try {
+      await db.update(plugins).set({ manifestJson: manifest(id, [driver(key, { title: "new" })]) }).where(eq(plugins.id, id));
+      expect((await lookup(raced))).toEqual({ title: "new" });
+    } finally { release(); }
+    expect((await oldRead)).toEqual({ title: "old" });
+    expect((await lookup(raced))).toEqual({ title: "new" });
+    expect((await db.execute<{ title: string }>(sql`select ${plugins.manifestJson}->'environmentDrivers'->0->'configSchema'->>'title' as title from ${plugins} where ${plugins.id}=${id}`))[0].title).toBe("new");
+  });
+});

server/src/services/environment-config.ts

@@ -17,7 +17,7 @@ import { unprocessable } from "../errors.js";
 import { parseObject } from "../adapters/utils.js";
 import { secretService } from "./secrets.js";
 import {
-  resolvePluginSandboxProviderDriverByKey,
+  resolvePluginSandboxProviderConfigSchema,
   validatePluginEnvironmentDriverConfig,
   validatePluginSandboxProviderConfig,
 } from "./plugin-environment-driver.js";
@@ -179,14 +179,10 @@ async function getSandboxProviderConfigSchema(
   db: Db,
   provider: string,
 ): Promise<Record<string, unknown> | null> {
-  const resolved = await resolvePluginSandboxProviderDriverByKey({
+  return resolvePluginSandboxProviderConfigSchema({
     db,
     driverKey: provider,
   });
-  const schema = resolved?.driver.configSchema;
-  return schema && typeof schema === "object" && !Array.isArray(schema)
-    ? schema as Record<string, unknown>
-    : null;
 }
 
 export async function resolveSandboxProviderSecretRefPaths(

server/src/services/plugin-environment-driver.ts

@@ -1,4 +1,5 @@
-import type { Db } from "@paperclipai/db";
+import { plugins, type Db } from "@paperclipai/db";
+import { sql } from "drizzle-orm";
 import type {
   EnvironmentProbeResult,
   PluginEnvironmentConfig,
@@ -162,6 +163,62 @@ export async function resolvePluginEnvironmentDriverByKey(input: {
   });
 }
 
+type CachedProviderSchema = { pluginId: string; revision: string; schema: Record<string, unknown> | null };
+const providerSchemas = new WeakMap<Db, Map<string, CachedProviderSchema>>();
+const MAX_CACHED_PROVIDER_SCHEMAS = 32;
+const MAX_CACHED_PROVIDER_SCHEMA_BYTES = 64 * 1024;
+
+export async function resolvePluginSandboxProviderConfigSchema(input: {
+  db: Db; driverKey: string;
+}): Promise<Record<string, unknown> | null> {
+  const { db, driverKey } = input;
+  let cache = providerSchemas.get(db);
+  const cached = cache?.get(driverKey);
+  const unchanged = sql`coalesce(plugin_id = ${cached?.pluginId ?? null}::uuid
+    and revision = ${cached?.revision ?? null}, false)`;
+  const [row] = await db.execute<{
+    pluginId: string; revision: string; unchanged: boolean; configSchema: unknown;
+  }>(sql`with selected_provider as materialized (
+    select ${plugins.id} as plugin_id, (
+      select declaration.value->'configSchema'
+      from jsonb_array_elements(${plugins.manifestJson}->'environmentDrivers')
+        with ordinality as declaration(value, position)
+      where declaration.value->'driverKey' = to_jsonb(${driverKey}::text)
+        and declaration.value->>'kind' = 'sandbox_provider'
+      order by declaration.position limit 1
+    ) as schema_value
+    from ${plugins}
+    where ${plugins.manifestJson} @> ${JSON.stringify({ environmentDrivers: [{ driverKey, kind: "sandbox_provider" }] })}::jsonb
+    order by ${plugins.installOrder} limit 1
+  ), versioned_provider as materialized (
+    select plugin_id, schema_value,
+      encode(sha256(convert_to(coalesce(schema_value::text, 'null'), 'UTF8')), 'hex') as revision
+    from selected_provider
+  )
+  select plugin_id as "pluginId", revision, ${unchanged} as unchanged,
+    case when ${unchanged} then null else schema_value end as "configSchema"
+  from versioned_provider`);
+  if (!row) {
+    cache?.delete(driverKey);
+    return null;
+  }
+  if (row.unchanged) {
+    if (!cached || cached.pluginId !== row.pluginId || cached.revision !== row.revision) {
+      throw new Error("Sandbox provider schema revision did not match the retained definition");
+    }
+    return structuredClone(cached.schema);
+  }
+  const schema = row.configSchema && typeof row.configSchema === "object" && !Array.isArray(row.configSchema)
+    ? row.configSchema as Record<string, unknown> : null;
+  cache?.delete(driverKey);
+  if (Buffer.byteLength(JSON.stringify(schema)) <= MAX_CACHED_PROVIDER_SCHEMA_BYTES) {
+    if (!cache) { cache = new Map(); providerSchemas.set(db, cache); }
+    if (cache.size >= MAX_CACHED_PROVIDER_SCHEMAS) cache.delete(cache.keys().next().value!);
+    cache.set(driverKey, { pluginId: row.pluginId, revision: row.revision, schema: structuredClone(schema) });
+  }
+  return schema;
+}
+
 export async function resolvePluginSandboxProviderDriverByKey(input: {
   db: Db;
   driverKey: string;