From 9e42d3fe07b2c48ac2fd7f01a0bedaa79afcaddd Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 14:04:37 -0500 Subject: [PATCH] fix(cloud): load and qualify image-linked plugin runtimes Co-Authored-By: Paperclip --- doc/cloud-configuration-migration.md | 24 +++- doc/plugins/PLUGIN_SPEC.md | 9 +- .../bundled-plugin-registration.test.ts | 61 +++++++++- .../__tests__/bundled-plugin-runtime.test.ts | 72 +++++++++++ .../cloud-bundled-package-digest.test.ts | 85 +++++++++++++ server/src/services/bundled-plugin-package.ts | 1 - server/src/services/bundled-plugin-runtime.ts | 37 ++++++ .../services/cloud-bundled-package-digest.ts | 114 ++++++++++++------ .../services/cloud-configuration-migration.ts | 1 + .../cloud-configuration-qualification.ts | 2 +- .../services/cloud-package-qualification.ts | 3 +- server/src/services/plugin-loader.ts | 8 +- 12 files changed, 368 insertions(+), 49 deletions(-) create mode 100644 server/src/__tests__/bundled-plugin-runtime.test.ts create mode 100644 server/src/__tests__/cloud-bundled-package-digest.test.ts create mode 100644 server/src/services/bundled-plugin-runtime.ts diff --git a/doc/cloud-configuration-migration.md b/doc/cloud-configuration-migration.md index 9c04159a7e6..519c0aee181 100644 --- a/doc/cloud-configuration-migration.md +++ b/doc/cloud-configuration-migration.md @@ -110,11 +110,18 @@ prevent future maintenance releases from establishing their own evidence. The package evidence digest covers selected packages, not container image bytes. Distribution packages reuse and reverify their catalog digest. Builtin packages -have a bounded inventory of physical file bytes, modes, and contained symlink -text, including standalone pnpm dependency trees. Dangling or escaping links, -special files, more than 10,000 entries, depth above 32, or more than 256 MiB per -package fail verification. Inventory work is synchronous and linear in selected -package bytes. It runs only during startup before the application is constructed +have a bounded inventory of physical file bytes, modes, and logical symlink +targets, including standalone pnpm dependency trees. With the selected catalog +root, inventory may follow links only into that provider, the catalog SDK, +`packages/shared`, and the current image's root `node_modules`. Reached pnpm +package units include their sibling dependency links; unrelated root packages +are not scanned. Canonical directory/file deduplication bounds cycles, and +logical root labels preserve the digest when identical image bytes relocate. +Dangling or escaping links, special files, more than 50,000 entries, depth above +32, metadata above 128 KiB, or more than 256 MiB per package fail verification. +The measured cached Cloud Daytona closure has about 22,253 entries and 130 MB; +that artifact predates this change and is not a complete current release proof. +Inventory work is synchronous and linear in selected package bytes. It runs only during startup before the application is constructed or in the explicit operator process, with no idle polling. The report includes `packageInspectionDurationMs`. Packages must remain immutable throughout a check. @@ -127,6 +134,13 @@ image fixture. They also cover concurrent migration, commit failure and retry, database failure, stale sources, durable external blockers, package mismatch, maintenance reconciliation, and changes during manifest inspection. +The bundled-plugin registration suite also has an actual prebuilt Daytona case. +Build its standalone package with the normal SDK linker before running that case; +it is skipped when the provider build is absent. It qualifies the real linked +SDK/shared package graph and uses the production loader and worker manager for +credential-free initialize/health calls. The runtime-path tests also invoke the +production server bundler to verify its preserved module locations. + Set `PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL` only to an owned disposable PostgreSQL database when embedded PostgreSQL is unavailable. The suite applies migrations and writes/deletes test records. A complete release image, hosted diff --git a/doc/plugins/PLUGIN_SPEC.md b/doc/plugins/PLUGIN_SPEC.md index 7d7b4d1fb33..9e5c8370823 100644 --- a/doc/plugins/PLUGIN_SPEC.md +++ b/doc/plugins/PLUGIN_SPEC.md @@ -40,13 +40,20 @@ Managed Cloud provisions only plugins selected from the current release image catalog. A Cloud tenant token or managed configuration keeps this image-only policy active. Without an explicit selected package list, Cloud selects no packages and cannot fall back to local installs. An explicit operator-managed -list also keeps this policy active outside Cloud. The database registration's plugin key, package name, version and +list also keeps this policy active outside Cloud. The database registration's +plugin key, package name, version and approved manifest identify that package. Its legacy `packagePath` remains for local/self-hosted compatibility; Cloud never uses that path to locate code. Workers, plugin migrations and UI files resolve through the same current-image package resolver. Cloud does not install npm dependencies, build packages, use the development UI proxy, or fall back to a server home directory. +Prebuilt workers use the current release's baked CLI `tsx` loader for the SDK's +workspace TypeScript dependencies. The loader is located from the owning server +package in both regular and bundled server layouts. Missing loaders and loader +links outside the release's CLI or root `node_modules` fail activation. This +loads image-supplied code without installing dependencies or building packages. + Equivalent packages at different image roots reuse the existing registration, configuration, approval status and company scopes without rewriting its stored path. Concurrent first registration of the same package retains one plugin ID. diff --git a/server/src/__tests__/bundled-plugin-registration.test.ts b/server/src/__tests__/bundled-plugin-registration.test.ts index e7f5c2b7de8..87b495dcf1f 100644 --- a/server/src/__tests__/bundled-plugin-registration.test.ts +++ b/server/src/__tests__/bundled-plugin-registration.test.ts @@ -1,7 +1,7 @@ import express from "express"; import request from "supertest"; import { execFileSync } from "node:child_process"; -import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync } from "node:fs"; +import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync, existsSync, mkdtempSync } from "node:fs"; import path from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { applyPendingMigrations, closeRegisteredClients, companies, createDb, plugins } from "@paperclipai/db"; @@ -9,6 +9,11 @@ import { eq } from "drizzle-orm"; import type { PluginRecord } from "@paperclipai/shared"; import { createPluginWorkerManager } from "../services/plugin-worker-manager.js"; import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js"; +import { linkSdkInto } from "../../../scripts/link-plugin-dev-sdk.mjs"; +import { fileURLToPath } from "node:url"; +import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js"; +import { inspectCloudPackages } from "../services/cloud-package-qualification.js"; +import { resolveBundledPluginInstalls } from "../services/bundled-plugins.js"; import { pluginRegistryService } from "../services/plugin-registry.js"; import { pluginUiStaticRoutes } from "../routes/plugin-ui-static.js"; import { errorHandler } from "../middleware/error-handler.js"; @@ -130,6 +135,60 @@ describePostgres("bundled image registration in PostgreSQL", () => { } finally { await manager.stopAll(); } }); + it("qualifies and initializes the current release SDK/shared exports through the baked loader", async () => { + const image = fixture(); + const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url)); + const provider = mkdtempSync(path.join(catalogRoot, "sandbox-providers", "image-sdk-proof-")); + roots.push(provider); + cpSync(image.packageRoot, provider, { recursive: true }); + linkSdkInto(provider); + writeFileSync(path.join(provider, "dist", "worker.js"), 'import { definePlugin, runWorker } from "@paperclipai/plugin-sdk"; runWorker(definePlugin({ async setup() {}, async onHealth() { return { status: "ok", message: "Current image SDK/shared loaded" }; } }), import.meta.url);'); + const installs = [{ key: "sdk-proof", pluginKey: image.manifest.id, localPath: provider }]; + const resolver = createBundledPluginPackageResolver(installs); + const manager = createPluginWorkerManager(); + const loader = pluginLoader(db, { + prebuiltOnly: true, resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation, + enableLocalFilesystem: false, enableNpmDiscovery: false, + }, { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices); + const registry = pluginRegistryService(db); + try { + await loader.installPlugin({ localPath: provider }); + const installed = (await registry.getByKey(image.manifest.id))!; + await registry.updateStatus(installed.id, { status: "ready" }); + const registration = (await registry.getById(installed.id))!; + expect((await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) })).blockers).toEqual([]); + expect((await loader.loadSingle(installed.id)).success).toBe(true); + expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Current image SDK/shared loaded" }); + } finally { await manager.stopAll(); } + }, 90_000); + + it.skipIf(!existsSync(fileURLToPath(new URL("../../../packages/plugins/sandbox-providers/daytona/dist/worker.js", import.meta.url))))("qualifies and initializes the actual prebuilt Daytona release package", async () => { + const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url)); + const installs = resolveBundledPluginInstalls(["daytona"], { catalogRoot, env: {}, enforceCatalogRoot: true, distributionPlugins: [] }); + const resolver = createBundledPluginPackageResolver(installs); + const manager = createPluginWorkerManager(); + const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices; + const loader = pluginLoader(db, { + localPluginDir: "/lost/release/home/plugins", prebuiltOnly: true, + resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation, + enableLocalFilesystem: false, enableNpmDiscovery: false, + }, services); + const registry = pluginRegistryService(db); + vi.stubEnv("DAYTONA_API_KEY", ""); + try { + await loader.installPlugin({ localPath: installs[0]!.localPath }); + const installed = (await registry.getByKey(daytonaManifest.id))!; + await registry.update(installed.id, { packagePath: "/lost/release/home/daytona" }); + await registry.updateStatus(installed.id, { status: "ready" }); + const registration = (await registry.getById(installed.id))!; + const inspected = await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) }); + expect(inspected.blockers).toEqual([]); + expect(inspected.packages[0]?.packageDigest).toMatch(/^sha256:/); + expect((await loader.loadSingle(installed.id)).success).toBe(true); + expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Daytona sandbox provider plugin healthy" }); + } finally { await manager.stopAll(); vi.unstubAllEnvs(); } + }, 90_000); + it("registers the same package concurrently from two roots without replacing the winner", async () => { const first = fixture(); const second = fixture(); await Promise.all([ diff --git a/server/src/__tests__/bundled-plugin-runtime.test.ts b/server/src/__tests__/bundled-plugin-runtime.test.ts new file mode 100644 index 00000000000..936687a91be --- /dev/null +++ b/server/src/__tests__/bundled-plugin-runtime.test.ts @@ -0,0 +1,72 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { build, transform } from "esbuild"; +import { bundleRuntime } from "../../scripts/bundle-runtime.mjs"; +import { resolveBundledPluginRuntimeLoader } from "../services/bundled-plugin-runtime.js"; + +const roots: string[] = []; +function fixture() { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-plugin-runtime-"))); + roots.push(root); + fs.mkdirSync(path.join(root, "server", "dist-runtime"), { recursive: true }); + fs.writeFileSync(path.join(root, "server", "package.json"), JSON.stringify({ name: "@paperclipai/server", type: "module" })); + const loader = path.join(root, "cli", "node_modules", "tsx", "dist", "loader.mjs"); + fs.mkdirSync(path.dirname(loader), { recursive: true }); + fs.writeFileSync(loader, "export {};\n"); + return { root, loader }; +} +afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); + +describe("current image plugin runtime loader", () => { + it.each(["src/services", "dist/services", "dist-runtime"])("resolves the owning release from server/%s", (location) => { + const { root, loader } = fixture(); + const moduleDirectory = path.join(root, "server", location); + fs.mkdirSync(moduleDirectory, { recursive: true }); + expect(resolveBundledPluginRuntimeLoader(moduleDirectory)).toBe(loader); + }); + + it("resolves a bundled entrypoint without preserved source module locations", async () => { + const { root, loader } = fixture(); + const runtime = path.join(root, "server", "dist-runtime", "index.mjs"); + await build({ + stdin: { contents: `import { resolveBundledPluginRuntimeLoader } from ${JSON.stringify(fileURLToPath(new URL("../services/bundled-plugin-runtime.ts", import.meta.url)))}; process.stdout.write(resolveBundledPluginRuntimeLoader());`, resolveDir: process.cwd() }, + outfile: runtime, bundle: true, platform: "node", format: "esm", target: "node24", logLevel: "silent", + }); + expect(execFileSync(process.execPath, [runtime], { encoding: "utf8" })).toBe(loader); + }); + + it("uses the production bundler's preserved module location at the dist-runtime entrypoint", async () => { + const { root, loader } = fixture(); + const service = path.join(root, "server", "dist", "services", "bundled-plugin-runtime.js"); + fs.mkdirSync(path.dirname(service), { recursive: true }); + const source = fs.readFileSync(new URL("../services/bundled-plugin-runtime.ts", import.meta.url), "utf8"); + fs.writeFileSync(service, (await transform(source, { loader: "ts", format: "esm", target: "node24" })).code); + const entry = path.join(root, "server", "dist", "probe.js"); + fs.writeFileSync(entry, 'import { resolveBundledPluginRuntimeLoader } from "./services/bundled-plugin-runtime.js"; process.stdout.write(resolveBundledPluginRuntimeLoader());'); + const bundled = await bundleRuntime({ entry, outdir: path.join(root, "server", "dist-runtime") }); + expect(execFileSync(process.execPath, [bundled.outfile], { encoding: "utf8" })).toBe(loader); + }); + + it("accepts the baked pnpm target inside the current image node_modules", () => { + const { root, loader } = fixture(); + const physical = path.join(root, "node_modules", ".pnpm", "tsx@1", "node_modules", "tsx", "dist", "loader.mjs"); + fs.mkdirSync(path.dirname(physical), { recursive: true }); + fs.writeFileSync(physical, "export {};\n"); + fs.rmSync(loader); fs.symlinkSync(path.relative(path.dirname(loader), physical), loader); + expect(resolveBundledPluginRuntimeLoader(path.join(root, "server", "dist-runtime"))).toBe(physical); + }); + + it("fails explicitly for missing or legacy-home loaders", () => { + const { root, loader } = fixture(); + const moduleDirectory = path.join(root, "server", "dist-runtime"); + fs.rmSync(loader); + expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("missing from the current image"); + const outside = fixture().loader; + fs.symlinkSync(outside, loader); + expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("escapes the current image"); + }); +}); diff --git a/server/src/__tests__/cloud-bundled-package-digest.test.ts b/server/src/__tests__/cloud-bundled-package-digest.test.ts new file mode 100644 index 00000000000..67491571762 --- /dev/null +++ b/server/src/__tests__/cloud-bundled-package-digest.test.ts @@ -0,0 +1,85 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js"; + +const roots: string[] = []; +function file(name: string, content = "export const value = 1;") { + fs.mkdirSync(path.dirname(name), { recursive: true }); fs.writeFileSync(name, content); +} +function link(target: string, name: string) { + fs.mkdirSync(path.dirname(name), { recursive: true }); fs.symlinkSync(path.relative(path.dirname(name), target), name); +} +function fixture() { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-package-graph-"))); roots.push(root); + const catalogRoot = path.join(root, "packages", "plugins"); + const provider = path.join(catalogRoot, "sandbox-providers", "daytona"); + const sdk = path.join(catalogRoot, "sdk"); + const shared = path.join(root, "packages", "shared"); + const dependency = path.join(root, "node_modules", ".pnpm", "dependency@1", "node_modules", "dependency"); + const transitive = path.join(root, "node_modules", ".pnpm", "transitive@1", "node_modules", "transitive"); + for (const directory of [provider, sdk, shared, dependency, transitive]) file(path.join(directory, "index.js")); + link(sdk, path.join(provider, "node_modules", "@paperclipai", "plugin-sdk")); + link(shared, path.join(sdk, "node_modules", "@paperclipai", "shared")); + link(dependency, path.join(shared, "node_modules", "dependency")); + link(transitive, path.join(path.dirname(dependency), "transitive")); + return { root, catalogRoot, provider, sdk, shared, dependency, transitive }; +} +afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); + +describe("bounded current-image package dependency inventory", () => { + it("hashes external SDK, shared, and pnpm sibling dependency bytes", () => { + const image = fixture(); + const digest = () => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot }); + let previous = digest(); + for (const directory of [image.sdk, image.shared, image.dependency, image.transitive]) { + file(path.join(directory, "index.js"), "export const value = 2;"); + const next = digest(); expect(next).not.toBe(previous); previous = next; + } + }); + + it("keeps the digest stable when an identical release moves to a new root", () => { + const original = fixture(); const replacement = fixture(); + expect(cloudBundledPackageDigest(original.provider, { catalogRoot: original.catalogRoot })) + .toBe(cloudBundledPackageDigest(replacement.provider, { catalogRoot: replacement.catalogRoot })); + }); + + it("rejects legacy-home links and preserves package-contained defaults", () => { + const image = fixture(); + expect(() => cloudBundledPackageDigest(image.provider)).toThrow("escapes its package"); + const outside = fixture(); + link(path.join(outside.sdk, "index.js"), path.join(image.shared, "legacy-home.js")); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("authorized image roots"); + }); + + it("deduplicates cycles without scanning unrelated image node_modules", () => { + const image = fixture(); + link(image.sdk, path.join(image.shared, "node_modules", "cycle")); + const unrelated = path.join(image.root, "node_modules", "unrelated", "special"); + file(unrelated); + const first = cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot }); + file(unrelated, "changed unrelated code"); + expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toBe(first); + }); + + it("accepts the shipped provider's observed entry class", () => { + const image = fixture(); + const directory = path.join(image.provider, "many"); fs.mkdirSync(directory); + for (let index = 0; index < 16_000; index++) fs.writeFileSync(path.join(directory, `${index}.js`), ""); + expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toMatch(/^sha256:/); + }); + + it("rejects oversized metadata and excessive depth", () => { + const image = fixture(); + file(path.join(image.sdk, "package.json"), " ".repeat(128 * 1024 + 1)); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("metadata"); + fs.rmSync(path.join(image.sdk, "package.json")); + const large = path.join(image.shared, "large.js"); + fs.closeSync(fs.openSync(large, "w")); fs.truncateSync(large, 256 * 1024 * 1024 + 1); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("byte limit"); + fs.rmSync(large); + file(path.join(image.sdk, ...Array.from({ length: 33 }, () => "deep"), "file.js")); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("depth limit"); + }); +}); diff --git a/server/src/services/bundled-plugin-package.ts b/server/src/services/bundled-plugin-package.ts index a80446a824b..4d85c4f93f1 100644 --- a/server/src/services/bundled-plugin-package.ts +++ b/server/src/services/bundled-plugin-package.ts @@ -106,4 +106,3 @@ export function createBundledPluginPackageResolver(installs: readonly ResolvedBu }, }; } - diff --git a/server/src/services/bundled-plugin-runtime.ts b/server/src/services/bundled-plugin-runtime.ts new file mode 100644 index 00000000000..cae4b5e80ab --- /dev/null +++ b/server/src/services/bundled-plugin-runtime.ts @@ -0,0 +1,37 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export function resolveBundledPluginRuntimeLoader( + moduleDirectory = path.dirname(fileURLToPath(import.meta.url)), +): string { + let directory = fs.realpathSync(moduleDirectory); + let imageRoot: string | undefined; + for (let depth = 0; depth < 5; depth++) { + const metadataPath = path.join(directory, "package.json"); + const metadata = fs.lstatSync(metadataPath, { throwIfNoEntry: false }); + if (metadata?.isFile() && metadata.size <= 128 * 1024 + && JSON.parse(fs.readFileSync(metadataPath, "utf8")).name === "@paperclipai/server") { + imageRoot = path.dirname(directory); + break; + } + const parent = path.dirname(directory); + if (parent === directory) break; + directory = parent; + } + if (!imageRoot) throw new Error("Managed plugin server package is missing from the current image"); + const declared = path.join(imageRoot, "cli/node_modules/tsx/dist/loader.mjs"); + let loader: string; + try { + loader = fs.realpathSync(declared); + if (!fs.statSync(loader).isFile()) throw new Error("not a file"); + } catch { + throw new Error("Managed plugin runtime loader is missing from the current image"); + } + const roots = [path.join(imageRoot, "cli"), path.join(imageRoot, "node_modules")]; + if (!roots.some((root) => { + const relative = path.relative(root, loader); + return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative); + })) throw new Error("Managed plugin runtime loader escapes the current image"); + return loader; +} diff --git a/server/src/services/cloud-bundled-package-digest.ts b/server/src/services/cloud-bundled-package-digest.ts index a7695fd31f0..47d3f209353 100644 --- a/server/src/services/cloud-bundled-package-digest.ts +++ b/server/src/services/cloud-bundled-package-digest.ts @@ -3,45 +3,89 @@ import path from "node:path"; import { createHash } from "node:crypto"; import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js"; -export function cloudBundledPackageDigest(packageRoot: string): string { +export function cloudBundledPackageDigest(packageRoot: string, options: { catalogRoot?: string } = {}): string { const root = fs.realpathSync(packageRoot); - const inventory: Array<[string, string, string, number]> = []; + const roots = [{ key: "package", directory: root }]; + let imageNodeModules: string | undefined; + if (options.catalogRoot) { + const catalogRoot = fs.realpathSync(options.catalogRoot); + const relative = path.relative(catalogRoot, root); + if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Image package is outside the selected catalog root"); + imageNodeModules = path.resolve(catalogRoot, "../../node_modules"); + roots.push( + { key: "plugin-sdk", directory: path.join(catalogRoot, "sdk") }, + { key: "shared", directory: path.resolve(catalogRoot, "../shared") }, + { key: "image-node-modules", directory: imageNodeModules }, + ); + } + roots.sort((left, right) => right.directory.length - left.directory.length); + const inventory = new Map(); + const directories = new Set(); + const files = new Set(); + const buffer = Buffer.alloc(65_536); let bytes = 0; let entries = 0; - function walk(directory: string, relative = "") { - if (relative.split("/").length > 32) throw new Error("Image package exceeds the directory depth limit"); - for (const name of fs.readdirSync(directory).sort()) { - if (++entries > 10_000) throw new Error("Image package exceeds the entry limit"); - const local = relative ? `${relative}/${name}` : name; - const file = path.join(directory, name); - const stat = fs.lstatSync(file); - if (stat.isSymbolicLink()) { - const target = fs.realpathSync(file); - const contained = path.relative(root, target); - if (!contained || contained.startsWith("..") || path.isAbsolute(contained)) throw new Error("Image package symlink escapes its package"); - const targetStat = fs.statSync(target); - if (!targetStat.isFile() && !targetStat.isDirectory()) throw new Error("Image package symlink has an unsupported target"); - inventory.push([local, "link", fs.readlinkSync(file), stat.mode & 0o777]); - } else if (stat.isDirectory()) walk(file, local); - else if (stat.isFile()) { - if (bytes + stat.size > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit"); - const handle = fs.openSync(file, "r"); - const hash = createHash("sha256"); - const buffer = Buffer.alloc(65_536); - try { - while (true) { - const count = fs.readSync(handle, buffer); - if (!count) break; - bytes += count; - if (bytes > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit"); - hash.update(buffer.subarray(0, count)); - } - } finally { fs.closeSync(handle); } - inventory.push([local, "file", `sha256:${hash.digest("hex")}`, stat.mode & 0o777]); - } else throw new Error("Image package contains a special file"); + + function logical(file: string): string { + for (const candidate of roots) { + const relative = path.relative(candidate.directory, file); + if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) { + if (relative.split(path.sep).length > 32) throw new Error("Image package exceeds the directory depth limit"); + return `${candidate.key}/${relative.split(path.sep).join("/")}`; + } } + throw new Error(options.catalogRoot ? "Image dependency escapes its authorized image roots" : "Image package symlink escapes its package"); + } + + function follow(target: string) { + logical(target); + if (imageNodeModules) { + const relative = path.relative(imageNodeModules, target).split(path.sep); + if (relative[0] === ".pnpm" && relative.length > 2 && relative[1] && relative[1] !== "node_modules") { + walk(path.join(imageNodeModules, ".pnpm", relative[1])); + return; + } + } + walk(target); + } + + function walk(file: string) { + const name = logical(file); + const stat = fs.lstatSync(file); + if (stat.isSymbolicLink()) { + const target = fs.realpathSync(file); + inventory.set(name, [name, "link", logical(target), stat.mode & 0o777]); + follow(target); + } else if (stat.isDirectory()) { + const real = fs.realpathSync(file); + logical(real); + if (directories.has(real)) return; + directories.add(real); + for (const child of fs.readdirSync(real).sort()) { + if (++entries > 50_000) throw new Error("Image package exceeds the entry limit"); + walk(path.join(real, child)); + } + } else if (stat.isFile()) { + const real = fs.realpathSync(file); + if (files.has(real)) return; + files.add(real); + if (path.basename(file) === "package.json" && stat.size > 128 * 1024) throw new Error("Image package metadata exceeds the byte limit"); + if (bytes + stat.size > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit"); + const handle = fs.openSync(file, "r"); + const hash = createHash("sha256"); + try { + while (true) { + const count = fs.readSync(handle, buffer); + if (!count) break; + bytes += count; + if (bytes > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit"); + hash.update(buffer.subarray(0, count)); + } + } finally { fs.closeSync(handle); } + inventory.set(name, [name, "file", `sha256:${hash.digest("hex")}`, stat.mode & 0o777]); + } else throw new Error("Image package contains a special file"); } walk(root); - if (!inventory.length) throw new Error("Image package is empty"); - return cloudConfigurationDigest(inventory); + if (!inventory.size) throw new Error("Image package is empty"); + return cloudConfigurationDigest([...inventory.values()].sort((left, right) => left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0)); } diff --git a/server/src/services/cloud-configuration-migration.ts b/server/src/services/cloud-configuration-migration.ts index b5c8e3450f0..3ce38234408 100644 --- a/server/src/services/cloud-configuration-migration.ts +++ b/server/src/services/cloud-configuration-migration.ts @@ -140,6 +140,7 @@ export function cloudConfigurationMigrationService(db: CloudConfigurationDb, opt }, async qualify(input: { installs: readonly ResolvedBundledPlugin[]; + catalogRoot?: string; loadManifest: (packageRoot: string) => Promise; expectedPackageEvidenceDigest?: string; imageCatalogUnavailable?: boolean; diff --git a/server/src/services/cloud-configuration-qualification.ts b/server/src/services/cloud-configuration-qualification.ts index 07094f0fc36..c96c97ae8e5 100644 --- a/server/src/services/cloud-configuration-qualification.ts +++ b/server/src/services/cloud-configuration-qualification.ts @@ -29,5 +29,5 @@ export async function qualifyCloudConfigurationForCurrentImage(db: Db, options: prebuiltOnly: true, enableLocalFilesystem: false, enableNpmDiscovery: false, assertPackageActivation: resolver.assertActivation, resolvePackageRoot: resolver.resolve, }); - return service.qualify({ installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest }); + return service.qualify({ installs, catalogRoot, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest }); } diff --git a/server/src/services/cloud-package-qualification.ts b/server/src/services/cloud-package-qualification.ts index ffdf05da861..d0d75f30cf7 100644 --- a/server/src/services/cloud-package-qualification.ts +++ b/server/src/services/cloud-package-qualification.ts @@ -20,6 +20,7 @@ export type CloudPackageEvidence = z.infer; export async function inspectCloudPackages(input: { installs: readonly ResolvedBundledPlugin[]; registrations: readonly BundledPluginRegistration[]; + catalogRoot?: string; loadManifest: (packageRoot: string) => Promise; }) { const started = performance.now(); @@ -41,7 +42,7 @@ export async function inspectCloudPackages(input: { try { resolver.assertActivation({ packageRoot: install.localPath }); stage = "unverified_package_integrity"; - evidence.packageDigest = install.distribution ? distributionBundleDigest(install.localPath) : cloudBundledPackageDigest(install.localPath); + evidence.packageDigest = install.distribution ? distributionBundleDigest(install.localPath) : cloudBundledPackageDigest(install.localPath, { catalogRoot: input.catalogRoot }); if (install.distribution && evidence.packageDigest !== install.distribution.digest) { blockers.push({ code: "package_integrity_mismatch", pluginKey: install.pluginKey, reason: "Current package bytes do not match the verified distribution digest" }); continue; diff --git a/server/src/services/plugin-loader.ts b/server/src/services/plugin-loader.ts index ab5fb092786..1928ff94c0f 100644 --- a/server/src/services/plugin-loader.ts +++ b/server/src/services/plugin-loader.ts @@ -1,3 +1,4 @@ +import { resolveBundledPluginRuntimeLoader } from "./bundled-plugin-runtime.js"; import { finishServiceShutdown } from "../shutdown.js"; /** * PluginLoader — discovery, installation, and runtime activation of plugins. @@ -2349,10 +2350,9 @@ export function pluginLoader( proactiveCompanyScopes: configRows.map((row) => row.companyId), }; - // Repo-local plugin installs can resolve workspace TS sources at runtime - // (for example @paperclipai/shared exports). Run those workers through - // the tsx loader so first-party example plugins work in development. - if (!prebuiltOnly && activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) { + if (prebuiltOnly) { + workerOptions.execArgv = ["--import", resolveBundledPluginRuntimeLoader()]; + } else if (activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) { workerOptions.execArgv = ["--import", DEV_TSX_LOADER_PATH]; }