From 8f822bfcf71ff5a9e91e9af4e43a5bf777e4d730 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:21:33 -0500 Subject: [PATCH] fix(cloud): persist configuration and load release-image packages (#18335) Land the approved Cloud configuration and release-image package stack. Includes the separately reviewed adapter settings (#18335), external adapters (#18333), bundled plugins (#18336), startup configuration (#18339), and migration/package checks (#18341). Includes the final image-owned loader and linked-dependency inventory corrections. Verified stack head: 462716170dca59fa2a0dd1b754b8644ed64462c4. Co-Authored-By: Paperclip --- doc/DATABASE.md | 63 ++++ doc/DEVELOPING.md | 13 +- doc/cloud-configuration-migration.md | 148 ++++++++ doc/cloud-runtime-configuration.md | 118 +++++++ doc/plugins/PLUGIN_SPEC.md | 36 ++ maps/agent-access-map/instance-operations.md | 32 ++ maps/agent-access-map/runs-adapters.md | 9 +- maps/feature-map/instance-operations.md | 30 ++ maps/feature-map/runs-adapters.md | 9 + .../sandbox-providers/daytona/package.json | 2 +- .../sandbox-providers/kubernetes/package.json | 2 +- .../__tests__/adapter-routes-authz.test.ts | 54 ++- .../agent-test-environment-routes.test.ts | 5 + .../__tests__/bundled-plugin-package.test.ts | 86 +++++ .../bundled-plugin-registration.test.ts | 287 ++++++++++++++++ .../__tests__/bundled-plugin-runtime.test.ts | 72 ++++ server/src/__tests__/bundled-plugins.test.ts | 16 + .../cloud-bundled-package-digest.test.ts | 93 +++++ ...d-configuration-migration-postgres.test.ts | 325 ++++++++++++++++++ .../cloud-configuration-readers.test.ts | 48 +++ .../__tests__/cloud-runtime-config.test.ts | 302 ++++++++++++++++ .../helpers/bundled-image-fixture.ts | 50 +++ .../server-startup-feedback-export.test.ts | 81 +++++ .../adapters/cloud-external-adapters.test.ts | 136 ++++++++ server/src/adapters/plugin-loader.ts | 7 + server/src/adapters/registry.ts | 15 +- server/src/app.ts | 17 +- server/src/config-file.ts | 3 + server/src/config.ts | 68 ++-- server/src/index.ts | 63 ++-- server/src/paths.ts | 2 + server/src/qualify-cloud-configuration.ts | 47 +++ server/src/routes/adapters.ts | 64 ++-- server/src/routes/agents.ts | 13 +- server/src/routes/plugin-ui-static.ts | 19 +- server/src/services/adapter-code-policy.ts | 10 + server/src/services/adapter-plugin-store.ts | 14 +- ...apter-registry-bootstrap.reconcile.test.ts | 3 +- .../services/adapter-registry-bootstrap.ts | 72 +--- .../src/services/adapter-registry-config.ts | 47 +++ .../services/adapter-settings-local.test.ts | 87 +++++ .../adapter-settings-postgres.test.ts | 228 ++++++++++++ server/src/services/adapter-settings.ts | 149 ++++++++ .../services/agent-runner-selection.test.ts | 18 +- server/src/services/agent-runner-selection.ts | 33 +- server/src/services/bundled-plugin-package.ts | 108 ++++++ server/src/services/bundled-plugin-runtime.ts | 37 ++ server/src/services/bundled-plugins.ts | 17 +- .../services/cloud-bundled-package-digest.ts | 95 +++++ .../services/cloud-configuration-evidence.ts | 74 ++++ .../services/cloud-configuration-migration.ts | 193 +++++++++++ .../cloud-configuration-qualification.ts | 33 ++ .../services/cloud-package-qualification.ts | 72 ++++ .../services/cloud-runtime-config-snapshot.ts | 129 +++++++ server/src/services/cloud-runtime-config.ts | 105 ++++++ server/src/services/plugin-loader.ts | 38 +- server/src/services/plugin-registry.ts | 23 +- 57 files changed, 3670 insertions(+), 250 deletions(-) create mode 100644 doc/cloud-configuration-migration.md create mode 100644 doc/cloud-runtime-configuration.md create mode 100644 server/src/__tests__/bundled-plugin-package.test.ts create mode 100644 server/src/__tests__/bundled-plugin-registration.test.ts create mode 100644 server/src/__tests__/bundled-plugin-runtime.test.ts create mode 100644 server/src/__tests__/cloud-bundled-package-digest.test.ts create mode 100644 server/src/__tests__/cloud-configuration-migration-postgres.test.ts create mode 100644 server/src/__tests__/cloud-configuration-readers.test.ts create mode 100644 server/src/__tests__/cloud-runtime-config.test.ts create mode 100644 server/src/__tests__/helpers/bundled-image-fixture.ts create mode 100644 server/src/adapters/cloud-external-adapters.test.ts create mode 100644 server/src/qualify-cloud-configuration.ts create mode 100644 server/src/services/adapter-code-policy.ts create mode 100644 server/src/services/adapter-registry-config.ts create mode 100644 server/src/services/adapter-settings-local.test.ts create mode 100644 server/src/services/adapter-settings-postgres.test.ts create mode 100644 server/src/services/adapter-settings.ts create mode 100644 server/src/services/bundled-plugin-package.ts create mode 100644 server/src/services/bundled-plugin-runtime.ts create mode 100644 server/src/services/cloud-bundled-package-digest.ts create mode 100644 server/src/services/cloud-configuration-evidence.ts create mode 100644 server/src/services/cloud-configuration-migration.ts create mode 100644 server/src/services/cloud-configuration-qualification.ts create mode 100644 server/src/services/cloud-package-qualification.ts create mode 100644 server/src/services/cloud-runtime-config-snapshot.ts create mode 100644 server/src/services/cloud-runtime-config.ts diff --git a/doc/DATABASE.md b/doc/DATABASE.md index ab7d6cf7cb4..b794f1e51ee 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -722,6 +722,60 @@ connection, before enabling the new server and UI. - The repo keeps only the newest 5 snapshots. `generate` runs `prune:snapshots` afterwards to delete older ones. Drizzle only reads the newest snapshot, and each snapshot is a full copy of the schema (over 1 MB each). Older snapshots are still in git history. - `packages/db/src/migration-snapshot-drift.test.ts` is the enforcement backstop. It repeats the diff that `generate` performs and fails when the newest snapshot no longer matches `packages/db/src/schema/`. +## Managed adapter settings + +Cloud-managed instances keep adapter selection preferences in the private +`instance_settings` singleton `adapter-settings:v1`. Its `general` JSON contains +`version: 1`, a monotonic `revision`, and `disabledTypes`. It is separate from +the public `default` settings row and never appears in the instance settings API. +Each update locks this row in a transaction. The update and company activity +records commit together. Discovery and new-agent admission read the database +at their asynchronous operation boundaries; no polling or mutable settings cache +is used. Disabled adapters remain available to existing agents. + +`PAPERCLIP_ADAPTERS` and `PAPERCLIP_ADAPTERS_FILE` declare the effective operator +adapter list. The declaration is parsed once for a process configuration and +validated at startup. It does not rewrite stored preferences. A conflicting +adapter toggle returns 403. Removing the declaration restores saved choices. +Operator file edits require a restart. Self-hosted instances continue to use +`adapter-settings.json` for saved preferences. + +The server-only adapter settings service exposes `inspectStored()` and +`seedIfAbsent({ disabledTypes }, transaction?)` for migration tooling. Seeding +uses the unique singleton key, returns `{ seeded, current }`, and never replaces +an existing record. The caller can use the same transaction for its import +receipt. Before admission, `initializeAdapterSettings` checks the durable row. +When absent, it imports a valid legacy settings file once, retaining unknown +adapter types, or seeds empty preferences if no file exists. It rejects files +larger than 64 KiB, malformed input, and read failures. An existing durable row +wins over every stale file. Ordinary Cloud reads do not read a legacy file. + +The focused PostgreSQL suite can use an owned disposable PostgreSQL database via +`PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL` when embedded PostgreSQL is +unavailable. It applies migrations and inserts test data, so never point this +variable at an existing application database. + +## Cloud configuration migration receipt + +The private `instance_settings` singleton `cloud-configuration-migration:v1` +records atomic import completion, semantic source digests, imported revisions, +and metadata-only external adapter blockers. It composes the adapter and +startup-config owners without replacing an existing authoritative row. Receipt +locking and the import transaction handle concurrent attempts and retries; no +local completion marker participates in conflict resolution. + +Qualification records current configuration revisions and selected image package +evidence before normal plugin reconciliation. Image inspection does not hold +admin write locks; configuration and registration snapshots are rechecked in a +short transaction before the report is saved. A historical successful report +never grants current eligibility. External metadata blockers survive loss of the +legacy home and require a deliberate revision-checked recheck of a present valid +original source file. Package mismatches remain recorded after maintenance +reconciliation. See the [bounded migration and qualification procedure](cloud-configuration-migration.md) +for the stopped-writer prerequisite, commands, retry semantics, integrity limits, +and the explicit exclusion of secret bytes, whole-server zero downtime, and +volume removal. + ## Cloud runtime identity singleton The private `instance_settings` row whose singleton key is @@ -1490,3 +1544,12 @@ It uses disposable embedded PostgreSQL by default. A disposable local database named `review_wait_benchmark...` can be supplied through `PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version. DataRow bytes exclude response metadata and transport overhead. + +## Cloud runtime configuration singleton + +The private `instance_settings` key `cloud-runtime-config:v1` holds a strictly +validated, versioned nonsecret configuration and revision. It is separate from +the ordinary instance settings API. Seed inserts preserve an existing winner; +replacements lock the row and reject stale revisions. Database/transaction +handles permit atomic migration receipts. Connection URLs, API keys and unknown +fields are excluded. See [field ownership and startup order](cloud-runtime-configuration.md). diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7e53190d2c1..04b519620da 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1542,7 +1542,9 @@ Paperclip also persists an empty `OPENAI_API_KEY` override for those agents so a If the `codex` CLI is not installed or not on `PATH`, `codex_local` agent runs fail at execution time with a clear adapter error. Quota polling uses a short-lived `codex app-server` subprocess: when `codex` cannot be spawned, that provider reports `ok: false` in aggregated quota results and the API server keeps running (it must not exit on a missing binary). -Local adapters require their corresponding CLI/session setup on the machine running Paperclip. External adapters are installed through the adapter/plugin flow and should not require hardcoded imports in `server/` or `ui/`. +Local adapters require their corresponding CLI/session setup on the machine running Paperclip. Self-hosted instances can install external adapters through the adapter/plugin flow without hardcoded imports in `server/` or `ui/`. + +Cloud-managed instances use only adapter implementations supplied by the release image. A tenant server token or the presence of `PAPERCLIP_MANAGED_CONFIG` enforces this boundary. Startup and UI parser requests do not read external adapter registrations or packages. Install, reinstall, reload, remove, and override operations return `403` with `adapter_install_platform_managed`, including for instance admins. Legacy `adapter-plugins.json` files stay untouched and do not change the built-in registry. ### Filesystem access boundary (2026-10-20) @@ -3869,3 +3871,12 @@ in `_test` for `PAPERCLIP_MCP_SLEEP_TEST_DATABASE_URL`, `PAPERCLIP_DOT_RUNNER_TEST_DATABASE_URL`. The interaction-service regression suite also accepts `PAPERCLIP_INTERACTIONS_TEST_DATABASE_URL`. The suites migrate and write to those URLs; the sleep suite truncates company data between tests. + +## Cloud startup configuration + +Cloud stores nonsecret startup choices in a private PostgreSQL singleton. +Bootstrap connection credentials stay in the supplied environment. A bounded +one-time config-file seed preserves existing choices, then later starts use the +database without loading home `.env` files or repairing worktree config. See +[Cloud runtime configuration](cloud-runtime-configuration.md) for field ownership, +secret handling, initialization order and migration interfaces. diff --git a/doc/cloud-configuration-migration.md b/doc/cloud-configuration-migration.md new file mode 100644 index 00000000000..519c0aee181 --- /dev/null +++ b/doc/cloud-configuration-migration.md @@ -0,0 +1,148 @@ +# Cloud configuration migration and package qualification + +This procedure covers saved adapter preferences, the allowlisted nonsecret +startup configuration, and selected bundled plugin packages. The result is a +point-in-time report. It does not authorize a deployment, guarantee zero +downtime, establish general Railway admission, or prove that a volume can be +removed. Secret and key bytes, database credentials, artifacts, logs, local +storage, workspaces, and plugin state outside PostgreSQL need their own storage +and replacement checks. This includes the decision signing key and local +encryption keys. Their byte authority is unchanged by this procedure. +Runtime environment and secret equality, plus an actual HTTP handoff between +application processes, remain separate rollout prerequisites. The package digest +does not establish those conditions. + +## Migration prerequisite + +Stop every legacy file writer before the first import. This includes old +application replicas and operator commands that can write configuration or +adapter settings. One maintenance window is sufficient. The importer cannot +make an old file writer participate in its database transaction. Keep those +writers stopped after import; later file edits do not update database authority. + +Keep the normal database backup and recovery procedure for the instance. Use the +new release with the same managed environment and database connection. The +server runs migration before constructing configuration-dependent services. +For an explicit source-checkout invocation, use: + +```sh +pnpm exec tsx server/src/qualify-cloud-configuration.ts --migrate +``` + +The command uses `DATABASE_URL` and the normal Cloud environment. It does not +accept credentials as arguments. It writes the receipt to PostgreSQL and prints +a JSON report; no report file or local completion marker is required. Exit code +0 means the requested configuration/package check passed, 2 means the report has +blockers, and 1 means the operation failed. A fresh installation can report a +missing plugin registration before normal boot installation completes. Run the +check again after that installation settles. + +## Database authority and retry behavior + +The private `instance_settings` singleton `cloud-configuration-migration:v1` +contains the versioned receipt. The transaction locks that row and composes the +adapter and startup-config owners' atomic seed methods. Completion and both +seeds commit together. A failed write or crash before commit leaves no completed +receipt. A retry uses committed database state and never replaces a later +settings revision. Earlier independently seeded owner records remain authoritative. + +The receipt records source kind, a digest of the nonsecret semantic candidate, +the imported revision, and bounded legacy external-registration evidence. +An absent startup configuration remains an authoritative `config: null`; it is +distinct from an uninitialized database record. No raw secret values are copied. +Once the receipt exists, ordinary boots do not reread legacy source files. +A replacement home can be empty or absent without changing saved authority. + +All legacy readers have a 64 KiB limit. They require regular files, reject +symlinks and FIFOs, and use bounded reads. External registration inspection reads +metadata only. It never loads a registered module or follows a registered package +path. A nonempty, invalid, or unreadable registration file creates a durable +blocker. Removing the old home does not clear that blocker. + +To resolve an external-registration blocker, stop its writer, deliberately +remove the unsupported registration metadata at the original source location, +and leave a present valid empty JSON array file. Read the current receipt revision +from the report, then run: + +```sh +pnpm exec tsx server/src/qualify-cloud-configuration.ts --recheck-external-revision 3 +``` + +Use the actual current revision in place of `3`. A changed revision, a different +source location, or a missing/invalid source file rejects the recheck. Reusing +the same container path with an empty home is insufficient. This is an explicit +operator resolution; qualification never clears the blocker automatically. + +## Current packages and overlap comparison + +Every qualification reads current settings revisions and plugin registrations. +It resolves selected packages from this image with the same bundled package +identity rules as normal loading. It verifies package bytes before importing a +trusted image manifest, then compares that manifest with the approved database +registration. Missing packages, missing evidence, incompatible versions, +unselected registrations, and integrity failures prevent qualification. Legacy +package paths and npm discovery cannot supply missing image bytes. + +Image inspection happens outside database write locks. A short transaction then +locks and rereads the current configuration and registration records. A change +during inspection produces a blocker and requires a new check. The report gives +`adapterRevision`, `configRevision`, `configurationDigest`, and +`packageEvidenceDigest`. A historical completed receipt is evidence, not current +authorization. Call qualification again for a current result. + +The server records this report before ordinary bundled-plugin reconciliation. +It retains `lastPackageMismatch` after later successful checks, so reconciliation +cannot erase the evidence seen before an upgrade. The report does not block +normal maintenance upgrades or replace capability approval. After an upgrade +settles, its current packages can qualify normally. + +For a same-package comparison, capture the current release's +`packageEvidenceDigest` and pass it explicitly on the replacement: + +```sh +pnpm exec tsx server/src/qualify-cloud-configuration.ts --expected-package-evidence sha256:REPLACE_WITH_THE_64_HEX_DIGEST +``` + +`configurationPackagesEligible` describes current database/configuration/package +compatibility. `samePackageOverlapEligible` is true only when an explicit expected +digest also matches. There is no permanent first-release baseline that would +prevent future maintenance releases from establishing their own evidence. + +The package evidence digest covers selected packages, not container image bytes. +Distribution packages reuse and reverify their catalog digest. Builtin packages +have a bounded inventory of physical file bytes, modes, and logical symlink +targets, including standalone pnpm dependency trees. With the selected catalog +root, inventory may follow links only into that provider, the catalog SDK, +`packages/shared`, and the current image's root `node_modules`. Reached pnpm +package units include their sibling dependency links; unrelated root packages +are not scanned. Canonical directory/file deduplication bounds cycles, and +logical root labels preserve the digest when identical image bytes relocate. +Dangling or escaping links, special files, more than 50,000 entries, depth above +32, metadata above 128 KiB, or more than 256 MiB per package fail verification. +The measured cached Cloud Daytona closure has about 22,253 entries and 130 MB; +that artifact predates this change and is not a complete current release proof. +Inventory work is synchronous and linear in selected package bytes. It runs only during startup before the application is constructed +or in the explicit operator process, with no idle polling. The report includes +`packageInspectionDurationMs`. Packages must remain immutable throughout a check. + +## Verification scope + +The focused PostgreSQL tests use two independent clients and separate home/image +roots. They import legacy settings, change saved revisions, remove the old roots, +reconstruct from the database, and start a real plugin worker from the replacement +image fixture. They also cover concurrent migration, commit failure and retry, +database failure, stale sources, durable external blockers, package mismatch, +maintenance reconciliation, and changes during manifest inspection. + +The bundled-plugin registration suite also has an actual prebuilt Daytona case. +Build its standalone package with the normal SDK linker before running that case; +it is skipped when the provider build is absent. It qualifies the real linked +SDK/shared package graph and uses the production loader and worker manager for +credential-free initialize/health calls. The runtime-path tests also invoke the +production server bundler to verify its preserved module locations. + +Set `PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL` only to an owned disposable +PostgreSQL database when embedded PostgreSQL is unavailable. The suite applies +migrations and writes/deletes test records. A complete release image, hosted +replacement, overlapping production processes, secret-byte migration, and volume +removal are not established by these local fixtures. diff --git a/doc/cloud-runtime-configuration.md b/doc/cloud-runtime-configuration.md new file mode 100644 index 00000000000..62f2eabb114 --- /dev/null +++ b/doc/cloud-runtime-configuration.md @@ -0,0 +1,118 @@ +# Cloud runtime configuration + +Cloud-managed servers store nonsecret startup and application choices in the +private `instance_settings` singleton `cloud-runtime-config:v1`. The row contains +`version: 1`, an increasing `revision`, and a strict allowlist of configuration +fields. It is separate from the normal instance settings row and is not exposed +by the instance settings API. + +The canonical managed-instance predicate applies: a nonblank Cloud tenant token +or the presence of `PAPERCLIP_MANAGED_CONFIG`, including a blank value, selects +this configuration path. Invalid managed feature documents still retain their +existing validation at application startup. + +## Startup and authority + +1. The hosting environment supplies `DATABASE_URL` and, if needed, + `DATABASE_MIGRATION_URL`. Cloud requires a PostgreSQL URL and never falls back + to embedded PostgreSQL. Credentials, trust keys, provider configuration and + deployment wiring needed before the database connection remain supplied by + the environment or the existing secret system. This phase parses only the + database connection record. Runtime mode, host binding and authentication + checks wait until the saved snapshot has loaded. +2. After schema migration and database connection, startup reads the private + runtime configuration row. An existing invalid row or a database error stops + startup. Neither condition selects a legacy file or an old process snapshot. +3. If no row exists, startup imports the legacy config once. It uses + `PAPERCLIP_CONFIG` when supplied, otherwise the instance home config path. + It does not search working-directory ancestors. A truly absent file saves + an authoritative `config: null` marker. This retains the existing dynamic + environment/image defaults, including authentication and secret strictness. + The marker never enables a later file fallback. A present malformed, linked, nonregular, or oversized file + stops initialization. Reads and stored snapshots are limited to 64 KiB. +4. The seed uses an insert that does not replace an existing row. Concurrent + initializers read the committed winner. Later starts do not inspect the + legacy file, even when it is absent, corrupt, or changed. +5. Startup publishes a detached configuration snapshot, restores the signed + Cloud runtime identity, then initializes telemetry, secret-provider defaults, + authentication and application services. A saved telemetry opt-out therefore + takes effect before a telemetry client can be created. + +Existing operator environment overlays keep precedence over saved values. +Reading an overlaid value does not save it back to the database. Credentials and +connection URLs are never copied into this JSONB row. + +Cloud does not load instance or working-directory `.env` files and does not run +legacy worktree configuration repair. The hosting system must supply those +bootstrap variables directly. In particular, a database connection kept only +in `.env` cannot bootstrap this path. Self-hosted configuration and dotenv +behavior remain unchanged. + +## Field ownership + +The table lists every field consumed from the existing config file and the +remaining known schema fields. Environment-only `Config` fields, such as +heartbeat maintenance flags and feedback-export credentials, retain their +existing environment sources and are not frozen into the saved snapshot. + +| Config file fields | Cloud owner | +| --- | --- | +| `server.deploymentMode`, `exposure`, `bind`, `customBindHost`, `host`, `port`, `allowedHostnames`, `serveUi` | Saved runtime configuration, with existing environment overlays. | +| `auth.baseUrlMode`, `publicBaseUrl`, `disableSignUp` | Saved runtime configuration, with environment overlays and the signed Cloud identity restoration. Saved URLs cannot contain userinfo, query parameters, or fragments. | +| `database.backup.enabled`, `intervalMinutes`, `retentionDays`, `dir` | Saved runtime configuration, with existing backup environment overlays. | +| `database.mode`, `connectionString` | Cloud bootstrap requires the environment `DATABASE_URL`. The legacy URL is never saved. | +| `database.embeddedPostgresDataDir`, `embeddedPostgresPort` | Local embedded-database settings. Cloud does not use embedded PostgreSQL or these paths. | +| `storage.provider`, `localDisk.baseDir`, `s3.bucket`, `s3.region`, `s3.endpoint`, `s3.prefix`, `s3.forcePathStyle` | Saved runtime configuration, with existing environment overlays. Saved endpoints cannot contain userinfo, query parameters, or fragments. | +| `secrets.provider`, `strictMode`, `localEncrypted.keyFilePath` | Saved nonsecret provider choice and locator, with existing environment overlays. No key bytes are imported or saved. | +| `telemetry.enabled` | Saved choice, passed to the existing telemetry initialization and opt-out policy before client creation. | +| `llm.provider` | Saved provider metadata. | +| `llm.apiKey` | Never saved. The existing OpenAI model-discovery consumer uses `OPENAI_API_KEY` first. A legacy OpenAI key requires that environment credential before import, or startup stops. Claude key metadata has no server consumer. | +| `logging.mode`, `logDir`, `updates.checkEnabled` | Nonsecret legacy/CLI metadata is preserved. Server logging continues to use its existing environment configuration. | +| `$meta` | CLI file-write metadata is not application authority. The compatibility view synthesizes valid metadata. | +| Unknown extension keys | No server config consumer exists. They are excluded from the strict saved allowlist, including any unknown credential fields. | + +A saved path is a locator. This change does not make local storage objects, +backup bytes, logging files, or secret-key bytes durable. Their storage owners +and existing environment/secret mechanisms retain responsibility for those +bytes. Restart tests prove configuration restoration under a different home; +they do not prove durable file, secret, or backup storage. + +The decision signing service still uses `PAPERCLIP_DECISION_SIGNING_SECRET` when +supplied and otherwise reads or creates its instance key file before the database +connection. The local encrypted secret provider still uses its environment key +or existing key-file mechanism. Those byte-storage fallbacks are outside this +configuration migration. Qualification must account for them separately; this +PR does not establish file-free secret storage on Cloud. + +## Internal update and migration interface + +`cloudRuntimeConfigService(db)` provides `inspectStored(handle?)`, +`seedIfAbsent(config, handle?)`, and `replace(config, expectedRevision, handle?)`. +Each method accepts a database or transaction handle. A migration can compose a +config seed, related settings and a receipt in one transaction. + +Replacement validates the same allowlist, locks the singleton row, checks the +expected revision, and increases it. A stale revision fails. There is no new +public configuration endpoint or settings UI. Startup consumers use a snapshot; +a saved replacement takes effect on the next initialization/restart. + +`readLegacyCloudRuntimeConfig(env?)` returns the validated nonsecret candidate +without writing it. `initializeCloudRuntimeConfig` performs the standalone +one-time seed and publishes its committed snapshot. Integrated migration +receipts and deployed Cloud qualification are separate consumers of this +interface. + +## Verification + +`server/src/__tests__/cloud-runtime-config.test.ts` exercises real PostgreSQL +clients, concurrent writes, transaction rollback, changed homes, a fresh Node +process, environment precedence, secret exclusion, invalid data and database +failure. It uses embedded PostgreSQL by default. A dedicated disposable database +can be selected with `PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL` when the host +cannot start embedded PostgreSQL. That database must be reserved for this test. +The test applies the current schema before use. + +The startup orchestration controls in +`server-startup-feedback-export.test.ts` hold configuration initialization and +verify that telemetry, authentication and app construction wait for it. These +local controls do not claim a live Cloud rollout. diff --git a/doc/plugins/PLUGIN_SPEC.md b/doc/plugins/PLUGIN_SPEC.md index a6c744476d8..9e5c8370823 100644 --- a/doc/plugins/PLUGIN_SPEC.md +++ b/doc/plugins/PLUGIN_SPEC.md @@ -34,6 +34,42 @@ Current limitations to keep in mind: In practice, that means the current implementation is a good fit for local development and self-hosted persistent deployments, but not yet for multi-instance cloud plugin distribution. +### Managed image packages + +Managed Cloud provisions only plugins selected from the current release image +catalog. A Cloud tenant token or managed configuration keeps this image-only +policy active. Without an explicit selected package list, Cloud selects no +packages and cannot fall back to local installs. An explicit operator-managed +list also keeps this policy active outside Cloud. The database registration's +plugin key, package name, version and +approved manifest identify that package. Its legacy `packagePath` remains for +local/self-hosted compatibility; Cloud never uses that path to locate code. +Workers, plugin migrations and UI files resolve through the same current-image +package resolver. Cloud does not install npm dependencies, build packages, use +the development UI proxy, or fall back to a server home directory. + +Prebuilt workers use the current release's baked CLI `tsx` loader for the SDK's +workspace TypeScript dependencies. The loader is located from the owning server +package in both regular and bundled server layouts. Missing loaders and loader +links outside the release's CLI or root `node_modules` fail activation. This +loads image-supplied code without installing dependencies or building packages. + +Equivalent packages at different image roots reuse the existing registration, +configuration, approval status and company scopes without rewriting its stored +path. Concurrent first registration of the same package retains one plugin ID. +Conflicting package identities fail explicitly. Normal maintenance boot retains +the existing bundled manifest/version reconciliation and approval gates. + +`createBundledPluginPackageResolver(...).inspect(registration, verifiedManifest)` +is a read-only qualification hook. Resolve the current image path, load its +manifest with the validated plugin loader, and inspect the registration before +normal boot reconciliation. Missing manifest evidence and incompatible packages +produce explicit blockers. Distribution packages retain their catalog digest +verification; built-in packages trust the immutable release image and must bump +their version when their code changes. Qualification supports identical package +sets only. This package-resolution boundary does not coordinate workers, +background jobs, or deployment overlap between servers. + ## 1. Scope This spec covers: diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md index e58f860289c..3e21ef72901 100644 --- a/maps/agent-access-map/instance-operations.md +++ b/maps/agent-access-map/instance-operations.md @@ -74,6 +74,9 @@ API: `present` — representative operation IDs: - `PATCH /api/instance/settings`. - `GET /api/instance/settings/experimental`. - `PATCH /api/instance/settings/experimental`. +- `GET /api/adapters`. +- `GET /api/adapters/{type}`. +- `PATCH /api/adapters/{type}`. Agent access: `restricted` — Board/admin instance settings; CLI configure/env; local configuration files. @@ -88,9 +91,38 @@ Verification: `not-run` — assert the documented agent denial with an owned tes verify any listed agent request/result path separately. A delegated operation needs its own policy and access review. +Cloud runtime configuration uses a separate private PostgreSQL singleton and +adds no public read or mutation endpoint. Existing instance settings APIs do not +expose it. Bootstrap database credentials and trust keys remain in the supplied +environment; the strict saved allowlist excludes API keys and connection URLs. +Internal configuration updates require a matching revision and transaction lock. +[Database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) cover +seed/restart, credential exclusion, conflicts and rollback. These are local +controls, not agent authorization or deployed Cloud qualification. + Gaps: Settings APIs cover parts of configuration; pre-start database/binding/storage configuration is local operator tooling. +Adapter toggles require instance-admin authority and remain blocked when +`instance.adapters` is hidden. Ordinary board members can read the catalog; +agent keys cannot toggle adapters. Managed Cloud preferences are stored in a +private database singleton. The operator adapter declaration sets the effective +list and rejects conflicting toggles. [PostgreSQL regression coverage](../../server/src/services/adapter-settings-postgres.test.ts) +exercises persistence, concurrent writers, admission freshness, activity audit, +and failed writes; [authorization tests](../../server/src/__tests__/adapter-routes-authz.test.ts) +cover admin and hidden-setting gates. These are local tests, not deployed proof. + +Cloud configuration migration and package qualification are operator commands, +not agent-accessible REST capabilities. They require the managed environment and +database operator access. The [command](../../server/src/qualify-cloud-configuration.ts) +composes transaction-safe owner seeds and writes a private receipt. Every +qualification rereads current revisions and image package evidence; a completed +historical receipt grants no deployment or execution authority. [PostgreSQL coverage](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts) +uses independent clients, lost-home reconstruction, a real replacement fixture +worker, rollback/retry, and durable blockers. [Reader coverage](../../server/src/__tests__/cloud-configuration-readers.test.ts) +checks bounded sources and pnpm-style link integrity. These are local fixtures; +see the [scope and operator procedure](../../doc/cloud-configuration-migration.md). + Cloud stack Admins may update experimental settings through the trusted Cloud actor without receiving other instance-admin powers. This uses the current stack role on each request; company membership alone does not grant it. diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index cf775968280..bc842d65d9e 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -55,7 +55,14 @@ Policy: Company model discovery requires company access; account-backed discover also enforces its account/pool and target-agent rules. Instance adapter inventory and installation retain board/admin gates. Delegated managed AI-account login operations are enumerated under `model-accounts`; they do not -grant raw host administration or execution-session reset. +grant raw host administration or execution-session reset. On Cloud-managed +instances, external adapter install, reinstall, reload, removal and override +operations return `403` with `adapter_install_platform_managed`, including for +instance admins. Built-in metadata remains available to board users. Startup +and UI parser reads do not consume saved external registrations or package code. +[Local fixture controls](../../server/src/adapters/cloud-external-adapters.test.ts) +and [route controls](../../server/src/__tests__/adapter-routes-authz.test.ts) +cover this boundary; they do not establish live agent or provider qualification. Evidence: `source-only` — [implementation](../../server/src/routes/agents.ts#L6608) at the reviewed commit above. diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md index f314556396a..7a01c24ebf6 100644 --- a/maps/feature-map/instance-operations.md +++ b/maps/feature-map/instance-operations.md @@ -63,8 +63,38 @@ unfinished application write must still prevent sleep after its client leaves. ### `settings-backup` +Cloud startup reads nonsecret application choices from a private, versioned +PostgreSQL singleton after a bounded one-time legacy config seed. A fresh +process and different home restore the saved choices without consulting a stale +`.env` file. Invalid saved data or an unavailable database prevents startup; +saved telemetry opt-out applies before client creation. The +[database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) and +[startup-order controls](../../server/src/__tests__/server-startup-feedback-export.test.ts) +cover these local boundaries. See [field ownership](../../doc/cloud-runtime-configuration.md). +These controls do not qualify a deployed Cloud rollout or durable file storage. + Automated: [experimental settings](../../ui/src/pages/InstanceExperimentalSettings.test.tsx) and [database backup routes](../../server/src/__tests__/instance-database-backups-routes.test.ts) cover their UI/API contracts. +Cloud migration: in one maintenance window, stop legacy file writers and run the +[configuration/package qualification command](../../doc/cloud-configuration-migration.md). +Inspect its database receipt, update a saved preference, then repeat against a +second owned application/image root with the first home unavailable. Use an +explicit package evidence digest for same-package comparison. Confirm stale +qualification and missing legacy files cannot clear blockers. This recipe covers +configuration/packages and does not qualify whole-server zero downtime, secret +bytes, or volume removal. [Local PostgreSQL evidence path](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts) +includes a real fixture worker; it does not establish a hosted replacement. + +Managed adapters: change a visible adapter preference as an instance admin, +then use a second application process against the same disposable database to +confirm catalog and new-agent selection reflect it. Verify saved agents remain +usable. Restart with an operator adapter declaration, confirm its effective +list, and reject conflicting toggles. Remove the declaration and confirm saved +preferences return. [Database tests](../../server/src/services/adapter-settings-postgres.test.ts) +cover local persistence, concurrent writers, audit rollback, and failure responses; +[local compatibility tests](../../server/src/services/adapter-settings-local.test.ts) +cover the legacy file. No deployed walkthrough is claimed. + Manual: Change a harmless experimental setting, reload, and confirm the affected route honors it. Back up a disposable instance, inspect the reported file and metadata, and restore using the documented procedure into a second isolated target. Verify a known record after restoration. Cloud staging: On a managed `*.staging.paperclip.app` instance, verify experiments diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index 21c853d8501..f9a92a8811d 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -170,6 +170,15 @@ controls do not establish live provider acceptance. ### `adapter-setup` +Cloud-managed instances load adapters from the release image. Saved external +registrations cannot replace built-ins or supply UI parser code. Install, +reinstall, reload, remove, and override operations remain platform-managed. +The [fixture loader controls](../../server/src/adapters/cloud-external-adapters.test.ts) +exercise real packages, import side effects, both managed-instance signals and +self-hosted compatibility. The [route authorization controls](../../server/src/__tests__/adapter-routes-authz.test.ts) +check managed denials before package or registration access. These are local +controls, not live Cloud provider qualification. + Automated: [adapter validation routes](../../server/src/__tests__/agent-adapter-validation-routes.test.ts) and [model refresh](../../server/src/__tests__/adapter-model-refresh-routes.test.ts) cover server behavior. Claude native setup uses the existing Anthropic API-key or subscription connection diff --git a/packages/plugins/sandbox-providers/daytona/package.json b/packages/plugins/sandbox-providers/daytona/package.json index ddc5acd934f..84007584468 100644 --- a/packages/plugins/sandbox-providers/daytona/package.json +++ b/packages/plugins/sandbox-providers/daytona/package.json @@ -1,6 +1,6 @@ { "name": "@paperclipai/plugin-daytona", - "version": "0.1.1", + "version": "0.1.9", "description": "Daytona sandbox provider plugin for Paperclip environments", "license": "MIT", "homepage": "https://github.com/paperclipai/paperclip", diff --git a/packages/plugins/sandbox-providers/kubernetes/package.json b/packages/plugins/sandbox-providers/kubernetes/package.json index d1238aa7073..057f6685b8b 100644 --- a/packages/plugins/sandbox-providers/kubernetes/package.json +++ b/packages/plugins/sandbox-providers/kubernetes/package.json @@ -1,6 +1,6 @@ { "name": "@paperclipai/plugin-kubernetes", - "version": "0.1.0", + "version": "0.1.0-alpha.1", "description": "Kubernetes sandbox provider plugin for Paperclip environments", "license": "MIT", "homepage": "https://github.com/paperclipai/paperclip", diff --git a/server/src/__tests__/adapter-routes-authz.test.ts b/server/src/__tests__/adapter-routes-authz.test.ts index 22aa7b2c268..10adbc978e6 100644 --- a/server/src/__tests__/adapter-routes-authz.test.ts +++ b/server/src/__tests__/adapter-routes-authz.test.ts @@ -300,7 +300,7 @@ describe("adapter management route authorization", () => { mocks.reloadExternalAdapter.mockImplementation(async (type: string) => createAdapter(type), ); - }, 20_000); + }, 60_000); afterEach(() => { unregisterServerAdapter(EXTERNAL_ADAPTER_TYPE); @@ -373,35 +373,31 @@ describe("adapter management route authorization", () => { }, ); - describe("cloud-managed adapter code install floor", () => { - beforeEach(() => { - process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "test-server-token"; + describe("cloud-managed adapter code floor", () => { + afterEach(() => vi.unstubAllEnvs()); + + it.each( + ["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"].flatMap((signal) => + ["install", "reinstall", "reload", "override", "delete"].map((routeName) => [signal, routeName]), + ), + )("blocks %s adapter %s before package or registration access", async (signal, routeName) => { + seedInstalledExternalAdapter(); + vi.clearAllMocks(); + vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-server-token"); + const app = createApp(instanceAdmin); + + const res = await sendMutatingRequest(app, routeName); + + expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(403); + expect(res.body.details).toMatchObject({ code: "adapter_install_platform_managed" }); + expect(mocks.execFile).not.toHaveBeenCalled(); + expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled(); + expect(mocks.reloadExternalAdapter).not.toHaveBeenCalled(); + expect(mocks.getAdapterPluginByType).not.toHaveBeenCalled(); + expect(mocks.addAdapterPlugin).not.toHaveBeenCalled(); + expect(mocks.removeAdapterPlugin).not.toHaveBeenCalled(); + expect(mocks.externalRecords.get(EXTERNAL_ADAPTER_TYPE)).toEqual(installedRecord()); }); - afterEach(() => { - delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; - }); - - it.each(["install", "reinstall"] as const)( - "floors adapter %s off for instance admins on cloud-managed instances", - async (routeName) => { - resetInstalledExternalAdapterState(); - if (routeName !== "install") { - seedInstalledExternalAdapter(); - } - const app = createApp(instanceAdmin); - - const res = await sendMutatingRequest(app, routeName); - - expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe( - 403, - ); - expect(res.body.details).toMatchObject({ - code: "adapter_install_platform_managed", - }); - expect(mocks.execFile).not.toHaveBeenCalled(); - expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled(); - }, - ); }); describe("operator-hidden adapter management floor", () => { diff --git a/server/src/__tests__/agent-test-environment-routes.test.ts b/server/src/__tests__/agent-test-environment-routes.test.ts index 4b5cf7b39d4..7a9088665ed 100644 --- a/server/src/__tests__/agent-test-environment-routes.test.ts +++ b/server/src/__tests__/agent-test-environment-routes.test.ts @@ -103,6 +103,11 @@ vi.mock("../services/instance-settings.js", () => ({ instanceSettingsService: () => mockInstanceSettingsService, })); +vi.mock("../services/adapter-settings.js", async (importOriginal) => ({ + ...(await importOriginal()), + adapterSettingsService: () => ({ getDisabledTypes: async () => [] }), +})); + vi.mock("../services/dot-runner-broker.js", () => ({ dotRunnerBroker: () => ({ enabled: async () => true, bindingForAgent: async () => null }), })); diff --git a/server/src/__tests__/bundled-plugin-package.test.ts b/server/src/__tests__/bundled-plugin-package.test.ts new file mode 100644 index 00000000000..367006a2c65 --- /dev/null +++ b/server/src/__tests__/bundled-plugin-package.test.ts @@ -0,0 +1,86 @@ +import { rmSync, writeFileSync, symlinkSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { bundledImageFixture } from "./helpers/bundled-image-fixture.js"; +import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js"; +import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../services/bundled-plugins.js"; +import { readDistributionPluginCatalog } from "../services/distribution-plugin-catalog.js"; + +import boxd from "../../../packages/plugins/sandbox-providers/boxd/src/manifest.js"; +import createos from "../../../packages/plugins/sandbox-providers/createos/src/manifest.js"; +import cloudflare from "../../../packages/plugins/sandbox-providers/cloudflare/src/manifest.js"; +import daytona from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js"; +import e2b from "../../../packages/plugins/sandbox-providers/e2b/src/manifest.js"; +import exe_dev from "../../../packages/plugins/sandbox-providers/exe-dev/src/manifest.js"; +import kubernetes from "../../../packages/plugins/sandbox-providers/kubernetes/src/manifest.js"; +import modal from "../../../packages/plugins/sandbox-providers/modal/src/manifest.js"; +import novita from "../../../packages/plugins/sandbox-providers/novita/src/manifest.js"; + +const manifests = { boxd, createos, cloudflare, daytona, e2b, "exe-dev": exe_dev, kubernetes, modal, novita }; +const roots: string[] = []; +function fixture() { const result = bundledImageFixture(); roots.push(result.root); return result; } +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +describe("bundled image package identity", () => { + it("resolves independent image roots from the same registration without any previous path", () => { + const first = fixture(); + const second = fixture(); + const registration = { ...first.registration, packagePath: "/lost/old/home/plugins/fixture" }; + expect(first.resolver.resolve(registration)).toBe(first.packageRoot); + expect(second.resolver.resolve(registration)).toBe(second.packageRoot); + expect(first.resolver.inspect(registration, first.manifest)).toEqual(second.resolver.inspect(registration, second.manifest)); + expect(registration.packagePath).toBe("/lost/old/home/plugins/fixture"); + }); + + it("requires verified manifest evidence and blocks same-version manifest drift", () => { + const { resolver, registration, manifest } = fixture(); + expect(resolver.inspect(registration)).toMatchObject({ compatible: false, code: "missing_manifest_evidence" }); + expect(resolver.inspect(registration, { ...manifest, description: "changed" })).toMatchObject({ compatible: false, code: "incompatible_bundled_package", reason: expect.stringContaining("Approved plugin manifest") }); + expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true, identity: { kind: "bundled_image", catalogKey: "fixture", pluginKey: manifest.id, version: "1.0.0" } }); + }); + + it("rejects mismatched package names, versions, identity and escaping entrypoints", () => { + const { resolver, registration } = fixture(); + for (const update of [ + { packageName: "@other/plugin" }, { version: "2.0.0" }, { pluginKey: "other.plugin" }, + { manifestJson: { ...registration.manifestJson, version: "2.0.0" } }, + { manifestJson: { ...registration.manifestJson, entrypoints: { worker: "../worker.js" } } }, + ]) expect(() => resolver.resolve({ ...registration, ...update })).toThrow(); + }); + + it("fails explicitly for missing or deselected bundles even when a stale npm package exists", () => { + const { root, packageRoot, resolver, registration } = fixture(); + const fallback = path.join(root, "home", "node_modules", registration.packageName); + mkdirSync(fallback, { recursive: true }); + writeFileSync(path.join(fallback, "worker.js"), "throw new Error('unsafe fallback');"); + rmSync(packageRoot, { recursive: true }); + expect(() => resolver.resolve(registration)).toThrow(/ENOENT/); + expect(() => createBundledPluginPackageResolver([]).resolve(registration)).toThrow(/absent or not selected/); + }); + + it("rejects tampering before manifest import and symlink escapes before worker resolution", () => { + const { root, packageRoot, registration, resolver } = fixture(); + writeFileSync(path.join(packageRoot, "dist", "manifest.js"), "throw new Error('must never import');"); + expect(() => readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG)).toThrow(/digest mismatch/); + rmSync(path.join(packageRoot, "dist", "worker.js")); + symlinkSync(path.join(root, "distribution", "catalog.json"), path.join(packageRoot, "dist", "worker.js")); + expect(() => resolver.resolve(registration)).toThrow(/escapes/); + }); + + it.each(BUNDLED_PLUGIN_CATALOG)("accepts the real shipped $key package and manifest contract", (entry) => { + const manifest = manifests[entry.key as keyof typeof manifests]; + const pkg = JSON.parse(readFileSync(new URL(`../../../packages/plugins/${entry.relativePath}/package.json`, import.meta.url), "utf8")); + const { root } = fixture(); + const packageRoot = path.join(root, entry.relativePath); + mkdirSync(path.join(packageRoot, "dist"), { recursive: true }); + writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(pkg)); + writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`); + writeFileSync(path.join(packageRoot, "dist", "worker.js"), "export default {};"); + const installs = resolveBundledPluginInstalls([entry.key], { catalogRoot: root, enforceCatalogRoot: true, distributionPlugins: [], env: {} }); + const resolver = createBundledPluginPackageResolver(installs); + const registration = { pluginKey: entry.pluginKey, packageName: pkg.name, version: manifest.version, manifestJson: manifest }; + expect(pkg.version).toBe(manifest.version); + expect(resolver.resolve(registration)).toBe(packageRoot); + expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true }); + }); +}); diff --git a/server/src/__tests__/bundled-plugin-registration.test.ts b/server/src/__tests__/bundled-plugin-registration.test.ts new file mode 100644 index 00000000000..87b495dcf1f --- /dev/null +++ b/server/src/__tests__/bundled-plugin-registration.test.ts @@ -0,0 +1,287 @@ +import express from "express"; +import request from "supertest"; +import { execFileSync } from "node:child_process"; +import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync, existsSync, mkdtempSync } from "node:fs"; +import path from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { applyPendingMigrations, closeRegisteredClients, companies, createDb, plugins } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import type { PluginRecord } from "@paperclipai/shared"; +import { createPluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js"; +import { linkSdkInto } from "../../../scripts/link-plugin-dev-sdk.mjs"; +import { fileURLToPath } from "node:url"; +import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js"; +import { inspectCloudPackages } from "../services/cloud-package-qualification.js"; +import { resolveBundledPluginInstalls } from "../services/bundled-plugins.js"; +import { pluginRegistryService } from "../services/plugin-registry.js"; +import { pluginUiStaticRoutes } from "../routes/plugin-ui-static.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { derivePluginDatabaseNamespace } from "../services/plugin-database.js"; +import { ensureBundledPlugins } from "../services/bundled-plugins.js"; +import daytonaManifest from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js"; +import { bundledImageFixture } from "./helpers/bundled-image-fixture.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const externalDatabaseUrl = process.env.PAPERCLIP_BUNDLED_PLUGIN_TEST_DATABASE_URL?.trim(); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; + +describePostgres("bundled image registration in PostgreSQL", () => { + let database: Awaited>; + let db: ReturnType; + let peer: ReturnType; + const roots: string[] = []; + const fixture = () => { const result = bundledImageFixture(); roots.push(result.root); return result; }; + beforeAll(async () => { + if (externalDatabaseUrl) { + await applyPendingMigrations(externalDatabaseUrl); + database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) }; + } else { + database = await startEmbeddedPostgresTestDatabase("bundled-image-registration-"); + } + db = createDb(database.connectionString); + peer = createDb(database.connectionString); + }, 90_000); + afterEach(async () => { + await db.delete(plugins); + await db.delete(companies); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); + }); + afterAll(async () => { await database?.cleanup(); }); + + function runtime() { + const output: string[] = []; + const services = { + workerManager: { + startWorker: vi.fn(async (_id: string, options: { entrypointPath: string }) => { output.push(execFileSync(process.execPath, [options.entrypointPath, "--source"], { encoding: "utf8" })); }), + call: vi.fn(async () => ({})), + }, + lifecycleManager: { markError: vi.fn() }, + jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() }, + toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) }, + buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" }, + }; + return { services, output }; + } + function loaderFor(image: ReturnType, databaseClient = db, services?: PluginRuntimeServices) { + return pluginLoader(databaseClient, { + localPluginDir: path.join(image.root, "lost-home"), prebuiltOnly: true, + resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation, + enableLocalFilesystem: false, enableNpmDiscovery: false, + }, services); + } + + it("starts current-image workers after losing the old home and retains each company's config", async () => { + const old = fixture(); + const first = fixture(); + const second = fixture(); + await loaderFor(old).installPlugin({ localPath: old.packageRoot }); + const registry = pluginRegistryService(db); + let row = (await registry.getByKey(old.manifest.id))!; + await registry.updateStatus(row.id, { status: "ready" }); + const [companyA, companyB, unconfigured] = await db.insert(companies).values([ + { name: "Company A", issuePrefix: "CFA" }, { name: "Company B", issuePrefix: "CFB" }, { name: "Unconfigured", issuePrefix: "CFC" }, + ]).returning(); + await registry.upsertConfig(row.id, companyA!.id, { configJson: { token: "only-A" } }); + await registry.upsertConfig(row.id, companyB!.id, { configJson: { token: "only-B" } }); + row = (await registry.getById(row.id))!; + rmSync(old.root, { recursive: true }); + const original = { ...row }; + for (const image of [first, second]) { + const current = runtime(); + const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices); + const result = await loader.loadSingle(row.id); + expect(result.success).toBe(true); + expect(current.output).toEqual([expect.stringContaining(image.packageRoot)]); + expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({ + entrypointPath: path.join(image.packageRoot, "dist", "worker.js"), config: {}, proactiveCompanyScopes: [companyA!.id, companyB!.id].sort(), + })); + expect(current.services.workerManager.call.mock.calls).toEqual(expect.arrayContaining([ + [row.id, "configChanged", { companyId: companyA!.id, config: { token: "only-A" } }], + [row.id, "configChanged", { companyId: companyB!.id, config: { token: "only-B" } }], + ])); + expect(current.services.workerManager.call).toHaveBeenCalledTimes(2); + expect(await registry.getConfig(row.id, unconfigured!.id)).toBeNull(); + const app = express(); + app.use((req, _res, next) => { + req.actor = { type: "board", userId: "fixture-user", source: "session", isInstanceAdmin: false, companyIds: [companyA!.id] }; + next(); + }); + app.use(pluginUiStaticRoutes(db, { localPluginDir: path.join(image.root, "lost-home"), resolvePackageRoot: image.resolver.resolve })); + app.use(errorHandler); + const served = await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyA!.id }); + expect(served.status).toBe(200); + expect(served.text).toContain("current-image"); + expect((await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyB!.id })).status).toBe(403); + + } + expect(await registry.getById(row.id)).toEqual(original); + }); + + it("completes the real worker initialize and health protocol from a relocated image", async () => { + const old = fixture(); const image = fixture(); + await loaderFor(old).installPlugin({ localPath: old.packageRoot }); + const registry = pluginRegistryService(db); + const row = (await registry.getByKey(old.manifest.id))!; + await registry.updateStatus(row.id, { status: "ready" }); + rmSync(old.root, { recursive: true }); + const manager = createPluginWorkerManager(); + try { + const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices; + const result = await loaderFor(image, db, services).loadSingle(row.id); + expect(result.success).toBe(true); + expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(image.packageRoot), configs: {} } }); + } finally { await manager.stopAll(); } + }); + + it("qualifies and initializes the current release SDK/shared exports through the baked loader", async () => { + const image = fixture(); + const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url)); + const provider = mkdtempSync(path.join(catalogRoot, "sandbox-providers", "image-sdk-proof-")); + roots.push(provider); + cpSync(image.packageRoot, provider, { recursive: true }); + linkSdkInto(provider); + writeFileSync(path.join(provider, "dist", "worker.js"), 'import { definePlugin, runWorker } from "@paperclipai/plugin-sdk"; runWorker(definePlugin({ async setup() {}, async onHealth() { return { status: "ok", message: "Current image SDK/shared loaded" }; } }), import.meta.url);'); + const installs = [{ key: "sdk-proof", pluginKey: image.manifest.id, localPath: provider }]; + const resolver = createBundledPluginPackageResolver(installs); + const manager = createPluginWorkerManager(); + const loader = pluginLoader(db, { + prebuiltOnly: true, resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation, + enableLocalFilesystem: false, enableNpmDiscovery: false, + }, { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices); + const registry = pluginRegistryService(db); + try { + await loader.installPlugin({ localPath: provider }); + const installed = (await registry.getByKey(image.manifest.id))!; + await registry.updateStatus(installed.id, { status: "ready" }); + const registration = (await registry.getById(installed.id))!; + expect((await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) })).blockers).toEqual([]); + expect((await loader.loadSingle(installed.id)).success).toBe(true); + expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Current image SDK/shared loaded" }); + } finally { await manager.stopAll(); } + }, 90_000); + + it.skipIf(!existsSync(fileURLToPath(new URL("../../../packages/plugins/sandbox-providers/daytona/dist/worker.js", import.meta.url))))("qualifies and initializes the actual prebuilt Daytona release package", async () => { + const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url)); + const installs = resolveBundledPluginInstalls(["daytona"], { catalogRoot, env: {}, enforceCatalogRoot: true, distributionPlugins: [] }); + const resolver = createBundledPluginPackageResolver(installs); + const manager = createPluginWorkerManager(); + const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices; + const loader = pluginLoader(db, { + localPluginDir: "/lost/release/home/plugins", prebuiltOnly: true, + resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation, + enableLocalFilesystem: false, enableNpmDiscovery: false, + }, services); + const registry = pluginRegistryService(db); + vi.stubEnv("DAYTONA_API_KEY", ""); + try { + await loader.installPlugin({ localPath: installs[0]!.localPath }); + const installed = (await registry.getByKey(daytonaManifest.id))!; + await registry.update(installed.id, { packagePath: "/lost/release/home/daytona" }); + await registry.updateStatus(installed.id, { status: "ready" }); + const registration = (await registry.getById(installed.id))!; + const inspected = await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) }); + expect(inspected.blockers).toEqual([]); + expect(inspected.packages[0]?.packageDigest).toMatch(/^sha256:/); + expect((await loader.loadSingle(installed.id)).success).toBe(true); + expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Daytona sandbox provider plugin healthy" }); + } finally { await manager.stopAll(); vi.unstubAllEnvs(); } + }, 90_000); + + it("registers the same package concurrently from two roots without replacing the winner", async () => { + const first = fixture(); const second = fixture(); + await Promise.all([ + loaderFor(first).installPlugin({ localPath: first.packageRoot }), + loaderFor(second, peer).installPlugin({ localPath: second.packageRoot }), + ]); + const rows = await pluginRegistryService(db).listInstalled(); + expect(rows).toHaveLength(1); + const original = rows[0]!; + expect([first.packageRoot, second.packageRoot]).toContain(original.packagePath); + await loaderFor(second, peer).installPlugin({ localPath: second.packageRoot }); + expect(await pluginRegistryService(db).getById(original.id)).toEqual(original); + await expect(pluginRegistryService(peer).install({ packageName: original.packageName }, { ...first.manifest, description: "different package metadata" }, { idempotent: true })).rejects.toThrow(/already installed/); + }); + + it("preserves disabled registrations and avoids path writes during managed boot", async () => { + const old = fixture(); const image = fixture(); + await loaderFor(old).installPlugin({ localPath: old.packageRoot }); + const registry = pluginRegistryService(db); + const row = (await registry.getByKey(old.manifest.id))!; + await db.update(plugins).set({ status: "disabled" }).where(eq(plugins.id, row.id)); + const original = await registry.getById(row.id); + await ensureBundledPlugins(image.installs, { registry, loader: loaderFor(image), lifecycle: { load: vi.fn() }, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }, { reinstallUninstalled: true, preservePackagePath: true }); + expect(await registry.getById(row.id)).toEqual(original); + }); + + it("fails before worker startup for missing image bytes without using a home fallback or building", async () => { + const image = fixture(); + await loaderFor(image).installPlugin({ localPath: image.packageRoot }); + const registry = pluginRegistryService(db); + const row = (await registry.getByKey(image.manifest.id))!; + await registry.updateStatus(row.id, { status: "ready" }); + const fallback = path.join(image.root, "lost-home", "node_modules", row.packageName); + mkdirSync(fallback, { recursive: true }); + writeFileSync(path.join(fallback, "package.json"), JSON.stringify({ name: row.packageName })); + writeFileSync(path.join(fallback, "worker.js"), "throw new Error('must not start');"); + rmSync(path.join(image.packageRoot, "dist", "worker.js")); + const current = runtime(); + const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices); + const result = await loader.loadSingle(row.id); + expect(result.success).toBe(false); + expect(result.error).toContain("ENOENT"); + expect(current.services.workerManager.startWorker).not.toHaveBeenCalled(); + await expect(loader.installPlugin({ packageName: row.packageName })).rejects.toThrow(/current image catalog/); + await loader.cleanupInstallArtifacts(row as PluginRecord); + expect(image.resolver.inspect(row, image.manifest).compatible).toBe(false); + }); + + it("applies plugin migrations from the current image after the legacy package disappears", async () => { + const image = bundledImageFixture({ + capabilities: ["database.namespace.migrate", "database.namespace.read", "database.namespace.write"], + database: { migrationsDir: "migrations" }, + }); + roots.push(image.root); + const [row] = await db.insert(plugins).values({ + ...image.registration, packagePath: "/lost/legacy/home/plugin", apiVersion: 1, categories: image.manifest.categories, status: "ready", + }).returning(); + const current = runtime(); + const result = await loaderFor(image, db, current.services as unknown as PluginRuntimeServices).loadSingle(row!.id); + expect(result.success).toBe(true); + expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row!.id, expect.objectContaining({ databaseNamespace: derivePluginDatabaseNamespace(image.manifest.id) })); + await db.execute((await import("drizzle-orm")).sql.raw(`DROP SCHEMA "${derivePluginDatabaseNamespace(image.manifest.id)}" CASCADE`)); + }); + + it("retains registry-install provider credentials for an ordinary npm package", async () => { + const npmManifest = { ...daytonaManifest, id: "paperclip.image-npm-fixture" }; + const image = bundledImageFixture(npmManifest); + roots.push(image.root); + const localPluginDir = path.join(image.root, "npm-home"); + const packageRoot = path.join(localPluginDir, "node_modules", "@paperclipai", "plugin-daytona"); + mkdirSync(path.dirname(packageRoot), { recursive: true }); + cpSync(image.packageRoot, packageRoot, { recursive: true }); + const metadata = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")); + metadata.name = "@paperclipai/plugin-daytona"; + writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(metadata)); + const registry = pluginRegistryService(db); + const row = (await registry.install({ packageName: metadata.name }, npmManifest))!; + await registry.updateStatus(row.id, { status: "ready" }); + vi.stubEnv("DAYTONA_API_KEY", "synthetic-provider-key"); + try { + const current = runtime(); + const loader = pluginLoader(db, { localPluginDir, enableNpmDiscovery: false, enableLocalFilesystem: false }, current.services as unknown as PluginRuntimeServices); + expect((await loader.loadSingle(row.id)).success).toBe(true); + expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({ env: expect.objectContaining({ DAYTONA_API_KEY: "synthetic-provider-key" }) })); + } finally { vi.unstubAllEnvs(); } + }); + + it("keeps self-hosted local-path loading available without a managed resolver", async () => { + const image = fixture(); + const loader = pluginLoader(db, { localPluginDir: path.join(image.root, "selfhost"), enableNpmDiscovery: false, enableLocalFilesystem: false }); + await loader.installPlugin({ localPath: image.packageRoot }); + const row = (await pluginRegistryService(db).getByKey(image.manifest.id))!; + expect(row.packagePath).toBe(image.packageRoot); + await expect(loader.installPlugin({ localPath: image.packageRoot })).rejects.toThrow(/already installed/); + }); +}); diff --git a/server/src/__tests__/bundled-plugin-runtime.test.ts b/server/src/__tests__/bundled-plugin-runtime.test.ts new file mode 100644 index 00000000000..936687a91be --- /dev/null +++ b/server/src/__tests__/bundled-plugin-runtime.test.ts @@ -0,0 +1,72 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { build, transform } from "esbuild"; +import { bundleRuntime } from "../../scripts/bundle-runtime.mjs"; +import { resolveBundledPluginRuntimeLoader } from "../services/bundled-plugin-runtime.js"; + +const roots: string[] = []; +function fixture() { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-plugin-runtime-"))); + roots.push(root); + fs.mkdirSync(path.join(root, "server", "dist-runtime"), { recursive: true }); + fs.writeFileSync(path.join(root, "server", "package.json"), JSON.stringify({ name: "@paperclipai/server", type: "module" })); + const loader = path.join(root, "cli", "node_modules", "tsx", "dist", "loader.mjs"); + fs.mkdirSync(path.dirname(loader), { recursive: true }); + fs.writeFileSync(loader, "export {};\n"); + return { root, loader }; +} +afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); + +describe("current image plugin runtime loader", () => { + it.each(["src/services", "dist/services", "dist-runtime"])("resolves the owning release from server/%s", (location) => { + const { root, loader } = fixture(); + const moduleDirectory = path.join(root, "server", location); + fs.mkdirSync(moduleDirectory, { recursive: true }); + expect(resolveBundledPluginRuntimeLoader(moduleDirectory)).toBe(loader); + }); + + it("resolves a bundled entrypoint without preserved source module locations", async () => { + const { root, loader } = fixture(); + const runtime = path.join(root, "server", "dist-runtime", "index.mjs"); + await build({ + stdin: { contents: `import { resolveBundledPluginRuntimeLoader } from ${JSON.stringify(fileURLToPath(new URL("../services/bundled-plugin-runtime.ts", import.meta.url)))}; process.stdout.write(resolveBundledPluginRuntimeLoader());`, resolveDir: process.cwd() }, + outfile: runtime, bundle: true, platform: "node", format: "esm", target: "node24", logLevel: "silent", + }); + expect(execFileSync(process.execPath, [runtime], { encoding: "utf8" })).toBe(loader); + }); + + it("uses the production bundler's preserved module location at the dist-runtime entrypoint", async () => { + const { root, loader } = fixture(); + const service = path.join(root, "server", "dist", "services", "bundled-plugin-runtime.js"); + fs.mkdirSync(path.dirname(service), { recursive: true }); + const source = fs.readFileSync(new URL("../services/bundled-plugin-runtime.ts", import.meta.url), "utf8"); + fs.writeFileSync(service, (await transform(source, { loader: "ts", format: "esm", target: "node24" })).code); + const entry = path.join(root, "server", "dist", "probe.js"); + fs.writeFileSync(entry, 'import { resolveBundledPluginRuntimeLoader } from "./services/bundled-plugin-runtime.js"; process.stdout.write(resolveBundledPluginRuntimeLoader());'); + const bundled = await bundleRuntime({ entry, outdir: path.join(root, "server", "dist-runtime") }); + expect(execFileSync(process.execPath, [bundled.outfile], { encoding: "utf8" })).toBe(loader); + }); + + it("accepts the baked pnpm target inside the current image node_modules", () => { + const { root, loader } = fixture(); + const physical = path.join(root, "node_modules", ".pnpm", "tsx@1", "node_modules", "tsx", "dist", "loader.mjs"); + fs.mkdirSync(path.dirname(physical), { recursive: true }); + fs.writeFileSync(physical, "export {};\n"); + fs.rmSync(loader); fs.symlinkSync(path.relative(path.dirname(loader), physical), loader); + expect(resolveBundledPluginRuntimeLoader(path.join(root, "server", "dist-runtime"))).toBe(physical); + }); + + it("fails explicitly for missing or legacy-home loaders", () => { + const { root, loader } = fixture(); + const moduleDirectory = path.join(root, "server", "dist-runtime"); + fs.rmSync(loader); + expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("missing from the current image"); + const outside = fixture().loader; + fs.symlinkSync(outside, loader); + expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("escapes the current image"); + }); +}); diff --git a/server/src/__tests__/bundled-plugins.test.ts b/server/src/__tests__/bundled-plugins.test.ts index 5dfd5e8f90e..d66b0d849da 100644 --- a/server/src/__tests__/bundled-plugins.test.ts +++ b/server/src/__tests__/bundled-plugins.test.ts @@ -9,6 +9,7 @@ import { ensureBundledPlugins, resolveBundledCatalogRoot, resolveBundledPluginInstalls, + resolveManagedBundledPluginKeys, type BundledPluginProvisionerDeps, type ResolvedBundledPlugin, } from "../services/bundled-plugins.js"; @@ -30,6 +31,21 @@ afterAll(() => { // Catalog-root resolution (fail-to-start allowlist) // --------------------------------------------------------------------------- +describe("managed bundled plugin selection", () => { + it.each([ + { env: {}, selected: undefined, expected: null }, + { env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: undefined, expected: [] }, + { env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: " " }, selected: undefined, expected: null }, + { env: { PAPERCLIP_MANAGED_CONFIG: "" }, selected: undefined, expected: [] }, + { env: { PAPERCLIP_MANAGED_CONFIG: "{}" }, selected: null, expected: [] }, + { env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: ["daytona"], expected: ["daytona"] }, + { env: {}, selected: ["modal"], expected: ["modal"] }, + { env: {}, selected: [], expected: [] }, + ])("uses the canonical Cloud floor and preserves explicit selection: $env / $selected", ({ env, selected, expected }) => { + expect(resolveManagedBundledPluginKeys(selected, env)).toEqual(expected); + }); +}); + describe("resolveBundledPluginInstalls", () => { it("resolves the optional CreateOS sandbox provider without making it a self-hosted default", () => { expect(resolveBundledPluginInstalls(["createos"], { diff --git a/server/src/__tests__/cloud-bundled-package-digest.test.ts b/server/src/__tests__/cloud-bundled-package-digest.test.ts new file mode 100644 index 00000000000..7978c46795e --- /dev/null +++ b/server/src/__tests__/cloud-bundled-package-digest.test.ts @@ -0,0 +1,93 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js"; + +const roots: string[] = []; +function file(name: string, content = "export const value = 1;") { + fs.mkdirSync(path.dirname(name), { recursive: true }); fs.writeFileSync(name, content); +} +function link(target: string, name: string) { + fs.mkdirSync(path.dirname(name), { recursive: true }); fs.symlinkSync(path.relative(path.dirname(name), target), name); +} +function fixture() { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-package-graph-"))); roots.push(root); + const catalogRoot = path.join(root, "packages", "plugins"); + const provider = path.join(catalogRoot, "sandbox-providers", "daytona"); + const sdk = path.join(catalogRoot, "sdk"); + const shared = path.join(root, "packages", "shared"); + const dependency = path.join(root, "node_modules", ".pnpm", "dependency@1", "node_modules", "dependency"); + const transitive = path.join(root, "node_modules", ".pnpm", "transitive@1", "node_modules", "transitive"); + for (const directory of [provider, sdk, shared, dependency, transitive]) file(path.join(directory, "index.js")); + link(sdk, path.join(provider, "node_modules", "@paperclipai", "plugin-sdk")); + link(shared, path.join(sdk, "node_modules", "@paperclipai", "shared")); + link(dependency, path.join(shared, "node_modules", "dependency")); + link(transitive, path.join(path.dirname(dependency), "transitive")); + return { root, catalogRoot, provider, sdk, shared, dependency, transitive }; +} +afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); + +describe("bounded current-image package dependency inventory", () => { + it("hashes external SDK, shared, and pnpm sibling dependency bytes", () => { + const image = fixture(); + const digest = () => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot }); + let previous = digest(); + for (const directory of [image.sdk, image.shared, image.dependency, image.transitive]) { + file(path.join(directory, "index.js"), "export const value = 2;"); + const next = digest(); expect(next).not.toBe(previous); previous = next; + } + }); + + it("keeps the digest stable when an identical release moves to a new root", () => { + const original = fixture(); const replacement = fixture(); + expect(cloudBundledPackageDigest(original.provider, { catalogRoot: original.catalogRoot })) + .toBe(cloudBundledPackageDigest(replacement.provider, { catalogRoot: replacement.catalogRoot })); + }); + + it("rejects legacy-home links and preserves package-contained defaults", () => { + const image = fixture(); + expect(() => cloudBundledPackageDigest(image.provider)).toThrow("escapes its package"); + const outside = fixture(); + link(path.join(outside.sdk, "index.js"), path.join(image.shared, "legacy-home.js")); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("authorized image roots"); + }); + + it("deduplicates cycles without scanning unrelated image node_modules", () => { + const image = fixture(); + link(image.sdk, path.join(image.shared, "node_modules", "cycle")); + const unrelated = path.join(image.root, "node_modules", "unrelated", "special"); + file(unrelated); + const first = cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot }); + file(unrelated, "changed unrelated code"); + expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toBe(first); + }); + + it.each(["", ".pnpm", ".pnpm/node_modules"])("rejects aggregate image dependency targets: %s", (aggregate) => { + const image = fixture(); + const target = path.join(image.root, "node_modules", aggregate); + fs.mkdirSync(target, { recursive: true }); + link(target, path.join(image.sdk, "node_modules", "aggregate")); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("select an individual package"); + }); + + it("accepts the shipped provider's observed entry class", () => { + const image = fixture(); + const directory = path.join(image.provider, "many"); fs.mkdirSync(directory); + for (let index = 0; index < 16_000; index++) fs.writeFileSync(path.join(directory, `${index}.js`), ""); + expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toMatch(/^sha256:/); + }); + + it("rejects oversized metadata and excessive depth", () => { + const image = fixture(); + file(path.join(image.sdk, "package.json"), " ".repeat(128 * 1024 + 1)); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("metadata"); + fs.rmSync(path.join(image.sdk, "package.json")); + const large = path.join(image.shared, "large.js"); + fs.closeSync(fs.openSync(large, "w")); fs.truncateSync(large, 256 * 1024 * 1024 + 1); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("byte limit"); + fs.rmSync(large); + file(path.join(image.sdk, ...Array.from({ length: 33 }, () => "deep"), "file.js")); + expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("depth limit"); + }); +}); diff --git a/server/src/__tests__/cloud-configuration-migration-postgres.test.ts b/server/src/__tests__/cloud-configuration-migration-postgres.test.ts new file mode 100644 index 00000000000..dd747d06ac6 --- /dev/null +++ b/server/src/__tests__/cloud-configuration-migration-postgres.test.ts @@ -0,0 +1,325 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, plugins, type Db } from "@paperclipai/db"; +import { eq, inArray, sql } from "drizzle-orm"; +import { paperclipConfigSchema } from "@paperclipai/shared"; +import { adapterSettingsService, initializeAdapterSettings } from "../services/adapter-settings.js"; +import { cloudRuntimeConfigService, initializeCloudRuntimeConfig } from "../services/cloud-runtime-config.js"; +import { cloudConfigurationMigrationService } from "../services/cloud-configuration-migration.js"; +import { qualifyCloudConfigurationForCurrentImage } from "../services/cloud-configuration-qualification.js"; +import { runCloudConfigurationQualification } from "../qualify-cloud-configuration.js"; +import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js"; +import { pluginRegistryService } from "../services/plugin-registry.js"; +import { createPluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { ensureBundledPlugins } from "../services/bundled-plugins.js"; +import { bundledImageFixture } from "./helpers/bundled-image-fixture.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const externalDatabaseUrl = process.env.PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL?.trim(); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const keys = ["adapter-settings:v1", "cloud-runtime-config:v1", "cloud-configuration-migration:v1"]; +(support.supported ? describe : describe.skip)("Cloud configuration migration and current-image qualification", () => { + let database: Awaited>; + let db: Db; + let peer: Db; + let homeA: string; + let homeB: string; + const roots: string[] = []; + const fixture = (version = "1.0.0") => { const image = bundledImageFixture({ version }); roots.push(image.root); return image; }; + const env = (home = homeA) => ({ PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "test-cloud-token", PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json") }); + const migration = (client = db, home = homeA) => cloudConfigurationMigrationService(client, { runtimeEnv: env(home) }); + beforeAll(async () => { + if (externalDatabaseUrl) { + await applyPendingMigrations(externalDatabaseUrl); + database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) }; + } else database = await startEmbeddedPostgresTestDatabase("cloud-configuration-migration-"); + db = createDb(database.connectionString); + peer = createDb(database.connectionString); + }, 90_000); + beforeEach(async () => { + homeA = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-legacy-A-")); + homeB = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-replacement-B-")); + roots.push(homeA, homeB); + await db.delete(instanceSettings).where(inArray(instanceSettings.singletonKey, keys)); + await db.delete(plugins); + }); + afterEach(async () => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); + }); + afterAll(async () => { await database?.cleanup(); }); + + async function legacy() { + const config = paperclipConfigSchema.parse({ + $meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" }, + database: { mode: "postgres" }, logging: { mode: "cloud" }, + server: { port: 3199 }, telemetry: { enabled: false }, + }); + await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config)); + await fs.writeFile(path.join(homeA, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local", "future_adapter"] })); + await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]"); + return config; + } + + function loaderFor(image: ReturnType, client = db, services?: PluginRuntimeServices) { + return pluginLoader(client, { + localPluginDir: path.join(homeB, "plugins"), prebuiltOnly: true, + resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation, + enableLocalFilesystem: false, enableNpmDiscovery: false, + }, services); + } + async function install(image: ReturnType) { + await loaderFor(image).installPlugin({ localPath: image.packageRoot }); + const registry = pluginRegistryService(db); + const row = (await registry.getByKey(image.manifest.id))!; + await registry.updateStatus(row.id, { status: "ready" }); + return row; + } + function qualify(image: ReturnType, client = db, home = homeA, expectedPackageEvidenceDigest?: string) { + const loader = loaderFor(image, client); + return migration(client, home).qualify({ installs: image.installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest }); + } + + it("imports A atomically, retains later DB revisions, and starts B's real image worker after A disappears", async () => { + await legacy(); + const oldImage = fixture(); + const newImage = fixture(); + const row = await install(oldImage); + const first = await migration().migrate(); + expect(first.migration.adapter.authority).toBe("legacy_file"); + expect(first.migration.config.authority).toBe("legacy_file"); + const original = await qualify(oldImage); + expect(original.configurationPackagesEligible).toBe(true); + expect(original.samePackageOverlapEligible).toBe(false); + await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true); + const config = (await cloudRuntimeConfigService(peer).inspectStored())!; + expect(config.config).not.toBeNull(); + await cloudRuntimeConfigService(peer).replace({ ...config.config!, server: { ...config.config!.server, port: 3299 } }, config.revision); + await fs.rm(homeA, { recursive: true }); + await fs.rm(oldImage.root, { recursive: true }); + expect((await migration(peer, homeB).migrate()).migration).toEqual(first.migration); + await initializeAdapterSettings(peer, { runtimeEnv: env(homeB) }); + const restored = await initializeCloudRuntimeConfig(peer, { runtimeEnv: env(homeB) }); + expect(restored?.current.config?.server.port).toBe(3299); + expect(await adapterSettingsService(peer, { runtimeEnv: env(homeB) }).getDisabledTypes()).toEqual(["claude_local", "codex_local", "future_adapter"]); + const replacement = await qualify(newImage, peer, homeB, original.packageEvidenceDigest); + expect(replacement).toMatchObject({ adapterRevision: 1, configRevision: 1, configurationPackagesEligible: true, samePackageOverlapEligible: true }); + expect(replacement.configurationDigest).not.toBe(original.configurationDigest); + const manager = createPluginWorkerManager(); + try { + const services = { + workerManager: manager, lifecycleManager: { markError: vi.fn() }, + jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() }, + toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) }, + buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "qualification-fixture", hostVersion: "1.0.0" }, + } as unknown as PluginRuntimeServices; + expect((await loaderFor(newImage, peer, services).loadSingle(row.id)).success).toBe(true); + expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(newImage.packageRoot) } }); + } finally { await manager.stopAll(); } + }); + + it("serializes concurrent attempts and never imports stale inputs over later DB changes", async () => { + await legacy(); + const [first, second] = await Promise.all([migration().migrate(), migration(peer).migrate()]); + expect(first).toEqual(second); + await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("claude_local", false); + await fs.writeFile(path.join(homeA, "config.json"), "invalid stale file"); + await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid stale file"); + expect(await migration().migrate()).toEqual(first); + expect(await adapterSettingsService(db, { runtimeEnv: env() }).getDisabledTypes()).toEqual(["future_adapter"]); + }); + + it("rolls back both seeds and the receipt if completion fails, then retries after the crash boundary", async () => { + await legacy(); + await db.execute(sql`CREATE FUNCTION reject_cloud_migration_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.singleton_key = 'cloud-configuration-migration:v1' THEN RAISE EXCEPTION 'test receipt failure'; END IF; RETURN NEW; END $$`); + await db.execute(sql`CREATE TRIGGER reject_cloud_migration_receipt BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_cloud_migration_receipt()`); + try { + await expect(migration().migrate()).rejects.toMatchObject({ cause: { message: expect.stringContaining("test receipt failure") } }); + expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]); + } finally { + await db.execute(sql`DROP TRIGGER reject_cloud_migration_receipt ON instance_settings`); + await db.execute(sql`DROP FUNCTION reject_cloud_migration_receipt()`); + } + expect((await migration(peer).migrate()).revision).toBe(0); + expect((await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).length).toBe(3); + }); + + it("fails without partial authority for malformed legacy config and preserves already-authoritative rows", async () => { + await legacy(); + await fs.writeFile(path.join(homeA, "config.json"), "invalid"); + await expect(migration().migrate()).rejects.toThrow("Legacy Cloud runtime configuration"); + expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]); + await adapterSettingsService(db, { runtimeEnv: env() }).seedIfAbsent({ disabledTypes: ["codex_local"] }); + await cloudRuntimeConfigService(db).seedIfAbsent(null); + await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid"); + const receipt = await migration(peer).migrate(); + expect(receipt.migration.adapter.authority).toBe("database"); + expect(receipt.migration.config.authority).toBe("database"); + expect((await cloudRuntimeConfigService(peer).inspectStored())?.config).toBeNull(); + }); + + it("returns database errors instead of falling back to files or a completed receipt", async () => { + await legacy(); + await migration().migrate(); + const broken = createDb(database.connectionString); + await broken.$client.end({ timeout: 0 }); + await expect(migration(broken).migrate()).rejects.toThrow(); + const image = fixture(); + await expect(qualify(image, broken)).rejects.toThrow(); + }); + + it("keeps external registration blockers durable when B has no legacy home and requires deliberate original-source resolution", async () => { + await legacy(); + const sentinel = path.join(homeA, "external-code-executed"); + await fs.writeFile(path.join(homeA, "external.js"), `require('node:fs').writeFileSync(${JSON.stringify(sentinel)}, 'executed');`); + await fs.writeFile(path.join(homeA, "adapter-plugins.json"), JSON.stringify([{ type: "legacy_external", packageName: "legacy-adapter", localPath: path.join(homeA, "external.js") }])); + const image = fixture(); + await install(image); + await migration().migrate(); + expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "unsupported_external_adapters" })); + await expect(fs.stat(sentinel)).rejects.toMatchObject({ code: "ENOENT" }); + await migration(peer, homeB).migrate(); + const replacement = await qualify(image, peer, homeB); + expect(replacement.configurationPackagesEligible).toBe(false); + await expect(migration(peer, homeB).recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("original legacy source"); + await fs.unlink(path.join(homeA, "adapter-plugins.json")); + await expect(migration().recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("present, valid registration file"); + expect((await migration().inspectStored())?.migration.externalRegistrations.blocker).toBe("unsupported_external_adapters"); + await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]"); + await expect(migration().recheckLegacyExternalRegistrations(0)).rejects.toThrow("revision conflict"); + await migration().recheckLegacyExternalRegistrations(replacement.receiptRevision); + expect((await qualify(image)).configurationPackagesEligible).toBe(true); + }); + + it("retains a metadata blocker for malformed or oversized registration files", async () => { + await legacy(); + await fs.writeFile(path.join(homeA, "adapter-plugins.json"), Buffer.alloc(65_537)); + const receipt = await migration().migrate(); + expect(receipt.migration.externalRegistrations.blocker).toBe("invalid_external_registration_metadata"); + await fs.rm(homeA, { recursive: true }); + expect((await migration(peer, homeB).migrate()).migration.externalRegistrations).toEqual(receipt.migration.externalRegistrations); + }); + + it("records image mismatches before reconciliation, permits maintenance upgrades, and requires explicit fresh overlap evidence", async () => { + await legacy(); + const oldImage = fixture(); + const nextImage = fixture("1.1.0"); + await install(oldImage); + await migration().migrate(); + const old = await qualify(oldImage); + const mismatch = await qualify(nextImage, peer, homeB, old.packageEvidenceDigest); + expect(mismatch.configurationPackagesEligible).toBe(false); + expect(mismatch.samePackageOverlapEligible).toBe(false); + expect(mismatch.blockers).toContainEqual(expect.objectContaining({ code: "incompatible_bundled_package", registeredVersion: "1.0.0", imageVersion: "1.1.0" })); + const registry = pluginRegistryService(db); + await ensureBundledPlugins(nextImage.installs, { + registry, loader: loaderFor(nextImage), lifecycle: { load: vi.fn() }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }, { reinstallUninstalled: true, preservePackagePath: true }); + const current = await qualify(nextImage); + expect(current.configurationPackagesEligible).toBe(true); + expect(current.samePackageOverlapEligible).toBe(false); + expect((await migration().inspectStored())?.lastPackageMismatch?.packageEvidenceDigest).toBe(mismatch.packageEvidenceDigest); + expect((await qualify(nextImage, peer, homeB, current.packageEvidenceDigest)).samePackageOverlapEligible).toBe(true); + }); + + it("fails qualification for missing package or manifest bytes without loading a home fallback", async () => { + await legacy(); + const image = fixture(); + await install(image); + await migration().migrate(); + const good = await qualify(image); + await fs.rm(path.join(image.packageRoot, "dist", "manifest.js")); + const broken = await qualify(image, peer, homeB, good.packageEvidenceDigest); + expect(broken.configurationPackagesEligible).toBe(false); + expect(broken.samePackageOverlapEligible).toBe(false); + expect(broken.blockers.length).toBeGreaterThan(0); + await fs.rm(image.packageRoot, { recursive: true }); + expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "missing_bundled_package" })); + }); + + it("rejects changed image bytes before manifest import and never uses stale completed qualification", async () => { + await legacy(); + const image = fixture(); + await install(image); + await migration().migrate(); + const original = await qualify(image); + await fs.appendFile(path.join(image.packageRoot, "dist", "worker.js"), "\nprocess.exit(99);\n"); + const loadManifest = vi.fn(); + const current = await migration(peer, homeB).qualify({ installs: image.installs, loadManifest, expectedPackageEvidenceDigest: original.packageEvidenceDigest }); + expect(current.configurationPackagesEligible).toBe(false); + expect(current.samePackageOverlapEligible).toBe(false); + expect(current.blockers).toContainEqual(expect.objectContaining({ code: "package_integrity_mismatch" })); + expect(loadManifest).not.toHaveBeenCalled(); + }); + + it("allows an administrator update while manifest inspection waits and refuses to certify the stale snapshot", async () => { + await legacy(); + const image = fixture(); + await install(image); + await migration().migrate(); + let started!: () => void; + let resume!: () => void; + const entered = new Promise((resolve) => { started = resolve; }); + const release = new Promise((resolve) => { resume = resolve; }); + const loader = loaderFor(image); + const pending = migration().qualify({ installs: image.installs, loadManifest: async (root) => { started(); await release; return loader.loadManifest(root); } }); + await entered; + try { + const change = adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true); + let timer: ReturnType | undefined; + try { + expect(await Promise.race([change, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error("Admin update was blocked by image inspection")), 2_000); })])).toBe(true); + } finally { clearTimeout(timer); } + } finally { resume(); } + const result = await pending; + expect(result).toMatchObject({ configurationPackagesEligible: false, adapterRevision: 1 }); + expect(result.blockers).toContainEqual(expect.objectContaining({ code: "configuration_changed_during_inspection" })); + expect((await qualify(image)).configurationPackagesEligible).toBe(true); + }); + + it("uses the current managed catalog through the operator command and durably records catalog failure", async () => { + await legacy(); + const image = fixture(); + await install(image); + await migration().migrate(); + const runtimeEnv = { + ...env(homeB), DATABASE_URL: database.connectionString, PAPERCLIP_BUNDLED_PLUGIN_ROOT: image.root, + PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "fixture", features: {}, plugins: { autoInstall: ["fixture"] } }), + }; + const initial = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv }); + expect(initial?.configurationPackagesEligible).toBe(true); + const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + try { + expect(await runCloudConfigurationQualification(["--expected-package-evidence", initial!.packageEvidenceDigest], runtimeEnv)).toBe(0); + const report = stdout.mock.calls.map((call) => String(call[0])).find((text) => text.startsWith('{\n "receiptRevision"')); + expect(report && JSON.parse(report).samePackageOverlapEligible).toBe(true); + } finally { stdout.mockRestore(); } + const missing = await qualifyCloudConfigurationForCurrentImage(peer, { runtimeEnv: { ...runtimeEnv, PAPERCLIP_MANAGED_CONFIG: runtimeEnv.PAPERCLIP_MANAGED_CONFIG.replace('"fixture"]', '"absent"]') } }); + expect(missing?.configurationPackagesEligible).toBe(false); + expect(missing?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" })); + expect((await migration().inspectStored())?.lastPackageMismatch?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" })); + }); + + it.skipIf(process.platform === "win32")("detects changed pnpm dependency bytes in a builtin package overlap comparison", async () => { + await legacy(); + const image = fixture(); + await install(image); + await migration().migrate(); + const dependency = path.join(image.packageRoot, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep"); + await fs.mkdir(dependency, { recursive: true }); + await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;"); + await fs.symlink(path.relative(path.join(image.packageRoot, "node_modules"), dependency), path.join(image.packageRoot, "node_modules", "dep")); + const installs = image.installs.map(({ distribution: _distribution, ...install }) => install); + const loader = loaderFor(image); + const original = await migration().qualify({ installs, loadManifest: (root) => loader.loadManifest(root) }); + expect(original.configurationPackagesEligible).toBe(true); + await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;"); + const changed = await migration(peer, homeB).qualify({ installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: original.packageEvidenceDigest }); + expect(changed.configurationPackagesEligible).toBe(true); + expect(changed.samePackageOverlapEligible).toBe(false); + expect(changed.overlapBlockers).toContainEqual(expect.objectContaining({ code: "package_evidence_changed" })); + }); +}); diff --git a/server/src/__tests__/cloud-configuration-readers.test.ts b/server/src/__tests__/cloud-configuration-readers.test.ts new file mode 100644 index 00000000000..9abfda97b7d --- /dev/null +++ b/server/src/__tests__/cloud-configuration-readers.test.ts @@ -0,0 +1,48 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { readLegacyAdapterSettings } from "../services/adapter-settings.js"; +import { inspectLegacyExternalRegistrations } from "../services/cloud-configuration-evidence.js"; +import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js"; + +vi.mock("../adapters/plugin-loader.js", () => ({ buildExternalAdapters: async () => [] })); + +describe("bounded Cloud legacy readers and image inventory", () => { + let root: string; + beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-config-reader-")); }); + afterEach(async () => { await fs.rm(root, { recursive: true, force: true }); }); + + it("distinguishes an absent adapter file from a saved empty preference", async () => { + expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toBeNull(); + await fs.writeFile(path.join(root, "adapter-settings.json"), '{"disabledTypes":[]}'); + expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toEqual({ disabledTypes: [] }); + }); + + it.skipIf(process.platform === "win32")("rejects symlinks and FIFOs without waiting for a writer", async () => { + const target = path.join(root, "target.json"); + const settings = path.join(root, "adapter-settings.json"); + await fs.writeFile(target, '{"disabledTypes":[]}'); + await fs.symlink(target, settings); + await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toMatchObject({ code: "ELOOP" }); + expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("unreadable_external_registration_metadata"); + await fs.unlink(settings); + execFileSync("mkfifo", [settings]); + await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toThrow("regular file"); + expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("invalid_external_registration_metadata"); + }); + + it.skipIf(process.platform === "win32")("hashes pnpm-style contained links and the physical dependency bytes", async () => { + const dependency = path.join(root, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep"); + await fs.mkdir(dependency, { recursive: true }); + await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;"); + await fs.symlink(path.relative(path.join(root, "node_modules"), dependency), path.join(root, "node_modules", "dep")); + const first = cloudBundledPackageDigest(root); + await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;"); + expect(cloudBundledPackageDigest(root)).not.toBe(first); + await fs.unlink(path.join(root, "node_modules", "dep")); + await fs.symlink("/", path.join(root, "node_modules", "dep")); + expect(() => cloudBundledPackageDigest(root)).toThrow("escapes its package"); + }); +}); diff --git a/server/src/__tests__/cloud-runtime-config.test.ts b/server/src/__tests__/cloud-runtime-config.test.ts new file mode 100644 index 00000000000..c7078f7cddc --- /dev/null +++ b/server/src/__tests__/cloud-runtime-config.test.ts @@ -0,0 +1,302 @@ +import fs from "node:fs"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, type Db } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { paperclipConfigSchema } from "@paperclipai/shared"; +import { loadCloudBootstrapConfig, loadConfig } from "../config.js"; +import { readConfigFile } from "../config-file.js"; +import { + CLOUD_RUNTIME_CONFIG_KEY, + cloudRuntimeConfigService, + initializeCloudRuntimeConfig, + readLegacyCloudRuntimeConfig, +} from "../services/cloud-runtime-config.js"; +import { + getCloudRuntimeConfigSnapshot, + projectCloudRuntimeConfig, + setCloudRuntimeConfigSnapshot, +} from "../services/cloud-runtime-config-snapshot.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const externalDatabaseUrl = process.env.PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL; +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; +let home: string; +let configPath: string; + +function legacyConfig() { + return { + $meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" }, + llm: { provider: "openai", apiKey: "legacy-openai-secret" }, + database: { + mode: "postgres", + connectionString: "postgres://legacy:database-secret@example.test/paperclip", + backup: { enabled: false, intervalMinutes: 73, retentionDays: 11, dir: "/tmp/legacy-backups" }, + }, + logging: { mode: "cloud", logDir: "/tmp/legacy-logs" }, + server: { deploymentMode: "authenticated", exposure: "private", port: 3210, serveUi: false, allowedHostnames: ["saved.example.test"] }, + auth: { baseUrlMode: "explicit", publicBaseUrl: "https://saved.example.test", disableSignUp: true }, + storage: { provider: "s3", s3: { bucket: "saved-bucket", region: "eu-west-1", prefix: "saved/", forcePathStyle: true } }, + secrets: { provider: "local_encrypted", strictMode: true, localEncrypted: { keyFilePath: "/tmp/legacy-master.key" } }, + telemetry: { enabled: false }, + updates: { checkEnabled: false }, + unsupportedSecret: "unknown-secret-value", + }; +} + +beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-runtime-config-")); + configPath = path.join(home, "config.json"); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_CONFIG", configPath); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-token"); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); + vi.stubEnv("DATABASE_MIGRATION_URL", undefined); + vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined); + vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", undefined); + vi.stubEnv("PAPERCLIP_AUTH_BASE_URL_MODE", undefined); + vi.stubEnv("PAPERCLIP_PUBLIC_URL", undefined); + vi.stubEnv("BETTER_AUTH_URL", undefined); + vi.stubEnv("BETTER_AUTH_BASE_URL", undefined); + vi.stubEnv("PAPERCLIP_ALLOWED_HOSTNAMES", undefined); + vi.stubEnv("PAPERCLIP_TAILNET_BIND_HOST", "100.100.0.1"); + vi.stubEnv("OPENAI_API_KEY", "environment-api-secret"); + setCloudRuntimeConfigSnapshot(null); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + setCloudRuntimeConfigSnapshot(null); + fs.rmSync(home, { recursive: true, force: true }); +}); + +describe("Cloud configuration file boundary", () => { + it.each(["null", "{", JSON.stringify({ unexpected: true })])("rejects malformed present legacy data %s", async (contents) => { + fs.writeFileSync(configPath, contents); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/Legacy Cloud runtime configuration/); + }); + + it("rejects oversized and linked legacy sources", async () => { + fs.writeFileSync(configPath, " ".repeat(65_537)); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/); + fs.rmSync(configPath); + const target = path.join(home, "target.json"); + fs.writeFileSync(target, JSON.stringify(legacyConfig())); + fs.symlinkSync(target, configPath); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/); + }); + + it("requires an environment credential for the legacy OpenAI consumer", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + vi.stubEnv("OPENAI_API_KEY", undefined); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/OPENAI_API_KEY/); + expect(() => readConfigFile()).toThrow(/not been initialized/); + }); + + it.each(["https://user:secret@saved.example.test", "https://saved.example.test?key=secret", "https://saved.example.test#secret"])("rejects secret-bearing URL %s", async (url) => { + const config = legacyConfig(); + config.auth.publicBaseUrl = url; + fs.writeFileSync(configPath, JSON.stringify(config)); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/configuration is invalid/); + }); + + it("loads the minimal Cloud bootstrap from environment only", () => { + fs.writeFileSync(configPath, "malformed"); + vi.stubEnv("DATABASE_URL", "postgres://bootstrap:bootstrap-secret@127.0.0.1:5432/runtime"); + expect(loadCloudBootstrapConfig()).toMatchObject({ databaseMode: "postgres", databaseUrl: process.env.DATABASE_URL }); + vi.stubEnv("DATABASE_URL", undefined); + expect(() => loadCloudBootstrapConfig()).toThrow(/requires DATABASE_URL/); + vi.stubEnv("DATABASE_URL", "sqlite:///tmp/db"); + expect(() => loadCloudBootstrapConfig()).toThrow(/must use PostgreSQL/); + }); + + it.each(["", " "])("treats an empty migration URL as absent (%j)", (migrationUrl) => { + vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime"); + vi.stubEnv("DATABASE_MIGRATION_URL", migrationUrl); + expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBeUndefined(); + }); + + it("normalizes and validates an explicit migration URL", () => { + vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime"); + vi.stubEnv("DATABASE_MIGRATION_URL", " postgres://migration:password@127.0.0.1/runtime "); + expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBe("postgres://migration:password@127.0.0.1/runtime"); + vi.stubEnv("DATABASE_MIGRATION_URL", "https://wrong.example.test"); + expect(() => loadCloudBootstrapConfig()).toThrow("DATABASE_MIGRATION_URL must use PostgreSQL"); + }); + + it("preserves local config file behavior without managed signals", () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + expect(readConfigFile()?.llm?.apiKey).toBe("legacy-openai-secret"); + expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" }); + }); +}); + +describePostgres("Cloud runtime configuration in PostgreSQL", () => { + let database: Awaited>; + let first: Db; + let second: Db; + + beforeAll(async () => { + if (externalDatabaseUrl) { + await applyPendingMigrations(externalDatabaseUrl); + database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl, { scope: "database" }) }; + } else { + database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-config-db-"); + } + first = createDb(database.connectionString); + second = createDb(database.connectionString); + }, 90_000); + + beforeEach(async () => { + await first.delete(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)); + vi.stubEnv("DATABASE_URL", database.connectionString); + }); + + afterAll(async () => { await database?.cleanup(); }); + + it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])("seeds a fresh database with %s", async (signal) => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token"); + const result = await initializeCloudRuntimeConfig(first); + expect(result).toMatchObject({ seeded: true, current: { version: 1, revision: 0, config: null } }); + expect(await cloudRuntimeConfigService(second).inspectStored()).toEqual(result?.current); + expect(fs.existsSync(configPath)).toBe(false); + vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", "authenticated"); + vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://environment.example.test"); + expect(loadConfig()).toMatchObject({ databaseMode: "postgres", authBaseUrlMode: "explicit", secretsStrictMode: true, telemetryEnabled: true }); + }); + + it("loads the saved authenticated mode before validating an environment host bind", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + vi.stubEnv("HOST", "0.0.0.0"); + vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined); + expect(loadCloudBootstrapConfig()).toEqual({ + databaseMode: "postgres", + databaseUrl: database.connectionString, + databaseMigrationUrl: process.env.DATABASE_MIGRATION_URL, + }); + await initializeCloudRuntimeConfig(first); + expect(loadConfig()).toMatchObject({ deploymentMode: "authenticated", host: "0.0.0.0", port: 3210 }); + }); + + it("imports existing choices without credentials and restores them from a second client and new home", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + const initial = await initializeCloudRuntimeConfig(first); + expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, databaseBackupEnabled: false, storageS3Bucket: "saved-bucket", authDisableSignUp: true }); + const [row] = await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)); + const persisted = JSON.stringify(row.general); + for (const secret of ["legacy-openai-secret", "database-secret", "unknown-secret-value", "environment-api-secret", database.connectionString]) { + expect(persisted).not.toContain(secret); + } + const replacementHome = fs.mkdtempSync(path.join(home, "replacement-")); + const replacementConfigDir = path.join(replacementHome, ".paperclip"); + fs.mkdirSync(replacementConfigDir); + const envPath = path.join(replacementConfigDir, ".env"); + const envContents = "PORT=9999\nPAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL=should-not-load\n"; + fs.writeFileSync(envPath, envContents); + vi.stubEnv("PAPERCLIP_HOME", replacementHome); + vi.stubEnv("PAPERCLIP_CONFIG", path.join(replacementConfigDir, "config.json")); + setCloudRuntimeConfigSnapshot(null); + const files = vi.spyOn(fs, "readFileSync"); + expect(await initializeCloudRuntimeConfig(second)).toEqual({ seeded: false, current: initial?.current }); + expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" }); + expect(files.mock.calls.map(([file]) => String(file)).filter(file => file.startsWith(replacementHome))).toEqual([]); + files.mockRestore(); + const root = fileURLToPath(new URL("../../../", import.meta.url)); + const childScript = path.join(home, "restore.mts"); + fs.writeFileSync(childScript, ` + import { createDb } from ${JSON.stringify(pathToFileURL(path.join(root, "packages/db/src/index.ts")).href)}; + import { initializeCloudRuntimeConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/services/cloud-runtime-config.ts")).href)}; + import { loadConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/config.ts")).href)}; + const db = createDb(process.env.DATABASE_URL); + await initializeCloudRuntimeConfig(db); + const config = loadConfig(); + console.log(JSON.stringify({ port: config.port, telemetryEnabled: config.telemetryEnabled, storageS3Bucket: config.storageS3Bucket, + envSentinel: process.env.PAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL ?? null })); + await db.$client.end({ timeout: 1 }); + `); + const child = await promisify(execFile)(process.execPath, [path.join(root, "server/node_modules/tsx/dist/cli.mjs"), childScript], { + cwd: replacementHome, + env: { ...process.env }, + timeout: 30_000, + }); + expect(JSON.parse(child.stdout.trim())).toEqual({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket", envSentinel: null }); + expect(fs.readFileSync(envPath, "utf8")).toBe(envContents); + }, 45_000); + + it("keeps environment overlays out of saved settings and ignores corrupt old files after seed", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + await initializeCloudRuntimeConfig(first); + vi.stubEnv("PORT", "4567"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "operator-bucket"); + vi.stubEnv("PAPERCLIP_AUTH_DISABLE_SIGN_UP", "false"); + fs.writeFileSync(configPath, "malformed legacy contents"); + await initializeCloudRuntimeConfig(second); + expect(loadConfig()).toMatchObject({ port: 4567, storageS3Bucket: "operator-bucket", authDisableSignUp: false, telemetryEnabled: false }); + expect((await cloudRuntimeConfigService(first).inspectStored())?.config).toMatchObject({ server: { port: 3210 }, storage: { s3: { bucket: "saved-bucket" } }, auth: { disableSignUp: true } }); + }); + + it("keeps one committed seed when two clients propose different initial values", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + const results = await Promise.all([ + cloudRuntimeConfigService(first).seedIfAbsent({ ...config, telemetry: { enabled: true } }), + cloudRuntimeConfigService(second).seedIfAbsent({ ...config, telemetry: { enabled: false } }), + ]); + expect(results.filter(result => result.seeded)).toHaveLength(1); + expect(results[0].current).toEqual(results[1].current); + expect(await cloudRuntimeConfigService(first).inspectStored()).toEqual(results[0].current); + }); + + it("rejects unknown or credential fields on the direct mutation path", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + await cloudRuntimeConfigService(first).seedIfAbsent(config); + await expect(cloudRuntimeConfigService(first).replace({ ...config, llm: { provider: "openai", apiKey: "must-not-store" } } as never, 0)).rejects.toThrow(/configuration is invalid/); + expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(0); + }); + + it("serializes revisions and rejects competing stale writes from two database clients", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + await cloudRuntimeConfigService(first).seedIfAbsent(config); + const results = await Promise.allSettled([ + cloudRuntimeConfigService(first).replace({ ...config, telemetry: { enabled: true } }, 0), + cloudRuntimeConfigService(second).replace({ ...config, telemetry: { enabled: false } }, 0), + ]); + expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1); + expect(results.filter(result => result.status === "rejected")).toHaveLength(1); + expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(1); + }); + + it("composes a seed with the caller transaction and rolls back both on failure", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + await expect(first.transaction(async tx => { + await cloudRuntimeConfigService(first).seedIfAbsent(config, tx); + await tx.insert(instanceSettings).values({ singletonKey: "runtime-config-test-receipt", general: {} }); + throw new Error("injected migration failure"); + })).rejects.toThrow(/injected migration failure/); + expect(await cloudRuntimeConfigService(second).inspectStored()).toBeNull(); + expect(await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "runtime-config-test-receipt"))).toEqual([]); + }); + + it("fails closed on invalid database settings without consulting a valid legacy file", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 99, config: { secret: "hidden" } } }); + await expect(initializeCloudRuntimeConfig(second)).rejects.toThrow(/Persisted Cloud runtime configuration is invalid/); + expect(() => getCloudRuntimeConfigSnapshot()).toThrow(/not been initialized/); + }); + + it("does not fall back to a file or old snapshot when the database fails", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + await initializeCloudRuntimeConfig(first); + const unavailable = createDb(database.connectionString); + await unavailable.$client.end({ timeout: 1 }); + await expect(initializeCloudRuntimeConfig(unavailable)).rejects.toThrow(); + expect(() => readConfigFile()).toThrow(/not been initialized/); + }); +}); diff --git a/server/src/__tests__/helpers/bundled-image-fixture.ts b/server/src/__tests__/helpers/bundled-image-fixture.ts new file mode 100644 index 00000000000..dcb958a5235 --- /dev/null +++ b/server/src/__tests__/helpers/bundled-image-fixture.ts @@ -0,0 +1,50 @@ +import { mkdtempSync, mkdirSync, realpathSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../../services/bundled-plugins.js"; +import { distributionBundleDigest, readDistributionPluginCatalog } from "../../services/distribution-plugin-catalog.js"; +import { derivePluginDatabaseNamespace } from "../../services/plugin-database.js"; +import { createBundledPluginPackageResolver } from "../../services/bundled-plugin-package.js"; + +export function bundledImageFixture(manifestOverrides: Partial = {}) { + const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "bundled-image-"))); + const packageRoot = path.join(root, "distribution", "fixture"); + const manifest: PaperclipPluginManifestV1 = { + id: "paperclip.image-fixture", apiVersion: 1, version: "1.0.0", displayName: "Image Fixture", + description: "Image package relocation fixture", author: "Paperclip", categories: ["automation"], capabilities: ["issues.read"], + entrypoints: { worker: "dist/worker.js", ui: "dist/ui" }, ...manifestOverrides, + }; + mkdirSync(path.join(packageRoot, "dist", "ui"), { recursive: true }); + writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify({ + name: "@paperclipai/plugin-image-fixture", version: manifest.version, type: "module", + paperclipPlugin: { manifest: "dist/manifest.js", ...manifest.entrypoints }, + })); + writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`); + writeFileSync(path.join(packageRoot, "dist", "worker.js"), `import readline from "node:readline"; +if (process.argv.includes("--source")) process.stdout.write(import.meta.url); +else { + const configs = {}; + readline.createInterface({ input: process.stdin }).on("line", line => { + const request = JSON.parse(line); + let result = {}; + if (request.method === "initialize") result = { ok: true, supportedMethods: ["configChanged", "health"] }; + if (request.method === "configChanged") configs[request.params.companyId] = request.params.config; + if (request.method === "health") result = { status: "ok", details: { source: import.meta.url, configs } }; + process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result }) + "\\n"); + if (request.method === "shutdown") setImmediate(() => process.exit(0)); + }); +}`); + writeFileSync(path.join(packageRoot, "dist", "ui", "index.js"), "export const source = 'current-image';"); + if (manifest.database) { + mkdirSync(path.join(packageRoot, manifest.database.migrationsDir), { recursive: true }); + writeFileSync(path.join(packageRoot, manifest.database.migrationsDir, "001_fixture.sql"), `CREATE TABLE ${derivePluginDatabaseNamespace(manifest.id)}.image_rows (id uuid PRIMARY KEY);`); + } + const entry = { key: "fixture", pluginKey: manifest.id, version: manifest.version, directory: "fixture", digest: distributionBundleDigest(packageRoot) }; + writeFileSync(path.join(root, "distribution", "catalog.json"), JSON.stringify({ schemaVersion: 1, plugins: [entry] })); + const catalog = readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG); + const installs = resolveBundledPluginInstalls(["fixture"], { catalogRoot: root, enforceCatalogRoot: true, env: {}, distributionPlugins: catalog }); + const resolver = createBundledPluginPackageResolver(installs); + const registration = { pluginKey: manifest.id, packageName: "@paperclipai/plugin-image-fixture", version: manifest.version, manifestJson: manifest }; + return { root, packageRoot, manifest, catalog, installs, resolver, registration }; +} diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 420a18bbc86..6942d87e8d3 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -20,6 +20,22 @@ import { gzipSync } from "node:zlib"; import { tmpdir } from "node:os"; import path from "node:path"; +const cloudRuntimeStartupMocks = vi.hoisted(() => ({ + initialize: vi.fn(async () => null), + bootstrap: vi.fn(), + telemetry: vi.fn(), +})); + +vi.mock("../services/adapter-settings.js", async () => ({ + ...await vi.importActual("../services/adapter-settings.js"), + initializeAdapterSettings: vi.fn(async () => null), +})); +vi.mock("../services/cloud-runtime-config.js", () => ({ initializeCloudRuntimeConfig: cloudRuntimeStartupMocks.initialize })); +const cloudMigrationMocks = vi.hoisted(() => ({ migrate: vi.fn(async () => null), qualify: vi.fn(async () => null) })); +vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudConfiguration: cloudMigrationMocks.migrate })); +vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify })); +vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) })); + const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON; @@ -343,6 +359,7 @@ vi.mock("../services/native-runtime/native-safe-replacement.js", () => ({ vi.mock("../config.js", () => ({ loadConfig: loadConfigMock, + loadCloudBootstrapConfig: cloudRuntimeStartupMocks.bootstrap, })); vi.mock("../middleware/logger.js", () => ({ @@ -586,6 +603,8 @@ describe("startServer feedback export wiring", () => { process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210"; process.env.PAPERCLIP_AGENT_JWT_SECRET = "0123456789abcdef0123456789abcdef"; loadConfigMock.mockReturnValue(buildTestConfig()); + cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null); + cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined }); resolveHeartbeatSchedulingSuppressionMock.mockReturnValue({ suppressed: false, reason: null, @@ -595,6 +614,64 @@ describe("startServer feedback export wiring", () => { process.env.BETTER_AUTH_SECRET = "test-secret"; }); + it("awaits saved Cloud configuration before telemetry, authentication and app construction", async () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + vi.stubEnv("HOST", "0.0.0.0"); + vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined); + cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined }); + let release: (() => void) | undefined; + cloudRuntimeStartupMocks.initialize.mockImplementationOnce(() => new Promise(resolve => { + release = () => { + loadConfigMock.mockReturnValue(buildTestConfig({ telemetryEnabled: false, authDisableSignUp: true, port: 4321, host: "0.0.0.0", bind: "all" })); + resolve(null); + }; + })); + try { + const starting = startServer(); + for (let index = 0; index < 30 && !release; index += 1) await new Promise(resolve => setImmediate(resolve)); + expect(release).toBeTypeOf("function"); + expect(cloudMigrationMocks.migrate.mock.invocationCallOrder[0]).toBeLessThan(cloudRuntimeStartupMocks.initialize.mock.invocationCallOrder[0]!); + expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled(); + expect(createBetterAuthInstanceMock).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); + expect(loadConfigMock).not.toHaveBeenCalled(); + release!(); + const started = await starting; + await settleStartupHousekeeping(); + expect(cloudRuntimeStartupMocks.telemetry).toHaveBeenCalledExactlyOnceWith({ enabled: false }); + expect(createBetterAuthInstanceMock).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ authDisableSignUp: true, port: 4321 }), expect.anything()); + expect(started.listenPort).toBe(4321); + expect(cloudMigrationMocks.qualify.mock.invocationCallOrder[0]).toBeLessThan(createAppMock.mock.invocationCallOrder[0]!); + } finally { + release?.(); + vi.unstubAllEnvs(); + } + }); + + it("does not create telemetry or the app when saved Cloud configuration fails", async () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + cloudRuntimeStartupMocks.initialize.mockRejectedValueOnce(new Error("persisted config unavailable")); + try { + await expect(startServer()).rejects.toThrow("persisted config unavailable"); + expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled(); + expect(createBetterAuthInstanceMock).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("stops before configuration consumers when the atomic Cloud migration fails", async () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + cloudMigrationMocks.migrate.mockRejectedValueOnce(new Error("migration did not commit")); + try { + await expect(startServer()).rejects.toThrow("migration did not commit"); + expect(cloudRuntimeStartupMocks.initialize).not.toHaveBeenCalled(); + expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); + } finally { vi.unstubAllEnvs(); } + }); + it.each([false, true])("registers runner evidence independently from the backup scan (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); @@ -1509,6 +1586,8 @@ describe("startServer authenticated auth origin setup", () => { vi.clearAllMocks(); process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210"; loadConfigMock.mockReturnValue(buildTestConfig()); + cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null); + cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined }); createBetterAuthInstanceMock.mockReturnValue({}); deriveAuthTrustedOriginsMock.mockReturnValue([]); process.env.BETTER_AUTH_SECRET = "test-secret"; @@ -1570,6 +1649,8 @@ describe("startServer PAPERCLIP_API_URL handling", () => { vi.clearAllMocks(); process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210"; loadConfigMock.mockReturnValue(buildTestConfig()); + cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null); + cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined }); process.env.BETTER_AUTH_SECRET = "test-secret"; delete process.env.PAPERCLIP_API_URL; }); diff --git a/server/src/adapters/cloud-external-adapters.test.ts b/server/src/adapters/cloud-external-adapters.test.ts new file mode 100644 index 00000000000..21da3bccbeb --- /dev/null +++ b/server/src/adapters/cloud-external-adapters.test.ts @@ -0,0 +1,136 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +let home: string; +let localPackage: string; +let npmPackage: string; +let storePath: string; +let sentinel: string; +const parserSource = "export function parseStdoutLine() { return { message: 'external parser' }; }"; +const managedError = { status: 403, details: { code: "adapter_install_platform_managed" } }; + +function writePackage(packageDir: string, type: string): void { + fs.mkdirSync(packageDir, { recursive: true }); + fs.writeFileSync(path.join(packageDir, "package.json"), JSON.stringify({ + name: `fixture-${type}`, + type: "module", + exports: { ".": "./index.mjs", "./ui-parser": "./ui-parser.mjs" }, + paperclip: { adapterUiParser: "1.0.0" }, + })); + fs.writeFileSync(path.join(packageDir, "index.mjs"), ` + import fs from "node:fs"; + fs.appendFileSync(${JSON.stringify(sentinel)}, ${JSON.stringify(type + "\n")}); + export function createServerAdapter() { + return { + type: ${JSON.stringify(type)}, + agentConfigurationDoc: "External fixture", + models: [], + execute: async () => ({ exitCode: 41, signal: null, timedOut: false }), + testEnvironment: async () => ({ status: "pass", checks: [] }), + }; + } + `); + fs.writeFileSync(path.join(packageDir, "ui-parser.mjs"), parserSource); +} + +beforeEach(() => { + vi.resetModules(); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); + home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-cloud-adapters-")); + vi.stubEnv("PAPERCLIP_HOME", home); + sentinel = path.join(home, "external-code-executed"); + localPackage = path.join(home, "local-package"); + npmPackage = path.join(home, "adapter-plugins", "node_modules", "fixture-npm"); + storePath = path.join(home, "adapter-plugins.json"); + writePackage(localPackage, "codex_local"); + writePackage(npmPackage, "fixture_npm"); + fs.writeFileSync(storePath, JSON.stringify([ + { packageName: "fixture-local", localPath: localPackage, type: "codex_local", installedAt: "2026-01-01" }, + { packageName: "fixture-npm", type: "fixture_npm", installedAt: "2026-01-01" }, + ])); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(home, { recursive: true, force: true }); +}); + +describe("Cloud image adapter boundary", () => { + it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])( + "cold boots built-ins without reading or executing persisted packages with %s", + async (signal) => { + vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token"); + const registrationBefore = fs.readFileSync(storePath, "utf8"); + const reads = vi.spyOn(fs, "readFileSync"); + const registry = await import("./registry.js"); + await registry.waitForExternalAdapters(); + const loader = await import("./plugin-loader.js"); + const store = await import("../services/adapter-plugin-store.js"); + + expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture"); + expect(registry.requireServerAdapter("process").execute).toBeTypeOf("function"); + expect(registry.findServerAdapter("fixture_npm")).toBeNull(); + expect(registry.hasActiveAdapterOverride("codex_local")).toBe(false); + expect(await loader.buildExternalAdapters()).toEqual([]); + expect(store.listAdapterPlugins()).toEqual([]); + expect(store.getAdapterPluginByType("codex_local")).toBeUndefined(); + expect(loader.getUiParserSource("codex_local")).toBeUndefined(); + expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined(); + await expect(loader.loadExternalAdapterPackage("fixture-local", localPackage)).rejects.toMatchObject(managedError); + await expect(loader.loadExternalAdapterPackage("fixture-npm")).rejects.toMatchObject(managedError); + await expect(loader.reloadExternalAdapter("codex_local")).rejects.toMatchObject(managedError); + expect(() => registry.registerServerAdapter(registry.requireServerAdapter("codex_local"))).toThrow(); + expect(() => registry.unregisterServerAdapter("codex_local")).toThrow(); + expect(() => registry.setOverridePaused("codex_local", false)).toThrow(); + expect(() => store.addAdapterPlugin({ packageName: "fixture", type: "fixture", installedAt: "2026-01-01" })).toThrow(); + expect(() => store.removeAdapterPlugin("codex_local")).toThrow(); + expect(() => store.getAdapterPluginsDir()).toThrow(); + expect(store.getAdapterPluginStorePath()).toBe(storePath); + expect(reads.mock.calls.map(([file]) => String(file)).filter(file => + file === storePath || file.startsWith(localPackage + path.sep) || file.startsWith(npmPackage + path.sep), + )).toEqual([]); + reads.mockRestore(); + expect(fs.existsSync(sentinel)).toBe(false); + expect(fs.readFileSync(storePath, "utf8")).toBe(registrationBefore); + }, + ); + + it("preserves local and npm package loading, overrides, reload, and lazy UI parsing on self-hosted instances", async () => { + const loader = await import("./plugin-loader.js"); + expect(loader.getOrExtractUiParserSource("codex_local")).toBe(parserSource); + const registry = await import("./registry.js"); + await registry.waitForExternalAdapters(); + + const override = registry.requireServerAdapter("codex_local"); + expect(override.agentConfigurationDoc).toBe("External fixture"); + expect(registry.hasActiveAdapterOverride("codex_local")).toBe(true); + expect(registry.requireServerAdapter("fixture_npm").agentConfigurationDoc).toBe("External fixture"); + expect(await override.execute({} as never)).toMatchObject({ exitCode: 41 }); + expect(fs.readFileSync(sentinel, "utf8").trim().split("\n")).toEqual(["codex_local", "fixture_npm"]); + expect(loader.getUiParserSource("fixture_npm")).toBe(parserSource); + const reloaded = await loader.reloadExternalAdapter("codex_local"); + expect(reloaded?.type).toBe("codex_local"); + expect(await reloaded?.execute({} as never)).toMatchObject({ exitCode: 41 }); + expect(registry.setOverridePaused("codex_local", true)).toBe(true); + expect(registry.findActiveServerAdapter("codex_local")?.agentConfigurationDoc).not.toBe("External fixture"); + expect(registry.setOverridePaused("codex_local", false)).toBe(true); + registry.unregisterServerAdapter("codex_local"); + expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture"); + }); + + it("does not expose a cached registration or parser after a managed signal becomes present", async () => { + const loader = await import("./plugin-loader.js"); + const store = await import("../services/adapter-plugin-store.js"); + await loader.loadExternalAdapterPackage("fixture-local", localPackage); + expect(loader.getUiParserSource("codex_local")).toBe(parserSource); + expect(store.listAdapterPlugins()).toHaveLength(2); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", "{}"); + expect(store.listAdapterPlugins()).toEqual([]); + expect(loader.getUiParserSource("codex_local")).toBeUndefined(); + expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined(); + }); +}); diff --git a/server/src/adapters/plugin-loader.ts b/server/src/adapters/plugin-loader.ts index 236e8ddd50f..2777d813af1 100644 --- a/server/src/adapters/plugin-loader.ts +++ b/server/src/adapters/plugin-loader.ts @@ -15,6 +15,8 @@ import { pathToFileURL } from "node:url"; import type { ServerAdapterModule } from "./types.js"; import { validateAdapterLoginCapability } from "@paperclipai/adapter-utils"; import { logger } from "../middleware/logger.js"; +import { isCloudManagedInstance } from "../services/cloud-instance.js"; +import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js"; import { listAdapterPlugins, @@ -30,6 +32,7 @@ import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js"; const uiParserCache = new Map(); export function getUiParserSource(adapterType: string): string | undefined { + if (isCloudManagedInstance()) return undefined; return uiParserCache.get(adapterType); } @@ -38,6 +41,7 @@ export function getUiParserSource(adapterType: string): string | undefined { * Makes the ui-parser.js endpoint self-healing. */ export function getOrExtractUiParserSource(adapterType: string): string | undefined { + if (isCloudManagedInstance()) return undefined; const cached = uiParserCache.get(adapterType); if (cached) return cached; @@ -182,6 +186,7 @@ export async function loadExternalAdapterPackage( packageName: string, localPath?: string, ): Promise { + assertExternalAdapterCodeAllowed(); const packageDir = localPath ? path.resolve(localPath) : path.resolve(getAdapterPluginsDir(), "node_modules", packageName); @@ -221,6 +226,7 @@ async function loadFromRecord(record: AdapterPluginRecord): Promise { + assertExternalAdapterCodeAllowed(); const record = getAdapterPluginByType(type); if (!record) return null; @@ -271,6 +277,7 @@ export async function reloadExternalAdapter( * Build all external adapter modules from the plugin store. */ export async function buildExternalAdapters(): Promise { + if (isCloudManagedInstance()) return []; const results: ServerAdapterModule[] = []; const storeRecords = listAdapterPlugins(); diff --git a/server/src/adapters/registry.ts b/server/src/adapters/registry.ts index 2a271a5d720..e4c6dddf53d 100644 --- a/server/src/adapters/registry.ts +++ b/server/src/adapters/registry.ts @@ -136,7 +136,8 @@ import { import { agentConfigurationDoc as piAgentConfigurationDoc } from "@paperclipai/adapter-pi-local"; import { BUILTIN_ADAPTER_TYPES } from "./builtin-adapter-types.js"; import { buildExternalAdapters } from "./plugin-loader.js"; -import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js"; +import { isCloudManagedInstance } from "../services/cloud-instance.js"; +import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js"; import { NativeEnvironmentProbeCleanupError } from "../vendor/paperclip-runner/index.js"; import { processAdapter } from "./process/index.js"; import { httpAdapter } from "./http/index.js"; @@ -997,11 +998,6 @@ registerBuiltInAdapters(); // applies the registry fallback for externals overriding a built-in type. // --------------------------------------------------------------------------- -/** Cached sync wrapper — the store is a simple JSON file read, safe to call frequently. */ -function getDisabledAdapterTypesFromStore(): string[] { - return getDisabledAdapterTypes(); -} - /** * Merge an external adapter module with host-provided session management. * @@ -1034,6 +1030,7 @@ export function resolveExternalAdapterRegistration( * and avoid racing against the loading window. */ const externalAdaptersReady: Promise = (async () => { + if (isCloudManagedInstance()) return; try { const externalAdapters = await buildExternalAdapters(); for (const externalAdapter of externalAdapters) { @@ -1069,6 +1066,7 @@ export function waitForExternalAdapters(): Promise { } export function registerServerAdapter(adapter: ServerAdapterModule): void { + assertExternalAdapterCodeAllowed(); if (BUILTIN_ADAPTER_TYPES.has(adapter.type) && !builtinFallbacks.has(adapter.type)) { const existing = adaptersByType.get(adapter.type); if (existing) { @@ -1079,6 +1077,7 @@ export function registerServerAdapter(adapter: ServerAdapterModule): void { } export function unregisterServerAdapter(type: string): void { + assertExternalAdapterCodeAllowed(); if (type === processAdapter.type || type === httpAdapter.type) return; if (builtinFallbacks.has(type)) { pausedOverrides.delete(type); @@ -1186,8 +1185,7 @@ export function listServerAdapters(): ServerAdapterModule[] { * Used for menus and agent creation flows — disabled adapters remain * functional for existing agents but hidden from selection. */ -export function listEnabledServerAdapters(): ServerAdapterModule[] { - const disabled = getDisabledAdapterTypesFromStore(); +export function listEnabledServerAdapters(disabled: string[]): ServerAdapterModule[] { const disabledSet = disabled.length > 0 ? new Set(disabled) : null; return disabledSet ? Array.from(adaptersByType.values()).filter((a) => !disabledSet.has(a.type)) @@ -1227,6 +1225,7 @@ export async function detectAdapterModel( * an override or was already in the requested state. */ export function setOverridePaused(type: string, paused: boolean): boolean { + assertExternalAdapterCodeAllowed(); if (!builtinFallbacks.has(type)) return false; const wasPaused = pausedOverrides.has(type); if (paused && !wasPaused) { diff --git a/server/src/app.ts b/server/src/app.ts index dcb51306a6b..130bfed7b60 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -183,7 +183,9 @@ import { ensureBundledPlugins, resolveBundledCatalogRoot, resolveBundledPluginInstalls, + resolveManagedBundledPluginKeys, } from "./services/bundled-plugins.js"; +import { createBundledPluginPackageResolver } from "./services/bundled-plugin-package.js"; import { readDistributionPluginCatalog, distributionPluginActivationGuard } from "./services/distribution-plugin-catalog.js"; import { createPluginWorkerManager, @@ -623,7 +625,7 @@ export async function createApp( res.set("Cache-Control", "no-store"); res.set("Referrer-Policy", "no-referrer"); res.redirect(303, await chatChannels.githubWizard.identityCallback(String(req.query.state ?? ""), String(req.query.code ?? ""), req.actor)); }); - const managedAutoInstallKeys = opts.managedPluginAutoInstall ?? null; + const managedAutoInstallKeys = resolveManagedBundledPluginKeys(opts.managedPluginAutoInstall); const bundledCatalogRoot = opts.bundledPluginCatalogRoot ?? resolveBundledCatalogRoot(process.env); const distributionPlugins = readDistributionPluginCatalog(bundledCatalogRoot, BUNDLED_PLUGIN_CATALOG); @@ -636,6 +638,7 @@ export async function createApp( distributionPlugins, }, ); + const bundledPackageResolver = managedAutoInstallKeys === null ? undefined : createBundledPluginPackageResolver(bundledPluginInstalls); const managedBundledPluginKeys = managedAutoInstallKeys !== null ? bundledPluginInstalls.map((install) => install.pluginKey) @@ -906,7 +909,11 @@ export async function createApp( { localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR, migrationDb: opts.pluginMigrationDb, - assertPackageActivation: distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys), + assertPackageActivation: bundledPackageResolver?.assertActivation ?? distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys), + resolvePackageRoot: bundledPackageResolver?.resolve, + prebuiltOnly: managedAutoInstallKeys !== null, + enableLocalFilesystem: managedAutoInstallKeys === null, + enableNpmDiscovery: managedAutoInstallKeys === null, }, { workerManager, @@ -965,6 +972,7 @@ export async function createApp( ); api.use( adapterRoutes({ + db, resolveRunnerTarget: (companyId, environmentId) => resolveAgentRunnerTargetForCompany(db, companyId, environmentId), getNativeRunnerEnabled: async () => (await instanceSettingsService(db).getExperimental()) @@ -1014,6 +1022,7 @@ export async function createApp( app.use( pluginUiStaticRoutes(db, { localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR, + resolvePackageRoot: bundledPackageResolver?.resolve, }), ); @@ -1253,7 +1262,7 @@ export async function createApp( void toolDispatcher.initialize().catch((err) => { logger.error({ err }, "Failed to initialize plugin tool dispatcher"); }); - const devWatcher = createPluginDevWatcher( + const devWatcher = managedAutoInstallKeys !== null ? null : createPluginDevWatcher( lifecycle, async (pluginId) => (await pluginRegistry.getById(pluginId))?.packagePath ?? null, @@ -1299,7 +1308,7 @@ export async function createApp( // owns provisioning); self-hosted leaves an operator's uninstall alone. // Disabled plugins never start automatically. Added distribution permissions // still enter upgrade_pending so enabling them requires an operator decision. - { reinstallUninstalled: managedAutoInstallKeys !== null }, + { reinstallUninstalled: managedAutoInstallKeys !== null, preservePackagePath: managedAutoInstallKeys !== null }, ) .then(() => loader.loadAll()) .then((result) => { diff --git a/server/src/config-file.ts b/server/src/config-file.ts index 957e8d05881..d750793c204 100644 --- a/server/src/config-file.ts +++ b/server/src/config-file.ts @@ -8,6 +8,8 @@ import { } from "@paperclipai/shared"; import { ZodError } from "zod"; import { resolvePaperclipConfigPath } from "./paths.js"; +import { isCloudManagedInstance } from "./services/cloud-instance.js"; +import { getCloudRuntimeConfigSnapshot, toPaperclipConfig } from "./services/cloud-runtime-config-snapshot.js"; function formatConfigValidationError(error: ZodError): string { return error.issues @@ -19,6 +21,7 @@ function formatConfigValidationError(error: ZodError): string { } export function readConfigFile(): PaperclipConfig | null { + if (isCloudManagedInstance()) return toPaperclipConfig(getCloudRuntimeConfigSnapshot().config); const configPath = resolvePaperclipConfigPath(); if (!fs.existsSync(configPath)) return null; diff --git a/server/src/config.ts b/server/src/config.ts index dac9dba2142..e5af468eedb 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -1,4 +1,7 @@ import { readConfigFile, resolveDeploymentMode } from "./config-file.js"; +import { isCloudManagedInstance } from "./services/cloud-instance.js"; +import { StartupRefusalError } from "./startup-refusals.js"; +import type { PaperclipConfig } from "@paperclipai/shared"; import { parseChatWebhookPublicBaseUrl } from "./chat-webhook-public-url.js"; import { execFileSync } from "node:child_process"; import { existsSync, realpathSync } from "node:fs"; @@ -32,27 +35,29 @@ import { resolveHomeAwarePath, } from "./home-paths.js"; -const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath(); -if (existsSync(PAPERCLIP_ENV_FILE_PATH)) { - loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true }); -} +if (!isCloudManagedInstance()) { + const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath(); + if (existsSync(PAPERCLIP_ENV_FILE_PATH)) { + loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true }); + } -const CWD_ENV_PATH = resolve(process.cwd(), ".env"); -const cwdEnvExists = existsSync(CWD_ENV_PATH); -const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH) - ? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH) - : CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH; -if (shouldLoadWorkingDirectoryEnv({ - cwdEnvExists, - isPaperclipEnvFile: isSameFile, -})) { - loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true }); -} + const CWD_ENV_PATH = resolve(process.cwd(), ".env"); + const cwdEnvExists = existsSync(CWD_ENV_PATH); + const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH) + ? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH) + : CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH; + if (shouldLoadWorkingDirectoryEnv({ + cwdEnvExists, + isPaperclipEnvFile: isSameFile, + })) { + loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true }); + } -if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") { - applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH))); + if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") { + applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH))); + } + maybeRepairLegacyWorktreeConfigAndEnvFiles(); } -maybeRepairLegacyWorktreeConfigAndEnvFiles(); const TAILSCALE_DETECT_TIMEOUT_MS = 3000; @@ -147,13 +152,12 @@ function detectTailnetBindHost(): string | undefined { } } -export function loadConfig(): Config { - const fileConfig = readConfigFile(); +export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()): Config { const fileDatabaseMode = - (fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode; + (isCloudManagedInstance() || fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode; const fileDbUrl = - fileDatabaseMode === "postgres" + !isCloudManagedInstance() && fileDatabaseMode === "postgres" ? fileConfig?.database.connectionString : undefined; const fileDatabaseBackup = fileConfig?.database.backup; @@ -460,3 +464,23 @@ export function loadConfig(): Config { announcementsFeedUrl: process.env.PAPERCLIP_ANNOUNCEMENTS_FEED_URL?.trim() || "https://pages.paperclip.ing/announcements/v1/current.json", }; } + +export type CloudBootstrapConfig = { + databaseMode: "postgres"; + databaseUrl: string; + databaseMigrationUrl: string | undefined; +}; + +export function loadCloudBootstrapConfig(): CloudBootstrapConfig { + if (!isCloudManagedInstance()) throw new Error("Cloud bootstrap configuration requires a managed instance"); + const url = process.env.DATABASE_URL?.trim(); + if (!url) throw new StartupRefusalError("database-contract-unmet", "Cloud bootstrap requires DATABASE_URL from the environment"); + const migrationUrl = process.env.DATABASE_MIGRATION_URL?.trim() || undefined; + for (const [name, value] of [["DATABASE_URL", url], ["DATABASE_MIGRATION_URL", migrationUrl]]) { + if (!value) continue; + let parsed: URL; + try { parsed = new URL(value); } catch { throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); } + if (!["postgres:", "postgresql:"].includes(parsed.protocol)) throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); + } + return { databaseMode: "postgres", databaseUrl: url, databaseMigrationUrl: migrationUrl }; +} diff --git a/server/src/index.ts b/server/src/index.ts index 424580c2a69..9ba0e2c02d2 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -104,7 +104,8 @@ import { } from "@paperclipai/db"; import detectPort from "detect-port"; import { createApp } from "./app.js"; -import { loadConfig } from "./config.js"; +import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js"; +import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js"; import { logger } from "./middleware/logger.js"; import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js"; import { setStartupRecoveryPhase } from "./startup-recovery-state.js"; @@ -186,6 +187,9 @@ import { loadWithoutCoordinatedShutdownSignalHooks, } from "./shutdown.js"; import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js"; +import { initializeAdapterSettings } from "./services/adapter-settings.js"; +import { migrateCloudConfiguration } from "./services/cloud-configuration-migration.js"; +import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js"; import { systemdNotify } from "./services/systemd-notify.js"; import { flushInFlightRunLogMirrors } from "./services/run-log-store.js"; import { @@ -251,6 +255,19 @@ async function endDatabaseClient(client: unknown, timeoutSeconds: number): Promi await sql.end({ timeout: timeoutSeconds }); } +function initializeRuntimeConfigConsumers(config: ReturnType): void { + initTelemetry({ enabled: config.telemetryEnabled }); + if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) { + process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider; + } + if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) { + process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false"; + } + if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) { + process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath; + } +} + export async function startServer(): Promise { const startupDatabase: StartupDatabaseTeardown = { close: null }; try { @@ -278,17 +295,8 @@ async function startServerWithDatabaseTeardown( // sentry.ts. await sentryReady; ensureDecisionSigningSecret(); - let config = loadConfig(); - initTelemetry({ enabled: config.telemetryEnabled }); - if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) { - process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider; - } - if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) { - process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false"; - } - if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) { - process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath; - } + const startupConfig = isCloudManagedInstance() ? loadCloudBootstrapConfig() : loadConfig(); + if ("deploymentMode" in startupConfig) initializeRuntimeConfigConsumers(startupConfig); type MigrationSummary = | "skipped" @@ -390,7 +398,7 @@ async function startServerWithDatabaseTeardown( } } - function assertCloudDatabaseContract(): void { + function assertCloudDatabaseContract(config: Config): void { if (config.deploymentMode !== "authenticated" || config.deploymentExposure !== "public") { return; } @@ -483,17 +491,19 @@ async function startServerWithDatabaseTeardown( let startupDbInfo: | { mode: "external-postgres"; connectionString: string } | { mode: "embedded-postgres"; dataDir: string; port: number }; - assertCloudDatabaseContract(); - if (config.databaseUrl) { - const migrationUrl = config.databaseMigrationUrl ?? config.databaseUrl; + if ("deploymentMode" in startupConfig) assertCloudDatabaseContract(startupConfig); + if (startupConfig.databaseUrl) { + const migrationUrl = startupConfig.databaseMigrationUrl ?? startupConfig.databaseUrl; migrationSummary = await ensureMigrations(migrationUrl, "PostgreSQL"); - - db = createDb(config.databaseUrl); - pluginMigrationDb = config.databaseMigrationUrl ? createDb(config.databaseMigrationUrl) : db; + + db = createDb(startupConfig.databaseUrl); + pluginMigrationDb = startupConfig.databaseMigrationUrl ? createDb(startupConfig.databaseMigrationUrl) : db; logger.info("Using external PostgreSQL via DATABASE_URL/config"); - activeDatabaseConnectionString = config.databaseUrl; - startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl }; + activeDatabaseConnectionString = startupConfig.databaseUrl; + startupDbInfo = { mode: "external-postgres", connectionString: startupConfig.databaseUrl }; } else { + if (!("deploymentMode" in startupConfig)) throw new Error("Cloud bootstrap requires an external PostgreSQL connection"); + const config = startupConfig; const moduleName = "embedded-postgres"; let EmbeddedPostgres: EmbeddedPostgresCtor; try { @@ -711,12 +721,19 @@ async function startServerWithDatabaseTeardown( }; startupDatabase.close = closeDatabaseClients; setSentryRunLinkResolver(createSentryRunLinkResolver(db)); + await migrateCloudConfiguration(db as any); + await initializeAdapterSettings(db as any); + await initializeCloudRuntimeConfig(db as any); + let config = "deploymentMode" in startupConfig ? startupConfig : loadConfig(); + // A claimed warm-pool stack may restart while its provider environment still // names the pool host. Restore the signed, durable identity before Better // Auth, routes, or child-runtime configuration capture any public URL. const restoredCloudRuntimeIdentity = await initializeCloudRuntimeIdentity(db as any); if (restoredCloudRuntimeIdentity) config = loadConfig(); + if (isCloudManagedInstance()) initializeRuntimeConfigConsumers(config); + const isWarmStandby = await createCloudWarmStandby(db as any); if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) { @@ -963,6 +980,10 @@ async function startServerWithDatabaseTeardown( // document parsed fail-closed above (`plugins.autoInstall`). Absent env means // self-hosted: createApp falls back to its built-in kubernetes-only default. const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null; + const cloudQualification = await qualifyCloudConfigurationForCurrentImage(db as any); + if (cloudQualification && !cloudQualification.configurationPackagesEligible) { + logger.warn({ scope: cloudQualification.scope, blockerCodes: cloudQualification.blockers.map((blocker) => blocker.code) }, "Cloud configuration and package qualification has blockers"); + } const app = await createApp(db as any, { prepareIdleDatabaseBackup: idleDatabaseBackupEnabled ? async () => (await runServerDatabaseBackup("idle")) !== null diff --git a/server/src/paths.ts b/server/src/paths.ts index 21856a6946e..5ed3971d4fd 100644 --- a/server/src/paths.ts +++ b/server/src/paths.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { resolveDefaultConfigPath } from "./home-paths.js"; +import { isCloudManagedInstance } from "./services/cloud-instance.js"; const PAPERCLIP_CONFIG_BASENAME = "config.json"; const PAPERCLIP_ENV_FILENAME = ".env"; @@ -26,6 +27,7 @@ function findConfigFileFromAncestors(startDir: string): string | null { export function resolvePaperclipConfigPath(overridePath?: string): string { if (overridePath) return path.resolve(overridePath); if (process.env.PAPERCLIP_CONFIG) return path.resolve(process.env.PAPERCLIP_CONFIG); + if (isCloudManagedInstance()) return resolveDefaultConfigPath(); return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath(); } diff --git a/server/src/qualify-cloud-configuration.ts b/server/src/qualify-cloud-configuration.ts new file mode 100644 index 00000000000..f2e50830a3f --- /dev/null +++ b/server/src/qualify-cloud-configuration.ts @@ -0,0 +1,47 @@ +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { createDb } from "@paperclipai/db"; +import { cloudConfigurationMigrationService } from "./services/cloud-configuration-migration.js"; +import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js"; + +export async function runCloudConfigurationQualification(args: string[], env: NodeJS.ProcessEnv = process.env): Promise { + const allowed = new Set(["--migrate", "--expected-package-evidence", "--recheck-external-revision"]); + let migrate = false; + let expectedPackageEvidenceDigest: string | undefined; + let recheckRevision: number | undefined; + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + if (!allowed.has(argument)) throw new Error(`Unknown qualification argument: ${argument}`); + if (argument === "--migrate") migrate = true; + if (argument === "--expected-package-evidence") { + expectedPackageEvidenceDigest = args[++index]; + if (!expectedPackageEvidenceDigest || !/^sha256:[a-f0-9]{64}$/.test(expectedPackageEvidenceDigest)) throw new Error("A SHA-256 package evidence digest is required"); + } + if (argument === "--recheck-external-revision") { + const value = args[++index]; + if (!value || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error("A nonnegative receipt revision is required"); + recheckRevision = Number(value); + } + } + const url = env.DATABASE_URL?.trim(); + if (!url) throw new Error("DATABASE_URL is required for Cloud configuration qualification"); + const db = createDb(url); + try { + const service = cloudConfigurationMigrationService(db, { runtimeEnv: env }); + if (migrate) await service.migrate(); + if (recheckRevision !== undefined) await service.recheckLegacyExternalRegistrations(recheckRevision); + const report = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv: env, expectedPackageEvidenceDigest }); + if (!report) throw new Error("Cloud configuration qualification requires a managed instance"); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + return report.configurationPackagesEligible && (!expectedPackageEvidenceDigest || report.samePackageOverlapEligible) ? 0 : 2; + } finally { + await db.$client.end({ timeout: 5 }); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + runCloudConfigurationQualification(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : "Cloud configuration qualification failed"}\n`); + process.exitCode = 1; + }); +} diff --git a/server/src/routes/adapters.ts b/server/src/routes/adapters.ts index 8e0d41e1d64..7a02043644b 100644 --- a/server/src/routes/adapters.ts +++ b/server/src/routes/adapters.ts @@ -19,11 +19,13 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; import { Router } from "express"; +import { companies, type Db } from "@paperclipai/db"; +import { adapterSettingsService } from "../services/adapter-settings.js"; +import { logActivity, publishActivity, type ActivityPublication } from "../services/activity-log.js"; import { listServerAdapters, findServerAdapter, findActiveServerAdapter, - listEnabledServerAdapters, registerServerAdapter, resolveExternalAdapterRegistration, unregisterServerAdapter, @@ -37,8 +39,6 @@ import { removeAdapterPlugin, getAdapterPluginByType, getAdapterPluginsDir, - getDisabledAdapterTypes, - setAdapterDisabled, } from "../services/adapter-plugin-store.js"; import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js"; import type { ServerAdapterModule, AdapterConfigSchema } from "../adapters/types.js"; @@ -49,9 +49,9 @@ import type { import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js"; import { logger } from "../middleware/logger.js"; import { badRequest, forbidden } from "../errors.js"; -import { isCloudManagedInstance } from "../services/cloud-instance.js"; +import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js"; import { getHiddenSettings } from "../services/settings-visibility.js"; -import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin } from "./authz.js"; +import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js"; import { BUILTIN_ADAPTER_TYPES } from "../adapters/builtin-adapter-types.js"; import { agentRunnerAvailability, type RunnerTarget } from "../services/agent-runner-selection.js"; import type { AgentRunnerAvailability } from "@paperclipai/shared"; @@ -60,22 +60,6 @@ import { resolveAntigravityCandidateAdmission } from "../services/native-runtime const execFileAsync = promisify(execFile); let warnedMalformedAntigravityAdmission = false; -/** - * Floor: on cloud-managed instances adapter code is bundled into the platform - * image; fetching and loading external adapter packages at runtime stays off - * for every actor, including instance admins. Adapter code executes in the - * server process, so a runtime install would let an instance admin read the - * platform trust anchors from the process environment (mirrors the - * bundled-only plugin install floor in plugin-install-guard.ts). - */ -function assertAdapterCodeInstallAllowed() { - if (isCloudManagedInstance()) { - throw forbidden("Adapter installation is platform-managed on cloud-managed instances", { - code: "adapter_install_platform_managed", - }); - } -} - /** * Floor: when the hosting operator hides the Adapters settings surface * (`instance.adapters` in PAPERCLIP_HIDDEN_SETTINGS), adapter management @@ -209,7 +193,7 @@ function buildAdapterInfo(adapter: ServerAdapterModule, externalRecord: AdapterP loaded: true, // If it's in the registry, it's loaded disabled: disabledSet.has(adapter.type), capabilities: buildAdapterCapabilities(adapter), - ...agentRunnerAvailability(adapter.type, target), + ...agentRunnerAvailability(adapter.type, target, [...disabledSet]), ...(adapter.acp ? { acp: adapter.acp } : {}), overriddenBuiltin: externalRecord ? BUILTIN_ADAPTER_TYPES.has(adapter.type) : undefined, overridePaused: BUILTIN_ADAPTER_TYPES.has(adapter.type) ? isOverridePaused(adapter.type) : undefined, @@ -265,11 +249,13 @@ function registerWithSessionManagement(adapter: ServerAdapterModule): void { // --------------------------------------------------------------------------- export function adapterRoutes(options: { + db?: Db; getNativeRunnerEnabled?: () => Promise; getOpenAiDotEnabled?: () => Promise; resolveRunnerTarget?: (companyId: string, environmentId?: string) => Promise; } = {}) { const router = Router(); + const settings = adapterSettingsService(options.db); /** * GET /api/adapters @@ -303,7 +289,7 @@ export function adapterRoutes(options: { const externalRecords = new Map( listAdapterPlugins().map((r) => [r.type, r]), ); - const disabledSet = new Set(getDisabledAdapterTypes()); + const disabledSet = new Set(await settings.getDisabledTypes()); // One shared implementation, with independent provider rollouts. Explicit // adapter-admin disabling still applies to both choices. @@ -343,7 +329,7 @@ export function adapterRoutes(options: { */ router.post("/adapters/install", async (req, res) => { assertInstanceAdmin(req); - assertAdapterCodeInstallAllowed(); + assertExternalAdapterCodeAllowed(); assertAdapterManagementVisible(); const { packageName, isLocalPath = false, version } = req.body as AdapterInstallRequest; @@ -477,7 +463,7 @@ export function adapterRoutes(options: { } const externalRecord = getAdapterPluginByType(adapterType); - const disabledSet = new Set(getDisabledAdapterTypes()); + const disabledSet = new Set(await settings.getDisabledTypes()); res.json(buildAdapterInfo(adapter, externalRecord, disabledSet)); }); @@ -509,7 +495,28 @@ export function adapterRoutes(options: { return; } - const changed = setAdapterDisabled(adapterType, disabled); + const publications: ActivityPublication[] = []; + const changed = await settings.setDisabled(adapterType, disabled, async (tx, current) => { + const actor = getActorInfo(req); + const companyIds = await tx.select({ id: companies.id }).from(companies); + for (const company of companyIds) { + await logActivity(tx as Db, { + companyId: company.id, + ...actor, + action: "instance.adapter.updated", + entityType: "instance_settings", + entityId: "adapter-settings:v1", + details: { adapterType, disabled, revision: current.revision }, + }, publications); + } + }); + for (const publication of publications) { + try { + publishActivity(publication); + } catch (error) { + logger.error({ err: error, companyId: publication.companyId }, "failed to publish adapter settings activity"); + } + } if (changed) { logger.info({ type: adapterType, disabled }, "Adapter enabled/disabled"); @@ -528,6 +535,7 @@ export function adapterRoutes(options: { */ router.patch("/adapters/:type/override", async (req, res) => { assertInstanceAdmin(req); + assertExternalAdapterCodeAllowed(); assertAdapterManagementVisible(); @@ -558,6 +566,7 @@ export function adapterRoutes(options: { */ router.delete("/adapters/:type", async (req, res) => { assertInstanceAdmin(req); + assertExternalAdapterCodeAllowed(); assertAdapterManagementVisible(); const adapterType = req.params.type; @@ -635,6 +644,7 @@ export function adapterRoutes(options: { */ router.post("/adapters/:type/reload", async (req, res) => { assertInstanceAdmin(req); + assertExternalAdapterCodeAllowed(); assertAdapterManagementVisible(); const type = req.params.type; @@ -688,7 +698,7 @@ export function adapterRoutes(options: { // package name, but without the risk of losing the store record. router.post("/adapters/:type/reinstall", async (req, res) => { assertInstanceAdmin(req); - assertAdapterCodeInstallAllowed(); + assertExternalAdapterCodeAllowed(); assertAdapterManagementVisible(); const type = req.params.type; diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 8b6cd0128f9..7f4b0990f48 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -154,7 +154,7 @@ import type { } from "@paperclipai/adapter-utils"; import { evaluateCodexCredentialReadiness } from "@paperclipai/adapter-codex-local/server"; import type { AdapterAuthSignal, AdapterAuthSignalResponse, CodexAccountBindingClaim } from "@paperclipai/shared"; -import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js"; +import { adapterSettingsService } from "../services/adapter-settings.js"; import { skillVersionSelectionMap } from "../services/runtime-skill-selections.js"; import { isFixedClaudeOAuthBinding, secretService } from "../services/secrets.js"; import { authorizationDeniedDetails } from "../services/authorization.js"; @@ -1859,15 +1859,6 @@ export function agentRoutes( * Adapter validation for the paths that CHOOSE a harness for a new agent * (hire + create), as opposed to the paths that operate on an existing one. * - * A disabled adapter is one this instance cannot run — most often because a - * declarative registry (PAPERCLIP_ADAPTERS) curated it out, which - * reconcileAdapterAvailability turns into a disabled type at boot. Registered - * but disabled still passes assertKnownAdapterType, so an agent could be - * created on it and then fail EVERY run at lease time with - * `Adapter "..." is not in the configured adapter registry` — an error that - * arrives minutes later, in a run log, with no way back to the choice that - * caused it. Refuse at selection time instead, and name what can be chosen. - * * Existing agents on a now-disabled adapter are deliberately untouched * (listEnabledServerAdapters documents the same rule: hidden from selection, * still functional for agents that already use them). @@ -1902,7 +1893,7 @@ export function agentRoutes( ); } } - const disabled = new Set(getDisabledAdapterTypes()); + const disabled = new Set(await adapterSettingsService(db).getDisabledTypes()); if (!disabled.has(adapterType)) return adapterType; const available = listServerAdapters() .map((a) => a.type) diff --git a/server/src/routes/plugin-ui-static.ts b/server/src/routes/plugin-ui-static.ts index ca9cae7db08..7a9efb4b249 100644 --- a/server/src/routes/plugin-ui-static.ts +++ b/server/src/routes/plugin-ui-static.ts @@ -32,6 +32,7 @@ import path from "node:path"; import fs from "node:fs"; import crypto from "node:crypto"; import type { Db } from "@paperclipai/db"; +import type { PluginRecord } from "@paperclipai/shared"; import { pluginRegistryService } from "../services/plugin-registry.js"; import { logger } from "../middleware/logger.js"; import { assertCompanyAccess } from "./authz.js"; @@ -192,6 +193,7 @@ export interface PluginUiStaticRouteOptions { * Defaults to the standard `~/.paperclip/plugins/` location. */ localPluginDir: string; + resolvePackageRoot?: (plugin: PluginRecord) => string; } /** @@ -304,7 +306,7 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions if (typeof devUiUrl === "string" && devUiUrl.length > 0) { // Dev proxy is only available in development mode - if (process.env.NODE_ENV === "production") { + if (options.resolvePackageRoot || process.env.NODE_ENV === "production") { log.warn( { pluginId: plugin.id }, "plugin-ui-static: devUiUrl ignored in production", @@ -405,12 +407,15 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions } // Step 3: Resolve the plugin's UI directory - const uiDir = resolvePluginUiDir( - options.localPluginDir, - plugin.packageName, - manifest.entrypoints.ui, - plugin.packagePath, - ); + let uiDir: string | null; + try { + uiDir = options.resolvePackageRoot + ? path.resolve(options.resolvePackageRoot(plugin), manifest.entrypoints.ui) + : resolvePluginUiDir(options.localPluginDir, plugin.packageName, manifest.entrypoints.ui, plugin.packagePath); + } catch { + res.status(503).json({ error: "Plugin package is unavailable in the current image" }); + return; + } if (!uiDir) { log.warn( diff --git a/server/src/services/adapter-code-policy.ts b/server/src/services/adapter-code-policy.ts new file mode 100644 index 00000000000..49c1e822f80 --- /dev/null +++ b/server/src/services/adapter-code-policy.ts @@ -0,0 +1,10 @@ +import { forbidden } from "../errors.js"; +import { isCloudManagedInstance } from "./cloud-instance.js"; + +export function assertExternalAdapterCodeAllowed(): void { + if (isCloudManagedInstance()) { + throw forbidden("Adapter installation is platform-managed on cloud-managed instances", { + code: "adapter_install_platform_managed", + }); + } +} diff --git a/server/src/services/adapter-plugin-store.ts b/server/src/services/adapter-plugin-store.ts index 13369323d4f..8ccdf3f6049 100644 --- a/server/src/services/adapter-plugin-store.ts +++ b/server/src/services/adapter-plugin-store.ts @@ -15,6 +15,8 @@ import fs from "node:fs"; import path from "node:path"; import { resolvePaperclipHomeDir } from "../home-paths.js"; +import { isCloudManagedInstance } from "./cloud-instance.js"; +import { assertExternalAdapterCodeAllowed } from "./adapter-code-policy.js"; // --------------------------------------------------------------------------- // Types @@ -43,8 +45,8 @@ interface AdapterSettings { // Paths // --------------------------------------------------------------------------- -function adapterPluginPaths() { - const paperclipDir = resolvePaperclipHomeDir(); +function adapterPluginPaths(homeOverride?: string) { + const paperclipDir = resolvePaperclipHomeDir(homeOverride); return { adapterPluginsDir: path.join(paperclipDir, "adapter-plugins"), adapterPluginsStorePath: path.join(paperclipDir, "adapter-plugins.json"), @@ -79,6 +81,7 @@ function ensureDirs(): string { } function readStore(): AdapterPluginRecord[] { + if (isCloudManagedInstance()) return []; const { adapterPluginsStorePath } = adapterPluginPaths(); if (storeCache?.path === adapterPluginsStorePath) return storeCache.records; try { @@ -130,11 +133,16 @@ function writeSettings(settings: AdapterSettings): void { // Public API // --------------------------------------------------------------------------- +export function getAdapterPluginStorePath(homeOverride?: string): string { + return adapterPluginPaths(homeOverride).adapterPluginsStorePath; +} + export function listAdapterPlugins(): AdapterPluginRecord[] { return readStore(); } export function addAdapterPlugin(record: AdapterPluginRecord): void { + assertExternalAdapterCodeAllowed(); const store = [...readStore()]; const idx = store.findIndex((r) => r.type === record.type); if (idx >= 0) { @@ -146,6 +154,7 @@ export function addAdapterPlugin(record: AdapterPluginRecord): void { } export function removeAdapterPlugin(type: string): boolean { + assertExternalAdapterCodeAllowed(); const store = [...readStore()]; const idx = store.findIndex((r) => r.type === type); if (idx < 0) return false; @@ -159,6 +168,7 @@ export function getAdapterPluginByType(type: string): AdapterPluginRecord | unde } export function getAdapterPluginsDir(): string { + assertExternalAdapterCodeAllowed(); return ensureDirs(); } diff --git a/server/src/services/adapter-registry-bootstrap.reconcile.test.ts b/server/src/services/adapter-registry-bootstrap.reconcile.test.ts index 1e32e3575fe..01409c571ec 100644 --- a/server/src/services/adapter-registry-bootstrap.reconcile.test.ts +++ b/server/src/services/adapter-registry-bootstrap.reconcile.test.ts @@ -32,8 +32,7 @@ describe("reconcileAdapterAvailability", () => { ]); expect(result.enabled).toEqual(["opencode_local"]); expect(result.disabled.sort()).toEqual(["claude_local", "pi_local"]); - expect(setAdapterDisabled).toHaveBeenCalledWith("claude_local", true); - expect(setAdapterDisabled).toHaveBeenCalledWith("opencode_local", false); + expect(setAdapterDisabled).not.toHaveBeenCalled(); }); it("throws when a declared adapter has no installed implementation", () => { diff --git a/server/src/services/adapter-registry-bootstrap.ts b/server/src/services/adapter-registry-bootstrap.ts index c8e7f4b0f0b..e4e609eed09 100644 --- a/server/src/services/adapter-registry-bootstrap.ts +++ b/server/src/services/adapter-registry-bootstrap.ts @@ -1,73 +1,8 @@ -/** - * Declarative adapter-registry bootstrap. - * - * One source (`PAPERCLIP_ADAPTERS` inline JSON, or `PAPERCLIP_ADAPTERS_FILE` a - * path to a JSON file) feeds two consumers: - * 1. Availability: reconcile the file-backed disabled-set so the picker shows - * exactly the declared, enabled set (runs for any instance). - * 2. k8s runtime: the same registry rides on the Kubernetes environment config - * (see execution-policy-bootstrap) so the plugin resolves runtime - * image/envKeys/allowFqdns/probe/defaultEnv from it. - * - * Parsing is pure + fails loud on malformed/invalid config, mirroring - * execution-policy-bootstrap. - */ -import fs from "node:fs"; -import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared"; +import type { AdapterRegistryEntryParsed } from "@paperclipai/shared"; import { logger } from "../middleware/logger.js"; import { listServerAdapters } from "../adapters/registry.js"; -import { setAdapterDisabled } from "./adapter-plugin-store.js"; +export { parseAdapterRegistryEnv, type AdapterRegistryEnv } from "./adapter-registry-config.js"; -export type AdapterRegistryEnv = Record; - -/** - * Parse the declarative registry from env. Returns null when unconfigured - * (built-in defaults; local/OSS unchanged). Throws on malformed/invalid input. - */ -export function parseAdapterRegistryEnv( - env: AdapterRegistryEnv = process.env, -): AdapterRegistryEntryParsed[] | null { - const inline = env.PAPERCLIP_ADAPTERS?.trim(); - const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim(); - if (!inline && !filePath) return null; - - let rawText: string; - if (inline) { - rawText = inline; - } else { - try { - rawText = fs.readFileSync(filePath as string, "utf-8"); - } catch (err) { - throw new Error( - `PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`, - ); - } - } - - let parsed: unknown; - try { - parsed = JSON.parse(rawText); - } catch (err) { - throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`); - } - - const result = adapterRegistrySchema.safeParse(parsed); - if (!result.success) { - throw new Error( - `PAPERCLIP_ADAPTERS failed validation: ${result.error.issues - .map((i) => `${i.path.join(".")}: ${i.message}`) - .join("; ")}`, - ); - } - return result.data; -} - -/** - * Reconcile availability: every known server adapter NOT enabled in the declared - * registry is disabled; declared+enabled ones are enabled. Throws if a declared - * adapterType has no installed adapter (cannot offer a harness with no - * implementation). No-op when `registry` is null. - */ export function reconcileAdapterAvailability( registry: AdapterRegistryEntryParsed[] | null, ): { enabled: string[]; disabled: string[] } { @@ -88,10 +23,9 @@ export function reconcileAdapterAvailability( for (const type of knownTypes) { const entry = declared.get(type); const shouldEnable = entry !== undefined && entry.enabled !== false; - setAdapterDisabled(type, !shouldEnable); (shouldEnable ? enabled : disabled).push(type); } - logger.info({ enabled, disabled }, "reconciled adapter availability from PAPERCLIP_ADAPTERS"); + logger.info({ enabled, disabled }, "validated adapter availability from PAPERCLIP_ADAPTERS"); return { enabled, disabled }; } diff --git a/server/src/services/adapter-registry-config.ts b/server/src/services/adapter-registry-config.ts new file mode 100644 index 00000000000..732f5428051 --- /dev/null +++ b/server/src/services/adapter-registry-config.ts @@ -0,0 +1,47 @@ +import fs from "node:fs"; +import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared"; + +export type AdapterRegistryEnv = Record; + +let cached: { inline: string | undefined; filePath: string | undefined; registry: AdapterRegistryEntryParsed[] } | undefined; + +export function parseAdapterRegistryEnv( + env: AdapterRegistryEnv = process.env, +): AdapterRegistryEntryParsed[] | null { + const inline = env.PAPERCLIP_ADAPTERS?.trim(); + const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim(); + if (!inline && !filePath) return null; + if (cached && cached.inline === inline && cached.filePath === filePath) return structuredClone(cached.registry); + + let rawText: string; + if (inline) { + rawText = inline; + } else { + try { + rawText = fs.readFileSync(filePath as string, "utf-8"); + } catch (err) { + throw new Error( + `PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`, + ); + } + } + + let parsed: unknown; + try { + parsed = JSON.parse(rawText); + } catch (err) { + throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`); + } + + const result = adapterRegistrySchema.safeParse(parsed); + if (!result.success) { + throw new Error( + `PAPERCLIP_ADAPTERS failed validation: ${result.error.issues + .map((i) => `${i.path.join(".")}: ${i.message}`) + .join("; ")}`, + ); + } + cached = { inline, filePath, registry: result.data }; + return structuredClone(result.data); +} + diff --git a/server/src/services/adapter-settings-local.test.ts b/server/src/services/adapter-settings-local.test.ts new file mode 100644 index 00000000000..03e56fcb941 --- /dev/null +++ b/server/src/services/adapter-settings-local.test.ts @@ -0,0 +1,87 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../adapters/registry.js", () => ({ listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }] })); +import { adapterSettingsService, type AdapterSettingsDb } from "./adapter-settings.js"; +import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js"; +import { parseAdapterRegistryEnv } from "./adapter-registry-config.js"; + +describe("local adapter settings compatibility", () => { + let directory: string; + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "adapter-settings-local-")); + vi.stubEnv("PAPERCLIP_HOME", directory); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); + vi.stubEnv("PAPERCLIP_ADAPTERS", undefined); + vi.stubEnv("PAPERCLIP_ADAPTERS_FILE", undefined); + }); + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("reads and writes the existing file without a database", async () => { + fs.writeFileSync(path.join(directory, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local"] })); + const service = adapterSettingsService(undefined); + expect(await service.getDisabledTypes()).toEqual(["claude_local"]); + expect(await service.setDisabled("codex_local", true)).toBe(true); + expect(getDisabledAdapterTypes()).toEqual(["claude_local", "codex_local"]); + expect(setAdapterDisabled("claude_local", false)).toBe(true); + expect(await service.getDisabledTypes()).toEqual(["codex_local"]); + expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: ["codex_local"] }); + }); + + it("loads an operator file once and leaves the saved file intact", async () => { + setAdapterDisabled("codex_local", true); + const file = path.join(directory, "operator-adapters.json"); + fs.writeFileSync(file, JSON.stringify([{ adapterType: "codex_local", enabled: true }])); + const env = { PAPERCLIP_ADAPTERS_FILE: file }; + parseAdapterRegistryEnv(env); + const read = vi.spyOn(fs, "readFileSync"); + const service = adapterSettingsService(undefined, { runtimeEnv: env }); + expect(await service.getDisabledTypes()).toEqual(["claude_local"]); + expect(await service.getDisabledTypes()).toEqual(["claude_local"]); + expect(read).not.toHaveBeenCalled(); + expect(getDisabledAdapterTypes()).toEqual(["codex_local"]); + }); + + it("restores the saved choice when its audit fails and audits a later retry", async () => { + const transaction = vi.fn(async (callback: (tx: AdapterSettingsDb) => Promise) => callback(db)); + const db = { transaction } as unknown as AdapterSettingsDb; + const service = adapterSettingsService(db); + const audit = vi.fn().mockRejectedValueOnce(new Error("audit unavailable")).mockResolvedValue(undefined); + + await expect(service.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable"); + expect(await service.getDisabledTypes()).toEqual([]); + expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: [] }); + expect(await service.setDisabled("codex_local", true, audit)).toBe(true); + expect(await service.getDisabledTypes()).toEqual(["codex_local"]); + expect(audit).toHaveBeenCalledTimes(2); + expect(transaction).toHaveBeenCalledTimes(2); + }); + + it("keeps a failed audit rollback from overwriting a concurrent save", async () => { + const db = { transaction: async (callback: (tx: AdapterSettingsDb) => Promise) => callback(db) } as unknown as AdapterSettingsDb; + const first = adapterSettingsService(db); + const second = adapterSettingsService(db); + let failAudit!: (reason: Error) => void; + const auditWait = new Promise((_resolve, reject) => { failAudit = reject; }); + const audit = vi.fn(() => auditWait); + const rejected = expect(first.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable"); + await vi.waitFor(() => expect(audit).toHaveBeenCalledOnce()); + const nextAudit = vi.fn(async () => undefined); + const next = second.setDisabled("codex_local", true, nextAudit); + await Promise.resolve(); + expect(nextAudit).not.toHaveBeenCalled(); + + failAudit(new Error("audit unavailable")); + await rejected; + expect(await next).toBe(true); + expect(await first.getDisabledTypes()).toEqual(["codex_local"]); + expect(nextAudit).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/services/adapter-settings-postgres.test.ts b/server/src/services/adapter-settings-postgres.test.ts new file mode 100644 index 00000000000..299c4b14f35 --- /dev/null +++ b/server/src/services/adapter-settings-postgres.test.ts @@ -0,0 +1,228 @@ +import express from "express"; +import request from "supertest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, applyPendingMigrations, companies, createDb, instanceSettings, type Db } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; + +const legacy = vi.hoisted(() => ({ get: vi.fn(() => [] as string[]), set: vi.fn() })); +vi.mock("./adapter-plugin-store.js", () => ({ + getDisabledAdapterTypes: legacy.get, + setAdapterDisabled: legacy.set, + listAdapterPlugins: () => [], + getAdapterPluginByType: () => undefined, +})); +vi.mock("../adapters/registry.js", () => ({ + listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }], + findServerAdapter: (type: string) => ({ type }), + findActiveServerAdapter: (type: string) => ({ type }), + isOverridePaused: () => false, + hasActiveAdapterOverride: () => false, + waitForExternalAdapters: async () => {}, +})); +vi.mock("../adapters/plugin-loader.js", () => ({})); +import { adapterSettingsService, initializeAdapterSettings } from "./adapter-settings.js"; +import { adapterRoutes } from "../routes/adapters.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; +import * as activities from "./activity-log.js"; + +const externalDatabaseUrl = process.env.PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL; +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("durable managed adapter settings", () => { + let database: Awaited>; + let db: Db; + let peer: Db; + let directory: string; + const managed = { runtimeEnv: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "synthetic-test-token" } }; + const service = () => adapterSettingsService(db, managed); + + beforeAll(async () => { + database = externalDatabaseUrl + ? { connectionString: externalDatabaseUrl, cleanup: async () => {} } + : await startEmbeddedPostgresTestDatabase("adapter-settings-"); + if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl); + db = createDb(database.connectionString); + peer = createDb(database.connectionString); + await db.insert(companies).values({ name: "Adapter settings test", issuePrefix: "AST" }); + }, 90_000); + afterAll(async () => { + await Promise.all([db?.$client.end({ timeout: 0 }), peer?.$client.end({ timeout: 0 })]); + await database?.cleanup(); + }, 60_000); + beforeEach(async () => { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "adapter-settings-import-")); + await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, "adapter-settings:v1")); + legacy.get.mockClear(); + legacy.set.mockClear(); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-test-token"); + }); + afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await fs.rm(directory, { recursive: true, force: true }); + }); + + function app() { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { type: "board", source: "local_implicit", userId: "local-board", isInstanceAdmin: true }; + next(); + }); + app.use(adapterRoutes({ db })); + app.use(errorHandler); + return app; + } + + it("persists across independent connections and reconstructed stores without file reads", async () => { + await expect(service().getDisabledTypes()).rejects.toThrow("not been initialized"); + await service().seedIfAbsent({ disabledTypes: [] }); + expect(await service().setDisabled("codex_local", true)).toBe(true); + expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]); + await db.$client.end({ timeout: 0 }); + db = createDb(database.connectionString); + expect(await service().inspectStored()).toEqual({ version: 1, revision: 1, disabledTypes: ["codex_local"] }); + expect(legacy.get).not.toHaveBeenCalled(); + expect(legacy.set).not.toHaveBeenCalled(); + }); + + it("acknowledges the committed save when its live activity event fails", async () => { + await service().seedIfAbsent({ disabledTypes: [] }); + const publication = vi.spyOn(activities, "publishActivity").mockImplementation(() => { throw new Error("event unavailable"); }); + const response = await request(app()).patch("/adapters/codex_local").send({ disabled: true }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(publication).toHaveBeenCalled(); + expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]); + const events = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated")); + expect(events.some((event) => event.details?.adapterType === "codex_local")).toBe(true); + }); + + it("preserves simultaneous first writes and concurrent removals from other connections", async () => { + const writers = [service(), adapterSettingsService(peer, managed)]; + await Promise.all(Array.from({ length: 12 }, (_, index) => writers[index % 2].setDisabled(`adapter_${index}`, true))); + expect((await service().inspectStored())?.disabledTypes).toHaveLength(12); + await Promise.all([ + writers[0].setDisabled("adapter_0", false), + writers[1].setDisabled("adapter_1", false), + writers[0].setDisabled("extra", true), + ]); + const stored = await service().inspectStored(); + expect(stored?.revision).toBe(15); + expect(stored?.disabledTypes).not.toContain("adapter_0"); + expect(stored?.disabledTypes).not.toContain("adapter_1"); + expect(stored?.disabledTypes).toContain("extra"); + expect(await writers[1].setDisabled("extra", true)).toBe(false); + expect((await service().inspectStored())?.revision).toBe(15); + }); + + it("imports legacy choices once before admission and ignores stale files after the DB record exists", async () => { + const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory }; + const file = path.join(directory, "adapter-settings.json"); + await fs.writeFile(file, JSON.stringify({ disabledTypes: ["codex_local", "future_adapter"] })); + const results = await Promise.all([ + initializeAdapterSettings(db, { runtimeEnv }), + initializeAdapterSettings(peer, { runtimeEnv }), + ]); + expect(results.filter((result) => result?.seeded)).toHaveLength(1); + expect(await service().getDisabledTypes()).toEqual(["codex_local", "future_adapter"]); + await service().setDisabled("codex_local", false); + await fs.writeFile(file, "invalid stale JSON"); + expect((await initializeAdapterSettings(peer, { runtimeEnv }))?.seeded).toBe(false); + expect(await service().getDisabledTypes()).toEqual(["future_adapter"]); + }); + + it("uses defaults only for an absent legacy file and rejects malformed or oversized input", async () => { + const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory }; + const file = path.join(directory, "adapter-settings.json"); + await fs.writeFile(file, JSON.stringify({ disabledTypes: [42] })); + await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("Persisted adapter settings are invalid"); + expect(await service().inspectStored()).toBeNull(); + await fs.writeFile(file, Buffer.alloc(65_537)); + await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("size limit"); + expect(await service().inspectStored()).toBeNull(); + await fs.unlink(file); + expect((await initializeAdapterSettings(db, { runtimeEnv }))?.current.disabledTypes).toEqual([]); + expect(await service().getDisabledTypes()).toEqual([]); + }); + + it("composes atomic import and receipts with the caller transaction", async () => { + await expect(db.transaction(async (tx) => { + expect(await service().seedIfAbsent({ disabledTypes: ["codex_local"] }, tx)).toEqual({ + seeded: true, current: { version: 1, revision: 0, disabledTypes: ["codex_local"] }, + }); + throw new Error("receipt failed"); + })).rejects.toThrow("receipt failed"); + expect(await service().inspectStored()).toBeNull(); + await Promise.all([ + service().seedIfAbsent({ disabledTypes: ["codex_local"] }), + adapterSettingsService(peer, managed).seedIfAbsent({ disabledTypes: ["claude_local"] }), + ]).then((results) => { + expect(results.filter((result) => result.seeded)).toHaveLength(1); + expect(results[0].current).toEqual(results[1].current); + }); + }); + + it("applies operator restrictions without rewriting saved choices", async () => { + await service().setDisabled("codex_local", true); + const env = { ...managed.runtimeEnv, PAPERCLIP_ADAPTERS: JSON.stringify([{ adapterType: "codex_local", enabled: true }]) }; + const overlay = adapterSettingsService(peer, { runtimeEnv: env }); + expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]); + expect(await overlay.inspectStored()).toEqual(await service().inspectStored()); + await expect(overlay.setDisabled("claude_local", false)).rejects.toThrow("hosting operator"); + await expect(overlay.setDisabled("codex_local", true)).rejects.toThrow("hosting operator"); + expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]); + expect(await service().getDisabledTypes()).toEqual(["codex_local"]); + }); + + it("refreshes the disabled harness at each new-agent admission", async () => { + await service().seedIfAbsent({ disabledTypes: [] }); + const input = { adapterType: "codex_local", runner: "legacy" as const }; + expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local"); + await adapterSettingsService(peer, managed).setDisabled("codex_local", true); + await expect(resolveNewAgentRunnerForCompany(db, "company", input)).rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } }); + await adapterSettingsService(peer, managed).setDisabled("codex_local", false); + expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local"); + }); + + it("rolls back the setting when its audit write fails", async () => { + await expect(service().setDisabled("codex_local", true, async () => { throw new Error("audit failed"); })).rejects.toThrow("audit failed"); + expect(await service().inspectStored()).toBeNull(); + await expect(adapterSettingsService(peer, managed).getDisabledTypes()).rejects.toThrow("not been initialized"); + }); + + it("publishes catalog changes across stores and records the instance mutation", async () => { + const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ type: "codex_local", disabled: true, changed: true }); + const catalog = await request(app()).get("/adapters"); + expect(catalog.status).toBe(200); + expect(catalog.body.find((entry: { type: string }) => entry.type === "codex_local").disabled).toBe(true); + const [audit] = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated")); + expect(audit.details).toMatchObject({ adapterType: "codex_local", disabled: true, revision: 1 }); + }); + + it("returns failure when PostgreSQL rejects the update and keeps the durable choice", async () => { + await service().seedIfAbsent({ disabledTypes: [] }); + await db.execute(sql`CREATE FUNCTION reject_adapter_settings() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test adapter settings failure'; END $$`); + await db.execute(sql`CREATE TRIGGER reject_adapter_settings BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_adapter_settings()`); + try { + const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true }); + expect(res.status).toBe(500); + expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual([]); + } finally { + await db.execute(sql`DROP TRIGGER reject_adapter_settings ON instance_settings`); + await db.execute(sql`DROP FUNCTION reject_adapter_settings()`); + } + }); + + it("fails closed on malformed stored settings and unavailable database", async () => { + await db.insert(instanceSettings).values({ singletonKey: "adapter-settings:v1", general: { version: 2, disabledTypes: [] } }); + await expect(service().getDisabledTypes()).rejects.toThrow("Persisted adapter settings are invalid"); + await expect(adapterSettingsService(undefined, managed).getDisabledTypes()).rejects.toThrow("database is required"); + expect(legacy.get).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/adapter-settings.ts b/server/src/services/adapter-settings.ts new file mode 100644 index 00000000000..4b9459b7165 --- /dev/null +++ b/server/src/services/adapter-settings.ts @@ -0,0 +1,149 @@ +import type { Db } from "@paperclipai/db"; +import { instanceSettings } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { open } from "node:fs/promises"; +import { constants } from "node:fs"; +import path from "node:path"; +import { z } from "zod"; +import { listServerAdapters } from "../adapters/registry.js"; +import { forbidden } from "../errors.js"; +import { resolvePaperclipHomeDir } from "../home-paths.js"; +import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js"; +import { parseAdapterRegistryEnv } from "./adapter-registry-config.js"; +import { isCloudManagedInstance } from "./cloud-instance.js"; + +const SINGLETON_KEY = "adapter-settings:v1"; +let localSettingsMutation: Promise = Promise.resolve(); +type Transaction = Parameters[0]>[0]; +export type AdapterSettingsDb = Pick; +export type StoredAdapterSettings = { version: 1; revision: number; disabledTypes: string[] }; +const storedSettingsSchema = z.object({ + version: z.literal(1), + revision: z.number().int().nonnegative(), + disabledTypes: z.array(z.string().refine((type) => type.trim().length > 0)), +}); + +function parseStored(value: Record): StoredAdapterSettings { + const parsed = storedSettingsSchema.safeParse(value); + if (!parsed.success) throw new Error("Persisted adapter settings are invalid"); + return { ...parsed.data, disabledTypes: [...new Set(parsed.data.disabledTypes)].sort() }; +} + +export function effectiveDisabledAdapterTypes(saved: string[], knownTypes: string[] | (() => string[]), env: Record = process.env): string[] { + const registry = parseAdapterRegistryEnv(env); + if (!registry) return [...saved]; + const enabled = new Set(registry.filter((entry) => entry.enabled !== false).map((entry) => entry.adapterType)); + const types = typeof knownTypes === "function" ? knownTypes() : knownTypes; + return [...new Set(types.filter((type) => !enabled.has(type)))].sort(); +} + +export function adapterSettingsService(db: AdapterSettingsDb | undefined, options: { runtimeEnv?: Record } = {}) { + const env = options.runtimeEnv ?? process.env; + const managed = isCloudManagedInstance(env); + + function requireDb(): AdapterSettingsDb { + if (!db) throw new Error("A database is required for managed adapter settings"); + return db; + } + + async function inspectStored(handle: AdapterSettingsDb = requireDb()): Promise { + const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings) + .where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).limit(1); + return row ? parseStored(row.general) : null; + } + + async function seedIfAbsent(value: { disabledTypes: string[] }, handle: AdapterSettingsDb = requireDb()) { + const snapshot = parseStored({ version: 1, revision: 0, disabledTypes: value.disabledTypes }); + const inserted = await handle.insert(instanceSettings).values({ singletonKey: SINGLETON_KEY, general: snapshot }) + .onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id }); + const current = await inspectStored(handle); + if (!current) throw new Error("Adapter settings seed was not persisted"); + return { seeded: inserted.length > 0, current }; + } + + return { + inspectStored, + seedIfAbsent, + async getDisabledTypes(): Promise { + const current = managed ? await inspectStored() : null; + if (managed && !current) throw new Error("Managed adapter settings have not been initialized"); + const stored = managed ? current!.disabledTypes : getDisabledAdapterTypes(); + return effectiveDisabledAdapterTypes(stored, () => listServerAdapters().map((adapter) => adapter.type), env); + }, + async setDisabled(type: string, disabled: boolean, afterChange?: (handle: AdapterSettingsDb, current: StoredAdapterSettings) => Promise): Promise { + if (!type.trim()) throw new Error("Adapter type is required"); + const registry = parseAdapterRegistryEnv(env); + if (registry && disabled === registry.some((entry) => entry.adapterType === type && entry.enabled !== false)) { + throw forbidden("Adapter availability is managed by the hosting operator", { code: "settings_operator_managed" }); + } + if (!managed) { + const mutation = localSettingsMutation.then(async () => { + const previous = getDisabledAdapterTypes().includes(type); + const changed = setAdapterDisabled(type, disabled); + if (!changed || !db || !afterChange) return changed; + try { + await db.transaction(async (tx) => { + await afterChange(tx, { version: 1, revision: 0, disabledTypes: getDisabledAdapterTypes() }); + }); + } catch (error) { + setAdapterDisabled(type, previous); + throw error; + } + return true; + }); + localSettingsMutation = mutation.catch(() => undefined); + return mutation; + } + return requireDb().transaction(async (tx) => { + await seedIfAbsent({ disabledTypes: [] }, tx); + const [row] = await tx.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).for("update"); + if (!row) throw new Error("Adapter settings are unavailable"); + const current = parseStored(row.general); + const types = new Set(current.disabledTypes); + if (types.has(type) === disabled) return false; + if (disabled) types.add(type); else types.delete(type); + const next: StoredAdapterSettings = { version: 1, revision: current.revision + 1, disabledTypes: [...types].sort() }; + await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id)); + await afterChange?.(tx, next); + return true; + }); + }, + }; +} + +export type AdapterSettingsService = Pick, "getDisabledTypes" | "setDisabled">; + +export async function initializeAdapterSettings(db: AdapterSettingsDb, options: { runtimeEnv?: Record } = {}) { + const env = options.runtimeEnv ?? process.env; + if (!isCloudManagedInstance(env)) return null; + const service = adapterSettingsService(db, options); + const current = await service.inspectStored(); + if (current) return { seeded: false, current }; + return service.seedIfAbsent(await readLegacyAdapterSettings(env) ?? { disabledTypes: [] }); +} + +export async function readLegacyAdapterSettings(env: Record = process.env): Promise<{ disabledTypes: string[] } | null> { + const filePath = path.join(resolvePaperclipHomeDir(env.PAPERCLIP_HOME), "adapter-settings.json"); + try { + const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size > 65_536) throw new Error("Legacy adapter settings exceed the size limit or are not a regular file"); + const buffer = Buffer.alloc(65_537); + let total = 0; + while (total < buffer.length) { + const { bytesRead } = await file.read(buffer, total, buffer.length - total, total); + if (bytesRead === 0) break; + total += bytesRead; + } + if (total > 65_536) throw new Error("Legacy adapter settings exceed the size limit"); + const value = JSON.parse(buffer.subarray(0, total).toString("utf8")); + return { disabledTypes: parseStored({ version: 1, revision: 0, disabledTypes: value?.disabledTypes }).disabledTypes }; + } finally { + await file.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return null; + } +} diff --git a/server/src/services/agent-runner-selection.test.ts b/server/src/services/agent-runner-selection.test.ts index 8b6fe94df2a..22114120f57 100644 --- a/server/src/services/agent-runner-selection.test.ts +++ b/server/src/services/agent-runner-selection.test.ts @@ -3,6 +3,10 @@ import { agentHarnessType, agentRunner, paperclipRunnerProfileForHarness, paperc const state = vi.hoisted(() => ({ disabled: [] as string[], overridden: new Set(), settings: {} as Record, getEnvironment: vi.fn(), bindings: vi.fn(), activeTemplate: vi.fn().mockResolvedValue(null), secretPaths: vi.fn().mockResolvedValue(new Set(["apiKey"])), managedEnvironment: vi.fn(), kubernetesEnvironment: vi.fn(), ensureKubernetesEnvironment: vi.fn(), ssh: vi.fn(), resolveEnvironment: vi.fn(), sandboxQualification: vi.fn(), pluginByKey: vi.fn() })); vi.mock("../adapters/registry.js", () => ({ findActiveServerAdapter: (type: string) => ({ type }), hasActiveAdapterOverride: (type: string) => state.overridden.has(type), waitForExternalAdapters: async () => {} })); vi.mock("./adapter-plugin-store.js", () => ({ getDisabledAdapterTypes: () => state.disabled })); +vi.mock("./adapter-settings.js", async (original) => ({ + ...await original(), + adapterSettingsService: () => ({ getDisabledTypes: async () => state.disabled }), +})); vi.mock("./instance-settings.js", () => ({ instanceSettingsService: () => ({ get: async () => state.settings }) })); vi.mock("./environments.js", () => ({ environmentService: () => ({ getById: state.getEnvironment, listBoundCompanyIds: state.bindings, findManagedSandboxEnvironment: state.managedEnvironment, findKubernetesEnvironment: state.kubernetesEnvironment, ensureKubernetesEnvironment: state.ensureKubernetesEnvironment }) })); vi.mock("./environment-custom-images.js", () => ({ environmentCustomImageService: () => ({ getActiveTemplate: state.activeTemplate }) })); @@ -351,7 +355,7 @@ describe("server-owned Codex runner selection", () => { state.settings = { general: { executionMode: "kubernetes" } }; const selected = await resolveAgentRunnerTargetForCompany({} as never, "company"); expect(selected).toEqual({ driver: "sandbox" }); - expect(agentRunnerAvailability("codex_local", selected)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" }); + expect(agentRunnerAvailability("codex_local", selected, state.disabled)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" }); expect(state.kubernetesEnvironment).toHaveBeenCalledWith("company"); expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled(); expect(state.resolveEnvironment).not.toHaveBeenCalled(); @@ -477,14 +481,14 @@ describe("server-owned Codex runner selection", () => { config: { provider: "daytona", snapshot: "replacement-snapshot" }, metadata: { managedByPaperclip: true } }); state.resolveEnvironment.mockRejectedValue(new Error("Saved provider secret is unavailable.")); const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox"); - expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("paperclip"); + expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("paperclip"); expect(await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType, defaultEnvironmentId: "sandbox", adapterConfig: { model: adapterType === "opencode_local" ? "provider/model" : undefined }, })).toMatchObject({ adapterType: "paperclip_runner" }); expect(state.sandboxQualification).not.toHaveBeenCalled(); expect(state.resolveEnvironment).not.toHaveBeenCalled(); state.disabled.push("paperclip_runner"); - expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("legacy"); + expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("legacy"); }); it.each(["self-hosted", "custom-environment", "captured-template", "stale-marker"])("keeps qualification for a %s sandbox", async kind => { vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", kind === "self-hosted" ? undefined : JSON.stringify({ ...managedCloudConfig, @@ -495,7 +499,7 @@ describe("server-owned Codex runner selection", () => { metadata: { managedByPaperclip: kind !== "custom-environment" } }); state.activeTemplate.mockResolvedValue(kind === "captured-template" ? { id: "template", provider: "daytona", templateRef: "custom-snapshot" } : null); const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox"); - expect(agentRunnerAvailability("codex_local", selected).defaultRunner).toBe("legacy"); + expect(agentRunnerAvailability("codex_local", selected, state.disabled).defaultRunner).toBe("legacy"); expect(state.sandboxQualification).toHaveBeenCalled(); }); it("ignores a stale captured image and preserves a matching custom image without resolving credentials", async () => { @@ -507,17 +511,17 @@ describe("server-owned Codex runner selection", () => { sourceEnvironmentConfigFingerprint: fingerprint({ ...config, snapshot: "previous-snapshot" }, { excludePaths: ["apiKey"] }) }; state.activeTemplate.mockResolvedValue(template); const current = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox"); - expect(agentRunnerAvailability("codex_local", current).defaultRunner).toBe("paperclip"); + expect(agentRunnerAvailability("codex_local", current, state.disabled).defaultRunner).toBe("paperclip"); expect(await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "codex_local", runner: "paperclip", defaultEnvironmentId: "sandbox" })) .toMatchObject({ adapterType: "paperclip_runner" }); expect(state.sandboxQualification).not.toHaveBeenCalled(); state.activeTemplate.mockResolvedValue({ ...template, sourceEnvironmentConfigFingerprint: fingerprint(config, { excludePaths: ["apiKey"] }) }); const custom = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox"); - expect(agentRunnerAvailability("codex_local", custom).defaultRunner).toBe("legacy"); + expect(agentRunnerAvailability("codex_local", custom, state.disabled).defaultRunner).toBe("legacy"); expect(state.sandboxQualification).toHaveBeenCalled(); state.getEnvironment.mockResolvedValue({ id: "sandbox", driver: "sandbox", status: "active", config: { ...config, timeoutMs: "invalid" }, metadata: { managedByPaperclip: true } }); const invalid = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox"); - expect(agentRunnerAvailability("codex_local", invalid).defaultRunner).toBe("legacy"); + expect(agentRunnerAvailability("codex_local", invalid, state.disabled).defaultRunner).toBe("legacy"); expect(state.resolveEnvironment).not.toHaveBeenCalled(); }); it("checks company access before trusting a managed Cloud image", async () => { diff --git a/server/src/services/agent-runner-selection.ts b/server/src/services/agent-runner-selection.ts index 7515b40dc81..843d69b699a 100644 --- a/server/src/services/agent-runner-selection.ts +++ b/server/src/services/agent-runner-selection.ts @@ -5,6 +5,9 @@ import { resolveClaudeModel } from "@paperclipai/adapter-claude-local"; import { DEFAULT_GROK_LOCAL_MODEL } from "@paperclipai/adapter-grok-local"; import { findActiveServerAdapter, hasActiveAdapterOverride, waitForExternalAdapters } from "../adapters/registry.js"; import { getDisabledAdapterTypes } from "./adapter-plugin-store.js"; +import { adapterSettingsService, effectiveDisabledAdapterTypes } from "./adapter-settings.js"; +import { listServerAdapters } from "../adapters/registry.js"; +import { isCloudManagedInstance } from "./cloud-instance.js"; import { claudeRunnerUnsupportedSettings, PaperclipRunnerProviderProfileError, resolvePaperclipRunnerNativeProviderInput } from "./native-runtime/provider-profile.js"; import { forbidden, unprocessable } from "../errors.js"; import { runSshCommand } from "@paperclipai/adapter-utils/ssh"; @@ -24,6 +27,7 @@ export type RunnerTarget = { export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi"; export interface AgentRunnerSelection { adapterType?: string | null; + disabledTypes?: string[]; adapterConfig?: Record; runner?: AgentRunnerChoice; target?: RunnerTarget; @@ -32,8 +36,13 @@ export interface AgentRunnerSelection { onConfigurationWarnings?: (warnings: AgentConfigurationWarning[]) => void; } -function runnerAdaptersAvailable(harness: string, driver: string): boolean { - const disabled = new Set(getDisabledAdapterTypes()); +function localDisabledTypes(): string[] { + if (isCloudManagedInstance()) throw new Error("Managed runner selection requires a fresh adapter settings snapshot"); + return effectiveDisabledAdapterTypes(getDisabledAdapterTypes(), () => listServerAdapters().map((adapter) => adapter.type)); +} + +function runnerAdaptersAvailable(harness: string, driver: string, disabledTypes: string[]): boolean { + const disabled = new Set(disabledTypes); return !!paperclipRunnerProfileForHarness(harness) && !disabled.has(harness) && !disabled.has("paperclip_runner") && !!findActiveServerAdapter(harness) && !!findActiveServerAdapter("paperclip_runner") @@ -41,13 +50,13 @@ function runnerAdaptersAvailable(harness: string, driver: string): boolean { && isEnvironmentDriverSupportedForAdapter("paperclip_runner", driver); } -export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget()): AgentRunnerAvailability { +export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget(), disabledTypes: string[] = localDisabledTypes()): AgentRunnerAvailability { if (target.runnerTasks || target.managedCloudImage) { - return runnerAdaptersAvailable(harness, target.driver) + return runnerAdaptersAvailable(harness, target.driver, disabledTypes) ? { supportedRunners: target.runnerTasks ? ["paperclip"] : ["paperclip", "legacy"], defaultRunner: "paperclip" } : { supportedRunners: ["legacy"], defaultRunner: "legacy" }; } - const supported = runnerAdaptersAvailable(harness, target.driver) + const supported = runnerAdaptersAvailable(harness, target.driver, disabledTypes) && (target.driver !== "sandbox" || Boolean(target.platform && target.architecture && (target.platform !== "linux" || (target.libc === "glibc" && target.glibcVersion)))) && (["claude_local", "grok_local"].includes(harness) @@ -166,11 +175,12 @@ export function resolveNewAgentRunner(input: AgentRunnerSelection): { adapterTyp function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; adapterConfig: Record } { const adapterType = input.adapterType ?? "process"; let config = { ...input.adapterConfig }; + const disabledTypes = input.disabledTypes ?? localDisabledTypes(); const harness = agentHarnessType(adapterType, config); const native = adapterType === "paperclip_runner"; - if (paperclipRunnerProfileForHarness(adapterType) && getDisabledAdapterTypes().includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" }); + if (paperclipRunnerProfileForHarness(adapterType) && disabledTypes.includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" }); if (native && (input.runner === undefined || input.runner === "paperclip" || (harness !== "codex_local" && input.runner !== "legacy"))) return { adapterType, adapterConfig: normalizeLegacyRunnerProvider(config) }; - const availability = agentRunnerAvailability(harness, input.target); + const availability = agentRunnerAvailability(harness, input.target, disabledTypes); const choice = input.runner ?? "auto"; const useNative = choice === "paperclip" || (choice === "auto" && availability.defaultRunner === "paperclip"); if (!useNative) { @@ -318,13 +328,14 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; } export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, input: AgentRunnerSelection & { defaultEnvironmentId?: string | null }) { + input = { ...input, disabledTypes: await adapterSettingsService(db).getDisabledTypes() }; if (!paperclipRunnerProfileForHarness(agentHarnessType(input.adapterType ?? "process", input.adapterConfig)) || input.runner === "legacy" || (input.adapterType === "paperclip_runner" && (input.runner === undefined || input.runner === "paperclip"))) return resolveNewAgentRunner(input); - const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId); + const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId, input.disabledTypes); return resolveNewAgentRunner({ ...input, target }); } -export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null): Promise { +export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null, disabledTypes?: string[]): Promise { await waitForExternalAdapters(); const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId); let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget() @@ -356,7 +367,9 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri const qualified = await readSandboxPlatformQualification(db, environment, companyId); if (qualified) target = { driver, ...qualified }; } - if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver))) { + const disabledForSsh = driver === "ssh" && environment + ? disabledTypes ?? await adapterSettingsService(db).getDisabledTypes() : []; + if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver, disabledForSsh))) { const parsed = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment); if (parsed.driver !== "ssh") throw unprocessable("The selected SSH environment is unavailable."); let output: string; diff --git a/server/src/services/bundled-plugin-package.ts b/server/src/services/bundled-plugin-package.ts new file mode 100644 index 00000000000..4d85c4f93f1 --- /dev/null +++ b/server/src/services/bundled-plugin-package.ts @@ -0,0 +1,108 @@ +import fs from "node:fs"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import type { PaperclipPluginManifestV1, PluginRecord } from "@paperclipai/shared"; +import type { ResolvedBundledPlugin } from "./bundled-plugins.js"; +import { assertDistributionManifestCapabilities } from "./distribution-plugin-catalog.js"; + +export type BundledPluginRegistration = Pick; + +export interface BundledPluginPackageIdentity { + kind: "bundled_image"; + catalogKey: string; + pluginKey: string; + packageName: string; + version: string; + digest?: string; +} + +export type BundledPluginPackageInspection = + | { compatible: true; identity: BundledPluginPackageIdentity } + | { compatible: false; pluginKey: string; code: "incompatible_bundled_package" | "missing_manifest_evidence"; reason: string }; + +function containedEntrypoint(root: string, declared: unknown, directory: boolean): string { + const relative = typeof declared === "string" ? declared.replace(/^\.\//, "").replace(/\/$/, "") : ""; + if (!relative || path.isAbsolute(relative) || path.win32.isAbsolute(relative) || relative.includes("\\") || relative.split("/").length > 32 || relative.split("/").some((part) => !part || part === "." || part === "..")) { + throw new Error("Bundled plugin entrypoint must stay inside its package"); + } + const target = fs.realpathSync(path.join(root, relative)); + const resolvedRelative = path.relative(root, target); + if (!resolvedRelative || resolvedRelative.startsWith("..") || path.isAbsolute(resolvedRelative)) { + throw new Error("Bundled plugin entrypoint escapes its package"); + } + const stat = fs.statSync(target); + if (directory ? !stat.isDirectory() : !stat.isFile()) throw new Error("Bundled plugin entrypoint is not prebuilt"); + return relative; +} + +export function createBundledPluginPackageResolver(installs: readonly ResolvedBundledPlugin[]) { + const byKey = new Map(installs.map((install) => [install.pluginKey, install])); + + function read(install: ResolvedBundledPlugin) { + const root = fs.realpathSync(install.localPath); + if (root !== path.resolve(install.localPath)) throw new Error("Bundled plugin package root changed after catalog resolution"); + containedEntrypoint(root, "package.json", false); + const metadataPath = path.join(root, "package.json"); + if (fs.statSync(metadataPath).size > 128 * 1024) throw new Error("Bundled plugin package metadata exceeds verification limits"); + const pkg = JSON.parse(fs.readFileSync(metadataPath, "utf8")); + if (typeof pkg.name !== "string" || typeof pkg.version !== "string" || !pkg.paperclipPlugin) { + throw new Error("Bundled plugin package identity or entrypoints are missing"); + } + if (install.distribution && pkg.version !== install.distribution.version) throw new Error("Bundled plugin package version does not match the catalog"); + const manifest = containedEntrypoint(root, pkg.paperclipPlugin.manifest, false); + const worker = containedEntrypoint(root, pkg.paperclipPlugin.worker, false); + const ui = pkg.paperclipPlugin.ui === undefined ? undefined : containedEntrypoint(root, pkg.paperclipPlugin.ui, true); + if (install.distribution && (worker !== install.distribution.entrypoints.worker || ui !== install.distribution.entrypoints.ui)) { + throw new Error("Bundled plugin entrypoints do not match the verified package"); + } + return { root, pkg, manifest, worker, ui }; + } + + function assertManifest(install: ResolvedBundledPlugin, manifest: PaperclipPluginManifestV1, previousManifest?: PaperclipPluginManifestV1) { + const bundle = read(install); + if (manifest.id !== install.pluginKey || manifest.version !== bundle.pkg.version) throw new Error("Bundled plugin manifest identity/version does not match the image"); + const worker = containedEntrypoint(bundle.root, manifest.entrypoints.worker, false); + const ui = manifest.entrypoints.ui === undefined ? undefined : containedEntrypoint(bundle.root, manifest.entrypoints.ui, true); + if (worker !== bundle.worker || ui !== bundle.ui) throw new Error("Bundled plugin manifest entrypoints do not match the image"); + assertDistributionManifestCapabilities(manifest); + if (previousManifest) { + const approved = new Set(previousManifest.capabilities); + if (manifest.capabilities.some((capability) => !approved.has(capability))) throw new Error("Bundled plugin capabilities require approval"); + } + return bundle; + } + + function resolve(plugin: BundledPluginRegistration): string { + const install = byKey.get(plugin.pluginKey); + if (!install) throw new Error(`Bundled plugin is absent or not selected in this deployment: ${plugin.pluginKey}`); + const bundle = assertManifest(install, plugin.manifestJson); + if (plugin.packageName !== bundle.pkg.name || plugin.version !== bundle.pkg.version) throw new Error("Bundled plugin registration identity/version does not match the image"); + return bundle.root; + } + + return { + resolve, + inspect(plugin: BundledPluginRegistration, verifiedManifest?: PaperclipPluginManifestV1): BundledPluginPackageInspection { + try { + resolve(plugin); + const install = byKey.get(plugin.pluginKey)!; + if (!verifiedManifest) return { compatible: false, pluginKey: plugin.pluginKey, code: "missing_manifest_evidence", reason: "Image manifest evidence is required" }; + assertManifest(install, verifiedManifest); + if (!isDeepStrictEqual(verifiedManifest, plugin.manifestJson)) throw new Error("Approved plugin manifest does not match the image"); + return { compatible: true, identity: { + kind: "bundled_image", catalogKey: install.key, pluginKey: plugin.pluginKey, + packageName: plugin.packageName, version: plugin.version, + ...(install.distribution ? { digest: install.distribution.digest } : {}), + } }; + } catch (error) { + return { compatible: false, pluginKey: plugin.pluginKey, code: "incompatible_bundled_package", reason: error instanceof Error ? error.message : String(error) }; + } + }, + assertActivation(input: { pluginKey?: string; packageRoot: string; manifest?: PaperclipPluginManifestV1; previousManifest?: PaperclipPluginManifestV1 }) { + const install = input.pluginKey ? byKey.get(input.pluginKey) : installs.find((candidate) => candidate.localPath === input.packageRoot); + if (!install || path.resolve(input.packageRoot) !== install.localPath) throw new Error("Bundled plugin is absent or not selected in this deployment"); + read(install); + if (input.manifest) assertManifest(install, input.manifest, input.previousManifest); + }, + }; +} diff --git a/server/src/services/bundled-plugin-runtime.ts b/server/src/services/bundled-plugin-runtime.ts new file mode 100644 index 00000000000..cae4b5e80ab --- /dev/null +++ b/server/src/services/bundled-plugin-runtime.ts @@ -0,0 +1,37 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export function resolveBundledPluginRuntimeLoader( + moduleDirectory = path.dirname(fileURLToPath(import.meta.url)), +): string { + let directory = fs.realpathSync(moduleDirectory); + let imageRoot: string | undefined; + for (let depth = 0; depth < 5; depth++) { + const metadataPath = path.join(directory, "package.json"); + const metadata = fs.lstatSync(metadataPath, { throwIfNoEntry: false }); + if (metadata?.isFile() && metadata.size <= 128 * 1024 + && JSON.parse(fs.readFileSync(metadataPath, "utf8")).name === "@paperclipai/server") { + imageRoot = path.dirname(directory); + break; + } + const parent = path.dirname(directory); + if (parent === directory) break; + directory = parent; + } + if (!imageRoot) throw new Error("Managed plugin server package is missing from the current image"); + const declared = path.join(imageRoot, "cli/node_modules/tsx/dist/loader.mjs"); + let loader: string; + try { + loader = fs.realpathSync(declared); + if (!fs.statSync(loader).isFile()) throw new Error("not a file"); + } catch { + throw new Error("Managed plugin runtime loader is missing from the current image"); + } + const roots = [path.join(imageRoot, "cli"), path.join(imageRoot, "node_modules")]; + if (!roots.some((root) => { + const relative = path.relative(root, loader); + return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative); + })) throw new Error("Managed plugin runtime loader escapes the current image"); + return loader; +} diff --git a/server/src/services/bundled-plugins.ts b/server/src/services/bundled-plugins.ts index 9a728f452fc..cb59fa5cbae 100644 --- a/server/src/services/bundled-plugins.ts +++ b/server/src/services/bundled-plugins.ts @@ -1,6 +1,7 @@ import path from "node:path"; import fs from "node:fs"; import { isDeepStrictEqual } from "node:util"; +import { isCloudManagedInstance } from "./cloud-instance.js"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import { assertDistributionManifestCapabilities, readDistributionPluginCatalog, type DistributionPlugin } from "./distribution-plugin-catalog.js"; @@ -118,6 +119,13 @@ export const BUNDLED_PLUGIN_CATALOG: readonly BundledPluginCatalogEntry[] = [ */ export const SELF_HOSTED_AUTO_INSTALL_KEYS: readonly string[] = ["kubernetes"]; +export function resolveManagedBundledPluginKeys( + keys: readonly string[] | null | undefined, + env: Record = process.env, +): readonly string[] | null { + return keys ?? (isCloudManagedInstance(env) ? [] : null); +} + export function resolveBundledCatalogRoot( env: Record, ): string { @@ -203,7 +211,7 @@ export function resolveBundledPluginInstalls( `bundled plugin "${key}" resolves to "${localPath}", outside the bundled catalog root "${opts.catalogRoot}"; refusing to start`, ); } - resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath }); + resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath: opts.enforceCatalogRoot ? canonicalize(localPath) : localPath }); } return resolved; } @@ -269,6 +277,7 @@ async function reconcileBundledPluginManifest( deps: BundledPluginProvisionerDeps, bundleManifestExists: (localPath: string) => boolean, verifiedManifest?: PaperclipPluginManifestV1, + preservePackagePath = false, ): Promise<"upgrade_pending" | undefined> { try { // Managed reinstalls take the install path instead; this branch means the @@ -283,7 +292,7 @@ async function reconcileBundledPluginManifest( const approved = new Set(existing.manifestJson.capabilities ?? []); requiresApproval = bundleManifest.capabilities.some((capability) => !approved.has(capability)); } - const rebindPackage = install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled"; + const rebindPackage = !preservePackagePath && install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled"; const refreshDistribution = install.distribution && !isDeepStrictEqual(bundleManifest, existing.manifestJson); if (bundleManifest.version === existing.version && !rebindPackage && !requiresApproval && !refreshDistribution) return; await deps.registry.update(existing.id, { @@ -389,7 +398,7 @@ async function reenableErroredBundledPlugin( export async function ensureBundledPlugins( installs: readonly ResolvedBundledPlugin[], deps: BundledPluginProvisionerDeps, - opts: { reinstallUninstalled: boolean }, + opts: { reinstallUninstalled: boolean; preservePackagePath?: boolean }, ): Promise { const bundleManifestExists = deps.bundleManifestExists ?? defaultBundleManifestExists; for (const install of installs) { @@ -411,7 +420,7 @@ export async function ensureBundledPlugins( // existing install, because the auto-install below skips a present // plugin. Distribution entries also replace a legacy/npm package path // before loadAll resolves the worker. Configuration and status stay put. - const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest); + const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest, opts.preservePackagePath); if (reconciledStatus === "upgrade_pending") continue; if (existing.status === "error") { await reenableErroredBundledPlugin(existing, install, deps); diff --git a/server/src/services/cloud-bundled-package-digest.ts b/server/src/services/cloud-bundled-package-digest.ts new file mode 100644 index 00000000000..40cacb3fa11 --- /dev/null +++ b/server/src/services/cloud-bundled-package-digest.ts @@ -0,0 +1,95 @@ +import fs from "node:fs"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js"; + +export function cloudBundledPackageDigest(packageRoot: string, options: { catalogRoot?: string } = {}): string { + const root = fs.realpathSync(packageRoot); + const roots = [{ key: "package", directory: root }]; + let imageNodeModules: string | undefined; + if (options.catalogRoot) { + const catalogRoot = fs.realpathSync(options.catalogRoot); + const relative = path.relative(catalogRoot, root); + if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Image package is outside the selected catalog root"); + imageNodeModules = path.resolve(catalogRoot, "../../node_modules"); + roots.push( + { key: "plugin-sdk", directory: path.join(catalogRoot, "sdk") }, + { key: "shared", directory: path.resolve(catalogRoot, "../shared") }, + { key: "image-node-modules", directory: imageNodeModules }, + ); + } + roots.sort((left, right) => right.directory.length - left.directory.length); + const inventory = new Map(); + const directories = new Set(); + const files = new Set(); + const buffer = Buffer.alloc(65_536); + let bytes = 0; + let entries = 0; + + function logical(file: string): string { + for (const candidate of roots) { + const relative = path.relative(candidate.directory, file); + if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) { + if (relative.split(path.sep).length > 32) throw new Error("Image package exceeds the directory depth limit"); + return `${candidate.key}/${relative.split(path.sep).join("/")}`; + } + } + throw new Error(options.catalogRoot ? "Image dependency escapes its authorized image roots" : "Image package symlink escapes its package"); + } + + function follow(target: string) { + logical(target); + if (imageNodeModules) { + const relative = path.relative(imageNodeModules, target).split(path.sep); + if ((relative.length === 1 && relative[0] === "") + || (relative[0] === ".pnpm" && (relative.length === 1 || (relative.length === 2 && relative[1] === "node_modules")))) { + throw new Error("Image dependency must select an individual package"); + } + if (relative[0] === ".pnpm" && relative.length > 2 && relative[1] && relative[1] !== "node_modules") { + walk(path.join(imageNodeModules, ".pnpm", relative[1])); + return; + } + } + walk(target); + } + + function walk(file: string) { + const name = logical(file); + const stat = fs.lstatSync(file); + if (stat.isSymbolicLink()) { + const target = fs.realpathSync(file); + inventory.set(name, [name, "link", logical(target), stat.mode & 0o777]); + follow(target); + } else if (stat.isDirectory()) { + const real = fs.realpathSync(file); + logical(real); + if (directories.has(real)) return; + directories.add(real); + for (const child of fs.readdirSync(real).sort()) { + if (++entries > 50_000) throw new Error("Image package exceeds the entry limit"); + walk(path.join(real, child)); + } + } else if (stat.isFile()) { + const real = fs.realpathSync(file); + if (files.has(real)) return; + files.add(real); + if (path.basename(file) === "package.json" && stat.size > 128 * 1024) throw new Error("Image package metadata exceeds the byte limit"); + if (bytes + stat.size > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit"); + const handle = fs.openSync(file, "r"); + const hash = createHash("sha256"); + try { + while (true) { + const count = fs.readSync(handle, buffer); + if (!count) break; + bytes += count; + if (bytes > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit"); + hash.update(buffer.subarray(0, count)); + } + } finally { fs.closeSync(handle); } + inventory.set(name, [name, "file", `sha256:${hash.digest("hex")}`, stat.mode & 0o777]); + } else throw new Error("Image package contains a special file"); + } + walk(root); + if (!inventory.size) throw new Error("Image package is empty"); + return cloudConfigurationDigest([...inventory.values()].sort((left, right) => left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0)); +} diff --git a/server/src/services/cloud-configuration-evidence.ts b/server/src/services/cloud-configuration-evidence.ts new file mode 100644 index 00000000000..63303fbf055 --- /dev/null +++ b/server/src/services/cloud-configuration-evidence.ts @@ -0,0 +1,74 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; +import { z } from "zod"; + +function canonical(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonical); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, child]) => [key, canonical(child)])); + } + return value; +} + +export function cloudConfigurationDigest(value: unknown): string { + return `sha256:${createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex")}`; +} + +export const externalRegistrationEvidenceSchema = z.object({ + sourceLocatorDigest: z.string(), + sourceDigest: z.string(), + state: z.enum(["missing", "parsed", "invalid", "unreadable"]), + adapterTypes: z.array(z.string()), + blocker: z.enum(["unsupported_external_adapters", "invalid_external_registration_metadata", "unreadable_external_registration_metadata"]).nullable(), +}).strict(); +export type ExternalRegistrationEvidence = z.infer; + +const legacyRegistrationSchema = z.array(z.object({ + type: z.string().regex(/^[a-zA-Z0-9_.-]{1,128}$/), + packageName: z.string().min(1).max(1024), + version: z.string().max(128).optional(), + localPath: z.string().max(4096).optional(), + disabled: z.boolean().optional(), +}).passthrough()).max(100); + +export async function inspectLegacyExternalRegistrations(filePath: string): Promise { + const sourceLocatorDigest = cloudConfigurationDigest(filePath); + try { + const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let raw: unknown; + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid legacy registration file"); + const buffer = Buffer.alloc(65_537); + let total = 0; + while (total < buffer.length) { + const { bytesRead } = await file.read(buffer, total, buffer.length - total, total); + if (!bytesRead) break; + total += bytesRead; + } + if (total > 65_536) throw new Error("oversized legacy registration file"); + raw = JSON.parse(buffer.subarray(0, total).toString("utf8")); + } finally { + await file.close(); + } + const records = legacyRegistrationSchema.safeParse(raw); + if (!records.success) { + return { sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state: "invalid" }), state: "invalid", adapterTypes: [], blocker: "invalid_external_registration_metadata" }; + } + const metadata = records.data.map((record) => ({ type: record.type, packageName: record.packageName, version: record.version, disabled: record.disabled, hasLocalPath: record.localPath !== undefined })); + return { + sourceLocatorDigest, sourceDigest: cloudConfigurationDigest(metadata), state: "parsed", + adapterTypes: [...new Set(metadata.map((record) => record.type))].sort(), + blocker: metadata.length ? "unsupported_external_adapters" : null, + }; + } catch (error) { + const absent = (error as NodeJS.ErrnoException).code === "ENOENT"; + const invalid = error instanceof SyntaxError || (error instanceof Error && /legacy registration file/.test(error.message)); + const state = absent ? "missing" : invalid ? "invalid" : "unreadable"; + return { + sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state }), state, adapterTypes: [], + blocker: absent ? null : invalid ? "invalid_external_registration_metadata" : "unreadable_external_registration_metadata", + }; + } +} diff --git a/server/src/services/cloud-configuration-migration.ts b/server/src/services/cloud-configuration-migration.ts new file mode 100644 index 00000000000..3ce38234408 --- /dev/null +++ b/server/src/services/cloud-configuration-migration.ts @@ -0,0 +1,193 @@ +import { instanceSettings, plugins, type Db } from "@paperclipai/db"; +import { eq, inArray, ne } from "drizzle-orm"; +import { z } from "zod"; +import { adapterSettingsService, readLegacyAdapterSettings } from "./adapter-settings.js"; +import { getAdapterPluginStorePath } from "./adapter-plugin-store.js"; +import { cloudRuntimeConfigService, readLegacyCloudRuntimeConfig } from "./cloud-runtime-config.js"; +import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js"; +import { cloudConfigurationDigest, externalRegistrationEvidenceSchema, inspectLegacyExternalRegistrations } from "./cloud-configuration-evidence.js"; +import { cloudPackageEvidenceSchema, inspectCloudPackages, qualificationBlockerSchema } from "./cloud-package-qualification.js"; +import type { ResolvedBundledPlugin } from "./bundled-plugins.js"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; + +const RECEIPT_KEY = "cloud-configuration-migration:v1"; +const CONFIGURATION_KEYS = ["adapter-settings:v1", "cloud-runtime-config:v1"]; +type Transaction = Parameters[0]>[0]; +export type CloudConfigurationDb = Pick; +const sourceSchema = z.object({ + authority: z.enum(["database", "legacy_file", "default"]), sourceDigest: z.string(), + importedRevision: z.number().int().nonnegative(), +}).strict(); +const qualificationSchema = z.object({ + scope: z.literal("configuration_and_packages"), checkedAt: z.string(), + adapterRevision: z.number().int().nonnegative(), configRevision: z.number().int().nonnegative(), + configurationDigest: z.string(), packageEvidenceDigest: z.string(), + packageInspectionDurationMs: z.number().nonnegative(), + expectedPackageEvidenceDigest: z.string().nullable(), + configurationPackagesEligible: z.boolean(), samePackageOverlapEligible: z.boolean(), + packages: z.array(cloudPackageEvidenceSchema), blockers: z.array(qualificationBlockerSchema), + overlapBlockers: z.array(qualificationBlockerSchema), +}).strict(); +const receiptSchema = z.object({ + version: z.literal(1), revision: z.number().int().nonnegative(), + migration: z.object({ completedAt: z.string(), adapter: sourceSchema, config: sourceSchema, externalRegistrations: externalRegistrationEvidenceSchema }).strict(), + qualification: qualificationSchema.nullable(), lastPackageMismatch: qualificationSchema.nullable(), +}).strict(); +export type CloudConfigurationReceipt = z.infer; + +function parseReceipt(value: unknown): CloudConfigurationReceipt { + const result = receiptSchema.safeParse(value); + if (!result.success) throw new Error("Persisted Cloud configuration migration receipt is invalid"); + return result.data; +} + +export function cloudConfigurationMigrationService(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv; now?: () => Date } = {}) { + const env = options.runtimeEnv ?? process.env; + const now = () => (options.now?.() ?? new Date()).toISOString(); + const adapters = adapterSettingsService(db, { runtimeEnv: env }); + const config = cloudRuntimeConfigService(db); + + function assertManaged() { + if (!isCloudManagedInstance(env)) throw new Error("Cloud configuration qualification requires a managed instance"); + } + async function readRow(handle: CloudConfigurationDb, lock = false) { + const query = handle.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, RECEIPT_KEY)); + const [row] = lock ? await query.for("update") : await query; + return row ?? null; + } + async function currentConfiguration(handle: CloudConfigurationDb) { + await handle.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS)) + .orderBy(instanceSettings.singletonKey).for("update"); + const adapter = await adapters.inspectStored(handle); + const runtime = await config.inspectStored(handle); + if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete"); + return { adapter, runtime }; + } + async function registrationSnapshot(handle: CloudConfigurationDb, lock = false) { + const query = handle.select({ + pluginKey: plugins.pluginKey, packageName: plugins.packageName, version: plugins.version, + manifestJson: plugins.manifestJson, status: plugins.status, + }).from(plugins).where(ne(plugins.status, "uninstalled")).orderBy(plugins.pluginKey); + return lock ? query.for("share") : query; + } + async function save(handle: CloudConfigurationDb, id: string, receipt: CloudConfigurationReceipt) { + const validated = parseReceipt(receipt); + const updated = await handle.update(instanceSettings).set({ general: validated, updatedAt: new Date() }).where(eq(instanceSettings.id, id)).returning({ id: instanceSettings.id }); + if (updated.length !== 1) throw new Error("Cloud configuration migration receipt was not persisted"); + return validated; + } + + return { + async inspectStored(): Promise { + const row = await readRow(db); + return row ? parseReceipt(row.general) : null; + }, + async migrate(): Promise { + assertManaged(); + return db.transaction(async (tx) => { + const inserted = await tx.insert(instanceSettings).values({ singletonKey: RECEIPT_KEY, general: { version: 1 } }) + .onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id }); + const row = await readRow(tx, true); + if (!row) throw new Error("Cloud configuration migration receipt is unavailable"); + if (!inserted.length) { + const receipt = parseReceipt(row.general); + await currentConfiguration(tx); + return receipt; + } + await tx.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS)) + .orderBy(instanceSettings.singletonKey).for("update"); + const storedAdapter = await adapters.inspectStored(tx); + const storedConfig = await config.inspectStored(tx); + const adapterCandidate = storedAdapter ? null : await readLegacyAdapterSettings(env); + const configCandidate = storedConfig ? null : await readLegacyCloudRuntimeConfig(env); + const adapterSeed = storedAdapter ? null : await adapters.seedIfAbsent({ disabledTypes: adapterCandidate?.disabledTypes ?? [] }, tx); + const configSeed = storedConfig ? null : await config.seedIfAbsent(configCandidate, tx); + const adapter = storedAdapter ?? adapterSeed!.current; + const runtime = storedConfig ?? configSeed!.current; + const adapterFromDatabase = Boolean(storedAdapter || !adapterSeed?.seeded); + const configFromDatabase = Boolean(storedConfig || !configSeed?.seeded); + const receipt: CloudConfigurationReceipt = { + version: 1, revision: 0, + migration: { + completedAt: now(), + adapter: { authority: adapterFromDatabase ? "database" : adapterCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(adapterFromDatabase ? { disabledTypes: adapter.disabledTypes } : adapterCandidate), importedRevision: adapter.revision }, + config: { authority: configFromDatabase ? "database" : configCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(configFromDatabase ? runtime.config : configCandidate), importedRevision: runtime.revision }, + externalRegistrations: await inspectLegacyExternalRegistrations(getAdapterPluginStorePath(env.PAPERCLIP_HOME)), + }, + qualification: null, lastPackageMismatch: null, + }; + return save(tx, row.id, receipt); + }); + }, + async recheckLegacyExternalRegistrations(expectedRevision: number): Promise { + assertManaged(); + return db.transaction(async (tx) => { + const row = await readRow(tx, true); + if (!row) throw new Error("Cloud configuration migration has not completed"); + const receipt = parseReceipt(row.general); + if (receipt.revision !== expectedRevision) throw new Error("Cloud configuration migration receipt revision conflict"); + const filePath = getAdapterPluginStorePath(env.PAPERCLIP_HOME); + if (cloudConfigurationDigest(filePath) !== receipt.migration.externalRegistrations.sourceLocatorDigest) { + throw new Error("External registration recheck requires the original legacy source location"); + } + const evidence = await inspectLegacyExternalRegistrations(filePath); + if (evidence.state !== "parsed") throw new Error("External registration recheck requires a present, valid registration file"); + receipt.migration.externalRegistrations = evidence; + receipt.revision += 1; + receipt.qualification = null; + return save(tx, row.id, receipt); + }); + }, + async qualify(input: { + installs: readonly ResolvedBundledPlugin[]; + catalogRoot?: string; + loadManifest: (packageRoot: string) => Promise; + expectedPackageEvidenceDigest?: string; + imageCatalogUnavailable?: boolean; + }) { + assertManaged(); + const adapter = await adapters.inspectStored(); + const runtime = await config.inspectStored(); + if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete"); + const configurationBefore = cloudConfigurationDigest({ adapter, runtime }); + const registrationsBefore = await registrationSnapshot(db); + const evidence = await inspectCloudPackages({ ...input, registrations: registrationsBefore }); + return db.transaction(async (tx) => { + const row = await readRow(tx, true); + if (!row) throw new Error("Cloud configuration migration has not completed"); + const receipt = parseReceipt(row.general); + const current = await currentConfiguration(tx); + const registrations = await registrationSnapshot(tx, true); + const blockers = [...evidence.blockers]; + if (configurationBefore !== cloudConfigurationDigest(current) || cloudConfigurationDigest(registrationsBefore) !== cloudConfigurationDigest(registrations)) { + blockers.push({ code: "configuration_changed_during_inspection", reason: "Configuration or plugin registration changed during image inspection; run qualification again" }); + } + if (input.imageCatalogUnavailable) blockers.push({ code: "current_image_catalog_unavailable", reason: "The current image package catalog could not be verified" }); + const external = receipt.migration.externalRegistrations; + if (external.blocker) blockers.push({ code: external.blocker, reason: "Legacy external adapter registration metadata requires an explicit operator resolution" }); + const overlapBlockers = input.expectedPackageEvidenceDigest && input.expectedPackageEvidenceDigest !== evidence.packageEvidenceDigest + ? [{ code: "package_evidence_changed", reason: "Current packages do not match the expected package evidence digest" }] : []; + const qualification: z.infer = { + scope: "configuration_and_packages", checkedAt: now(), + adapterRevision: current.adapter.revision, configRevision: current.runtime.revision, + configurationDigest: cloudConfigurationDigest(current), packageEvidenceDigest: evidence.packageEvidenceDigest, + packageInspectionDurationMs: evidence.packageInspectionDurationMs, + expectedPackageEvidenceDigest: input.expectedPackageEvidenceDigest ?? null, + configurationPackagesEligible: blockers.length === 0, + samePackageOverlapEligible: Boolean(input.expectedPackageEvidenceDigest && blockers.length === 0 && overlapBlockers.length === 0), + packages: evidence.packages, blockers, overlapBlockers, + }; + receipt.revision += 1; + receipt.qualification = qualification; + if (evidence.blockers.length || input.imageCatalogUnavailable || overlapBlockers.length) receipt.lastPackageMismatch = qualification; + await save(tx, row.id, receipt); + return { receiptRevision: receipt.revision, ...qualification }; + }); + }, + }; +} + +export async function migrateCloudConfiguration(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) { + if (!isCloudManagedInstance(options.runtimeEnv ?? process.env)) return null; + return cloudConfigurationMigrationService(db, options).migrate(); +} diff --git a/server/src/services/cloud-configuration-qualification.ts b/server/src/services/cloud-configuration-qualification.ts new file mode 100644 index 00000000000..c96c97ae8e5 --- /dev/null +++ b/server/src/services/cloud-configuration-qualification.ts @@ -0,0 +1,33 @@ +import type { Db } from "@paperclipai/db"; +import { BUNDLED_PLUGIN_CATALOG, resolveBundledCatalogRoot, resolveBundledPluginInstalls, resolveManagedBundledPluginKeys, type ResolvedBundledPlugin } from "./bundled-plugins.js"; +import { readDistributionPluginCatalog } from "./distribution-plugin-catalog.js"; +import { createBundledPluginPackageResolver } from "./bundled-plugin-package.js"; +import { cloudConfigurationMigrationService } from "./cloud-configuration-migration.js"; +import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js"; +import { getManagedInstanceConfig } from "./managed-config.js"; +import { pluginLoader } from "./plugin-loader.js"; + +export async function qualifyCloudConfigurationForCurrentImage(db: Db, options: { + runtimeEnv?: CloudInstanceEnv; + expectedPackageEvidenceDigest?: string; +} = {}) { + const env = options.runtimeEnv ?? process.env; + if (!isCloudManagedInstance(env)) return null; + const service = cloudConfigurationMigrationService(db, { runtimeEnv: env }); + const catalogRoot = resolveBundledCatalogRoot(env); + let installs: ResolvedBundledPlugin[]; + try { + const distribution = readDistributionPluginCatalog(catalogRoot, BUNDLED_PLUGIN_CATALOG); + installs = resolveBundledPluginInstalls(resolveManagedBundledPluginKeys(getManagedInstanceConfig(env)?.plugins.autoInstall, env) ?? [], { + catalogRoot, env, enforceCatalogRoot: true, distributionPlugins: distribution, + }); + } catch { + return service.qualify({ installs: [], loadManifest: async () => null, imageCatalogUnavailable: true, expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest }); + } + const resolver = createBundledPluginPackageResolver(installs); + const loader = pluginLoader(db, { + prebuiltOnly: true, enableLocalFilesystem: false, enableNpmDiscovery: false, + assertPackageActivation: resolver.assertActivation, resolvePackageRoot: resolver.resolve, + }); + return service.qualify({ installs, catalogRoot, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest }); +} diff --git a/server/src/services/cloud-package-qualification.ts b/server/src/services/cloud-package-qualification.ts new file mode 100644 index 00000000000..d0d75f30cf7 --- /dev/null +++ b/server/src/services/cloud-package-qualification.ts @@ -0,0 +1,72 @@ +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import { z } from "zod"; +import { createBundledPluginPackageResolver, type BundledPluginRegistration } from "./bundled-plugin-package.js"; +import type { ResolvedBundledPlugin } from "./bundled-plugins.js"; +import { distributionBundleDigest } from "./distribution-plugin-catalog.js"; +import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js"; +import { cloudBundledPackageDigest } from "./cloud-bundled-package-digest.js"; + +export const qualificationBlockerSchema = z.object({ + code: z.string(), pluginKey: z.string().optional(), reason: z.string(), + registeredVersion: z.string().optional(), imageVersion: z.string().optional(), +}).strict(); +export type QualificationBlocker = z.infer; +export const cloudPackageEvidenceSchema = z.object({ + catalogKey: z.string(), pluginKey: z.string(), packageDigest: z.string().nullable(), + imageManifestDigest: z.string().nullable(), approvedManifestDigest: z.string().nullable(), +}).strict(); +export type CloudPackageEvidence = z.infer; + +export async function inspectCloudPackages(input: { + installs: readonly ResolvedBundledPlugin[]; + registrations: readonly BundledPluginRegistration[]; + catalogRoot?: string; + loadManifest: (packageRoot: string) => Promise; +}) { + const started = performance.now(); + const resolver = createBundledPluginPackageResolver(input.installs); + const registrations = new Map(input.registrations.map((record) => [record.pluginKey, record])); + const selected = new Set(input.installs.map((install) => install.pluginKey)); + const blockers: QualificationBlocker[] = input.registrations.filter((record) => !selected.has(record.pluginKey)).map((record) => ({ + code: "unsupported_plugin_registration", pluginKey: record.pluginKey, reason: "An installed plugin is not selected from the current image", registeredVersion: record.version, + })); + const packages: CloudPackageEvidence[] = []; + for (const install of [...input.installs].sort((left, right) => left.pluginKey < right.pluginKey ? -1 : left.pluginKey > right.pluginKey ? 1 : 0)) { + const registration = registrations.get(install.pluginKey); + const evidence: CloudPackageEvidence = { + catalogKey: install.key, pluginKey: install.pluginKey, packageDigest: null, + imageManifestDigest: null, approvedManifestDigest: registration ? cloudConfigurationDigest(registration.manifestJson) : null, + }; + packages.push(evidence); + let stage = "missing_bundled_package"; + try { + resolver.assertActivation({ packageRoot: install.localPath }); + stage = "unverified_package_integrity"; + evidence.packageDigest = install.distribution ? distributionBundleDigest(install.localPath) : cloudBundledPackageDigest(install.localPath, { catalogRoot: input.catalogRoot }); + if (install.distribution && evidence.packageDigest !== install.distribution.digest) { + blockers.push({ code: "package_integrity_mismatch", pluginKey: install.pluginKey, reason: "Current package bytes do not match the verified distribution digest" }); + continue; + } + stage = "missing_manifest_evidence"; + const manifest = await input.loadManifest(install.localPath); + if (!manifest) { + blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image has no verified plugin manifest" }); + continue; + } + evidence.imageManifestDigest = cloudConfigurationDigest(manifest); + if (!registration) { + blockers.push({ code: "missing_plugin_registration", pluginKey: install.pluginKey, reason: "The current image package is not registered in the database", imageVersion: manifest.version }); + continue; + } + const inspected = resolver.inspect(registration, manifest); + if (!inspected.compatible) blockers.push({ + code: inspected.code, pluginKey: install.pluginKey, reason: inspected.reason, + registeredVersion: registration.version, imageVersion: manifest.version, + }); + } catch { + blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image package could not be verified" }); + } + } + const packageEvidenceDigest = cloudConfigurationDigest(packages.map(({ approvedManifestDigest: _approved, ...evidence }) => evidence)); + return { packages, blockers, packageEvidenceDigest, packageInspectionDurationMs: Math.round(performance.now() - started) }; +} diff --git a/server/src/services/cloud-runtime-config-snapshot.ts b/server/src/services/cloud-runtime-config-snapshot.ts new file mode 100644 index 00000000000..e97cd534b24 --- /dev/null +++ b/server/src/services/cloud-runtime-config-snapshot.ts @@ -0,0 +1,129 @@ +import { + authConfigSchema, + databaseBackupConfigSchema, + llmConfigSchema, + loggingConfigSchema, + paperclipConfigSchema, + secretsConfigSchema, + secretsLocalEncryptedConfigSchema, + serverConfigSchema, + storageConfigSchema, + storageLocalDiskConfigSchema, + storageS3ConfigSchema, + type PaperclipConfig, +} from "@paperclipai/shared"; +import { z } from "zod"; + +const publicUrl = z.string().url().refine((value) => { + const url = new URL(value); + return !url.username && !url.password && !url.search && !url.hash; +}); + +export const cloudRuntimeConfigSchema = z.object({ + server: serverConfigSchema.strict(), + auth: authConfigSchema.extend({ publicBaseUrl: publicUrl.optional() }).strict(), + backup: databaseBackupConfigSchema.strict(), + storage: storageConfigSchema.extend({ + localDisk: storageLocalDiskConfigSchema.strict(), + s3: storageS3ConfigSchema.extend({ endpoint: publicUrl.optional() }).strict(), + }).strict(), + secrets: secretsConfigSchema.extend({ + localEncrypted: secretsLocalEncryptedConfigSchema.strict(), + }).strict(), + telemetry: z.object({ enabled: z.boolean() }).strict(), + logging: loggingConfigSchema.strict(), + updates: z.object({ checkEnabled: z.boolean() }).strict().optional(), + llm: llmConfigSchema.omit({ apiKey: true }).strict().optional(), +}).strict(); + +export type CloudRuntimeConfig = z.infer; +export type StoredCloudRuntimeConfig = { version: 1; revision: number; config: CloudRuntimeConfig | null }; + +const storedSchema = z.object({ + version: z.literal(1), + revision: z.number().int().nonnegative(), + config: cloudRuntimeConfigSchema.nullable(), +}).strict(); + +export function parseStoredCloudRuntimeConfig(value: unknown): StoredCloudRuntimeConfig { + const parsed = storedSchema.safeParse(value); + if (!parsed.success || Buffer.byteLength(JSON.stringify(parsed.data), "utf8") > 65_536) { + throw new Error("Persisted Cloud runtime configuration is invalid"); + } + const full = parsed.data.config === null ? { success: true } : paperclipConfigSchema.safeParse(toPaperclipConfig(parsed.data.config)); + if (!full.success) throw new Error("Persisted Cloud runtime configuration is invalid"); + return parsed.data; +} + +export function toPaperclipConfig(config: CloudRuntimeConfig | null): PaperclipConfig | null { + if (config === null) return null; + const { backup, ...settings } = structuredClone(config); + return { + $meta: { version: 1, updatedAt: "1970-01-01T00:00:00.000Z", source: "configure" }, + database: { + mode: "postgres", + embeddedPostgresDataDir: "", + embeddedPostgresPort: 54329, + backup, + }, + ...settings, + }; +} + +export function projectCloudRuntimeConfig(source: PaperclipConfig): CloudRuntimeConfig { + const raw = { + server: { + deploymentMode: source.server.deploymentMode, + exposure: source.server.exposure, + bind: source.server.bind, + customBindHost: source.server.customBindHost, + host: source.server.host, + port: source.server.port, + allowedHostnames: source.server.allowedHostnames, + serveUi: source.server.serveUi, + }, + auth: { + baseUrlMode: source.auth.baseUrlMode, + publicBaseUrl: source.auth.publicBaseUrl, + disableSignUp: source.auth.disableSignUp, + }, + backup: { + enabled: source.database.backup.enabled, + intervalMinutes: source.database.backup.intervalMinutes, + retentionDays: source.database.backup.retentionDays, + dir: source.database.backup.dir, + }, + storage: { + provider: source.storage.provider, + localDisk: { baseDir: source.storage.localDisk.baseDir }, + s3: { + bucket: source.storage.s3.bucket, + region: source.storage.s3.region, + endpoint: source.storage.s3.endpoint, + prefix: source.storage.s3.prefix, + forcePathStyle: source.storage.s3.forcePathStyle, + }, + }, + secrets: { + provider: source.secrets.provider, + strictMode: source.secrets.strictMode, + localEncrypted: { keyFilePath: source.secrets.localEncrypted.keyFilePath }, + }, + telemetry: { enabled: source.telemetry.enabled }, + logging: { mode: source.logging.mode, logDir: source.logging.logDir }, + updates: source.updates ? { checkEnabled: source.updates.checkEnabled } : undefined, + llm: source.llm ? { provider: source.llm.provider } : undefined, + }; + return parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config: raw }).config!; +} + +let snapshot: StoredCloudRuntimeConfig | null = null; + +export function setCloudRuntimeConfigSnapshot(value: StoredCloudRuntimeConfig | null): void { + snapshot = value ? parseStoredCloudRuntimeConfig(value) : null; +} + +export function getCloudRuntimeConfigSnapshot(): StoredCloudRuntimeConfig { + if (!snapshot) throw new Error("Cloud runtime configuration has not been initialized"); + return structuredClone(snapshot); +} diff --git a/server/src/services/cloud-runtime-config.ts b/server/src/services/cloud-runtime-config.ts new file mode 100644 index 00000000000..61082ae07c7 --- /dev/null +++ b/server/src/services/cloud-runtime-config.ts @@ -0,0 +1,105 @@ +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { instanceSettings, type Db } from "@paperclipai/db"; +import { paperclipConfigSchema } from "@paperclipai/shared"; +import { resolvePaperclipConfigPathForInstance } from "@paperclipai/shared/home-paths"; +import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js"; +import { + parseStoredCloudRuntimeConfig, + projectCloudRuntimeConfig, + setCloudRuntimeConfigSnapshot, + type CloudRuntimeConfig, + type StoredCloudRuntimeConfig, +} from "./cloud-runtime-config-snapshot.js"; + +export const CLOUD_RUNTIME_CONFIG_KEY = "cloud-runtime-config:v1"; +type Transaction = Parameters[0]>[0]; +export type CloudRuntimeConfigDb = Pick; + +export function cloudRuntimeConfigService(db: CloudRuntimeConfigDb) { + async function inspectStored(handle: CloudRuntimeConfigDb = db): Promise { + const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings) + .where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).limit(1); + return row ? parseStoredCloudRuntimeConfig(row.general) : null; + } + + async function seedIfAbsent(config: CloudRuntimeConfig | null, handle: CloudRuntimeConfigDb = db) { + const next = parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config }); + const inserted = await handle.insert(instanceSettings) + .values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: next }) + .onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id }); + const current = await inspectStored(handle); + if (!current) throw new Error("Cloud runtime configuration seed was not persisted"); + return { seeded: inserted.length > 0, current }; + } + + return { + inspectStored, + seedIfAbsent, + async replace(config: CloudRuntimeConfig | null, expectedRevision: number, handle: CloudRuntimeConfigDb = db) { + const validated = parseStoredCloudRuntimeConfig({ version: 1, revision: expectedRevision, config }); + return handle.transaction(async (tx) => { + const [row] = await tx.select().from(instanceSettings) + .where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).for("update"); + if (!row) throw new Error("Cloud runtime configuration has not been initialized"); + const current = parseStoredCloudRuntimeConfig(row.general); + if (current.revision !== expectedRevision) throw new Error("Cloud runtime configuration revision conflict"); + const next = { ...validated, revision: current.revision + 1 }; + await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id)); + return next; + }); + }, + }; +} + +export function legacyCloudRuntimeConfigPath(env: CloudInstanceEnv = process.env): string { + return env.PAPERCLIP_CONFIG + ? path.resolve(env.PAPERCLIP_CONFIG) + : resolvePaperclipConfigPathForInstance({ homeDir: env.PAPERCLIP_HOME, instanceId: env.PAPERCLIP_INSTANCE_ID }); +} + +export async function readLegacyCloudRuntimeConfig(env: CloudInstanceEnv = process.env): Promise { + let raw: unknown; + try { + const file = await open(legacyCloudRuntimeConfigPath(env), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid file"); + const buffer = Buffer.alloc(65_537); + let total = 0; + while (total < buffer.length) { + const { bytesRead } = await file.read(buffer, total, buffer.length - total, total); + if (bytesRead === 0) break; + total += bytesRead; + } + if (total > 65_536) throw new Error("oversized file"); + raw = JSON.parse(buffer.subarray(0, total).toString("utf8")); + } finally { + await file.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw new Error("Legacy Cloud runtime configuration could not be read safely"); + return null; + } + const parsed = paperclipConfigSchema.safeParse(raw); + if (!parsed.success) throw new Error("Legacy Cloud runtime configuration is invalid"); + if (parsed.data.llm?.provider === "openai" && parsed.data.llm.apiKey?.trim() && !env.OPENAI_API_KEY?.trim()) { + throw new Error("Move the legacy OpenAI API key to OPENAI_API_KEY before initializing Cloud runtime configuration"); + } + return projectCloudRuntimeConfig(parsed.data); +} + +export async function initializeCloudRuntimeConfig(db: CloudRuntimeConfigDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) { + const env = options.runtimeEnv ?? process.env; + if (!isCloudManagedInstance(env)) return null; + setCloudRuntimeConfigSnapshot(null); + const service = cloudRuntimeConfigService(db); + const current = await service.inspectStored(); + const result = current + ? { seeded: false, current } + : await service.seedIfAbsent(await readLegacyCloudRuntimeConfig(env)); + setCloudRuntimeConfigSnapshot(result.current); + return result; +} diff --git a/server/src/services/plugin-loader.ts b/server/src/services/plugin-loader.ts index d4da285265b..1928ff94c0f 100644 --- a/server/src/services/plugin-loader.ts +++ b/server/src/services/plugin-loader.ts @@ -1,3 +1,4 @@ +import { resolveBundledPluginRuntimeLoader } from "./bundled-plugin-runtime.js"; import { finishServiceShutdown } from "../shutdown.js"; /** * PluginLoader — discovery, installation, and runtime activation of plugins. @@ -31,7 +32,7 @@ import { execFile } from "node:child_process"; import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { promisify } from "node:util"; +import { isDeepStrictEqual, promisify } from "node:util"; import type { Db } from "@paperclipai/db"; import { PLUGIN_RPC_ERROR_CODES } from "@paperclipai/plugin-sdk"; import type { @@ -271,6 +272,8 @@ function getDeclaredPageRoutePaths(manifest: PaperclipPluginManifestV1): string[ * Options for the plugin loader service. */ export interface PluginLoaderOptions { + resolvePackageRoot?: (plugin: PluginRecord) => string; + prebuiltOnly?: boolean; /** Image-owned deployment policy, checked before importing code and starting workers. */ assertPackageActivation?: (input: { pluginKey?: string; @@ -1157,6 +1160,8 @@ export function pluginLoader( enableLocalFilesystem = true, enableNpmDiscovery = true, assertPackageActivation, + resolvePackageRoot, + prebuiltOnly = false, } = options; const registry = pluginRegistryService(db); @@ -1211,6 +1216,7 @@ export function pluginLoader( installOptions: PluginInstallOptions, ): Promise { const { packageName, localPath, version, installDir } = installOptions; + if (prebuiltOnly && !localPath) throw new Error("Managed plugins must come from the current image catalog"); if (!packageName && !localPath) { throw new Error("Either packageName or localPath must be provided"); @@ -1286,7 +1292,7 @@ export function pluginLoader( const pkgJson = await readPackageJson(resolvedPackagePath); if (!pkgJson) throw new Error(`Missing package.json at ${resolvedPackagePath}`); - if (localPath) { + if (localPath && !prebuiltOnly) { await ensureLocalPluginBuilt(resolvedPackagePath, pkgJson); } @@ -1398,7 +1404,7 @@ export function pluginLoader( } assertPackageActivation?.({ packageRoot, installedPackagePath: plugin.packagePath, pluginKey: plugin.pluginKey, manifest, previousManifest: plugin.manifestJson }); - if (JSON.stringify(manifest) === JSON.stringify(plugin.manifestJson)) { + if (isDeepStrictEqual(manifest, plugin.manifestJson)) { return plugin; } @@ -1749,6 +1755,7 @@ export function pluginLoader( packagePath: discovered.source === "local-filesystem" ? discovered.packagePath : undefined, }, manifest, + { idempotent: prebuiltOnly }, ); if (!installed) { @@ -1803,12 +1810,7 @@ export function pluginLoader( newManifest: PaperclipPluginManifestV1; discovered: DiscoveredPlugin; }> { - const plugin = (await registry.getById(pluginId)) as { - id: string; - packageName: string; - packagePath: string | null; - manifestJson: PaperclipPluginManifestV1; - } | null; + const plugin = await registry.getById(pluginId); if (!plugin) throw new Error(`Plugin not found: ${pluginId}`); const oldManifest = plugin.manifestJson; @@ -1817,7 +1819,7 @@ export function pluginLoader( // For local-path installs, fall back to the stored packagePath so // `upgradePlugin` can re-read the manifest from disk without needing // the caller to re-supply the path every time. - localPath = plugin.packagePath ?? undefined, + localPath = resolvePackageRoot ? resolvePackageRoot(plugin) : plugin.packagePath ?? undefined, version, } = upgradeOptions; @@ -1887,6 +1889,7 @@ export function pluginLoader( // ----------------------------------------------------------------------- async cleanupInstallArtifacts(plugin: PluginRecord): Promise { + if (prebuiltOnly) return; const managedTargets = new Set(); const managedNodeModulesDir = resolveManagedInstallPackageDir(localPluginDir, plugin.packageName); const directManagedDir = path.join(localPluginDir, plugin.packageName); @@ -2262,12 +2265,14 @@ export function pluginLoader( // ------------------------------------------------------------------ // 1. Resolve worker entrypoint // ------------------------------------------------------------------ - const packageRoot = resolvePluginPackageRoot(activePlugin, localPluginDir); + const packageRoot = resolvePackageRoot ? resolvePackageRoot(activePlugin) : resolvePluginPackageRoot(activePlugin, localPluginDir); assertPackageActivation?.({ pluginKey, packageRoot, installedPackagePath: activePlugin.packagePath }); activePlugin = await refreshPluginManifestFromPackage(activePlugin, packageRoot); manifest = activePlugin.manifestJson; assertPackageActivation?.({ pluginKey, packageRoot, installedPackagePath: activePlugin.packagePath, manifest }); - const workerEntrypoint = resolveWorkerEntrypoint(activePlugin, localPluginDir); + const workerEntrypoint = resolvePackageRoot + ? path.resolve(packageRoot, manifest.entrypoints.worker) + : resolveWorkerEntrypoint(activePlugin, localPluginDir); // ------------------------------------------------------------------ // 2. Apply restricted database migrations before worker startup @@ -2334,7 +2339,7 @@ export function pluginLoader( env: buildPluginWorkerEnv({ manifest, packageName: activePlugin.packageName, - packagePath: activePlugin.packagePath, + packagePath: resolvePackageRoot ? packageRoot : activePlugin.packagePath, instanceInfo, }), // Authorize the worker to act on each configured company from its @@ -2345,10 +2350,9 @@ export function pluginLoader( proactiveCompanyScopes: configRows.map((row) => row.companyId), }; - // Repo-local plugin installs can resolve workspace TS sources at runtime - // (for example @paperclipai/shared exports). Run those workers through - // the tsx loader so first-party example plugins work in development. - if (activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) { + if (prebuiltOnly) { + workerOptions.execArgv = ["--import", resolveBundledPluginRuntimeLoader()]; + } else if (activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) { workerOptions.execArgv = ["--import", DEV_TSX_LOADER_PATH]; } diff --git a/server/src/services/plugin-registry.ts b/server/src/services/plugin-registry.ts index f7574bb293d..239c3c50490 100644 --- a/server/src/services/plugin-registry.ts +++ b/server/src/services/plugin-registry.ts @@ -1,3 +1,4 @@ +import { isDeepStrictEqual } from "node:util"; import { writePluginJobChange } from "./plugin-job-work-notifications.js"; import { asc, eq, isNull, ne, sql, and } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -31,6 +32,10 @@ import type { import { conflict, notFound } from "../errors.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +function sameBundledPluginRegistration(existing: typeof plugins.$inferSelect, packageName: string, manifest: PaperclipPluginManifestV1): boolean { + return existing.pluginKey === manifest.id && existing.packageName === packageName && existing.version === manifest.version && isDeepStrictEqual(existing.manifestJson, manifest); +} + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -140,10 +145,11 @@ export function pluginRegistryService(db: Db) { * manifest from the package. This method persists the plugin row and * assigns the next install order. */ - install: async (input: InstallPlugin, manifest: PaperclipPluginManifestV1) => { + install: async (input: InstallPlugin, manifest: PaperclipPluginManifestV1, options: { idempotent?: boolean } = {}) => { const existing = await getByKey(manifest.id); if (existing) { if (existing.status !== "uninstalled") { + if (options.idempotent && sameBundledPluginRegistration(existing, input.packageName, manifest)) return existing; throw conflict(`Plugin already installed: ${manifest.id}`); } @@ -153,7 +159,7 @@ export function pluginRegistryService(db: Db) { .update(plugins) .set({ packageName: input.packageName, - packagePath: input.packagePath ?? null, + packagePath: options.idempotent ? existing.packagePath : input.packagePath ?? null, version: manifest.version, apiVersion: manifest.apiVersion, categories: manifest.categories, @@ -170,7 +176,7 @@ export function pluginRegistryService(db: Db) { const installOrder = await nextInstallOrder(); try { - const rows = await db + const insertion = db .insert(plugins) .values({ pluginKey: manifest.id, @@ -182,9 +188,14 @@ export function pluginRegistryService(db: Db) { status: "installed" as PluginStatus, installOrder, packagePath: input.packagePath ?? null, - }) - .returning(); - return rows[0]; + }); + const rows = await (options.idempotent + ? insertion.onConflictDoNothing({ target: plugins.pluginKey }) + : insertion).returning(); + if (rows[0]) return rows[0]; + const concurrent = await getByKey(manifest.id); + if (concurrent && concurrent.status !== "uninstalled" && sameBundledPluginRegistration(concurrent, input.packageName, manifest)) return concurrent; + throw conflict(`Plugin registration differs from the current image: ${manifest.id}`); } catch (error) { if (isPluginKeyConflict(error)) { throw conflict(`Plugin already installed: ${manifest.id}`);