From cc42171101eb900788bec090da38d4dace48f4b4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 11:18:47 -0500 Subject: [PATCH] fix(config): persist managed Cloud runtime choices in Postgres Co-Authored-By: Paperclip --- doc/DATABASE.md | 9 + doc/DEVELOPING.md | 9 + doc/cloud-runtime-configuration.md | 109 +++++++ maps/agent-access-map/instance-operations.md | 9 + maps/feature-map/instance-operations.md | 10 + .../__tests__/cloud-runtime-config.test.ts | 274 ++++++++++++++++++ .../server-startup-feedback-export.test.ts | 57 ++++ server/src/config-file.ts | 3 + server/src/config.ts | 57 ++-- server/src/index.ts | 34 ++- server/src/paths.ts | 2 + .../services/cloud-runtime-config-snapshot.ts | 129 +++++++++ server/src/services/cloud-runtime-config.ts | 105 +++++++ 13 files changed, 773 insertions(+), 34 deletions(-) create mode 100644 doc/cloud-runtime-configuration.md create mode 100644 server/src/__tests__/cloud-runtime-config.test.ts create mode 100644 server/src/services/cloud-runtime-config-snapshot.ts create mode 100644 server/src/services/cloud-runtime-config.ts diff --git a/doc/DATABASE.md b/doc/DATABASE.md index c63a2bcb168..b77d312c735 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -1523,3 +1523,12 @@ It uses disposable embedded PostgreSQL by default. A disposable local database named `review_wait_benchmark...` can be supplied through `PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version. DataRow bytes exclude response metadata and transport overhead. + +## Cloud runtime configuration singleton + +The private `instance_settings` key `cloud-runtime-config:v1` holds a strictly +validated, versioned nonsecret configuration and revision. It is separate from +the ordinary instance settings API. Seed inserts preserve an existing winner; +replacements lock the row and reject stale revisions. Database/transaction +handles permit atomic migration receipts. Connection URLs, API keys and unknown +fields are excluded. See [field ownership and startup order](cloud-runtime-configuration.md). diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 27fa5f94a28..6b316a9ea02 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -3816,3 +3816,12 @@ in `_test` for `PAPERCLIP_MCP_SLEEP_TEST_DATABASE_URL`, `PAPERCLIP_DOT_RUNNER_TEST_DATABASE_URL`. The interaction-service regression suite also accepts `PAPERCLIP_INTERACTIONS_TEST_DATABASE_URL`. The suites migrate and write to those URLs; the sleep suite truncates company data between tests. + +## Cloud startup configuration + +Cloud stores nonsecret startup choices in a private PostgreSQL singleton. +Bootstrap connection credentials stay in the supplied environment. A bounded +one-time config-file seed preserves existing choices, then later starts use the +database without loading home `.env` files or repairing worktree config. See +[Cloud runtime configuration](cloud-runtime-configuration.md) for field ownership, +secret handling, initialization order and migration interfaces. diff --git a/doc/cloud-runtime-configuration.md b/doc/cloud-runtime-configuration.md new file mode 100644 index 00000000000..0f7ce7ed446 --- /dev/null +++ b/doc/cloud-runtime-configuration.md @@ -0,0 +1,109 @@ +# Cloud runtime configuration + +Cloud-managed servers store nonsecret startup and application choices in the +private `instance_settings` singleton `cloud-runtime-config:v1`. The row contains +`version: 1`, an increasing `revision`, and a strict allowlist of configuration +fields. It is separate from the normal instance settings row and is not exposed +by the instance settings API. + +The canonical managed-instance predicate applies: a nonblank Cloud tenant token +or the presence of `PAPERCLIP_MANAGED_CONFIG`, including a blank value, selects +this configuration path. Invalid managed feature documents still retain their +existing validation at application startup. + +## Startup and authority + +1. The hosting environment supplies `DATABASE_URL` and, if needed, + `DATABASE_MIGRATION_URL`. Cloud requires a PostgreSQL URL and never falls back + to embedded PostgreSQL. Credentials, trust keys, provider configuration and + deployment wiring needed before the database connection remain supplied by + the environment or the existing secret system. +2. After schema migration and database connection, startup reads the private + runtime configuration row. An existing invalid row or a database error stops + startup. Neither condition selects a legacy file or an old process snapshot. +3. If no row exists, startup imports the legacy config once. It uses + `PAPERCLIP_CONFIG` when supplied, otherwise the instance home config path. + It does not search working-directory ancestors. A truly absent file saves + an authoritative `config: null` marker. This retains the existing dynamic + environment/image defaults, including authentication and secret strictness. + The marker never enables a later file fallback. A present malformed, linked, nonregular, or oversized file + stops initialization. Reads and stored snapshots are limited to 64 KiB. +4. The seed uses an insert that does not replace an existing row. Concurrent + initializers read the committed winner. Later starts do not inspect the + legacy file, even when it is absent, corrupt, or changed. +5. Startup publishes a detached configuration snapshot, restores the signed + Cloud runtime identity, then initializes telemetry, secret-provider defaults, + authentication and application services. A saved telemetry opt-out therefore + takes effect before a telemetry client can be created. + +Existing operator environment overlays keep precedence over saved values. +Reading an overlaid value does not save it back to the database. Credentials and +connection URLs are never copied into this JSONB row. + +Cloud does not load instance or working-directory `.env` files and does not run +legacy worktree configuration repair. The hosting system must supply those +bootstrap variables directly. In particular, a database connection kept only +in `.env` cannot bootstrap this path. Self-hosted configuration and dotenv +behavior remain unchanged. + +## Field ownership + +The table lists every field consumed from the existing config file and the +remaining known schema fields. Environment-only `Config` fields, such as +heartbeat maintenance flags and feedback-export credentials, retain their +existing environment sources and are not frozen into the saved snapshot. + +| Config file fields | Cloud owner | +| --- | --- | +| `server.deploymentMode`, `exposure`, `bind`, `customBindHost`, `host`, `port`, `allowedHostnames`, `serveUi` | Saved runtime configuration, with existing environment overlays. | +| `auth.baseUrlMode`, `publicBaseUrl`, `disableSignUp` | Saved runtime configuration, with environment overlays and the signed Cloud identity restoration. Saved URLs cannot contain userinfo, query parameters, or fragments. | +| `database.backup.enabled`, `intervalMinutes`, `retentionDays`, `dir` | Saved runtime configuration, with existing backup environment overlays. | +| `database.mode`, `connectionString` | Cloud bootstrap requires the environment `DATABASE_URL`. The legacy URL is never saved. | +| `database.embeddedPostgresDataDir`, `embeddedPostgresPort` | Local embedded-database settings. Cloud does not use embedded PostgreSQL or these paths. | +| `storage.provider`, `localDisk.baseDir`, `s3.bucket`, `s3.region`, `s3.endpoint`, `s3.prefix`, `s3.forcePathStyle` | Saved runtime configuration, with existing environment overlays. Saved endpoints cannot contain userinfo, query parameters, or fragments. | +| `secrets.provider`, `strictMode`, `localEncrypted.keyFilePath` | Saved nonsecret provider choice and locator, with existing environment overlays. No key bytes are imported or saved. | +| `telemetry.enabled` | Saved choice, passed to the existing telemetry initialization and opt-out policy before client creation. | +| `llm.provider` | Saved provider metadata. | +| `llm.apiKey` | Never saved. The existing OpenAI model-discovery consumer uses `OPENAI_API_KEY` first. A legacy OpenAI key requires that environment credential before import, or startup stops. Claude key metadata has no server consumer. | +| `logging.mode`, `logDir`, `updates.checkEnabled` | Nonsecret legacy/CLI metadata is preserved. Server logging continues to use its existing environment configuration. | +| `$meta` | CLI file-write metadata is not application authority. The compatibility view synthesizes valid metadata. | +| Unknown extension keys | No server config consumer exists. They are excluded from the strict saved allowlist, including any unknown credential fields. | + +A saved path is a locator. This change does not make local storage objects, +backup bytes, logging files, or secret-key bytes durable. Their storage owners +and existing environment/secret mechanisms retain responsibility for those +bytes. Restart tests prove configuration restoration under a different home; +they do not prove durable file, secret, or backup storage. + +## Internal update and migration interface + +`cloudRuntimeConfigService(db)` provides `inspectStored(handle?)`, +`seedIfAbsent(config, handle?)`, and `replace(config, expectedRevision, handle?)`. +Each method accepts a database or transaction handle. A migration can compose a +config seed, related settings and a receipt in one transaction. + +Replacement validates the same allowlist, locks the singleton row, checks the +expected revision, and increases it. A stale revision fails. There is no new +public configuration endpoint or settings UI. Startup consumers use a snapshot; +a saved replacement takes effect on the next initialization/restart. + +`readLegacyCloudRuntimeConfig(env?)` returns the validated nonsecret candidate +without writing it. `initializeCloudRuntimeConfig` performs the standalone +one-time seed and publishes its committed snapshot. Integrated migration +receipts and deployed Cloud qualification are separate consumers of this +interface. + +## Verification + +`server/src/__tests__/cloud-runtime-config.test.ts` exercises real PostgreSQL +clients, concurrent writes, transaction rollback, changed homes, a fresh Node +process, environment precedence, secret exclusion, invalid data and database +failure. It uses embedded PostgreSQL by default. A dedicated disposable database +can be selected with `PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL` when the host +cannot start embedded PostgreSQL. That database must be reserved for this test. +The test applies the current schema before use. + +The startup orchestration controls in +`server-startup-feedback-export.test.ts` hold configuration initialization and +verify that telemetry, authentication and app construction wait for it. These +local controls do not claim a live Cloud rollout. diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md index 0f70b631c58..b58eb870b6e 100644 --- a/maps/agent-access-map/instance-operations.md +++ b/maps/agent-access-map/instance-operations.md @@ -91,6 +91,15 @@ Verification: `not-run` — assert the documented agent denial with an owned tes verify any listed agent request/result path separately. A delegated operation needs its own policy and access review. +Cloud runtime configuration uses a separate private PostgreSQL singleton and +adds no public read or mutation endpoint. Existing instance settings APIs do not +expose it. Bootstrap database credentials and trust keys remain in the supplied +environment; the strict saved allowlist excludes API keys and connection URLs. +Internal configuration updates require a matching revision and transaction lock. +[Database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) cover +seed/restart, credential exclusion, conflicts and rollback. These are local +controls, not agent authorization or deployed Cloud qualification. + Gaps: Settings APIs cover parts of configuration; pre-start database/binding/storage configuration is local operator tooling. diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md index 53d5520cdd1..daae7ef5955 100644 --- a/maps/feature-map/instance-operations.md +++ b/maps/feature-map/instance-operations.md @@ -63,6 +63,16 @@ unfinished application write must still prevent sleep after its client leaves. ### `settings-backup` +Cloud startup reads nonsecret application choices from a private, versioned +PostgreSQL singleton after a bounded one-time legacy config seed. A fresh +process and different home restore the saved choices without consulting a stale +`.env` file. Invalid saved data or an unavailable database prevents startup; +saved telemetry opt-out applies before client creation. The +[database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) and +[startup-order controls](../../server/src/__tests__/server-startup-feedback-export.test.ts) +cover these local boundaries. See [field ownership](../../doc/cloud-runtime-configuration.md). +These controls do not qualify a deployed Cloud rollout or durable file storage. + Automated: [experimental settings](../../ui/src/pages/InstanceExperimentalSettings.test.tsx) and [database backup routes](../../server/src/__tests__/instance-database-backups-routes.test.ts) cover their UI/API contracts. Managed adapters: change a visible adapter preference as an instance admin, diff --git a/server/src/__tests__/cloud-runtime-config.test.ts b/server/src/__tests__/cloud-runtime-config.test.ts new file mode 100644 index 00000000000..93f1d7900eb --- /dev/null +++ b/server/src/__tests__/cloud-runtime-config.test.ts @@ -0,0 +1,274 @@ +import fs from "node:fs"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, type Db } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { paperclipConfigSchema } from "@paperclipai/shared"; +import { loadCloudBootstrapConfig, loadConfig } from "../config.js"; +import { readConfigFile } from "../config-file.js"; +import { + CLOUD_RUNTIME_CONFIG_KEY, + cloudRuntimeConfigService, + initializeCloudRuntimeConfig, + readLegacyCloudRuntimeConfig, +} from "../services/cloud-runtime-config.js"; +import { + getCloudRuntimeConfigSnapshot, + projectCloudRuntimeConfig, + setCloudRuntimeConfigSnapshot, +} from "../services/cloud-runtime-config-snapshot.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const externalDatabaseUrl = process.env.PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL; +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; +let home: string; +let configPath: string; + +function legacyConfig() { + return { + $meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" }, + llm: { provider: "openai", apiKey: "legacy-openai-secret" }, + database: { + mode: "postgres", + connectionString: "postgres://legacy:database-secret@example.test/paperclip", + backup: { enabled: false, intervalMinutes: 73, retentionDays: 11, dir: "/tmp/legacy-backups" }, + }, + logging: { mode: "cloud", logDir: "/tmp/legacy-logs" }, + server: { deploymentMode: "authenticated", exposure: "private", port: 3210, serveUi: false, allowedHostnames: ["saved.example.test"] }, + auth: { baseUrlMode: "explicit", publicBaseUrl: "https://saved.example.test", disableSignUp: true }, + storage: { provider: "s3", s3: { bucket: "saved-bucket", region: "eu-west-1", prefix: "saved/", forcePathStyle: true } }, + secrets: { provider: "local_encrypted", strictMode: true, localEncrypted: { keyFilePath: "/tmp/legacy-master.key" } }, + telemetry: { enabled: false }, + updates: { checkEnabled: false }, + unsupportedSecret: "unknown-secret-value", + }; +} + +beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-runtime-config-")); + configPath = path.join(home, "config.json"); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_CONFIG", configPath); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-token"); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); + vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined); + vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", undefined); + vi.stubEnv("PAPERCLIP_AUTH_BASE_URL_MODE", undefined); + vi.stubEnv("PAPERCLIP_PUBLIC_URL", undefined); + vi.stubEnv("BETTER_AUTH_URL", undefined); + vi.stubEnv("BETTER_AUTH_BASE_URL", undefined); + vi.stubEnv("PAPERCLIP_ALLOWED_HOSTNAMES", undefined); + vi.stubEnv("PAPERCLIP_TAILNET_BIND_HOST", "100.100.0.1"); + vi.stubEnv("OPENAI_API_KEY", "environment-api-secret"); + setCloudRuntimeConfigSnapshot(null); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + setCloudRuntimeConfigSnapshot(null); + fs.rmSync(home, { recursive: true, force: true }); +}); + +describe("Cloud configuration file boundary", () => { + it.each(["null", "{", JSON.stringify({ unexpected: true })])("rejects malformed present legacy data %s", async (contents) => { + fs.writeFileSync(configPath, contents); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/Legacy Cloud runtime configuration/); + }); + + it("rejects oversized and linked legacy sources", async () => { + fs.writeFileSync(configPath, " ".repeat(65_537)); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/); + fs.rmSync(configPath); + const target = path.join(home, "target.json"); + fs.writeFileSync(target, JSON.stringify(legacyConfig())); + fs.symlinkSync(target, configPath); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/); + }); + + it("requires an environment credential for the legacy OpenAI consumer", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + vi.stubEnv("OPENAI_API_KEY", undefined); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/OPENAI_API_KEY/); + expect(() => readConfigFile()).toThrow(/not been initialized/); + }); + + it.each(["https://user:secret@saved.example.test", "https://saved.example.test?key=secret", "https://saved.example.test#secret"])("rejects secret-bearing URL %s", async (url) => { + const config = legacyConfig(); + config.auth.publicBaseUrl = url; + fs.writeFileSync(configPath, JSON.stringify(config)); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/configuration is invalid/); + }); + + it("loads the minimal Cloud bootstrap from environment only", () => { + fs.writeFileSync(configPath, "malformed"); + vi.stubEnv("DATABASE_URL", "postgres://bootstrap:bootstrap-secret@127.0.0.1:5432/runtime"); + expect(loadCloudBootstrapConfig()).toMatchObject({ databaseMode: "postgres", databaseUrl: process.env.DATABASE_URL }); + vi.stubEnv("DATABASE_URL", undefined); + expect(() => loadCloudBootstrapConfig()).toThrow(/requires DATABASE_URL/); + vi.stubEnv("DATABASE_URL", "sqlite:///tmp/db"); + expect(() => loadCloudBootstrapConfig()).toThrow(/must use PostgreSQL/); + }); + + it("preserves local config file behavior without managed signals", () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + expect(readConfigFile()?.llm?.apiKey).toBe("legacy-openai-secret"); + expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" }); + }); +}); + +describePostgres("Cloud runtime configuration in PostgreSQL", () => { + let database: Awaited>; + let first: Db; + let second: Db; + + beforeAll(async () => { + if (externalDatabaseUrl) { + await applyPendingMigrations(externalDatabaseUrl); + database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl, { scope: "database" }) }; + } else { + database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-config-db-"); + } + first = createDb(database.connectionString); + second = createDb(database.connectionString); + }, 90_000); + + beforeEach(async () => { + await first.delete(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)); + vi.stubEnv("DATABASE_URL", database.connectionString); + }); + + afterAll(async () => { await database?.cleanup(); }); + + it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])("seeds a fresh database with %s", async (signal) => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token"); + const result = await initializeCloudRuntimeConfig(first); + expect(result).toMatchObject({ seeded: true, current: { version: 1, revision: 0, config: null } }); + expect(await cloudRuntimeConfigService(second).inspectStored()).toEqual(result?.current); + expect(fs.existsSync(configPath)).toBe(false); + vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", "authenticated"); + vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://environment.example.test"); + expect(loadConfig()).toMatchObject({ databaseMode: "postgres", authBaseUrlMode: "explicit", secretsStrictMode: true, telemetryEnabled: true }); + }); + + it("imports existing choices without credentials and restores them from a second client and new home", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + const initial = await initializeCloudRuntimeConfig(first); + expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, databaseBackupEnabled: false, storageS3Bucket: "saved-bucket", authDisableSignUp: true }); + const [row] = await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)); + const persisted = JSON.stringify(row.general); + for (const secret of ["legacy-openai-secret", "database-secret", "unknown-secret-value", "environment-api-secret", database.connectionString]) { + expect(persisted).not.toContain(secret); + } + const replacementHome = fs.mkdtempSync(path.join(home, "replacement-")); + const replacementConfigDir = path.join(replacementHome, ".paperclip"); + fs.mkdirSync(replacementConfigDir); + const envPath = path.join(replacementConfigDir, ".env"); + const envContents = "PORT=9999\nPAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL=should-not-load\n"; + fs.writeFileSync(envPath, envContents); + vi.stubEnv("PAPERCLIP_HOME", replacementHome); + vi.stubEnv("PAPERCLIP_CONFIG", path.join(replacementConfigDir, "config.json")); + setCloudRuntimeConfigSnapshot(null); + const files = vi.spyOn(fs, "readFileSync"); + expect(await initializeCloudRuntimeConfig(second)).toEqual({ seeded: false, current: initial?.current }); + expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" }); + expect(files.mock.calls.map(([file]) => String(file)).filter(file => file.startsWith(replacementHome))).toEqual([]); + files.mockRestore(); + const root = fileURLToPath(new URL("../../../", import.meta.url)); + const childScript = path.join(home, "restore.mts"); + fs.writeFileSync(childScript, ` + import { createDb } from ${JSON.stringify(pathToFileURL(path.join(root, "packages/db/src/index.ts")).href)}; + import { initializeCloudRuntimeConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/services/cloud-runtime-config.ts")).href)}; + import { loadConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/config.ts")).href)}; + const db = createDb(process.env.DATABASE_URL); + await initializeCloudRuntimeConfig(db); + const config = loadConfig(); + console.log(JSON.stringify({ port: config.port, telemetryEnabled: config.telemetryEnabled, storageS3Bucket: config.storageS3Bucket, + envSentinel: process.env.PAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL ?? null })); + await db.$client.end({ timeout: 1 }); + `); + const child = await promisify(execFile)(process.execPath, [path.join(root, "server/node_modules/tsx/dist/cli.mjs"), childScript], { + cwd: replacementHome, + env: { ...process.env }, + timeout: 30_000, + }); + expect(JSON.parse(child.stdout.trim())).toEqual({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket", envSentinel: null }); + expect(fs.readFileSync(envPath, "utf8")).toBe(envContents); + }, 45_000); + + it("keeps environment overlays out of saved settings and ignores corrupt old files after seed", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + await initializeCloudRuntimeConfig(first); + vi.stubEnv("PORT", "4567"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "operator-bucket"); + vi.stubEnv("PAPERCLIP_AUTH_DISABLE_SIGN_UP", "false"); + fs.writeFileSync(configPath, "malformed legacy contents"); + await initializeCloudRuntimeConfig(second); + expect(loadConfig()).toMatchObject({ port: 4567, storageS3Bucket: "operator-bucket", authDisableSignUp: false, telemetryEnabled: false }); + expect((await cloudRuntimeConfigService(first).inspectStored())?.config).toMatchObject({ server: { port: 3210 }, storage: { s3: { bucket: "saved-bucket" } }, auth: { disableSignUp: true } }); + }); + + it("keeps one committed seed when two clients propose different initial values", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + const results = await Promise.all([ + cloudRuntimeConfigService(first).seedIfAbsent({ ...config, telemetry: { enabled: true } }), + cloudRuntimeConfigService(second).seedIfAbsent({ ...config, telemetry: { enabled: false } }), + ]); + expect(results.filter(result => result.seeded)).toHaveLength(1); + expect(results[0].current).toEqual(results[1].current); + expect(await cloudRuntimeConfigService(first).inspectStored()).toEqual(results[0].current); + }); + + it("rejects unknown or credential fields on the direct mutation path", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + await cloudRuntimeConfigService(first).seedIfAbsent(config); + await expect(cloudRuntimeConfigService(first).replace({ ...config, llm: { provider: "openai", apiKey: "must-not-store" } } as never, 0)).rejects.toThrow(/configuration is invalid/); + expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(0); + }); + + it("serializes revisions and rejects competing stale writes from two database clients", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + await cloudRuntimeConfigService(first).seedIfAbsent(config); + const results = await Promise.allSettled([ + cloudRuntimeConfigService(first).replace({ ...config, telemetry: { enabled: true } }, 0), + cloudRuntimeConfigService(second).replace({ ...config, telemetry: { enabled: false } }, 0), + ]); + expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1); + expect(results.filter(result => result.status === "rejected")).toHaveLength(1); + expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(1); + }); + + it("composes a seed with the caller transaction and rolls back both on failure", async () => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + await expect(first.transaction(async tx => { + await cloudRuntimeConfigService(first).seedIfAbsent(config, tx); + await tx.insert(instanceSettings).values({ singletonKey: "runtime-config-test-receipt", general: {} }); + throw new Error("injected migration failure"); + })).rejects.toThrow(/injected migration failure/); + expect(await cloudRuntimeConfigService(second).inspectStored()).toBeNull(); + expect(await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "runtime-config-test-receipt"))).toEqual([]); + }); + + it("fails closed on invalid database settings without consulting a valid legacy file", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 99, config: { secret: "hidden" } } }); + await expect(initializeCloudRuntimeConfig(second)).rejects.toThrow(/Persisted Cloud runtime configuration is invalid/); + expect(() => getCloudRuntimeConfigSnapshot()).toThrow(/not been initialized/); + }); + + it("does not fall back to a file or old snapshot when the database fails", async () => { + fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); + await initializeCloudRuntimeConfig(first); + const unavailable = createDb(database.connectionString); + await unavailable.$client.end({ timeout: 1 }); + await expect(initializeCloudRuntimeConfig(unavailable)).rejects.toThrow(); + expect(() => readConfigFile()).toThrow(/not been initialized/); + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 420a18bbc86..8d9e9f96cb9 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -20,6 +20,15 @@ import { gzipSync } from "node:zlib"; import { tmpdir } from "node:os"; import path from "node:path"; +const cloudRuntimeStartupMocks = vi.hoisted(() => ({ + initialize: vi.fn(async () => null), + bootstrap: vi.fn(), + telemetry: vi.fn(), +})); + +vi.mock("../services/cloud-runtime-config.js", () => ({ initializeCloudRuntimeConfig: cloudRuntimeStartupMocks.initialize })); +vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) })); + const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON; @@ -343,6 +352,7 @@ vi.mock("../services/native-runtime/native-safe-replacement.js", () => ({ vi.mock("../config.js", () => ({ loadConfig: loadConfigMock, + loadCloudBootstrapConfig: cloudRuntimeStartupMocks.bootstrap, })); vi.mock("../middleware/logger.js", () => ({ @@ -586,6 +596,8 @@ describe("startServer feedback export wiring", () => { process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210"; process.env.PAPERCLIP_AGENT_JWT_SECRET = "0123456789abcdef0123456789abcdef"; loadConfigMock.mockReturnValue(buildTestConfig()); + cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null); + cloudRuntimeStartupMocks.bootstrap.mockReturnValue(buildTestConfig()); resolveHeartbeatSchedulingSuppressionMock.mockReturnValue({ suppressed: false, reason: null, @@ -595,6 +607,47 @@ describe("startServer feedback export wiring", () => { process.env.BETTER_AUTH_SECRET = "test-secret"; }); + it("awaits saved Cloud configuration before telemetry, authentication and app construction", async () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + let release: (() => void) | undefined; + cloudRuntimeStartupMocks.initialize.mockImplementationOnce(() => new Promise(resolve => { + release = () => { + loadConfigMock.mockReturnValue(buildTestConfig({ telemetryEnabled: false, authDisableSignUp: true, port: 4321 })); + resolve(null); + }; + })); + try { + const starting = startServer(); + for (let index = 0; index < 30 && !release; index += 1) await new Promise(resolve => setImmediate(resolve)); + expect(release).toBeTypeOf("function"); + expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled(); + expect(createBetterAuthInstanceMock).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); + release!(); + const started = await starting; + await settleStartupHousekeeping(); + expect(cloudRuntimeStartupMocks.telemetry).toHaveBeenCalledExactlyOnceWith({ enabled: false }); + expect(createBetterAuthInstanceMock).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ authDisableSignUp: true, port: 4321 }), expect.anything()); + expect(started.listenPort).toBe(4321); + } finally { + release?.(); + vi.unstubAllEnvs(); + } + }); + + it("does not create telemetry or the app when saved Cloud configuration fails", async () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + cloudRuntimeStartupMocks.initialize.mockRejectedValueOnce(new Error("persisted config unavailable")); + try { + await expect(startServer()).rejects.toThrow("persisted config unavailable"); + expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled(); + expect(createBetterAuthInstanceMock).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + } + }); + it.each([false, true])("registers runner evidence independently from the backup scan (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); @@ -1509,6 +1562,8 @@ describe("startServer authenticated auth origin setup", () => { vi.clearAllMocks(); process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210"; loadConfigMock.mockReturnValue(buildTestConfig()); + cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null); + cloudRuntimeStartupMocks.bootstrap.mockReturnValue(buildTestConfig()); createBetterAuthInstanceMock.mockReturnValue({}); deriveAuthTrustedOriginsMock.mockReturnValue([]); process.env.BETTER_AUTH_SECRET = "test-secret"; @@ -1570,6 +1625,8 @@ describe("startServer PAPERCLIP_API_URL handling", () => { vi.clearAllMocks(); process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210"; loadConfigMock.mockReturnValue(buildTestConfig()); + cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null); + cloudRuntimeStartupMocks.bootstrap.mockReturnValue(buildTestConfig()); process.env.BETTER_AUTH_SECRET = "test-secret"; delete process.env.PAPERCLIP_API_URL; }); diff --git a/server/src/config-file.ts b/server/src/config-file.ts index 957e8d05881..d750793c204 100644 --- a/server/src/config-file.ts +++ b/server/src/config-file.ts @@ -8,6 +8,8 @@ import { } from "@paperclipai/shared"; import { ZodError } from "zod"; import { resolvePaperclipConfigPath } from "./paths.js"; +import { isCloudManagedInstance } from "./services/cloud-instance.js"; +import { getCloudRuntimeConfigSnapshot, toPaperclipConfig } from "./services/cloud-runtime-config-snapshot.js"; function formatConfigValidationError(error: ZodError): string { return error.issues @@ -19,6 +21,7 @@ function formatConfigValidationError(error: ZodError): string { } export function readConfigFile(): PaperclipConfig | null { + if (isCloudManagedInstance()) return toPaperclipConfig(getCloudRuntimeConfigSnapshot().config); const configPath = resolvePaperclipConfigPath(); if (!fs.existsSync(configPath)) return null; diff --git a/server/src/config.ts b/server/src/config.ts index dac9dba2142..133f28aff75 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -1,4 +1,6 @@ import { readConfigFile, resolveDeploymentMode } from "./config-file.js"; +import { isCloudManagedInstance } from "./services/cloud-instance.js"; +import type { PaperclipConfig } from "@paperclipai/shared"; import { parseChatWebhookPublicBaseUrl } from "./chat-webhook-public-url.js"; import { execFileSync } from "node:child_process"; import { existsSync, realpathSync } from "node:fs"; @@ -32,27 +34,29 @@ import { resolveHomeAwarePath, } from "./home-paths.js"; -const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath(); -if (existsSync(PAPERCLIP_ENV_FILE_PATH)) { - loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true }); -} +if (!isCloudManagedInstance()) { + const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath(); + if (existsSync(PAPERCLIP_ENV_FILE_PATH)) { + loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true }); + } -const CWD_ENV_PATH = resolve(process.cwd(), ".env"); -const cwdEnvExists = existsSync(CWD_ENV_PATH); -const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH) - ? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH) - : CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH; -if (shouldLoadWorkingDirectoryEnv({ - cwdEnvExists, - isPaperclipEnvFile: isSameFile, -})) { - loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true }); -} + const CWD_ENV_PATH = resolve(process.cwd(), ".env"); + const cwdEnvExists = existsSync(CWD_ENV_PATH); + const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH) + ? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH) + : CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH; + if (shouldLoadWorkingDirectoryEnv({ + cwdEnvExists, + isPaperclipEnvFile: isSameFile, + })) { + loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true }); + } -if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") { - applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH))); + if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") { + applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH))); + } + maybeRepairLegacyWorktreeConfigAndEnvFiles(); } -maybeRepairLegacyWorktreeConfigAndEnvFiles(); const TAILSCALE_DETECT_TIMEOUT_MS = 3000; @@ -147,13 +151,12 @@ function detectTailnetBindHost(): string | undefined { } } -export function loadConfig(): Config { - const fileConfig = readConfigFile(); +export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()): Config { const fileDatabaseMode = - (fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode; + (isCloudManagedInstance() || fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode; const fileDbUrl = - fileDatabaseMode === "postgres" + !isCloudManagedInstance() && fileDatabaseMode === "postgres" ? fileConfig?.database.connectionString : undefined; const fileDatabaseBackup = fileConfig?.database.backup; @@ -460,3 +463,13 @@ export function loadConfig(): Config { announcementsFeedUrl: process.env.PAPERCLIP_ANNOUNCEMENTS_FEED_URL?.trim() || "https://pages.paperclip.ing/announcements/v1/current.json", }; } + +export function loadCloudBootstrapConfig(): Config { + if (!isCloudManagedInstance()) throw new Error("Cloud bootstrap configuration requires a managed instance"); + const url = process.env.DATABASE_URL?.trim(); + if (!url) throw new Error("Cloud bootstrap requires DATABASE_URL from the environment"); + let parsed: URL; + try { parsed = new URL(url); } catch { throw new Error("Cloud bootstrap DATABASE_URL must use PostgreSQL"); } + if (!["postgres:", "postgresql:"].includes(parsed.protocol)) throw new Error("Cloud bootstrap DATABASE_URL must use PostgreSQL"); + return { ...loadConfig(null), databaseMode: "postgres", databaseUrl: url }; +} diff --git a/server/src/index.ts b/server/src/index.ts index 1973e68113d..b08485754f4 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -104,7 +104,8 @@ import { } from "@paperclipai/db"; import detectPort from "detect-port"; import { createApp } from "./app.js"; -import { loadConfig } from "./config.js"; +import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js"; +import { loadCloudBootstrapConfig, loadConfig } from "./config.js"; import { logger } from "./middleware/logger.js"; import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js"; import { setStartupRecoveryPhase } from "./startup-recovery-state.js"; @@ -252,6 +253,19 @@ async function endDatabaseClient(client: unknown, timeoutSeconds: number): Promi await sql.end({ timeout: timeoutSeconds }); } +function initializeRuntimeConfigConsumers(config: ReturnType): void { + initTelemetry({ enabled: config.telemetryEnabled }); + if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) { + process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider; + } + if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) { + process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false"; + } + if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) { + process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath; + } +} + export async function startServer(): Promise { const startupDatabase: StartupDatabaseTeardown = { close: null }; try { @@ -279,17 +293,8 @@ async function startServerWithDatabaseTeardown( // sentry.ts. await sentryReady; ensureDecisionSigningSecret(); - let config = loadConfig(); - initTelemetry({ enabled: config.telemetryEnabled }); - if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) { - process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider; - } - if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) { - process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false"; - } - if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) { - process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath; - } + let config = isCloudManagedInstance() ? loadCloudBootstrapConfig() : loadConfig(); + if (!isCloudManagedInstance()) initializeRuntimeConfigConsumers(config); type MigrationSummary = | "skipped" @@ -714,11 +719,16 @@ async function startServerWithDatabaseTeardown( setSentryRunLinkResolver(createSentryRunLinkResolver(db)); await initializeAdapterSettings(db as any); + await initializeCloudRuntimeConfig(db as any); + if (isCloudManagedInstance()) config = loadConfig(); + // A claimed warm-pool stack may restart while its provider environment still // names the pool host. Restore the signed, durable identity before Better // Auth, routes, or child-runtime configuration capture any public URL. const restoredCloudRuntimeIdentity = await initializeCloudRuntimeIdentity(db as any); if (restoredCloudRuntimeIdentity) config = loadConfig(); + if (isCloudManagedInstance()) initializeRuntimeConfigConsumers(config); + const isWarmStandby = await createCloudWarmStandby(db as any); if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) { diff --git a/server/src/paths.ts b/server/src/paths.ts index 21856a6946e..5ed3971d4fd 100644 --- a/server/src/paths.ts +++ b/server/src/paths.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { resolveDefaultConfigPath } from "./home-paths.js"; +import { isCloudManagedInstance } from "./services/cloud-instance.js"; const PAPERCLIP_CONFIG_BASENAME = "config.json"; const PAPERCLIP_ENV_FILENAME = ".env"; @@ -26,6 +27,7 @@ function findConfigFileFromAncestors(startDir: string): string | null { export function resolvePaperclipConfigPath(overridePath?: string): string { if (overridePath) return path.resolve(overridePath); if (process.env.PAPERCLIP_CONFIG) return path.resolve(process.env.PAPERCLIP_CONFIG); + if (isCloudManagedInstance()) return resolveDefaultConfigPath(); return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath(); } diff --git a/server/src/services/cloud-runtime-config-snapshot.ts b/server/src/services/cloud-runtime-config-snapshot.ts new file mode 100644 index 00000000000..e97cd534b24 --- /dev/null +++ b/server/src/services/cloud-runtime-config-snapshot.ts @@ -0,0 +1,129 @@ +import { + authConfigSchema, + databaseBackupConfigSchema, + llmConfigSchema, + loggingConfigSchema, + paperclipConfigSchema, + secretsConfigSchema, + secretsLocalEncryptedConfigSchema, + serverConfigSchema, + storageConfigSchema, + storageLocalDiskConfigSchema, + storageS3ConfigSchema, + type PaperclipConfig, +} from "@paperclipai/shared"; +import { z } from "zod"; + +const publicUrl = z.string().url().refine((value) => { + const url = new URL(value); + return !url.username && !url.password && !url.search && !url.hash; +}); + +export const cloudRuntimeConfigSchema = z.object({ + server: serverConfigSchema.strict(), + auth: authConfigSchema.extend({ publicBaseUrl: publicUrl.optional() }).strict(), + backup: databaseBackupConfigSchema.strict(), + storage: storageConfigSchema.extend({ + localDisk: storageLocalDiskConfigSchema.strict(), + s3: storageS3ConfigSchema.extend({ endpoint: publicUrl.optional() }).strict(), + }).strict(), + secrets: secretsConfigSchema.extend({ + localEncrypted: secretsLocalEncryptedConfigSchema.strict(), + }).strict(), + telemetry: z.object({ enabled: z.boolean() }).strict(), + logging: loggingConfigSchema.strict(), + updates: z.object({ checkEnabled: z.boolean() }).strict().optional(), + llm: llmConfigSchema.omit({ apiKey: true }).strict().optional(), +}).strict(); + +export type CloudRuntimeConfig = z.infer; +export type StoredCloudRuntimeConfig = { version: 1; revision: number; config: CloudRuntimeConfig | null }; + +const storedSchema = z.object({ + version: z.literal(1), + revision: z.number().int().nonnegative(), + config: cloudRuntimeConfigSchema.nullable(), +}).strict(); + +export function parseStoredCloudRuntimeConfig(value: unknown): StoredCloudRuntimeConfig { + const parsed = storedSchema.safeParse(value); + if (!parsed.success || Buffer.byteLength(JSON.stringify(parsed.data), "utf8") > 65_536) { + throw new Error("Persisted Cloud runtime configuration is invalid"); + } + const full = parsed.data.config === null ? { success: true } : paperclipConfigSchema.safeParse(toPaperclipConfig(parsed.data.config)); + if (!full.success) throw new Error("Persisted Cloud runtime configuration is invalid"); + return parsed.data; +} + +export function toPaperclipConfig(config: CloudRuntimeConfig | null): PaperclipConfig | null { + if (config === null) return null; + const { backup, ...settings } = structuredClone(config); + return { + $meta: { version: 1, updatedAt: "1970-01-01T00:00:00.000Z", source: "configure" }, + database: { + mode: "postgres", + embeddedPostgresDataDir: "", + embeddedPostgresPort: 54329, + backup, + }, + ...settings, + }; +} + +export function projectCloudRuntimeConfig(source: PaperclipConfig): CloudRuntimeConfig { + const raw = { + server: { + deploymentMode: source.server.deploymentMode, + exposure: source.server.exposure, + bind: source.server.bind, + customBindHost: source.server.customBindHost, + host: source.server.host, + port: source.server.port, + allowedHostnames: source.server.allowedHostnames, + serveUi: source.server.serveUi, + }, + auth: { + baseUrlMode: source.auth.baseUrlMode, + publicBaseUrl: source.auth.publicBaseUrl, + disableSignUp: source.auth.disableSignUp, + }, + backup: { + enabled: source.database.backup.enabled, + intervalMinutes: source.database.backup.intervalMinutes, + retentionDays: source.database.backup.retentionDays, + dir: source.database.backup.dir, + }, + storage: { + provider: source.storage.provider, + localDisk: { baseDir: source.storage.localDisk.baseDir }, + s3: { + bucket: source.storage.s3.bucket, + region: source.storage.s3.region, + endpoint: source.storage.s3.endpoint, + prefix: source.storage.s3.prefix, + forcePathStyle: source.storage.s3.forcePathStyle, + }, + }, + secrets: { + provider: source.secrets.provider, + strictMode: source.secrets.strictMode, + localEncrypted: { keyFilePath: source.secrets.localEncrypted.keyFilePath }, + }, + telemetry: { enabled: source.telemetry.enabled }, + logging: { mode: source.logging.mode, logDir: source.logging.logDir }, + updates: source.updates ? { checkEnabled: source.updates.checkEnabled } : undefined, + llm: source.llm ? { provider: source.llm.provider } : undefined, + }; + return parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config: raw }).config!; +} + +let snapshot: StoredCloudRuntimeConfig | null = null; + +export function setCloudRuntimeConfigSnapshot(value: StoredCloudRuntimeConfig | null): void { + snapshot = value ? parseStoredCloudRuntimeConfig(value) : null; +} + +export function getCloudRuntimeConfigSnapshot(): StoredCloudRuntimeConfig { + if (!snapshot) throw new Error("Cloud runtime configuration has not been initialized"); + return structuredClone(snapshot); +} diff --git a/server/src/services/cloud-runtime-config.ts b/server/src/services/cloud-runtime-config.ts new file mode 100644 index 00000000000..61082ae07c7 --- /dev/null +++ b/server/src/services/cloud-runtime-config.ts @@ -0,0 +1,105 @@ +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { instanceSettings, type Db } from "@paperclipai/db"; +import { paperclipConfigSchema } from "@paperclipai/shared"; +import { resolvePaperclipConfigPathForInstance } from "@paperclipai/shared/home-paths"; +import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js"; +import { + parseStoredCloudRuntimeConfig, + projectCloudRuntimeConfig, + setCloudRuntimeConfigSnapshot, + type CloudRuntimeConfig, + type StoredCloudRuntimeConfig, +} from "./cloud-runtime-config-snapshot.js"; + +export const CLOUD_RUNTIME_CONFIG_KEY = "cloud-runtime-config:v1"; +type Transaction = Parameters[0]>[0]; +export type CloudRuntimeConfigDb = Pick; + +export function cloudRuntimeConfigService(db: CloudRuntimeConfigDb) { + async function inspectStored(handle: CloudRuntimeConfigDb = db): Promise { + const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings) + .where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).limit(1); + return row ? parseStoredCloudRuntimeConfig(row.general) : null; + } + + async function seedIfAbsent(config: CloudRuntimeConfig | null, handle: CloudRuntimeConfigDb = db) { + const next = parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config }); + const inserted = await handle.insert(instanceSettings) + .values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: next }) + .onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id }); + const current = await inspectStored(handle); + if (!current) throw new Error("Cloud runtime configuration seed was not persisted"); + return { seeded: inserted.length > 0, current }; + } + + return { + inspectStored, + seedIfAbsent, + async replace(config: CloudRuntimeConfig | null, expectedRevision: number, handle: CloudRuntimeConfigDb = db) { + const validated = parseStoredCloudRuntimeConfig({ version: 1, revision: expectedRevision, config }); + return handle.transaction(async (tx) => { + const [row] = await tx.select().from(instanceSettings) + .where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).for("update"); + if (!row) throw new Error("Cloud runtime configuration has not been initialized"); + const current = parseStoredCloudRuntimeConfig(row.general); + if (current.revision !== expectedRevision) throw new Error("Cloud runtime configuration revision conflict"); + const next = { ...validated, revision: current.revision + 1 }; + await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id)); + return next; + }); + }, + }; +} + +export function legacyCloudRuntimeConfigPath(env: CloudInstanceEnv = process.env): string { + return env.PAPERCLIP_CONFIG + ? path.resolve(env.PAPERCLIP_CONFIG) + : resolvePaperclipConfigPathForInstance({ homeDir: env.PAPERCLIP_HOME, instanceId: env.PAPERCLIP_INSTANCE_ID }); +} + +export async function readLegacyCloudRuntimeConfig(env: CloudInstanceEnv = process.env): Promise { + let raw: unknown; + try { + const file = await open(legacyCloudRuntimeConfigPath(env), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid file"); + const buffer = Buffer.alloc(65_537); + let total = 0; + while (total < buffer.length) { + const { bytesRead } = await file.read(buffer, total, buffer.length - total, total); + if (bytesRead === 0) break; + total += bytesRead; + } + if (total > 65_536) throw new Error("oversized file"); + raw = JSON.parse(buffer.subarray(0, total).toString("utf8")); + } finally { + await file.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw new Error("Legacy Cloud runtime configuration could not be read safely"); + return null; + } + const parsed = paperclipConfigSchema.safeParse(raw); + if (!parsed.success) throw new Error("Legacy Cloud runtime configuration is invalid"); + if (parsed.data.llm?.provider === "openai" && parsed.data.llm.apiKey?.trim() && !env.OPENAI_API_KEY?.trim()) { + throw new Error("Move the legacy OpenAI API key to OPENAI_API_KEY before initializing Cloud runtime configuration"); + } + return projectCloudRuntimeConfig(parsed.data); +} + +export async function initializeCloudRuntimeConfig(db: CloudRuntimeConfigDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) { + const env = options.runtimeEnv ?? process.env; + if (!isCloudManagedInstance(env)) return null; + setCloudRuntimeConfigSnapshot(null); + const service = cloudRuntimeConfigService(db); + const current = await service.inspectStored(); + const result = current + ? { seeded: false, current } + : await service.seedIfAbsent(await readLegacyCloudRuntimeConfig(env)); + setCloudRuntimeConfigSnapshot(result.current); + return result; +}