diff --git a/cli/src/checks/secrets-check.ts b/cli/src/checks/secrets-check.ts index 49c6a90b28e..a60d8e4e863 100644 --- a/cli/src/checks/secrets-check.ts +++ b/cli/src/checks/secrets-check.ts @@ -135,11 +135,20 @@ export function secretsCheck(config: PaperclipConfig, configPath?: string): Chec }; } + const keyMode = fs.statSync(keyFilePath).mode & 0o777; + const permissionWarning = + (keyMode & 0o077) !== 0 + ? `; key file permissions are ${keyMode.toString(8)} (run chmod 600 ${keyFilePath})` + : ""; + return withStrictModeNote( { name: "Secrets adapter", - status: "pass", - message: `Local encrypted provider configured with key file ${keyFilePath}`, + status: permissionWarning ? "warn" : "pass", + message: `Local encrypted provider configured with key file ${keyFilePath}${permissionWarning}`, + repairHint: permissionWarning + ? "Restrict the local encrypted secrets key file to owner read/write permissions" + : undefined, }, config, ); diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 23abd32d415..aa112581cae 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -171,6 +171,8 @@ For local/default installs, the active provider is `local_encrypted`: - Secret material is encrypted at rest with a local master key. - Default key file: `~/.paperclip/instances/default/secrets/master.key` (auto-created if missing). - CLI config location: `~/.paperclip/instances/default/config.json` under `secrets.localEncrypted.keyFilePath`. +- Backup/restore requires both the database metadata and the local master key file; either artifact alone is insufficient. +- The server best-effort enforces `0600` key file permissions and provider health reports permission warnings. Optional overrides: diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index d95bb04dd78..1dd0e384e59 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -462,6 +462,7 @@ Agent env vars now support secret references. By default, secret values are stor - Default local key path: `~/.paperclip/instances/default/secrets/master.key` - Override key material directly: `PAPERCLIP_SECRETS_MASTER_KEY` - Override key file path: `PAPERCLIP_SECRETS_MASTER_KEY_FILE` +- Back up the key file and database together; either one alone is not enough to restore local encrypted secrets. Strict mode (recommended outside local trusted machines): @@ -470,12 +471,14 @@ PAPERCLIP_SECRETS_STRICT_MODE=true ``` When strict mode is enabled, sensitive env keys (for example `*_API_KEY`, `*_TOKEN`, `*_SECRET`) must use secret references instead of inline plain values. +Authenticated deployments default strict mode on unless explicitly overridden. CLI configuration support: - `pnpm paperclipai onboard` writes a default `secrets` config section (`local_encrypted`, strict mode off, key file path set) and creates a local key file when needed. - `pnpm paperclipai configure --section secrets` lets you update provider/strict mode/key path and creates the local key file when needed. - `pnpm paperclipai doctor` validates secrets adapter configuration and can create a missing local key file with `--repair`. +- Provider health is available at `GET /api/companies/:companyId/secret-providers/health` and reports local key permission warnings plus backup guidance. Migration helper for existing inline env secrets: diff --git a/docs/api/secrets.md b/docs/api/secrets.md index 49a36e0e434..8808463682d 100644 --- a/docs/api/secrets.md +++ b/docs/api/secrets.md @@ -25,10 +25,36 @@ POST /api/companies/{companyId}/secrets The value is encrypted at rest. Only the secret ID and metadata are returned. +To link a provider-owned secret without copying the value into Paperclip, create +an external-reference secret: + +```json +{ + "name": "prod-stripe-key", + "provider": "aws_secrets_manager", + "managedMode": "external_reference", + "externalRef": "arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/prod/stripe", + "providerVersionRef": "version-id-or-label" +} +``` + +Paperclip stores the provider reference and a non-sensitive fingerprint only. +The value is resolved, when the provider is configured, through the server +runtime path that enforces binding context and records access events. + +## Provider Health + +``` +GET /api/companies/{companyId}/secret-providers/health +``` + +Returns provider setup diagnostics, warnings, and local backup guidance. Health +responses must not include secret values or provider credentials. + ## Update Secret ``` -PATCH /api/secrets/{secretId} +POST /api/secrets/{secretId}/rotate { "value": "sk-ant-new-value..." } diff --git a/docs/deploy/secrets.md b/docs/deploy/secrets.md index 3ef1c689f85..46fabd67ceb 100644 --- a/docs/deploy/secrets.md +++ b/docs/deploy/secrets.md @@ -14,6 +14,13 @@ Secrets are encrypted with a local master key stored at: ``` This key is auto-created during onboarding. The key never leaves your machine. +Paperclip best-effort enforces `0600` permissions when it creates or loads the +key file. `paperclipai doctor` and the provider health API warn when the file is +readable by group or other users. + +Back up the key file together with database backups. A database backup without +the key cannot decrypt local secrets, and a key backup without the database +metadata is not enough to restore named secret versions. ## Configuration @@ -55,6 +62,21 @@ PAPERCLIP_SECRETS_STRICT_MODE=true Recommended for any deployment beyond local trusted. +Authenticated deployments default strict mode on unless explicitly overridden by +configuration or `PAPERCLIP_SECRETS_STRICT_MODE=false`. + +## External References + +Provider-owned secrets can be linked without copying values into Paperclip by +using `managedMode: "external_reference"` plus a provider `externalRef`. +Paperclip stores metadata and a non-sensitive fingerprint, never the value. +Runtime resolution remains server-side and binding-enforced. + +The built-in AWS, GCP, and Vault provider IDs currently accept external +reference metadata, but runtime resolution requires provider configuration in the +deployment. Their provider health check reports this as a warning until +configured. + ## Migrating Inline Secrets If you have existing agents with inline API keys in their config, migrate them to encrypted secret refs: diff --git a/packages/shared/src/types/secrets.ts b/packages/shared/src/types/secrets.ts index ebffb18706c..cd1b566623c 100644 --- a/packages/shared/src/types/secrets.ts +++ b/packages/shared/src/types/secrets.ts @@ -53,6 +53,9 @@ export interface SecretProviderDescriptor { id: SecretProvider; label: string; requiresExternalRef: boolean; + supportsManagedValues?: boolean; + supportsExternalReferences?: boolean; + configured?: boolean; } export interface CompanySecretVersion { diff --git a/packages/shared/src/validators/secret.ts b/packages/shared/src/validators/secret.ts index 18848f831f7..74ec04fe500 100644 --- a/packages/shared/src/validators/secret.ts +++ b/packages/shared/src/validators/secret.ts @@ -31,16 +31,35 @@ export const createSecretSchema = z.object({ key: z.string().min(1).regex(/^[a-zA-Z0-9_.-]+$/).optional(), provider: z.enum(SECRET_PROVIDERS).optional(), managedMode: z.enum(SECRET_MANAGED_MODES).optional(), - value: z.string().min(1), + value: z.string().min(1).optional().nullable(), description: z.string().optional().nullable(), externalRef: z.string().optional().nullable(), providerMetadata: z.record(z.unknown()).optional().nullable(), + providerVersionRef: z.string().optional().nullable(), +}).superRefine((value, ctx) => { + if ((value.managedMode ?? "paperclip_managed") === "external_reference") { + if (!value.externalRef?.trim()) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["externalRef"], + message: "External reference secrets require externalRef", + }); + } + return; + } + if (!value.value?.trim()) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["value"], + message: "Managed secrets require value", + }); + } }); export type CreateSecret = z.infer; export const rotateSecretSchema = z.object({ - value: z.string().min(1), + value: z.string().min(1).optional().nullable(), externalRef: z.string().optional().nullable(), providerVersionRef: z.string().optional().nullable(), }); diff --git a/server/src/__tests__/secret-provider-registry.test.ts b/server/src/__tests__/secret-provider-registry.test.ts new file mode 100644 index 00000000000..207c5ea127c --- /dev/null +++ b/server/src/__tests__/secret-provider-registry.test.ts @@ -0,0 +1,70 @@ +import { randomBytes } from "node:crypto"; +import { chmodSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { checkSecretProviders, listSecretProviders } from "../secrets/provider-registry.js"; + +describe("secret provider registry", () => { + const previousKeyFile = process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; + const previousMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + const tmpDirs: string[] = []; + + afterEach(() => { + if (previousKeyFile === undefined) { + delete process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; + } else { + process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = previousKeyFile; + } + if (previousMasterKey === undefined) { + delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; + } else { + process.env.PAPERCLIP_SECRETS_MASTER_KEY = previousMasterKey; + } + for (const dir of tmpDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("describes managed and external-reference provider capabilities", () => { + const descriptors = listSecretProviders(); + + expect(descriptors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "local_encrypted", + supportsManagedValues: true, + supportsExternalReferences: false, + configured: true, + }), + expect.objectContaining({ + id: "aws_secrets_manager", + supportsManagedValues: false, + supportsExternalReferences: true, + configured: false, + }), + ]), + ); + }); + + it("warns when the local encrypted key file is readable by group or others", async () => { + const dir = path.join(os.tmpdir(), `paperclip-secret-provider-${randomBytes(6).toString("hex")}`); + tmpDirs.push(dir); + mkdirSync(dir, { recursive: true }); + const keyFile = path.join(dir, "master.key"); + writeFileSync(keyFile, randomBytes(32).toString("base64"), { encoding: "utf8", mode: 0o644 }); + chmodSync(keyFile, 0o644); + process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = keyFile; + delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; + + const checks = await checkSecretProviders(); + const local = checks.find((check) => check.provider === "local_encrypted"); + + expect(local).toMatchObject({ + status: "warn", + details: { keyFilePath: keyFile }, + }); + expect(local?.warnings?.join("\n")).toContain("chmod 600"); + expect(local?.backupGuidance?.join("\n")).toContain("database"); + }); +}); diff --git a/server/src/__tests__/secrets-routes.test.ts b/server/src/__tests__/secrets-routes.test.ts new file mode 100644 index 00000000000..140e77c5761 --- /dev/null +++ b/server/src/__tests__/secrets-routes.test.ts @@ -0,0 +1,67 @@ +import express from "express"; +import request from "supertest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { secretRoutes } from "../routes/secrets.js"; +import { errorHandler } from "../middleware/error-handler.js"; + +const mockSecretService = vi.hoisted(() => ({ + listProviders: vi.fn(), + checkProviders: vi.fn(), +})); +const mockLogActivity = vi.hoisted(() => vi.fn()); + +vi.mock("../services/index.js", () => ({ + secretService: () => mockSecretService, + logActivity: mockLogActivity, +})); + +function createApp() { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + (req as any).actor = { + type: "board", + userId: "user-1", + source: "session", + companyIds: ["company-1"], + memberships: [{ companyId: "company-1", status: "active", membershipRole: "admin" }], + }; + next(); + }); + app.use("/api", secretRoutes({} as any)); + app.use(errorHandler); + return app; +} + +describe("secret routes", () => { + beforeEach(() => { + mockSecretService.listProviders.mockReset(); + mockSecretService.checkProviders.mockReset(); + mockLogActivity.mockReset(); + }); + + it("returns provider health checks for board callers with company access", async () => { + mockSecretService.checkProviders.mockResolvedValue([ + { + provider: "local_encrypted", + status: "ok", + message: "Local encrypted provider configured", + backupGuidance: ["Back up the key file together with database backups."], + }, + ]); + + const res = await request(createApp()).get("/api/companies/company-1/secret-providers/health"); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + providers: [ + { + provider: "local_encrypted", + status: "ok", + message: "Local encrypted provider configured", + backupGuidance: ["Back up the key file together with database backups."], + }, + ], + }); + }); +}); diff --git a/server/src/__tests__/secrets-service.test.ts b/server/src/__tests__/secrets-service.test.ts index 2f64fbbe926..2d3571dd247 100644 --- a/server/src/__tests__/secrets-service.test.ts +++ b/server/src/__tests__/secrets-service.test.ts @@ -1,5 +1,9 @@ import { randomUUID } from "node:crypto"; +import { mkdirSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; import { companies, companySecretBindings, @@ -23,8 +27,12 @@ if (!embeddedPostgresSupport.supported) { describeEmbeddedPostgres("secretService", () => { let stopDb: (() => Promise) | null = null; let db!: ReturnType; + const previousKeyFile = process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; + const secretsTmpDir = path.join(os.tmpdir(), `paperclip-secrets-service-${randomUUID()}`); beforeAll(async () => { + mkdirSync(secretsTmpDir, { recursive: true }); + process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = path.join(secretsTmpDir, "master.key"); const started = await startEmbeddedPostgresTestDatabase("secrets-service"); stopDb = started.stop; db = createDb(started.connectionString); @@ -40,6 +48,12 @@ describeEmbeddedPostgres("secretService", () => { afterAll(async () => { await stopDb?.(); + if (previousKeyFile === undefined) { + delete process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; + } else { + process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = previousKeyFile; + } + rmSync(secretsTmpDir, { recursive: true, force: true }); }); async function seedCompany(name = "Acme") { @@ -141,4 +155,37 @@ describeEmbeddedPostgres("secretService", () => { expect(events.map((event) => event.outcome).sort()).toEqual(["failure", "success"]); expect(JSON.stringify(events)).not.toContain("runtime-secret"); }); + + it("stores external references without requiring or persisting secret values", async () => { + const companyId = await seedCompany(); + const svc = secretService(db); + + const secret = await svc.create(companyId, { + name: `external-${randomUUID()}`, + provider: "aws_secrets_manager", + managedMode: "external_reference", + externalRef: "arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/test", + providerVersionRef: "version-1", + }); + + expect(secret.managedMode).toBe("external_reference"); + expect(secret.externalRef).toBe("arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/test"); + + const versions = await db + .select() + .from(companySecretVersions) + .where(eq(companySecretVersions.secretId, secret.id)); + expect(versions).toHaveLength(1); + expect(versions[0]?.providerVersionRef).toBe("version-1"); + expect(JSON.stringify(versions[0])).not.toContain("runtime-secret"); + expect(JSON.stringify(versions[0])).not.toContain("sk-"); + + await expect( + svc.resolveSecretValue(companyId, secret.id, "latest", { + consumerType: "system", + consumerId: "system", + configPath: "env.EXTERNAL_SECRET", + }), + ).rejects.toThrow(/not bound/i); + }); }); diff --git a/server/src/config.ts b/server/src/config.ts index 77b8a3f0cf9..90d6cbf6b14 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -120,11 +120,6 @@ export function loadConfig(): Config { const fileDatabaseBackup = fileConfig?.database.backup; const fileSecrets = fileConfig?.secrets; const fileStorage = fileConfig?.storage; - const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE; - const secretsStrictMode = - strictModeFromEnv !== undefined - ? strictModeFromEnv === "true" - : (fileSecrets?.strictMode ?? false); const providerFromEnvRaw = process.env.PAPERCLIP_SECRETS_PROVIDER; const providerFromEnv = @@ -168,6 +163,11 @@ export function loadConfig(): Config { ? (deploymentModeFromEnvRaw as DeploymentMode) : null; const deploymentMode: DeploymentMode = deploymentModeFromEnv ?? fileConfig?.server.deploymentMode ?? "local_trusted"; + const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE; + const secretsStrictMode = + strictModeFromEnv !== undefined + ? strictModeFromEnv === "true" + : (fileSecrets?.strictMode ?? deploymentMode === "authenticated"); const deploymentExposureFromEnvRaw = process.env.PAPERCLIP_DEPLOYMENT_EXPOSURE; const deploymentExposureFromEnv = deploymentExposureFromEnvRaw && diff --git a/server/src/routes/secrets.ts b/server/src/routes/secrets.ts index e6e2f0205e5..984d27070f4 100644 --- a/server/src/routes/secrets.ts +++ b/server/src/routes/secrets.ts @@ -28,6 +28,14 @@ export function secretRoutes(db: Db) { res.json(svc.listProviders()); }); + router.get("/companies/:companyId/secret-providers/health", async (req, res) => { + assertBoard(req); + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + const checks = await svc.checkProviders(); + res.json({ providers: checks }); + }); + router.get("/companies/:companyId/secrets", async (req, res) => { assertBoard(req); const companyId = req.params.companyId as string; @@ -51,6 +59,7 @@ export function secretRoutes(db: Db) { value: req.body.value, description: req.body.description, externalRef: req.body.externalRef, + providerVersionRef: req.body.providerVersionRef, providerMetadata: req.body.providerMetadata, }, { userId: req.actor.userId ?? "board", agentId: null }, @@ -84,6 +93,7 @@ export function secretRoutes(db: Db) { { value: req.body.value, externalRef: req.body.externalRef, + providerVersionRef: req.body.providerVersionRef, }, { userId: req.actor.userId ?? "board", agentId: null }, ); diff --git a/server/src/secrets/external-stub-providers.ts b/server/src/secrets/external-stub-providers.ts index 3e808abf808..bd2d2f2374b 100644 --- a/server/src/secrets/external-stub-providers.ts +++ b/server/src/secrets/external-stub-providers.ts @@ -1,23 +1,78 @@ import { unprocessable } from "../errors.js"; -import type { SecretProviderModule } from "./types.js"; +import type { PreparedSecretVersion, SecretProviderModule } from "./types.js"; +import { createHash } from "node:crypto"; function unavailableProvider( id: "aws_secrets_manager" | "gcp_secret_manager" | "vault", label: string, ): SecretProviderModule { + function externalFingerprint(externalRef: string, providerVersionRef: string | null): string { + return createHash("sha256") + .update(`${id}:${externalRef}:${providerVersionRef ?? ""}`) + .digest("hex"); + } + + function prepareExternalReference(input: { + externalRef: string; + providerVersionRef?: string | null; + }): PreparedSecretVersion { + const externalRef = input.externalRef.trim(); + const providerVersionRef = input.providerVersionRef?.trim() || null; + const fingerprint = externalFingerprint(externalRef, providerVersionRef); + return { + material: { + scheme: "external_reference_v1", + provider: id, + externalRef, + providerVersionRef, + }, + valueSha256: fingerprint, + fingerprintSha256: fingerprint, + externalRef, + providerVersionRef, + }; + } + return { id, - descriptor: { - id, - label, - requiresExternalRef: true, + descriptor() { + return { + id, + label, + requiresExternalRef: true, + supportsManagedValues: false, + supportsExternalReferences: true, + configured: false, + }; + }, + async validateConfig() { + return { ok: false, warnings: [`${id} provider is not configured in this deployment`] }; + }, + async createSecret() { + throw unprocessable(`${id} provider is not configured for Paperclip-managed values`); }, async createVersion() { - throw unprocessable(`${id} provider is not configured in this deployment`); + throw unprocessable(`${id} provider is not configured for Paperclip-managed values`); + }, + async linkExternalSecret(input) { + return prepareExternalReference(input); }, async resolveVersion() { throw unprocessable(`${id} provider is not configured in this deployment`); }, + async deleteOrArchive() { + // External references are metadata-only in Paperclip for unconfigured providers. + }, + async healthCheck() { + return { + provider: id, + status: "warn", + message: `${id} provider is available for external references but not configured for runtime resolution`, + warnings: [ + "Linked external references can be stored as metadata, but runtime resolution will fail until this provider is configured.", + ], + }; + }, }; } diff --git a/server/src/secrets/local-encrypted-provider.ts b/server/src/secrets/local-encrypted-provider.ts index a92ded202f6..e19ccc470e4 100644 --- a/server/src/secrets/local-encrypted-provider.ts +++ b/server/src/secrets/local-encrypted-provider.ts @@ -1,7 +1,14 @@ import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; -import { mkdirSync, readFileSync, writeFileSync, existsSync, chmodSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import path from "node:path"; -import type { SecretProviderModule, StoredSecretVersionMaterial } from "./types.js"; +import { resolveDefaultSecretsKeyFilePath } from "../home-paths.js"; +import type { + PreparedSecretVersion, + SecretProviderHealthCheck, + SecretProviderModule, + SecretProviderValidationResult, + StoredSecretVersionMaterial, +} from "./types.js"; import { badRequest } from "../errors.js"; interface LocalEncryptedMaterial extends StoredSecretVersionMaterial { @@ -14,7 +21,7 @@ interface LocalEncryptedMaterial extends StoredSecretVersionMaterial { function resolveMasterKeyFilePath() { const fromEnv = process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; if (fromEnv && fromEnv.trim().length > 0) return path.resolve(fromEnv.trim()); - return path.resolve(process.cwd(), "data/secrets/master.key"); + return resolveDefaultSecretsKeyFilePath(); } function decodeMasterKey(raw: string): Buffer | null { @@ -52,6 +59,7 @@ function loadOrCreateMasterKey(): Buffer { const keyPath = resolveMasterKeyFilePath(); if (existsSync(keyPath)) { + enforceKeyFilePermissionsBestEffort(keyPath); const raw = readFileSync(keyPath, "utf8"); const decoded = decodeMasterKey(raw); if (!decoded) { @@ -72,10 +80,118 @@ function loadOrCreateMasterKey(): Buffer { return generated; } +function enforceKeyFilePermissionsBestEffort(keyPath: string) { + try { + const mode = statSync(keyPath).mode & 0o777; + if ((mode & 0o077) !== 0) { + chmodSync(keyPath, 0o600); + } + } catch { + // best effort only; health checks surface persistent permission problems. + } +} + function sha256Hex(value: string): string { return createHash("sha256").update(value).digest("hex"); } +function prepareManagedVersion(value: string): PreparedSecretVersion { + const masterKey = loadOrCreateMasterKey(); + const valueSha256 = sha256Hex(value); + return { + material: encryptValue(masterKey, value), + valueSha256, + fingerprintSha256: valueSha256, + externalRef: null, + }; +} + +async function inspectLocalEncryptedHealth(): Promise { + const envKeyRaw = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + if (envKeyRaw && envKeyRaw.trim().length > 0) { + if (!decodeMasterKey(envKeyRaw)) { + return { + provider: "local_encrypted", + status: "error", + message: + "PAPERCLIP_SECRETS_MASTER_KEY is invalid; expected 32-byte base64, 64-char hex, or raw 32-char string", + }; + } + return { + provider: "local_encrypted", + status: "ok", + message: "Local encrypted provider is using PAPERCLIP_SECRETS_MASTER_KEY", + backupGuidance: [ + "Back up the configured master key separately from the database.", + "A restore needs both the database metadata and the same master key.", + ], + details: { keySource: "env" }, + }; + } + + const keyPath = resolveMasterKeyFilePath(); + if (!existsSync(keyPath)) { + return { + provider: "local_encrypted", + status: "warn", + message: `Secrets key file does not exist yet: ${keyPath}`, + warnings: ["The first managed secret write will create this key file with 0600 permissions."], + backupGuidance: [ + "Back up the key file together with database backups.", + "The database alone cannot restore local encrypted secret values.", + ], + details: { keySource: "file", keyFilePath: keyPath }, + }; + } + + let mode: number | null = null; + try { + mode = statSync(keyPath).mode & 0o777; + } catch (err) { + return { + provider: "local_encrypted", + status: "error", + message: `Could not stat secrets key file: ${err instanceof Error ? err.message : String(err)}`, + details: { keySource: "file", keyFilePath: keyPath }, + }; + } + + try { + const raw = readFileSync(keyPath, "utf8"); + if (!decodeMasterKey(raw)) { + return { + provider: "local_encrypted", + status: "error", + message: `Invalid key material in ${keyPath}`, + details: { keySource: "file", keyFilePath: keyPath }, + }; + } + } catch (err) { + return { + provider: "local_encrypted", + status: "error", + message: `Could not read secrets key file: ${err instanceof Error ? err.message : String(err)}`, + details: { keySource: "file", keyFilePath: keyPath }, + }; + } + + const warnings = + mode !== null && (mode & 0o077) !== 0 + ? [`Secrets key file permissions are ${mode.toString(8)}; run chmod 600 ${keyPath}`] + : []; + return { + provider: "local_encrypted", + status: warnings.length > 0 ? "warn" : "ok", + message: `Local encrypted provider configured with key file ${keyPath}`, + warnings, + backupGuidance: [ + "Back up the key file together with database backups.", + "The database alone cannot restore local encrypted secret values.", + ], + details: { keySource: "file", keyFilePath: keyPath }, + }; +} + function encryptValue(masterKey: Buffer, value: string): LocalEncryptedMaterial { const iv = randomBytes(12); const cipher = createCipheriv("aes-256-gcm", masterKey, iv); @@ -115,21 +231,45 @@ function asLocalEncryptedMaterial(value: StoredSecretVersionMaterial): LocalEncr export const localEncryptedProvider: SecretProviderModule = { id: "local_encrypted", - descriptor: { - id: "local_encrypted", - label: "Local encrypted (default)", - requiresExternalRef: false, + descriptor() { + return { + id: "local_encrypted", + label: "Local encrypted (default)", + requiresExternalRef: false, + supportsManagedValues: true, + supportsExternalReferences: false, + configured: true, + }; + }, + async validateConfig(input): Promise { + const warnings: string[] = []; + if (input?.deploymentMode === "authenticated" && input.strictMode !== true) { + warnings.push("Strict secret mode should be enabled for authenticated deployments"); + } + const health = await inspectLocalEncryptedHealth(); + if (health.status === "error") { + throw badRequest(health.message); + } + warnings.push(...(health.warnings ?? [])); + return { ok: true, warnings }; + }, + async createSecret(input) { + return prepareManagedVersion(input.value); }, async createVersion(input) { - const masterKey = loadOrCreateMasterKey(); - return { - material: encryptValue(masterKey, input.value), - valueSha256: sha256Hex(input.value), - externalRef: null, - }; + return prepareManagedVersion(input.value); + }, + async linkExternalSecret() { + throw badRequest("local_encrypted does not support external reference secrets"); }, async resolveVersion(input) { const masterKey = loadOrCreateMasterKey(); return decryptValue(masterKey, asLocalEncryptedMaterial(input.material)); }, + async deleteOrArchive() { + // Secret metadata deletion is handled in Paperclip DB; the local key is shared and must remain. + }, + async healthCheck() { + return inspectLocalEncryptedHealth(); + }, }; diff --git a/server/src/secrets/provider-registry.ts b/server/src/secrets/provider-registry.ts index 95e16de8d7c..390a13ada8f 100644 --- a/server/src/secrets/provider-registry.ts +++ b/server/src/secrets/provider-registry.ts @@ -5,7 +5,7 @@ import { gcpSecretManagerProvider, vaultProvider, } from "./external-stub-providers.js"; -import type { SecretProviderModule } from "./types.js"; +import type { SecretProviderHealthCheck, SecretProviderModule } from "./types.js"; import { unprocessable } from "../errors.js"; const providers: SecretProviderModule[] = [ @@ -26,5 +26,9 @@ export function getSecretProvider(id: SecretProvider): SecretProviderModule { } export function listSecretProviders(): SecretProviderDescriptor[] { - return providers.map((provider) => provider.descriptor); + return providers.map((provider) => provider.descriptor()); +} + +export async function checkSecretProviders(): Promise { + return Promise.all(providers.map((provider) => provider.healthCheck())); } diff --git a/server/src/secrets/types.ts b/server/src/secrets/types.ts index 5f9ed1b9001..56545e0bd2b 100644 --- a/server/src/secrets/types.ts +++ b/server/src/secrets/types.ts @@ -1,22 +1,88 @@ import type { SecretProvider, SecretProviderDescriptor } from "@paperclipai/shared"; +import type { DeploymentMode } from "@paperclipai/shared"; export interface StoredSecretVersionMaterial { [key: string]: unknown; } +export type SecretProviderHealthStatus = "ok" | "warn" | "error"; + +export interface SecretProviderHealthCheck { + provider: SecretProvider; + status: SecretProviderHealthStatus; + message: string; + warnings?: string[]; + backupGuidance?: string[]; + details?: Record; +} + +export interface SecretProviderValidationResult { + ok: boolean; + warnings: string[]; +} + +export interface PreparedSecretVersion { + material: StoredSecretVersionMaterial; + valueSha256: string; + fingerprintSha256?: string; + externalRef: string | null; + providerVersionRef?: string | null; +} + +export interface SecretProviderRuntimeContext { + companyId: string; + secretId: string; + secretKey: string; + version: number; +} + +export interface SecretProviderWriteContext { + companyId: string; + secretKey: string; + secretName: string; + version: number; +} + export interface SecretProviderModule { id: SecretProvider; - descriptor: SecretProviderDescriptor; + descriptor(): SecretProviderDescriptor; + validateConfig(input?: { + deploymentMode?: DeploymentMode; + strictMode?: boolean; + }): Promise; + createSecret(input: { + value: string; + externalRef?: string | null; + context?: SecretProviderWriteContext; + }): Promise; createVersion(input: { value: string; - externalRef: string | null; - }): Promise<{ - material: StoredSecretVersionMaterial; - valueSha256: string; - externalRef: string | null; - }>; + externalRef?: string | null; + context?: SecretProviderWriteContext; + }): Promise; + linkExternalSecret(input: { + externalRef: string; + providerVersionRef?: string | null; + context?: SecretProviderWriteContext; + }): Promise; resolveVersion(input: { material: StoredSecretVersionMaterial; externalRef: string | null; + providerVersionRef?: string | null; + context?: SecretProviderRuntimeContext; }): Promise; + rotate?(input: { + material: StoredSecretVersionMaterial; + externalRef: string | null; + providerVersionRef?: string | null; + }): Promise; + deleteOrArchive(input: { + material?: StoredSecretVersionMaterial | null; + externalRef: string | null; + mode: "archive" | "delete"; + }): Promise; + healthCheck(input?: { + deploymentMode?: DeploymentMode; + strictMode?: boolean; + }): Promise; } diff --git a/server/src/services/secrets.ts b/server/src/services/secrets.ts index c2c0a02cb37..f5cb389a959 100644 --- a/server/src/services/secrets.ts +++ b/server/src/services/secrets.ts @@ -15,7 +15,11 @@ import type { } from "@paperclipai/shared"; import { envBindingSchema } from "@paperclipai/shared"; import { conflict, notFound, unprocessable } from "../errors.js"; -import { getSecretProvider, listSecretProviders } from "../secrets/provider-registry.js"; +import { + checkSecretProviders, + getSecretProvider, + listSecretProviders, +} from "../secrets/provider-registry.js"; const ENV_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; const SENSITIVE_ENV_KEY_RE = @@ -207,6 +211,13 @@ export function secretService(db: Db) { const value = await provider.resolveVersion({ material: versionRow.material as Record, externalRef: secret.externalRef, + providerVersionRef: versionRow.providerVersionRef, + context: { + companyId, + secretId: secret.id, + secretKey: secret.key, + version: resolvedVersion, + }, }); await Promise.all([ db @@ -296,6 +307,8 @@ export function secretService(db: Db) { return { listProviders: () => listSecretProviders(), + checkProviders: () => checkSecretProviders(), + list: (companyId: string) => db .select() @@ -330,11 +343,12 @@ export function secretService(db: Db) { input: { name: string; provider: SecretProvider; - value: string; + value?: string | null; key?: string | null; managedMode?: "paperclip_managed" | "external_reference"; description?: string | null; externalRef?: string | null; + providerVersionRef?: string | null; providerMetadata?: Record | null; }, actor?: { userId?: string | null; agentId?: string | null }, @@ -350,11 +364,24 @@ export function secretService(db: Db) { .then((rows) => rows[0] ?? null); if (duplicateKey) throw conflict(`Secret key already exists: ${key}`); + const managedMode = input.managedMode ?? "paperclip_managed"; const provider = getSecretProvider(input.provider); - const prepared = await provider.createVersion({ - value: input.value, - externalRef: input.externalRef ?? null, - }); + if (managedMode === "external_reference" && !input.externalRef?.trim()) { + throw unprocessable("External reference secrets require externalRef"); + } + if (managedMode === "paperclip_managed" && !input.value?.trim()) { + throw unprocessable("Managed secrets require value"); + } + const prepared = + managedMode === "external_reference" + ? await provider.linkExternalSecret({ + externalRef: input.externalRef ?? "", + providerVersionRef: input.providerVersionRef ?? null, + }) + : await provider.createSecret({ + value: input.value ?? "", + externalRef: input.externalRef ?? null, + }); return db.transaction(async (tx) => { const secret = await tx @@ -365,7 +392,7 @@ export function secretService(db: Db) { name: input.name, provider: input.provider, status: "active", - managedMode: input.managedMode ?? "paperclip_managed", + managedMode, externalRef: prepared.externalRef, providerMetadata: input.providerMetadata ?? null, latestVersion: 1, @@ -382,7 +409,8 @@ export function secretService(db: Db) { version: 1, material: prepared.material, valueSha256: prepared.valueSha256, - fingerprintSha256: prepared.valueSha256, + fingerprintSha256: prepared.fingerprintSha256 ?? prepared.valueSha256, + providerVersionRef: prepared.providerVersionRef ?? null, status: "current", createdByAgentId: actor?.agentId ?? null, createdByUserId: actor?.userId ?? null, @@ -394,17 +422,29 @@ export function secretService(db: Db) { rotate: async ( secretId: string, - input: { value: string; externalRef?: string | null }, + input: { value?: string | null; externalRef?: string | null; providerVersionRef?: string | null }, actor?: { userId?: string | null; agentId?: string | null }, ) => { const secret = await getById(secretId); if (!secret) throw notFound("Secret not found"); const provider = getSecretProvider(secret.provider as SecretProvider); const nextVersion = secret.latestVersion + 1; - const prepared = await provider.createVersion({ - value: input.value, - externalRef: input.externalRef ?? secret.externalRef ?? null, - }); + if (secret.managedMode === "external_reference" && !(input.externalRef ?? secret.externalRef)?.trim()) { + throw unprocessable("External reference secrets require externalRef"); + } + if (secret.managedMode !== "external_reference" && !input.value?.trim()) { + throw unprocessable("Managed secrets require value"); + } + const prepared = + secret.managedMode === "external_reference" + ? await provider.linkExternalSecret({ + externalRef: input.externalRef ?? secret.externalRef ?? "", + providerVersionRef: input.providerVersionRef ?? null, + }) + : await provider.createVersion({ + value: input.value ?? "", + externalRef: input.externalRef ?? secret.externalRef ?? null, + }); return db.transaction(async (tx) => { await tx @@ -416,7 +456,8 @@ export function secretService(db: Db) { version: nextVersion, material: prepared.material, valueSha256: prepared.valueSha256, - fingerprintSha256: prepared.valueSha256, + fingerprintSha256: prepared.fingerprintSha256 ?? prepared.valueSha256, + providerVersionRef: prepared.providerVersionRef ?? null, status: "current", createdByAgentId: actor?.agentId ?? null, createdByUserId: actor?.userId ?? null, @@ -587,6 +628,13 @@ export function secretService(db: Db) { remove: async (secretId: string) => { const secret = await getById(secretId); if (!secret) return null; + const versionRow = await getSecretVersion(secret.id, secret.latestVersion); + const provider = getSecretProvider(secret.provider as SecretProvider); + await provider.deleteOrArchive({ + material: versionRow?.material as Record | undefined, + externalRef: secret.externalRef, + mode: "delete", + }); await db.delete(companySecrets).where(eq(companySecrets.id, secretId)); return secret; }, diff --git a/ui/storybook/stories/agent-management.stories.tsx b/ui/storybook/stories/agent-management.stories.tsx index 9e9ead37dba..095b624c6be 100644 --- a/ui/storybook/stories/agent-management.stories.tsx +++ b/ui/storybook/stories/agent-management.stories.tsx @@ -250,28 +250,44 @@ const runtimeState: AgentRuntimeState = { }; const storybookSecrets: CompanySecret[] = [ - { - id: "secret-openai", - companyId: COMPANY_ID, - name: "OPENAI_API_KEY", - provider: "local_encrypted", - externalRef: null, - latestVersion: 5, - description: "Primary coding model key for local Codex agents.", - createdByAgentId: null, - createdByUserId: "user-board", - createdAt: recent(21_000), + { + id: "secret-openai", + companyId: COMPANY_ID, + key: "openai-api-key", + name: "OPENAI_API_KEY", + provider: "local_encrypted", + status: "active", + managedMode: "paperclip_managed", + externalRef: null, + providerConfigId: null, + providerMetadata: null, + latestVersion: 5, + description: "Primary coding model key for local Codex agents.", + lastResolvedAt: recent(600), + lastRotatedAt: recent(21_000), + deletedAt: null, + createdByAgentId: null, + createdByUserId: "user-board", + createdAt: recent(21_000), updatedAt: recent(400), }, - { - id: "secret-ops-webhook", - companyId: COMPANY_ID, - name: "OPS_WEBHOOK_TOKEN", - provider: "local_encrypted", - externalRef: null, - latestVersion: 3, - description: "Webhook token for runtime observability callbacks.", - createdByAgentId: "agent-cto", + { + id: "secret-ops-webhook", + companyId: COMPANY_ID, + key: "ops-webhook-token", + name: "OPS_WEBHOOK_TOKEN", + provider: "local_encrypted", + status: "active", + managedMode: "paperclip_managed", + externalRef: null, + providerConfigId: null, + providerMetadata: null, + latestVersion: 3, + description: "Webhook token for runtime observability callbacks.", + lastResolvedAt: recent(120), + lastRotatedAt: recent(12_000), + deletedAt: null, + createdByAgentId: "agent-cto", createdByUserId: null, createdAt: recent(12_000), updatedAt: recent(80), diff --git a/ui/storybook/stories/forms-editors.stories.tsx b/ui/storybook/stories/forms-editors.stories.tsx index 2a5e6b8c4ed..bdf863459ab 100644 --- a/ui/storybook/stories/forms-editors.stories.tsx +++ b/ui/storybook/stories/forms-editors.stories.tsx @@ -199,28 +199,44 @@ const adapterErrors = { }; const storybookSecrets: CompanySecret[] = [ - { - id: "secret-openai", - companyId: "company-storybook", - name: "OPENAI_API_KEY", - provider: "local_encrypted", - externalRef: null, - latestVersion: 3, - description: null, - createdByAgentId: null, + { + id: "secret-openai", + companyId: "company-storybook", + key: "openai-api-key", + name: "OPENAI_API_KEY", + provider: "local_encrypted", + status: "active", + managedMode: "paperclip_managed", + externalRef: null, + providerConfigId: null, + providerMetadata: null, + latestVersion: 3, + description: null, + lastResolvedAt: new Date("2026-04-20T09:00:00.000Z"), + lastRotatedAt: new Date("2026-04-18T10:00:00.000Z"), + deletedAt: null, + createdByAgentId: null, createdByUserId: "user-board", createdAt: new Date("2026-04-18T10:00:00.000Z"), updatedAt: new Date("2026-04-20T10:00:00.000Z"), }, - { - id: "secret-github", - companyId: "company-storybook", - name: "GITHUB_TOKEN", - provider: "local_encrypted", - externalRef: null, - latestVersion: 1, - description: null, - createdByAgentId: null, + { + id: "secret-github", + companyId: "company-storybook", + key: "github-token", + name: "GITHUB_TOKEN", + provider: "local_encrypted", + status: "active", + managedMode: "paperclip_managed", + externalRef: null, + providerConfigId: null, + providerMetadata: null, + latestVersion: 1, + description: null, + lastResolvedAt: null, + lastRotatedAt: new Date("2026-04-19T10:00:00.000Z"), + deletedAt: null, + createdByAgentId: null, createdByUserId: "user-board", createdAt: new Date("2026-04-19T10:00:00.000Z"), updatedAt: new Date("2026-04-19T10:00:00.000Z"),