From 9690a55dbae6c411af8f831a6aab13413c376ef8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 17 May 2026 19:42:09 -0500 Subject: [PATCH] Implement AWS provider vault discovery Co-Authored-By: Paperclip --- packages/shared/src/api.ts | 1 + packages/shared/src/index.ts | 6 + packages/shared/src/types/index.ts | 4 + packages/shared/src/types/secrets.ts | 37 +++ packages/shared/src/validators/index.ts | 2 + packages/shared/src/validators/secret.test.ts | 35 +++ packages/shared/src/validators/secret.ts | 24 ++ .../aws-secrets-manager-provider.test.ts | 97 +++++++ server/src/__tests__/secrets-routes.test.ts | 103 +++++++ server/src/__tests__/secrets-service.test.ts | 105 +++++++ server/src/routes/secrets.ts | 36 +++ .../secrets/aws-secrets-manager-provider.ts | 262 +++++++++++++++++- server/src/secrets/types.ts | 13 +- server/src/services/secrets.ts | 63 +++++ 14 files changed, 786 insertions(+), 2 deletions(-) diff --git a/packages/shared/src/api.ts b/packages/shared/src/api.ts index 38988c6fc2e..7fe53f4c3d3 100644 --- a/packages/shared/src/api.ts +++ b/packages/shared/src/api.ts @@ -12,6 +12,7 @@ export const API = { approvals: `${API_PREFIX}/approvals`, secrets: `${API_PREFIX}/secrets`, secretProviderConfigs: `${API_PREFIX}/secret-provider-configs`, + secretProviderConfigDiscoveryPreview: `${API_PREFIX}/companies/:companyId/secret-provider-configs/discovery/preview`, costs: `${API_PREFIX}/costs`, activity: `${API_PREFIX}/activity`, dashboard: `${API_PREFIX}/dashboard`, diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index dcd92bcfe62..a851acf0aca 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -558,6 +558,10 @@ export type { SecretProviderConfigPayload, SecretProviderConfigHealthDetails, SecretProviderConfigHealthResponse, + SecretProviderConfigDiscoveryCandidate, + SecretProviderConfigDiscoveryPreviewResult, + SecretProviderConfigDiscoverySample, + SecretProviderConfigDiscoverySignal, CompanySecretBinding, CompanySecretBindingTarget, CompanySecretUsageBinding, @@ -885,6 +889,7 @@ export { createSecretSchema, createSecretProviderConfigSchema, updateSecretProviderConfigSchema, + secretProviderConfigDiscoveryPreviewSchema, remoteSecretImportPreviewSchema, remoteSecretImportSchema, remoteSecretImportSelectionSchema, @@ -911,6 +916,7 @@ export { type CreateSecret, type CreateSecretProviderConfig, type UpdateSecretProviderConfig, + type SecretProviderConfigDiscoveryPreview, type RemoteSecretImportPreview, type RemoteSecretImport, type RemoteSecretImportSelection, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index a23547572a3..cfd7c74b924 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -258,6 +258,10 @@ export type { SecretProviderConfigPayload, SecretProviderConfigHealthDetails, SecretProviderConfigHealthResponse, + SecretProviderConfigDiscoveryCandidate, + SecretProviderConfigDiscoveryPreviewResult, + SecretProviderConfigDiscoverySample, + SecretProviderConfigDiscoverySignal, CompanySecretBinding, CompanySecretBindingTarget, CompanySecretUsageBinding, diff --git a/packages/shared/src/types/secrets.ts b/packages/shared/src/types/secrets.ts index 7a4f0ae3243..9c01ff40fff 100644 --- a/packages/shared/src/types/secrets.ts +++ b/packages/shared/src/types/secrets.ts @@ -138,6 +138,43 @@ export interface SecretProviderConfigHealthResponse { checkedAt: Date; } +export interface SecretProviderConfigDiscoverySignal { + namespace: string | null; + secretNamePrefix: string | null; + environmentTag: string | null; + ownerTag: string | null; + kmsKeyId: string | null; + hasKmsKey: boolean; + sampleCount: number; + paperclipManagedSampleCount: number; + skippedForeignPaperclipSampleCount: number; +} + +export interface SecretProviderConfigDiscoverySample { + name: string; + hasKmsKey: boolean; + tagKeys: string[]; +} + +export interface SecretProviderConfigDiscoveryCandidate { + provider: SecretProvider; + displayName: string; + config: SecretProviderConfigPayload; + sampleCount: number; + samples: SecretProviderConfigDiscoverySample[]; + signals: SecretProviderConfigDiscoverySignal; + warnings: string[]; +} + +export interface SecretProviderConfigDiscoveryPreviewResult { + provider: SecretProvider; + nextToken: string | null; + sampledSecretCount: number; + skippedForeignPaperclipSampleCount: number; + candidates: SecretProviderConfigDiscoveryCandidate[]; + warnings: string[]; +} + export interface CompanySecretVersion { id: string; secretId: string; diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 3380c5534a3..e8410a01bf6 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -293,6 +293,7 @@ export { createSecretSchema, createSecretProviderConfigSchema, updateSecretProviderConfigSchema, + secretProviderConfigDiscoveryPreviewSchema, remoteSecretImportPreviewSchema, remoteSecretImportSchema, remoteSecretImportSelectionSchema, @@ -309,6 +310,7 @@ export { type CreateSecret, type CreateSecretProviderConfig, type UpdateSecretProviderConfig, + type SecretProviderConfigDiscoveryPreview, type RemoteSecretImportPreview, type RemoteSecretImport, type RemoteSecretImportSelection, diff --git a/packages/shared/src/validators/secret.test.ts b/packages/shared/src/validators/secret.test.ts index c8a8163d3d2..10d81a9d587 100644 --- a/packages/shared/src/validators/secret.test.ts +++ b/packages/shared/src/validators/secret.test.ts @@ -4,6 +4,7 @@ import { createSecretSchema, remoteSecretImportPreviewSchema, remoteSecretImportSchema, + secretProviderConfigDiscoveryPreviewSchema, secretProviderConfigPayloadSchema, updateSecretProviderConfigSchema, } from "./secret.js"; @@ -140,6 +141,40 @@ describe("secret validators", () => { }); }); + it("validates AWS provider vault discovery draft config without allowing sensitive keys", () => { + expect( + secretProviderConfigDiscoveryPreviewSchema.parse({ + provider: "aws_secrets_manager", + config: { + region: "us-east-1", + namespace: "production", + secretNamePrefix: "paperclip", + }, + query: "paperclip", + pageSize: 50, + }), + ).toEqual({ + provider: "aws_secrets_manager", + config: { + region: "us-east-1", + namespace: "production", + secretNamePrefix: "paperclip", + }, + query: "paperclip", + pageSize: 50, + }); + + expect(() => + secretProviderConfigDiscoveryPreviewSchema.parse({ + provider: "aws_secrets_manager", + config: { + region: "us-east-1", + accessKeyId: "AKIA...", + }, + }), + ).toThrow(/sensitive field/i); + }); + it("caps AWS remote import paging and row counts", () => { expect(() => remoteSecretImportPreviewSchema.parse({ diff --git a/packages/shared/src/validators/secret.ts b/packages/shared/src/validators/secret.ts index 1ee1952ca7d..a90bf064f18 100644 --- a/packages/shared/src/validators/secret.ts +++ b/packages/shared/src/validators/secret.ts @@ -262,6 +262,30 @@ export const remoteSecretImportPreviewSchema = z.object({ export type RemoteSecretImportPreview = z.infer; +export const secretProviderConfigDiscoveryPreviewSchema = z.object({ + provider: z.enum(SECRET_PROVIDERS), + config: z.record(z.unknown()).default({}), + query: z.string().trim().max(200).optional().nullable(), + nextToken: z.string().trim().min(1).max(4096).optional().nullable(), + pageSize: z.number().int().min(1).max(100).optional(), +}).superRefine((value, ctx) => { + rejectSensitiveProviderConfigKeys(value.config, ctx); + const parsed = secretProviderConfigPayloadSchema.safeParse({ + provider: value.provider, + config: value.config, + }); + if (!parsed.success) { + for (const issue of parsed.error.issues) { + ctx.addIssue({ + ...issue, + path: issue.path[0] === "config" ? issue.path : ["config", ...issue.path], + }); + } + } +}); + +export type SecretProviderConfigDiscoveryPreview = z.infer; + export const remoteSecretImportSelectionSchema = z.object({ externalRef: z.string().trim().min(1).max(2048), name: z.string().trim().min(1).max(160).optional().nullable(), diff --git a/server/src/__tests__/aws-secrets-manager-provider.test.ts b/server/src/__tests__/aws-secrets-manager-provider.test.ts index 488f3415998..90fec29575c 100644 --- a/server/src/__tests__/aws-secrets-manager-provider.test.ts +++ b/server/src/__tests__/aws-secrets-manager-provider.test.ts @@ -454,6 +454,103 @@ describe("awsSecretsManagerProvider", () => { expect(JSON.stringify(listed)).not.toContain("team"); }); + it("discovers AWS provider vault prefill candidates from metadata without reading values", async () => { + const calls: Array<{ op: string; input: Record }> = []; + const provider = createAwsSecretsManagerProvider({ + gateway: { + async createSecret() { + throw new Error("not used"); + }, + async putSecretValue() { + throw new Error("not used"); + }, + async getSecretValue() { + throw new Error("GetSecretValue must not be used for provider vault discovery"); + }, + async deleteSecret() { + throw new Error("not used"); + }, + async listSecrets(input) { + calls.push({ op: "listSecrets", input }); + return { + NextToken: "next-page", + SecretList: [ + { + ARN: "arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/prod-use1/company-1/openai", + Name: "paperclip/prod-use1/company-1/openai", + KmsKeyId: "arn:aws:kms:us-east-1:123456789012:key/prod", + Tags: [ + { Key: "paperclip:managed-by", Value: "paperclip" }, + { Key: "paperclip:deployment-id", Value: "prod-use1" }, + { Key: "paperclip:company-id", Value: "company-1" }, + { Key: "paperclip:environment", Value: "production" }, + { Key: "paperclip:provider-owner", Value: "platform" }, + ], + }, + { + ARN: "arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/prod-use1/company-2/stripe", + Name: "paperclip/prod-use1/company-2/stripe", + Tags: [ + { Key: "paperclip:managed-by", Value: "paperclip" }, + { Key: "paperclip:company-id", Value: "company-2" }, + ], + }, + ], + }; + }, + }, + }); + + const preview = await provider.discoverProviderConfigs?.({ + companyId: "company-1", + providerConfig: { + id: "draft", + provider: "aws_secrets_manager", + status: "ready", + config: { region: "us-east-1" }, + }, + query: "paperclip", + pageSize: 25, + }); + + expect(calls).toEqual([ + { + op: "listSecrets", + input: { + MaxResults: 25, + NextToken: undefined, + IncludePlannedDeletion: false, + Filters: [{ Key: "all", Values: ["paperclip"] }], + }, + }, + ]); + expect(preview).toMatchObject({ + provider: "aws_secrets_manager", + nextToken: "next-page", + sampledSecretCount: 1, + skippedForeignPaperclipSampleCount: 1, + candidates: [ + expect.objectContaining({ + displayName: "AWS production", + config: expect.objectContaining({ + region: "us-east-1", + namespace: "prod-use1", + secretNamePrefix: "paperclip", + kmsKeyId: "arn:aws:kms:us-east-1:123456789012:key/prod", + ownerTag: "platform", + environmentTag: "production", + }), + signals: expect.objectContaining({ + paperclipManagedSampleCount: 1, + skippedForeignPaperclipSampleCount: 1, + }), + }), + ], + }); + expect(JSON.stringify(preview)).not.toContain("SecretString"); + expect(JSON.stringify(preview)).not.toContain("company-2/stripe"); + }); + it("redacts AWS provider exception text when remote listing fails", async () => { const rawProviderMessage = "AccessDeniedException: User: arn:aws:sts::123456789012:assumed-role/prod/Paperclip is not authorized to perform secretsmanager:ListSecrets on arn:aws:secretsmanager:us-east-1:123456789012:secret:prod/openai"; diff --git a/server/src/__tests__/secrets-routes.test.ts b/server/src/__tests__/secrets-routes.test.ts index 86d4b7cb1ed..5ec31f021ab 100644 --- a/server/src/__tests__/secrets-routes.test.ts +++ b/server/src/__tests__/secrets-routes.test.ts @@ -9,6 +9,7 @@ const mockSecretService = vi.hoisted(() => ({ listProviders: vi.fn(), checkProviders: vi.fn(), listProviderConfigs: vi.fn(), + previewProviderConfigDiscovery: vi.fn(), getProviderConfigById: vi.fn(), createProviderConfig: vi.fn(), updateProviderConfig: vi.fn(), @@ -117,6 +118,22 @@ describe("secret routes", () => { expect(mockSecretService.listProviderConfigs).not.toHaveBeenCalled(); }); + it("rejects provider vault discovery preview for non-board actors", async () => { + const res = await request(createApp({ + type: "agent", + agentId: "agent-1", + companyId: "company-1", + })) + .post("/api/companies/company-1/secret-provider-configs/discovery/preview") + .send({ + provider: "aws_secrets_manager", + config: { region: "us-east-1" }, + }); + + expect(res.status).toBe(403); + expect(mockSecretService.previewProviderConfigDiscovery).not.toHaveBeenCalled(); + }); + it("rejects sensitive provider vault config fields", async () => { const res = await request(createApp()).post("/api/companies/company-1/secret-provider-configs").send({ provider: "aws_secrets_manager", @@ -132,6 +149,92 @@ describe("secret routes", () => { expect(mockSecretService.createProviderConfig).not.toHaveBeenCalled(); }); + it("rejects sensitive provider vault discovery draft config fields", async () => { + const res = await request(createApp()) + .post("/api/companies/company-1/secret-provider-configs/discovery/preview") + .send({ + provider: "aws_secrets_manager", + config: { + region: "us-east-1", + secretAccessKey: "secret", + }, + }); + + expect(res.status).toBe(400); + expect(JSON.stringify(res.body)).toMatch(/sensitive field/i); + expect(mockSecretService.previewProviderConfigDiscovery).not.toHaveBeenCalled(); + }); + + it("previews provider vault discovery and logs only aggregate metadata", async () => { + mockSecretService.previewProviderConfigDiscovery.mockResolvedValue({ + provider: "aws_secrets_manager", + nextToken: null, + sampledSecretCount: 2, + skippedForeignPaperclipSampleCount: 0, + candidates: [ + { + provider: "aws_secrets_manager", + displayName: "AWS production", + config: { + region: "us-east-1", + namespace: "prod-use1", + secretNamePrefix: "paperclip", + environmentTag: "production", + ownerTag: "platform", + kmsKeyId: null, + }, + sampleCount: 2, + samples: [ + { name: "paperclip/prod-use1/company-1/openai", hasKmsKey: false, tagKeys: ["environment"] }, + ], + signals: { + namespace: "prod-use1", + secretNamePrefix: "paperclip", + environmentTag: "production", + ownerTag: "platform", + kmsKeyId: null, + hasKmsKey: false, + sampleCount: 2, + paperclipManagedSampleCount: 0, + skippedForeignPaperclipSampleCount: 0, + }, + warnings: [], + }, + ], + warnings: [], + }); + + const res = await request(createApp()) + .post("/api/companies/company-1/secret-provider-configs/discovery/preview") + .send({ + provider: "aws_secrets_manager", + config: { region: "us-east-1" }, + query: "paperclip", + pageSize: 25, + }); + + expect(res.status).toBe(200); + expect(mockSecretService.previewProviderConfigDiscovery).toHaveBeenCalledWith("company-1", { + provider: "aws_secrets_manager", + config: { region: "us-east-1" }, + query: "paperclip", + nextToken: undefined, + pageSize: 25, + }); + expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + action: "secret_provider_config.discovery_previewed", + entityType: "secret_provider_config_discovery", + entityId: "company-1", + details: { + provider: "aws_secrets_manager", + candidateCount: 1, + sampledSecretCount: 2, + warningCount: 0, + }, + })); + expect(JSON.stringify(mockLogActivity.mock.calls)).not.toContain("paperclip/prod-use1/company-1/openai"); + }); + it("rejects ready status for coming-soon provider vaults", async () => { const res = await request(createApp()).post("/api/companies/company-1/secret-provider-configs").send({ provider: "vault", diff --git a/server/src/__tests__/secrets-service.test.ts b/server/src/__tests__/secrets-service.test.ts index d01b677707b..bfd0ec68669 100644 --- a/server/src/__tests__/secrets-service.test.ts +++ b/server/src/__tests__/secrets-service.test.ts @@ -1207,6 +1207,111 @@ describeEmbeddedPostgres("secretService", () => { expect(thrown instanceof Error ? thrown.message : String(thrown)).not.toContain("arn:aws"); }); + it("previews AWS provider vault discovery from draft config without persisting a provider vault", async () => { + const companyId = await seedCompany(); + const svc = secretService(db); + const discoverSpy = vi.spyOn(awsSecretsManagerProvider, "discoverProviderConfigs").mockResolvedValue({ + provider: "aws_secrets_manager", + nextToken: null, + sampledSecretCount: 1, + skippedForeignPaperclipSampleCount: 0, + candidates: [ + { + provider: "aws_secrets_manager", + displayName: "AWS production", + config: { + region: "us-east-1", + namespace: "prod-use1", + secretNamePrefix: "paperclip", + kmsKeyId: null, + ownerTag: "platform", + environmentTag: "production", + }, + sampleCount: 1, + samples: [ + { name: "paperclip/prod-use1/company-1/openai", hasKmsKey: false, tagKeys: ["paperclip:environment"] }, + ], + signals: { + namespace: "prod-use1", + secretNamePrefix: "paperclip", + environmentTag: "production", + ownerTag: "platform", + kmsKeyId: null, + hasKmsKey: false, + sampleCount: 1, + paperclipManagedSampleCount: 0, + skippedForeignPaperclipSampleCount: 0, + }, + warnings: [], + }, + ], + warnings: [], + }); + + const preview = await svc.previewProviderConfigDiscovery(companyId, { + provider: "aws_secrets_manager", + config: { region: "us-east-1" }, + query: "openai", + pageSize: 25, + }); + + expect(discoverSpy).toHaveBeenCalledWith({ + companyId, + providerConfig: { + id: `discovery-preview-${companyId}`, + provider: "aws_secrets_manager", + status: "ready", + config: { region: "us-east-1" }, + }, + query: "openai", + nextToken: undefined, + pageSize: 25, + }); + expect(preview.candidates[0]?.config).toMatchObject({ + region: "us-east-1", + namespace: "prod-use1", + }); + expect(JSON.stringify(preview)).not.toContain("runtime-secret"); + const persistedVaults = await db.select().from(companySecretProviderConfigs); + expect(persistedVaults).toHaveLength(0); + }); + + it("sanitizes AWS provider vault discovery errors before crossing the service boundary", async () => { + const companyId = await seedCompany(); + const svc = secretService(db); + const rawProviderMessage = + "AccessDeniedException: User: arn:aws:sts::123456789012:assumed-role/prod/Paperclip is not authorized to perform secretsmanager:ListSecrets"; + + vi.spyOn(awsSecretsManagerProvider, "discoverProviderConfigs").mockRejectedValueOnce( + new SecretProviderClientError({ + code: "access_denied", + provider: "aws_secrets_manager", + operation: "discoverProviderConfigs", + message: "AWS Secrets Manager denied the request. Check IAM permissions for this provider vault.", + rawMessage: rawProviderMessage, + }), + ); + + let thrown: unknown; + try { + await svc.previewProviderConfigDiscovery(companyId, { + provider: "aws_secrets_manager", + config: { region: "us-east-1" }, + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toMatchObject({ + status: 403, + message: "AWS Secrets Manager denied the request. Check IAM permissions for this provider vault.", + details: { code: "access_denied" }, + }); + expect(JSON.stringify(thrown)).not.toContain("arn:aws"); + expect(JSON.stringify(thrown)).not.toContain("123456789012"); + expect(thrown instanceof Error ? thrown.message : String(thrown)).not.toContain("arn:aws"); + }); + it("imports AWS remote references row-by-row without fetching plaintext", async () => { const companyId = await seedCompany(); const svc = secretService(db); diff --git a/server/src/routes/secrets.ts b/server/src/routes/secrets.ts index be9d503f3de..632600654eb 100644 --- a/server/src/routes/secrets.ts +++ b/server/src/routes/secrets.ts @@ -6,6 +6,7 @@ import { remoteSecretImportPreviewSchema, remoteSecretImportSchema, rotateSecretSchema, + secretProviderConfigDiscoveryPreviewSchema, updateSecretProviderConfigSchema, updateSecretSchema, } from "@paperclipai/shared"; @@ -41,6 +42,41 @@ export function secretRoutes(db: Db) { res.json(await svc.listProviderConfigs(companyId)); }); + router.post( + "/companies/:companyId/secret-provider-configs/discovery/preview", + validate(secretProviderConfigDiscoveryPreviewSchema), + async (req, res) => { + assertBoard(req); + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + + const preview = await svc.previewProviderConfigDiscovery(companyId, { + provider: req.body.provider, + config: req.body.config, + query: req.body.query, + nextToken: req.body.nextToken, + pageSize: req.body.pageSize, + }); + + await logActivity(db, { + companyId, + actorType: "user", + actorId: req.actor.userId ?? "board", + action: "secret_provider_config.discovery_previewed", + entityType: "secret_provider_config_discovery", + entityId: companyId, + details: { + provider: preview.provider, + candidateCount: preview.candidates.length, + sampledSecretCount: preview.sampledSecretCount, + warningCount: preview.warnings.length, + }, + }); + + res.json(preview); + }, + ); + router.post("/companies/:companyId/secret-provider-configs", validate(createSecretProviderConfigSchema), async (req, res) => { assertBoard(req); const companyId = req.params.companyId as string; diff --git a/server/src/secrets/aws-secrets-manager-provider.ts b/server/src/secrets/aws-secrets-manager-provider.ts index 8c638594ae9..a697556a169 100644 --- a/server/src/secrets/aws-secrets-manager-provider.ts +++ b/server/src/secrets/aws-secrets-manager-provider.ts @@ -1,6 +1,6 @@ import { createHash, createHmac } from "node:crypto"; import { S3Client } from "@aws-sdk/client-s3"; -import type { DeploymentMode } from "@paperclipai/shared"; +import type { DeploymentMode, SecretProviderConfigDiscoveryPreviewResult } from "@paperclipai/shared"; import { unprocessable } from "../errors.js"; import type { PreparedSecretVersion, @@ -24,6 +24,8 @@ const DEFAULT_DELETE_RECOVERY_WINDOW_DAYS = 30; const AWS_SECRETS_MANAGER_REQUEST_TIMEOUT_MS = 30_000; const AWS_CREDENTIAL_CACHE_TTL_MS = 5 * 60_000; const AWS_CREDENTIAL_EXPIRATION_SKEW_MS = 60_000; +const PROVIDER_CONFIG_DISCOVERY_SAMPLE_LIMIT = 3; +const PROVIDER_CONFIG_DISCOVERY_CANDIDATE_LIMIT = 6; const AWS_RUNTIME_CREDENTIAL_WARNING = "AWS bootstrap credentials must be available to the Paperclip server runtime through the AWS SDK default credential provider chain: IAM role/workload identity, AWS_PROFILE/SSO/shared credentials, web identity, container/instance metadata, or short-lived shell credentials."; const AWS_CREDENTIAL_CUSTODY_WARNING = @@ -590,6 +592,234 @@ function createRemoteSecretMetadata(entry: AwsSecretsManagerListSecretEntry): Re }; } +function tagValue(tags: Map, keys: string[]) { + for (const key of keys) { + const value = tags.get(key.toLowerCase()); + if (value) return value; + } + return null; +} + +function normalizeAwsTags(tags: AwsSecretsManagerTag[] | undefined) { + const normalized = new Map(); + for (const tag of tags ?? []) { + const key = tag.Key?.trim(); + const value = tag.Value?.trim(); + if (key && value) normalized.set(key.toLowerCase(), value); + } + return normalized; +} + +function commonValue(values: Array) { + const nonEmpty = values.filter((value): value is string => Boolean(value?.trim())); + if (nonEmpty.length === 0) return null; + const first = nonEmpty[0]; + return nonEmpty.every((value) => value === first) ? first : null; +} + +function uniqueValues(values: Array) { + return [...new Set(values.filter((value): value is string => Boolean(value?.trim())))]; +} + +function pathSegments(name: string) { + return name.split("/").map((segment) => segment.trim()).filter(Boolean); +} + +function inferPathSignals(entry: AwsSecretsManagerListSecretEntry, tags: Map) { + const name = entry.Name?.trim() || entry.ARN?.trim() || ""; + const segments = pathSegments(name); + const paperclipDeploymentId = tagValue(tags, ["paperclip:deployment-id"]); + const paperclipManaged = tagValue(tags, ["paperclip:managed-by"])?.toLowerCase() === "paperclip"; + + if (paperclipDeploymentId || paperclipManaged) { + return { + prefix: segments[0] ?? DEFAULT_PREFIX, + namespace: paperclipDeploymentId ?? segments[1] ?? null, + }; + } + + if (segments.length >= 3) { + return { + prefix: segments[0] ?? null, + namespace: segments[1] ?? null, + }; + } + + return { + prefix: segments[0] ?? null, + namespace: null, + }; +} + +function discoveryDisplayName(input: { + environmentTag: string | null; + ownerTag: string | null; + namespace: string | null; + secretNamePrefix: string | null; +}) { + const qualifier = + input.environmentTag ?? + input.namespace ?? + input.secretNamePrefix ?? + input.ownerTag ?? + "discovered"; + return `AWS ${qualifier}`; +} + +function discoverAwsProviderConfigCandidates(input: { + companyId: string; + config: AwsSecretsManagerConfig; + draftConfig: Record; + entries: AwsSecretsManagerListSecretEntry[]; + nextToken: string | null; +}): SecretProviderConfigDiscoveryPreviewResult { + type DiscoverySample = { + entry: AwsSecretsManagerListSecretEntry; + name: string; + tags: Map; + prefix: string | null; + namespace: string | null; + environmentTag: string | null; + ownerTag: string | null; + kmsKeyId: string | null; + paperclipManaged: boolean; + paperclipCompanyId: string | null; + }; + + const skippedWarnings: string[] = []; + let skippedForeignPaperclipSampleCount = 0; + const samples: DiscoverySample[] = []; + + for (const entry of input.entries) { + const name = entry.Name?.trim() || entry.ARN?.trim(); + if (!name) continue; + const tags = normalizeAwsTags(entry.Tags); + const paperclipManaged = tagValue(tags, ["paperclip:managed-by"])?.toLowerCase() === "paperclip"; + const paperclipCompanyId = tagValue(tags, ["paperclip:company-id"]); + if (paperclipManaged && paperclipCompanyId !== input.companyId) { + skippedForeignPaperclipSampleCount += 1; + continue; + } + const path = inferPathSignals(entry, tags); + samples.push({ + entry, + name, + tags, + prefix: path.prefix, + namespace: path.namespace, + environmentTag: tagValue(tags, ["paperclip:environment", "environment", "env", "stage"]), + ownerTag: tagValue(tags, ["paperclip:provider-owner", "owner", "team", "service", "application"]), + kmsKeyId: asOptionalNonEmptyString(entry.KmsKeyId), + paperclipManaged, + paperclipCompanyId, + }); + } + + if (skippedForeignPaperclipSampleCount > 0) { + skippedWarnings.push( + `Skipped ${skippedForeignPaperclipSampleCount} Paperclip-managed AWS secret sample(s) that were not tagged for this company.`, + ); + } + + const draftNamespace = asOptionalNonEmptyString(input.draftConfig.namespace); + const draftPrefix = asOptionalNonEmptyString(input.draftConfig.secretNamePrefix); + const draftKmsKeyId = asOptionalNonEmptyString(input.draftConfig.kmsKeyId); + const draftEnvironmentTag = asOptionalNonEmptyString(input.draftConfig.environmentTag); + const draftOwnerTag = asOptionalNonEmptyString(input.draftConfig.ownerTag); + const groups = new Map(); + + for (const sample of samples) { + const key = [ + draftPrefix ?? sample.prefix ?? "", + draftNamespace ?? sample.namespace ?? "", + ].join("\0"); + groups.set(key, [...(groups.get(key) ?? []), sample]); + } + + const candidates = [...groups.values()] + .sort((a, b) => b.length - a.length) + .slice(0, PROVIDER_CONFIG_DISCOVERY_CANDIDATE_LIMIT) + .map((group) => { + const prefix = draftPrefix ?? commonValue(group.map((sample) => sample.prefix)) ?? input.config.prefix; + const namespace = draftNamespace ?? commonValue(group.map((sample) => sample.namespace)) ?? null; + const environmentTag = draftEnvironmentTag ?? commonValue(group.map((sample) => sample.environmentTag)); + const ownerTag = draftOwnerTag ?? commonValue(group.map((sample) => sample.ownerTag)); + const kmsKeys = uniqueValues(group.map((sample) => sample.kmsKeyId)); + const commonKmsKey = commonValue(group.map((sample) => sample.kmsKeyId)); + const kmsKeyId = draftKmsKeyId ?? commonKmsKey; + const candidateWarnings: string[] = []; + + if (!namespace) { + candidateWarnings.push("No stable namespace signal was found in the sampled AWS secret names or tags."); + } + if (!environmentTag) { + candidateWarnings.push("No common environment tag was found in the sampled AWS secrets."); + } + if (!ownerTag) { + candidateWarnings.push("No common owner/team tag was found in the sampled AWS secrets."); + } + if (kmsKeys.length > 1 && !draftKmsKeyId) { + candidateWarnings.push("Sampled AWS secrets use multiple KMS keys; choose the intended KMS key before saving."); + } + if (group.some((sample) => sample.paperclipManaged && sample.paperclipCompanyId === input.companyId)) { + candidateWarnings.push("Sample includes Paperclip-managed secrets for this company; do not import them as external references."); + } + + return { + provider: "aws_secrets_manager" as const, + displayName: discoveryDisplayName({ + environmentTag, + ownerTag, + namespace, + secretNamePrefix: prefix, + }), + config: { + region: input.config.region, + namespace, + secretNamePrefix: prefix, + kmsKeyId: kmsKeyId ?? null, + ownerTag, + environmentTag, + }, + sampleCount: group.length, + samples: group.slice(0, PROVIDER_CONFIG_DISCOVERY_SAMPLE_LIMIT).map((sample) => ({ + name: sample.name, + hasKmsKey: Boolean(sample.kmsKeyId), + tagKeys: [...sample.tags.keys()].sort(), + })), + signals: { + namespace, + secretNamePrefix: prefix, + environmentTag, + ownerTag, + kmsKeyId: kmsKeyId ?? null, + hasKmsKey: kmsKeys.length > 0, + sampleCount: group.length, + paperclipManagedSampleCount: group.filter((sample) => sample.paperclipManaged).length, + skippedForeignPaperclipSampleCount, + }, + warnings: candidateWarnings, + }; + }); + + const warnings = [...skippedWarnings]; + if (samples.length === 0) { + warnings.push("AWS Secrets Manager returned no metadata samples for this draft provider vault config."); + } + if (groups.size > PROVIDER_CONFIG_DISCOVERY_CANDIDATE_LIMIT) { + warnings.push("Additional AWS secret name groups were omitted from this preview; refine the query to inspect them."); + } + + return { + provider: "aws_secrets_manager", + nextToken: input.nextToken, + sampledSecretCount: samples.length, + skippedForeignPaperclipSampleCount, + candidates, + warnings, + }; +} + function asAwsSecretsManagerMaterial(value: StoredSecretVersionMaterial): AwsSecretsManagerMaterial { if ( value && @@ -983,6 +1213,36 @@ export function createAwsSecretsManagerProvider( normalizeAwsError("listSecrets", error); } }, + async discoverProviderConfigs(input): Promise { + const config = resolveConfig(input.providerConfig); + const gateway = resolveGateway(config); + const query = input.query?.trim(); + const pageSize = + input.pageSize && Number.isFinite(input.pageSize) + ? Math.min(Math.max(Math.trunc(input.pageSize), 1), 100) + : 100; + + try { + if (!gateway.listSecrets) { + throw new Error("ListSecrets gateway operation is unavailable"); + } + const listed = await gateway.listSecrets({ + MaxResults: pageSize, + NextToken: input.nextToken?.trim() || undefined, + IncludePlannedDeletion: false, + Filters: query ? [{ Key: "all", Values: [query] }] : undefined, + }); + return discoverAwsProviderConfigCandidates({ + companyId: input.companyId, + config, + draftConfig: input.providerConfig.config, + entries: listed.SecretList ?? [], + nextToken: listed.NextToken ?? null, + }); + } catch (error) { + normalizeAwsError("discoverProviderConfigs", error); + } + }, async resolveVersion(input) { const config = resolveConfig(input.providerConfig); const gateway = resolveGateway(config); diff --git a/server/src/secrets/types.ts b/server/src/secrets/types.ts index 341163e6bec..9edf7766be1 100644 --- a/server/src/secrets/types.ts +++ b/server/src/secrets/types.ts @@ -1,4 +1,8 @@ -import type { SecretProvider, SecretProviderDescriptor } from "@paperclipai/shared"; +import type { + SecretProvider, + SecretProviderConfigDiscoveryPreviewResult, + SecretProviderDescriptor, +} from "@paperclipai/shared"; import type { DeploymentMode } from "@paperclipai/shared"; export interface StoredSecretVersionMaterial { @@ -152,6 +156,13 @@ export interface SecretProviderModule { nextToken?: string | null; pageSize?: number; }): Promise; + discoverProviderConfigs?(input: { + companyId: string; + providerConfig: SecretProviderVaultRuntimeConfig; + query?: string | null; + nextToken?: string | null; + pageSize?: number; + }): Promise; resolveVersion(input: { material: StoredSecretVersionMaterial; externalRef: string | null; diff --git a/server/src/services/secrets.ts b/server/src/services/secrets.ts index ce9ee967812..0f05d922c61 100644 --- a/server/src/services/secrets.ts +++ b/server/src/services/secrets.ts @@ -20,6 +20,7 @@ import type { RemoteSecretImportCandidate, RemoteSecretImportConflict, RemoteSecretImportRowResult, + SecretProviderConfigDiscoveryPreviewResult, SecretBindingTargetType, SecretProvider, SecretProviderConfigHealthResponse, @@ -34,6 +35,7 @@ import { isUuidLike, normalizeAgentUrlKey, secretProviderConfigPayloadSchema, + secretProviderConfigDiscoveryPreviewSchema, updateSecretProviderConfigSchema, } from "@paperclipai/shared"; import { conflict, HttpError, notFound, unprocessable } from "../errors.js"; @@ -471,6 +473,19 @@ export function secretService(db: Db) { return parsed.data.config; } + function toDraftProviderVaultRuntimeConfig(input: { + companyId: string; + provider: SecretProvider; + config: Record; + }): SecretProviderVaultRuntimeConfig { + return { + id: `discovery-preview-${input.companyId}`, + provider: input.provider, + status: "ready", + config: validateProviderConfigPayload(input.provider, input.config), + }; + } + function providerConfigHealth(input: { id: string; provider: SecretProvider; @@ -949,6 +964,54 @@ export function secretService(db: Db) { checkProviders: () => checkSecretProviders(), + previewProviderConfigDiscovery: async ( + companyId: string, + input: { + provider: SecretProvider; + config?: Record; + query?: string | null; + nextToken?: string | null; + pageSize?: number; + }, + ): Promise => { + const parsed = secretProviderConfigDiscoveryPreviewSchema.safeParse({ + provider: input.provider, + config: input.config ?? {}, + query: input.query, + nextToken: input.nextToken, + pageSize: input.pageSize, + }); + if (!parsed.success) { + throw unprocessable("Invalid provider vault discovery config", parsed.error.flatten()); + } + const providerId = parsed.data.provider as SecretProvider; + const provider = getSecretProvider(providerId); + if (!provider.discoverProviderConfigs) { + throw unprocessable(`${providerId} provider does not support provider vault discovery`); + } + const runtimeConfig = toDraftProviderVaultRuntimeConfig({ + companyId, + provider: providerId, + config: parsed.data.config, + }); + try { + return await provider.discoverProviderConfigs({ + companyId, + providerConfig: runtimeConfig, + query: parsed.data.query, + nextToken: parsed.data.nextToken, + pageSize: parsed.data.pageSize, + }); + } catch (error) { + throw remoteProviderHttpError(error, { + companyId, + provider: providerId, + providerConfigId: "discovery-preview", + operation: "secret_provider_config.discovery.preview", + }); + } + }, + listProviderConfigs: (companyId: string) => db .select()