mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
Implement AWS provider vault discovery
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
bfe6369ef5
commit
9690a55dba
14 files changed
+786
-2
No files matched your search
@@ -12,6 +12,7 @@ export const API = {
|
||||
approvals: `${API_PREFIX}/approvals`,
|
||||
secrets: `${API_PREFIX}/secrets`,
|
||||
secretProviderConfigs: `${API_PREFIX}/secret-provider-configs`,
|
||||
secretProviderConfigDiscoveryPreview: `${API_PREFIX}/companies/:companyId/secret-provider-configs/discovery/preview`,
|
||||
costs: `${API_PREFIX}/costs`,
|
||||
activity: `${API_PREFIX}/activity`,
|
||||
dashboard: `${API_PREFIX}/dashboard`,
|
||||
|
||||
@@ -558,6 +558,10 @@ export type {
|
||||
SecretProviderConfigPayload,
|
||||
SecretProviderConfigHealthDetails,
|
||||
SecretProviderConfigHealthResponse,
|
||||
SecretProviderConfigDiscoveryCandidate,
|
||||
SecretProviderConfigDiscoveryPreviewResult,
|
||||
SecretProviderConfigDiscoverySample,
|
||||
SecretProviderConfigDiscoverySignal,
|
||||
CompanySecretBinding,
|
||||
CompanySecretBindingTarget,
|
||||
CompanySecretUsageBinding,
|
||||
@@ -885,6 +889,7 @@ export {
|
||||
createSecretSchema,
|
||||
createSecretProviderConfigSchema,
|
||||
updateSecretProviderConfigSchema,
|
||||
secretProviderConfigDiscoveryPreviewSchema,
|
||||
remoteSecretImportPreviewSchema,
|
||||
remoteSecretImportSchema,
|
||||
remoteSecretImportSelectionSchema,
|
||||
@@ -911,6 +916,7 @@ export {
|
||||
type CreateSecret,
|
||||
type CreateSecretProviderConfig,
|
||||
type UpdateSecretProviderConfig,
|
||||
type SecretProviderConfigDiscoveryPreview,
|
||||
type RemoteSecretImportPreview,
|
||||
type RemoteSecretImport,
|
||||
type RemoteSecretImportSelection,
|
||||
|
||||
@@ -258,6 +258,10 @@ export type {
|
||||
SecretProviderConfigPayload,
|
||||
SecretProviderConfigHealthDetails,
|
||||
SecretProviderConfigHealthResponse,
|
||||
SecretProviderConfigDiscoveryCandidate,
|
||||
SecretProviderConfigDiscoveryPreviewResult,
|
||||
SecretProviderConfigDiscoverySample,
|
||||
SecretProviderConfigDiscoverySignal,
|
||||
CompanySecretBinding,
|
||||
CompanySecretBindingTarget,
|
||||
CompanySecretUsageBinding,
|
||||
|
||||
@@ -138,6 +138,43 @@ export interface SecretProviderConfigHealthResponse {
|
||||
checkedAt: Date;
|
||||
}
|
||||
|
||||
export interface SecretProviderConfigDiscoverySignal {
|
||||
namespace: string | null;
|
||||
secretNamePrefix: string | null;
|
||||
environmentTag: string | null;
|
||||
ownerTag: string | null;
|
||||
kmsKeyId: string | null;
|
||||
hasKmsKey: boolean;
|
||||
sampleCount: number;
|
||||
paperclipManagedSampleCount: number;
|
||||
skippedForeignPaperclipSampleCount: number;
|
||||
}
|
||||
|
||||
export interface SecretProviderConfigDiscoverySample {
|
||||
name: string;
|
||||
hasKmsKey: boolean;
|
||||
tagKeys: string[];
|
||||
}
|
||||
|
||||
export interface SecretProviderConfigDiscoveryCandidate {
|
||||
provider: SecretProvider;
|
||||
displayName: string;
|
||||
config: SecretProviderConfigPayload;
|
||||
sampleCount: number;
|
||||
samples: SecretProviderConfigDiscoverySample[];
|
||||
signals: SecretProviderConfigDiscoverySignal;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export interface SecretProviderConfigDiscoveryPreviewResult {
|
||||
provider: SecretProvider;
|
||||
nextToken: string | null;
|
||||
sampledSecretCount: number;
|
||||
skippedForeignPaperclipSampleCount: number;
|
||||
candidates: SecretProviderConfigDiscoveryCandidate[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export interface CompanySecretVersion {
|
||||
id: string;
|
||||
secretId: string;
|
||||
|
||||
@@ -293,6 +293,7 @@ export {
|
||||
createSecretSchema,
|
||||
createSecretProviderConfigSchema,
|
||||
updateSecretProviderConfigSchema,
|
||||
secretProviderConfigDiscoveryPreviewSchema,
|
||||
remoteSecretImportPreviewSchema,
|
||||
remoteSecretImportSchema,
|
||||
remoteSecretImportSelectionSchema,
|
||||
@@ -309,6 +310,7 @@ export {
|
||||
type CreateSecret,
|
||||
type CreateSecretProviderConfig,
|
||||
type UpdateSecretProviderConfig,
|
||||
type SecretProviderConfigDiscoveryPreview,
|
||||
type RemoteSecretImportPreview,
|
||||
type RemoteSecretImport,
|
||||
type RemoteSecretImportSelection,
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
createSecretSchema,
|
||||
remoteSecretImportPreviewSchema,
|
||||
remoteSecretImportSchema,
|
||||
secretProviderConfigDiscoveryPreviewSchema,
|
||||
secretProviderConfigPayloadSchema,
|
||||
updateSecretProviderConfigSchema,
|
||||
} from "./secret.js";
|
||||
@@ -140,6 +141,40 @@ describe("secret validators", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("validates AWS provider vault discovery draft config without allowing sensitive keys", () => {
|
||||
expect(
|
||||
secretProviderConfigDiscoveryPreviewSchema.parse({
|
||||
provider: "aws_secrets_manager",
|
||||
config: {
|
||||
region: "us-east-1",
|
||||
namespace: "production",
|
||||
secretNamePrefix: "paperclip",
|
||||
},
|
||||
query: "paperclip",
|
||||
pageSize: 50,
|
||||
}),
|
||||
).toEqual({
|
||||
provider: "aws_secrets_manager",
|
||||
config: {
|
||||
region: "us-east-1",
|
||||
namespace: "production",
|
||||
secretNamePrefix: "paperclip",
|
||||
},
|
||||
query: "paperclip",
|
||||
pageSize: 50,
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
secretProviderConfigDiscoveryPreviewSchema.parse({
|
||||
provider: "aws_secrets_manager",
|
||||
config: {
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIA...",
|
||||
},
|
||||
}),
|
||||
).toThrow(/sensitive field/i);
|
||||
});
|
||||
|
||||
it("caps AWS remote import paging and row counts", () => {
|
||||
expect(() =>
|
||||
remoteSecretImportPreviewSchema.parse({
|
||||
|
||||
@@ -262,6 +262,30 @@ export const remoteSecretImportPreviewSchema = z.object({
|
||||
|
||||
export type RemoteSecretImportPreview = z.infer<typeof remoteSecretImportPreviewSchema>;
|
||||
|
||||
export const secretProviderConfigDiscoveryPreviewSchema = z.object({
|
||||
provider: z.enum(SECRET_PROVIDERS),
|
||||
config: z.record(z.unknown()).default({}),
|
||||
query: z.string().trim().max(200).optional().nullable(),
|
||||
nextToken: z.string().trim().min(1).max(4096).optional().nullable(),
|
||||
pageSize: z.number().int().min(1).max(100).optional(),
|
||||
}).superRefine((value, ctx) => {
|
||||
rejectSensitiveProviderConfigKeys(value.config, ctx);
|
||||
const parsed = secretProviderConfigPayloadSchema.safeParse({
|
||||
provider: value.provider,
|
||||
config: value.config,
|
||||
});
|
||||
if (!parsed.success) {
|
||||
for (const issue of parsed.error.issues) {
|
||||
ctx.addIssue({
|
||||
...issue,
|
||||
path: issue.path[0] === "config" ? issue.path : ["config", ...issue.path],
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export type SecretProviderConfigDiscoveryPreview = z.infer<typeof secretProviderConfigDiscoveryPreviewSchema>;
|
||||
|
||||
export const remoteSecretImportSelectionSchema = z.object({
|
||||
externalRef: z.string().trim().min(1).max(2048),
|
||||
name: z.string().trim().min(1).max(160).optional().nullable(),
|
||||
|
||||
@@ -454,6 +454,103 @@ describe("awsSecretsManagerProvider", () => {
|
||||
expect(JSON.stringify(listed)).not.toContain("team");
|
||||
});
|
||||
|
||||
it("discovers AWS provider vault prefill candidates from metadata without reading values", async () => {
|
||||
const calls: Array<{ op: string; input: Record<string, unknown> }> = [];
|
||||
const provider = createAwsSecretsManagerProvider({
|
||||
gateway: {
|
||||
async createSecret() {
|
||||
throw new Error("not used");
|
||||
},
|
||||
async putSecretValue() {
|
||||
throw new Error("not used");
|
||||
},
|
||||
async getSecretValue() {
|
||||
throw new Error("GetSecretValue must not be used for provider vault discovery");
|
||||
},
|
||||
async deleteSecret() {
|
||||
throw new Error("not used");
|
||||
},
|
||||
async listSecrets(input) {
|
||||
calls.push({ op: "listSecrets", input });
|
||||
return {
|
||||
NextToken: "next-page",
|
||||
SecretList: [
|
||||
{
|
||||
ARN: "arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/prod-use1/company-1/openai",
|
||||
Name: "paperclip/prod-use1/company-1/openai",
|
||||
KmsKeyId: "arn:aws:kms:us-east-1:123456789012:key/prod",
|
||||
Tags: [
|
||||
{ Key: "paperclip:managed-by", Value: "paperclip" },
|
||||
{ Key: "paperclip:deployment-id", Value: "prod-use1" },
|
||||
{ Key: "paperclip:company-id", Value: "company-1" },
|
||||
{ Key: "paperclip:environment", Value: "production" },
|
||||
{ Key: "paperclip:provider-owner", Value: "platform" },
|
||||
],
|
||||
},
|
||||
{
|
||||
ARN: "arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/prod-use1/company-2/stripe",
|
||||
Name: "paperclip/prod-use1/company-2/stripe",
|
||||
Tags: [
|
||||
{ Key: "paperclip:managed-by", Value: "paperclip" },
|
||||
{ Key: "paperclip:company-id", Value: "company-2" },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const preview = await provider.discoverProviderConfigs?.({
|
||||
companyId: "company-1",
|
||||
providerConfig: {
|
||||
id: "draft",
|
||||
provider: "aws_secrets_manager",
|
||||
status: "ready",
|
||||
config: { region: "us-east-1" },
|
||||
},
|
||||
query: "paperclip",
|
||||
pageSize: 25,
|
||||
});
|
||||
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
op: "listSecrets",
|
||||
input: {
|
||||
MaxResults: 25,
|
||||
NextToken: undefined,
|
||||
IncludePlannedDeletion: false,
|
||||
Filters: [{ Key: "all", Values: ["paperclip"] }],
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(preview).toMatchObject({
|
||||
provider: "aws_secrets_manager",
|
||||
nextToken: "next-page",
|
||||
sampledSecretCount: 1,
|
||||
skippedForeignPaperclipSampleCount: 1,
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
displayName: "AWS production",
|
||||
config: expect.objectContaining({
|
||||
region: "us-east-1",
|
||||
namespace: "prod-use1",
|
||||
secretNamePrefix: "paperclip",
|
||||
kmsKeyId: "arn:aws:kms:us-east-1:123456789012:key/prod",
|
||||
ownerTag: "platform",
|
||||
environmentTag: "production",
|
||||
}),
|
||||
signals: expect.objectContaining({
|
||||
paperclipManagedSampleCount: 1,
|
||||
skippedForeignPaperclipSampleCount: 1,
|
||||
}),
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(JSON.stringify(preview)).not.toContain("SecretString");
|
||||
expect(JSON.stringify(preview)).not.toContain("company-2/stripe");
|
||||
});
|
||||
|
||||
it("redacts AWS provider exception text when remote listing fails", async () => {
|
||||
const rawProviderMessage =
|
||||
"AccessDeniedException: User: arn:aws:sts::123456789012:assumed-role/prod/Paperclip is not authorized to perform secretsmanager:ListSecrets on arn:aws:secretsmanager:us-east-1:123456789012:secret:prod/openai";
|
||||
|
||||
@@ -9,6 +9,7 @@ const mockSecretService = vi.hoisted(() => ({
|
||||
listProviders: vi.fn(),
|
||||
checkProviders: vi.fn(),
|
||||
listProviderConfigs: vi.fn(),
|
||||
previewProviderConfigDiscovery: vi.fn(),
|
||||
getProviderConfigById: vi.fn(),
|
||||
createProviderConfig: vi.fn(),
|
||||
updateProviderConfig: vi.fn(),
|
||||
@@ -117,6 +118,22 @@ describe("secret routes", () => {
|
||||
expect(mockSecretService.listProviderConfigs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects provider vault discovery preview for non-board actors", async () => {
|
||||
const res = await request(createApp({
|
||||
type: "agent",
|
||||
agentId: "agent-1",
|
||||
companyId: "company-1",
|
||||
}))
|
||||
.post("/api/companies/company-1/secret-provider-configs/discovery/preview")
|
||||
.send({
|
||||
provider: "aws_secrets_manager",
|
||||
config: { region: "us-east-1" },
|
||||
});
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(mockSecretService.previewProviderConfigDiscovery).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects sensitive provider vault config fields", async () => {
|
||||
const res = await request(createApp()).post("/api/companies/company-1/secret-provider-configs").send({
|
||||
provider: "aws_secrets_manager",
|
||||
@@ -132,6 +149,92 @@ describe("secret routes", () => {
|
||||
expect(mockSecretService.createProviderConfig).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects sensitive provider vault discovery draft config fields", async () => {
|
||||
const res = await request(createApp())
|
||||
.post("/api/companies/company-1/secret-provider-configs/discovery/preview")
|
||||
.send({
|
||||
provider: "aws_secrets_manager",
|
||||
config: {
|
||||
region: "us-east-1",
|
||||
secretAccessKey: "secret",
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(JSON.stringify(res.body)).toMatch(/sensitive field/i);
|
||||
expect(mockSecretService.previewProviderConfigDiscovery).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("previews provider vault discovery and logs only aggregate metadata", async () => {
|
||||
mockSecretService.previewProviderConfigDiscovery.mockResolvedValue({
|
||||
provider: "aws_secrets_manager",
|
||||
nextToken: null,
|
||||
sampledSecretCount: 2,
|
||||
skippedForeignPaperclipSampleCount: 0,
|
||||
candidates: [
|
||||
{
|
||||
provider: "aws_secrets_manager",
|
||||
displayName: "AWS production",
|
||||
config: {
|
||||
region: "us-east-1",
|
||||
namespace: "prod-use1",
|
||||
secretNamePrefix: "paperclip",
|
||||
environmentTag: "production",
|
||||
ownerTag: "platform",
|
||||
kmsKeyId: null,
|
||||
},
|
||||
sampleCount: 2,
|
||||
samples: [
|
||||
{ name: "paperclip/prod-use1/company-1/openai", hasKmsKey: false, tagKeys: ["environment"] },
|
||||
],
|
||||
signals: {
|
||||
namespace: "prod-use1",
|
||||
secretNamePrefix: "paperclip",
|
||||
environmentTag: "production",
|
||||
ownerTag: "platform",
|
||||
kmsKeyId: null,
|
||||
hasKmsKey: false,
|
||||
sampleCount: 2,
|
||||
paperclipManagedSampleCount: 0,
|
||||
skippedForeignPaperclipSampleCount: 0,
|
||||
},
|
||||
warnings: [],
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
const res = await request(createApp())
|
||||
.post("/api/companies/company-1/secret-provider-configs/discovery/preview")
|
||||
.send({
|
||||
provider: "aws_secrets_manager",
|
||||
config: { region: "us-east-1" },
|
||||
query: "paperclip",
|
||||
pageSize: 25,
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockSecretService.previewProviderConfigDiscovery).toHaveBeenCalledWith("company-1", {
|
||||
provider: "aws_secrets_manager",
|
||||
config: { region: "us-east-1" },
|
||||
query: "paperclip",
|
||||
nextToken: undefined,
|
||||
pageSize: 25,
|
||||
});
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
|
||||
action: "secret_provider_config.discovery_previewed",
|
||||
entityType: "secret_provider_config_discovery",
|
||||
entityId: "company-1",
|
||||
details: {
|
||||
provider: "aws_secrets_manager",
|
||||
candidateCount: 1,
|
||||
sampledSecretCount: 2,
|
||||
warningCount: 0,
|
||||
},
|
||||
}));
|
||||
expect(JSON.stringify(mockLogActivity.mock.calls)).not.toContain("paperclip/prod-use1/company-1/openai");
|
||||
});
|
||||
|
||||
it("rejects ready status for coming-soon provider vaults", async () => {
|
||||
const res = await request(createApp()).post("/api/companies/company-1/secret-provider-configs").send({
|
||||
provider: "vault",
|
||||
|
||||
@@ -1207,6 +1207,111 @@ describeEmbeddedPostgres("secretService", () => {
|
||||
expect(thrown instanceof Error ? thrown.message : String(thrown)).not.toContain("arn:aws");
|
||||
});
|
||||
|
||||
it("previews AWS provider vault discovery from draft config without persisting a provider vault", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const svc = secretService(db);
|
||||
const discoverSpy = vi.spyOn(awsSecretsManagerProvider, "discoverProviderConfigs").mockResolvedValue({
|
||||
provider: "aws_secrets_manager",
|
||||
nextToken: null,
|
||||
sampledSecretCount: 1,
|
||||
skippedForeignPaperclipSampleCount: 0,
|
||||
candidates: [
|
||||
{
|
||||
provider: "aws_secrets_manager",
|
||||
displayName: "AWS production",
|
||||
config: {
|
||||
region: "us-east-1",
|
||||
namespace: "prod-use1",
|
||||
secretNamePrefix: "paperclip",
|
||||
kmsKeyId: null,
|
||||
ownerTag: "platform",
|
||||
environmentTag: "production",
|
||||
},
|
||||
sampleCount: 1,
|
||||
samples: [
|
||||
{ name: "paperclip/prod-use1/company-1/openai", hasKmsKey: false, tagKeys: ["paperclip:environment"] },
|
||||
],
|
||||
signals: {
|
||||
namespace: "prod-use1",
|
||||
secretNamePrefix: "paperclip",
|
||||
environmentTag: "production",
|
||||
ownerTag: "platform",
|
||||
kmsKeyId: null,
|
||||
hasKmsKey: false,
|
||||
sampleCount: 1,
|
||||
paperclipManagedSampleCount: 0,
|
||||
skippedForeignPaperclipSampleCount: 0,
|
||||
},
|
||||
warnings: [],
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
const preview = await svc.previewProviderConfigDiscovery(companyId, {
|
||||
provider: "aws_secrets_manager",
|
||||
config: { region: "us-east-1" },
|
||||
query: "openai",
|
||||
pageSize: 25,
|
||||
});
|
||||
|
||||
expect(discoverSpy).toHaveBeenCalledWith({
|
||||
companyId,
|
||||
providerConfig: {
|
||||
id: `discovery-preview-${companyId}`,
|
||||
provider: "aws_secrets_manager",
|
||||
status: "ready",
|
||||
config: { region: "us-east-1" },
|
||||
},
|
||||
query: "openai",
|
||||
nextToken: undefined,
|
||||
pageSize: 25,
|
||||
});
|
||||
expect(preview.candidates[0]?.config).toMatchObject({
|
||||
region: "us-east-1",
|
||||
namespace: "prod-use1",
|
||||
});
|
||||
expect(JSON.stringify(preview)).not.toContain("runtime-secret");
|
||||
const persistedVaults = await db.select().from(companySecretProviderConfigs);
|
||||
expect(persistedVaults).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("sanitizes AWS provider vault discovery errors before crossing the service boundary", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const svc = secretService(db);
|
||||
const rawProviderMessage =
|
||||
"AccessDeniedException: User: arn:aws:sts::123456789012:assumed-role/prod/Paperclip is not authorized to perform secretsmanager:ListSecrets";
|
||||
|
||||
vi.spyOn(awsSecretsManagerProvider, "discoverProviderConfigs").mockRejectedValueOnce(
|
||||
new SecretProviderClientError({
|
||||
code: "access_denied",
|
||||
provider: "aws_secrets_manager",
|
||||
operation: "discoverProviderConfigs",
|
||||
message: "AWS Secrets Manager denied the request. Check IAM permissions for this provider vault.",
|
||||
rawMessage: rawProviderMessage,
|
||||
}),
|
||||
);
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await svc.previewProviderConfigDiscovery(companyId, {
|
||||
provider: "aws_secrets_manager",
|
||||
config: { region: "us-east-1" },
|
||||
});
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(thrown).toMatchObject({
|
||||
status: 403,
|
||||
message: "AWS Secrets Manager denied the request. Check IAM permissions for this provider vault.",
|
||||
details: { code: "access_denied" },
|
||||
});
|
||||
expect(JSON.stringify(thrown)).not.toContain("arn:aws");
|
||||
expect(JSON.stringify(thrown)).not.toContain("123456789012");
|
||||
expect(thrown instanceof Error ? thrown.message : String(thrown)).not.toContain("arn:aws");
|
||||
});
|
||||
|
||||
it("imports AWS remote references row-by-row without fetching plaintext", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const svc = secretService(db);
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
remoteSecretImportPreviewSchema,
|
||||
remoteSecretImportSchema,
|
||||
rotateSecretSchema,
|
||||
secretProviderConfigDiscoveryPreviewSchema,
|
||||
updateSecretProviderConfigSchema,
|
||||
updateSecretSchema,
|
||||
} from "@paperclipai/shared";
|
||||
@@ -41,6 +42,41 @@ export function secretRoutes(db: Db) {
|
||||
res.json(await svc.listProviderConfigs(companyId));
|
||||
});
|
||||
|
||||
router.post(
|
||||
"/companies/:companyId/secret-provider-configs/discovery/preview",
|
||||
validate(secretProviderConfigDiscoveryPreviewSchema),
|
||||
async (req, res) => {
|
||||
assertBoard(req);
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
|
||||
const preview = await svc.previewProviderConfigDiscovery(companyId, {
|
||||
provider: req.body.provider,
|
||||
config: req.body.config,
|
||||
query: req.body.query,
|
||||
nextToken: req.body.nextToken,
|
||||
pageSize: req.body.pageSize,
|
||||
});
|
||||
|
||||
await logActivity(db, {
|
||||
companyId,
|
||||
actorType: "user",
|
||||
actorId: req.actor.userId ?? "board",
|
||||
action: "secret_provider_config.discovery_previewed",
|
||||
entityType: "secret_provider_config_discovery",
|
||||
entityId: companyId,
|
||||
details: {
|
||||
provider: preview.provider,
|
||||
candidateCount: preview.candidates.length,
|
||||
sampledSecretCount: preview.sampledSecretCount,
|
||||
warningCount: preview.warnings.length,
|
||||
},
|
||||
});
|
||||
|
||||
res.json(preview);
|
||||
},
|
||||
);
|
||||
|
||||
router.post("/companies/:companyId/secret-provider-configs", validate(createSecretProviderConfigSchema), async (req, res) => {
|
||||
assertBoard(req);
|
||||
const companyId = req.params.companyId as string;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { createHash, createHmac } from "node:crypto";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import type { DeploymentMode } from "@paperclipai/shared";
|
||||
import type { DeploymentMode, SecretProviderConfigDiscoveryPreviewResult } from "@paperclipai/shared";
|
||||
import { unprocessable } from "../errors.js";
|
||||
import type {
|
||||
PreparedSecretVersion,
|
||||
@@ -24,6 +24,8 @@ const DEFAULT_DELETE_RECOVERY_WINDOW_DAYS = 30;
|
||||
const AWS_SECRETS_MANAGER_REQUEST_TIMEOUT_MS = 30_000;
|
||||
const AWS_CREDENTIAL_CACHE_TTL_MS = 5 * 60_000;
|
||||
const AWS_CREDENTIAL_EXPIRATION_SKEW_MS = 60_000;
|
||||
const PROVIDER_CONFIG_DISCOVERY_SAMPLE_LIMIT = 3;
|
||||
const PROVIDER_CONFIG_DISCOVERY_CANDIDATE_LIMIT = 6;
|
||||
const AWS_RUNTIME_CREDENTIAL_WARNING =
|
||||
"AWS bootstrap credentials must be available to the Paperclip server runtime through the AWS SDK default credential provider chain: IAM role/workload identity, AWS_PROFILE/SSO/shared credentials, web identity, container/instance metadata, or short-lived shell credentials.";
|
||||
const AWS_CREDENTIAL_CUSTODY_WARNING =
|
||||
@@ -590,6 +592,234 @@ function createRemoteSecretMetadata(entry: AwsSecretsManagerListSecretEntry): Re
|
||||
};
|
||||
}
|
||||
|
||||
function tagValue(tags: Map<string, string>, keys: string[]) {
|
||||
for (const key of keys) {
|
||||
const value = tags.get(key.toLowerCase());
|
||||
if (value) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeAwsTags(tags: AwsSecretsManagerTag[] | undefined) {
|
||||
const normalized = new Map<string, string>();
|
||||
for (const tag of tags ?? []) {
|
||||
const key = tag.Key?.trim();
|
||||
const value = tag.Value?.trim();
|
||||
if (key && value) normalized.set(key.toLowerCase(), value);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function commonValue(values: Array<string | null | undefined>) {
|
||||
const nonEmpty = values.filter((value): value is string => Boolean(value?.trim()));
|
||||
if (nonEmpty.length === 0) return null;
|
||||
const first = nonEmpty[0];
|
||||
return nonEmpty.every((value) => value === first) ? first : null;
|
||||
}
|
||||
|
||||
function uniqueValues(values: Array<string | null | undefined>) {
|
||||
return [...new Set(values.filter((value): value is string => Boolean(value?.trim())))];
|
||||
}
|
||||
|
||||
function pathSegments(name: string) {
|
||||
return name.split("/").map((segment) => segment.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function inferPathSignals(entry: AwsSecretsManagerListSecretEntry, tags: Map<string, string>) {
|
||||
const name = entry.Name?.trim() || entry.ARN?.trim() || "";
|
||||
const segments = pathSegments(name);
|
||||
const paperclipDeploymentId = tagValue(tags, ["paperclip:deployment-id"]);
|
||||
const paperclipManaged = tagValue(tags, ["paperclip:managed-by"])?.toLowerCase() === "paperclip";
|
||||
|
||||
if (paperclipDeploymentId || paperclipManaged) {
|
||||
return {
|
||||
prefix: segments[0] ?? DEFAULT_PREFIX,
|
||||
namespace: paperclipDeploymentId ?? segments[1] ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
if (segments.length >= 3) {
|
||||
return {
|
||||
prefix: segments[0] ?? null,
|
||||
namespace: segments[1] ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
prefix: segments[0] ?? null,
|
||||
namespace: null,
|
||||
};
|
||||
}
|
||||
|
||||
function discoveryDisplayName(input: {
|
||||
environmentTag: string | null;
|
||||
ownerTag: string | null;
|
||||
namespace: string | null;
|
||||
secretNamePrefix: string | null;
|
||||
}) {
|
||||
const qualifier =
|
||||
input.environmentTag ??
|
||||
input.namespace ??
|
||||
input.secretNamePrefix ??
|
||||
input.ownerTag ??
|
||||
"discovered";
|
||||
return `AWS ${qualifier}`;
|
||||
}
|
||||
|
||||
function discoverAwsProviderConfigCandidates(input: {
|
||||
companyId: string;
|
||||
config: AwsSecretsManagerConfig;
|
||||
draftConfig: Record<string, unknown>;
|
||||
entries: AwsSecretsManagerListSecretEntry[];
|
||||
nextToken: string | null;
|
||||
}): SecretProviderConfigDiscoveryPreviewResult {
|
||||
type DiscoverySample = {
|
||||
entry: AwsSecretsManagerListSecretEntry;
|
||||
name: string;
|
||||
tags: Map<string, string>;
|
||||
prefix: string | null;
|
||||
namespace: string | null;
|
||||
environmentTag: string | null;
|
||||
ownerTag: string | null;
|
||||
kmsKeyId: string | null;
|
||||
paperclipManaged: boolean;
|
||||
paperclipCompanyId: string | null;
|
||||
};
|
||||
|
||||
const skippedWarnings: string[] = [];
|
||||
let skippedForeignPaperclipSampleCount = 0;
|
||||
const samples: DiscoverySample[] = [];
|
||||
|
||||
for (const entry of input.entries) {
|
||||
const name = entry.Name?.trim() || entry.ARN?.trim();
|
||||
if (!name) continue;
|
||||
const tags = normalizeAwsTags(entry.Tags);
|
||||
const paperclipManaged = tagValue(tags, ["paperclip:managed-by"])?.toLowerCase() === "paperclip";
|
||||
const paperclipCompanyId = tagValue(tags, ["paperclip:company-id"]);
|
||||
if (paperclipManaged && paperclipCompanyId !== input.companyId) {
|
||||
skippedForeignPaperclipSampleCount += 1;
|
||||
continue;
|
||||
}
|
||||
const path = inferPathSignals(entry, tags);
|
||||
samples.push({
|
||||
entry,
|
||||
name,
|
||||
tags,
|
||||
prefix: path.prefix,
|
||||
namespace: path.namespace,
|
||||
environmentTag: tagValue(tags, ["paperclip:environment", "environment", "env", "stage"]),
|
||||
ownerTag: tagValue(tags, ["paperclip:provider-owner", "owner", "team", "service", "application"]),
|
||||
kmsKeyId: asOptionalNonEmptyString(entry.KmsKeyId),
|
||||
paperclipManaged,
|
||||
paperclipCompanyId,
|
||||
});
|
||||
}
|
||||
|
||||
if (skippedForeignPaperclipSampleCount > 0) {
|
||||
skippedWarnings.push(
|
||||
`Skipped ${skippedForeignPaperclipSampleCount} Paperclip-managed AWS secret sample(s) that were not tagged for this company.`,
|
||||
);
|
||||
}
|
||||
|
||||
const draftNamespace = asOptionalNonEmptyString(input.draftConfig.namespace);
|
||||
const draftPrefix = asOptionalNonEmptyString(input.draftConfig.secretNamePrefix);
|
||||
const draftKmsKeyId = asOptionalNonEmptyString(input.draftConfig.kmsKeyId);
|
||||
const draftEnvironmentTag = asOptionalNonEmptyString(input.draftConfig.environmentTag);
|
||||
const draftOwnerTag = asOptionalNonEmptyString(input.draftConfig.ownerTag);
|
||||
const groups = new Map<string, DiscoverySample[]>();
|
||||
|
||||
for (const sample of samples) {
|
||||
const key = [
|
||||
draftPrefix ?? sample.prefix ?? "",
|
||||
draftNamespace ?? sample.namespace ?? "",
|
||||
].join("\0");
|
||||
groups.set(key, [...(groups.get(key) ?? []), sample]);
|
||||
}
|
||||
|
||||
const candidates = [...groups.values()]
|
||||
.sort((a, b) => b.length - a.length)
|
||||
.slice(0, PROVIDER_CONFIG_DISCOVERY_CANDIDATE_LIMIT)
|
||||
.map((group) => {
|
||||
const prefix = draftPrefix ?? commonValue(group.map((sample) => sample.prefix)) ?? input.config.prefix;
|
||||
const namespace = draftNamespace ?? commonValue(group.map((sample) => sample.namespace)) ?? null;
|
||||
const environmentTag = draftEnvironmentTag ?? commonValue(group.map((sample) => sample.environmentTag));
|
||||
const ownerTag = draftOwnerTag ?? commonValue(group.map((sample) => sample.ownerTag));
|
||||
const kmsKeys = uniqueValues(group.map((sample) => sample.kmsKeyId));
|
||||
const commonKmsKey = commonValue(group.map((sample) => sample.kmsKeyId));
|
||||
const kmsKeyId = draftKmsKeyId ?? commonKmsKey;
|
||||
const candidateWarnings: string[] = [];
|
||||
|
||||
if (!namespace) {
|
||||
candidateWarnings.push("No stable namespace signal was found in the sampled AWS secret names or tags.");
|
||||
}
|
||||
if (!environmentTag) {
|
||||
candidateWarnings.push("No common environment tag was found in the sampled AWS secrets.");
|
||||
}
|
||||
if (!ownerTag) {
|
||||
candidateWarnings.push("No common owner/team tag was found in the sampled AWS secrets.");
|
||||
}
|
||||
if (kmsKeys.length > 1 && !draftKmsKeyId) {
|
||||
candidateWarnings.push("Sampled AWS secrets use multiple KMS keys; choose the intended KMS key before saving.");
|
||||
}
|
||||
if (group.some((sample) => sample.paperclipManaged && sample.paperclipCompanyId === input.companyId)) {
|
||||
candidateWarnings.push("Sample includes Paperclip-managed secrets for this company; do not import them as external references.");
|
||||
}
|
||||
|
||||
return {
|
||||
provider: "aws_secrets_manager" as const,
|
||||
displayName: discoveryDisplayName({
|
||||
environmentTag,
|
||||
ownerTag,
|
||||
namespace,
|
||||
secretNamePrefix: prefix,
|
||||
}),
|
||||
config: {
|
||||
region: input.config.region,
|
||||
namespace,
|
||||
secretNamePrefix: prefix,
|
||||
kmsKeyId: kmsKeyId ?? null,
|
||||
ownerTag,
|
||||
environmentTag,
|
||||
},
|
||||
sampleCount: group.length,
|
||||
samples: group.slice(0, PROVIDER_CONFIG_DISCOVERY_SAMPLE_LIMIT).map((sample) => ({
|
||||
name: sample.name,
|
||||
hasKmsKey: Boolean(sample.kmsKeyId),
|
||||
tagKeys: [...sample.tags.keys()].sort(),
|
||||
})),
|
||||
signals: {
|
||||
namespace,
|
||||
secretNamePrefix: prefix,
|
||||
environmentTag,
|
||||
ownerTag,
|
||||
kmsKeyId: kmsKeyId ?? null,
|
||||
hasKmsKey: kmsKeys.length > 0,
|
||||
sampleCount: group.length,
|
||||
paperclipManagedSampleCount: group.filter((sample) => sample.paperclipManaged).length,
|
||||
skippedForeignPaperclipSampleCount,
|
||||
},
|
||||
warnings: candidateWarnings,
|
||||
};
|
||||
});
|
||||
|
||||
const warnings = [...skippedWarnings];
|
||||
if (samples.length === 0) {
|
||||
warnings.push("AWS Secrets Manager returned no metadata samples for this draft provider vault config.");
|
||||
}
|
||||
if (groups.size > PROVIDER_CONFIG_DISCOVERY_CANDIDATE_LIMIT) {
|
||||
warnings.push("Additional AWS secret name groups were omitted from this preview; refine the query to inspect them.");
|
||||
}
|
||||
|
||||
return {
|
||||
provider: "aws_secrets_manager",
|
||||
nextToken: input.nextToken,
|
||||
sampledSecretCount: samples.length,
|
||||
skippedForeignPaperclipSampleCount,
|
||||
candidates,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
function asAwsSecretsManagerMaterial(value: StoredSecretVersionMaterial): AwsSecretsManagerMaterial {
|
||||
if (
|
||||
value &&
|
||||
@@ -983,6 +1213,36 @@ export function createAwsSecretsManagerProvider(
|
||||
normalizeAwsError("listSecrets", error);
|
||||
}
|
||||
},
|
||||
async discoverProviderConfigs(input): Promise<SecretProviderConfigDiscoveryPreviewResult> {
|
||||
const config = resolveConfig(input.providerConfig);
|
||||
const gateway = resolveGateway(config);
|
||||
const query = input.query?.trim();
|
||||
const pageSize =
|
||||
input.pageSize && Number.isFinite(input.pageSize)
|
||||
? Math.min(Math.max(Math.trunc(input.pageSize), 1), 100)
|
||||
: 100;
|
||||
|
||||
try {
|
||||
if (!gateway.listSecrets) {
|
||||
throw new Error("ListSecrets gateway operation is unavailable");
|
||||
}
|
||||
const listed = await gateway.listSecrets({
|
||||
MaxResults: pageSize,
|
||||
NextToken: input.nextToken?.trim() || undefined,
|
||||
IncludePlannedDeletion: false,
|
||||
Filters: query ? [{ Key: "all", Values: [query] }] : undefined,
|
||||
});
|
||||
return discoverAwsProviderConfigCandidates({
|
||||
companyId: input.companyId,
|
||||
config,
|
||||
draftConfig: input.providerConfig.config,
|
||||
entries: listed.SecretList ?? [],
|
||||
nextToken: listed.NextToken ?? null,
|
||||
});
|
||||
} catch (error) {
|
||||
normalizeAwsError("discoverProviderConfigs", error);
|
||||
}
|
||||
},
|
||||
async resolveVersion(input) {
|
||||
const config = resolveConfig(input.providerConfig);
|
||||
const gateway = resolveGateway(config);
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { SecretProvider, SecretProviderDescriptor } from "@paperclipai/shared";
|
||||
import type {
|
||||
SecretProvider,
|
||||
SecretProviderConfigDiscoveryPreviewResult,
|
||||
SecretProviderDescriptor,
|
||||
} from "@paperclipai/shared";
|
||||
import type { DeploymentMode } from "@paperclipai/shared";
|
||||
|
||||
export interface StoredSecretVersionMaterial {
|
||||
@@ -152,6 +156,13 @@ export interface SecretProviderModule {
|
||||
nextToken?: string | null;
|
||||
pageSize?: number;
|
||||
}): Promise<RemoteSecretListResult>;
|
||||
discoverProviderConfigs?(input: {
|
||||
companyId: string;
|
||||
providerConfig: SecretProviderVaultRuntimeConfig;
|
||||
query?: string | null;
|
||||
nextToken?: string | null;
|
||||
pageSize?: number;
|
||||
}): Promise<SecretProviderConfigDiscoveryPreviewResult>;
|
||||
resolveVersion(input: {
|
||||
material: StoredSecretVersionMaterial;
|
||||
externalRef: string | null;
|
||||
|
||||
@@ -20,6 +20,7 @@ import type {
|
||||
RemoteSecretImportCandidate,
|
||||
RemoteSecretImportConflict,
|
||||
RemoteSecretImportRowResult,
|
||||
SecretProviderConfigDiscoveryPreviewResult,
|
||||
SecretBindingTargetType,
|
||||
SecretProvider,
|
||||
SecretProviderConfigHealthResponse,
|
||||
@@ -34,6 +35,7 @@ import {
|
||||
isUuidLike,
|
||||
normalizeAgentUrlKey,
|
||||
secretProviderConfigPayloadSchema,
|
||||
secretProviderConfigDiscoveryPreviewSchema,
|
||||
updateSecretProviderConfigSchema,
|
||||
} from "@paperclipai/shared";
|
||||
import { conflict, HttpError, notFound, unprocessable } from "../errors.js";
|
||||
@@ -471,6 +473,19 @@ export function secretService(db: Db) {
|
||||
return parsed.data.config;
|
||||
}
|
||||
|
||||
function toDraftProviderVaultRuntimeConfig(input: {
|
||||
companyId: string;
|
||||
provider: SecretProvider;
|
||||
config: Record<string, unknown>;
|
||||
}): SecretProviderVaultRuntimeConfig {
|
||||
return {
|
||||
id: `discovery-preview-${input.companyId}`,
|
||||
provider: input.provider,
|
||||
status: "ready",
|
||||
config: validateProviderConfigPayload(input.provider, input.config),
|
||||
};
|
||||
}
|
||||
|
||||
function providerConfigHealth(input: {
|
||||
id: string;
|
||||
provider: SecretProvider;
|
||||
@@ -949,6 +964,54 @@ export function secretService(db: Db) {
|
||||
|
||||
checkProviders: () => checkSecretProviders(),
|
||||
|
||||
previewProviderConfigDiscovery: async (
|
||||
companyId: string,
|
||||
input: {
|
||||
provider: SecretProvider;
|
||||
config?: Record<string, unknown>;
|
||||
query?: string | null;
|
||||
nextToken?: string | null;
|
||||
pageSize?: number;
|
||||
},
|
||||
): Promise<SecretProviderConfigDiscoveryPreviewResult> => {
|
||||
const parsed = secretProviderConfigDiscoveryPreviewSchema.safeParse({
|
||||
provider: input.provider,
|
||||
config: input.config ?? {},
|
||||
query: input.query,
|
||||
nextToken: input.nextToken,
|
||||
pageSize: input.pageSize,
|
||||
});
|
||||
if (!parsed.success) {
|
||||
throw unprocessable("Invalid provider vault discovery config", parsed.error.flatten());
|
||||
}
|
||||
const providerId = parsed.data.provider as SecretProvider;
|
||||
const provider = getSecretProvider(providerId);
|
||||
if (!provider.discoverProviderConfigs) {
|
||||
throw unprocessable(`${providerId} provider does not support provider vault discovery`);
|
||||
}
|
||||
const runtimeConfig = toDraftProviderVaultRuntimeConfig({
|
||||
companyId,
|
||||
provider: providerId,
|
||||
config: parsed.data.config,
|
||||
});
|
||||
try {
|
||||
return await provider.discoverProviderConfigs({
|
||||
companyId,
|
||||
providerConfig: runtimeConfig,
|
||||
query: parsed.data.query,
|
||||
nextToken: parsed.data.nextToken,
|
||||
pageSize: parsed.data.pageSize,
|
||||
});
|
||||
} catch (error) {
|
||||
throw remoteProviderHttpError(error, {
|
||||
companyId,
|
||||
provider: providerId,
|
||||
providerConfigId: "discovery-preview",
|
||||
operation: "secret_provider_config.discovery.preview",
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
listProviderConfigs: (companyId: string) =>
|
||||
db
|
||||
.select()
|
||||
|
||||
Reference in new issue
Block a user