Harden Hermes gateway transport and secrets

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-06-26 13:53:36 +00:00
1 parent 401da17d37
commit f0e2bffe42
14 files changed
+799 -65

No files matched your search

@@ -1,4 +1,18 @@
import { envBindingSchema, type SecretVersionSelector } from "@paperclipai/shared";
interface AgentSecretBindingSyncService {
syncSecretRefsForTarget?: (
companyId: string,
target: { targetType: "agent"; targetId: string },
refs: Array<{
secretId: string;
configPath: string;
versionSelector?: SecretVersionSelector;
required?: boolean;
label?: string | null;
}>,
options?: { replaceAll?: boolean },
) => Promise<unknown>;
syncEnvBindingsForTarget?: (
companyId: string,
target: { targetType: "agent"; targetId: string; pathPrefix?: string },
@@ -11,12 +25,63 @@ function asRecord(value: unknown): Record<string, unknown> | null {
return value as Record<string, unknown>;
}
function collectSecretRefs(adapterConfig: unknown): Array<{
secretId: string;
configPath: string;
versionSelector?: SecretVersionSelector;
}> {
const config = asRecord(adapterConfig);
if (!config) return [];
const refs: Array<{
secretId: string;
configPath: string;
versionSelector?: SecretVersionSelector;
}> = [];
const envValue = asRecord(config.env);
for (const [key, rawBinding] of Object.entries(envValue ?? {})) {
const parsed = envBindingSchema.safeParse(rawBinding);
if (!parsed.success) continue;
const binding = parsed.data;
if (typeof binding !== "object" || binding === null || binding.type !== "secret_ref") continue;
refs.push({
secretId: binding.secretId,
configPath: `env.${key}`,
versionSelector: binding.version ?? "latest",
});
}
for (const [key, rawBinding] of Object.entries(config)) {
if (key === "env") continue;
const parsed = envBindingSchema.safeParse(rawBinding);
if (!parsed.success) continue;
const binding = parsed.data;
if (typeof binding !== "object" || binding === null || binding.type !== "secret_ref") continue;
refs.push({
secretId: binding.secretId,
configPath: key,
versionSelector: binding.version ?? "latest",
});
}
return refs;
}
export async function syncAgentAdapterEnvBindings(input: {
secretsSvc: AgentSecretBindingSyncService;
companyId: string;
agentId: string;
adapterConfig: unknown;
}) {
if (input.secretsSvc.syncSecretRefsForTarget) {
await input.secretsSvc.syncSecretRefsForTarget(
input.companyId,
{ targetType: "agent", targetId: input.agentId },
collectSecretRefs(input.adapterConfig),
{ replaceAll: true },
);
return;
}
const envValue = asRecord(asRecord(input.adapterConfig)?.env);
await input.secretsSvc.syncEnvBindingsForTarget?.(
input.companyId,
+24 -1
View File
@@ -428,6 +428,16 @@ export function agentService(db: Db) {
const role = (data.role ?? existing.role) as string;
normalizedPatch.permissions = normalizeAgentPermissions(data.permissions, role);
}
if (
Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterConfig") &&
isPlainRecord(normalizedPatch.adapterConfig)
) {
normalizedPatch.adapterConfig = await secretsSvc.normalizeAdapterConfigForPersistence(
existing.companyId,
normalizedPatch.adapterConfig,
{ adapterType: (normalizedPatch.adapterType ?? existing.adapterType) as string },
);
}
const shouldRecordRevision = Boolean(options?.recordRevision) && hasConfigPatchFields(normalizedPatch);
const beforeConfig = shouldRecordRevision ? buildConfigSnapshot(existing) : null;
@@ -503,11 +513,24 @@ export function agentService(db: Db) {
const role = data.role ?? "general";
const normalizedPermissions = normalizeAgentPermissions(data.permissions, role);
const runtimeConfig = normalizeRuntimeConfigForNewAgent(data.runtimeConfig);
const adapterType = data.adapterType ?? "process";
const adapterConfig = isPlainRecord(data.adapterConfig)
? await secretsSvc.normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType })
: {};
return db.transaction(async (tx) => {
const txDb = tx as unknown as Db;
const created = await tx
.insert(agents)
.values({ ...data, name: uniqueName, companyId, role, permissions: normalizedPermissions, runtimeConfig })
.values({
...data,
name: uniqueName,
companyId,
role,
adapterType,
adapterConfig,
permissions: normalizedPermissions,
runtimeConfig,
})
.returning()
.then((rows) => rows[0]);
await syncAgentSecretBindings(created, txDb);
+1 -1
View File
@@ -3050,7 +3050,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
const normalizedAdapterConfig = await secrets.normalizeAdapterConfigForPersistence(
companyId,
nextAdapterConfig,
{ strictMode: strictSecretsMode },
{ strictMode: strictSecretsMode, adapterType: effectiveAdapterType },
);
await assertImportAdapterConfigConstraints(effectiveAdapterType, normalizedAdapterConfig);
return {
+17 -1
View File
@@ -412,7 +412,10 @@ const INLINE_BASE64_IMAGE_DATA_RE = /("type":"image","source":\{"type":"base64",
type RuntimeConfigSecretResolver = Pick<
ReturnType<typeof secretService>,
"resolveAdapterConfigForRuntime" | "resolveEnvBindings" | "collectMissingRuntimeBindings"
| "resolveAdapterConfigForRuntime"
| "resolveEnvBindings"
| "collectMissingRuntimeBindings"
| "collectMissingAdapterConfigRuntimeBindings"
>;
function formatMissingBindingForOperator(missing: MissingRuntimeBinding): string {
@@ -494,6 +497,7 @@ function assertLowTrustEnvConfigAllowed(envValue: unknown, source: string) {
export async function resolveExecutionRunAdapterConfig(input: {
companyId: string;
agentId?: string | null;
adapterType?: string | null;
issueId?: string | null;
heartbeatRunId?: string | null;
environmentId?: string | null;
@@ -574,6 +578,16 @@ export async function resolveExecutionRunAdapterConfig(input: {
{ consumerType: "agent", consumerId: input.agentId },
)),
);
if (typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings === "function") {
missingBindings.push(
...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(
input.companyId,
executionRunConfig,
input.adapterType ?? null,
{ consumerType: "agent", consumerId: input.agentId },
)),
);
}
}
if (projectEnv && input.projectId) {
missingBindings.push(
@@ -666,6 +680,7 @@ export async function resolveExecutionRunAdapterConfig(input: {
...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}),
}
: undefined,
{ adapterType: input.adapterType ?? null },
);
if (Object.keys(environmentEnvResolution.env).length > 0) {
resolvedConfig.env = {
@@ -8943,6 +8958,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const { resolvedConfig, secretKeys, secretManifest } = await resolveExecutionRunAdapterConfig({
companyId: agent.companyId,
agentId: agent.id,
adapterType: agent.adapterType,
issueId,
heartbeatRunId: run.id,
environmentId: selectedEnvironmentForConfig?.id ?? null,
+325 -42
View File
@@ -1,3 +1,4 @@
import { randomUUID } from "node:crypto";
import { and, desc, eq, inArray, like, ne, notInArray, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
@@ -55,6 +56,7 @@ import type {
} from "../secrets/types.js";
import { isSecretProviderClientError } from "../secrets/types.js";
import { authorizationService } from "./authorization.js";
import { findActiveServerAdapter } from "../adapters/index.js";
const ENV_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
const SENSITIVE_ENV_KEY_RE =
@@ -64,6 +66,9 @@ const COMING_SOON_SECRET_PROVIDERS: ReadonlySet<SecretProvider> = new Set([
"gcp_secret_manager",
"vault",
]);
const FALLBACK_ADAPTER_SCHEMA_SECRET_FIELDS: Readonly<Record<string, readonly string[]>> = {
hermes_gateway: ["apiKey"],
};
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type SecretBindingDb = Pick<Db | DbTransaction, "select" | "delete" | "insert">;
@@ -357,6 +362,11 @@ export function secretService(db: Db) {
strictMode?: boolean;
fieldPath?: string;
};
type NormalizeAdapterConfigOptions = {
strictMode?: boolean;
adapterType?: string | null;
actor?: { userId?: string | null; agentId?: string | null };
};
async function getById(id: string, source: Pick<Db | DbTransaction, "select"> = db) {
return source
@@ -809,16 +819,209 @@ export function secretService(db: Db) {
async function normalizeAdapterConfigForPersistenceInternal(
companyId: string,
adapterConfig: Record<string, unknown>,
opts?: { strictMode?: boolean },
opts?: NormalizeAdapterConfigOptions,
) {
const normalized = { ...adapterConfig };
if (!Object.prototype.hasOwnProperty.call(adapterConfig, "env")) {
return normalized;
if (Object.prototype.hasOwnProperty.call(adapterConfig, "env")) {
normalized.env = await normalizeEnvConfig(companyId, adapterConfig.env, opts);
}
const secretFieldKeys = await listAdapterSchemaSecretFieldKeys(opts?.adapterType);
for (const key of secretFieldKeys) {
if (!Object.prototype.hasOwnProperty.call(adapterConfig, key)) continue;
const value = await normalizeSchemaSecretFieldForPersistence(companyId, {
adapterType: opts?.adapterType ?? null,
key,
rawValue: adapterConfig[key],
actor: opts?.actor,
});
if (value === undefined) {
delete normalized[key];
} else {
normalized[key] = value;
}
}
normalized.env = await normalizeEnvConfig(companyId, adapterConfig.env, opts);
return normalized;
}
async function listAdapterSchemaSecretFieldKeys(adapterType: string | null | undefined): Promise<string[]> {
if (!adapterType) return [];
const adapter = findActiveServerAdapter(adapterType);
const fallback = [...(FALLBACK_ADAPTER_SCHEMA_SECRET_FIELDS[adapterType] ?? [])];
if (!adapter?.getConfigSchema) return fallback;
try {
const schema = await adapter.getConfigSchema();
return [...new Set([
...fallback,
...schema.fields
.filter((field) => field.meta?.secret === true)
.map((field) => field.key),
])];
} catch (err) {
logger.warn({ err, adapterType }, "adapter config schema unavailable while normalizing secret fields");
return fallback;
}
}
async function normalizeSchemaSecretFieldForPersistence(
companyId: string,
input: {
adapterType: string | null;
key: string;
rawValue: unknown;
actor?: { userId?: string | null; agentId?: string | null };
},
): Promise<EnvBinding | undefined> {
if (input.rawValue === null || input.rawValue === undefined) return undefined;
const parsed = envBindingSchema.safeParse(input.rawValue);
if (!parsed.success) {
throw unprocessable(`${input.key} must be a string, plain binding, or secret reference`);
}
const binding = canonicalizeBinding(parsed.data as EnvBinding);
if (binding.type === "secret_ref") {
await assertSecretInCompany(companyId, binding.secretId);
return {
type: "secret_ref",
secretId: binding.secretId,
version: binding.version,
};
}
const value = binding.value.trim();
if (!value) return undefined;
if (value === REDACTED_SENTINEL) {
throw unprocessable(`Refusing to persist redacted placeholder for key: ${input.key}`);
}
const id = randomUUID();
const adapterPart = normalizeSecretKey(input.adapterType ?? "adapter");
const fieldPart = normalizeSecretKey(input.key);
const secret = await createManagedLocalSecret(companyId, {
name: `${adapterPart}.${fieldPart}.${id}`,
key: `${adapterPart}.${fieldPart}.${id}`,
value,
description: `Adapter config secret for ${input.adapterType ?? "adapter"}.${input.key}`,
}, input.actor);
return {
type: "secret_ref",
secretId: secret.id,
version: "latest",
};
}
async function createManagedLocalSecret(
companyId: string,
input: {
name: string;
key: string;
value: string;
description?: string | null;
},
actor?: { userId?: string | null; agentId?: string | null },
) {
const existing = await getByName(companyId, input.name);
if (existing) throw conflict(`Secret already exists: ${input.name}`);
const key = normalizeSecretKey(input.key);
if (!key) throw unprocessable("Secret key is required");
const duplicateKey = await db
.select()
.from(companySecrets)
.where(and(
eq(companySecrets.companyId, companyId),
eq(companySecrets.key, key),
ne(companySecrets.status, "deleted"),
))
.then((rows) => rows[0] ?? null);
if (duplicateKey) throw conflict(`Secret key already exists: ${key}`);
const provider = getSecretProvider("local_encrypted");
const providerConfig = await getSelectableRuntimeProviderConfig({
companyId,
provider: "local_encrypted",
providerConfigId: null,
});
const providerWriteContext = {
companyId,
secretKey: key,
secretName: input.name,
version: 1,
};
const reservedSecret = await db
.insert(companySecrets)
.values({
companyId,
key,
name: input.name,
provider: "local_encrypted",
providerConfigId: null,
status: "archived",
managedMode: "paperclip_managed",
externalRef: null,
providerMetadata: null,
latestVersion: 0,
description: input.description ?? null,
createdByAgentId: actor?.agentId ?? null,
createdByUserId: actor?.userId ?? null,
})
.returning()
.then((rows) => rows[0]);
let prepared: PreparedSecretVersion | null = null;
try {
prepared = await provider.createSecret({
value: input.value,
externalRef: null,
providerConfig,
context: providerWriteContext,
});
const preparedSecret = prepared;
await db.insert(companySecretVersions).values({
secretId: reservedSecret.id,
version: 1,
material: preparedSecret.material,
valueSha256: preparedSecret.valueSha256,
fingerprintSha256: preparedSecret.fingerprintSha256 ?? preparedSecret.valueSha256,
providerVersionRef: preparedSecret.providerVersionRef ?? null,
status: "disabled",
createdByAgentId: actor?.agentId ?? null,
createdByUserId: actor?.userId ?? null,
});
return await db.transaction(async (tx) => {
await tx
.update(companySecretVersions)
.set({ status: "current" })
.where(and(
eq(companySecretVersions.secretId, reservedSecret.id),
eq(companySecretVersions.version, 1),
));
const secret = await tx
.update(companySecrets)
.set({
status: "active",
externalRef: preparedSecret.externalRef,
latestVersion: 1,
lastRotatedAt: new Date(),
updatedAt: new Date(),
})
.where(eq(companySecrets.id, reservedSecret.id))
.returning()
.then((rows) => rows[0]);
if (!secret) throw notFound("Secret not found");
return secret;
});
} catch (error) {
if (prepared) {
await cleanupPreparedProviderWrite({
provider,
prepared,
providerConfig,
context: providerWriteContext,
mode: "delete",
operation: "adapter_config_secret.create_rollback",
}).catch(() => false);
}
await db.delete(companySecrets).where(eq(companySecrets.id, reservedSecret.id)).catch(() => undefined);
throw error;
}
}
function collectTargetIds(
bindings: Array<typeof companySecretBindings.$inferSelect>,
targetType: SecretBindingTargetType,
@@ -2159,6 +2362,7 @@ export function secretService(db: Db) {
required?: boolean;
label?: string | null;
}>,
options?: { replaceAll?: boolean },
) => {
const normalizedRefs: Array<{
secretId: string;
@@ -2181,7 +2385,17 @@ export function secretService(db: Db) {
const pathPrefixes = [...new Set(normalizedRefs.map((ref) => ref.configPath.split(".")[0]))];
await db.transaction(async (tx) => {
if (pathPrefixes.length > 0) {
if (options?.replaceAll) {
await tx
.delete(companySecretBindings)
.where(
and(
eq(companySecretBindings.companyId, companyId),
eq(companySecretBindings.targetType, target.targetType),
eq(companySecretBindings.targetId, target.targetId),
),
);
} else if (pathPrefixes.length > 0) {
for (const pathPrefix of pathPrefixes) {
await tx
.delete(companySecretBindings)
@@ -2351,7 +2565,7 @@ export function secretService(db: Db) {
normalizeAdapterConfigForPersistence: async (
companyId: string,
adapterConfig: Record<string, unknown>,
opts?: { strictMode?: boolean },
opts?: NormalizeAdapterConfigOptions,
) => normalizeAdapterConfigForPersistenceInternal(companyId, adapterConfig, opts),
normalizeEnvBindingsForPersistence: async (
@@ -2363,7 +2577,7 @@ export function secretService(db: Db) {
normalizeHireApprovalPayloadForPersistence: async (
companyId: string,
payload: Record<string, unknown>,
opts?: { strictMode?: boolean },
opts?: NormalizeAdapterConfigOptions,
) => {
const normalized = { ...payload };
const adapterConfig = asRecord(payload.adapterConfig);
@@ -2473,52 +2687,121 @@ export function secretService(db: Db) {
}));
},
collectMissingAdapterConfigRuntimeBindings: async (
companyId: string,
adapterConfig: Record<string, unknown>,
adapterType: string | null | undefined,
context: Omit<SecretConsumerContext, "configPath">,
): Promise<MissingRuntimeBinding[]> => {
const secretFieldKeys = await listAdapterSchemaSecretFieldKeys(adapterType);
const secretRefs = secretFieldKeys.flatMap((key) => {
const parsed = envBindingSchema.safeParse(adapterConfig[key]);
if (!parsed.success) return [];
const binding = canonicalizeBinding(parsed.data as EnvBinding);
if (binding.type !== "secret_ref") return [];
return [{ key, configPath: key, secretId: binding.secretId }];
});
if (secretRefs.length === 0) return [];
const bindingChecks = await Promise.all(secretRefs.map(async (entry) => ({
entry,
found: await getBinding({
companyId,
secretId: entry.secretId,
consumerType: context.consumerType,
consumerId: context.consumerId,
configPath: entry.configPath,
}),
})));
const missingEntries = bindingChecks
.filter((check) => !check.found)
.map((check) => check.entry);
if (missingEntries.length === 0) return [];
const secretRows = await Promise.all(
[...new Set(missingEntries.map((entry) => entry.secretId))].map(async (secretId) => [
secretId,
await getById(secretId).catch(() => null),
] as const),
);
const secretsById = new Map(secretRows);
return missingEntries.map((entry) => ({
consumerType: context.consumerType,
consumerId: context.consumerId,
configPath: entry.configPath,
envKey: entry.key,
secretId: entry.secretId,
secretName: secretsById.get(entry.secretId)?.name ?? null,
}));
},
resolveAdapterConfigForRuntime: async (
companyId: string,
adapterConfig: Record<string, unknown>,
context?: Omit<SecretConsumerContext, "configPath">,
opts?: { adapterType?: string | null },
): Promise<{ config: Record<string, unknown>; secretKeys: Set<string>; manifest: RuntimeSecretManifestEntry[] }> => {
const resolved = { ...adapterConfig };
const secretKeys = new Set<string>();
const manifest: RuntimeSecretManifestEntry[] = [];
if (!Object.prototype.hasOwnProperty.call(adapterConfig, "env")) {
return { config: resolved, secretKeys, manifest };
}
const record = asRecord(adapterConfig.env);
if (!record) {
resolved.env = {};
return { config: resolved, secretKeys, manifest };
}
const env: Record<string, string> = {};
for (const [key, rawBinding] of Object.entries(record)) {
if (!ENV_KEY_RE.test(key)) {
throw unprocessable(`Invalid environment variable name: ${key}`);
}
const parsed = envBindingSchema.safeParse(rawBinding);
if (!parsed.success) {
throw unprocessable(`Invalid environment binding for key: ${key}`);
}
const binding = canonicalizeBinding(parsed.data as EnvBinding);
if (binding.type === "plain") {
env[key] = binding.value;
if (Object.prototype.hasOwnProperty.call(adapterConfig, "env")) {
const record = asRecord(adapterConfig.env);
if (!record) {
resolved.env = {};
} else {
const secretResolution = await resolveSecretValueInternal(
companyId,
binding.secretId,
binding.version,
context
? {
bindingContext: { ...context, configPath: `env.${key}` },
accessContext: { ...context, configPath: `env.${key}` },
}
: undefined,
);
env[key] = secretResolution.value;
manifest.push(secretResolution.manifestEntry);
secretKeys.add(key);
const env: Record<string, string> = {};
for (const [key, rawBinding] of Object.entries(record)) {
if (!ENV_KEY_RE.test(key)) {
throw unprocessable(`Invalid environment variable name: ${key}`);
}
const parsed = envBindingSchema.safeParse(rawBinding);
if (!parsed.success) {
throw unprocessable(`Invalid environment binding for key: ${key}`);
}
const binding = canonicalizeBinding(parsed.data as EnvBinding);
if (binding.type === "plain") {
env[key] = binding.value;
} else {
const secretResolution = await resolveSecretValueInternal(
companyId,
binding.secretId,
binding.version,
context
? {
bindingContext: { ...context, configPath: `env.${key}` },
accessContext: { ...context, configPath: `env.${key}` },
}
: undefined,
);
env[key] = secretResolution.value;
manifest.push(secretResolution.manifestEntry);
secretKeys.add(key);
}
}
resolved.env = env;
}
}
resolved.env = env;
const secretFieldKeys = await listAdapterSchemaSecretFieldKeys(opts?.adapterType);
for (const key of secretFieldKeys) {
const parsed = envBindingSchema.safeParse(adapterConfig[key]);
if (!parsed.success) continue;
const binding = canonicalizeBinding(parsed.data as EnvBinding);
if (binding.type === "plain") continue;
const secretResolution = await resolveSecretValueInternal(
companyId,
binding.secretId,
binding.version,
context
? {
bindingContext: { ...context, configPath: key },
accessContext: { ...context, configPath: key },
}
: undefined,
);
resolved[key] = secretResolution.value;
manifest.push(secretResolution.manifestEntry);
secretKeys.add(key);
}
return { config: resolved, secretKeys, manifest };
},
};