Implement dynamic secret runtime resolver

- Resolve dynamic_command secrets through the runtime injection path with static binding argv only
- Add memory-only TTL caching, fail-closed command handling, and metadata-only task activity for fresh mints
- Cover cache hit/miss/expiry and generator failure paths in secret service tests
- Sync CLI/UI fixtures with dynamic secret fields from the shared contract

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky LeachandPaperclip committed 2026-06-25 22:12:33 +00:00
1 parent d495974f95
commit d2492cf628
10 files changed
+416 -6

No files matched your search

+1
View File
@@ -52,6 +52,7 @@ function secret(partial: Partial<CompanySecret>): CompanySecret {
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 1,
+3 -3
View File
@@ -6,7 +6,7 @@ import {
BIND_MODES,
DEPLOYMENT_EXPOSURES,
DEPLOYMENT_MODES,
SECRET_PROVIDERS,
SECRET_STORAGE_PROVIDERS,
STORAGE_PROVIDERS,
inferBindModeFromHost,
resolveRuntimeBind,
@@ -14,7 +14,7 @@ import {
type AuthBaseUrlMode,
type DeploymentExposure,
type DeploymentMode,
type SecretProvider,
type SecretStorageProvider,
type StorageProvider,
} from "@paperclipai/shared";
import { configExists, readConfig, resolveConfigPath, writeConfig } from "../config/store.js";
@@ -197,7 +197,7 @@ function quickstartDefaultsFromEnv(opts?: { preferTrustedLocal?: boolean }): {
parseEnumFromEnv<StorageProvider>(process.env.PAPERCLIP_STORAGE_PROVIDER, STORAGE_PROVIDERS) ??
defaultStorage.provider;
const secretsProvider =
parseEnumFromEnv<SecretProvider>(process.env.PAPERCLIP_SECRETS_PROVIDER, SECRET_PROVIDERS) ??
parseEnumFromEnv<SecretStorageProvider>(process.env.PAPERCLIP_SECRETS_PROVIDER, SECRET_STORAGE_PROVIDERS) ??
defaultSecrets.provider;
const databaseBackupEnabled = parseBooleanFromEnv(process.env.PAPERCLIP_DB_BACKUP_ENABLED) ?? true;
const databaseBackupIntervalMinutes = Math.max(
+2 -2
View File
@@ -1,5 +1,5 @@
import * as p from "@clack/prompts";
import type { SecretProvider } from "@paperclipai/shared";
import type { SecretStorageProvider } from "@paperclipai/shared";
import type { SecretsConfig } from "../config/schema.js";
import { resolveDefaultSecretsKeyFilePath, resolvePaperclipInstanceId } from "../config/home.js";
@@ -92,7 +92,7 @@ export async function promptSecrets(current?: SecretsConfig): Promise<SecretsCon
}
return {
provider: provider as SecretProvider,
provider: provider as SecretStorageProvider,
strictMode,
localEncrypted: {
keyFilePath,
+152 -1
View File
@@ -1,5 +1,5 @@
import { randomUUID } from "node:crypto";
import { mkdirSync, rmSync } from "node:fs";
import { chmodSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -13,6 +13,8 @@ import {
companySecretVersions,
companySecrets,
createDb,
activityLog,
issues,
secretAccessEvents,
} from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -46,11 +48,13 @@ describeEmbeddedPostgres("secretService", () => {
afterEach(async () => {
vi.restoreAllMocks();
await db.delete(activityLog);
await db.delete(secretAccessEvents);
await db.delete(companySecretBindings);
await db.delete(companySecretVersions);
await db.delete(companySecrets);
await db.delete(companySecretProviderConfigs);
await db.delete(issues);
await db.delete(companyMemberships);
await db.delete(agents);
await db.delete(companies);
@@ -79,6 +83,26 @@ describeEmbeddedPostgres("secretService", () => {
return companyId;
}
async function seedIssue(companyId: string) {
return db
.insert(issues)
.values({
companyId,
title: `Issue ${randomUUID()}`,
status: "in_progress",
priority: "medium",
})
.returning()
.then((rows) => rows[0]);
}
function writeGeneratorScript(name: string, body: string) {
const scriptPath = path.join(secretsTmpDir, `${name}-${randomUUID()}.js`);
writeFileSync(scriptPath, body, { mode: 0o700 });
chmodSync(scriptPath, 0o700);
return scriptPath;
}
async function seedCompanyMember(
companyId: string,
userId: string,
@@ -189,6 +213,133 @@ describeEmbeddedPostgres("secretService", () => {
expect(binding.staticArgv).toEqual(["--installation", "12345"]);
});
it("resolves dynamic command secrets with a TTL cache and metadata-only task activity", async () => {
const companyId = await seedCompany();
const issue = await seedIssue(companyId);
const countFile = path.join(secretsTmpDir, `count-${randomUUID()}.txt`);
const scriptPath = writeGeneratorScript("dynamic-success", `
const { readFileSync, writeFileSync } = require("node:fs");
const countFile = ${JSON.stringify(countFile)};
let count = 0;
try { count = Number(readFileSync(countFile, "utf8")) || 0; } catch {}
count += 1;
writeFileSync(countFile, String(count));
console.log("minted-" + count + ":" + process.argv.slice(2).join("|"));
`);
const svc = secretService(db);
const secret = await svc.create(companyId, {
name: `dynamic-resolve-${randomUUID()}`,
provider: "host_command",
managedMode: "dynamic_command",
dynamicCommand: {
provider: "host-command",
command: process.execPath,
ttlSeconds: 1,
},
});
await svc.createBinding({
companyId,
secretId: secret.id,
targetType: "agent",
targetId: "agent-1",
configPath: "env.GITHUB_TOKEN",
staticArgv: [scriptPath, "--installation", "12345"],
});
const config = { env: { GITHUB_TOKEN: { type: "secret_ref", secretId: secret.id } } };
const context = {
consumerType: "agent" as const,
consumerId: "agent-1",
actorType: "system" as const,
actorId: "system",
issueId: issue.id,
};
const first = await svc.resolveAdapterConfigForRuntime(companyId, config, context);
const second = await svc.resolveAdapterConfigForRuntime(companyId, config, context);
await new Promise((resolve) => setTimeout(resolve, 1100));
const third = await svc.resolveAdapterConfigForRuntime(companyId, config, context);
expect(first.config.env).toMatchObject({ GITHUB_TOKEN: "minted-1:--installation|12345" });
expect(second.config.env).toMatchObject({ GITHUB_TOKEN: "minted-1:--installation|12345" });
expect(third.config.env).toMatchObject({ GITHUB_TOKEN: "minted-2:--installation|12345" });
expect(readFileSync(countFile, "utf8")).toBe("2");
const activities = await db
.select()
.from(activityLog)
.where(eq(activityLog.action, "secret.dynamic_command_resolved"));
expect(activities).toHaveLength(2);
const activityJson = JSON.stringify(activities.map((activity) => activity.details));
expect(activityJson).toContain("GITHUB_TOKEN");
expect(activityJson).toContain("***REDACTED***");
expect(activityJson).not.toContain(secret.id);
expect(activityJson).not.toContain("minted-");
expect(activityJson).not.toContain("--installation");
expect(activityJson).not.toContain("12345");
expect(activityJson).not.toContain(scriptPath);
});
it("fails closed for dynamic command error, timeout, and empty output", async () => {
const companyId = await seedCompany();
const svc = secretService(db);
const previousTimeout = process.env.PAPERCLIP_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS;
process.env.PAPERCLIP_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS = "50";
async function expectDynamicFailure(scriptBody: string, expectedMessage: RegExp) {
const scriptPath = writeGeneratorScript("dynamic-failure", scriptBody);
const targetId = `agent-${randomUUID()}`;
const secret = await svc.create(companyId, {
name: `dynamic-failure-${randomUUID()}`,
provider: "host_command",
managedMode: "dynamic_command",
dynamicCommand: {
provider: "host-command",
command: process.execPath,
ttlSeconds: 60,
},
});
await svc.createBinding({
companyId,
secretId: secret.id,
targetType: "agent",
targetId,
configPath: "env.RUNTIME_TOKEN",
staticArgv: [scriptPath],
});
await expect(
svc.resolveAdapterConfigForRuntime(
companyId,
{ env: { RUNTIME_TOKEN: { type: "secret_ref", secretId: secret.id } } },
{
consumerType: "agent",
consumerId: targetId,
actorType: "system",
actorId: "system",
},
),
).rejects.toThrow(expectedMessage);
}
try {
await expectDynamicFailure("process.stderr.write('do-not-leak'); process.exit(3);", /generator failed/i);
await expectDynamicFailure("setTimeout(() => console.log('late-token'), 1000);", /timed out/i);
await expectDynamicFailure("console.log(' ');", /empty value/i);
} finally {
if (previousTimeout === undefined) {
delete process.env.PAPERCLIP_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS;
} else {
process.env.PAPERCLIP_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS = previousTimeout;
}
}
const activities = await db
.select()
.from(activityLog)
.where(eq(activityLog.action, "secret.dynamic_command_resolved"));
expect(activities).toHaveLength(0);
});
it("syncs top-level secret refs idempotently", async () => {
const companyId = await seedCompany();
const svc = secretService(db);
+243
View File
@@ -1,3 +1,5 @@
import { createHash } from "node:crypto";
import { spawn } from "node:child_process";
import { and, desc, eq, inArray, like, ne, notInArray, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
@@ -56,6 +58,7 @@ import type {
} from "../secrets/types.js";
import { isSecretProviderClientError } from "../secrets/types.js";
import { authorizationService } from "./authorization.js";
import { logActivity } from "./activity-log.js";
const ENV_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
const SENSITIVE_ENV_KEY_RE =
@@ -66,6 +69,8 @@ const COMING_SOON_SECRET_PROVIDERS: ReadonlySet<SecretProvider> = new Set([
"vault",
]);
const EMPTY_STATIC_ARGV: string[] = [];
const DEFAULT_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS = 30_000;
const dynamicSecretCache = new Map<string, { value: string; expiresAt: number }>();
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type SecretBindingDb = Pick<Db | DbTransaction, "select" | "delete" | "insert">;
@@ -259,12 +264,17 @@ type RuntimeSecretResolution = {
manifestEntry: RuntimeSecretManifestEntry;
};
type SecretBindingRow = typeof companySecretBindings.$inferSelect;
type SecretResolutionErrorCode =
| "binding_missing"
| "secret_deleted"
| "secret_inactive"
| "version_missing"
| "version_inactive"
| "dynamic_secret_command_failed"
| "dynamic_secret_command_timeout"
| "dynamic_secret_command_empty"
| "provider_error";
function asRecord(value: unknown): Record<string, unknown> | null {
@@ -285,6 +295,101 @@ function normalizeSecretKey(input: string) {
.slice(0, 120);
}
function dynamicSecretCacheKey(input: { bindingId: string; staticArgv: string[] }) {
const argvHash = createHash("sha256")
.update(JSON.stringify(input.staticArgv))
.digest("hex");
return `${input.bindingId}:${argvHash}`;
}
function normalizeStaticArgv(value: unknown): string[] {
if (!Array.isArray(value)) return EMPTY_STATIC_ARGV;
if (value.some((entry) => typeof entry !== "string")) {
throw unprocessable("Dynamic secret binding has invalid static argv", {
code: "dynamic_secret_command_failed",
});
}
return value;
}
function dynamicSecretHttpError(code: SecretResolutionErrorCode, message: string) {
return new HttpError(422, message, { code });
}
function dynamicSecretCommandTimeoutMs() {
const configured = Number(process.env.PAPERCLIP_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS);
return Number.isFinite(configured) && configured > 0
? Math.min(configured, DEFAULT_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS)
: DEFAULT_DYNAMIC_SECRET_COMMAND_TIMEOUT_MS;
}
async function runDynamicSecretCommand(input: {
command: string;
staticArgv: string[];
timeoutMs?: number;
}): Promise<string> {
return new Promise((resolve, reject) => {
let stdout = "";
let stderrBytes = 0;
let settled = false;
const child = spawn(input.command, input.staticArgv, {
shell: false,
windowsHide: true,
stdio: ["ignore", "pipe", "pipe"],
env: process.env,
});
const timeout = setTimeout(() => {
if (settled) return;
settled = true;
child.kill("SIGTERM");
reject(dynamicSecretHttpError(
"dynamic_secret_command_timeout",
"Dynamic secret generator timed out before producing a fresh value.",
));
}, input.timeoutMs ?? dynamicSecretCommandTimeoutMs());
child.stdout.setEncoding("utf8");
child.stdout.on("data", (chunk) => {
stdout += chunk;
});
child.stderr.on("data", (chunk: Buffer | string) => {
stderrBytes += Buffer.byteLength(chunk);
});
child.on("error", (error) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
reject(dynamicSecretHttpError(
"dynamic_secret_command_failed",
error instanceof Error && error.message.includes("ENOENT")
? "Dynamic secret generator command was not found."
: "Dynamic secret generator failed before producing a fresh value.",
));
});
child.on("close", (code, signal) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
if (code !== 0 || signal) {
reject(dynamicSecretHttpError(
"dynamic_secret_command_failed",
`Dynamic secret generator failed before producing a fresh value.${stderrBytes > 0 ? " Check host logs for details." : ""}`,
));
return;
}
const value = stdout.trim();
if (!value) {
reject(dynamicSecretHttpError(
"dynamic_secret_command_empty",
"Dynamic secret generator produced an empty value.",
));
return;
}
resolve(value);
});
});
}
function deriveSecretNameFromExternalRef(externalRef: string) {
const trimmed = externalRef.trim();
const arnMatch = /^arn:[^:]+:secretsmanager:[^:]*:[^:]*:secret:(.+)$/i.exec(trimmed);
@@ -320,6 +425,9 @@ function secretResolutionErrorCode(error: unknown): SecretResolutionErrorCode {
case "secret_inactive":
case "version_missing":
case "version_inactive":
case "dynamic_secret_command_failed":
case "dynamic_secret_command_timeout":
case "dynamic_secret_command_empty":
case "provider_error":
return details.code;
}
@@ -640,6 +748,29 @@ export function secretService(db: Db) {
throw unprocessable("Secret is not active", { code: "secret_inactive" });
}
const binding = await assertBindingContext(companyId, secret.id, bindingContext);
if (secret.managedMode === "dynamic_command") {
if (providerId !== "host_command" || !secret.dynamicCommand) {
throw unprocessable("Dynamic secret is missing generator command config", {
code: "dynamic_secret_command_failed",
});
}
if (!binding) {
throw unprocessable("Dynamic secret resolution requires a runtime binding", { code: "binding_missing" });
}
const resolution = await resolveDynamicSecretValue({
companyId,
secret: {
id: secret.id,
key: secret.key,
provider: providerId,
dynamicCommand: secret.dynamicCommand,
},
binding,
configPath,
accessContext,
});
return resolution;
}
const versionRow = await getSecretVersion(secret.id, resolvedVersion);
if (!versionRow) throw new HttpError(404, "Secret version not found", { code: "version_missing" });
if (versionRow.status === "disabled" || versionRow.status === "destroyed" || versionRow.revokedAt) {
@@ -706,6 +837,118 @@ export function secretService(db: Db) {
}
}
async function resolveDynamicSecretValue(input: {
companyId: string;
secret: {
id: string;
key: string;
provider: SecretProvider;
dynamicCommand: DynamicSecretCommandConfig;
};
binding: SecretBindingRow;
configPath: string | null;
accessContext: SecretConsumerContext | undefined;
}): Promise<RuntimeSecretResolution> {
const staticArgv = normalizeStaticArgv(input.binding.staticArgv);
const cacheKey = dynamicSecretCacheKey({ bindingId: input.binding.id, staticArgv });
const now = Date.now();
const cached = dynamicSecretCache.get(cacheKey);
if (cached && cached.expiresAt > now) {
return {
value: cached.value,
manifestEntry: {
configPath: input.configPath ?? "",
envKey: input.configPath?.startsWith("env.") ? input.configPath.slice("env.".length) : null,
secretId: input.secret.id,
bindingId: input.binding.id,
secretKey: input.secret.key,
version: 0,
provider: input.secret.provider,
outcome: "success",
},
};
}
if (cached) {
dynamicSecretCache.delete(cacheKey);
}
const value = await runDynamicSecretCommand({
command: input.secret.dynamicCommand.command,
staticArgv,
});
dynamicSecretCache.set(cacheKey, {
value,
expiresAt: now + input.secret.dynamicCommand.ttlSeconds * 1000,
});
await Promise.all([
db
.update(companySecrets)
.set({ lastResolvedAt: new Date(), updatedAt: new Date() })
.where(eq(companySecrets.id, input.secret.id))
.catch(() => undefined),
recordAccessEvent({
companyId: input.companyId,
secretId: input.secret.id,
version: 0,
provider: input.secret.provider,
context: input.accessContext,
outcome: "success",
}).catch(() => undefined),
recordDynamicSecretActivity({
companyId: input.companyId,
secretId: input.secret.id,
bindingId: input.binding.id,
configPath: input.configPath,
ttlSeconds: input.secret.dynamicCommand.ttlSeconds,
context: input.accessContext,
}).catch(() => undefined),
]);
return {
value,
manifestEntry: {
configPath: input.configPath ?? "",
envKey: input.configPath?.startsWith("env.") ? input.configPath.slice("env.".length) : null,
secretId: input.secret.id,
bindingId: input.binding.id,
secretKey: input.secret.key,
version: 0,
provider: input.secret.provider,
outcome: "success",
},
};
}
async function recordDynamicSecretActivity(input: {
companyId: string;
secretId: string;
bindingId: string;
configPath: string | null;
ttlSeconds: number;
context: SecretConsumerContext | undefined;
}) {
if (!input.context?.issueId) return;
await logActivity(db, {
companyId: input.companyId,
actorType: input.context.actorType ?? "system",
actorId: input.context.actorId ?? "system",
action: "secret.dynamic_command_resolved",
entityType: "issue",
entityId: input.context.issueId,
agentId: input.context.actorType === "agent" ? input.context.actorId ?? null : null,
runId: input.context.heartbeatRunId ?? null,
details: {
secretId: input.secretId,
bindingId: input.bindingId,
configPath: input.configPath,
envKey: input.configPath?.startsWith("env.") ? input.configPath.slice("env.".length) : null,
ttlSeconds: input.ttlSeconds,
cache: "miss",
},
});
}
async function resolveSecretValue(
companyId: string,
secretId: string,
@@ -352,6 +352,7 @@ describe("RoutineHistoryTab", () => {
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 4,
@@ -414,6 +415,7 @@ describe("RoutineHistoryTab", () => {
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 1,
@@ -435,6 +437,7 @@ describe("RoutineHistoryTab", () => {
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 1,
@@ -292,6 +292,7 @@ describe("ImportFromVaultDialog", () => {
status: "active",
managedMode: "external_reference",
externalRef: "arn:aws:secretsmanager:us-east-1:1:secret:other-XYZ",
dynamicCommand: null,
providerConfigId: awsVault.id,
providerMetadata: null,
latestVersion: 1,
+7
View File
@@ -1429,6 +1429,7 @@ export const storybookSecrets: CompanySecret[] = [
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 3,
@@ -1450,6 +1451,7 @@ export const storybookSecrets: CompanySecret[] = [
status: "active",
managedMode: "external_reference",
externalRef: "arn:aws:secretsmanager:us-east-1:123456789012:secret:paperclip/prod/aws-deploy-AbCdEf",
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 2,
@@ -1471,6 +1473,7 @@ export const storybookSecrets: CompanySecret[] = [
status: "disabled",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 1,
@@ -1492,6 +1495,7 @@ export const storybookSecrets: CompanySecret[] = [
status: "archived",
managedMode: "external_reference",
externalRef: "secret/data/payments/stripe-legacy",
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 4,
@@ -1517,6 +1521,7 @@ export const storybookSecretBindings: CompanySecretBinding[] = [
versionSelector: "latest",
required: true,
label: "Codex agent env",
staticArgv: [],
createdAt: new Date("2026-03-02T09:00:00.000Z"),
updatedAt: new Date("2026-03-02T09:00:00.000Z"),
},
@@ -1530,6 +1535,7 @@ export const storybookSecretBindings: CompanySecretBinding[] = [
versionSelector: "latest",
required: true,
label: "Paperclip App project env",
staticArgv: [],
createdAt: new Date("2026-03-02T09:00:00.000Z"),
updatedAt: new Date("2026-03-02T09:00:00.000Z"),
},
@@ -1543,6 +1549,7 @@ export const storybookSecretBindings: CompanySecretBinding[] = [
versionSelector: 2,
required: true,
label: "Prod environment",
staticArgv: [],
createdAt: new Date("2026-04-22T14:01:00.000Z"),
updatedAt: new Date("2026-04-22T14:01:00.000Z"),
},
@@ -260,6 +260,7 @@ const storybookSecrets: CompanySecret[] = [
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 5,
@@ -281,6 +282,7 @@ const storybookSecrets: CompanySecret[] = [
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 3,
@@ -208,6 +208,7 @@ const storybookSecrets: CompanySecret[] = [
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 3,
@@ -229,6 +230,7 @@ const storybookSecrets: CompanySecret[] = [
status: "active",
managedMode: "paperclip_managed",
externalRef: null,
dynamicCommand: null,
providerConfigId: null,
providerMetadata: null,
latestVersion: 1,