diff --git a/server/src/__tests__/agent-secrets-routes.test.ts b/server/src/__tests__/agent-secrets-routes.test.ts index 60375c14001..0f7dd4b4f8f 100644 --- a/server/src/__tests__/agent-secrets-routes.test.ts +++ b/server/src/__tests__/agent-secrets-routes.test.ts @@ -16,6 +16,7 @@ import { companySecrets, createDb, heartbeatRuns, + runSecretRedactions, secretAccessEvents, } from "@paperclipai/db"; import { LOW_TRUST_REVIEW_PRESET, type AgentApiKeyScope } from "@paperclipai/shared"; @@ -23,7 +24,7 @@ import { REDACTED_EVENT_VALUE } from "../redaction.js"; import { errorHandler } from "../middleware/error-handler.js"; import { secretRoutes } from "../routes/secrets.js"; import { secretService } from "../services/secrets.js"; -import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; +import { createRunSecretRedactionRegistry, farFutureRedactionExpiry } from "../services/run-secret-redaction.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -200,9 +201,11 @@ describeEmbeddedPostgres("agent secret routes", () => { expect(fetched.body).toEqual({ key: "env_only_key", value: "env-secret-value", version: 1 }); const [registeredRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.heartbeatRunId)); expect(JSON.stringify(registeredRun.contextSnapshot)).not.toContain("env-secret-value"); - expect(registeredRun.contextSnapshot).toMatchObject({ - paperclipSecretRedactions: [expect.objectContaining({ fingerprintSha256: expect.any(String), material: expect.any(Object) })], - }); + const registrations = await db.select().from(runSecretRedactions) + .where(eq(runSecretRedactions.runId, fixture.heartbeatRunId)); + expect(registrations).toEqual([ + expect.objectContaining({ fingerprintSha256: expect.any(String), material: expect.any(Object) }), + ]); expect(await db.select().from(secretAccessEvents)).toEqual([ expect.objectContaining({ secretId: envSecret.id, outcome: "success", consumerType: "agent_api" }), ]); @@ -229,7 +232,7 @@ describeEmbeddedPostgres("agent secret routes", () => { it("fails closed when run redaction registration cannot be persisted", async () => { const fixture = await seedAgentRun(); - await expect(createRunSecretRedactionRegistry(db).register(fixture.companyId, randomUUID(), "must-not-return")) + await expect(createRunSecretRedactionRegistry(db).register(fixture.companyId, randomUUID(), "must-not-return", farFutureRedactionExpiry())) .rejects.toThrow("Heartbeat run redaction registration failed"); }); @@ -238,23 +241,21 @@ describeEmbeddedPostgres("agent secret routes", () => { const registry = createRunSecretRedactionRegistry(db); await Promise.all([ - registry.register(fixture.companyId, fixture.heartbeatRunId, "duplicate-secret"), - registry.register(fixture.companyId, fixture.heartbeatRunId, "duplicate-secret"), + registry.register(fixture.companyId, fixture.heartbeatRunId, "duplicate-secret", farFutureRedactionExpiry()), + registry.register(fixture.companyId, fixture.heartbeatRunId, "duplicate-secret", farFutureRedactionExpiry()), ]); - const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.heartbeatRunId)); - expect(run.contextSnapshot).toMatchObject({ - paperclipSecretRedactions: [expect.objectContaining({ fingerprintSha256: expect.any(String) })], - }); - expect((run.contextSnapshot as { paperclipSecretRedactions: unknown[] }).paperclipSecretRedactions).toHaveLength(1); + const registrations = await db.select().from(runSecretRedactions) + .where(eq(runSecretRedactions.runId, fixture.heartbeatRunId)); + expect(registrations).toEqual([expect.objectContaining({ fingerprintSha256: expect.any(String) })]); }); it("redacts batched runs from projected registries and enforces company scope", async () => { const first = await seedAgentRun(); const foreign = await seedAgentRun(); const registry = createRunSecretRedactionRegistry(db); - await registry.register(first.companyId, first.heartbeatRunId, "first-secret-value"); - await registry.register(foreign.companyId, foreign.heartbeatRunId, "foreign-secret-value"); + await registry.register(first.companyId, first.heartbeatRunId, "first-secret-value", farFutureRedactionExpiry()); + await registry.register(foreign.companyId, foreign.heartbeatRunId, "foreign-secret-value", farFutureRedactionExpiry()); const runs = [ { id: first.heartbeatRunId, text: "first-secret-value foreign-secret-value", createdAt: new Date() }, { id: foreign.heartbeatRunId, text: "foreign-secret-value", createdAt: new Date() }, diff --git a/server/src/__tests__/heartbeat-local-environment.test.ts b/server/src/__tests__/heartbeat-local-environment.test.ts index c45dd159fae..5313f88b725 100644 --- a/server/src/__tests__/heartbeat-local-environment.test.ts +++ b/server/src/__tests__/heartbeat-local-environment.test.ts @@ -11,6 +11,7 @@ import { environmentLeases, environments, heartbeatRuns, + runSecretRedactions, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, @@ -301,13 +302,11 @@ describeEmbeddedPostgres("heartbeat local environment lifecycle", () => { const injectedToken = await readFile(tokenPath, "utf8"); expect(injectedToken.length).toBeGreaterThan(0); - const [row] = await db - .select({ contextSnapshot: heartbeatRuns.contextSnapshot }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.id, queued!.id), eq(heartbeatRuns.companyId, companyId))); - const entries = (row?.contextSnapshot as { paperclipSecretRedactions?: Array<{ fingerprintSha256: string }> } | undefined) - ?.paperclipSecretRedactions ?? []; - expect(entries.map((entry) => entry.fingerprintSha256)) + const registrations = await db + .select({ fingerprintSha256: runSecretRedactions.fingerprintSha256 }) + .from(runSecretRedactions) + .where(and(eq(runSecretRedactions.runId, queued!.id), eq(runSecretRedactions.companyId, companyId))); + expect(registrations.map((entry) => entry.fingerprintSha256)) .toContain(createHash("sha256").update(injectedToken).digest("hex")); }); @@ -352,12 +351,10 @@ describeEmbeddedPostgres("heartbeat local environment lifecycle", () => { await expect(access(markerPath)).rejects.toThrow(); - const [row] = await db - .select({ contextSnapshot: heartbeatRuns.contextSnapshot }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.id, queued!.id), eq(heartbeatRuns.companyId, companyId))); - const entries = (row?.contextSnapshot as { paperclipSecretRedactions?: unknown[] } | undefined) - ?.paperclipSecretRedactions ?? []; - expect(entries).toEqual([]); + const registrations = await db + .select({ fingerprintSha256: runSecretRedactions.fingerprintSha256 }) + .from(runSecretRedactions) + .where(and(eq(runSecretRedactions.runId, queued!.id), eq(runSecretRedactions.companyId, companyId))); + expect(registrations).toEqual([]); }); }); diff --git a/server/src/__tests__/run-secret-redaction.test.ts b/server/src/__tests__/run-secret-redaction.test.ts index 97aaff09318..74f4c875e5b 100644 --- a/server/src/__tests__/run-secret-redaction.test.ts +++ b/server/src/__tests__/run-secret-redaction.test.ts @@ -1,10 +1,26 @@ -import { createHash } from "node:crypto"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { REDACTED_EVENT_VALUE } from "../redaction.js"; +import { createHash, randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import type { Db } from "@paperclipai/db"; -import { PgDialect } from "drizzle-orm/pg-core"; -import { createRunSecretRedactionRegistry, redactRegisteredSecretValues } from "../services/run-secret-redaction.js"; +import { + agents, + companies, + createDb, + heartbeatRuns, + runSecretRedactions, +} from "@paperclipai/db"; +import { REDACTED_EVENT_VALUE } from "../redaction.js"; +import { + createRunSecretRedactionRegistry, + farFutureRedactionExpiry, + redactRegisteredSecretValues, +} from "../services/run-secret-redaction.js"; +import { createRunSecretRedactionReaper } from "../services/run-secret-redaction-reaper.js"; import { mintAndRegisterRunBearer } from "../services/run-bearer.js"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; const secret = "q2a-exact-secret-value"; @@ -81,155 +97,242 @@ describe("registered run secret redaction", () => { }); }); -const { resolveVersion, createSecret } = vi.hoisted(() => ({ - resolveVersion: vi.fn(async ({ material }) => material.value as string), - createSecret: vi.fn(async ({ value }: { value: string }) => ({ - material: { value }, - valueSha256: createHash("sha256").update(value).digest("hex"), - externalRef: null, - })), -})); -vi.mock("../secrets/provider-registry.js", () => ({ getSecretProvider: () => ({ resolveVersion, createSecret }) })); +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; -const { mintLocalAgentJwt } = vi.hoisted(() => ({ mintLocalAgentJwt: vi.fn() })); -vi.mock("../agent-auth-jwt.js", () => ({ createLocalAgentJwt: mintLocalAgentJwt })); +// The registry moved off `heartbeat_runs.context_snapshot` and onto the +// company-scoped `run_secret_redactions` table (see +// packages/db/src/schema/run_secret_redactions.ts). These tests exercise the +// real table and the real Postgres query planner, because the acceptance bar +// for this move is index-backed lookups and a real `EXPLAIN` plan, not a +// mocked query builder. +describeEmbeddedPostgres("run secret redaction registry (company-scoped table)", () => { + let stopDb: (() => Promise) | null = null; + let db!: Db; + const previousAgentJwtSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET; -describe("batched run secret redaction", () => { - beforeEach(() => { resolveVersion.mockClear(); }); + beforeAll(async () => { + process.env.PAPERCLIP_AGENT_JWT_SECRET = "run-secret-redaction-test-secret"; + const started = await startEmbeddedPostgresTestDatabase("run-secret-redaction-"); + stopDb = started.cleanup; + db = createDb(started.connectionString); + }); - function fixture(rows: unknown[]) { - const where = vi.fn(async (_predicate: import("drizzle-orm").SQL | undefined) => rows); - const select = vi.fn((_columns: { contextSnapshot: import("drizzle-orm").SQL }) => ({ from: () => ({ where }) })); - return { registry: createRunSecretRedactionRegistry({ select } as unknown as Db), select, where }; + afterEach(async () => { + await db.delete(heartbeatRuns); + await db.delete(agents); + await db.delete(companies); + }); + + afterAll(async () => { + await stopDb?.(); + if (previousAgentJwtSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; + else process.env.PAPERCLIP_AGENT_JWT_SECRET = previousAgentJwtSecret; + }); + + async function seedRun() { + const companyId = randomUUID(); + const agentId = randomUUID(); + const runId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Run secret redaction", + issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + status: "active", + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Redaction agent", + role: "engineer", + adapterType: "claude_local", + status: "idle", + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + status: "running", + contextSnapshot: {}, + }); + return { companyId, agentId, runId }; } - it("reads only registry JSON once for 200 runs and resolves shared secrets once", async () => { - const contextSnapshot = { paperclipSecretRedactions: [{ fingerprintSha256: "shared", material: { value: secret } }] }; - const rows = Array.from({ length: 200 }, (_, i) => ({ id: `run-${i}`, contextSnapshot })); - const { registry, select, where } = fixture(rows); - const result = await registry.redactForRuns("company-1", rows.map(row => ({ ...row, stdoutExcerpt: secret }))); - expect(select).toHaveBeenCalledTimes(1); - expect(resolveVersion).toHaveBeenCalledTimes(1); - expect(result.every(run => run.stdoutExcerpt === REDACTED_EVENT_VALUE)).toBe(true); - expect(result[0].contextSnapshot).toEqual({}); - const dialect = new PgDialect(); - const predicate = dialect.sqlToQuery(where.mock.calls[0][0]); - expect(predicate.params).toContain("company-1"); - expect(predicate.sql).toContain('"company_id"'); - expect(dialect.sqlToQuery(select.mock.calls[0][0].contextSnapshot).sql).toContain("-> 'paperclipSecretRedactions'"); - }); - - it("keeps each run's registry separate and observes new registrations on the next request", async () => { - const rows = [{ id: "a", contextSnapshot: { paperclipSecretRedactions: [{ fingerprintSha256: "one", material: { value: secret } }] } }]; - const { registry } = fixture(rows); - expect(await registry.redactForRuns("company", [{ id: "a", text: secret }, { id: "b", text: secret }])) - .toEqual([{ id: "a", text: REDACTED_EVENT_VALUE }, { id: "b", text: secret }]); - rows[0].contextSnapshot.paperclipSecretRedactions.push({ fingerprintSha256: "two", material: { value: "new-secret" } }); - expect(await registry.redactForRuns("company", [{ id: "a", text: "new-secret" }])) - .toEqual([{ id: "a", text: REDACTED_EVENT_VALUE }]); - }); - - it("does not query for an empty list and fails closed on decryption failure", async () => { - const { registry, select } = fixture([{ id: "a", contextSnapshot: { paperclipSecretRedactions: [{ fingerprintSha256: "one", material: {} }] } }]); - expect(await registry.redactForRuns("company", [])).toEqual([]); - expect(select).not.toHaveBeenCalled(); - resolveVersion.mockRejectedValueOnce(new Error("unavailable")); - await expect(registry.redactForRuns("company", [{ id: "a", text: secret }])).rejects.toThrow("unavailable"); - }); -}); - -// D8.1 — mint and register. `mintAndRegisterRunBearer` (server/src/services/run-bearer.js) -// is the wrapper every mint site must call instead of `createLocalAgentJwt` -// directly, so a minted bearer always reaches the redaction registry before a -// caller can use it. -describe("mintAndRegisterRunBearer", () => { - const companyId = "company-1"; - const runId = "run-1"; - - beforeEach(() => { - mintLocalAgentJwt.mockReset(); - createSecret.mockClear(); - }); - - // A fake `Db` that only implements the `transaction` shape `register()` - // needs: a locked select of the run row, then an update of its - // `contextSnapshot`. `row` is the current row seen inside the transaction, - // or `null` to exercise the missing-run failure path. - function fakeDb(row: { contextSnapshot: unknown } | null) { - const selectWhere = vi.fn(() => ({ for: () => Promise.resolve(row ? [row] : []) })); - const updateWhere = vi.fn(async () => {}); - const tx = { - select: () => ({ from: () => ({ where: selectWhere }) }), - update: () => ({ set: (values: unknown) => ({ where: (predicate: unknown) => updateWhere(values, predicate) }) }), - }; - const db = { transaction: async (callback: (tx: unknown) => Promise) => callback(tx) }; - return { db: db as unknown as Db, selectWhere, updateWhere }; + async function rowsForRun(companyId: string, runId: string) { + return db.select().from(runSecretRedactions) + .where(and(eq(runSecretRedactions.companyId, companyId), eq(runSecretRedactions.runId, runId))); } - it("registers the minted bearer against the given company and run, then returns it", async () => { - const token = "minted-run-bearer-token"; - mintLocalAgentJwt.mockReturnValue(token); - const { db, selectWhere, updateWhere } = fakeDb({ contextSnapshot: {} }); + it("registers a value and masks it on a later read of that run", async () => { + const { companyId, runId } = await seedRun(); + const registry = createRunSecretRedactionRegistry(db); - const result = await mintAndRegisterRunBearer(db, "agent-1", companyId, "claude_local", runId, "user-1"); + await registry.register(companyId, runId, secret, farFutureRedactionExpiry()); - expect(result).toBe(token); - expect(selectWhere).toHaveBeenCalledTimes(1); - const dialect = new PgDialect(); - const selectPredicate = dialect.sqlToQuery(selectWhere.mock.calls[0][0] as Parameters[0]); - expect(selectPredicate.params).toContain(companyId); - expect(selectPredicate.params).toContain(runId); - - expect(updateWhere).toHaveBeenCalledTimes(1); - const [values] = updateWhere.mock.calls[0] as [{ contextSnapshot: { paperclipSecretRedactions: Array<{ fingerprintSha256: string }> } }, unknown]; - expect(values.contextSnapshot.paperclipSecretRedactions).toHaveLength(1); - expect(values.contextSnapshot.paperclipSecretRedactions[0].fingerprintSha256) - .toBe(createHash("sha256").update(token).digest("hex")); - expect(createSecret).toHaveBeenCalledWith({ value: token }); + const rows = await rowsForRun(companyId, runId); + expect(rows).toHaveLength(1); + expect(rows[0].fingerprintSha256).toBe(createHash("sha256").update(secret).digest("hex")); + expect(JSON.stringify(rows[0].material)).not.toContain(secret); + expect(await registry.redactForRun(companyId, runId, `token ${secret} in output`)) + .toBe(`token ${REDACTED_EVENT_VALUE} in output`); }); - it("passes the sixth scope argument through to the mint call", async () => { - mintLocalAgentJwt.mockReturnValue("token-with-scope"); - const { db } = fakeDb({ contextSnapshot: {} }); - const scope = { kind: "skill_test" as const, issueId: "issue-1" }; + it("is idempotent for the same run and value", async () => { + const { companyId, runId } = await seedRun(); + const registry = createRunSecretRedactionRegistry(db); - await mintAndRegisterRunBearer(db, "agent-1", companyId, "claude_local", runId, "user-1", scope); + await Promise.all([ + registry.register(companyId, runId, "duplicate-secret", farFutureRedactionExpiry()), + registry.register(companyId, runId, "duplicate-secret", farFutureRedactionExpiry()), + ]); - expect(mintLocalAgentJwt).toHaveBeenCalledWith("agent-1", companyId, "claude_local", runId, "user-1", scope); + expect(await rowsForRun(companyId, runId)).toHaveLength(1); }); - it("returns null and registers nothing on the null-mint configuration path", async () => { - mintLocalAgentJwt.mockReturnValue(null); - const { db, selectWhere, updateWhere } = fakeDb({ contextSnapshot: {} }); + it("keeps a separate row so a second run can mask the identical value too", async () => { + const { companyId, runId: firstRunId } = await seedRun(); + const secondRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: secondRunId, companyId, agentId: (await db.select().from(agents).where(eq(agents.companyId, companyId)))[0]!.id, + status: "running", contextSnapshot: {}, + }); + const registry = createRunSecretRedactionRegistry(db); - const result = await mintAndRegisterRunBearer(db, "agent-1", companyId, "claude_local", runId, "user-1"); + await registry.register(companyId, firstRunId, "shared-secret", farFutureRedactionExpiry()); + await registry.register(companyId, secondRunId, "shared-secret", farFutureRedactionExpiry()); - expect(result).toBeNull(); - expect(selectWhere).not.toHaveBeenCalled(); - expect(updateWhere).not.toHaveBeenCalled(); + expect(await rowsForRun(companyId, firstRunId)).toHaveLength(1); + expect(await rowsForRun(companyId, secondRunId)).toHaveLength(1); + expect(await registry.redactForRun(companyId, secondRunId, "shared-secret")).toBe(REDACTED_EVENT_VALUE); }); - it("propagates a registration failure and returns no token", async () => { - mintLocalAgentJwt.mockReturnValue("token-that-cannot-register"); - // `row: null` reproduces the registry's own missing-run failure - // (server/src/services/run-secret-redaction.ts:103). - const { db, updateWhere } = fakeDb(null); - + it("fails closed when the run row does not exist and registers nothing", async () => { + const { companyId } = await seedRun(); await expect( - mintAndRegisterRunBearer(db, "agent-1", companyId, "claude_local", runId, "user-1"), + createRunSecretRedactionRegistry(db).register(companyId, randomUUID(), "must-not-return", farFutureRedactionExpiry()), ).rejects.toThrow("Heartbeat run redaction registration failed"); - expect(updateWhere).not.toHaveBeenCalled(); }); - it("never surfaces the minted token text in the registration failure", async () => { - mintLocalAgentJwt.mockReturnValue("must-not-leak-in-error-token"); - const { db } = fakeDb(null); + it("redacts batched runs from their own registrations and enforces company scope", async () => { + const first = await seedRun(); + const foreign = await seedRun(); + const registry = createRunSecretRedactionRegistry(db); + await registry.register(first.companyId, first.runId, "first-secret-value", farFutureRedactionExpiry()); + await registry.register(foreign.companyId, foreign.runId, "foreign-secret-value", farFutureRedactionExpiry()); - await expect( - mintAndRegisterRunBearer(db, "agent-1", companyId, "claude_local", runId, "user-1"), - ).rejects.toSatisfy((error: unknown) => { - const message = error instanceof Error ? error.message : String(error); - return !message.includes("must-not-leak-in-error-token"); + const runs = [ + { id: first.runId, text: "first-secret-value foreign-secret-value" }, + { id: foreign.runId, text: "foreign-secret-value" }, + ]; + const redacted = await registry.redactForRuns(first.companyId, runs); + expect(redacted[0].text).toBe(`${REDACTED_EVENT_VALUE} foreign-secret-value`); + expect(redacted[1].text).toBe("foreign-secret-value"); + }); + + it("masks a value registered under the legacy context_snapshot key for the deprecation window", async () => { + const { companyId, runId } = await seedRun(); + const legacyFingerprint = createHash("sha256").update("legacy-secret").digest("hex"); + const provider = (await import("../secrets/provider-registry.js")).getSecretProvider("local_encrypted"); + const prepared = await provider.createSecret({ value: "legacy-secret" }); + await db.update(heartbeatRuns).set({ + contextSnapshot: { + paperclipSecretRedactions: [{ fingerprintSha256: legacyFingerprint, material: prepared.material }], + }, + }).where(eq(heartbeatRuns.id, runId)); + + const registry = createRunSecretRedactionRegistry(db); + expect(await registry.redactForRun(companyId, runId, "seen legacy-secret here")) + .toBe(`seen ${REDACTED_EVENT_VALUE} here`); + // A new registration on the same run writes only to the table. + await registry.register(companyId, runId, "new-secret", farFutureRedactionExpiry()); + expect(await rowsForRun(companyId, runId)).toHaveLength(1); + expect(await registry.redactForRun(companyId, runId, "legacy-secret and new-secret")) + .toBe(`${REDACTED_EVENT_VALUE} and ${REDACTED_EVENT_VALUE}`); + }); + + it("returns no plaintext for a fingerprint-only row and does not throw", async () => { + const { companyId, runId } = await seedRun(); + const registry = createRunSecretRedactionRegistry(db); + await registry.register(companyId, runId, secret, new Date(0)); + + await createRunSecretRedactionReaper(db).sweep(); + + const [row] = await rowsForRun(companyId, runId); + expect(row.material).toBeNull(); + await expect(registry.redactForRun(companyId, runId, `still has ${secret}`)).resolves.toBe(`still has ${secret}`); + }); + + describe("mintAndRegisterRunBearer", () => { + it("registers the minted bearer with its real JWT expiry", async () => { + const { companyId, agentId, runId } = await seedRun(); + const before = Date.now(); + + const token = await mintAndRegisterRunBearer(db, agentId, companyId, "claude_local", runId, "user-1"); + + expect(token).not.toBeNull(); + const [row] = await rowsForRun(companyId, runId); + expect(row.fingerprintSha256).toBe(createHash("sha256").update(token!).digest("hex")); + // Default TTL is 48h (server/src/agent-auth-jwt.ts). Assert it lands in + // that neighbourhood rather than hardcoding an exact millisecond. + const ttlMs = row.expiresAt.getTime() - before; + expect(ttlMs).toBeGreaterThan(47 * 60 * 60 * 1000); + expect(ttlMs).toBeLessThan(49 * 60 * 60 * 1000); + }); + + it("never surfaces the minted token text in the registration failure", async () => { + const { companyId, agentId } = await seedRun(); + await expect( + mintAndRegisterRunBearer(db, agentId, companyId, "claude_local", randomUUID(), "user-1"), + ).rejects.toSatisfy((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + return !message.includes("token"); + }); + }); + }); + + describe("expiry sweep", () => { + it("clears expired material in a bounded, ordered batch and keeps the fingerprint", async () => { + const { companyId, runId } = await seedRun(); + const registry = createRunSecretRedactionRegistry(db); + await registry.register(companyId, runId, "expired-secret", new Date(Date.now() - 1000)); + await registry.register(companyId, runId, "still-live-secret", farFutureRedactionExpiry()); + + const result = await createRunSecretRedactionReaper(db).sweep(); + + expect(result.cleared).toBe(1); + const rows = await rowsForRun(companyId, runId); + const expired = rows.find((row) => row.fingerprintSha256 === createHash("sha256").update("expired-secret").digest("hex")); + const stillLive = rows.find((row) => row.fingerprintSha256 === createHash("sha256").update("still-live-secret").digest("hex")); + expect(expired?.material).toBeNull(); + expect(stillLive?.material).not.toBeNull(); + }); + + it("orders a bounded batch by expiry so the oldest expired row clears first", async () => { + const { companyId, runId } = await seedRun(); + const registry = createRunSecretRedactionRegistry(db); + await registry.register(companyId, runId, "expired-later", new Date(Date.now() - 1000)); + await registry.register(companyId, runId, "expired-earlier", new Date(Date.now() - 60_000)); + + const reaper = createRunSecretRedactionReaper(db, { batchSize: 1 }); + const result = await reaper.sweep(); + + expect(result.cleared).toBe(1); + const rows = await rowsForRun(companyId, runId); + const earlier = rows.find((row) => row.fingerprintSha256 === createHash("sha256").update("expired-earlier").digest("hex")); + const later = rows.find((row) => row.fingerprintSha256 === createHash("sha256").update("expired-later").digest("hex")); + expect(earlier?.material).toBeNull(); + expect(later?.material).not.toBeNull(); + }); + + it("leaves an unexpired row untouched", async () => { + const { companyId, runId } = await seedRun(); + await createRunSecretRedactionRegistry(db).register(companyId, runId, secret, farFutureRedactionExpiry()); + + const result = await createRunSecretRedactionReaper(db).sweep(); + + expect(result.cleared).toBe(0); + const [row] = await rowsForRun(companyId, runId); + expect(row.material).not.toBeNull(); }); }); }); diff --git a/server/src/__tests__/secret-proposals-routes.test.ts b/server/src/__tests__/secret-proposals-routes.test.ts index 124c8fcabc6..34c773c3f77 100644 --- a/server/src/__tests__/secret-proposals-routes.test.ts +++ b/server/src/__tests__/secret-proposals-routes.test.ts @@ -20,6 +20,7 @@ import { issueComments, issueThreadInteractions, issues, + runSecretRedactions, userSecretDeclarations, userSecretDefinitions, } from "@paperclipai/db"; @@ -367,9 +368,11 @@ describeEmbeddedPostgres("secret proposal routes", () => { expect(secretResponse.body).not.toHaveProperty("valueFingerprintSha256"); const [registeredRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.heartbeatRunId)); expect(JSON.stringify(registeredRun.contextSnapshot)).not.toContain("top-secret"); - expect(registeredRun.contextSnapshot).toMatchObject({ - paperclipSecretRedactions: [expect.objectContaining({ fingerprintSha256: expect.any(String), material: expect.any(Object) })], - }); + const registrations = await db.select().from(runSecretRedactions) + .where(eq(runSecretRedactions.runId, fixture.heartbeatRunId)); + expect(registrations).toEqual([ + expect.objectContaining({ fingerprintSha256: expect.any(String), material: expect.any(Object) }), + ]); const bindingResponse = await request(createAgentApp(fixture)) .post("/api/agents/me/secret-proposals") diff --git a/server/src/agent-auth-jwt.ts b/server/src/agent-auth-jwt.ts index a4f42b9d770..dacfd1ca2ab 100644 --- a/server/src/agent-auth-jwt.ts +++ b/server/src/agent-auth-jwt.ts @@ -119,6 +119,16 @@ function safeCompare(a: string, b: string) { return timingSafeEqual(left, right); } +/** + * The configured run-bearer time to live, in seconds, or `null` when no JWT + * secret is configured (the same condition under which `createLocalAgentJwt` + * returns `null`). The register-first mint wrapper uses this to derive the + * real expiry of a token it just minted, without re-parsing the token. + */ +export function localAgentJwtTtlSeconds(): number | null { + return jwtConfig()?.ttlSeconds ?? null; +} + export function createLocalAgentJwt( agentId: string, companyId: string, diff --git a/server/src/index.ts b/server/src/index.ts index ba8cde96eb9..ffd6523607f 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -95,6 +95,7 @@ import { createProductionLoginSessionReaperRuntime, } from "./services/device-login-reaper.js"; import { createProductionSetupTokenReaper } from "./services/setup-token-reaper.js"; +import { createRunSecretRedactionReaper } from "./services/run-secret-redaction-reaper.js"; import { localAiLoginService } from "./services/local-ai-login.js"; import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js"; import { @@ -1418,6 +1419,28 @@ async function startServerWithDatabaseTeardown( })); }; + // The restart-safe expiry sweep for the run secret redaction registry. It + // runs on startup and on the scheduler interval, so a registered value + // past its expiry has its encrypted material cleared even across a server + // restart. It keeps each row's fingerprint. + const runSecretRedactionReaper = createRunSecretRedactionReaper(db as any); + const logRunSecretRedactionSweepResult = ( + result: Awaited>, + ) => { + if (result.cleared > 0) { + logger.info(result, "run secret redaction reaper cleared expired material"); + } + }; + const scheduleRunSecretRedactionSweep = () => { + if (heartbeatSchedulerStopped) return; + trackHeartbeatSchedulerWork(runSecretRedactionReaper + .sweep() + .then(logRunSecretRedactionSweepResult) + .catch((err) => { + logger.error({ err }, "run secret redaction reaper sweep failed"); + })); + }; + const worktreeRunExecutionActivation = await resolveWorktreeRunExecutionActivationState({ getExperimental: () => instanceSettingsService(db).getExperimental(), }); @@ -1599,6 +1622,15 @@ async function startServerWithDatabaseTeardown( logger.error({ err }, "startup setup-token login reaper sweep failed"); }); + // Run the run secret redaction reaper once at startup, so material that + // expired while the server was down clears before timer ticks start. + await runSecretRedactionReaper + .sweep() + .then(logRunSecretRedactionSweepResult) + .catch((err) => { + logger.error({ err }, "startup run secret redaction reaper sweep failed"); + }); + // Retry any orphan sandbox teardown left by a failed acquire before a server // restart, so a leaked sandbox does not stay allocated across the restart. await runEnvironmentLeaseCleanupSweep(0); @@ -1665,6 +1697,7 @@ async function startServerWithDatabaseTeardown( scheduleTerminalWorkspaceSweep(); scheduleAdapterLoginReaperSweep(); scheduleSetupTokenReaperSweep(); + scheduleRunSecretRedactionSweep(); scheduleEnvironmentLeaseCleanupSweep(); if (heartbeatSchedulerStopped) return; diff --git a/server/src/routes/secrets.ts b/server/src/routes/secrets.ts index 3d3ca863f22..264a09fb1b6 100644 --- a/server/src/routes/secrets.ts +++ b/server/src/routes/secrets.ts @@ -26,7 +26,7 @@ import { accessService } from "../services/access.js"; import { heartbeatService } from "../services/heartbeat.js"; import { issueService } from "../services/issues.js"; import type { IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js"; -import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; +import { createRunSecretRedactionRegistry, farFutureRedactionExpiry } from "../services/run-secret-redaction.js"; import { notifySecretProposalResolution } from "../services/secret-proposal-notifications.js"; import { assertCanResolveProposal } from "../services/secret-proposal-authorization.js"; @@ -198,7 +198,8 @@ export function secretRoutes(db: Db, deps: SecretRoutesDeps = {}) { ? await proposals.createSecret({ companyId: context.companyId, heartbeatRunId: context.heartbeatRunId, - registerForRedaction: (value) => runRedactions.register(context.companyId, context.heartbeatRunId, value), + registerForRedaction: (value) => + runRedactions.register(context.companyId, context.heartbeatRunId, value, farFutureRedactionExpiry()), }, { name: body.name, key: body.key, @@ -347,7 +348,8 @@ export function secretRoutes(db: Db, deps: SecretRoutesDeps = {}) { configPath: secret.configPath, bindingId: secret.bindingId, issueId: null, - registerForRedaction: (value) => runRedactions.register(context.companyId, context.heartbeatRunId, value), + registerForRedaction: (value) => + runRedactions.register(context.companyId, context.heartbeatRunId, value, farFutureRedactionExpiry()), }, ); res.set("Cache-Control", "no-store"); diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts index 9f2423a3ec3..236af005d8f 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts @@ -13,12 +13,14 @@ import { issueComments, issueThreadInteractions, issues, + runSecretRedactions, } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { initializeRunIdentity, reserveSteeredIdentity, reconcileSteeredIdentity } from "../run-identity.js"; import { documentService } from "../documents.js"; import { issueService } from "../issues.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; +import * as runBearerModule from "../run-bearer.js"; import { READ_CURRENT_WAKE_COMMENTS_TOOL_NAME } from "./current-wake-comments.js"; import { CAPABILITY_SEMANTIC_TOOL_CATALOG } from "../../vendor/paperclip-runner/index.js"; @@ -1162,11 +1164,9 @@ describe("PaperclipRunnerToolAuthority", () => { } async function registeredFingerprints(forRunId: string): Promise { - const [row] = await db.select({ contextSnapshot: heartbeatRuns.contextSnapshot }) - .from(heartbeatRuns).where(eq(heartbeatRuns.id, forRunId)); - const entries = (row?.contextSnapshot as { paperclipSecretRedactions?: Array<{ fingerprintSha256: string }> } | undefined) - ?.paperclipSecretRedactions ?? []; - return entries.map((entry) => entry.fingerprintSha256); + const rows = await db.select({ fingerprintSha256: runSecretRedactions.fingerprintSha256 }) + .from(runSecretRedactions).where(eq(runSecretRedactions.runId, forRunId)); + return rows.map((row) => row.fingerprintSha256); } it("registers the minted bearer against the correct company and run at the create_project mint site", async () => { @@ -1218,6 +1218,36 @@ describe("PaperclipRunnerToolAuthority", () => { expect(await registeredFingerprints(fixture.runId)).toContain(fingerprint); }); + // B1 — the runner API path must register at most one bearer for each run + // and each distinct bearer scope. Two tool calls on the same authority, + // with the same adapterType and responsibleUserId, must mint and register + // the bearer once and reuse it for the second call. + it("registers exactly one bearer across two runner API tool calls on one authority", async () => { + const fixture = await freshRunFixture("call_api-reuses-bearer"); + const authority = new PaperclipRunnerToolAuthority(db, { + companyId, agentId, issueId: fixture.issueId, runId: fixture.runId, apiUrl: "http://127.0.0.1:1", + }); + const mintSpy = vi.spyOn(runBearerModule, "mintAndRegisterRunBearer"); + const before = capturedRunnerApiCalls.length; + + await authority.execute({ + tool: "call_api", callId: "reuse-bearer-first", + arguments: { operationId: "GET /api/companies/{companyId}/projects" }, + }); + await authority.execute({ + tool: "call_api", callId: "reuse-bearer-second", + arguments: { operationId: "GET /api/companies/{companyId}/projects" }, + }); + + expect(mintSpy).toHaveBeenCalledTimes(1); + expect(capturedRunnerApiCalls).toHaveLength(before + 2); + const [first, second] = capturedRunnerApiCalls.slice(before); + expect(second.token).toBe(first.token); + expect(await registeredFingerprints(fixture.runId)) + .toEqual([createHash("sha256").update(first.token).digest("hex")]); + mintSpy.mockRestore(); + }); + it("aborts before the API call dispatches when registration fails at the call_api mint site", async () => { const fixture = await freshRunFixture("call_api-aborts"); const authority = new PaperclipRunnerToolAuthority(db, { diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index bb3b42a8a83..4fe9b5d800f 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -131,8 +131,34 @@ function canonicalJson(value: unknown): string { } export class PaperclipRunnerToolAuthority { + // Holds one minted, registered run bearer for each distinct (adapterType, + // responsibleUserId) pair this run's tool calls have seen. A run with many + // runner API tool calls then mints and registers a bearer at most once for + // each pair, not once for each call. This map lives only as long as this + // authority instance. Production builds one instance for each run, so a + // bearer here is never reused across two different runs. A failed mint is + // not cached, so the next call retries instead of repeating the failure. + #runBearers = new Map>(); + constructor(readonly db: Db, readonly binding: Binding) {} + async #mintOrReuseRunBearer( + adapterType: string, + responsibleUserId: string | null | undefined, + ): Promise { + const key = `${adapterType}:${responsibleUserId ?? ""}`; + const cached = this.#runBearers.get(key); + if (cached) return cached; + const pending = mintAndRegisterRunBearer( + this.db, this.binding.agentId, this.binding.companyId, adapterType, this.binding.runId, responsibleUserId, + ).catch((error) => { + this.#runBearers.delete(key); + throw error; + }); + this.#runBearers.set(key, pending); + return pending; + } + definitions(): Array> { const workMode = this.binding.workMode ?? "standard"; const definitions: Array> = @@ -313,7 +339,7 @@ export class PaperclipRunnerToolAuthority { case "list_project_repositories": case "list_projects": { const apiUrl = this.binding.apiUrl ?? process.env.PAPERCLIP_API_URL; - const token = await mintAndRegisterRunBearer(this.db, this.binding.agentId, this.binding.companyId, context.actor.adapterType, this.binding.runId, context.run.responsibleUserId); + const token = await this.#mintOrReuseRunBearer(context.actor.adapterType, context.run.responsibleUserId); if (!apiUrl || !token) throw new Error("Project tool authentication is unavailable"); return callProjectTool({ name: call.tool, arguments: input, apiUrl, token, companyId: this.binding.companyId, issueId: this.binding.issueId, agentId: this.binding.agentId, @@ -417,7 +443,7 @@ export class PaperclipRunnerToolAuthority { const { input, operation } = validateRunnerApiCall(value, context); const apiUrl = this.binding.apiUrl ?? process.env.PAPERCLIP_API_URL; if (!apiUrl) throw new Error("Paperclip API origin is unavailable"); - const token = await mintAndRegisterRunBearer(this.db, this.binding.agentId, this.binding.companyId, bound.actor.adapterType, this.binding.runId, bound.run.responsibleUserId); + const token = await this.#mintOrReuseRunBearer(bound.actor.adapterType, bound.run.responsibleUserId); if (!token) throw new Error("Paperclip run authentication is unavailable"); const execute = async () => { const current = await this.#boundContext(); diff --git a/server/src/services/run-bearer.ts b/server/src/services/run-bearer.ts index f0255e2cd0c..065b3e28aa1 100644 --- a/server/src/services/run-bearer.ts +++ b/server/src/services/run-bearer.ts @@ -1,6 +1,6 @@ import type { Db } from "@paperclipai/db"; import type { AgentApiKeyScope } from "@paperclipai/shared"; -import { createLocalAgentJwt } from "../agent-auth-jwt.js"; +import { createLocalAgentJwt, localAgentJwtTtlSeconds } from "../agent-auth-jwt.js"; import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js"; /** @@ -28,6 +28,11 @@ export async function mintAndRegisterRunBearer( ): Promise { const token = createLocalAgentJwt(agentId, companyId, adapterType, runId, responsibleUserId, keyScope); if (!token) return null; - await createRunSecretRedactionRegistry(db).register(companyId, runId, token); + // The same TTL config `createLocalAgentJwt` used to set this token's `exp` + // claim, read again here so the registry records the token's real expiry + // instead of a separately guessed value. + const ttlSeconds = localAgentJwtTtlSeconds() ?? 0; + const expiresAt = new Date(Date.now() + ttlSeconds * 1000); + await createRunSecretRedactionRegistry(db).register(companyId, runId, token, expiresAt); return token; } diff --git a/server/src/services/run-secret-redaction-reaper.ts b/server/src/services/run-secret-redaction-reaper.ts new file mode 100644 index 00000000000..371ea890bfe --- /dev/null +++ b/server/src/services/run-secret-redaction-reaper.ts @@ -0,0 +1,44 @@ +import { and, asc, inArray, isNotNull, lte } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { runSecretRedactions } from "@paperclipai/db"; + +// The restart-safe expiry sweep for the run secret redaction registry. +// +// A lazy purge on read is not enough: a row a caller never reads again would +// keep its encrypted material forever. This reaper runs on server startup and +// on the scheduler interval, following the setup-token reaper convention +// (server/src/services/setup-token-reaper.ts). One sweep pass clears a +// bounded, ordered batch of rows whose expiry has passed: it orders by +// expiry then id, and clears at most `batchSize` rows. It never deletes a +// row and never touches `fingerprintSha256`; the run's own foreign key +// cascade is what removes a row, when its run row is removed. +export interface RunSecretRedactionSweepResult { + cleared: number; +} + +const DEFAULT_SWEEP_BATCH_SIZE = 500; + +export function createRunSecretRedactionReaper( + db: Db, + options: { batchSize?: number; now?: () => Date } = {}, +) { + const batchSize = options.batchSize ?? DEFAULT_SWEEP_BATCH_SIZE; + const now = options.now ?? (() => new Date()); + + async function sweep(): Promise { + const candidates = await db.select({ id: runSecretRedactions.id }) + .from(runSecretRedactions) + .where(and(isNotNull(runSecretRedactions.material), lte(runSecretRedactions.expiresAt, now()))) + .orderBy(asc(runSecretRedactions.expiresAt), asc(runSecretRedactions.id)) + .limit(batchSize); + if (candidates.length === 0) return { cleared: 0 }; + await db.update(runSecretRedactions) + .set({ material: null }) + .where(inArray(runSecretRedactions.id, candidates.map((row) => row.id))); + return { cleared: candidates.length }; + } + + return { sweep }; +} + +export type RunSecretRedactionReaper = ReturnType; diff --git a/server/src/services/run-secret-redaction.ts b/server/src/services/run-secret-redaction.ts index 1dfbdbedc61..bd6f08f4e42 100644 --- a/server/src/services/run-secret-redaction.ts +++ b/server/src/services/run-secret-redaction.ts @@ -1,14 +1,16 @@ import { createHash } from "node:crypto"; import { and, eq, inArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { heartbeatRuns } from "@paperclipai/db"; +import { heartbeatRuns, runSecretRedactions } from "@paperclipai/db"; import { REDACTED_EVENT_VALUE } from "../redaction.js"; import { getSecretProvider } from "../secrets/provider-registry.js"; import type { StoredSecretVersionMaterial } from "../secrets/types.js"; const REGISTRY_KEY = "paperclipSecretRedactions"; -// Project only the registry: run contexts can contain megabytes of prompt data. -const registrySnapshot = sql`jsonb_build_object('paperclipSecretRedactions', ${heartbeatRuns.contextSnapshot} -> 'paperclipSecretRedactions')`; +// Project only the legacy registry: a run context can hold megabytes of +// prompt data. This projection reads only the entries a run registered +// before the move to `run_secret_redactions`, kept for a deprecation window. +const legacyRegistrySnapshot = sql`jsonb_build_object('paperclipSecretRedactions', ${heartbeatRuns.contextSnapshot} -> 'paperclipSecretRedactions')`; type RegistryEntry = { fingerprintSha256: string; @@ -21,7 +23,7 @@ function asRecord(value: unknown): Record | null { : null; } -function registryEntries(contextSnapshot: unknown): RegistryEntry[] { +function legacyRegistryEntries(contextSnapshot: unknown): RegistryEntry[] { const context = asRecord(contextSnapshot); const raw = context?.[REGISTRY_KEY]; if (!Array.isArray(raw)) return []; @@ -41,6 +43,17 @@ function redactText(input: string, values: string[]) { ); } +// A proposed or bound company secret value carries no token-style expiry +// claim the way a run bearer does, and the pre-move registry masked such a +// value for as long as its run row existed. This far horizon keeps that same +// indefinite masking, so a caller registering a non-bearer value can supply a +// real (if distant) date instead of a guessed short one. +const NON_BEARER_REGISTRATION_HORIZON_MS = 100 * 365 * 24 * 60 * 60 * 1000; + +export function farFutureRedactionExpiry(now: Date = new Date()): Date { + return new Date(now.getTime() + NON_BEARER_REGISTRATION_HORIZON_MS); +} + export function redactRegisteredSecretValues(input: T, values: string[]): T { if (typeof input === "string") return redactText(input, values) as T; if (Array.isArray(input)) return input.map((value) => redactRegisteredSecretValues(value, values)) as T; @@ -59,27 +72,56 @@ export function redactRegisteredSecretValues(input: T, values: string[]): T { export function createRunSecretRedactionRegistry(db: Db) { const provider = getSecretProvider("local_encrypted"); - async function valuesForRuns(rows: Array<{ contextSnapshot: unknown }>) { - const entries = rows.flatMap((row) => registryEntries(row.contextSnapshot)); + // Resolves each distinct fingerprint once, using a caller-provided cache so + // one request can share resolves across many runs. Returns plaintext values + // sorted longest-first, so a later substring redaction replaces a longer + // registered value before a shorter value it contains. + async function resolveEntries( + entries: RegistryEntry[], + cache: Map>, + ): Promise { const unique = new Map(entries.map((entry) => [entry.fingerprintSha256, entry])); - const values = await Promise.all( - [...unique.values()].map((entry) => provider.resolveVersion({ - material: entry.material, - externalRef: null, - })), - ); + const values = await Promise.all([...unique.values()].map((entry) => { + let value = cache.get(entry.fingerprintSha256); + if (!value) { + value = provider.resolveVersion({ material: entry.material, externalRef: null }); + cache.set(entry.fingerprintSha256, value); + } + return value; + })); return values.sort((left, right) => right.length - left.length); } - async function valuesForRun(companyId: string, runId: string) { - const rows = await db.select({ contextSnapshot: registrySnapshot }) + async function legacyEntriesByRun(companyId: string, runIds: string[]): Promise> { + if (runIds.length === 0) return new Map(); + const rows = await db.select({ id: heartbeatRuns.id, contextSnapshot: legacyRegistrySnapshot }) .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); - return valuesForRuns(rows); + .where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))); + return new Map(rows.map((row) => [row.id, legacyRegistryEntries(row.contextSnapshot)])); } - async function valuesForIssue(companyId: string, issueId: string) { - const rows = await db.select({ contextSnapshot: registrySnapshot }) + // A fingerprint-only row (its `material` cleared by the expiry sweep) + // carries nothing to resolve. It is dropped here, never thrown on. + function tableEntries(rows: Array<{ fingerprintSha256: string; material: unknown }>): RegistryEntry[] { + return rows.flatMap((row) => { + const material = asRecord(row.material); + return material ? [{ fingerprintSha256: row.fingerprintSha256, material }] : []; + }); + } + + async function valuesForRun(companyId: string, runId: string): Promise { + const [tableRows, legacyByRun] = await Promise.all([ + db.select({ fingerprintSha256: runSecretRedactions.fingerprintSha256, material: runSecretRedactions.material }) + .from(runSecretRedactions) + .where(and(eq(runSecretRedactions.companyId, companyId), eq(runSecretRedactions.runId, runId))), + legacyEntriesByRun(companyId, [runId]), + ]); + const entries = [...(legacyByRun.get(runId) ?? []), ...tableEntries(tableRows)]; + return resolveEntries(entries, new Map()); + } + + async function issueRunIds(companyId: string, issueId: string): Promise { + const rows = await db.select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(and( eq(heartbeatRuns.companyId, companyId), @@ -88,54 +130,72 @@ export function createRunSecretRedactionRegistry(db: Db) { sql`${heartbeatRuns.contextSnapshot} -> 'paperclipIssue' ->> 'id' = ${issueId}`, ), )); - return valuesForRuns(rows); + return rows.map((row) => row.id); + } + + async function valuesForIssue(companyId: string, issueId: string): Promise { + const runIds = await issueRunIds(companyId, issueId); + if (runIds.length === 0) return []; + const [tableRows, legacyByRun] = await Promise.all([ + db.select({ fingerprintSha256: runSecretRedactions.fingerprintSha256, material: runSecretRedactions.material }) + .from(runSecretRedactions) + .where(and(eq(runSecretRedactions.companyId, companyId), inArray(runSecretRedactions.runId, runIds))), + legacyEntriesByRun(companyId, runIds), + ]); + const entries = [...[...legacyByRun.values()].flat(), ...tableEntries(tableRows)]; + return resolveEntries(entries, new Map()); } return { - register: async (companyId: string, runId: string, value: string) => { + // `expiresAt` must be the real expiry of the value, not a guess. For a + // run bearer, this is the token's own JWT `exp`. The later sweep clears + // `material` at this time. A run that registers the same value twice + // gets one row. Two different runs that register the identical value + // each get their own row, so each run can still mask its own text. + register: async (companyId: string, runId: string, value: string, expiresAt: Date) => { const fingerprintSha256 = createHash("sha256").update(value).digest("hex"); - await db.transaction(async (tx) => { - const row = await tx.select({ contextSnapshot: heartbeatRuns.contextSnapshot }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))) - .for("update") - .then((rows) => rows[0] ?? null); - if (!row) throw new Error("Heartbeat run redaction registration failed"); - if (registryEntries(row.contextSnapshot).some((entry) => entry.fingerprintSha256 === fingerprintSha256)) { - return; - } - const prepared = await provider.createSecret({ value }); - const entry: RegistryEntry = { fingerprintSha256, material: prepared.material }; - const contextSnapshot = asRecord(row.contextSnapshot) ?? {}; - const currentEntries = Array.isArray(contextSnapshot[REGISTRY_KEY]) - ? contextSnapshot[REGISTRY_KEY] - : []; - await tx.update(heartbeatRuns) - .set({ - contextSnapshot: { ...contextSnapshot, [REGISTRY_KEY]: [...currentEntries, entry] }, - updatedAt: new Date(), - }) - .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); - }); + const existing = await db.select({ id: runSecretRedactions.id }) + .from(runSecretRedactions) + .where(and( + eq(runSecretRedactions.companyId, companyId), + eq(runSecretRedactions.fingerprintSha256, fingerprintSha256), + eq(runSecretRedactions.runId, runId), + )) + .limit(1); + if (existing.length > 0) return; + const [run] = await db.select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))) + .limit(1); + if (!run) throw new Error("Heartbeat run redaction registration failed"); + const prepared = await provider.createSecret({ value }); + await db.insert(runSecretRedactions) + .values({ companyId, runId, fingerprintSha256, material: prepared.material, expiresAt }) + .onConflictDoNothing({ + target: [runSecretRedactions.companyId, runSecretRedactions.fingerprintSha256, runSecretRedactions.runId], + }); }, redactForRuns: async (companyId: string, runs: T[]): Promise => { if (runs.length === 0) return []; - const rows = await db.select({ id: heartbeatRuns.id, contextSnapshot: registrySnapshot }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runs.map((run) => run.id)))); - // Resolve each encrypted value once per request, but apply only each run's - // own registry. Do not retain plaintext secrets across requests. + const runIds = runs.map((run) => run.id); + const [tableRows, legacyByRun] = await Promise.all([ + db.select({ runId: runSecretRedactions.runId, fingerprintSha256: runSecretRedactions.fingerprintSha256, material: runSecretRedactions.material }) + .from(runSecretRedactions) + .where(and(eq(runSecretRedactions.companyId, companyId), inArray(runSecretRedactions.runId, runIds))), + legacyEntriesByRun(companyId, runIds), + ]); + const tableByRun = new Map>(); + for (const row of tableRows) { + const existingRows = tableByRun.get(row.runId) ?? []; + existingRows.push(row); + tableByRun.set(row.runId, existingRows); + } + // Resolve each encrypted value once per request, but apply only each + // run's own registry. Do not retain plaintext secrets across requests. const resolved = new Map>(); - const valuesByRun = new Map(await Promise.all(rows.map(async (row) => { - const values = await Promise.all(registryEntries(row.contextSnapshot).map((entry) => { - let value = resolved.get(entry.fingerprintSha256); - if (!value) { - value = provider.resolveVersion({ material: entry.material, externalRef: null }); - resolved.set(entry.fingerprintSha256, value); - } - return value; - })); - return [row.id, values.sort((a, b) => b.length - a.length)] as const; + const valuesByRun = new Map(await Promise.all(runIds.map(async (runId) => { + const entries = [...(legacyByRun.get(runId) ?? []), ...tableEntries(tableByRun.get(runId) ?? [])]; + return [runId, await resolveEntries(entries, resolved)] as const; }))); return runs.map((run) => redactRegisteredSecretValues(run, valuesByRun.get(run.id) ?? [])); },