mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-28 20:21:50 +01:00
fix(server): redact an agent-authored run bearer before persistence
Close a gap in the JWT candidate scanner: it took only the first three dot-joined segments of a chain, so a dotted prefix right before a bearer hid the real token from the fingerprint lookup. The scanner now offers every three-segment window in a chain, and both the read mask and the new write-time redactor share this one scanner. Add redactAuthoredRunBearer(): an exact registered-bearer match first, then a narrow JWT-shaped fallback that requires the header segment to decode as JSON with an alg member. This keeps an unexpanded variable, a placeholder, and the redaction marker untouched. Call it from addComment() for an agent-authored body, and from create() and update() for an agent-authored issue description — the two writers of issues.description. Each read runs on the caller's own transaction connection, and a failed lookup aborts the write instead of persisting unredacted text. Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
f3144b17ca
commit
ba465523ea
5 files changed
+527
-32
No files matched your search
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
@@ -12,15 +12,18 @@ import {
|
||||
vi,
|
||||
} from "vitest";
|
||||
import {
|
||||
agents,
|
||||
assets,
|
||||
companies,
|
||||
closeRegisteredClients,
|
||||
companyMemberships,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueAttachments,
|
||||
issueComments,
|
||||
issueReferenceMentions,
|
||||
issues,
|
||||
runSecretRedactions,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
companySearchQuerySchema,
|
||||
@@ -31,11 +34,16 @@ import {
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { REDACTED_EVENT_VALUE } from "../redaction.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { companySearchService } from "../services/company-search.js";
|
||||
import { buildPaperclipWakePayload } from "../services/heartbeat.js";
|
||||
import { issueReferenceService } from "../services/issue-references.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import {
|
||||
createRunSecretRedactionRegistry,
|
||||
farFutureRedactionExpiry,
|
||||
} from "../services/run-secret-redaction.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
|
||||
const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL;
|
||||
@@ -76,6 +84,9 @@ describeEmbeddedPostgres("deleted issue comment redaction", () => {
|
||||
await db.delete(issueComments);
|
||||
await db.delete(assets);
|
||||
await db.delete(issues);
|
||||
await db.delete(runSecretRedactions);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agents);
|
||||
await db.delete(companyMemberships);
|
||||
await db.delete(companies);
|
||||
});
|
||||
@@ -579,4 +590,72 @@ describeEmbeddedPostgres("deleted issue comment redaction", () => {
|
||||
(await refs.listIssueReferenceSummary(sourceIssueId)).outbound,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
// Write-time bearer redaction (PAP-6528). Step C already proves the
|
||||
// read-mask side of this fingerprint registry in
|
||||
// run-secret-redaction.test.ts; these tests cover the write side:
|
||||
// addComment() must redact a registered run bearer out of an
|
||||
// agent-authored body before the row is written.
|
||||
async function seedAgentRun(companyId: string) {
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Redaction agent",
|
||||
role: "engineer",
|
||||
adapterType: "claude_local",
|
||||
status: "idle",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId,
|
||||
status: "running",
|
||||
contextSnapshot: {},
|
||||
});
|
||||
return { agentId, runId };
|
||||
}
|
||||
|
||||
function jwtShapedBearer(seed: string): string {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "HS256", typ: "JWT" })).toString("base64url");
|
||||
const payload = Buffer.from(`payload-${seed}`).toString("base64url");
|
||||
const signature = Buffer.from(`sig-${seed}`).toString("base64url");
|
||||
return `${header}.${payload}.${signature}`;
|
||||
}
|
||||
|
||||
it("redacts a registered run bearer from an agent-authored comment body before it is written (PAP-6528)", async () => {
|
||||
const { companyId, issueId } = await seedIssue();
|
||||
const { agentId, runId } = await seedAgentRun(companyId);
|
||||
const bearer = jwtShapedBearer("comment-agent");
|
||||
await createRunSecretRedactionRegistry(db).register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const comment = await issueService(db).addComment(
|
||||
issueId,
|
||||
`Authorization: Bearer ${bearer}`,
|
||||
{ agentId, runId },
|
||||
);
|
||||
|
||||
expect(comment.body).toBe(`Authorization: Bearer ${REDACTED_EVENT_VALUE}`);
|
||||
const [stored] = await db
|
||||
.select()
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.id, comment.id));
|
||||
expect(stored?.body).toBe(`Authorization: Bearer ${REDACTED_EVENT_VALUE}`);
|
||||
});
|
||||
|
||||
it("does not redact a human-authored comment body carrying the same bearer shape (out of scope for PAP-6528)", async () => {
|
||||
const { companyId, issueId } = await seedIssue();
|
||||
const { runId } = await seedAgentRun(companyId);
|
||||
const bearer = jwtShapedBearer("comment-human");
|
||||
await createRunSecretRedactionRegistry(db).register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const comment = await issueService(db).addComment(
|
||||
issueId,
|
||||
`Authorization: Bearer ${bearer}`,
|
||||
{ userId: "board-user-1" },
|
||||
);
|
||||
|
||||
expect(comment.body).toBe(`Authorization: Bearer ${bearer}`);
|
||||
});
|
||||
});
|
||||
@@ -42,6 +42,10 @@ import {
|
||||
ISSUE_LIST_MAX_LIMIT,
|
||||
issueService,
|
||||
} from "../services/issues.ts";
|
||||
import {
|
||||
createRunSecretRedactionRegistry,
|
||||
farFutureRedactionExpiry,
|
||||
} from "../services/run-secret-redaction.ts";
|
||||
import {
|
||||
WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE,
|
||||
WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE,
|
||||
@@ -7223,3 +7227,165 @@ describeEmbeddedPostgres("issueService.addComment createdByRunId", () => {
|
||||
expect(duplicates).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// Write-time redaction of an agent-authored issue description (PAP-6528).
|
||||
// `create` and `update` are the only two writers of `issues.description`
|
||||
// (the child-create helper and the accepted-plan-decomposition path both
|
||||
// funnel through `create`), so covering these two methods covers every
|
||||
// writer.
|
||||
describeEmbeddedPostgres("issueService write-time run bearer redaction", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let svc!: ReturnType<typeof issueService>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-description-redaction-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
svc = issueService(db);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
function jwtShapedBearer(seed: string): string {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "HS256", typ: "JWT" })).toString("base64url");
|
||||
const payload = Buffer.from(`payload-${seed}`).toString("base64url");
|
||||
const signature = Buffer.from(`sig-${seed}`).toString("base64url");
|
||||
return `${header}.${payload}.${signature}`;
|
||||
}
|
||||
|
||||
async function seedCompanyAgentRun() {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Description Redaction Co",
|
||||
issuePrefix: `D${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Redaction agent",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" });
|
||||
return { companyId, agentId, runId };
|
||||
}
|
||||
|
||||
it("redacts a registered run bearer from an agent-created issue's description before it is written", async () => {
|
||||
const { companyId, agentId, runId } = await seedCompanyAgentRun();
|
||||
const bearer = jwtShapedBearer("create-exact");
|
||||
await createRunSecretRedactionRegistry(db).register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Agent-created task",
|
||||
description: `Authorization: Bearer ${bearer}`,
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
createdByAgentId: agentId,
|
||||
});
|
||||
|
||||
expect(issue.description).toBe("Authorization: Bearer ***REDACTED***");
|
||||
const [stored] = await db.select({ description: issues.description }).from(issues).where(eq(issues.id, issue.id));
|
||||
expect(stored?.description).toBe("Authorization: Bearer ***REDACTED***");
|
||||
});
|
||||
|
||||
it("redacts an unregistered JWT-shaped bearer from an agent-created description via the narrow fallback", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAgentRun();
|
||||
const bearer = jwtShapedBearer("create-fallback");
|
||||
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Agent-created task",
|
||||
description: `Authorization: Bearer ${bearer}`,
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
createdByAgentId: agentId,
|
||||
});
|
||||
|
||||
expect(issue.description).toBe("Authorization: Bearer ***REDACTED***");
|
||||
});
|
||||
|
||||
it("does not redact a registered bearer from a description with no agent author", async () => {
|
||||
const { companyId, runId } = await seedCompanyAgentRun();
|
||||
const bearer = jwtShapedBearer("create-no-agent");
|
||||
await createRunSecretRedactionRegistry(db).register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Manually created task",
|
||||
description: `Authorization: Bearer ${bearer}`,
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
});
|
||||
|
||||
expect(issue.description).toBe(`Authorization: Bearer ${bearer}`);
|
||||
});
|
||||
|
||||
it("leaves an unexpanded environment variable in an agent-created description unchanged", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAgentRun();
|
||||
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Agent-created task",
|
||||
description: "Authorization: Bearer $PAPERCLIP_API_KEY",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
createdByAgentId: agentId,
|
||||
});
|
||||
|
||||
expect(issue.description).toBe("Authorization: Bearer $PAPERCLIP_API_KEY");
|
||||
});
|
||||
|
||||
it("redacts a registered run bearer from an agent-actor's description update before it is written", async () => {
|
||||
const { companyId, agentId, runId } = await seedCompanyAgentRun();
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Task to update",
|
||||
description: "original description",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
});
|
||||
const bearer = jwtShapedBearer("update-exact");
|
||||
await createRunSecretRedactionRegistry(db).register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const updated = await svc.update(issue.id, {
|
||||
description: `Authorization: Bearer ${bearer}`,
|
||||
actorAgentId: agentId,
|
||||
});
|
||||
|
||||
expect(updated?.description).toBe("Authorization: Bearer ***REDACTED***");
|
||||
const [stored] = await db.select({ description: issues.description }).from(issues).where(eq(issues.id, issue.id));
|
||||
expect(stored?.description).toBe("Authorization: Bearer ***REDACTED***");
|
||||
});
|
||||
|
||||
it("does not redact a registered bearer from a description update with no agent actor", async () => {
|
||||
const { companyId, runId } = await seedCompanyAgentRun();
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Task to update",
|
||||
description: "original description",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
});
|
||||
const bearer = jwtShapedBearer("update-no-agent");
|
||||
await createRunSecretRedactionRegistry(db).register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const updated = await svc.update(issue.id, {
|
||||
description: `Authorization: Bearer ${bearer}`,
|
||||
actorUserId: "local-board",
|
||||
});
|
||||
|
||||
expect(updated?.description).toBe(`Authorization: Bearer ${bearer}`);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,7 @@ import { REDACTED_EVENT_VALUE } from "../redaction.js";
|
||||
import {
|
||||
createRunSecretRedactionRegistry,
|
||||
farFutureRedactionExpiry,
|
||||
redactAuthoredRunBearer,
|
||||
redactRegisteredSecretValues,
|
||||
} from "../services/run-secret-redaction.js";
|
||||
import { createRunSecretRedactionReaper } from "../services/run-secret-redaction-reaper.js";
|
||||
@@ -386,6 +387,22 @@ describeEmbeddedPostgres("run secret redaction registry (company-scoped table)",
|
||||
.toBe(`token ${bearer} here`);
|
||||
});
|
||||
|
||||
it("masks a bearer whose chain a dotted prefix word joins into a longer run (PAP-6528)", async () => {
|
||||
// Before the fix, the scanner took only the first three dot-joined
|
||||
// segments of a chain. "paperclip.local.<bearer>" then offered
|
||||
// "paperclip.local.<header>" as the sole candidate, and the real
|
||||
// bearer's own three segments were never offered to the digest
|
||||
// lookup, so it survived in plain text.
|
||||
const { companyId, runId } = await seedRun();
|
||||
const registry = createRunSecretRedactionRegistry(db);
|
||||
const bearer = jwtShaped("dot-prefix");
|
||||
await registry.register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const result = await registry.redactForRun(companyId, runId, `paperclip.local.${bearer}`);
|
||||
|
||||
expect(result).toBe(`paperclip.local.${REDACTED_EVENT_VALUE}`);
|
||||
});
|
||||
|
||||
it("leaves an unregistered JWT-shaped string and a literal environment variable reference unchanged", async () => {
|
||||
const { companyId, runId } = await seedRun();
|
||||
const registry = createRunSecretRedactionRegistry(db);
|
||||
@@ -421,4 +438,88 @@ describeEmbeddedPostgres("run secret redaction registry (company-scoped table)",
|
||||
selectSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
// The write-time redactor for an agent-authored comment body or issue
|
||||
// description (PAP-6528). Step C already proves the cross-issue paste and
|
||||
// the fingerprint-only row above; these tests cover the two write-time
|
||||
// arms and the text a false positive must never touch.
|
||||
describe("redactAuthoredRunBearer (write-time redaction)", () => {
|
||||
function jwtShaped(seed: string): string {
|
||||
const segment = (label: string) => Buffer.from(`${label}-${seed}`).toString("base64url");
|
||||
return `${segment("header")}.${segment("payload")}.${segment("sig")}`;
|
||||
}
|
||||
|
||||
// A candidate whose first segment decodes as a real JWT header, the
|
||||
// shape the D4 fallback arm requires. `jwtShaped` above is enough for an
|
||||
// exact-match test, because that arm only compares hashes, but the
|
||||
// fallback arm needs the header to actually decode this way.
|
||||
function realJwtShaped(seed: string): string {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "HS256", typ: "JWT" })).toString("base64url");
|
||||
const payload = Buffer.from(`payload-${seed}`).toString("base64url");
|
||||
const signature = Buffer.from(`sig-${seed}`).toString("base64url");
|
||||
return `${header}.${payload}.${signature}`;
|
||||
}
|
||||
|
||||
it("redacts an exact registered bearer and keeps the header words", async () => {
|
||||
const { companyId, runId } = await seedRun();
|
||||
const registry = createRunSecretRedactionRegistry(db);
|
||||
const bearer = jwtShaped("write-exact");
|
||||
await registry.register(companyId, runId, bearer, farFutureRedactionExpiry());
|
||||
|
||||
const result = await redactAuthoredRunBearer(db, companyId, `Authorization: Bearer ${bearer}`);
|
||||
|
||||
expect(result).toBe(`Authorization: Bearer ${REDACTED_EVENT_VALUE}`);
|
||||
});
|
||||
|
||||
it("redacts a JWT-shaped value the registry does not yet hold, via the narrow shape fallback", async () => {
|
||||
const unregistered = realJwtShaped("write-fallback");
|
||||
|
||||
const result = await redactAuthoredRunBearer(db, randomUUID(), `Authorization: Bearer ${unregistered}`);
|
||||
|
||||
expect(result).toBe(`Authorization: Bearer ${REDACTED_EVENT_VALUE}`);
|
||||
});
|
||||
|
||||
it("leaves an unexpanded environment variable, its brace form, and the exact incident-report line unchanged", async () => {
|
||||
const companyId = randomUUID();
|
||||
|
||||
await expect(redactAuthoredRunBearer(db, companyId, "$PAPERCLIP_API_KEY"))
|
||||
.resolves.toBe("$PAPERCLIP_API_KEY");
|
||||
await expect(redactAuthoredRunBearer(db, companyId, "${PAPERCLIP_API_KEY}"))
|
||||
.resolves.toBe("${PAPERCLIP_API_KEY}");
|
||||
await expect(redactAuthoredRunBearer(db, companyId, "Authorization: Bearer $PAPERCLIP_API_KEY"))
|
||||
.resolves.toBe("Authorization: Bearer $PAPERCLIP_API_KEY");
|
||||
});
|
||||
|
||||
it("leaves a placeholder and the redaction marker unchanged", async () => {
|
||||
const companyId = randomUUID();
|
||||
|
||||
await expect(redactAuthoredRunBearer(db, companyId, "Authorization: Bearer <token>"))
|
||||
.resolves.toBe("Authorization: Bearer <token>");
|
||||
await expect(redactAuthoredRunBearer(db, companyId, "Authorization: Bearer YOUR_TOKEN"))
|
||||
.resolves.toBe("Authorization: Bearer YOUR_TOKEN");
|
||||
await expect(redactAuthoredRunBearer(db, companyId, REDACTED_EVENT_VALUE))
|
||||
.resolves.toBe(REDACTED_EVENT_VALUE);
|
||||
});
|
||||
|
||||
it("leaves an ordinary dotted identifier and a file name unchanged (PAP-6528 D4)", async () => {
|
||||
const companyId = randomUUID();
|
||||
|
||||
await expect(redactAuthoredRunBearer(db, companyId, "com.example.Foo.bar"))
|
||||
.resolves.toBe("com.example.Foo.bar");
|
||||
await expect(redactAuthoredRunBearer(db, companyId, "a.b.c"))
|
||||
.resolves.toBe("a.b.c");
|
||||
});
|
||||
|
||||
it("propagates a failed fingerprint lookup instead of returning unredacted text", async () => {
|
||||
const bearer = jwtShaped("write-fail-closed");
|
||||
const failingDbOrTx = {
|
||||
select: () => {
|
||||
throw new Error("simulated fingerprint lookup failure");
|
||||
},
|
||||
} as unknown as Db;
|
||||
|
||||
await expect(redactAuthoredRunBearer(failingDbOrTx, randomUUID(), `has ${bearer}`))
|
||||
.rejects.toThrow("simulated fingerprint lookup failure");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -136,6 +136,7 @@ import {
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { redactCurrentUserText } from "../log-redaction.js";
|
||||
import { redactSensitiveText } from "../redaction.js";
|
||||
import { redactAuthoredRunBearer } from "./run-secret-redaction.js";
|
||||
import {
|
||||
resolveIssueGoalId,
|
||||
resolveNextIssueGoalId,
|
||||
@@ -10064,6 +10065,13 @@ export function issueService(db: Db) {
|
||||
assigneeUserId: values.assigneeUserId ?? null,
|
||||
}),
|
||||
);
|
||||
// An agent-authored description can carry a run bearer. Redact it on
|
||||
// the transaction connection before the row is written. A failed
|
||||
// lookup propagates and aborts the create instead of persisting an
|
||||
// unredacted description.
|
||||
if (values.createdByAgentId && typeof values.description === "string") {
|
||||
values.description = await redactAuthoredRunBearer(tx, companyId, values.description);
|
||||
}
|
||||
|
||||
const [issue] = await tx.insert(issues).values(values).returning();
|
||||
if (idempotencyKey) {
|
||||
@@ -10828,6 +10836,13 @@ export function issueService(db: Db) {
|
||||
(blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) {
|
||||
patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number;
|
||||
}
|
||||
// An agent-authored description can carry a run bearer. Redact it on
|
||||
// `tx`, the connection this update already holds the row lock on,
|
||||
// before the row is written. A failed lookup propagates and aborts
|
||||
// the update instead of persisting an unredacted description.
|
||||
if (actorAgentId && typeof patch.description === "string") {
|
||||
patch.description = await redactAuthoredRunBearer(tx, existing.companyId, patch.description);
|
||||
}
|
||||
const updated = await tx
|
||||
.update(issues)
|
||||
.set(patch)
|
||||
@@ -12037,7 +12052,15 @@ export function issueService(db: Db) {
|
||||
enabled: (await instanceSettings.getGeneral({ db: dbOrTx }))
|
||||
.censorUsernameInLogs,
|
||||
};
|
||||
const redactedBody = redactCurrentUserText(body, currentUserRedactionOptions);
|
||||
const currentUserRedactedBody = redactCurrentUserText(body, currentUserRedactionOptions);
|
||||
// Only an agent-authored body can carry a run bearer, so the lookup
|
||||
// below runs only for that case. It reads on `dbOrTx`, the caller's
|
||||
// own connection, for the same reason as the setting read above. A
|
||||
// failed lookup propagates and aborts this write instead of
|
||||
// persisting an unredacted body.
|
||||
const redactedBody = actor.agentId
|
||||
? await redactAuthoredRunBearer(dbOrTx, issue.companyId, currentUserRedactedBody)
|
||||
: currentUserRedactedBody;
|
||||
if (actor.userId && options?.clientRequestId) {
|
||||
const [existing] = await dbOrTx.select().from(issueComments).where(and(eq(issueComments.issueId, issueId),
|
||||
eq(issueComments.authorUserId, actor.userId), eq(issueComments.clientRequestId, options.clientRequestId)));
|
||||
|
||||
@@ -6,6 +6,11 @@ import { REDACTED_EVENT_VALUE } from "../redaction.js";
|
||||
import { getSecretProvider } from "../secrets/provider-registry.js";
|
||||
import type { StoredSecretVersionMaterial } from "../secrets/types.js";
|
||||
|
||||
// Lets a caller pass either the pooled `Db` or an open transaction. A
|
||||
// write-time caller must read on its own transaction connection, not
|
||||
// re-enter the outer pool while that transaction still holds locks.
|
||||
type DbOrTx = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
|
||||
const REGISTRY_KEY = "paperclipSecretRedactions";
|
||||
// Project only the legacy registry: a run context can hold megabytes of
|
||||
// prompt data. This projection reads only the entries a run registered
|
||||
@@ -43,20 +48,45 @@ function redactText(input: string, values: string[]) {
|
||||
);
|
||||
}
|
||||
|
||||
// A JWT-shaped candidate: three non-empty base64url segments joined by two
|
||||
// dot characters. A registered run bearer that appears in text is the plain
|
||||
// text of that bearer, so this pass hashes each candidate it finds and looks
|
||||
// up the hash. It never decrypts stored material to find a match.
|
||||
const JWT_CANDIDATE_RE = /[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
|
||||
// A maximal chain of base64url segments joined by dots, at least three
|
||||
// segments long. A registered run bearer that appears in text is the plain
|
||||
// text of that bearer, so a later pass hashes each candidate this scanner
|
||||
// offers and looks up the hash. It never decrypts stored material to find a
|
||||
// match.
|
||||
const JWT_CHAIN_RE = /[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){2,}/g;
|
||||
|
||||
// A caller may supply a batch of many runs, or one run with a large payload.
|
||||
// This bounds each fingerprint lookup to a fixed number of query round trips
|
||||
// instead of one round trip per candidate.
|
||||
const FINGERPRINT_QUERY_BATCH_SIZE = 1000;
|
||||
|
||||
// Every window of three consecutive segments inside one maximal chain. A
|
||||
// dotted word right before a bearer (for example "paperclip.local.<bearer>")
|
||||
// joins into one longer chain, so the bearer's own three segments are not
|
||||
// always the chain's first three. Offering every window, not only the
|
||||
// first, closes that gap. No window is dropped here: a later pass bounds
|
||||
// its database lookups, but this candidate set itself stays unbounded.
|
||||
function jwtWindowsInChain(chain: string): string[] {
|
||||
const segments = chain.split(".");
|
||||
const windows: string[] = [];
|
||||
for (let start = 0; start + 3 <= segments.length; start += 1) {
|
||||
windows.push(segments.slice(start, start + 3).join("."));
|
||||
}
|
||||
return windows;
|
||||
}
|
||||
|
||||
// The one candidate scanner. Both the read mask below and the write-time
|
||||
// redactor in `redactAuthoredRunBearer` call this, so a shape fix here
|
||||
// applies to both paths at once.
|
||||
function collectJwtCandidateStrings(text: string, into: Set<string>): void {
|
||||
for (const chain of text.matchAll(JWT_CHAIN_RE)) {
|
||||
for (const candidate of jwtWindowsInChain(chain[0])) into.add(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
function collectJwtCandidates(value: unknown, into: Set<string>): void {
|
||||
if (typeof value === "string") {
|
||||
for (const match of value.matchAll(JWT_CANDIDATE_RE)) into.add(match[0]);
|
||||
collectJwtCandidateStrings(value, into);
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
@@ -72,6 +102,32 @@ function collectJwtCandidates(value: unknown, into: Set<string>): void {
|
||||
}
|
||||
}
|
||||
|
||||
// Replaces only the matched windows inside each maximal chain, scanning
|
||||
// left to right and consuming three segments at a time on a match. Every
|
||||
// other character of the chain, including a dotted prefix such as
|
||||
// "paperclip.local.", stays exactly as it was.
|
||||
function replaceJwtWindowsInText(text: string, matched: Set<string>): string {
|
||||
if (matched.size === 0) return text;
|
||||
return text.replace(JWT_CHAIN_RE, (chain) => {
|
||||
const segments = chain.split(".");
|
||||
const tokens: string[] = [];
|
||||
let index = 0;
|
||||
while (index < segments.length) {
|
||||
const window = index + 3 <= segments.length
|
||||
? segments.slice(index, index + 3).join(".")
|
||||
: null;
|
||||
if (window && matched.has(window)) {
|
||||
tokens.push(REDACTED_EVENT_VALUE);
|
||||
index += 3;
|
||||
} else {
|
||||
tokens.push(segments[index]!);
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
return tokens.join(".");
|
||||
});
|
||||
}
|
||||
|
||||
// Runs after the value pass has already rebuilt the object graph and
|
||||
// stripped the legacy registry key, so this pass does not need to repeat
|
||||
// that filter. It only replaces a candidate substring whose fingerprint
|
||||
@@ -79,10 +135,7 @@ function collectJwtCandidates(value: unknown, into: Set<string>): void {
|
||||
function replaceMatchedCandidates<T>(input: T, matched: Set<string>): T {
|
||||
if (matched.size === 0) return input;
|
||||
if (typeof input === "string") {
|
||||
return input.replace(
|
||||
JWT_CANDIDATE_RE,
|
||||
(candidate) => matched.has(candidate) ? REDACTED_EVENT_VALUE : candidate,
|
||||
) as T;
|
||||
return replaceJwtWindowsInText(input, matched) as T;
|
||||
}
|
||||
if (Array.isArray(input)) return input.map((item) => replaceMatchedCandidates(item, matched)) as T;
|
||||
if (input instanceof Date) return input;
|
||||
@@ -119,6 +172,94 @@ export function redactRegisteredSecretValues<T>(input: T, values: string[]): T {
|
||||
) as T;
|
||||
}
|
||||
|
||||
// Looks up a company-scoped set of digests against the registry, in query
|
||||
// batches bounded to `FINGERPRINT_QUERY_BATCH_SIZE`. The lookup filters on
|
||||
// `company_id` only, so a registered value matches wherever it appears in
|
||||
// that company's text. It never filters on `run_id`, and it never matches a
|
||||
// row of another company. Shared by the read mask inside
|
||||
// `createRunSecretRedactionRegistry` and by the write-time
|
||||
// `redactAuthoredRunBearer` below, so both read the registry the same way.
|
||||
async function matchedFingerprintsFor(
|
||||
dbOrTx: DbOrTx,
|
||||
companyId: string,
|
||||
digests: string[],
|
||||
): Promise<Set<string>> {
|
||||
const matched = new Set<string>();
|
||||
for (let start = 0; start < digests.length; start += FINGERPRINT_QUERY_BATCH_SIZE) {
|
||||
const batch = digests.slice(start, start + FINGERPRINT_QUERY_BATCH_SIZE);
|
||||
const rows = await dbOrTx.select({ fingerprintSha256: runSecretRedactions.fingerprintSha256 })
|
||||
.from(runSecretRedactions)
|
||||
.where(and(
|
||||
eq(runSecretRedactions.companyId, companyId),
|
||||
inArray(runSecretRedactions.fingerprintSha256, batch),
|
||||
));
|
||||
for (const row of rows) matched.add(row.fingerprintSha256);
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
// A base64url segment that decodes to a JSON object carrying an `alg`
|
||||
// member. A real JWT header always has this shape. An ordinary dotted
|
||||
// identifier, a file name, or a placeholder never decodes this way, so this
|
||||
// is the test that keeps the write-time fallback below narrow.
|
||||
function decodesAsJwtHeader(segment: string): boolean {
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = Buffer.from(segment, "base64url").toString("utf8");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(decoded);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
const header = asRecord(parsed);
|
||||
return header !== null && typeof header.alg === "string";
|
||||
}
|
||||
|
||||
function isJwtShapedCandidate(candidate: string): boolean {
|
||||
const header = candidate.split(".")[0];
|
||||
return header !== undefined && decodesAsJwtHeader(header);
|
||||
}
|
||||
|
||||
// Write-time redaction for an agent-authored comment body or issue
|
||||
// description, applied before the row is written. Two arms, in this order:
|
||||
//
|
||||
// 1. An exact registered-bearer match, using the same fingerprint lookup
|
||||
// the read mask uses. A digest match cannot be a false positive, so
|
||||
// this arm is safe to run on any text.
|
||||
// 2. A narrow JWT-shaped fallback for a candidate the registry does not
|
||||
// yet hold. It fires only when the candidate's first segment decodes
|
||||
// as a JSON header carrying an `alg` member, so it does not fire on an
|
||||
// unexpanded variable, a placeholder, or the redaction marker — none
|
||||
// of those decode this way.
|
||||
//
|
||||
// Reads on `dbOrTx`, the caller's own connection. A failed lookup
|
||||
// propagates instead of being swallowed, so the caller's write aborts
|
||||
// rather than persist unredacted text.
|
||||
export async function redactAuthoredRunBearer(
|
||||
dbOrTx: DbOrTx,
|
||||
companyId: string,
|
||||
text: string,
|
||||
): Promise<string> {
|
||||
const candidates = new Set<string>();
|
||||
collectJwtCandidateStrings(text, candidates);
|
||||
if (candidates.size === 0) return text;
|
||||
const digestByCandidate = new Map(
|
||||
[...candidates].map((candidate) => [candidate, createHash("sha256").update(candidate).digest("hex")] as const),
|
||||
);
|
||||
const digests = [...new Set(digestByCandidate.values())];
|
||||
const matchedDigests = await matchedFingerprintsFor(dbOrTx, companyId, digests);
|
||||
const matched = new Set<string>();
|
||||
for (const [candidate, digest] of digestByCandidate) {
|
||||
if (matchedDigests.has(digest) || isJwtShapedCandidate(candidate)) matched.add(candidate);
|
||||
}
|
||||
if (matched.size === 0) return text;
|
||||
return replaceJwtWindowsInText(text, matched);
|
||||
}
|
||||
|
||||
export function createRunSecretRedactionRegistry(db: Db) {
|
||||
const provider = getSecretProvider("local_encrypted");
|
||||
|
||||
@@ -196,26 +337,11 @@ export function createRunSecretRedactionRegistry(db: Db) {
|
||||
return resolveEntries(entries, new Map());
|
||||
}
|
||||
|
||||
// Looks up a company-scoped set of digests against the registry, in query
|
||||
// batches bounded to `FINGERPRINT_QUERY_BATCH_SIZE`. The lookup filters on
|
||||
// `company_id` only, so a registered value masks wherever it appears in
|
||||
// that company's text, not only in the run or issue text that registered
|
||||
// it. It never filters on `run_id`, and it never matches a row of another
|
||||
// company.
|
||||
async function matchedFingerprints(companyId: string, digests: string[]): Promise<Set<string>> {
|
||||
const matched = new Set<string>();
|
||||
for (let start = 0; start < digests.length; start += FINGERPRINT_QUERY_BATCH_SIZE) {
|
||||
const batch = digests.slice(start, start + FINGERPRINT_QUERY_BATCH_SIZE);
|
||||
const rows = await db.select({ fingerprintSha256: runSecretRedactions.fingerprintSha256 })
|
||||
.from(runSecretRedactions)
|
||||
.where(and(
|
||||
eq(runSecretRedactions.companyId, companyId),
|
||||
inArray(runSecretRedactions.fingerprintSha256, batch),
|
||||
));
|
||||
for (const row of rows) matched.add(row.fingerprintSha256);
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
// Delegates to the module-level `matchedFingerprintsFor`, bound to this
|
||||
// registry's own `db`, so this read mask and `redactAuthoredRunBearer`
|
||||
// share one query implementation.
|
||||
const matchedFingerprints = (companyId: string, digests: string[]) =>
|
||||
matchedFingerprintsFor(db, companyId, digests);
|
||||
|
||||
// Walks a value for JWT-shaped candidates, hashes each distinct candidate
|
||||
// once, and masks only the candidates whose hash the registry holds. A
|
||||
|
||||
Reference in new issue
Block a user