mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-01 23:43:16 +01:00
feat(db): add a company-scoped run secret redaction table
Add `run_secret_redactions`: a new table with an index on (company_id, fingerprint_sha256, run_id), an index on (company_id, run_id), and a partial index on expires_at for a bounded expiry sweep. The table has a foreign key to heartbeat_runs with a cascade delete, so a fingerprint-only row cannot outlive its run row. Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
74b3e024eb
commit
fac9c37f64
6 files changed
+48061
No files matched your search
@@ -0,0 +1,15 @@
|
||||
CREATE TABLE "run_secret_redactions" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"company_id" uuid NOT NULL,
|
||||
"run_id" uuid NOT NULL,
|
||||
"fingerprint_sha256" text NOT NULL,
|
||||
"material" jsonb,
|
||||
"expires_at" timestamp with time zone NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "run_secret_redactions" ADD CONSTRAINT "run_secret_redactions_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "run_secret_redactions" ADD CONSTRAINT "run_secret_redactions_run_id_heartbeat_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."heartbeat_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "run_secret_redactions_company_fingerprint_run_uq" ON "run_secret_redactions" USING btree ("company_id","fingerprint_sha256","run_id");--> statement-breakpoint
|
||||
CREATE INDEX "run_secret_redactions_company_run_idx" ON "run_secret_redactions" USING btree ("company_id","run_id");--> statement-breakpoint
|
||||
CREATE INDEX "run_secret_redactions_sweep_idx" ON "run_secret_redactions" USING btree ("expires_at") WHERE "run_secret_redactions"."material" is not null;
|
||||
+47951
File diff suppressed because it is too large.
Load diff
+7
@@ -1947,6 +1947,13 @@
|
||||
"when": 1789390281190,
|
||||
"tag": "0279_tired_deathstrike",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 280,
|
||||
"version": "7",
|
||||
"when": 1789542093896,
|
||||
"tag": "0280_overjoyed_bushwacker",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { describe, expect, it, afterEach } from "vitest";
|
||||
import postgres from "postgres";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./test-embedded-postgres.js";
|
||||
|
||||
const cleanups: Array<() => Promise<void>> = [];
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const d = support.supported ? describe : describe.skip;
|
||||
|
||||
afterEach(async () => {
|
||||
while (cleanups.length > 0) await cleanups.pop()?.();
|
||||
});
|
||||
|
||||
d("run_secret_redactions migration", () => {
|
||||
it("creates the table and its indexes, and the fingerprint lookup uses the index", async () => {
|
||||
const dbh = await startEmbeddedPostgresTestDatabase("pap6522-run-secret-redactions-");
|
||||
cleanups.push(() => dbh.cleanup());
|
||||
const sql = postgres(dbh.connectionString, { max: 1 });
|
||||
cleanups.push(async () => { await sql.end(); });
|
||||
|
||||
const idx = await sql`SELECT indexname FROM pg_indexes WHERE tablename = 'run_secret_redactions'`;
|
||||
const names = idx.map((r) => r.indexname as string);
|
||||
expect(names).toContain("run_secret_redactions_company_fingerprint_run_uq");
|
||||
expect(names).toContain("run_secret_redactions_company_run_idx");
|
||||
expect(names).toContain("run_secret_redactions_sweep_idx");
|
||||
|
||||
await sql.unsafe("SET enable_seqscan = off");
|
||||
// Companies id 1 is never inserted; EXPLAIN plans the query without
|
||||
// executing it, so the query needs no rows to prove the planner's choice.
|
||||
const plan = await sql.unsafe(
|
||||
"EXPLAIN SELECT id FROM run_secret_redactions WHERE company_id = '00000000-0000-0000-0000-000000000001' AND fingerprint_sha256 = 'x'",
|
||||
);
|
||||
const planText = plan.map((r) => Object.values(r)[0]).join("\n");
|
||||
expect(planText).toContain("run_secret_redactions_company_fingerprint_run_uq");
|
||||
|
||||
const runLookupPlan = await sql.unsafe(
|
||||
"EXPLAIN SELECT id FROM run_secret_redactions WHERE company_id = '00000000-0000-0000-0000-000000000001' AND run_id = '00000000-0000-0000-0000-000000000002'",
|
||||
);
|
||||
const runLookupText = runLookupPlan.map((r) => Object.values(r)[0]).join("\n");
|
||||
expect(runLookupText).toContain("run_secret_redactions_company_run_idx");
|
||||
}, 240_000);
|
||||
});
|
||||
@@ -126,6 +126,7 @@ export { documentAnnotationThreads } from "./document_annotation_threads.js";
|
||||
export { documentAnnotationComments } from "./document_annotation_comments.js";
|
||||
export { documentAnnotationAnchorSnapshots } from "./document_annotation_anchor_snapshots.js";
|
||||
export { heartbeatRuns } from "./heartbeat_runs.js";
|
||||
export { runSecretRedactions } from "./run_secret_redactions.js";
|
||||
export { heartbeatRunEvents } from "./heartbeat_run_events.js";
|
||||
export { providerTraceRecords } from "./provider_trace_records.js";
|
||||
export { completionContracts } from "./completion_contracts.js";
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { pgTable, uuid, text, timestamp, jsonb, index, uniqueIndex } from "drizzle-orm/pg-core";
|
||||
import { companies } from "./companies.js";
|
||||
import { heartbeatRuns } from "./heartbeat_runs.js";
|
||||
|
||||
// A company-scoped registry of secret text a run may leak into plain text (a
|
||||
// run bearer token, a proposed secret value). A later read of run or issue
|
||||
// text uses this registry to mask any value it finds.
|
||||
//
|
||||
// `fingerprintSha256` is a one-way hash of the registered value. A periodic
|
||||
// sweep clears `material` at or after `expiresAt` and keeps the row and its
|
||||
// fingerprint, so the row still blocks a repeat registration of the same
|
||||
// value. The fingerprint never appears in an API response or a log line.
|
||||
export const runSecretRedactions = pgTable(
|
||||
"run_secret_redactions",
|
||||
{
|
||||
id: uuid("id").primaryKey().defaultRandom(),
|
||||
companyId: uuid("company_id").notNull().references(() => companies.id),
|
||||
// The foreign key cascades on delete. This bounds the life of a
|
||||
// fingerprint-only row to the life of its run row, so the table needs no
|
||||
// separate retention policy.
|
||||
runId: uuid("run_id").notNull().references(() => heartbeatRuns.id, { onDelete: "cascade" }),
|
||||
fingerprintSha256: text("fingerprint_sha256").notNull(),
|
||||
material: jsonb("material").$type<Record<string, unknown>>(),
|
||||
expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(table) => ({
|
||||
// This index gives register() its idempotency key: one run cannot
|
||||
// register the same value twice. Its first two columns also give a
|
||||
// fingerprint lookup the (company_id, fingerprint_sha256) shape it needs.
|
||||
// The run_id column lets two different runs register the identical value
|
||||
// without a conflict, so each run keeps its own row to mask its own text.
|
||||
companyFingerprintRunUq: uniqueIndex("run_secret_redactions_company_fingerprint_run_uq")
|
||||
.on(table.companyId, table.fingerprintSha256, table.runId),
|
||||
companyRunIdx: index("run_secret_redactions_company_run_idx").on(table.companyId, table.runId),
|
||||
// Supports the bounded, ordered expiry sweep: only a row that still holds
|
||||
// material and has passed its expiry is a sweep candidate.
|
||||
sweepIdx: index("run_secret_redactions_sweep_idx")
|
||||
.on(table.expiresAt)
|
||||
.where(sql`${table.material} is not null`),
|
||||
}),
|
||||
);
|
||||
Reference in new issue
Block a user