diff --git a/.github/workflows/backup-checkpoint-qualification.yml b/.github/workflows/backup-checkpoint-qualification.yml index 8da917ce4dc..3b90802cc13 100644 --- a/.github/workflows/backup-checkpoint-qualification.yml +++ b/.github/workflows/backup-checkpoint-qualification.yml @@ -4,6 +4,7 @@ on: paths: - 'packages/db/**' - 'server/src/services/*backup*' + - 'server/src/lib/dedicated-test-database-url.ts' - 'server/src/__tests__/*backup*' - 'server/src/__tests__/fixtures/database-backup-worker.ts' - 'server/src/index.ts' diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index ef45a370ed9..31042f687c6 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -362,9 +362,12 @@ pnpm --filter @paperclipai/server exec tsx scripts/qualify-idle-database-backup. Use a fresh source/restore pair and output directory for a second run with `javascript` as the last argument. This exercises the supported fallback engine separately. The command verifies the full archive, syncs its bytes and directory, -restores it, checks relational and binary (including empty bytes)/Unicode/null values, enforces the -restored foreign key, and checks identity sequence progress. It writes a JSON +restores it, checks relational, Unicode and null values, and verifies binary +scalars and one- and two-dimensional arrays, including empty bytes, empty arrays +and null elements, with array dimensions and bounds preserved. It enforces the +restored foreign key and checks identity sequence progress. It writes a JSON receipt with binary versions, database names, archive size/hash, and results. +Archive hashing reads a bounded stream rather than loading the archive into memory. It does not write credentials to the receipt. The fixture rows remain available for inspection. Dispose of these owned databases through the normal test-owner workflow after closing their clients. diff --git a/packages/db/src/backup-lib-binary.test.ts b/packages/db/src/backup-lib-binary.test.ts new file mode 100644 index 00000000000..f6e303059c0 --- /dev/null +++ b/packages/db/src/backup-lib-binary.test.ts @@ -0,0 +1,67 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import { describe, expect, it, vi } from "vitest"; +import { runDatabaseBackup } from "./backup-lib.js"; + +const transport = vi.hoisted(() => ({ rows: [] as unknown[][], projection: "", end: vi.fn() })); +vi.mock("postgres", () => ({ + default: () => { + const columns = [ + { column_name: "bytes", data_type: "bytea", udt_name: "bytea", udt_schema: "pg_catalog" }, + { column_name: "payloads", data_type: "ARRAY", udt_name: "_bytea", udt_schema: "pg_catalog" }, + { column_name: "metadata", data_type: "jsonb", udt_name: "jsonb", udt_schema: "pg_catalog" }, + { column_name: "redacted", data_type: "ARRAY", udt_name: "_bytea", udt_schema: "pg_catalog" }, + ]; + const sql = async (parts: TemplateStringsArray) => { + const query = parts.join("?"); + if (query.includes("FROM information_schema.tables")) return [{ schema_name: "public", tablename: "binary_fixture" }]; + if (query.includes("FROM information_schema.columns")) return columns.map(column => ({ ...column, udt_schema: "pg_catalog", is_nullable: "YES", is_identity: "NO", column_default: null })); + return []; + }; + return Object.assign(sql, { + end: transport.end, + unsafe: (query: string) => query.startsWith("SELECT count(*)") ? Promise.resolve([{ n: transport.rows.length }]) + : query.startsWith("SELECT ") && query.includes('FROM "public"."binary_fixture"') + ? { values: () => ({ cursor: async function* () { transport.projection = query; yield transport.rows; } }) } + : query, + }); + }, +})); + +describe("JavaScript backup binary SQL", () => { + it("writes exact binary scalars and nested arrays without changing JSON or redaction", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "backup-binary-sql-")); + const binary = Buffer.from([0, 255, ...Buffer.from("雪")]); + const metadata = { bytes: binary, values: [null, "雪"] }; + const hex = binary.toString("hex"); + const payloads = `{"\\\\x${hex}","\\\\x",NULL}`; + const boundedPayloads = `[0:1][3:4]={{"\\\\x${hex}",NULL},{"\\\\x","\\\\x00ff"}}`; + transport.rows = [ + [binary, payloads, metadata, payloads], + [new Uint8Array(), "{}", [], payloads], + [null, null, null, payloads], + [new Uint8Array([0, 255]), boundedPayloads, {}, boundedPayloads], + ]; + transport.end.mockClear(); + try { + const archive = await runDatabaseBackup({ connectionString: "postgres://unused/binary_test", backupDir: directory, + backupEngine: "javascript", retention: { dailyDays: 1, weeklyWeeks: 1, monthlyMonths: 1 }, + nullifyColumns: { binary_fixture: ["redacted"] } }); + const body = gunzipSync(await readFile(archive.backupFile)).toString("utf8"); + const inserts = body.split("\n").filter(line => line.startsWith("INSERT INTO")); + const literal = (value: string) => `$paperclip$${value}$paperclip$`; + const array = (value: string) => literal(value); + const prefix = 'INSERT INTO "public"."binary_fixture" ("bytes", "payloads", "metadata", "redacted") VALUES '; + expect(inserts).toEqual([ + `${prefix}(decode('${hex}', 'hex'), ${array(`{"\\\\x${hex}","\\\\x",NULL}`)}, ${literal(JSON.stringify(metadata))}, NULL);`, + `${prefix}(decode('', 'hex'), ${array("{}")}, ${literal("[]")}, NULL);`, + `${prefix}(NULL, NULL, NULL, NULL);`, + `${prefix}(decode('00ff', 'hex'), ${array(boundedPayloads)}, ${literal("{}")}, NULL);`, + ]); + expect(transport.projection).toBe('SELECT "bytes", "payloads"::text AS "payloads", "metadata", "redacted"::text AS "redacted" FROM "public"."binary_fixture"'); + expect(transport.end).toHaveBeenCalledTimes(1); + } finally { await rm(directory, { recursive: true, force: true }); } + }); +}); diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index 00b03218242..06b54478ab4 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -965,8 +965,8 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise if (excludedTableNames.has(currentTableKey) || (count[0]?.n ?? 0) === 0) continue; // Get column info for this table - const cols = await sql<{ column_name: string; data_type: string }[]>` - SELECT column_name, data_type + const cols = await sql<{ column_name: string; data_type: string; udt_schema: string; udt_name: string }[]>` + SELECT column_name, data_type, udt_schema, udt_name FROM information_schema.columns WHERE table_schema = ${schema_name} AND table_name = ${tablename} ORDER BY ordinal_position @@ -996,8 +996,13 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise continue; } + const binaryArray = (column: (typeof cols)[number]) => column.data_type === "ARRAY" + && column.udt_schema === "pg_catalog" && column.udt_name === "_bytea"; + const projection = cols.some(binaryArray) ? cols.map(column => binaryArray(column) + ? `${quoteIdentifier(column.column_name)}::text AS ${quoteIdentifier(column.column_name)}` + : quoteIdentifier(column.column_name)).join(", ") : "*"; const rowCursor = sql - .unsafe(`SELECT * FROM ${qualifiedTableName}`) + .unsafe(`SELECT ${projection} FROM ${qualifiedTableName}`) .values() .cursor(BACKUP_DATA_CURSOR_ROWS) as AsyncIterable; for await (const rows of rowCursor) { diff --git a/server/scripts/lib/backup-qualification.ts b/server/scripts/lib/backup-qualification.ts index e60cd0a4421..ad12b00091c 100644 --- a/server/scripts/lib/backup-qualification.ts +++ b/server/scripts/lib/backup-qualification.ts @@ -1,7 +1,7 @@ import { randomBytes, createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; +import { createReadStream } from "node:fs"; import { createDb, runDatabaseBackup, runDatabaseRestore, type RunDatabaseBackupOptions } from "@paperclipai/db"; -import { resolveExternalTestDatabaseUrl } from "../../src/__tests__/helpers/external-test-database.js"; +import { resolveExternalTestDatabaseUrl } from "../../src/lib/dedicated-test-database-url.js"; import { verifyIdleDatabaseBackup } from "../../src/services/idle-database-backup.js"; export const SOURCE_ENV = "PAPERCLIP_BACKUP_CHECKPOINT_SOURCE_TEST_DATABASE_URL"; @@ -52,9 +52,15 @@ export async function seedBackupQualificationFixture(sourceUrl: string) { await source`CREATE TABLE backup_qualification.parents (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, value text NOT NULL)`; await source`CREATE TABLE backup_qualification.children (id integer PRIMARY KEY, parent_id bigint REFERENCES backup_qualification.parents(id), value text, bytes bytea)`; + await source`CREATE TABLE backup_qualification.binary_arrays (id integer PRIMARY KEY, payloads bytea[])`; await source.begin(async tx => { await tx`INSERT INTO backup_qualification.parents (value) VALUES ('last committed state — 雪 ☃')`; await tx`INSERT INTO backup_qualification.children VALUES (1, 1, NULL, decode('00ff10275c', 'hex')), (2, 1, 'empty bytes', decode('', 'hex')), (3, 1, 'null bytes', NULL)`; + await tx`INSERT INTO backup_qualification.binary_arrays VALUES + (1, ARRAY[decode('00ff10275c', 'hex'), convert_to('雪 ☃', 'UTF8'), decode('', 'hex'), NULL]::bytea[]), + (2, ARRAY[]::bytea[]), (3, NULL), + (4, ARRAY[[decode('00ff', 'hex'), NULL], [decode('', 'hex'), convert_to('雪', 'UTF8')]]::bytea[]), + (5, array_fill(decode('00ff', 'hex'), ARRAY[2,2], ARRAY[0,3]))`; }); } finally { await source.end({ timeout: 5 }); } } @@ -71,13 +77,29 @@ export async function inspectRestoredFixture(restoreUrl: string) { if (empty?.size !== 0) throw new Error("Restored empty binary value differs"); const [nullBytes] = await target`SELECT bytes IS NULL AS is_null FROM backup_qualification.children WHERE id = 3`; if (nullBytes?.is_null !== true) throw new Error("Restored null binary value differs"); + const arrays = await target`SELECT id, pg_typeof(payloads)::text AS type, array_dims(payloads) AS dimensions, + CASE id + WHEN 1 THEN payloads IS NOT DISTINCT FROM ARRAY[decode('00ff10275c', 'hex'), convert_to('雪 ☃', 'UTF8'), decode('', 'hex'), NULL]::bytea[] + WHEN 2 THEN payloads IS NOT DISTINCT FROM ARRAY[]::bytea[] + WHEN 3 THEN payloads IS NULL + WHEN 4 THEN payloads IS NOT DISTINCT FROM ARRAY[[decode('00ff', 'hex'), NULL], [decode('', 'hex'), convert_to('雪', 'UTF8')]]::bytea[] + WHEN 5 THEN payloads IS NOT DISTINCT FROM array_fill(decode('00ff', 'hex'), ARRAY[2,2], ARRAY[0,3]) + END AS matches FROM backup_qualification.binary_arrays ORDER BY id`; + const expected = [ + { id: 1, type: "bytea[]", dimensions: "[1:4]", matches: true }, + { id: 2, type: "bytea[]", dimensions: null, matches: true }, + { id: 3, type: "bytea[]", dimensions: null, matches: true }, + { id: 4, type: "bytea[]", dimensions: "[1:2][1:2]", matches: true }, + { id: 5, type: "bytea[]", dimensions: "[0:1][3:4]", matches: true }, + ]; + if (JSON.stringify(arrays) !== JSON.stringify(expected)) throw new Error("Restored binary array values, dimensions or types differ"); const [next] = await target`INSERT INTO backup_qualification.parents (value) VALUES ('sequence after restore') RETURNING id::text`; if (next?.id !== "2") throw new Error("Restored identity sequence did not resume at 2"); try { await target`INSERT INTO backup_qualification.children VALUES (999, 999, NULL, decode('', 'hex'))`; throw new Error("Restored foreign key was not enforced"); } catch (error) { if ((error as { code?: string }).code !== "23503") throw error; } - return { relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, nullable: true, nextIdentity: next.id, foreignKeyEnforced: true }; + return { relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, binaryArrays: true, nestedBinaryArrays: true, binaryArrayBounds: true, nullable: true, nextIdentity: next.id, foreignKeyEnforced: true }; } finally { await target.end({ timeout: 5 }); } } @@ -87,7 +109,9 @@ export async function qualifyBackupRestore(pair: QualificationPair, backupDir: s backupEngine: engine, verifyBeforePrune: verifyIdleDatabaseBackup }); await assertDistinctEmptyRestore(pair); await runDatabaseRestore({ connectionString: pair.restoreUrl, backupFile: archive.backupFile }); + const hash = createHash("sha256"); + for await (const chunk of createReadStream(archive.backupFile)) hash.update(chunk); return { engine, archive: archive.backupFile, sizeBytes: archive.sizeBytes, - sha256: createHash("sha256").update(await readFile(archive.backupFile)).digest("hex"), + sha256: hash.digest("hex"), restored: await inspectRestoredFixture(pair.restoreUrl) }; } diff --git a/server/src/__tests__/database-backup-checkpoint-qualification.integration.ts b/server/src/__tests__/database-backup-checkpoint-qualification.integration.ts index 951150fb297..bd589548d65 100644 --- a/server/src/__tests__/database-backup-checkpoint-qualification.integration.ts +++ b/server/src/__tests__/database-backup-checkpoint-qualification.integration.ts @@ -32,6 +32,18 @@ describe("real automatic backup checkpoint qualification", () => { await assertDistinctEmptyRestore(pair); ownsRestoreFixture = true; await seedBackupQualificationFixture(pair.sourceUrl); + const [column] = await source.$client`SELECT data_type, udt_name FROM information_schema.columns + WHERE table_schema = 'backup_qualification' AND table_name = 'binary_arrays' AND column_name = 'payloads'`; + expect(column).toEqual({ data_type: "ARRAY", udt_name: "_bytea" }); + const [driver] = await source.$client`SELECT payloads FROM backup_qualification.binary_arrays WHERE id = 4`; + expect(Array.isArray(driver!.payloads[0])).toBe(true); + expect(Buffer.isBuffer(driver!.payloads[0][0])).toBe(true); + const [sqlNull] = await source.$client`SELECT payloads[1][2] IS NULL AS is_null + FROM backup_qualification.binary_arrays WHERE id = 4`; + expect(sqlNull?.is_null).toBe(true); + receipts.push({ case: "binary-array-driver-representation", column, nestedBuffers: true, + sqlNullElement: true, parsedNullElement: driver!.payloads[0][1] === null ? "null" + : Buffer.isBuffer(driver!.payloads[0][1]) ? "Buffer" : typeof driver!.payloads[0][1] }); }, 120_000); afterAll(async () => { try { @@ -77,7 +89,7 @@ describe("real automatic backup checkpoint qualification", () => { const versions = Object.fromEntries([process.env.PAPERCLIP_PG_DUMP_PATH || "pg_dump", process.env.PAPERCLIP_PSQL_PATH || "psql"] .map(binary => [binary, execFileSync(binary, ["--version"], { encoding: "utf8" }).trim()])); const result = await qualifyBackupRestore(pair, directory, engine); - expect(result.restored).toEqual({ relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, nullable: true, nextIdentity: "2", foreignKeyEnforced: true }); + expect(result.restored).toEqual({ relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, binaryArrays: true, nestedBinaryArrays: true, binaryArrayBounds: true, nullable: true, nextIdentity: "2", foreignKeyEnforced: true }); expect(gunzipSync(await readFile(result.archive)).length).toBeGreaterThan(0); receipts.push({ ...result, versions }); }, 120_000); diff --git a/server/src/__tests__/helpers/external-test-database.ts b/server/src/__tests__/helpers/external-test-database.ts index 0d622f930c9..de177cb097b 100644 --- a/server/src/__tests__/helpers/external-test-database.ts +++ b/server/src/__tests__/helpers/external-test-database.ts @@ -1,11 +1 @@ -export function resolveExternalTestDatabaseUrl(name: string, value = process.env[name]): string | undefined { - if (!value?.trim()) return undefined; - try { - const url = new URL(value); - const database = decodeURIComponent(url.pathname.slice(1)); - if (['postgres:', 'postgresql:'].includes(url.protocol) - && /^[a-zA-Z0-9_]+_test$/.test(database) - && !['database', 'dbname', 'options'].some(key => url.searchParams.has(key))) return value; - } catch {} - throw new Error(`${name} must name a dedicated PostgreSQL database ending in _test, without database or options overrides`); -} +export { resolveExternalTestDatabaseUrl } from "../../lib/dedicated-test-database-url.js"; diff --git a/server/src/lib/dedicated-test-database-url.ts b/server/src/lib/dedicated-test-database-url.ts new file mode 100644 index 00000000000..0d622f930c9 --- /dev/null +++ b/server/src/lib/dedicated-test-database-url.ts @@ -0,0 +1,11 @@ +export function resolveExternalTestDatabaseUrl(name: string, value = process.env[name]): string | undefined { + if (!value?.trim()) return undefined; + try { + const url = new URL(value); + const database = decodeURIComponent(url.pathname.slice(1)); + if (['postgres:', 'postgresql:'].includes(url.protocol) + && /^[a-zA-Z0-9_]+_test$/.test(database) + && !['database', 'dbname', 'options'].some(key => url.searchParams.has(key))) return value; + } catch {} + throw new Error(`${name} must name a dedicated PostgreSQL database ending in _test, without database or options overrides`); +}