fix(db): preserve binary array values through backups

Read bytea arrays as PostgreSQL text before driver parsing so null elements, dimensions, and bounds survive JavaScript restores. Gate binary SQL in the default suite and verify both real engines and automatic fallback. Stream qualification archive hashes and share the dedicated database URL guard.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/backup-checkpoint-qualification.yml

@@ -4,6 +4,7 @@ on:
     paths:
       - 'packages/db/**'
       - 'server/src/services/*backup*'
+      - 'server/src/lib/dedicated-test-database-url.ts'
       - 'server/src/__tests__/*backup*'
       - 'server/src/__tests__/fixtures/database-backup-worker.ts'
       - 'server/src/index.ts'

doc/idle-sleep-safety.md

@@ -362,9 +362,12 @@ pnpm --filter @paperclipai/server exec tsx scripts/qualify-idle-database-backup.
 Use a fresh source/restore pair and output directory for a second run with
 `javascript` as the last argument. This exercises the supported fallback engine
 separately. The command verifies the full archive, syncs its bytes and directory,
-restores it, checks relational and binary (including empty bytes)/Unicode/null values, enforces the
-restored foreign key, and checks identity sequence progress. It writes a JSON
+restores it, checks relational, Unicode and null values, and verifies binary
+scalars and one- and two-dimensional arrays, including empty bytes, empty arrays
+and null elements, with array dimensions and bounds preserved. It enforces the
+restored foreign key and checks identity sequence progress. It writes a JSON
 receipt with binary versions, database names, archive size/hash, and results.
+Archive hashing reads a bounded stream rather than loading the archive into memory.
 It does not write credentials to the receipt. The fixture rows remain available
 for inspection. Dispose of these owned databases through the normal test-owner
 workflow after closing their clients.

packages/db/src/backup-lib-binary.test.ts

@@ -0,0 +1,67 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { gunzipSync } from "node:zlib";
+import { describe, expect, it, vi } from "vitest";
+import { runDatabaseBackup } from "./backup-lib.js";
+
+const transport = vi.hoisted(() => ({ rows: [] as unknown[][], projection: "", end: vi.fn() }));
+vi.mock("postgres", () => ({
+  default: () => {
+    const columns = [
+      { column_name: "bytes", data_type: "bytea", udt_name: "bytea", udt_schema: "pg_catalog" },
+      { column_name: "payloads", data_type: "ARRAY", udt_name: "_bytea", udt_schema: "pg_catalog" },
+      { column_name: "metadata", data_type: "jsonb", udt_name: "jsonb", udt_schema: "pg_catalog" },
+      { column_name: "redacted", data_type: "ARRAY", udt_name: "_bytea", udt_schema: "pg_catalog" },
+    ];
+    const sql = async (parts: TemplateStringsArray) => {
+      const query = parts.join("?");
+      if (query.includes("FROM information_schema.tables")) return [{ schema_name: "public", tablename: "binary_fixture" }];
+      if (query.includes("FROM information_schema.columns")) return columns.map(column => ({ ...column, udt_schema: "pg_catalog", is_nullable: "YES", is_identity: "NO", column_default: null }));
+      return [];
+    };
+    return Object.assign(sql, {
+      end: transport.end,
+      unsafe: (query: string) => query.startsWith("SELECT count(*)") ? Promise.resolve([{ n: transport.rows.length }])
+        : query.startsWith("SELECT ") && query.includes('FROM "public"."binary_fixture"')
+          ? { values: () => ({ cursor: async function* () { transport.projection = query; yield transport.rows; } }) }
+          : query,
+    });
+  },
+}));
+
+describe("JavaScript backup binary SQL", () => {
+  it("writes exact binary scalars and nested arrays without changing JSON or redaction", async () => {
+    const directory = await mkdtemp(path.join(os.tmpdir(), "backup-binary-sql-"));
+    const binary = Buffer.from([0, 255, ...Buffer.from("雪")]);
+    const metadata = { bytes: binary, values: [null, "雪"] };
+    const hex = binary.toString("hex");
+    const payloads = `{"\\\\x${hex}","\\\\x",NULL}`;
+    const boundedPayloads = `[0:1][3:4]={{"\\\\x${hex}",NULL},{"\\\\x","\\\\x00ff"}}`;
+    transport.rows = [
+      [binary, payloads, metadata, payloads],
+      [new Uint8Array(), "{}", [], payloads],
+      [null, null, null, payloads],
+      [new Uint8Array([0, 255]), boundedPayloads, {}, boundedPayloads],
+    ];
+    transport.end.mockClear();
+    try {
+      const archive = await runDatabaseBackup({ connectionString: "postgres://unused/binary_test", backupDir: directory,
+        backupEngine: "javascript", retention: { dailyDays: 1, weeklyWeeks: 1, monthlyMonths: 1 },
+        nullifyColumns: { binary_fixture: ["redacted"] } });
+      const body = gunzipSync(await readFile(archive.backupFile)).toString("utf8");
+      const inserts = body.split("\n").filter(line => line.startsWith("INSERT INTO"));
+      const literal = (value: string) => `$paperclip$${value}$paperclip$`;
+      const array = (value: string) => literal(value);
+      const prefix = 'INSERT INTO "public"."binary_fixture" ("bytes", "payloads", "metadata", "redacted") VALUES ';
+      expect(inserts).toEqual([
+        `${prefix}(decode('${hex}', 'hex'), ${array(`{"\\\\x${hex}","\\\\x",NULL}`)}, ${literal(JSON.stringify(metadata))}, NULL);`,
+        `${prefix}(decode('', 'hex'), ${array("{}")}, ${literal("[]")}, NULL);`,
+        `${prefix}(NULL, NULL, NULL, NULL);`,
+        `${prefix}(decode('00ff', 'hex'), ${array(boundedPayloads)}, ${literal("{}")}, NULL);`,
+      ]);
+      expect(transport.projection).toBe('SELECT "bytes", "payloads"::text AS "payloads", "metadata", "redacted"::text AS "redacted" FROM "public"."binary_fixture"');
+      expect(transport.end).toHaveBeenCalledTimes(1);
+    } finally { await rm(directory, { recursive: true, force: true }); }
+  });
+});

packages/db/src/backup-lib.ts

@@ -965,8 +965,8 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
       if (excludedTableNames.has(currentTableKey) || (count[0]?.n ?? 0) === 0) continue;
 
       // Get column info for this table
-      const cols = await sql<{ column_name: string; data_type: string }[]>`
-        SELECT column_name, data_type
+      const cols = await sql<{ column_name: string; data_type: string; udt_schema: string; udt_name: string }[]>`
+        SELECT column_name, data_type, udt_schema, udt_name
         FROM information_schema.columns
         WHERE table_schema = ${schema_name} AND table_name = ${tablename}
         ORDER BY ordinal_position
@@ -996,8 +996,13 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
         continue;
       }
 
+      const binaryArray = (column: (typeof cols)[number]) => column.data_type === "ARRAY"
+        && column.udt_schema === "pg_catalog" && column.udt_name === "_bytea";
+      const projection = cols.some(binaryArray) ? cols.map(column => binaryArray(column)
+        ? `${quoteIdentifier(column.column_name)}::text AS ${quoteIdentifier(column.column_name)}`
+        : quoteIdentifier(column.column_name)).join(", ") : "*";
       const rowCursor = sql
-        .unsafe(`SELECT * FROM ${qualifiedTableName}`)
+        .unsafe(`SELECT ${projection} FROM ${qualifiedTableName}`)
         .values()
         .cursor(BACKUP_DATA_CURSOR_ROWS) as AsyncIterable<unknown[][]>;
       for await (const rows of rowCursor) {

server/scripts/lib/backup-qualification.ts

@@ -1,7 +1,7 @@
 import { randomBytes, createHash } from "node:crypto";
-import { readFile } from "node:fs/promises";
+import { createReadStream } from "node:fs";
 import { createDb, runDatabaseBackup, runDatabaseRestore, type RunDatabaseBackupOptions } from "@paperclipai/db";
-import { resolveExternalTestDatabaseUrl } from "../../src/__tests__/helpers/external-test-database.js";
+import { resolveExternalTestDatabaseUrl } from "../../src/lib/dedicated-test-database-url.js";
 import { verifyIdleDatabaseBackup } from "../../src/services/idle-database-backup.js";
 
 export const SOURCE_ENV = "PAPERCLIP_BACKUP_CHECKPOINT_SOURCE_TEST_DATABASE_URL";
@@ -52,9 +52,15 @@ export async function seedBackupQualificationFixture(sourceUrl: string) {
     await source`CREATE TABLE backup_qualification.parents (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, value text NOT NULL)`;
     await source`CREATE TABLE backup_qualification.children (id integer PRIMARY KEY, parent_id bigint REFERENCES backup_qualification.parents(id),
       value text, bytes bytea)`;
+    await source`CREATE TABLE backup_qualification.binary_arrays (id integer PRIMARY KEY, payloads bytea[])`;
     await source.begin(async tx => {
       await tx`INSERT INTO backup_qualification.parents (value) VALUES ('last committed state — 雪 ☃')`;
       await tx`INSERT INTO backup_qualification.children VALUES (1, 1, NULL, decode('00ff10275c', 'hex')), (2, 1, 'empty bytes', decode('', 'hex')), (3, 1, 'null bytes', NULL)`;
+      await tx`INSERT INTO backup_qualification.binary_arrays VALUES
+        (1, ARRAY[decode('00ff10275c', 'hex'), convert_to('雪 ☃', 'UTF8'), decode('', 'hex'), NULL]::bytea[]),
+        (2, ARRAY[]::bytea[]), (3, NULL),
+        (4, ARRAY[[decode('00ff', 'hex'), NULL], [decode('', 'hex'), convert_to('雪', 'UTF8')]]::bytea[]),
+        (5, array_fill(decode('00ff', 'hex'), ARRAY[2,2], ARRAY[0,3]))`;
     });
   } finally { await source.end({ timeout: 5 }); }
 }
@@ -71,13 +77,29 @@ export async function inspectRestoredFixture(restoreUrl: string) {
     if (empty?.size !== 0) throw new Error("Restored empty binary value differs");
     const [nullBytes] = await target`SELECT bytes IS NULL AS is_null FROM backup_qualification.children WHERE id = 3`;
     if (nullBytes?.is_null !== true) throw new Error("Restored null binary value differs");
+    const arrays = await target`SELECT id, pg_typeof(payloads)::text AS type, array_dims(payloads) AS dimensions,
+      CASE id
+        WHEN 1 THEN payloads IS NOT DISTINCT FROM ARRAY[decode('00ff10275c', 'hex'), convert_to('雪 ☃', 'UTF8'), decode('', 'hex'), NULL]::bytea[]
+        WHEN 2 THEN payloads IS NOT DISTINCT FROM ARRAY[]::bytea[]
+        WHEN 3 THEN payloads IS NULL
+        WHEN 4 THEN payloads IS NOT DISTINCT FROM ARRAY[[decode('00ff', 'hex'), NULL], [decode('', 'hex'), convert_to('雪', 'UTF8')]]::bytea[]
+        WHEN 5 THEN payloads IS NOT DISTINCT FROM array_fill(decode('00ff', 'hex'), ARRAY[2,2], ARRAY[0,3])
+      END AS matches FROM backup_qualification.binary_arrays ORDER BY id`;
+    const expected = [
+      { id: 1, type: "bytea[]", dimensions: "[1:4]", matches: true },
+      { id: 2, type: "bytea[]", dimensions: null, matches: true },
+      { id: 3, type: "bytea[]", dimensions: null, matches: true },
+      { id: 4, type: "bytea[]", dimensions: "[1:2][1:2]", matches: true },
+      { id: 5, type: "bytea[]", dimensions: "[0:1][3:4]", matches: true },
+    ];
+    if (JSON.stringify(arrays) !== JSON.stringify(expected)) throw new Error("Restored binary array values, dimensions or types differ");
     const [next] = await target`INSERT INTO backup_qualification.parents (value) VALUES ('sequence after restore') RETURNING id::text`;
     if (next?.id !== "2") throw new Error("Restored identity sequence did not resume at 2");
     try {
       await target`INSERT INTO backup_qualification.children VALUES (999, 999, NULL, decode('', 'hex'))`;
       throw new Error("Restored foreign key was not enforced");
     } catch (error) { if ((error as { code?: string }).code !== "23503") throw error; }
-    return { relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, nullable: true, nextIdentity: next.id, foreignKeyEnforced: true };
+    return { relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, binaryArrays: true, nestedBinaryArrays: true, binaryArrayBounds: true, nullable: true, nextIdentity: next.id, foreignKeyEnforced: true };
   } finally { await target.end({ timeout: 5 }); }
 }
 
@@ -87,7 +109,9 @@ export async function qualifyBackupRestore(pair: QualificationPair, backupDir: s
     backupEngine: engine, verifyBeforePrune: verifyIdleDatabaseBackup });
   await assertDistinctEmptyRestore(pair);
   await runDatabaseRestore({ connectionString: pair.restoreUrl, backupFile: archive.backupFile });
+  const hash = createHash("sha256");
+  for await (const chunk of createReadStream(archive.backupFile)) hash.update(chunk);
   return { engine, archive: archive.backupFile, sizeBytes: archive.sizeBytes,
-    sha256: createHash("sha256").update(await readFile(archive.backupFile)).digest("hex"),
+    sha256: hash.digest("hex"),
     restored: await inspectRestoredFixture(pair.restoreUrl) };
 }

server/src/__tests__/database-backup-checkpoint-qualification.integration.ts

@@ -32,6 +32,18 @@ describe("real automatic backup checkpoint qualification", () => {
     await assertDistinctEmptyRestore(pair);
     ownsRestoreFixture = true;
     await seedBackupQualificationFixture(pair.sourceUrl);
+    const [column] = await source.$client`SELECT data_type, udt_name FROM information_schema.columns
+      WHERE table_schema = 'backup_qualification' AND table_name = 'binary_arrays' AND column_name = 'payloads'`;
+    expect(column).toEqual({ data_type: "ARRAY", udt_name: "_bytea" });
+    const [driver] = await source.$client`SELECT payloads FROM backup_qualification.binary_arrays WHERE id = 4`;
+    expect(Array.isArray(driver!.payloads[0])).toBe(true);
+    expect(Buffer.isBuffer(driver!.payloads[0][0])).toBe(true);
+    const [sqlNull] = await source.$client`SELECT payloads[1][2] IS NULL AS is_null
+      FROM backup_qualification.binary_arrays WHERE id = 4`;
+    expect(sqlNull?.is_null).toBe(true);
+    receipts.push({ case: "binary-array-driver-representation", column, nestedBuffers: true,
+      sqlNullElement: true, parsedNullElement: driver!.payloads[0][1] === null ? "null"
+        : Buffer.isBuffer(driver!.payloads[0][1]) ? "Buffer" : typeof driver!.payloads[0][1] });
   }, 120_000);
   afterAll(async () => {
     try {
@@ -77,7 +89,7 @@ describe("real automatic backup checkpoint qualification", () => {
     const versions = Object.fromEntries([process.env.PAPERCLIP_PG_DUMP_PATH || "pg_dump", process.env.PAPERCLIP_PSQL_PATH || "psql"]
       .map(binary => [binary, execFileSync(binary, ["--version"], { encoding: "utf8" }).trim()]));
     const result = await qualifyBackupRestore(pair, directory, engine);
-    expect(result.restored).toEqual({ relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, nullable: true, nextIdentity: "2", foreignKeyEnforced: true });
+    expect(result.restored).toEqual({ relational: true, unicode: true, binary: true, emptyBinary: true, nullBinary: true, binaryArrays: true, nestedBinaryArrays: true, binaryArrayBounds: true, nullable: true, nextIdentity: "2", foreignKeyEnforced: true });
     expect(gunzipSync(await readFile(result.archive)).length).toBeGreaterThan(0);
     receipts.push({ ...result, versions });
   }, 120_000);

server/src/__tests__/helpers/external-test-database.ts

@@ -1,11 +1 @@
-export function resolveExternalTestDatabaseUrl(name: string, value = process.env[name]): string | undefined {
-  if (!value?.trim()) return undefined;
-  try {
-    const url = new URL(value);
-    const database = decodeURIComponent(url.pathname.slice(1));
-    if (['postgres:', 'postgresql:'].includes(url.protocol)
-      && /^[a-zA-Z0-9_]+_test$/.test(database)
-      && !['database', 'dbname', 'options'].some(key => url.searchParams.has(key))) return value;
-  } catch {}
-  throw new Error(`${name} must name a dedicated PostgreSQL database ending in _test, without database or options overrides`);
-}
+export { resolveExternalTestDatabaseUrl } from "../../lib/dedicated-test-database-url.js";

server/src/lib/dedicated-test-database-url.ts

@@ -0,0 +1,11 @@
+export function resolveExternalTestDatabaseUrl(name: string, value = process.env[name]): string | undefined {
+  if (!value?.trim()) return undefined;
+  try {
+    const url = new URL(value);
+    const database = decodeURIComponent(url.pathname.slice(1));
+    if (['postgres:', 'postgresql:'].includes(url.protocol)
+      && /^[a-zA-Z0-9_]+_test$/.test(database)
+      && !['database', 'dbname', 'options'].some(key => url.searchParams.has(key))) return value;
+  } catch {}
+  throw new Error(`${name} must name a dedicated PostgreSQL database ending in _test, without database or options overrides`);
+}