diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 5d69f5356e5..e11ddaef860 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -342,6 +342,14 @@ Your Drizzle schema (`packages/db/src/schema/`) stays the same regardless of mod ## Migration authoring checklist +Older embedded databases can use `SQL_ASCII`. Test encoding-sensitive SQL on +both `SQL_ASCII` and UTF-8 databases. Migration `0355` constructs its whitespace +pattern from UTF-8 bytes so legacy databases can compile it, and matches whole +characters instead of trimming individual bytes. Its shipped hash remains a +valid migration receipt; otherwise an upgrade could replay its older connector +triggers over the replacements from `0356`. The encoding migration regression +checks both selector behavior and preservation of that later trigger definition. + The 0126 issue comment attribution backfill showed the failure mode this checklist is meant to prevent: each batch looked for the next rows with an unindexed predicate, so PostgreSQL repeatedly scanned the same table and the migration became O(n²) as the table grew. When authoring migrations or one-time backfills: diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 1f4a9257a39..5efa60e511c 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -580,6 +580,9 @@ async function mapHashesToMigrationFiles(migrationFiles: string[]): Promise { + let database: EmbeddedPostgresTestDatabase | undefined; + let utf8: ReturnType | undefined; + let ascii: ReturnType | undefined; + let asciiUrl: string; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-connector-encoding-"); + utf8 = postgres(database.connectionString, { max: 1, onnotice: () => {} }); + await utf8.unsafe("CREATE DATABASE legacy_ascii TEMPLATE template0 ENCODING 'SQL_ASCII' LC_COLLATE 'C' LC_CTYPE 'C'"); + const url = new URL(database.connectionString); + url.pathname = "/legacy_ascii"; + asciiUrl = url.toString(); + ascii = postgres(asciiUrl, { max: 1, onnotice: () => {} }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterAll(async () => { + try { + await Promise.all([utf8?.end(), ascii?.end()]); + } finally { + await database?.cleanup(); + } + }); + + it.each(["UTF8", "SQL_ASCII"])("migrates %s and matches JavaScript selector trimming", async (encoding) => { + const sql = encoding === "UTF8" ? utf8! : ascii!; + if (encoding === "SQL_ASCII") await applyPendingMigrations(asciiUrl); + expect((await sql`SHOW server_encoding`)[0]?.server_encoding).toBe(encoding); + for (const input of [...whitespace, whitespace, `${whitespace}${agent}${whitespace}`, "\u200b", "é", "中"]) { + const actual = await sql`SELECT * FROM paperclip_connector_policy_targets(${company}::uuid, ${sql.json({ agentIds: input })})`; + expect(Array.from(actual), JSON.stringify(input)).toEqual(input.trim() + ? [{ target_type: "agent", target_id: input.trim() }] + : [{ target_type: "company", target_id: company }]); + } + const arrays = await sql`SELECT * FROM paperclip_connector_policy_targets(${company}::uuid, + ${sql.json({ agentIds: [whitespace, agent, ` ${agent} `, null, 12] })})`; + expect(Array.from(arrays)).toEqual([ + { target_type: "agent", target_id: agent }, + { target_type: "agent", target_id: ` ${agent} ` }, + ]); + const intersection = await sql`SELECT * FROM paperclip_connector_policy_targets(${company}::uuid, + ${sql.json({ agentId: company, agentIds: [agent] })})`; + expect(Array.from(intersection)).toEqual([]); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + it("recognizes the shipped receipt without replaying older connector triggers", async () => { + const sql = utf8!; + const content = await readFile(new URL(`./migrations/${migration}`, import.meta.url), "utf8"); + const hash = createHash("sha256").update(content).digest("hex"); + const originalHash = "7ac76adac0220da522c78e51da20490aeb0f90512b4370a05af9f8dd0f75eda6"; + const before = await sql`SELECT pg_get_functiondef('paperclip_connector_runtime_changed()'::regprocedure) AS definition`; + const count = await sql`SELECT count(*)::int AS count FROM drizzle.__drizzle_migrations`; + await sql`UPDATE drizzle.__drizzle_migrations SET hash = ${originalHash} WHERE hash = ${hash}`; + await applyPendingMigrations(database!.connectionString); + expect((await inspectMigrations(database!.connectionString)).status).toBe("upToDate"); + expect(await sql`SELECT pg_get_functiondef('paperclip_connector_runtime_changed()'::regprocedure) AS definition`).toEqual(before); + expect(await sql`SELECT count(*)::int AS count FROM drizzle.__drizzle_migrations`).toEqual(count); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); +}); diff --git a/packages/db/src/migrations/0355_agent_tool_definition_hooks.sql b/packages/db/src/migrations/0355_agent_tool_definition_hooks.sql index 2a3bb66d273..00f647a5fb4 100644 --- a/packages/db/src/migrations/0355_agent_tool_definition_hooks.sql +++ b/packages/db/src/migrations/0355_agent_tool_definition_hooks.sql @@ -3,13 +3,13 @@ RETURNS TABLE(target_type text, target_id text) LANGUAGE plpgsql IMMUTABLE SET search_path = public, pg_temp AS $$ DECLARE single_agent text := CASE WHEN jsonb_typeof(selectors->'agentId') = 'string' THEN nullif(selectors->>'agentId', '') END; many_agents text[]; - trim_chars text := U&'\0009\000A\000B\000C\000D\0020\00A0\1680\2000\2001\2002\2003\2004\2005\2006\2007\2008\2009\200A\2028\2029\202F\205F\3000\FEFF'; + trim_pattern text := '[\x09-\x0d ]|' || convert_from(decode('c2a07ce19a807ce280807ce280817ce280827ce280837ce280847ce280857ce280867ce280877ce280887ce280897ce2808a7ce280a87ce280a97ce280af7ce2819f7ce380807cefbbbf', 'hex'), 'UTF8'); BEGIN SELECT array_agg(value#>>'{}') INTO many_agents FROM jsonb_array_elements(CASE jsonb_typeof(selectors->'agentIds') WHEN 'array' THEN selectors->'agentIds' - WHEN 'string' THEN jsonb_build_array(btrim(selectors->>'agentIds', trim_chars)) ELSE '[]'::jsonb END) value - WHERE jsonb_typeof(value) = 'string' AND btrim(value#>>'{}', trim_chars) <> ''; + WHEN 'string' THEN jsonb_build_array(regexp_replace(selectors->>'agentIds', '^(' || trim_pattern || ')+|(' || trim_pattern || ')+$', '', 'g')) ELSE '[]'::jsonb END) value + WHERE jsonb_typeof(value) = 'string' AND regexp_replace(value#>>'{}', '^(' || trim_pattern || ')+|(' || trim_pattern || ')+$', '', 'g') <> ''; IF single_agent IS NOT NULL THEN IF coalesce(cardinality(many_agents), 0) = 0 OR single_agent = ANY(many_agents) THEN RETURN QUERY SELECT 'agent'::text, single_agent;