diff --git a/doc/DATABASE.md b/doc/DATABASE.md index e2ac1e50d99..f89eaae0b52 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -461,6 +461,21 @@ pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-chat-admission-i pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-chat-admission-index-migration.ts ``` +Native harness backup expiration finds the leases that reference one backup root +by run, by `metadata.nativeHarnessBackup.sessionScopeSha256`, and by the +identity's own lease ID, in one `OR` query. The partial expression index +`environment_leases_native_backup_scope_idx` from migration 0459 keys leases by +that scope digest. With it, every arm of the `OR` has an index, so PostgreSQL +combines three index probes instead of reading every lease of the company and its +metadata. The expression must stay identical to the one in +`server/src/services/native-runtime/native-harness-backup-expiration.ts`; a +regression test checks that the planner uses the index. The migration builds the +index concurrently. To compare both plans on a disposable local fixture: + +```sh +pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-environment-lease-backup-lookup.ts +``` + The chat wake-notice sweep compares UUID columns directly against guarded, canonical lowercase UUID payload values. Malformed and noncanonical payloads still match nothing. The query can use existing primary keys without adding a diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 320df7b461b..0e0f80b846d 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -621,6 +621,7 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([ "0456_chat_review_lookup_indexes.sql", "0457_native_session_run_index.sql", "0458_chat_admission_issue_index.sql", + "0459_environment_lease_backup_scope_index.sql", "0384_funny_adam_warlock.sql", diff --git a/packages/db/src/migrations/0459_environment_lease_backup_scope_index.sql b/packages/db/src/migrations/0459_environment_lease_backup_scope_index.sql new file mode 100644 index 00000000000..0912ccba441 --- /dev/null +++ b/packages/db/src/migrations/0459_environment_lease_backup_scope_index.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "environment_leases_native_backup_scope_idx" ON "environment_leases" USING btree (("metadata"->'nativeHarnessBackup'->>'sessionScopeSha256')) WHERE ("environment_leases"."metadata"->'nativeHarnessBackup'->>'sessionScopeSha256') is not null; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0454_snapshot.json b/packages/db/src/migrations/meta/0459_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0454_snapshot.json rename to packages/db/src/migrations/meta/0459_snapshot.json index 4e0bb003c30..4ec4872e79f 100644 --- a/packages/db/src/migrations/meta/0454_snapshot.json +++ b/packages/db/src/migrations/meta/0459_snapshot.json @@ -1,6 +1,6 @@ { - "id": "f98af245-4a2e-4c3d-b89e-6ed3e81a074c", - "prevId": "d54bfeb4-dedb-4f8b-9195-a4d4aa67d71f", + "id": "e070677d-cfb8-4438-847c-29c4da7ca8c5", + "prevId": "3347b37d-b35b-4357-b08c-a78c017086ac", "version": "7", "dialect": "postgresql", "tables": { @@ -10188,6 +10188,40 @@ "method": "btree", "with": {} }, + "chat_actions_company_issue_kind_idx": { + "name": "chat_actions_company_issue_kind_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"payload\"->>'issueId')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat_actions\".\"kind\" IN ('inbound_wakeup', 'failed_run_retry')", + "concurrently": true, + "method": "btree", + "with": {} + }, "chat_actions_github_stale_head_idx": { "name": "chat_actions_github_stale_head_idx", "columns": [ @@ -10955,6 +10989,28 @@ "concurrently": false, "method": "btree", "with": {} + }, + "chat_deliveries_github_automatic_delivery_idx": { + "name": "chat_deliveries_github_automatic_delivery_idx", + "columns": [ + { + "expression": "endpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"normalized_event\"->'githubAutomatic'->'context'->>'deliveryId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "(\"chat_deliveries\".\"normalized_event\"->'githubAutomatic'->'context'->>'deliveryId') is not null", + "concurrently": true, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -12393,6 +12449,44 @@ "concurrently": false, "method": "btree", "with": {} + }, + "chat_message_links_company_comment_idx": { + "name": "chat_message_links_company_comment_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "comment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat_message_links\".\"comment_id\" is not null", + "concurrently": true, + "method": "btree", + "with": {} + }, + "chat_message_links_delivery_idx": { + "name": "chat_message_links_delivery_idx", + "columns": [ + { + "expression": "delivery_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat_message_links\".\"delivery_id\" is not null", + "concurrently": true, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -29073,6 +29167,22 @@ "concurrently": false, "method": "btree", "with": {} + }, + "environment_leases_native_backup_scope_idx": { + "name": "environment_leases_native_backup_scope_idx", + "columns": [ + { + "expression": "(\"metadata\"->'nativeHarnessBackup'->>'sessionScopeSha256')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "(\"environment_leases\".\"metadata\"->'nativeHarnessBackup'->>'sessionScopeSha256') is not null", + "concurrently": true, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -35421,6 +35531,22 @@ "method": "btree", "with": {} }, + "heartbeat_runs_pending_budget_cancellation_idx": { + "name": "heartbeat_runs_pending_budget_cancellation_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "jsonb_typeof(\"heartbeat_runs\".\"result_json\"->'startupCancellation'->'budgetEnforcement') = 'object'\n and \"heartbeat_runs\".\"result_json\"->'startupCancellation'->'budgetEnforcement'->>'deliveredAt' is null", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_tree_hold_suppression_idx": { "name": "heartbeat_runs_tree_hold_suppression_idx", "columns": [ @@ -35459,6 +35585,27 @@ "method": "btree", "with": {} }, + "heartbeat_runs_company_wake_comment_idx": { + "name": "heartbeat_runs_company_wake_comment_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"context_snapshot\"->>'wakeCommentId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_cost_accounting_pending_idx": { "name": "heartbeat_runs_cost_accounting_pending_idx", "columns": [ @@ -35611,6 +35758,28 @@ "method": "btree", "with": {} }, + "heartbeat_runs_company_native_session_idx": { + "name": "heartbeat_runs_company_native_session_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "native_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"native_session_id\" is not null", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_company_issue_created_idx": { "name": "heartbeat_runs_company_issue_created_idx", "columns": [ diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 28cd5ff66b5..45a14d43c57 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3200,6 +3200,13 @@ "when": 1794326875077, "tag": "0458_chat_admission_issue_index", "breakpoints": true + }, + { + "idx": 459, + "version": "7", + "when": 1794329161946, + "tag": "0459_environment_lease_backup_scope_index", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/environment_leases.ts b/packages/db/src/schema/environment_leases.ts index 948d372f237..02ff6d624f9 100644 --- a/packages/db/src/schema/environment_leases.ts +++ b/packages/db/src/schema/environment_leases.ts @@ -54,5 +54,12 @@ export const environmentLeases = pgTable( heartbeatRunIdx: index("environment_leases_heartbeat_run_idx").on(table.heartbeatRunId), companyLastUsedIdx: index("environment_leases_company_last_used_idx").on(table.companyId, table.lastUsedAt), providerLeaseIdx: index("environment_leases_provider_lease_idx").on(table.providerLeaseId), + // Leases that reference one native harness backup root by its scope digest + // (backup expiration, retention and session retention checks). Partial: + // most leases carry no backup stamp. + nativeBackupScopeIdx: index("environment_leases_native_backup_scope_idx") + .on(sql`(${table.metadata}->'nativeHarnessBackup'->>'sessionScopeSha256')`) + .where(sql`(${table.metadata}->'nativeHarnessBackup'->>'sessionScopeSha256') is not null`) + .concurrently(), }), ); diff --git a/scripts/benchmark-environment-lease-backup-lookup.ts b/scripts/benchmark-environment-lease-backup-lookup.ts new file mode 100644 index 00000000000..2f062dd43d2 --- /dev/null +++ b/scripts/benchmark-environment-lease-backup-lookup.ts @@ -0,0 +1,181 @@ +/** + * Local proof for the native backup lease lookup (migration 0459). + * + * `reconcileBackup` reads the leases that reference one backup root: the + * session's run leases, leases whose `metadata.nativeHarnessBackup` stamp has + * the root's scope digest, and the identity's own lease. The metadata arm had + * no index, so PostgreSQL read every lease of the company and its metadata. + * + * The script captures the SQL that the real service generates, then measures + * it without and with the new index (no index, index, no index again). It + * fails unless the service's lease IDs and the complete query rows match. + * + * pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-environment-lease-backup-lookup.ts + * + * Optional sizes: LEASES (default 100000), STAMPED_PERCENT (30), + * METADATA_KIB (4), OTHER_COMPANY_LEASES (20000). + */ +import assert from "node:assert/strict"; +import { createHash, randomUUID } from "node:crypto"; +import { promises as fs, readFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { agents, companies, createDb, environmentLeases, heartbeatRuns, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; +import { nativeHarnessBackupExpirationService } from "../server/src/services/native-runtime/native-harness-backup-expiration.js"; + +const size = (name: string, fallback: number) => Number(process.env[name] ?? fallback); +const LEASES = size("LEASES", 100_000), STAMPED_PERCENT = size("STAMPED_PERCENT", 30); +const METADATA_KIB = size("METADATA_KIB", 4), OTHER_COMPANY_LEASES = size("OTHER_COMPANY_LEASES", 20_000); +const migration = readFileSync(process.env.INDEX_SQL_FILE ?? new URL( + "../packages/db/src/migrations/0459_environment_lease_backup_scope_index.sql", import.meta.url), "utf8"); +const indexStatements = migration.split("--> statement-breakpoint").map(statement => statement.trim()).filter(Boolean); +const indexNames = indexStatements.map(statement => statement.match(/IF NOT EXISTS "([a-z_]+)"/)![1]!); + +type Captured = { sql: string; params: unknown[] }; +/** Records the SQL of each `db.select(...).from(...)` the service awaits. */ +function capture(db: Db, queries: Captured[]): Db { + return new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), then = selection.then.bind(selection); + selection.then = ((...handlers: Parameters) => { + queries.push(selection.toSQL()); + return then(...handlers); + }) as typeof selection.then; + return selection; + }) as typeof builder.from; + return builder; + }; + }}); +} + +const keepAlive = setInterval(() => {}, 1_000); +let temporary: Awaited> | undefined; +let db: ReturnType | undefined; +let stateBase: string | undefined; +try { + temporary = await startEmbeddedPostgresTestDatabase("environment-lease-backup-lookup-"); + db = createDb(temporary.connectionString); + const sql = db.$client; + for (const name of indexNames) await sql.unsafe(`DROP INDEX IF EXISTS "${name}"`); + await sql.unsafe("create extension if not exists pgcrypto"); + + const companyId = randomUUID(), otherCompanyId = randomUUID(); + await db.insert(companies).values([{ id: companyId, name: "Leases", issuePrefix: "LEA" }, + { id: otherCompanyId, name: "Other", issuePrefix: "OTH" }]); + const [agent] = await db.insert(agents).values({ companyId, name: "Coder" }).returning(); + const sessionId = randomUUID(); + // Three runs of the backed-up session; the root's identity names the first. + const runs = await db.insert(heartbeatRuns).values([0, 1, 2].map(() => ({ + companyId, agentId: agent!.id, status: "succeeded", nativeSessionId: sessionId }))).returning(); + const scope = createHash("sha256").update(randomUUID()).digest("hex"); + + const loadStart = performance.now(); + // History: incompressible sandbox metadata; some leases carry another + // session's backup stamp, most carry none. + const seed = (company: string, count: number) => sql.unsafe(`insert into environment_leases + (company_id, status, lease_policy, provider, provider_lease_id, metadata, created_at, last_used_at) + select $1::uuid, case when g % 10 = 0 then 'active' else 'released' end, 'retain_on_failure', 'sandbox', + 'sbx-' || g, jsonb_build_object( + 'nativeHarnessBackup', case when g % 100 < $2::int then jsonb_build_object( + 'sessionScopeSha256', encode(digest(gen_random_uuid()::text, 'sha256'), 'hex'), + 'manifestSha256', encode(digest(g::text, 'sha256'), 'hex')) end, + 'sandbox', (select string_agg(encode(gen_random_bytes(768), 'base64'), '') from generate_series(1, $3::int) s(i) where g > 0)), + now() - make_interval(secs => g), now() - make_interval(secs => g) + from generate_series(1, $4::int) g`, [company, STAMPED_PERCENT, METADATA_KIB, count]); + await seed(companyId, LEASES); + await seed(otherCompanyId, OTHER_COMPANY_LEASES); + // References to the root: a lease of each run; one lease known only by its + // backup stamp (its run row was deleted); the identity's own lease; and the + // same digest in another company, which must stay out. + const runLeases = await db.insert(environmentLeases).values(runs.map(run => ({ + companyId, heartbeatRunId: run.id, status: "released", metadata: { sandbox: "run" } }))).returning(); + const [stamped] = await db.insert(environmentLeases).values({ companyId, status: "retained", + metadata: { nativeHarnessBackup: { sessionScopeSha256: scope } } }).returning(); + const [own] = await db.insert(environmentLeases).values({ companyId, status: "active", metadata: {} }).returning(); + await db.insert(environmentLeases).values({ companyId: otherCompanyId, status: "retained", + metadata: { nativeHarnessBackup: { sessionScopeSha256: scope } } }); + await sql.unsafe("vacuum analyze"); + const loadMs = performance.now() - loadStart; + const expectedLeaseIds = [...runLeases.map(lease => lease.id), stamped!.id, own!.id].sort(); + + stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "environment-lease-backup-lookup-")); + const root = path.join(stateBase, scope); + await fs.mkdir(path.join(root, "control-plane"), { recursive: true }); + await fs.mkdir(path.join(root, "failover-backups", "current"), { recursive: true }); + await fs.writeFile(path.join(root, "control-plane", "control-plane-state.json"), JSON.stringify({ + identity: { runId: runs[0]!.id, normalizedSessionId: sessionId, environmentLeaseId: own!.id } })); + const service = (target: Db) => nativeHarnessBackupExpirationService(target, { + stateBase, closedForDays: 7, isSessionBusy: () => false }); + + const queries: Captured[] = []; + const reconciled = await service(capture(db, queries)).reconcileBackup(scope); + assert.deepEqual([...reconciled!.dependencies.leaseIds].sort(), expectedLeaseIds); + const leaseQuery = queries.find(query => query.sql.includes('from "environment_leases"') && query.sql.includes("sessionScopeSha256")); + assert.ok(leaseQuery, "The service must read the root's leases"); + + const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]!; + const round = (value: number) => Number(value.toFixed(3)); + const summary = (samples: number[]) => ({ samplesMs: samples.map(round), medianMs: round(median(samples)), + minMs: round(Math.min(...samples)), maxMs: round(Math.max(...samples)) }); + async function explain() { + const [result] = await sql.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${leaseQuery!.sql}`, leaseQuery!.params as never[]); + return (result!["QUERY PLAN"] as Array<{ Plan: Record; "Execution Time": number }>)[0]!; + } + function nodes(plan: Record, out: string[] = []) { + out.push(`${plan["Node Type"]}${plan["Index Name"] ? ` using ${plan["Index Name"]}` : ""}`); + for (const child of (plan.Plans as Array> | undefined) ?? []) nodes(child, out); + return out; + } + async function measure(indexed: boolean) { + const builds: Array<{ index: string; ms: number }> = []; + if (!indexed) for (const name of indexNames) await sql.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS "${name}"`); + else for (const [n, statement] of indexStatements.entries()) { + const start = performance.now(); + await sql.unsafe(statement); + builds.push({ index: indexNames[n]!, ms: Math.round(performance.now() - start) }); + } + await sql.unsafe("analyze environment_leases"); + const rows = (await sql.unsafe(leaseQuery!.sql, leaseQuery!.params as never[]).values()) + .map(row => row.map(value => value?.toString() ?? null)).sort(); + const ids = [...(await service(db!).reconcileBackup(scope))!.dependencies.leaseIds].sort(); + for (let n = 0; n < 2; n++) await explain(); + const execution: number[] = [], wall: number[] = []; + for (let n = 0; n < 11; n++) { + execution.push((await explain())["Execution Time"]); + const start = performance.now(); + await service(db!).reconcileBackup(scope); + wall.push(performance.now() - start); + } + const plan = (await explain()).Plan; + return { indexed, builds, rows, ids, serverExecution: summary(execution), reconcileBackupWall: summary(wall), + plan: { nodes: nodes(plan), sharedHitBlocks: plan["Shared Hit Blocks"], sharedReadBlocks: plan["Shared Read Blocks"] } }; + } + + const results = [await measure(false), await measure(true), await measure(false), await measure(true)]; + for (const result of results) { + assert.deepEqual(result.ids, expectedLeaseIds, "Service lease IDs must match"); + assert.deepEqual(result.rows, results[0]!.rows, "Complete query rows must match"); + } + console.log(JSON.stringify({ + postgresVersion: (await sql.unsafe("select version()"))[0]!.version, + settings: await sql.unsafe("select name, setting from pg_settings where name in ('shared_buffers','work_mem','jit') order by name"), + fixture: { synthetic: true, leases: LEASES, stampedPercent: STAMPED_PERCENT, metadataKiB: METADATA_KIB, + otherCompanyLeases: OTHER_COMPANY_LEASES, loadMs: Math.round(loadMs), + tableSize: (await sql.unsafe("select pg_size_pretty(pg_total_relation_size('environment_leases')) as size"))[0]!.size, + indexSizes: await sql.unsafe(`select c.relname, pg_size_pretty(pg_relation_size(c.oid)) as size from pg_class c + where c.relname in (${indexNames.map((_, n) => `$${n + 1}`).join(", ")})`, indexNames) }, + statement: leaseQuery.sql, + correctness: { serviceLeaseIdsEqual: true, completeRowsEqual: true, matchedLeases: expectedLeaseIds.length, + cases: ["run leases", "stamp-only lease", "identity lease", "same digest in another company excluded"] }, + results: results.map(({ rows: _rows, ids: _ids, ...result }) => result), + }, null, 2)); +} finally { + try { + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + if (stateBase) await fs.rm(stateBase, { recursive: true, force: true }); + } finally { clearInterval(keepAlive); } +} diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index 1251a29bb67..28740b3bf9c 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -985,6 +985,60 @@ describeEmbeddedPostgres("native harness backup expiration", () => { await quiet(); }); + it("finds a backup root's leases by run, backup stamp and identity through indexes, within one company", async () => { + const session = await seedSession({ leaseStatus: "retained" }); + const [stamped] = await db.select({ id: environmentLeases.id }).from(environmentLeases) + .where(eq(environmentLeases.companyId, session.companyId)); + const [runLease] = await db.insert(environmentLeases).values({ companyId: session.companyId, + heartbeatRunId: session.runId, status: "released", metadata: {} }).returning(); + const [own] = await db.insert(environmentLeases).values({ companyId: session.companyId, status: "active", + metadata: { nativeHarnessBackup: { sessionScopeSha256: "0".repeat(64) } } }).returning(); + // Not references: another digest, a stamp that is not a string match, and + // the same digest in another company. + const otherCompanyId = randomUUID(); + await db.insert(companies).values({ id: otherCompanyId, name: "Other", issuePrefix: `O${otherCompanyId.slice(0, 6)}` }); + await db.insert(environmentLeases).values([ + { companyId: session.companyId, status: "retained", metadata: { nativeHarnessBackup: { sessionScopeSha256: "f".repeat(64) } } }, + { companyId: session.companyId, status: "retained", metadata: { nativeHarnessBackup: { sessionScopeSha256: [session.scope] } } }, + { companyId: session.companyId, status: "retained", metadata: { sessionScopeSha256: session.scope } }, + { companyId: otherCompanyId, status: "retained", metadata: { nativeHarnessBackup: { sessionScopeSha256: session.scope } } }, + ]); + // Unrelated lease history, so the planner prefers indexes to a company scan. + await db.$client.unsafe(`insert into environment_leases (company_id, status, metadata) + select $1::uuid, 'released', jsonb_build_object('sandbox', repeat('x', 512)) from generate_series(1, 2000)`, [session.companyId]); + await db.$client.unsafe("analyze environment_leases"); + const statePath = path.join(session.root, "control-plane", "control-plane-state.json"); + await fs.writeFile(statePath, JSON.stringify({ identity: { + runId: session.runId, normalizedSessionId: session.sessionId, environmentLeaseId: own!.id } })); + + const queries: Array<{ text: string; params: unknown[] }> = []; + const previous = db.$client.options.debug; + db.$client.options.debug = (_connection, text, params) => { queries.push({ text, params: params as unknown[] }); }; + let reconciled: Awaited["reconcileBackup"]>>; + try { reconciled = await service().reconcileBackup(session.scope); } finally { db.$client.options.debug = previous; } + expect([...reconciled!.dependencies.leaseIds].sort()).toEqual([stamped!.id, runLease!.id, own!.id].sort()); + + // Every arm of the company's lease lookup must be index-backed, or the + // planner reads each lease's metadata (environment_leases_native_backup_scope_idx). + const lookup = queries.find(query => query.text.startsWith('select "id" from "environment_leases"') + && query.text.includes("sessionScopeSha256")); + expect(lookup).toBeDefined(); + const custom = JSON.stringify(await db.$client.unsafe(`explain (format json) ${lookup!.text}`, lookup!.params as never[])); + // A reused prepared statement gets a generic plan; the bound digest must + // still prove the partial index predicate. + const generic = JSON.stringify(await db.$client.begin(async tx => { + await tx.unsafe("set local plan_cache_mode = force_generic_plan"); + await tx.unsafe(`prepare backup_lease_lookup as ${lookup!.text}`); + const values = lookup!.params.map(value => `'${String(value).replaceAll("'", "''")}'`).join(", "); + return tx.unsafe(`explain (format json) execute backup_lease_lookup(${values})`); + })); + for (const text of [custom, generic]) { + expect(text).toContain("environment_leases_native_backup_scope_idx"); + expect(text).toContain("BitmapOr"); + expect(text).not.toContain("Seq Scan"); + } + }); + it("rejects destructive backup scopes outside the controller digest namespace", async () => { await emptyStateBase(); const remove = vi.spyOn(fs, "rm"); await expect(service().expireBackup("../outside")).rejects.toThrow("Invalid native harness backup scope");