diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index 28740b3bf9c..4c956f5de5b 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -1020,22 +1020,29 @@ describeEmbeddedPostgres("native harness backup expiration", () => { // Every arm of the company's lease lookup must be index-backed, or the // planner reads each lease's metadata (environment_leases_native_backup_scope_idx). - const lookup = queries.find(query => query.text.startsWith('select "id" from "environment_leases"') - && query.text.includes("sessionScopeSha256")); + const lookup = queries.find(query => query.text.includes("jsonb_to_recordset") && query.text.includes("sessionScopeSha256")); expect(lookup).toBeDefined(); - const custom = JSON.stringify(await db.$client.unsafe(`explain (format json) ${lookup!.text}`, lookup!.params as never[])); - // A reused prepared statement gets a generic plan; the bound digest must - // still prove the partial index predicate. - const generic = JSON.stringify(await db.$client.begin(async tx => { + const leaseScans = (plan: unknown): string[] => { + const nodes: string[] = []; + const visit = (node: Record) => { + if (String(node["One-Time Filter"] ?? "") === "$3") return; + if (node["Relation Name"] === "environment_leases" || String(node["Index Name"] ?? "").startsWith("environment_leases")) + nodes.push(`${node["Node Type"]}:${node["Index Name"] ?? ""}`); + for (const child of (node.Plans as Record[] | undefined) ?? []) visit(child); + }; + visit(((plan as Array<{ "QUERY PLAN": Array<{ Plan: Record }> }>)[0]!["QUERY PLAN"][0]!).Plan); + return nodes; + }; + const custom = leaseScans(await db.$client.unsafe(`explain (format json) ${lookup!.text}`, lookup!.params as never[])); + const generic = leaseScans(await db.$client.begin(async tx => { await tx.unsafe("set local plan_cache_mode = force_generic_plan"); await tx.unsafe(`prepare backup_lease_lookup as ${lookup!.text}`); const values = lookup!.params.map(value => `'${String(value).replaceAll("'", "''")}'`).join(", "); return tx.unsafe(`explain (format json) execute backup_lease_lookup(${values})`); })); - for (const text of [custom, generic]) { - expect(text).toContain("environment_leases_native_backup_scope_idx"); - expect(text).toContain("BitmapOr"); - expect(text).not.toContain("Seq Scan"); + for (const scans of [custom, generic]) { + expect(scans.some(scan => scan.endsWith("environment_leases_native_backup_scope_idx"))).toBe(true); + expect(scans.filter(scan => scan.startsWith("Seq Scan"))).toEqual([]); } }); diff --git a/server/src/__tests__/native-retention-blocker-batch.test.ts b/server/src/__tests__/native-retention-blocker-batch.test.ts new file mode 100644 index 00000000000..bcfb995efd9 --- /dev/null +++ b/server/src/__tests__/native-retention-blocker-batch.test.ts @@ -0,0 +1,164 @@ +import { createHash, randomUUID } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import { promises as fs } from "node:fs"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { + agents, agentTaskSessions, companies, createDb, environmentLeases, heartbeatRuns, issues, + nativeRunFinalizations, nativeRunnerCheckpoints, +} from "@paperclipai/db"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; +import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; +import { registerNativeHarnessBackupExpirationWork } from "../services/native-runtime/native-harness-backup-expiration-work.js"; +import { notifyRetainedNativeSessionCleanup } from "../services/retained-native-session-cleanup-notifications.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const DAY_MS = 24 * 60 * 60 * 1000; +const describeEmbeddedPostgres = recoveryPostgresSupport.supported ? describe : describe.skip; + +describeEmbeddedPostgres("native retention blockers", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + let meter!: ReturnType; + let stateBase = ""; + const now = new Date(); + + beforeAll(async () => { + tempDb = await startRecoveryPostgres("paperclip-native-retention-blockers-"); + db = createDb(tempDb.connectionString); + meter = meterDatabaseStatements(db, tempDb.connectionString); + }, 30_000); + afterEach(async () => { + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(nativeRunnerCheckpoints); + await db.delete(agentTaskSessions); + await db.delete(nativeRunFinalizations); + await db.delete(environmentLeases); + await db.delete(heartbeatRuns); + await db.delete(issues); + await db.delete(agents); + await db.delete(companies); + if (stateBase) await fs.rm(stateBase, { recursive: true, force: true }); + stateBase = ""; + }); + afterAll(async () => { + meter?.uninstall(); + await db?.$client.end({ timeout: 1 }); + await tempDb?.cleanup(); + }); + + async function seedSessions(count: number, input: { runsPerSession?: number; issueStatus?: string; closedDaysAgo?: number; backup?: boolean } = {}) { + if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-retention-blockers-")); + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `R${companyId.slice(0, 6)}` }); + const [agent] = await db.insert(agents).values({ companyId, name: "Coder" }).returning(); + const finishedAt = new Date(now.getTime() - 10 * DAY_MS); + const status = input.issueStatus ?? "todo"; + const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS); + const sessions = []; + for (let index = 0; index < count; index++) { + const [issue] = await db.insert(issues).values({ companyId, title: "Work", status, + completedAt: status === "done" ? closedAt : null }).returning(); + const sessionId = randomUUID(); + const runs = await db.insert(heartbeatRuns).values(Array.from({ length: input.runsPerSession ?? 3 }, () => ({ + companyId, agentId: agent!.id, status: "succeeded", issueId: issue!.id, nativeIssueId: issue!.id, + nativeSessionId: sessionId, finishedAt, + }))).returning(); + await db.insert(nativeRunFinalizations).values(runs.map(run => ({ runId: run.id, companyId, issueId: issue!.id, phase: "committed" }))); + await db.insert(nativeRunnerCheckpoints).values(runs.map(run => ({ + companyId, issueId: issue!.id, runId: run.id, part: "header", material: { encrypted: true }, + }))); + const scope = createHash("sha256").update(randomUUID()).digest("hex"); + if (input.backup) { + const root = path.join(stateBase, scope); + const current = path.join(root, "failover-backups", "current"); + await fs.mkdir(path.join(root, "control-plane"), { recursive: true }); + await fs.writeFile(path.join(root, "control-plane", "control-plane-state.json"), + JSON.stringify({ identity: { runId: runs[0]!.id, normalizedSessionId: sessionId } })); + await fs.mkdir(current, { recursive: true }); + await fs.writeFile(path.join(current, "manifest.json"), "{}"); + await fs.utimes(current, finishedAt, finishedAt); + } + sessions.push({ companyId, agentId: agent!.id, issueId: issue!.id, sessionId, scope, runIds: runs.map(run => run.id) }); + } + return sessions; + } + + function service() { + return nativeHarnessBackupExpirationService(db, { stateBase, closedForDays: 7, isSessionBusy: () => false }); + } + + async function startup() { + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: () => {} }); + const before = meter.read(); + try { + await service().registerCheckpointExpiration(coordinator).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 60_000 }); + } finally { await coordinator.stop(); } + const after = meter.read(); + return { statements: after.statements - before.statements, transactions: after.transactions - before.transactions }; + } + + it("evaluates blocked checkpoint sessions in one statement per page and repeats nothing per checkpoint on restart", async () => { + await seedSessions(100); + const first = await startup(); + const second = await startup(); + expect(first).toEqual({ statements: 4, transactions: 0 }); + expect(second).toEqual(first); + expect(await db.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)).toHaveLength(300); + }, 120_000); + + it("expires the checkpoints of a session on the next startup once its blocker clears", async () => { + const [blocked] = await seedSessions(1); + const cleared = await seedSessions(2, { issueStatus: "done", closedDaysAgo: 9 }); + await startup(); + const remaining = await db.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints); + expect(remaining.map(row => row.runId).sort()).toEqual([...blocked!.runIds].sort()); + expect(cleared.flatMap(session => session.runIds).some(runId => remaining.some(row => row.runId === runId))).toBe(false); + await db.update(issues).set({ status: "done", completedAt: new Date(now.getTime() - 8 * DAY_MS) }).where(eq(issues.id, blocked!.issueId)); + await startup(); + expect(await db.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)).toEqual([]); + }, 120_000); + + it("keeps a checkpoint whose session still has a resume pointer", async () => { + const [session] = await seedSessions(1, { issueStatus: "done", closedDaysAgo: 9 }); + await db.insert(agentTaskSessions).values({ companyId: session!.companyId, agentId: session!.agentId, adapterType: "paperclip_runner", + taskKey: session!.issueId, sessionParamsJson: { sessionId: session!.sessionId }, lastRunId: session!.runIds[0]! }); + expect(await startup()).toEqual({ statements: 2, transactions: 0 }); + expect(await db.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)).toHaveLength(3); + }, 120_000); + + it("lets two servers run the startup pass at once without duplicate or conflicting retirement", async () => { + const sessions = await seedSessions(20, { issueStatus: "done", closedDaysAgo: 9 }); + const [left, right] = await Promise.all([service().tickCheckpoints(), service().tickCheckpoints()]); + expect(left.failed + right.failed).toBe(0); + expect(left.expiredCheckpoints + right.expiredCheckpoints).toBe(sessions.length * 3); + expect(await db.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)).toEqual([]); + }, 120_000); + + it("reads a blocked backup in one statement for each run hint that reaches it", async () => { + const sessions = await seedSessions(50, { backup: true, runsPerSession: 1 }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: () => {} }); + const worker = registerNativeHarnessBackupExpirationWork(coordinator, db, { stateBase, isSessionBusy: () => false }, () => true); + try { + await worker.ready; + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }); + const before = meter.read(); + meter.startLog(); + for (const session of sessions) await db.transaction(tx => notifyRetainedNativeSessionCleanup(tx, { companyId: session.companyId, runId: session.runIds[0]! })); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 30_000 }); + await vi.advanceTimersByTimeAsync(5000); + const log = meter.stopLog(); + const after = meter.read(); + const hintWrites = sessions.length * 3; + expect(after.transactions - before.transactions).toBe(sessions.length); + expect(after.statements - before.statements - hintWrites).toBeLessThanOrEqual(sessions.length * 2); + expect(log.filter(entry => entry.sql.includes("jsonb_to_recordset")).length).toBe(sessions.length); + for (const session of sessions) expect(await fs.lstat(path.join(stateBase, session.scope, "failover-backups", "current")).then(() => true)).toBe(true); + } finally { await worker.stop(); await coordinator.stop(); } + }, 120_000); +}); diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index c51bcbd30ff..6f10184e23e 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -16,6 +16,7 @@ import { type Db, } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; +import { assessSessionRetention } from "./native-session-retention-blockers.js"; import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js"; import { nativeSessionScopeDigestIsBusy } from "./native-session-executor.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; @@ -185,7 +186,7 @@ type Tx = Parameters[0]>[0]; async function sessionRetentionBlocker( tx: Tx, - input: { scopeDigest: string | null; identity: SessionIdentity; closedBefore: Date; checkpoint?: boolean; lock?: boolean }, + input: { scopeDigest: string | null; identity: SessionIdentity; closedBefore: Date; checkpoint?: boolean }, ): Promise<{ reason: string } | { companyId: string; agentId: string; runIds: string[]; issueIds: string[]; closedAt: number }> { const [owner] = await tx .select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId }) @@ -225,7 +226,7 @@ async function sessionRetentionBlocker( }) .from(issues) .where(and(eq(issues.companyId, owner.companyId), inArray(issues.id, issueIds))); - const issueRows = await (input.lock === false ? issueQuery : issueQuery.for("share")); + const issueRows = await issueQuery.for("share"); if (issueRows.length !== issueIds.length) return { reason: "issue_unknown" }; for (const issue of issueRows) { if (!CLOSED_ISSUE_STATUSES.includes(issue.status)) return { reason: "issue_open" }; @@ -314,6 +315,11 @@ export function nativeHarnessBackupExpirationService( .where(and(durableBackupCursor ? gt(nativeHarnessBackups.id, durableBackupCursor) : undefined, or(eq(nativeHarnessBackups.status, "retired"), lt(nativeHarnessBackups.createdAt, options.closedBefore)))) .orderBy(asc(nativeHarnessBackups.id)).limit(options.maxEntries + 1); + const assessments = await assessSessionRetention(db, candidates.slice(0, options.maxEntries) + .filter(candidate => candidate.survivingRunId && candidate.sessionId && !["retired", "pending", "superseded"].includes(candidate.status)) + .map(candidate => ({ key: candidate.id, scopeDigest: candidate.scopeDigest, + identity: { runId: candidate.runId, normalizedSessionId: candidate.sessionId!, environmentLeaseId: null } })), + { closedBefore: options.closedBefore, checkpoint: true }); let processed = 0; for (const candidate of candidates) { if (!options.shouldContinue() || processed++ >= options.maxEntries || result.expired >= options.maxRemovals) { @@ -321,6 +327,7 @@ export function nativeHarnessBackupExpirationService( } durableBackupCursor = candidate.id; if (candidate.survivingRunId && !candidate.sessionId) continue; + if (assessments.get(candidate.id)?.reason) continue; try { const retired = candidate.status === "retired" || await db.transaction(async tx => { if (["pending", "superseded"].includes(candidate.status)) { @@ -385,6 +392,11 @@ export function nativeHarnessBackupExpirationService( isNull(nativeRunFinalizations.leaseOwner), isNull(nativeRunFinalizations.recoveryState), )) .orderBy(asc(nativeRunnerCheckpoints.runId)).limit(options.maxEntries + 1); + const assessments = await assessSessionRetention(db, candidates.slice(0, options.maxEntries) + .filter(candidate => candidate.sessionId && !candidate.retirement) + .map(candidate => ({ key: candidate.runId, scopeDigest: null, + identity: { runId: candidate.runId, normalizedSessionId: candidate.sessionId!, environmentLeaseId: null } })), + { closedBefore: options.closedBefore, checkpoint: true }); let processed = 0; let removed = 0; for (const candidate of candidates) { @@ -395,6 +407,9 @@ export function nativeHarnessBackupExpirationService( processed++; checkpointCursor = candidate.runId; if (!candidate.sessionId) continue; // Unknown identity is never expiration evidence. + const assessment = assessments.get(candidate.runId); + if (assessment && (assessment.reason || assessment.resumable + || assessment.runIds.length > MAX_NATIVE_CHECKPOINT_RETIREMENT_RUNS)) continue; try { const previousRetirement = readNativeCheckpointRetirement(candidate.retirement, candidate); if (previousRetirement && previousRetirement.sessionId !== candidate.sessionId) throw new Error("native_runner_checkpoint_retirement_invalid"); @@ -861,25 +876,6 @@ export function nativeHarnessBackupExpirationService( }); } - async function backupDependencies(scopeDigest: string, identity: SessionIdentity): Promise { - const [owner] = await db.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, - issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, - }).from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)); - if (!owner) return null; - const runs = await db.select({ id: heartbeatRuns.id, - issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, - }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, owner.companyId), - eq(heartbeatRuns.agentId, owner.agentId), eq(heartbeatRuns.nativeSessionId, identity.normalizedSessionId))); - const runIds = [...new Set([identity.runId, ...runs.map(run => run.id)])]; - const issueIds = [...new Set([owner.issueId, ...runs.map(run => run.issueId)].filter((id): id is string => Boolean(id)))]; - const leases = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( - eq(environmentLeases.companyId, owner.companyId), or(inArray(environmentLeases.heartbeatRunId, runIds), - sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'sessionScopeSha256' = ${scopeDigest}`, - identity.environmentLeaseId ? eq(environmentLeases.id, identity.environmentLeaseId) : undefined))); - return { companyId: owner.companyId, agentId: owner.agentId, sessionId: identity.normalizedSessionId, - runIds, issueIds, leaseIds: leases.map(lease => lease.id) }; - } - async function resolveBackupScope(scope: NativeHarnessBackupExpirationScope) { let runId = scope.runId; let issueId = scope.issueId; @@ -910,18 +906,20 @@ export function nativeHarnessBackupExpirationService( if (!stat?.isDirectory() || stat.isSymbolicLink()) return null; const identity = await readSessionIdentity(root); if (!identity) return null; - const dependencies = await backupDependencies(scopeDigest, identity); - if (!dependencies) return null; + const assessment = (await assessSessionRetention(db, [{ key: scopeDigest, identity, scopeDigest }], + { closedBefore: "infinity", checkpoint: false })).get(scopeDigest); + if (!assessment?.companyId || !assessment.agentId) return null; + const dependencies: NativeHarnessBackupDependencies = { companyId: assessment.companyId, agentId: assessment.agentId, + sessionId: identity.normalizedSessionId, runIds: [...new Set([identity.runId, ...assessment.runIds])], + issueIds: [...new Set([assessment.ownerIssueId, ...assessment.issueIds].filter((id): id is string => Boolean(id)))], + leaseIds: assessment.leaseIds }; const source = { id: scopeDigest, companyId: dependencies.companyId, agentId: dependencies.agentId, dependencies }; if (await fs.lstat(path.join(root, "cleanup-activation.json")).then(() => true, () => false)) { return { ...source, dueAt: null }; } - const blocker = await db.transaction(tx => sessionRetentionBlocker(tx, { - scopeDigest, identity, closedBefore: new Date(8_000_000_000_000_000), lock: false, - })); - if ("reason" in blocker) return { ...source, dueAt: null }; + if (assessment.reason || assessment.closedAt === null) return { ...source, dueAt: null }; const retentionMs = Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS; - return { ...source, dueAt: Math.max(stat!.mtime.getTime(), blocker.closedAt) + retentionMs + 1 }; + return { ...source, dueAt: Math.max(stat!.mtime.getTime(), assessment.closedAt) + retentionMs + 1 }; } async function backupScopes(after?: string, limit = 100) { diff --git a/server/src/services/native-runtime/native-session-retention-blockers.ts b/server/src/services/native-runtime/native-session-retention-blockers.ts new file mode 100644 index 00000000000..9d44a386002 --- /dev/null +++ b/server/src/services/native-runtime/native-session-retention-blockers.ts @@ -0,0 +1,84 @@ +import type { Db } from "@paperclipai/db"; + +const SESSION_RETENTION_BLOCKERS_SQL = ` +with input as ( + select * from jsonb_to_recordset($1::jsonb) as i(key text, run_id uuid, session_id uuid, scope_digest text, lease_id uuid) +), owner as ( + select i.*, r.company_id, r.agent_id, coalesce(r.native_issue_id, r.issue_id) as owner_issue_id + from input i join heartbeat_runs r on r.id = i.run_id +), sessions as ( + select o.key, o.run_id, o.session_id, o.scope_digest, o.lease_id, o.company_id, o.agent_id, o.owner_issue_id, + coalesce(array_agg(r.id order by r.id) filter (where r.id is not null), '{}') as run_ids, + coalesce(array_agg(distinct coalesce(r.native_issue_id, r.issue_id)) filter (where coalesce(r.native_issue_id, r.issue_id) is not null), '{}') as issue_ids, + coalesce(bool_or(r.status in ('queued', 'scheduled_retry', 'running') or r.finished_at is null), false) as live, + coalesce(bool_or((r.runner_profile_json->'nativeControllerRecovery'->>'schema' = 'paperclip.native-controller-recovery.v1' + and r.runner_profile_json->'nativeControllerRecovery'->'checkpoint' = 'null'::jsonb) is true), false) as incomplete, + coalesce(bool_or(r.finished_at >= $2::timestamptz), false) as recent + from owner o + left join heartbeat_runs r on r.company_id = o.company_id and r.agent_id = o.agent_id and r.native_session_id = o.session_id + group by o.key, o.run_id, o.session_id, o.scope_digest, o.lease_id, o.company_id, o.agent_id, o.owner_issue_id +), assessed as ( + select s.*, + (select count(*) from issues x where x.company_id = s.company_id and x.id = any(s.issue_ids)) as issues_found, + exists (select 1 from issues x where x.company_id = s.company_id and x.id = any(s.issue_ids) + and x.status not in ('done', 'cancelled')) as issue_open, + exists (select 1 from issues x where x.company_id = s.company_id and x.id = any(s.issue_ids) + and x.execution_run_id is not null) as issue_executing, + exists (select 1 from issues x where x.company_id = s.company_id and x.id = any(s.issue_ids) + and (coalesce(x.completed_at, x.cancelled_at) is null or coalesce(x.completed_at, x.cancelled_at) >= $2::timestamptz)) as issue_recent, + (select (extract(epoch from max(coalesce(x.completed_at, x.cancelled_at))) * 1000)::float8 from issues x + where x.company_id = s.company_id and x.id = any(s.issue_ids)) as closed_at, + (select count(*) from native_run_finalizations f where f.run_id = any(s.run_ids)) as finalizations, + exists (select 1 from native_run_finalizations f where f.run_id = any(s.run_ids) + and (f.phase not in ('committed', 'terminal_failure') or f.lease_owner is not null)) as finalization_pending, + exists (select 1 from ( + select l.status from environment_leases l where l.heartbeat_run_id = any(s.run_ids) + union all select l.status from environment_leases l + where l.metadata->'nativeHarnessBackup'->>'sessionScopeSha256' = s.scope_digest + union all select l.status from environment_leases l where $3::boolean and l.company_id = s.company_id and l.issue_id = any(s.issue_ids) + union all select l.status from environment_leases l where $3::boolean and l.company_id = s.company_id + and l.metadata->'nativeHarnessBackup'->>'normalizedSessionId' = s.session_id::text + union all select l.status from environment_leases l where l.id = s.lease_id + ) referenced where referenced.status in ('active', 'released', 'retained', 'pending_cleanup')) as lease_live, + coalesce((select array_agg(distinct referenced.id order by referenced.id) from ( + select l.id from environment_leases l where l.company_id = s.company_id and l.heartbeat_run_id = any(s.run_ids || s.run_id) + union all select l.id from environment_leases l where l.company_id = s.company_id + and l.metadata->'nativeHarnessBackup'->>'sessionScopeSha256' = s.scope_digest + union all select l.id from environment_leases l where l.company_id = s.company_id and l.id = s.lease_id + ) referenced), '{}') as lease_ids, + exists (select 1 from agent_task_sessions t where t.company_id = s.company_id and t.agent_id = s.agent_id + and (t.last_run_id = any(s.run_ids) or t.session_params_json->>'sessionId' = s.session_id::text)) as resumable + from sessions s +) +select i.key, a.company_id, a.agent_id, a.run_ids, a.issue_ids, a.owner_issue_id, a.lease_ids, a.closed_at, coalesce(a.resumable, false) as resumable, + case + when a.key is null then 'run_missing' + when a.live then 'live_run' + when a.incomplete then 'native_recovery_incomplete' + when $3::boolean and a.recent then 'recent_run' + when cardinality(a.issue_ids) = 0 or a.issues_found <> cardinality(a.issue_ids) then 'issue_unknown' + when a.issue_open then 'issue_open' + when $3::boolean and a.issue_executing then 'live_run' + when a.issue_recent then 'recently_closed' + when ($3::boolean and a.finalizations <> cardinality(a.run_ids)) or a.finalization_pending then 'pending_finalization' + when a.lease_live then 'live_lease' + end as reason +from input i left join assessed a on a.key = i.key`; + +export type SessionRetentionAssessment = { + reason: string | null; companyId: string | null; agentId: string | null; runIds: string[]; issueIds: string[]; + ownerIssueId: string | null; leaseIds: string[]; closedAt: number | null; resumable: boolean; +}; + +export async function assessSessionRetention(db: Db, sessions: Array<{ key: string; identity: { runId: string; normalizedSessionId: string; environmentLeaseId: string | null }; scopeDigest: string | null }>, + options: { closedBefore: Date | "infinity"; checkpoint: boolean }): Promise> { + if (sessions.length === 0) return new Map(); + const rows = await db.$client.unsafe>( + SESSION_RETENTION_BLOCKERS_SQL, [JSON.stringify(sessions.map(session => ({ key: session.key, run_id: session.identity.runId, + session_id: session.identity.normalizedSessionId, scope_digest: session.scopeDigest, lease_id: session.identity.environmentLeaseId }))), + options.closedBefore === "infinity" ? "infinity" : options.closedBefore.toISOString(), options.checkpoint]); + return new Map(rows.map(row => [row.key, { reason: row.reason, companyId: row.company_id, agentId: row.agent_id, + runIds: row.run_ids ?? [], issueIds: row.issue_ids ?? [], ownerIssueId: row.owner_issue_id, leaseIds: row.lease_ids ?? [], + closedAt: row.closed_at === null ? null : Number(row.closed_at), resumable: row.resumable }])); +}