diff --git a/server/src/__tests__/execution-workspaces-service.test.ts b/server/src/__tests__/execution-workspaces-service.test.ts index 71341434ea9..64d2e4e1851 100644 --- a/server/src/__tests__/execution-workspaces-service.test.ts +++ b/server/src/__tests__/execution-workspaces-service.test.ts @@ -606,7 +606,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { expect(scan.mock.calls.filter(([input]) => input.operation === "execution_workspaces.close_readiness_status")) .toHaveLength(3); expect(select.mock.calls.filter(([fields]) => fields && "completedAt" in fields && "cancelledAt" in fields)) - .toHaveLength(4); + .toHaveLength(1); } finally { scan.mockRestore(); select.mockRestore(); diff --git a/server/src/__tests__/terminal-workspace-page-batch.test.ts b/server/src/__tests__/terminal-workspace-page-batch.test.ts new file mode 100644 index 00000000000..038708b7e65 --- /dev/null +++ b/server/src/__tests__/terminal-workspace-page-batch.test.ts @@ -0,0 +1,94 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, executionWorkspaces, heartbeatRuns, issues, projects, type Db } from "@paperclipai/db"; +import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_TERMINAL_WORKSPACE_PAGE_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const DAY = 86_400_000; + +(support.supported ? describe : describe.skip)("terminal workspace page reads", () => { + let temporary: Awaited> | undefined; + let db: Db; + let meter: ReturnType; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("terminal-workspace-page-"); + const url = external ?? temporary!.connectionString; + db = createDb(url); + db.$client.options.onnotice = () => {}; + meter = meterDatabaseStatements(db, url); + }, 120_000); + afterEach(async () => { await db.execute(sql`truncate companies cascade`); }); + afterAll(async () => { meter?.uninstall(); await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + async function seed(input: { open: number; held: number; companies: number }) { + const old = new Date(Date.now() - 10 * DAY); + const held: string[] = []; + for (let company = 0; company < input.companies; company++) { + const companyId = randomUUID(), projectId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Page", issuePrefix: `P${companyId.slice(0, 6)}` }); + await db.insert(projects).values({ id: projectId, companyId, name: "Project", status: "in_progress" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", role: "engineer", status: "idle", adapterType: "process" }); + for (const kind of [...Array(input.open).fill("open"), ...Array(input.held).fill("held")] as Array<"open" | "held">) { + const id = randomUUID(), issueId = randomUUID(), childId = randomUUID(); + const runId = kind === "held" ? randomUUID() : null; + if (runId) await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" }); + await db.insert(executionWorkspaces).values({ id, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", + name: id, status: "active", providerType: "local_fs", cwd: `/missing/${id}`, createdAt: old, updatedAt: old }); + await db.insert(issues).values([ + { id: issueId, companyId, projectId, title: kind, status: "done", executionWorkspaceId: id, checkoutRunId: runId, completedAt: old, updatedAt: old }, + { id: childId, companyId, projectId, parentId: issueId, title: `${kind} child`, status: kind === "open" ? "todo" : "done", + completedAt: kind === "open" ? null : old, updatedAt: old }, + ]); + await db.update(executionWorkspaces).set({ sourceIssueId: issueId, updatedAt: old }).where(eq(executionWorkspaces.id, id)); + if (kind === "held") held.push(id); + } + } + return { held }; + } + + it("does not archive a workspace whose run starts after the page was read", async () => { + await seed({ open: 3, held: 0, companies: 1 }); + const [workspace] = await db.select().from(executionWorkspaces).limit(1); + const [source] = await db.select().from(issues).where(eq(issues.id, workspace!.sourceIssueId!)); + await db.update(issues).set({ status: "done", completedAt: new Date(Date.now() - 10 * DAY) }).where(eq(issues.parentId, source!.id)); + const [agent] = await db.select().from(agents).limit(1); + let started: string | null = null; + const service = executionWorkspaceService(db, { workspaceReaperCooldownDays: 1, + inspectGitCloseReadiness: async inspected => { + if (inspected.id === workspace!.id && !started) { + started = randomUUID(); + await db.insert(heartbeatRuns).values({ id: started, companyId: workspace!.companyId, agentId: agent!.id, status: "running" }); + await db.update(issues).set({ checkoutRunId: started }).where(eq(issues.id, source!.id)); + } + return { statusInspectionSucceeded: true, warnings: [], git: { repoRoot: null, workspacePath: inspected.cwd, branchName: null, + baseRef: null, hasDirtyTrackedFiles: false, hasUntrackedFiles: false, dirtyEntryCount: 0, untrackedEntryCount: 0, + aheadCount: 0, behindCount: 0, isMergedIntoBase: true, createdByRuntime: false } }; + } }); + const result = await service.sweepTerminalWorkspaces(50); + expect(started).not.toBeNull(); + expect(result).toMatchObject({ eligible: 1, archived: 0, skippedRace: 1 }); + const [row] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace!.id)); + expect(row!.status).toBe("active"); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, started!)); + expect(await service.sweepTerminalWorkspaces(50)).toMatchObject({ archived: 1 }); + }); + + it("reads the task trees and active runs of a page in a fixed number of statements", async () => { + await seed({ open: 20, held: 5, companies: 2 }); + const service = executionWorkspaceService(db, { workspaceReaperCooldownDays: 1 }); + meter.startLog(); + const result = await service.sweepTerminalWorkspaces(50); + const statements = meter.stopLog(); + expect(result).toMatchObject({ checked: 50, skippedNonTerminalTree: 40, skippedActiveRun: 10 }); + const treeReads = statements.filter(entry => /issue_tree/i.test(entry.sql)); + const runReads = statements.filter(entry => /"checkout_run_id", "execution_run_id"|live_run/.test(entry.sql)); + expect(treeReads).toHaveLength(1); + expect(runReads).toHaveLength(1); + expect(statements.length).toBeLessThan(10); + }); +}); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 7eeb921e750..999a2424f70 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -2234,6 +2234,67 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } + type IssueTreeRow = Awaited>[number]; + + async function readPageIssueTrees(workspaces: Array>) { + const trees = new Map(); + const roots = workspaces.filter(workspace => workspace.sourceIssueId) + .map(workspace => ({ companyId: workspace.companyId, issueId: workspace.sourceIssueId! })); + if (roots.length === 0) return trees; + const rows = await retryIdempotentDatabaseOperation(() => db + .select({ id: issues.id, companyId: issues.companyId, parentId: issues.parentId, status: issues.status, + completedAt: issues.completedAt, cancelledAt: issues.cancelledAt, updatedAt: issues.updatedAt }) + .from(issues) + .where(sql`(${issues.companyId}, ${issues.id}) IN ( + WITH RECURSIVE issue_tree(company_id, id) AS ( + SELECT (root->>'companyId')::uuid, (root->>'issueId')::uuid FROM jsonb_array_elements(${JSON.stringify(roots)}::jsonb) root + UNION + SELECT child.company_id, child.id FROM ${issues} child + JOIN issue_tree parent ON child.parent_id = parent.id AND child.company_id = parent.company_id + ) + SELECT company_id, id FROM issue_tree + )`)); + const byId = new Map(rows.map(row => [row.id, row])); + const children = new Map(); + for (const row of rows) { + if (!row.parentId) continue; + const siblings = children.get(row.parentId); + if (siblings) siblings.push(row); else children.set(row.parentId, [row]); + } + for (const workspace of workspaces) { + const root = workspace.sourceIssueId ? byId.get(workspace.sourceIssueId) : undefined; + if (!root || root.companyId !== workspace.companyId) { trees.set(workspace.id, []); continue; } + const tree: IssueTreeRow[] = []; + const seen = new Set(); + const stack = [root]; + while (stack.length) { + const row = stack.pop()!; + if (seen.has(row.id)) continue; + seen.add(row.id); + tree.push({ id: row.id, status: row.status, completedAt: row.completedAt, cancelledAt: row.cancelledAt, updatedAt: row.updatedAt }); + for (const child of children.get(row.id) ?? []) if (child.companyId === workspace.companyId) stack.push(child); + } + trees.set(workspace.id, tree); + } + return trees; + } + + async function readPageActiveRunWorkspaces(workspaces: Array>) { + if (workspaces.length === 0) return new Set(); + const pairs = workspaces.map(workspace => ({ id: workspace.id, companyId: workspace.companyId, sourceIssueId: workspace.sourceIssueId })); + const rows = await db.execute<{ id: string }>(sql`SELECT workspace.id::text AS id + FROM jsonb_to_recordset(${JSON.stringify(pairs)}::jsonb) AS workspace(id uuid, "companyId" uuid, "sourceIssueId" uuid) + WHERE EXISTS ( + SELECT 1 FROM ${issues} linked + JOIN ${heartbeatRuns} live_run ON live_run.company_id = linked.company_id + AND live_run.id IN (linked.checkout_run_id, linked.execution_run_id) + AND live_run.status IN ('queued', 'running') + WHERE linked.company_id = workspace."companyId" + AND (linked.execution_workspace_id = workspace.id OR linked.id = workspace."sourceIssueId") + )`); + return new Set(rows.map(row => row.id)); + } + async function workspaceHasActiveRun(workspace: Pick) { const linkedIssues = await db .select({ @@ -3662,6 +3723,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }; if (targets) for (const id of [...targets.workspaceIds, ...candidates.map(candidate => candidate.workspace.id)]) terminalWorkspaceDeadlines.delete(id); const readinessByWorkingTree = new Map>(); + const pageWorkspaces = candidates.map(candidate => candidate.workspace); + const [pageIssueTrees, pageActiveRunWorkspaces] = await Promise.all([ + readPageIssueTrees(pageWorkspaces), readPageActiveRunWorkspaces(pageWorkspaces)]); const backOffTerminalWorkspace = async (snapshot: ExecutionWorkspaceRow, failure: TerminalWorkspaceFailure) => { const [target] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, snapshot.id)); if (!target) return; @@ -3787,7 +3851,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic await expireTaskDirectory(workspace, now()); continue; } - const issueTree = await listWorkspaceIssueTree(workspace); + const issueTree = workspace.sourceIssueId === candidate.workspace.sourceIssueId + ? pageIssueTrees.get(workspace.id) ?? await listWorkspaceIssueTree(workspace) : await listWorkspaceIssueTree(workspace); const sourceIssue = issueTree.find((issue) => issue.id === workspace.sourceIssueId); const reopenPending = metadataHasReopenPendingConsumption( workspace.metadata as Record | null, @@ -3912,7 +3977,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic result.skippedBackoff += 1; continue; } - if (await workspaceHasActiveRun(workspace)) { + if (pageActiveRunWorkspaces.has(workspace.id)) { if (reopenPending) result.skippedReopened += 1; else result.skippedActiveRun += 1; terminalWorkspaceDeadlines.delete(workspace.id); @@ -4154,7 +4219,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } continue; } - if (await workspaceHasActiveRun(workspace)) { + if (pageActiveRunWorkspaces.has(workspace.id)) { result.skippedActiveRun += 1; terminalWorkspaceDeadlines.delete(workspace.id); continue;