From 7db92ea3194f7afdd82ce0c37681203dcc4ed8f6 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:51:28 -0600 Subject: [PATCH] perf(workspaces): read a sweep page's task trees and active runs in two statements (#21618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The terminal workspace sweep inspects candidate workspaces 50 per page. For each workspace it read the task tree with its own recursive query. For many of them it also ran one or two queries to check for an active run. > - The cost of a page therefore grew with the page: about 50 to 110 statements. On the hosted instance, building the SQL text of these statements costs about as much main-thread time as parsing their results. > - The archive statement re-checks the terminal tree and the absence of a live run in the same statement, and the remote and plugin cleanups re-check active runs under the lifecycle lock. So these early reads are only pre-filters. > - This pull request reads the task trees and the active-run workspaces of a whole page in two fixed-text statements. > - The benefit is that a page costs a constant number of these reads, whatever its size. ## Linked Issues or Issue Description No issue exists. This describes the problem with the fields of `.github/ISSUE_TEMPLATE/enhancement.yml`. **What existing behavior does this improve?** The per-page reads of the terminal workspace sweep (`sweepTerminalWorkspaces` in `server/src/services/execution-workspaces.ts`). **Current behavior** Each candidate calls `listWorkspaceIssueTree`, one recursive query. A candidate that reaches the active-run checks calls `workspaceHasActiveRun`: one query for the linked tasks' run ids, then one for live runs. **Proposed behavior** - `readPageIssueTrees` reads the tree rows of every root on the page in one statement. Its SQL text is fixed: the roots are one `jsonb` parameter. The trees are then built in memory and keep the same row shape. - `readPageActiveRunWorkspaces` returns, in one fixed-text statement, the page workspaces that have a `queued` or `running` run on a linked task. - The two pre-filter checks in the loop use these results. A workspace whose source task changed earlier in the same pass (rehome) still reads its own tree. - The authoritative checks are unchanged: the archive `UPDATE` with its terminal-tree and live-run predicates, the active-run check inside the remote and plugin cleanup under the lifecycle lock, and the per-workspace check before a stale reopen flag is cleared. **Reason and benefit** One rotation pass should issue a bounded number of statements per page. For a page of 50 workspaces (40 with open trees, 10 held by a run), the tree reads fall from 50 to 1 and the active-run reads from 20 to 1. ## What Changed - `server/src/services/execution-workspaces.ts`: add `readPageIssueTrees` and `readPageActiveRunWorkspaces`, and use them for the per-page pre-filter. - `server/src/__tests__/terminal-workspace-page-batch.test.ts`: two new tests on real PostgreSQL. ## Verification - `reads the task trees and active runs of a page in a fixed number of statements`: 2 companies, 50 workspaces, each with a child task, 10 held by a `running` run. - Before the change it fails: 50 tree reads. - After the change: one tree read, one active-run read, fewer than 10 statements for the page. The results are the same (`checked: 50`, `skippedNonTerminalTree: 40`, `skippedActiveRun: 10`). - `does not archive a workspace whose run starts after the page was read`: - The page is read first. Then, during that workspace's close-readiness check, a `running` run is inserted and set as its task's checkout run. - The sweep counts the workspace `eligible: 1`, `archived: 0`, `skippedRace: 1`, and it stays `active`. - After the run succeeds, the next sweep archives it (`archived: 1`). - `npx vitest run src/__tests__/terminal-workspace-page-batch.test.ts src/__tests__/execution-workspaces-service.test.ts src/__tests__/isolated-task-directory-retention.test.ts src/__tests__/git-workspace-expiration.test.ts src/__tests__/workspace-delivery-cleanup.integration.test.ts src/__tests__/nested-clone-migration.test.ts src/__tests__/execution-workspace-compaction.test.ts src/__tests__/git-restore-point-expiration.test.ts src/__tests__/plugin-workspaces.integration.test.ts` (in `server/`). - `pnpm check:module-boundaries` passes. ## Risks - **The pre-filter reads happen once, at the start of the page.** A page can take long, because cleanups do remote and Git I/O. A run that starts mid-page on a workspace the pre-filter saw as idle is still caught by a check made at the destructive step itself, in the same statement or under a lock. Line numbers are on this head: - **Archive** of an active workspace: the archive `UPDATE` in `sweepTerminalWorkspaces` carries `NOT EXISTS (… live_run.status IN ('queued', 'running'))` (`server/src/services/execution-workspaces.ts:4291`). It changes 0 rows and the workspace counts as `skippedRace` (`:4340`). - **Remote cleanup** (destroying the sandbox): `runTerminalWorkspaceCleanup` re-runs `workspaceHasActiveRun` under the lifecycle lock before `cleanupRemoteWorkspaceAllocation` (`:2702`). - **Plugin workspace removal**: the same check under the lifecycle lock before `removePluginWorkspace` (`:2706`). - **Git cache expiration** and **isolated task directory expiry**: `isGitWorkspaceExpirationEligible` re-reads the tree and refuses any run that is not finished, inside the locked transaction (`server/src/services/git-workspace-expiration.ts:311` and `:416`, `server/src/services/isolated-task-directory-retention.ts:244` and `:290`). - A run that ends mid-page leaves its workspace for the run-release hint or the next inspection. - **Tree rows are now deduplicated** (`UNION` instead of `UNION ALL`). Parent cycles therefore cannot repeat rows. - Low risk otherwise. No schema change. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../execution-workspaces-service.test.ts | 2 +- .../terminal-workspace-page-batch.test.ts | 94 +++++++++++++++++++ server/src/services/execution-workspaces.ts | 71 +++++++++++++- 3 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 server/src/__tests__/terminal-workspace-page-batch.test.ts diff --git a/server/src/__tests__/execution-workspaces-service.test.ts b/server/src/__tests__/execution-workspaces-service.test.ts index 71341434ea9..64d2e4e1851 100644 --- a/server/src/__tests__/execution-workspaces-service.test.ts +++ b/server/src/__tests__/execution-workspaces-service.test.ts @@ -606,7 +606,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { expect(scan.mock.calls.filter(([input]) => input.operation === "execution_workspaces.close_readiness_status")) .toHaveLength(3); expect(select.mock.calls.filter(([fields]) => fields && "completedAt" in fields && "cancelledAt" in fields)) - .toHaveLength(4); + .toHaveLength(1); } finally { scan.mockRestore(); select.mockRestore(); diff --git a/server/src/__tests__/terminal-workspace-page-batch.test.ts b/server/src/__tests__/terminal-workspace-page-batch.test.ts new file mode 100644 index 00000000000..038708b7e65 --- /dev/null +++ b/server/src/__tests__/terminal-workspace-page-batch.test.ts @@ -0,0 +1,94 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, executionWorkspaces, heartbeatRuns, issues, projects, type Db } from "@paperclipai/db"; +import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_TERMINAL_WORKSPACE_PAGE_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const DAY = 86_400_000; + +(support.supported ? describe : describe.skip)("terminal workspace page reads", () => { + let temporary: Awaited> | undefined; + let db: Db; + let meter: ReturnType; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("terminal-workspace-page-"); + const url = external ?? temporary!.connectionString; + db = createDb(url); + db.$client.options.onnotice = () => {}; + meter = meterDatabaseStatements(db, url); + }, 120_000); + afterEach(async () => { await db.execute(sql`truncate companies cascade`); }); + afterAll(async () => { meter?.uninstall(); await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + async function seed(input: { open: number; held: number; companies: number }) { + const old = new Date(Date.now() - 10 * DAY); + const held: string[] = []; + for (let company = 0; company < input.companies; company++) { + const companyId = randomUUID(), projectId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Page", issuePrefix: `P${companyId.slice(0, 6)}` }); + await db.insert(projects).values({ id: projectId, companyId, name: "Project", status: "in_progress" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", role: "engineer", status: "idle", adapterType: "process" }); + for (const kind of [...Array(input.open).fill("open"), ...Array(input.held).fill("held")] as Array<"open" | "held">) { + const id = randomUUID(), issueId = randomUUID(), childId = randomUUID(); + const runId = kind === "held" ? randomUUID() : null; + if (runId) await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" }); + await db.insert(executionWorkspaces).values({ id, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", + name: id, status: "active", providerType: "local_fs", cwd: `/missing/${id}`, createdAt: old, updatedAt: old }); + await db.insert(issues).values([ + { id: issueId, companyId, projectId, title: kind, status: "done", executionWorkspaceId: id, checkoutRunId: runId, completedAt: old, updatedAt: old }, + { id: childId, companyId, projectId, parentId: issueId, title: `${kind} child`, status: kind === "open" ? "todo" : "done", + completedAt: kind === "open" ? null : old, updatedAt: old }, + ]); + await db.update(executionWorkspaces).set({ sourceIssueId: issueId, updatedAt: old }).where(eq(executionWorkspaces.id, id)); + if (kind === "held") held.push(id); + } + } + return { held }; + } + + it("does not archive a workspace whose run starts after the page was read", async () => { + await seed({ open: 3, held: 0, companies: 1 }); + const [workspace] = await db.select().from(executionWorkspaces).limit(1); + const [source] = await db.select().from(issues).where(eq(issues.id, workspace!.sourceIssueId!)); + await db.update(issues).set({ status: "done", completedAt: new Date(Date.now() - 10 * DAY) }).where(eq(issues.parentId, source!.id)); + const [agent] = await db.select().from(agents).limit(1); + let started: string | null = null; + const service = executionWorkspaceService(db, { workspaceReaperCooldownDays: 1, + inspectGitCloseReadiness: async inspected => { + if (inspected.id === workspace!.id && !started) { + started = randomUUID(); + await db.insert(heartbeatRuns).values({ id: started, companyId: workspace!.companyId, agentId: agent!.id, status: "running" }); + await db.update(issues).set({ checkoutRunId: started }).where(eq(issues.id, source!.id)); + } + return { statusInspectionSucceeded: true, warnings: [], git: { repoRoot: null, workspacePath: inspected.cwd, branchName: null, + baseRef: null, hasDirtyTrackedFiles: false, hasUntrackedFiles: false, dirtyEntryCount: 0, untrackedEntryCount: 0, + aheadCount: 0, behindCount: 0, isMergedIntoBase: true, createdByRuntime: false } }; + } }); + const result = await service.sweepTerminalWorkspaces(50); + expect(started).not.toBeNull(); + expect(result).toMatchObject({ eligible: 1, archived: 0, skippedRace: 1 }); + const [row] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace!.id)); + expect(row!.status).toBe("active"); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, started!)); + expect(await service.sweepTerminalWorkspaces(50)).toMatchObject({ archived: 1 }); + }); + + it("reads the task trees and active runs of a page in a fixed number of statements", async () => { + await seed({ open: 20, held: 5, companies: 2 }); + const service = executionWorkspaceService(db, { workspaceReaperCooldownDays: 1 }); + meter.startLog(); + const result = await service.sweepTerminalWorkspaces(50); + const statements = meter.stopLog(); + expect(result).toMatchObject({ checked: 50, skippedNonTerminalTree: 40, skippedActiveRun: 10 }); + const treeReads = statements.filter(entry => /issue_tree/i.test(entry.sql)); + const runReads = statements.filter(entry => /"checkout_run_id", "execution_run_id"|live_run/.test(entry.sql)); + expect(treeReads).toHaveLength(1); + expect(runReads).toHaveLength(1); + expect(statements.length).toBeLessThan(10); + }); +}); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 7eeb921e750..999a2424f70 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -2234,6 +2234,67 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } + type IssueTreeRow = Awaited>[number]; + + async function readPageIssueTrees(workspaces: Array>) { + const trees = new Map(); + const roots = workspaces.filter(workspace => workspace.sourceIssueId) + .map(workspace => ({ companyId: workspace.companyId, issueId: workspace.sourceIssueId! })); + if (roots.length === 0) return trees; + const rows = await retryIdempotentDatabaseOperation(() => db + .select({ id: issues.id, companyId: issues.companyId, parentId: issues.parentId, status: issues.status, + completedAt: issues.completedAt, cancelledAt: issues.cancelledAt, updatedAt: issues.updatedAt }) + .from(issues) + .where(sql`(${issues.companyId}, ${issues.id}) IN ( + WITH RECURSIVE issue_tree(company_id, id) AS ( + SELECT (root->>'companyId')::uuid, (root->>'issueId')::uuid FROM jsonb_array_elements(${JSON.stringify(roots)}::jsonb) root + UNION + SELECT child.company_id, child.id FROM ${issues} child + JOIN issue_tree parent ON child.parent_id = parent.id AND child.company_id = parent.company_id + ) + SELECT company_id, id FROM issue_tree + )`)); + const byId = new Map(rows.map(row => [row.id, row])); + const children = new Map(); + for (const row of rows) { + if (!row.parentId) continue; + const siblings = children.get(row.parentId); + if (siblings) siblings.push(row); else children.set(row.parentId, [row]); + } + for (const workspace of workspaces) { + const root = workspace.sourceIssueId ? byId.get(workspace.sourceIssueId) : undefined; + if (!root || root.companyId !== workspace.companyId) { trees.set(workspace.id, []); continue; } + const tree: IssueTreeRow[] = []; + const seen = new Set(); + const stack = [root]; + while (stack.length) { + const row = stack.pop()!; + if (seen.has(row.id)) continue; + seen.add(row.id); + tree.push({ id: row.id, status: row.status, completedAt: row.completedAt, cancelledAt: row.cancelledAt, updatedAt: row.updatedAt }); + for (const child of children.get(row.id) ?? []) if (child.companyId === workspace.companyId) stack.push(child); + } + trees.set(workspace.id, tree); + } + return trees; + } + + async function readPageActiveRunWorkspaces(workspaces: Array>) { + if (workspaces.length === 0) return new Set(); + const pairs = workspaces.map(workspace => ({ id: workspace.id, companyId: workspace.companyId, sourceIssueId: workspace.sourceIssueId })); + const rows = await db.execute<{ id: string }>(sql`SELECT workspace.id::text AS id + FROM jsonb_to_recordset(${JSON.stringify(pairs)}::jsonb) AS workspace(id uuid, "companyId" uuid, "sourceIssueId" uuid) + WHERE EXISTS ( + SELECT 1 FROM ${issues} linked + JOIN ${heartbeatRuns} live_run ON live_run.company_id = linked.company_id + AND live_run.id IN (linked.checkout_run_id, linked.execution_run_id) + AND live_run.status IN ('queued', 'running') + WHERE linked.company_id = workspace."companyId" + AND (linked.execution_workspace_id = workspace.id OR linked.id = workspace."sourceIssueId") + )`); + return new Set(rows.map(row => row.id)); + } + async function workspaceHasActiveRun(workspace: Pick) { const linkedIssues = await db .select({ @@ -3662,6 +3723,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }; if (targets) for (const id of [...targets.workspaceIds, ...candidates.map(candidate => candidate.workspace.id)]) terminalWorkspaceDeadlines.delete(id); const readinessByWorkingTree = new Map>(); + const pageWorkspaces = candidates.map(candidate => candidate.workspace); + const [pageIssueTrees, pageActiveRunWorkspaces] = await Promise.all([ + readPageIssueTrees(pageWorkspaces), readPageActiveRunWorkspaces(pageWorkspaces)]); const backOffTerminalWorkspace = async (snapshot: ExecutionWorkspaceRow, failure: TerminalWorkspaceFailure) => { const [target] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, snapshot.id)); if (!target) return; @@ -3787,7 +3851,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic await expireTaskDirectory(workspace, now()); continue; } - const issueTree = await listWorkspaceIssueTree(workspace); + const issueTree = workspace.sourceIssueId === candidate.workspace.sourceIssueId + ? pageIssueTrees.get(workspace.id) ?? await listWorkspaceIssueTree(workspace) : await listWorkspaceIssueTree(workspace); const sourceIssue = issueTree.find((issue) => issue.id === workspace.sourceIssueId); const reopenPending = metadataHasReopenPendingConsumption( workspace.metadata as Record | null, @@ -3912,7 +3977,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic result.skippedBackoff += 1; continue; } - if (await workspaceHasActiveRun(workspace)) { + if (pageActiveRunWorkspaces.has(workspace.id)) { if (reopenPending) result.skippedReopened += 1; else result.skippedActiveRun += 1; terminalWorkspaceDeadlines.delete(workspace.id); @@ -4154,7 +4219,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } continue; } - if (await workspaceHasActiveRun(workspace)) { + if (pageActiveRunWorkspaces.has(workspace.id)) { result.skippedActiveRun += 1; terminalWorkspaceDeadlines.delete(workspace.id); continue;