diff --git a/server/src/__tests__/execution-workspaces-service.test.ts b/server/src/__tests__/execution-workspaces-service.test.ts index 48a39517fb6..71341434ea9 100644 --- a/server/src/__tests__/execution-workspaces-service.test.ts +++ b/server/src/__tests__/execution-workspaces-service.test.ts @@ -675,6 +675,50 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { expect(await service.nextTerminalWorkspaceSweepAt()).toBeNull(); }); + it("inspects only the workspaces a hint reaches, however many trees stay open", async () => { + const open = [await seedTerminalWorkspace({ childStatus: "todo" }), await seedTerminalWorkspace({ childStatus: "todo" }), + await seedTerminalWorkspace({ childStatus: "todo" })]; + const seed = await seedTerminalWorkspace({ mergedPr: true, childStatus: "done" }); + const [child] = await db.select().from(issues).where(eq(issues.parentId, seed.sourceIssueId)); + const service = executionWorkspaceService(db, { + workspaceReaperCooldownDays: 0, + resolvePullRequestDetails: async () => ({ state: "merged", headRef: "PAP-16015-delivery", headSha: seed.headSha }), + }); + const original = db.select.bind(db); + let treeReads = 0; + const select = vi.spyOn(db, "select").mockImplementation(((fields: any) => { + if (fields?.completedAt && fields?.cancelledAt) treeReads += 1; + return original(fields); + }) as typeof db.select); + try { + expect(await service.sweepTerminalWorkspaces(50, undefined, { targets: { workspaceIds: [], issueIds: [child!.id] } })) + .toMatchObject({ checked: 1, archived: 1 }); + expect(treeReads).toBe(1); + expect(await service.sweepTerminalWorkspaces(50, undefined, { + targets: { workspaceIds: [open[0]!.executionWorkspaceId], issueIds: [] } })).toMatchObject({ checked: 1, skippedNonTerminalTree: 1 }); + expect(treeReads).toBe(2); + } finally { select.mockRestore(); } + }); + + it("hints the former parent and workspace when a task moves or is removed", async () => { + const seed = await seedTerminalWorkspace({ childStatus: "todo" }); + const [child] = await db.select().from(issues).where(eq(issues.parentId, seed.sourceIssueId)); + await db.update(issues).set({ executionWorkspaceId: seed.executionWorkspaceId }).where(eq(issues.id, child!.id)); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.terminalWorkspaces, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await issueService(db).update(child!.id, { parentId: null }); + expect(hints).toEqual(expect.arrayContaining([`issue:${child!.id}`, `issue:${seed.sourceIssueId}`])); + await db.update(issues).set({ parentId: seed.sourceIssueId }).where(eq(issues.id, child!.id)); + hints.length = 0; + await issueService(db).remove(child!.id); + expect(hints).toEqual(expect.arrayContaining([`issue:${child!.id}`, `issue:${seed.sourceIssueId}`, + `workspace:${seed.executionWorkspaceId}`])); + } finally { unsubscribe(); } + }); + it("retries unread rows from a failed full page before it can park recovery", async () => { const seed = await seedTerminalWorkspace({ mergedPr: true }); await seedTerminalWorkspace({ childStatus: "todo" }); diff --git a/server/src/__tests__/terminal-workspace-work.test.ts b/server/src/__tests__/terminal-workspace-work.test.ts index e44d37f14a6..4ac80c30906 100644 --- a/server/src/__tests__/terminal-workspace-work.test.ts +++ b/server/src/__tests__/terminal-workspace-work.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; -import { notifyTerminalWorkspaceWork, registerTerminalWorkspaceWork, TERMINAL_WORKSPACE_RETRY_MS } from "../services/terminal-workspace-work.js"; +import { notifyTerminalWorkspaceWork, registerTerminalWorkspaceWork, TERMINAL_WORKSPACE_RETRY_MS, type TerminalWorkspaceTargets } from "../services/terminal-workspace-work.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; const coordinators: ReturnType[] = []; @@ -22,7 +22,7 @@ function setup() { }); coordinators.push(coordinator); const service = { - sweepTerminalWorkspaces: vi.fn(async (_limit?: number, _signal?: AbortSignal, _options?: { compact?: boolean; allowRemoteDeliveryChecks?: boolean }) => ({ checked: 0 })), + sweepTerminalWorkspaces: vi.fn(async (_limit?: number, _signal?: AbortSignal, _options?: { compact?: boolean; allowRemoteDeliveryChecks?: boolean; targets?: TerminalWorkspaceTargets }) => ({ checked: 0 })), hasTerminalWorkspaceSweepContinuation: vi.fn(() => false), nextTerminalWorkspaceSweepAt: vi.fn(async (): Promise => null), }; @@ -107,6 +107,55 @@ describe("terminal workspace work", () => { expect(vi.getTimerCount()).toBe(0); }); + it("inspects only hinted workspaces after a commit, and keeps unscoped commits and deadlines on the full rotation", async () => { + const s = setup(), due = Date.now() + 60_000; + s.service.nextTerminalWorkspaceSweepAt.mockResolvedValue(due); + await s.start().ready; + const workspaceId = "6b1f9c2e-8a4d-4f3b-9c1e-2d7a5b8e0f13", issueId = "0c9e7d6b-5a4f-4e3d-8c2b-1a0f9e8d7c6b"; + await s.owner.transaction(async tx => { + await notifyTerminalWorkspaceWork(tx, `workspace:${workspaceId}`); + await notifyTerminalWorkspaceWork(tx, `issue:${issueId}`); + }); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(2); + expect(s.service.sweepTerminalWorkspaces.mock.calls[1]?.[2]).toMatchObject({ targets: { workspaceIds: [workspaceId], issueIds: [issueId] } }); + await s.owner.transaction(tx => notifyTerminalWorkspaceWork(tx)); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(3); + expect(s.service.sweepTerminalWorkspaces.mock.calls[2]?.[2]?.targets).toBeUndefined(); + await vi.advanceTimersByTimeAsync(due - Date.now()); + expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(4); + expect(s.service.sweepTerminalWorkspaces.mock.calls[3]?.[2]?.targets).toBeUndefined(); + }); + + it("falls back to the full rotation when more workspaces are hinted than one inspection reads", async () => { + const s = setup(); + await s.start().ready; + const ids = Array.from({ length: 51 }, (_, index) => `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`); + await s.owner.transaction(async tx => { for (const id of ids) await notifyTerminalWorkspaceWork(tx, `workspace:${id}`); }); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(2); + expect(s.service.sweepTerminalWorkspaces.mock.calls[1]?.[2]?.targets).toBeUndefined(); + }); + + it("keeps the pause between rotation pages while hinted workspaces wait", async () => { + const s = setup(); + s.service.hasTerminalWorkspaceSweepContinuation.mockReturnValue(true); + s.service.sweepTerminalWorkspaces.mockImplementation(async (_limit, _signal, options) => { + if (options?.compact && !options.targets) await s.owner.transaction(tx => notifyTerminalWorkspaceWork(tx, "workspace:6b1f9c2e-8a4d-4f3b-9c1e-2d7a5b8e0f13")); + return { checked: 1 }; + }); + await s.start().ready; + await vi.advanceTimersByTimeAsync(1_000); + const pageAt = Date.now(); + await vi.advanceTimersByTimeAsync(1); + expect(s.service.sweepTerminalWorkspaces.mock.calls.filter(call => !call[2]?.targets)).toHaveLength(2); + expect(s.service.sweepTerminalWorkspaces.mock.calls.at(-1)?.[2]?.targets).toEqual({ + workspaceIds: ["6b1f9c2e-8a4d-4f3b-9c1e-2d7a5b8e0f13"], issueIds: [] }); + expect(s.coordinator.nextWakeAt()).toBe(pageAt + 1_000); + s.service.hasTerminalWorkspaceSweepContinuation.mockReturnValue(false); + }); + it("finishes bounded recovery pages then retries only existing overdue work", async () => { const s = setup(); s.service.hasTerminalWorkspaceSweepContinuation.mockReturnValueOnce(true); diff --git a/server/src/index.ts b/server/src/index.ts index a7cea216ccf..8da56303243 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1705,8 +1705,8 @@ async function startServerWithDatabaseTeardown( let lastTerminalWorkspaceSkipLogAt = 0; const terminalWorkspaceWorker = registerTerminalWorkspaceWork(deliveryWork, { ...terminalWorkspaces, - terminalWorkspaceWorkSettled: (hints) => { - terminalWorkspaces.terminalWorkspaceWorkSettled?.(hints); + terminalWorkspaceWorkSettled: (hints, options) => { + terminalWorkspaces.terminalWorkspaceWorkSettled?.(hints, options); gitRestorePointWorker.issuesChanged(); }, sweepTerminalWorkspaces: async (limit, signal, options) => { diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 0f0accc81e8..bb411786a65 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -39,7 +39,7 @@ import { authorizationService, executionWorkspaceReadSqlCondition, projectReadSq import type { SQL } from "drizzle-orm"; import { nativeSessionProviderEvidence } from "./native-runtime/native-session-resume.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; -import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; +import { notifyTerminalWorkspaceWork, terminalWorkspaceTargetsFromHints, type TerminalWorkspaceTargets } from "./terminal-workspace-work.js"; import { clearTerminalWorkspaceRetry, readTerminalWorkspaceRetry, recordTerminalWorkspaceFailure, terminalWorkspaceFailure, TERMINAL_WORKSPACE_RETRY_KEY, type TerminalWorkspaceFailure, @@ -269,6 +269,20 @@ export function clearMetadataReopenPendingConsumption( return next; } +function terminalWorkspaceTargetFilter(targets: TerminalWorkspaceTargets) { + const ids = (values: string[]) => sql.join(values.map(id => sql`${id}::uuid`), sql`, `); + const byWorkspace = targets.workspaceIds.length ? sql`${executionWorkspaces.id} in (${ids(targets.workspaceIds)})` : undefined; + const byIssue = targets.issueIds.length ? sql`(${executionWorkspaces.sourceIssueId} in ( + with recursive ancestry(id, parent_id) as ( + select id, parent_id from ${issues} where id in (${ids(targets.issueIds)}) + union + select parent.id, parent.parent_id from ${issues} parent inner join ancestry on parent.id = ancestry.parent_id + ) select id from ancestry) + or ${executionWorkspaces.id} in (select execution_workspace_id from ${issues} + where id in (${ids(targets.issueIds)}) and execution_workspace_id is not null))` : undefined; + return or(byWorkspace, byIssue) ?? sql`false`; +} + const workspaceSourceKeys = ["providerType", "cwd", "providerRef", "projectId", "projectWorkspaceId", "sourceIssueId", "mode", "strategyType", "branchName", "repoUrl", "baseRef"] as const; function preserveWorkspaceMetadataForPatch( @@ -3518,7 +3532,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // `compact: false` skips workspace compaction and compaction-trash deletion // for this sweep only. Pending compactions keep their deadlines, so a later // sweep runs them. - sweepTerminalWorkspaces: async (limit = 50, signal?: AbortSignal, options: { compact?: boolean; allowRemoteDeliveryChecks?: boolean } = {}) => { + sweepTerminalWorkspaces: async (limit = 50, signal?: AbortSignal, options: { compact?: boolean; allowRemoteDeliveryChecks?: boolean; targets?: TerminalWorkspaceTargets } = {}) => { // Skip this sweep while another sweep runs. A concurrent sweep would share // the cursor and the boundary and could corrupt the rotation state. A // skipped tick is safe: the next tick runs the sweep with intact state. @@ -3567,13 +3581,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // predicate uses the same (updatedAt, id) order as the query, so each // sweep advances past the rows it already inspected instead of re-reading // the oldest ineligible candidates. - const cursor = terminalSweepCursor; + const targets = options.targets; + const cursor = targets ? null : terminalSweepCursor; // Freeze an upper bound on updatedAt at the start of each rotation. Without // a bound, a steady stream of newer candidates keeps every page full, so // the cursor never reaches the end and never resets, and older candidates // that became eligible are never revisited. The frozen bound makes the // rotation cover a finite set, so the cursor always reaches a short page. - if (!cursor) { + if (!targets && !cursor) { terminalSweepBoundary = now(); terminalWorkspaceDeadlines.clear(); isolatedRehomeDeadlines.clear(); @@ -3606,7 +3621,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic const cursorFilter = cursor ? sql`(${executionWorkspaces.updatedAt}, ${executionWorkspaces.id}) > (${cursor.updatedAt}::timestamptz, ${cursor.id}::uuid)` : undefined; - const scanFilter = and(baseCandidateFilter, boundaryFilter, cursorFilter); + const scanFilter = targets ? and(baseCandidateFilter, terminalWorkspaceTargetFilter(targets)) + : and(baseCandidateFilter, boundaryFilter, cursorFilter); const candidates = await db .select({ workspace: executionWorkspaces, cursorAt: sql`${executionWorkspaces.updatedAt}::text` }) .from(executionWorkspaces) @@ -3616,10 +3632,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // Advance the cursor to this page's last row. A short page means the scan // reached the end of the bounded candidate set, so reset the cursor and // the bound to start a new rotation on the next sweep. - if (candidates.length < limit) { + if (targets && candidates.length >= limit) terminalSweepRescanRequested = true; + if (!targets && candidates.length < limit) { terminalSweepCursor = null; terminalSweepBoundary = null; - } else { + } else if (!targets) { const lastCandidate = candidates[candidates.length - 1]!; terminalSweepCursor = { updatedAt: lastCandidate.cursorAt, id: lastCandidate.workspace.id }; } @@ -4306,8 +4323,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } return result; } catch (error) { - terminalSweepCursor = previousCursor; - terminalSweepBoundary = previousBoundary; + if (!options.targets) { + terminalSweepCursor = previousCursor; + terminalSweepBoundary = previousBoundary; + } terminalSweepRescanRequested = true; throw error; } finally { @@ -4317,8 +4336,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic hasTerminalWorkspaceSweepContinuation: () => terminalSweepCursor !== null || terminalSweepRescanRequested, - terminalWorkspaceWorkSettled: (hints?: ReadonlySet) => { - if (terminalSweepInProgress || terminalSweepCursor !== null) terminalSweepRescanRequested = true; + terminalWorkspaceWorkSettled: (hints?: ReadonlySet, options: { targeted?: boolean } = {}) => { + if ((terminalSweepInProgress || terminalSweepCursor !== null) && !(options.targeted && terminalWorkspaceTargetsFromHints(hints))) { + terminalSweepRescanRequested = true; + } pullRequestStateCache.clear(); if (opts.taskDirectoryRetention) { if (!hints) taskDirectoryRowSkips.clear(); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index dfa75c8734c..c2cf51ce77a 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -12184,6 +12184,9 @@ export function issueService(db: Db) { || patch.completedAt !== undefined || patch.cancelledAt !== undefined || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + if (issueData.parentId !== undefined && receiptExisting.parentId && receiptExisting.parentId !== issueData.parentId) { + await notifyTerminalWorkspaceWork(tx, `issue:${receiptExisting.parentId}`); + } } await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId }); if (patch.status !== undefined || patch.hiddenAt !== undefined) { @@ -12647,7 +12650,8 @@ export function issueService(db: Db) { remove: (id: string, actorAgentId?: string | null) => db.transaction(async (tx) => { - const [task] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); + const [task] = await tx.select({ companyId: issues.companyId, parentId: issues.parentId, + executionWorkspaceId: issues.executionWorkspaceId }).from(issues).where(eq(issues.id, id)).for("update"); if (task) await assertAgentInteraction(tx, { companyId: task.companyId, actorAgentId, operation: "edit", issueId: id, lock: true }); const attachmentAssetIds = await tx @@ -12662,6 +12666,8 @@ export function issueService(db: Db) { let removedIssue; try { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + if (task?.parentId) await notifyTerminalWorkspaceWork(tx, `issue:${task.parentId}`); + if (task?.executionWorkspaceId) await notifyTerminalWorkspaceWork(tx, `workspace:${task.executionWorkspaceId}`); await requestRestorePointDeletion(tx, { sourceIssueId: id }); const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { diff --git a/server/src/services/terminal-workspace-work.ts b/server/src/services/terminal-workspace-work.ts index 0a479d459f5..d7248aae517 100644 --- a/server/src/services/terminal-workspace-work.ts +++ b/server/src/services/terminal-workspace-work.ts @@ -2,22 +2,40 @@ import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator. import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export const TERMINAL_WORKSPACE_RETRY_MS = 5 * 60 * 1000; +export const MAX_TERMINAL_WORKSPACE_TARGETS = 50; export function notifyTerminalWorkspaceWork(transaction: object, hint?: string) { return notifyDeliveryWork(transaction, DELIVERY_QUEUES.terminalWorkspaces, hint); } +export type TerminalWorkspaceTargets = { workspaceIds: string[]; issueIds: string[] }; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export function terminalWorkspaceTargetsFromHints(hints?: ReadonlySet): TerminalWorkspaceTargets | null { + if (!hints || hints.size === 0) return null; + const targets: TerminalWorkspaceTargets = { workspaceIds: [], issueIds: [] }; + for (const hint of hints) { + const [kind, id] = hint.split(":", 2); + if (!id || !UUID.test(id) || `${kind}:${id}` !== hint) return null; + if (kind === "workspace") targets.workspaceIds.push(id); + else if (kind === "issue") targets.issueIds.push(id); + else return null; + } + return targets; +} + export function registerTerminalWorkspaceWork( coordinator: ReturnType, service: { sweepTerminalWorkspaces( limit?: number, signal?: AbortSignal, - options?: { compact?: boolean; allowRemoteDeliveryChecks?: boolean }, + options?: { compact?: boolean; allowRemoteDeliveryChecks?: boolean; targets?: TerminalWorkspaceTargets }, ): Promise<{ checked: number }>; nextTerminalWorkspaceSweepAt(): Promise; hasTerminalWorkspaceSweepContinuation(): boolean; - terminalWorkspaceWorkSettled?(hints?: ReadonlySet): void; + terminalWorkspaceWorkSettled?(hints?: ReadonlySet, options?: { targeted?: boolean }): void; }, canRun: () => boolean, ) { @@ -28,23 +46,56 @@ export function registerTerminalWorkspaceWork( // can take seconds per workspace, so the first sweep skips it and later // sweeps do it. let startupSweepStarted = false; + let rotationRequested = false; + let dueAt: number | null = null; + let lastRotationPageAt = 0; + const targetWorkspaceIds = new Set(), targetIssueIds = new Set(); const worker = coordinator.register(DELIVERY_QUEUES.terminalWorkspaces, { retryMs: TERMINAL_WORKSPACE_RETRY_MS, - onSettled: service.terminalWorkspaceWorkSettled, - run: signal => { + onSettled: hints => { + const targets = terminalWorkspaceTargetsFromHints(hints); + for (const id of targets?.workspaceIds ?? []) targetWorkspaceIds.add(id); + for (const id of targets?.issueIds ?? []) targetIssueIds.add(id); + const overflow = targetWorkspaceIds.size + targetIssueIds.size > MAX_TERMINAL_WORKSPACE_TARGETS; + service.terminalWorkspaceWorkSettled?.(hints, { targeted: Boolean(targets) && !overflow }); + if (!targets || overflow) { + rotationRequested = true; + targetWorkspaceIds.clear(); targetIssueIds.clear(); + } + }, + run: async signal => { recovered = false; + const due = dueAt !== null && dueAt <= Date.now(); + dueAt = null; next = Date.now() + TERMINAL_WORKSPACE_RETRY_MS; const startup = !startupSweepStarted; startupSweepStarted = true; - return service.sweepTerminalWorkspaces(50, signal, { compact: !startup, allowRemoteDeliveryChecks: !startup }); + const options = { compact: !startup, allowRemoteDeliveryChecks: !startup }; + const targeted = targetWorkspaceIds.size + targetIssueIds.size > 0; + if (startup || due || rotationRequested || !targeted + || (service.hasTerminalWorkspaceSweepContinuation() && Date.now() >= lastRotationPageAt + 1000)) { + rotationRequested = false; + lastRotationPageAt = Date.now(); + return service.sweepTerminalWorkspaces(50, signal, options); + } + const targets = { workspaceIds: [...targetWorkspaceIds], issueIds: [...targetIssueIds] }; + targetWorkspaceIds.clear(); targetIssueIds.clear(); + try { + return await service.sweepTerminalWorkspaces(50, signal, { ...options, targets }); + } catch (error) { + rotationRequested = true; + throw error; + } }, hasPending: async () => { if (!canRun()) next = Date.now() + TERMINAL_WORKSPACE_RETRY_MS; - else if (service.hasTerminalWorkspaceSweepContinuation()) next = Date.now() + 1000; + else if (service.hasTerminalWorkspaceSweepContinuation()) next = Math.max(Date.now(), lastRotationPageAt + 1000); + else if (rotationRequested || targetWorkspaceIds.size + targetIssueIds.size > 0) next = Date.now(); else { const deadline = await service.nextTerminalWorkspaceSweepAt(); next = deadline === null ? null : deadline > Date.now() ? deadline : Date.now() + TERMINAL_WORKSPACE_RETRY_MS; + dueAt = next; recovered = true; } return false;