diff --git a/server/src/__tests__/terminal-workspace-rotation-limits.test.ts b/server/src/__tests__/terminal-workspace-rotation-limits.test.ts new file mode 100644 index 00000000000..892ce9624f5 --- /dev/null +++ b/server/src/__tests__/terminal-workspace-rotation-limits.test.ts @@ -0,0 +1,172 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { companies, createDb, executionWorkspaceSnapshots, executionWorkspaces, issues, projects, type Db } from "@paperclipai/db"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js"; +import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { + notifyTerminalWorkspaceWork, registerTerminalWorkspaceWork, TERMINAL_WORKSPACE_BACKSTOP_MS, TERMINAL_WORKSPACE_PLUGIN_HINT, + TERMINAL_WORKSPACE_RETRY_MS, +} from "../services/terminal-workspace-work.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_TERMINAL_WORKSPACE_ROTATION_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const HOUR = 3_600_000; +const realSetTimeout = globalThis.setTimeout; +const realSleep = (ms: number) => new Promise(resolve => realSetTimeout(resolve, ms)); + +(support.supported ? describe : describe.skip)("terminal workspace full rotation limits", () => { + let temporary: Awaited> | undefined; + let db: Db; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("terminal-workspace-rotation-"); + db = createDb(external ?? temporary!.connectionString); + db.$client.options.onnotice = () => {}; + }, 120_000); + afterEach(async () => { vi.useRealTimers(); await db.execute(sql`truncate companies cascade`); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + async function seed(input: { remote: number; plugin?: number }) { + const companyId = randomUUID(), projectId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Rotation", issuePrefix: `R${companyId.slice(0, 6)}` }); + await db.insert(projects).values({ id: projectId, companyId, name: "Project", status: "in_progress" }); + const old = new Date(Date.now() - 5 * 24 * HOUR); + const remote = Array.from({ length: input.remote }, () => ({ workspaceId: randomUUID(), issueId: randomUUID(), snapshotId: randomUUID() })); + const plugin = Array.from({ length: input.plugin ?? 0 }, () => ({ workspaceId: randomUUID(), issueId: randomUUID() })); + for (let index = 0; index < remote.length; index += 250) { + const page = remote.slice(index, index + 250); + await db.insert(executionWorkspaces).values(page.map(row => ({ id: row.workspaceId, companyId, projectId, mode: "isolated_workspace", + strategyType: "git_worktree", name: row.workspaceId, status: "active", providerType: "remote", createdAt: old, updatedAt: old, + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: row.snapshotId, source: { kind: "empty" }, + writerRunId: null, dirty: false, location: { environmentId: randomUUID(), leaseId: randomUUID(), + providerLeaseId: `sandbox-${row.workspaceId}`, remoteCwd: "/workspace" } } } }))); + await db.insert(executionWorkspaceSnapshots).values(page.map(row => ({ id: row.snapshotId, companyId, executionWorkspaceId: row.workspaceId, + sourceOperationId: randomUUID(), generation: 1, epoch: 1, status: "committed" as const }))); + } + if (plugin.length) await db.insert(executionWorkspaces).values(plugin.map(row => ({ id: row.workspaceId, companyId, projectId, + mode: "isolated_workspace", strategyType: "git_worktree", name: row.workspaceId, status: "active", providerType: "plugin", + createdAt: old, updatedAt: old }))); + const all = [...remote, ...plugin]; + for (let index = 0; index < all.length; index += 250) { + const page = all.slice(index, index + 250); + await db.insert(issues).values(page.map(row => ({ id: row.issueId, companyId, projectId, title: "done", status: "done", + executionWorkspaceId: row.workspaceId, completedAt: old, updatedAt: old }))); + } + await db.execute(sql`update execution_workspaces set source_issue_id = issues.id, updated_at = ${old.toISOString()}::timestamptz + from issues where issues.execution_workspace_id = execution_workspaces.id and issues.company_id = ${companyId}::uuid`); + return { companyId, remote, plugin }; + } + + function start(canRun: () => boolean = () => true) { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const service = executionWorkspaceService(db, { workspaceReaperCooldownDays: 1, terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS }); + const sweeps: Array<{ targeted: boolean; checked: number; workspaceIds: number }> = []; + let inFlight = 0; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun, onError: () => {} }); + const worker = registerTerminalWorkspaceWork(coordinator, { ...service, + sweepTerminalWorkspaces: async (limit, signal, options) => { + inFlight += 1; + const result = await service.sweepTerminalWorkspaces(limit, signal, options).finally(() => { inFlight -= 1; }); + sweeps.push({ targeted: Boolean(options?.targets), checked: result.checked, + workspaceIds: (options?.targets?.workspaceIds.length ?? 0) + (options?.targets?.issueIds.length ?? 0) }); + return result; + } }, canRun); + const advance = async (ms: number) => { + for (let elapsed = 0; elapsed < ms; elapsed += 10_000) { + await vi.advanceTimersByTimeAsync(10_000); + do { await realSleep(10); await vi.advanceTimersByTimeAsync(1); } while (inFlight > 0); + } + }; + const settle = async (operation: Promise) => { + let done = false; + const result = operation.finally(() => { done = true; }); + while (!done) { await realSleep(5); await vi.advanceTimersByTimeAsync(1); } + return result; + }; + const fullChecked = (from = 0) => sweeps.slice(from).filter(sweep => !sweep.targeted).reduce((sum, sweep) => sum + sweep.checked, 0); + return { sweeps, advance, settle, fullChecked, stop: async () => { vi.useRealTimers(); await worker.stop(); await coordinator.stop(); } }; + } + + it("works through a burst of run-finish hints in targeted batches without a full pass", async () => { + const f = await seed({ remote: 300 }); + const work = start(); + try { + await work.advance(60_000); + expect(work.fullChecked()).toBe(300); + const mark = work.sweeps.length; + const burst = f.remote.slice(0, 120); + for (let index = 0; index < burst.length; index += 40) { + await work.settle(db.transaction(async tx => { + for (const row of burst.slice(index, index + 40)) await notifyTerminalWorkspaceWork(tx, `issue:${row.issueId}`); + })); + } + await work.advance(5 * 60_000); + const window = work.sweeps.slice(mark); + expect(work.fullChecked(mark)).toBe(0); + expect(window.every(sweep => sweep.targeted && sweep.workspaceIds <= 50)).toBe(true); + expect(window.reduce((sum, sweep) => sum + sweep.checked, 0)).toBe(120); + } finally { await work.stop(); } + }, 300_000); + + it("arms the hourly backstop for a hint without a target instead of starting a pass", async () => { + await seed({ remote: 120 }); + const work = start(); + try { + await work.advance(60_000); + const mark = work.sweeps.length; + await work.settle(db.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.terminalWorkspaces))); + await work.settle(db.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.terminalWorkspaces, "not-a-target"))); + await work.advance(30 * 60_000); + expect(work.fullChecked(mark)).toBe(0); + await work.advance(TERMINAL_WORKSPACE_BACKSTOP_MS); + expect(work.fullChecked(mark)).toBe(120); + } finally { await work.stop(); } + }, 300_000); + + it("inspects plugin workspaces when a workspace provider becomes ready without a full pass", async () => { + await seed({ remote: 120, plugin: 3 }); + const work = start(); + try { + await work.advance(60_000); + const mark = work.sweeps.length; + await work.settle(db.transaction(tx => notifyTerminalWorkspaceWork(tx, TERMINAL_WORKSPACE_PLUGIN_HINT))); + await work.advance(60_000); + expect(work.fullChecked(mark)).toBe(0); + expect(work.sweeps.slice(mark).reduce((sum, sweep) => sum + sweep.checked, 0)).toBe(3); + } finally { await work.stop(); } + }, 300_000); + + it("runs one full pass across a restart: none while the old instance drains and one at the new start", async () => { + const f = await seed({ remote: 200, plugin: 2 }); + let draining = false; + const old = start(() => !draining); + try { + await old.advance(60_000); + const mark = old.sweeps.length; + draining = true; + for (let index = 0; index < 120; index += 40) { + await old.settle(db.transaction(async tx => { + for (const row of f.remote.slice(index, index + 40)) await notifyTerminalWorkspaceWork(tx, `issue:${row.issueId}`); + })); + } + await old.settle(db.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.terminalWorkspaces))); + await old.advance(5 * 60_000); + expect(old.sweeps.slice(mark)).toEqual([]); + } finally { await old.stop(); } + const fresh = start(); + try { + await fresh.advance(10_000); + await fresh.settle(db.transaction(tx => notifyTerminalWorkspaceWork(tx, TERMINAL_WORKSPACE_PLUGIN_HINT))); + for (let index = 0; index < 120; index += 40) { + await fresh.settle(db.transaction(async tx => { + for (const row of f.remote.slice(index, index + 40)) await notifyTerminalWorkspaceWork(tx, `issue:${row.issueId}`); + })); + } + await fresh.advance(30 * 60_000); + expect(fresh.fullChecked()).toBe(202); + } finally { await fresh.stop(); } + }, 300_000); +}); diff --git a/server/src/__tests__/terminal-workspace-work.test.ts b/server/src/__tests__/terminal-workspace-work.test.ts index 4ac80c30906..5189bc03eb0 100644 --- a/server/src/__tests__/terminal-workspace-work.test.ts +++ b/server/src/__tests__/terminal-workspace-work.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; -import { notifyTerminalWorkspaceWork, registerTerminalWorkspaceWork, TERMINAL_WORKSPACE_RETRY_MS, type TerminalWorkspaceTargets } from "../services/terminal-workspace-work.js"; +import { notifyTerminalWorkspaceWork, registerTerminalWorkspaceWork, TERMINAL_WORKSPACE_BACKSTOP_MS, TERMINAL_WORKSPACE_PLUGIN_HINT, TERMINAL_WORKSPACE_RETRY_MS, type TerminalWorkspaceTargets } from "../services/terminal-workspace-work.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; const coordinators: ReturnType[] = []; @@ -88,18 +88,21 @@ describe("terminal workspace work", () => { } finally { release?.(); } }); - it("restores the cooldown deadline, lets commits preempt it, and parks after cleanup", async () => { + it("restores the cooldown deadline, leaves unscoped commits to the hourly backstop, and parks after cleanup", async () => { const s = setup(), due = Date.now() + 86_400_000; s.service.nextTerminalWorkspaceSweepAt.mockResolvedValue(due); const worker = s.start(); await worker.ready; expect(worker.hasUnrecoveredWork()).toBe(false); - await vi.advanceTimersByTimeAsync(3_600_000); + await vi.advanceTimersByTimeAsync(30 * 60_000); expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); await s.owner.transaction(tx => notifyTerminalWorkspaceWork(tx)); await s.owner.transaction(tx => notifyTerminalWorkspaceWork(tx)); await vi.advanceTimersByTimeAsync(1); + expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(TERMINAL_WORKSPACE_BACKSTOP_MS); expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(2); + expect(s.service.sweepTerminalWorkspaces.mock.calls[1]?.[2]?.targets).toBeUndefined(); expect(s.coordinator.nextWakeAt()).toBe(due); s.service.nextTerminalWorkspaceSweepAt.mockResolvedValue(null); await vi.advanceTimersByTimeAsync(due - Date.now()); @@ -107,7 +110,7 @@ describe("terminal workspace work", () => { expect(vi.getTimerCount()).toBe(0); }); - it("inspects only hinted workspaces after a commit, and keeps unscoped commits and deadlines on the full rotation", async () => { + it("inspects only hinted workspaces after a commit, and leaves an unscoped commit to the backstop", async () => { const s = setup(), due = Date.now() + 60_000; s.service.nextTerminalWorkspaceSweepAt.mockResolvedValue(due); await s.start().ready; @@ -121,21 +124,34 @@ describe("terminal workspace work", () => { expect(s.service.sweepTerminalWorkspaces.mock.calls[1]?.[2]).toMatchObject({ targets: { workspaceIds: [workspaceId], issueIds: [issueId] } }); await s.owner.transaction(tx => notifyTerminalWorkspaceWork(tx)); await vi.advanceTimersByTimeAsync(1); + expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(due - Date.now()); expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(3); expect(s.service.sweepTerminalWorkspaces.mock.calls[2]?.[2]?.targets).toBeUndefined(); - await vi.advanceTimersByTimeAsync(due - Date.now()); - expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(4); - expect(s.service.sweepTerminalWorkspaces.mock.calls[3]?.[2]?.targets).toBeUndefined(); }); - it("falls back to the full rotation when more workspaces are hinted than one inspection reads", async () => { + it("retries the plugin workspace listing after it fails instead of waiting for the backstop", async () => { + const s = setup(); + const pluginWorkspaceId = "6b1f9c2e-8a4d-4f3b-9c1e-2d7a5b8e0f13"; + const listPluginTerminalWorkspaceIds = vi.fn<() => Promise>() + .mockRejectedValueOnce(new Error("database unavailable")).mockResolvedValue([pluginWorkspaceId]); + await registerTerminalWorkspaceWork(s.coordinator, { ...s.service, listPluginTerminalWorkspaceIds }, s.enabled).ready; + await s.owner.transaction(tx => notifyTerminalWorkspaceWork(tx, TERMINAL_WORKSPACE_PLUGIN_HINT)); + await vi.advanceTimersByTimeAsync(1); + expect(listPluginTerminalWorkspaceIds).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(TERMINAL_WORKSPACE_RETRY_MS); + expect(listPluginTerminalWorkspaceIds).toHaveBeenCalledTimes(2); + expect(s.service.sweepTerminalWorkspaces.mock.calls.at(-1)?.[2]?.targets).toEqual({ workspaceIds: [pluginWorkspaceId], issueIds: [] }); + }); + + it("works through more hinted workspaces than one inspection reads in targeted batches", async () => { const s = setup(); await s.start().ready; const ids = Array.from({ length: 51 }, (_, index) => `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`); await s.owner.transaction(async tx => { for (const id of ids) await notifyTerminalWorkspaceWork(tx, `workspace:${id}`); }); - await vi.advanceTimersByTimeAsync(1); - expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(2); - expect(s.service.sweepTerminalWorkspaces.mock.calls[1]?.[2]?.targets).toBeUndefined(); + await vi.advanceTimersByTimeAsync(10); + expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(3); + expect(s.service.sweepTerminalWorkspaces.mock.calls.slice(1).map(call => call[2]?.targets?.workspaceIds.length)).toEqual([50, 1]); }); it("keeps the pause between rotation pages while hinted workspaces wait", async () => { diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 999a2424f70..9bb81f59c19 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -39,7 +39,7 @@ import { authorizationService, executionWorkspaceReadSqlCondition, projectReadSq import type { SQL } from "drizzle-orm"; import { nativeSessionProviderEvidence } from "./native-runtime/native-session-resume.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; -import { notifyTerminalWorkspaceWork, terminalWorkspaceTargetsFromHints, type TerminalWorkspaceTargets } from "./terminal-workspace-work.js"; +import { notifyTerminalWorkspaceWork, type TerminalWorkspaceTargets } from "./terminal-workspace-work.js"; import { clearTerminalWorkspaceRetry, readTerminalWorkspaceRetry, recordTerminalWorkspaceFailure, terminalWorkspaceFailure, TERMINAL_WORKSPACE_RETRY_KEY, type TerminalWorkspaceFailure, @@ -3688,16 +3688,15 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic : undefined; const scanFilter = targets ? and(baseCandidateFilter, terminalWorkspaceTargetFilter(targets)) : and(baseCandidateFilter, boundaryFilter, cursorFilter); - const candidates = await db + const candidateQuery = db .select({ workspace: executionWorkspaces, cursorAt: sql`${executionWorkspaces.updatedAt}::text` }) .from(executionWorkspaces) .where(scanFilter) - .orderBy(asc(executionWorkspaces.updatedAt), asc(executionWorkspaces.id)) - .limit(limit); + .orderBy(asc(executionWorkspaces.updatedAt), asc(executionWorkspaces.id)); + const candidates = await (targets ? candidateQuery : candidateQuery.limit(limit)); // Advance the cursor to this page's last row. A short page means the scan // reached the end of the bounded candidate set, so reset the cursor and // the bound to start a new rotation on the next sweep. - if (targets && candidates.length >= limit) terminalSweepRescanRequested = true; if (!targets && candidates.length < limit) { terminalSweepCursor = null; terminalSweepBoundary = null; @@ -4398,8 +4397,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic if (!options.targets) { terminalSweepCursor = previousCursor; terminalSweepBoundary = previousBoundary; + terminalSweepRescanRequested = true; } - terminalSweepRescanRequested = true; throw error; } finally { terminalSweepInProgress = false; @@ -4409,7 +4408,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic hasTerminalWorkspaceSweepContinuation: () => terminalSweepCursor !== null || terminalSweepRescanRequested, terminalWorkspaceWorkSettled: (hints?: ReadonlySet, options: { targeted?: boolean } = {}) => { - if ((terminalSweepInProgress || terminalSweepCursor !== null) && !(options.targeted && terminalWorkspaceTargetsFromHints(hints))) { + if ((terminalSweepInProgress || terminalSweepCursor !== null) && !options.targeted) { terminalSweepRescanRequested = true; } pullRequestStateCache.clear(); @@ -4432,11 +4431,24 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } }, + listPluginTerminalWorkspaceIds: async () => { + const rows = await db.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and( + eq(executionWorkspaces.providerType, "plugin"), + sql`${executionWorkspaces.sourceIssueId} IS NOT NULL`, + or(and(inArray(executionWorkspaces.status, ["active", "idle", "in_review"]), isNull(executionWorkspaces.closedAt)), + and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]), + sql`(${executionWorkspaces.metadata}->>'workspaceProviderRemoved') IS DISTINCT FROM 'true'`)), + )); + return rows.map(row => row.id); + }, + takeDueTerminalWorkspaceTargets: (): { workspaceIds: string[]; rotation: boolean } => { const at = now().getTime(); const due = (deadlines: Map) => [...deadlines].filter(([, deadline]) => deadline <= at).map(([id]) => id); - const rotation = due(isolatedRehomeDeadlines).length + due(nestedCloneMigrationDeadlines).length + due(taskDirectoryDeadlines).length > 0; - return { workspaceIds: due(terminalWorkspaceDeadlines), rotation }; + const workspaceKey = (key: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(key); + const migrations = [...due(isolatedRehomeDeadlines), ...due(nestedCloneMigrationDeadlines)]; + const rotation = due(taskDirectoryDeadlines).length > 0 || migrations.some(key => !workspaceKey(key)); + return { workspaceIds: [...new Set([...due(terminalWorkspaceDeadlines), ...migrations.filter(workspaceKey)])], rotation }; }, nextTerminalWorkspaceSweepAt: async () => { diff --git a/server/src/services/plugin-registry.ts b/server/src/services/plugin-registry.ts index ab9eabc658d..30f2259d958 100644 --- a/server/src/services/plugin-registry.ts +++ b/server/src/services/plugin-registry.ts @@ -32,6 +32,7 @@ import type { } from "@paperclipai/shared"; import { conflict, notFound } from "../errors.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { TERMINAL_WORKSPACE_PLUGIN_HINT } from "./terminal-workspace-work.js"; function sameBundledPluginRegistration(existing: typeof plugins.$inferSelect, packageName: string, manifest: PaperclipPluginManifestV1): boolean { return existing.pluginKey === manifest.id && existing.packageName === packageName && existing.version === manifest.version && isDeepStrictEqual(existing.manifestJson, manifest); @@ -92,7 +93,7 @@ export function pluginRegistryService(db: Db) { async function writeWorkspaceProviderChange(reconsiderWorkspaces: boolean, write: (database: Db) => Promise): Promise { return db.transaction(async tx => { await notifyExternalObjectRefresh(tx); - if (reconsiderWorkspaces) await notifyDeliveryWork(tx, DELIVERY_QUEUES.terminalWorkspaces); + if (reconsiderWorkspaces) await notifyDeliveryWork(tx, DELIVERY_QUEUES.terminalWorkspaces, TERMINAL_WORKSPACE_PLUGIN_HINT); return write(tx as unknown as Db); }); } @@ -517,7 +518,7 @@ export function pluginRegistryService(db: Db) { .then((rows) => rows[0] ?? null); if (plugin.manifestJson.workspaceProvider && existing?.enabled === false && input.enabled === true) { - await notifyDeliveryWork(db, DELIVERY_QUEUES.terminalWorkspaces); + await notifyDeliveryWork(db, DELIVERY_QUEUES.terminalWorkspaces, TERMINAL_WORKSPACE_PLUGIN_HINT); } if (existing) { diff --git a/server/src/services/terminal-workspace-work.ts b/server/src/services/terminal-workspace-work.ts index b9845bdbb5d..6f8220f0854 100644 --- a/server/src/services/terminal-workspace-work.ts +++ b/server/src/services/terminal-workspace-work.ts @@ -4,6 +4,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export const TERMINAL_WORKSPACE_RETRY_MS = 5 * 60 * 1000; export const MAX_TERMINAL_WORKSPACE_TARGETS = 50; export const TERMINAL_WORKSPACE_BACKSTOP_MS = 60 * 60 * 1000; +export const TERMINAL_WORKSPACE_PLUGIN_HINT = "plugin-workspaces"; export function notifyTerminalWorkspaceWork(transaction: object, hint?: string) { return notifyDeliveryWork(transaction, DELIVERY_QUEUES.terminalWorkspaces, hint); @@ -38,6 +39,7 @@ export function registerTerminalWorkspaceWork( hasTerminalWorkspaceSweepContinuation(): boolean; terminalWorkspaceWorkSettled?(hints?: ReadonlySet, options?: { targeted?: boolean }): void; takeDueTerminalWorkspaceTargets?(): { workspaceIds: string[]; rotation: boolean }; + listPluginTerminalWorkspaceIds?(): Promise; }, canRun: () => boolean, ) { @@ -48,26 +50,31 @@ export function registerTerminalWorkspaceWork( // can take seconds per workspace, so the first sweep skips it and later // sweeps do it. let startupSweepStarted = false; - let rotationRequested = false; let dueAt: number | null = null; let lastRotationPageAt = 0; let backstopArmed = false; let lastFullRotationAt = Date.now(); let continuing = false; - const targetWorkspaceIds = new Set(), targetIssueIds = new Set(); + let pluginRefresh = false; + const pendingWorkspaceIds = new Set(), pendingIssueIds = new Set(); + const pendingCount = () => pendingWorkspaceIds.size + pendingIssueIds.size; + const takeTargets = (): TerminalWorkspaceTargets => { + const workspaceIds = [...pendingWorkspaceIds].slice(0, MAX_TERMINAL_WORKSPACE_TARGETS); + for (const id of workspaceIds) pendingWorkspaceIds.delete(id); + const issueIds = [...pendingIssueIds].slice(0, MAX_TERMINAL_WORKSPACE_TARGETS - workspaceIds.length); + for (const id of issueIds) pendingIssueIds.delete(id); + return { workspaceIds, issueIds }; + }; const worker = coordinator.register(DELIVERY_QUEUES.terminalWorkspaces, { retryMs: TERMINAL_WORKSPACE_RETRY_MS, onSettled: hints => { backstopArmed = true; - const targets = terminalWorkspaceTargetsFromHints(hints); - for (const id of targets?.workspaceIds ?? []) targetWorkspaceIds.add(id); - for (const id of targets?.issueIds ?? []) targetIssueIds.add(id); - const overflow = targetWorkspaceIds.size + targetIssueIds.size > MAX_TERMINAL_WORKSPACE_TARGETS; - service.terminalWorkspaceWorkSettled?.(hints, { targeted: Boolean(targets) && !overflow }); - if (!targets || overflow) { - rotationRequested = true; - targetWorkspaceIds.clear(); targetIssueIds.clear(); - } + if (hints?.has(TERMINAL_WORKSPACE_PLUGIN_HINT)) pluginRefresh = true; + const scoped = hints ? new Set([...hints].filter(hint => hint !== TERMINAL_WORKSPACE_PLUGIN_HINT)) : undefined; + const targets = terminalWorkspaceTargetsFromHints(scoped); + for (const id of targets?.workspaceIds ?? []) pendingWorkspaceIds.add(id); + for (const id of targets?.issueIds ?? []) pendingIssueIds.add(id); + service.terminalWorkspaceWorkSettled?.(hints, { targeted: true }); }, run: async signal => { recovered = false; @@ -76,44 +83,44 @@ export function registerTerminalWorkspaceWork( next = Date.now() + TERMINAL_WORKSPACE_RETRY_MS; const startup = !startupSweepStarted; startupSweepStarted = true; - let dueRotation = due; + const options = { compact: !startup, allowRemoteDeliveryChecks: !startup }; + let prelude = false; + const legacyDue = due && !startup && !service.takeDueTerminalWorkspaceTargets; if (due && !startup && service.takeDueTerminalWorkspaceTargets) { const dueTargets = service.takeDueTerminalWorkspaceTargets(); - dueRotation = dueTargets.rotation; - for (const id of dueTargets.workspaceIds) targetWorkspaceIds.add(id); - if (targetWorkspaceIds.size + targetIssueIds.size > MAX_TERMINAL_WORKSPACE_TARGETS) { - dueRotation = true; - targetWorkspaceIds.clear(); targetIssueIds.clear(); + for (const id of dueTargets.workspaceIds) pendingWorkspaceIds.add(id); + if (dueTargets.rotation) { prelude = true; backstopArmed = true; } + } + if (pluginRefresh && !startup && service.listPluginTerminalWorkspaceIds) { + pluginRefresh = false; + try { + for (const id of await service.listPluginTerminalWorkspaceIds()) pendingWorkspaceIds.add(id); + } catch (error) { + pluginRefresh = true; + throw error; } } - const options = { compact: !startup, allowRemoteDeliveryChecks: !startup }; - const targeted = targetWorkspaceIds.size + targetIssueIds.size > 0; const backstopDue = backstopArmed && Date.now() >= lastFullRotationAt + TERMINAL_WORKSPACE_BACKSTOP_MS; - if (due && service.takeDueTerminalWorkspaceTargets && !startup && !dueRotation && !targeted && !rotationRequested && !backstopDue - && !service.hasTerminalWorkspaceSweepContinuation()) return { checked: 0 }; - if (startup || dueRotation || rotationRequested || backstopDue || !targeted - || (service.hasTerminalWorkspaceSweepContinuation() && Date.now() >= lastRotationPageAt + 1000)) { - rotationRequested = false; + if (startup || backstopDue || legacyDue || (continuing && Date.now() >= lastRotationPageAt + 1000)) { lastRotationPageAt = Date.now(); - if (!continuing) { lastFullRotationAt = Date.now(); backstopArmed = false; } + if (!continuing) { lastFullRotationAt = Date.now(); backstopArmed = false; pluginRefresh = false; } return service.sweepTerminalWorkspaces(50, signal, options); } - const targets = { workspaceIds: [...targetWorkspaceIds], issueIds: [...targetIssueIds] }; - targetWorkspaceIds.clear(); targetIssueIds.clear(); + if (pendingCount() === 0 && !prelude) return { checked: 0 }; + const targets = takeTargets(); try { return await service.sweepTerminalWorkspaces(50, signal, { ...options, targets }); } catch (error) { - rotationRequested = true; + for (const id of targets.workspaceIds) pendingWorkspaceIds.add(id); + for (const id of targets.issueIds) pendingIssueIds.add(id); throw error; } }, hasPending: async () => { - continuing = false; + if (canRun()) continuing = service.hasTerminalWorkspaceSweepContinuation(); if (!canRun()) next = Date.now() + TERMINAL_WORKSPACE_RETRY_MS; - else if (service.hasTerminalWorkspaceSweepContinuation()) { - continuing = true; - next = Math.max(Date.now(), lastRotationPageAt + 1000); - } else if (rotationRequested || targetWorkspaceIds.size + targetIssueIds.size > 0) next = Date.now(); + else if (continuing) next = Math.max(Date.now(), lastRotationPageAt + 1000); + else if (pendingCount() > 0 || pluginRefresh) next = Date.now(); else { const deadline = await service.nextTerminalWorkspaceSweepAt(); next = deadline === null ? null : deadline > Date.now()