diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index c7bbeba4d55..09bce17810c 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -294,6 +294,19 @@ removal clears the record, and a reopen, which raises the lifecycle generation, discards it. Other wakes, including provider upgrades, wait for the next attempt time. Core keeps the record through workspace metadata updates. +When a successful terminal cleanup inspection keeps the workspace, Core records +`metadata.terminalInspectionHold` and does not send `inspect` again while the +hold applies. A workspace kept for local data (dirty or untracked files, or +local-only data) is held until something changes it. A clean workspace that is +neither merged nor recoverable is checked again after 1 hour, then on an +interval that doubles up to 24 hours, because the remote can change without +touching the workspace. Any of these ends a hold early: a change notice that +names the workspace or a task in its tree, a new lifecycle generation, a write +to the workspace row (for example a new binding or status), a task in its tree +finishing again, or new delivery evidence. A new inspected HEAD restarts the +interval. Only a kept workspace is held. An inspection that finds the workspace +removable proceeds to removal as before and clears any hold. + The optional `prepared.destinationRef` records the resolved delivery branch as `refs/heads/`, up to 266 characters including the prefix. Providers should return null when preparation cannot establish an unambiguous branch destination diff --git a/server/src/__tests__/plugin-workspace-terminal-hold.integration.test.ts b/server/src/__tests__/plugin-workspace-terminal-hold.integration.test.ts new file mode 100644 index 00000000000..b0a51eb8ebe --- /dev/null +++ b/server/src/__tests__/plugin-workspace-terminal-hold.integration.test.ts @@ -0,0 +1,201 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { createDb, executionWorkspaces, heartbeatRuns, issues } from "@paperclipai/db"; +import { createPluginWorkspaceFixture } from "./helpers/plugin-workspaces.js"; +import { preparePluginWorkspace, type WorkspaceWorkers } from "../services/plugin-workspaces.js"; +import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; + +type Inspection = { headSha: string; hasDirtyTrackedFiles: boolean; hasUntrackedFiles: boolean; hasLocalOnlyData: boolean; + isMergedIntoBase: boolean | null; remotelyRecoverable: boolean }; +type Hold = { reason: string; checks: number; checkedAt: string; recheckAt: string | null; headSha: string | null }; + +(support.supported ? describe : describe.skip)("plugin workspace terminal inspection hold", () => { + let db: ReturnType, stopDb: (() => Promise) | undefined; + beforeAll(async () => { + const started = await startEmbeddedPostgresTestDatabase("plugin-workspace-terminal-hold"); + db = createDb(started.connectionString, { maxConnections: 1 }); + stopDb = started.cleanup; + }); + afterAll(async () => { + await db?.$client.end(); + await stopDb?.(); + }); + + async function setup(inspection: Partial) { + const f = await createPluginWorkspaceFixture(db); + const state = { inspection: { headSha: "a".repeat(40), hasDirtyTrackedFiles: false, hasUntrackedFiles: false, hasLocalOnlyData: false, + isMergedIntoBase: true, remotelyRecoverable: true, ...inspection } as Inspection, failInspect: false }; + const workers = { ...f.workers, call: async (...args: Parameters) => { + const params = args[2] as { operation: { kind: string } }; + if (params.operation.kind === "inspect" && state.failInspect) { + f.calls.push(params); + throw new Error("provider outage"); + } + const result = await f.workers.call(...args); + return params.operation.kind === "inspect" ? { ...(result as object), ...state.inspection } : result; + } } as WorkspaceWorkers; + const workspace = (await preparePluginWorkspace(db, workers, f.input))!; + await db.update(issues).set({ status: "done", completedAt: new Date() }).where(eq(issues.id, f.issueId)); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId)); + let clock = Date.now(); + const svc = executionWorkspaceService(db, { pluginWorkerManager: workers, workspaceReaperCooldownDays: 0, now: () => new Date(clock) }); + const read = async () => (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace.id)))[0]!; + const hold = async () => (await read()).metadata?.terminalInspectionHold as Hold | undefined; + const inspections = () => f.calls.filter(call => call.executionWorkspaceId === workspace.id && call.operation.kind === "inspect").length; + const removals = () => f.calls.filter(call => call.executionWorkspaceId === workspace.id && call.operation.kind === "remove").length; + const sweep = () => svc.sweepTerminalWorkspaces(100); + const due = () => svc.takeDueTerminalWorkspaceTargets().workspaceIds.includes(workspace.id); + return { f, state, svc, workspace, read, hold, inspections, removals, sweep, due, + get clock() { return clock; }, set clock(value: number) { clock = value; } }; + } + + it("holds a workspace kept for local data until the workspace, its tasks, or a hint changes it", async () => { + const s = await setup({ hasLocalOnlyData: true }); + await s.sweep(); + expect(s.inspections()).toBe(1); + expect((await s.read()).status).toBe("active"); + expect(await s.hold()).toMatchObject({ reason: "local_data", checks: 1, recheckAt: null }); + expect(s.due()).toBe(false); + + s.clock += 30 * 24 * HOUR; + expect((await s.sweep()).skippedBackoff).toBeGreaterThanOrEqual(1); + expect(s.inspections()).toBe(1); + expect(s.due()).toBe(false); + + s.svc.terminalWorkspaceWorkSettled(new Set([`workspace:${s.workspace.id}`]), { targeted: true }); + s.clock += 1; + await s.sweep(); + expect(s.inspections()).toBe(2); + await s.sweep(); + expect(s.inspections()).toBe(2); + + s.svc.terminalWorkspaceWorkSettled(new Set([`issue:${s.f.issueId}`]), { targeted: true }); + s.clock += 1; + await s.sweep(); + expect(s.inspections()).toBe(3); + + await db.update(executionWorkspaces).set({ metadata: sql`jsonb_set(${executionWorkspaces.metadata}, '{lifecycleGeneration}', + to_jsonb(coalesce((${executionWorkspaces.metadata}->>'lifecycleGeneration')::int, 0) + 1))` }) + .where(eq(executionWorkspaces.id, s.workspace.id)); + await s.sweep(); + expect(s.inspections()).toBe(4); + + await executionWorkspaceService(db).update(s.workspace.id, { metadata: { note: "operator edit" } }); + expect(await s.hold()).toMatchObject({ reason: "local_data" }); + await s.sweep(); + expect(s.inspections()).toBe(5); + + await db.update(issues).set({ completedAt: new Date(s.clock) }).where(eq(issues.id, s.f.issueId)); + await s.sweep(); + expect(s.inspections()).toBe(6); + await s.sweep(); + expect(s.inspections()).toBe(6); + + s.svc.terminalWorkspaceWorkSettled(new Set(["plugin-workspaces"]), { targeted: true }); + s.clock += 1; + await s.sweep(); + expect(s.inspections()).toBe(7); + await s.sweep(); + expect(s.inspections()).toBe(7); + expect(s.removals()).toBe(0); + }); + + it("re-checks a clean unmerged workspace on a doubling interval and resets it on a new head or a hint", async () => { + const s = await setup({ isMergedIntoBase: false, remotelyRecoverable: false }); + const start = s.clock; + await s.sweep(); + expect(await s.hold()).toMatchObject({ reason: "unmerged", checks: 1, recheckAt: new Date(start + HOUR).toISOString() }); + + s.clock = start + 59 * MINUTE; + await s.sweep(); + expect(s.inspections()).toBe(1); + expect(s.due()).toBe(false); + + const delays: number[] = []; + for (let check = 0; check < 7; check++) { + const recheckAt = Date.parse((await s.hold())!.recheckAt!); + delays.push((recheckAt - s.clock) / HOUR); + s.clock = recheckAt; + expect(s.due()).toBe(true); + await s.sweep(); + } + expect(s.inspections()).toBe(8); + expect(delays.slice(1)).toEqual([2, 4, 8, 16, 24, 24]); + expect(await s.hold()).toMatchObject({ checks: 8, recheckAt: new Date(s.clock + 24 * HOUR).toISOString() }); + + s.state.inspection.headSha = "b".repeat(40); + s.clock = Date.parse((await s.hold())!.recheckAt!); + await s.sweep(); + expect(await s.hold()).toMatchObject({ checks: 1, headSha: "b".repeat(40), recheckAt: new Date(s.clock + HOUR).toISOString() }); + + s.clock = Date.parse((await s.hold())!.recheckAt!); + await s.sweep(); + expect(await s.hold()).toMatchObject({ checks: 2 }); + s.clock += 10 * MINUTE; + s.svc.terminalWorkspaceWorkSettled(new Set([`workspace:${s.workspace.id}`]), { targeted: true }); + s.clock += 1; + await s.sweep(); + expect(s.inspections()).toBe(11); + expect(await s.hold()).toMatchObject({ checks: 1, recheckAt: new Date(s.clock + HOUR).toISOString() }); + }); + + it("removes a workspace as soon as it is removable, including one that was held", async () => { + const merged = await setup({}); + await merged.sweep(); + expect(merged.inspections()).toBe(2); + expect(merged.removals()).toBe(1); + expect(await merged.read()).toMatchObject({ status: "archived", metadata: { workspaceProviderRemoved: true } }); + + const s = await setup({ hasDirtyTrackedFiles: true }); + await s.sweep(); + expect(await s.hold()).toMatchObject({ reason: "local_data" }); + s.state.inspection.hasDirtyTrackedFiles = false; + s.svc.terminalWorkspaceWorkSettled(new Set([`workspace:${s.workspace.id}`]), { targeted: true }); + s.clock += 1; + await s.sweep(); + expect(s.removals()).toBe(1); + const removed = await s.read(); + expect(removed).toMatchObject({ status: "archived", metadata: { workspaceProviderRemoved: true } }); + expect(removed.metadata).not.toHaveProperty("terminalInspectionHold"); + }); + + it("holds a closed workspace with local data and removes it after an unscoped change notice", async () => { + const s = await setup({}); + await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date() }).where(eq(executionWorkspaces.id, s.workspace.id)); + s.state.inspection.hasUntrackedFiles = true; + await s.sweep(); + expect(await s.hold()).toMatchObject({ reason: "local_data" }); + await s.sweep(); + expect(s.inspections()).toBe(1); + s.state.inspection.hasUntrackedFiles = false; + s.svc.terminalWorkspaceWorkSettled(); + s.clock += 1; + await s.sweep(); + expect(s.removals()).toBe(1); + expect(await s.read()).toMatchObject({ metadata: { workspaceProviderRemoved: true } }); + }); + + it("keeps the inspection failure backoff and does not hold a workspace it could not inspect", async () => { + const s = await setup({ hasLocalOnlyData: true }); + s.state.failInspect = true; + await s.sweep(); + expect(await s.hold()).toBeUndefined(); + expect((await s.read()).metadata?.terminalCleanupRetry).toMatchObject({ failures: 1, + retryAt: new Date(s.clock + 5 * MINUTE).toISOString() }); + s.clock += 5 * MINUTE; + await s.sweep(); + expect((await s.read()).metadata?.terminalCleanupRetry).toMatchObject({ failures: 2, + retryAt: new Date(s.clock + 10 * MINUTE).toISOString() }); + s.state.failInspect = false; + s.clock += 10 * MINUTE; + await s.sweep(); + expect(s.inspections()).toBe(3); + expect((await s.read()).metadata).not.toHaveProperty("terminalCleanupRetry"); + expect(await s.hold()).toMatchObject({ reason: "local_data", checks: 1 }); + }); +}); diff --git a/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts b/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts index 3961b5667a2..45f429ba6b6 100644 --- a/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts +++ b/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts @@ -204,6 +204,8 @@ function providerRefusal() { await svc.sweepTerminalWorkspaces(100); expect(s.callsFor(workspace.id, "inspect")).toHaveLength(4); expect(await retryAfter()).toBeUndefined(); + // The successful inspection kept the dirty workspace on hold; a change notice ends the hold. + svc.terminalWorkspaceWorkSettled(new Set([`workspace:${workspace.id}`]), { targeted: true }); s.failures.inspect = true; clock += MINUTE; await svc.sweepTerminalWorkspaces(100); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 9bb81f59c19..9f122bb070f 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -39,10 +39,12 @@ import { authorizationService, executionWorkspaceReadSqlCondition, projectReadSq import type { SQL } from "drizzle-orm"; import { nativeSessionProviderEvidence } from "./native-runtime/native-session-resume.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; -import { notifyTerminalWorkspaceWork, type TerminalWorkspaceTargets } from "./terminal-workspace-work.js"; +import { notifyTerminalWorkspaceWork, TERMINAL_WORKSPACE_PLUGIN_HINT, type TerminalWorkspaceTargets } from "./terminal-workspace-work.js"; import { clearTerminalWorkspaceRetry, readTerminalWorkspaceRetry, recordTerminalWorkspaceFailure, terminalWorkspaceFailure, TERMINAL_WORKSPACE_RETRY_KEY, type TerminalWorkspaceFailure, + clearTerminalWorkspaceHold, readTerminalWorkspaceHold, recordTerminalWorkspaceHold, terminalWorkspaceRecheckDelayMs, + TERMINAL_WORKSPACE_HOLD_KEY, type TerminalWorkspaceHold, } from "./terminal-workspace-retry.js"; import { deleteWorkspaceCompactionTrash, @@ -269,6 +271,19 @@ export function clearMetadataReopenPendingConsumption( return next; } +/** What a kept provider workspace's inspection depends on, apart from its lifecycle generation and the remote. */ +function terminalHoldFingerprint(workspace: typeof executionWorkspaces.$inferSelect, + issueTree: Array<{ id: string; status: string; completedAt: Date | null; cancelledAt: Date | null; updatedAt: Date }>, + deliveryCandidate: { operationId: string; headSha: string; destinationRef: string } | null) { + return createHash("sha256").update(JSON.stringify([ + workspace.updatedAt.toISOString(), workspace.status, workspace.providerRef, workspace.repoUrl, workspace.branchName, workspace.baseRef, + readWorkspaceProviderBinding(workspace.metadata) ?? null, + deliveryCandidate && [deliveryCandidate.operationId, deliveryCandidate.headSha, deliveryCandidate.destinationRef], + issueTree.map(issue => [issue.id, issue.status, issueTerminalTimestamp(issue)?.toISOString() ?? null] as const) + .sort((a, b) => a[0].localeCompare(b[0])), + ])).digest("hex"); +} + function terminalWorkspaceTargetFilter(targets: TerminalWorkspaceTargets) { const ids = (values: string[]) => sql.join(values.map(id => sql`${id}::uuid`), sql`, `); const byWorkspace = targets.workspaceIds.length ? sql`${executionWorkspaces.id} in (${ids(targets.workspaceIds)})` : undefined; @@ -314,7 +329,8 @@ function preserveWorkspaceMetadataForPatch( } } } - const coreOwnedKeys = ["workspaceProviderFresh", "runnerTaskFreshWorkspace", "runnerTaskWorkspaceOwner", TERMINAL_WORKSPACE_RETRY_KEY] as const; + const coreOwnedKeys = ["workspaceProviderFresh", "runnerTaskFreshWorkspace", "runnerTaskWorkspaceOwner", TERMINAL_WORKSPACE_RETRY_KEY, + TERMINAL_WORKSPACE_HOLD_KEY] as const; if (patch.metadata !== undefined && (patch.metadata !== null || coreOwnedKeys.some(key => current.metadata?.[key] !== undefined))) { nextPatch.metadata = { ...(nextPatch.metadata ?? patch.metadata) }; for (const key of coreOwnedKeys) { @@ -1563,6 +1579,7 @@ export async function releaseAncestorWorkspaceBinding(tx: DbTransaction, input: // move and the guard finish in one short transaction. const WORKSPACE_COMPACTION_TRASH_STALE_MS = 60 * 60 * 1000; const WORKSPACE_COMPACTION_TRASH_PURGE_INTERVAL_MS = 30 * 60 * 1000; +const TERMINAL_HOLD_RESET_LIMIT = 65_536; export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServiceOptions = {}) { const destroyRemoteLease: DestroyRemoteWorkspaceLease = input => opts.destroyRemoteWorkspaceLease @@ -1937,6 +1954,13 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // when a sweep reaches the end of the candidate set. let terminalSweepCursor: { updatedAt: string; id: string } | null = null; const terminalInspections = new Map; nextAt: number; parked?: boolean }>(); + // A `workspace:` or `issue:` hint ends the persisted inspection holds it names + // that were checked at or before the hint. A sweep visit clears such a hold + // from the row, so an entry is dropped once a full rotation that started after + // it completes. A hint without a scope ends every hold. + const terminalHoldResets = new Map(); + let terminalHoldResetAllAt = Number.NEGATIVE_INFINITY; + let terminalHoldRotationStartedAt: number | null = null; const terminalWorkspaceDeadlines = new Map(); const isolatedRehomeDeadlines = new Map(); const isolatedRehomeNotBefore = new Map(); @@ -2279,6 +2303,21 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic return trees; } + async function readPageLinkedIssueIds(workspaces: Array>) { + const linked = new Map(); + if (workspaces.length === 0) return linked; + const rows = await db.select({ id: issues.id, workspaceId: issues.executionWorkspaceId }).from(issues).where(and( + inArray(issues.companyId, [...new Set(workspaces.map(workspace => workspace.companyId))]), + inArray(issues.executionWorkspaceId, workspaces.map(workspace => workspace.id)))); + for (const row of rows) { + if (!row.workspaceId) continue; + const ids = linked.get(row.workspaceId); + if (ids) ids.push(row.id); + else linked.set(row.workspaceId, [row.id]); + } + return linked; + } + async function readPageActiveRunWorkspaces(workspaces: Array>) { if (workspaces.length === 0) return new Set(); const pairs = workspaces.map(workspace => ({ id: workspace.id, companyId: workspace.companyId, sourceIssueId: workspace.sourceIssueId })); @@ -3655,6 +3694,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // rotation cover a finite set, so the cursor always reaches a short page. if (!targets && !cursor) { terminalSweepBoundary = now(); + terminalHoldRotationStartedAt = terminalSweepBoundary.getTime(); terminalWorkspaceDeadlines.clear(); isolatedRehomeDeadlines.clear(); nestedCloneMigrationDeadlines.clear(); @@ -3697,7 +3737,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // Advance the cursor to this page's last row. A short page means the scan // reached the end of the bounded candidate set, so reset the cursor and // the bound to start a new rotation on the next sweep. - if (!targets && candidates.length < limit) { + const rotationCompleted = !targets && candidates.length < limit; + const completedRotationStartedAt = terminalHoldRotationStartedAt; + if (rotationCompleted) { terminalSweepCursor = null; terminalSweepBoundary = null; } else if (!targets) { @@ -3723,8 +3765,24 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic if (targets) for (const id of [...targets.workspaceIds, ...candidates.map(candidate => candidate.workspace.id)]) terminalWorkspaceDeadlines.delete(id); const readinessByWorkingTree = new Map>(); const pageWorkspaces = candidates.map(candidate => candidate.workspace); - const [pageIssueTrees, pageActiveRunWorkspaces] = await Promise.all([ - readPageIssueTrees(pageWorkspaces), readPageActiveRunWorkspaces(pageWorkspaces)]); + const [pageIssueTrees, pageActiveRunWorkspaces, pageLinkedIssueIds] = await Promise.all([ + readPageIssueTrees(pageWorkspaces), readPageActiveRunWorkspaces(pageWorkspaces), + readPageLinkedIssueIds(pageWorkspaces.filter(workspace => workspace.metadata?.[TERMINAL_WORKSPACE_HOLD_KEY] !== undefined))]); + // Clear a persisted hold that a hint or a malformed record has ended, so + // the reset survives this process and the hint entry can be pruned. + const releaseEndedTerminalHold = async (target: ExecutionWorkspaceRow) => { + const hold = target.metadata?.[TERMINAL_WORKSPACE_HOLD_KEY] as { checkedAt?: unknown } | undefined; + if (hold === undefined) return target; + const checkedAt = typeof hold?.checkedAt === "string" ? Date.parse(hold.checkedAt) : Number.NaN; + const issueIds = [...(pageIssueTrees.get(target.id) ?? []).map(issue => issue.id), ...(pageLinkedIssueIds.get(target.id) ?? [])]; + const resetAt = Math.max(terminalHoldResetAllAt, terminalHoldResets.get(`workspace:${target.id}`) ?? Number.NEGATIVE_INFINITY, + ...issueIds.map(id => terminalHoldResets.get(`issue:${id}`) ?? Number.NEGATIVE_INFINITY)); + if (Number.isFinite(checkedAt) && resetAt < checkedAt) return target; + await clearTerminalWorkspaceHold(db, target.id); + const metadata = { ...target.metadata }; + delete metadata[TERMINAL_WORKSPACE_HOLD_KEY]; + return { ...target, metadata }; + }; const backOffTerminalWorkspace = async (snapshot: ExecutionWorkspaceRow, failure: TerminalWorkspaceFailure) => { const [target] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, snapshot.id)); if (!target) return; @@ -3736,6 +3794,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic logger.warn({ event: "execution_workspace.terminal_cleanup_retry", executionWorkspaceId: target.id, sourceIssueId: target.sourceIssueId, failures: retry.failures, retryAt: retry.retryAt, failure: retry.failure }, "terminal workspace cleanup failed; a later sweep retries"); }; + const setTerminalWorkspaceDeadline = (workspaceId: string, recheckAt: number | null, compactionAt: number | null) => { + const nextAt = Math.min(recheckAt ?? Number.POSITIVE_INFINITY, compactionAt ?? Number.POSITIVE_INFINITY); + if (Number.isFinite(nextAt)) terminalWorkspaceDeadlines.set(workspaceId, nextAt); + else terminalWorkspaceDeadlines.delete(workspaceId); + }; const settleTerminalWorkspaceRetry = async (target: ExecutionWorkspaceRow) => { if (target.metadata?.[TERMINAL_WORKSPACE_RETRY_KEY] !== undefined) await clearTerminalWorkspaceRetry(db, target.id); }; @@ -3769,8 +3832,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }; for (const candidate of candidates) { - let workspace = candidate.workspace; signal?.throwIfAborted(); + let workspace = await releaseEndedTerminalHold(candidate.workspace); // Trash deletion is part of compaction, so a sweep that skips compaction // also leaves leftover trash for a later sweep. if (options.compact !== false) await purgeCompactionTrashIfDue(workspace); @@ -4021,6 +4084,21 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic result.skippedBackoff += 1; continue; } + // Each inspection of a provider workspace is a provider operation. Skip + // it while an earlier inspection kept the workspace and nothing that + // could change that answer has changed. + const pendingCompactionAt = compactionPending && compactedTerminalAt.get(workspace.id) !== latestTerminalAt ? compactionDueAt : null; + const holdGeneration = readExecutionWorkspaceLifecycleGeneration(workspace.metadata); + const holdFingerprint = workspace.providerType === "plugin" && !reopenPending + && !workspace.metadata?.workspaceProviderRemoval && !workspace.metadata?.workspaceProviderRestoration + ? terminalHoldFingerprint(workspace, issueTree, deliveryCandidate) : null; + const priorHold = holdFingerprint ? readTerminalWorkspaceHold(workspace.metadata, holdGeneration) : null; + if (priorHold && priorHold.fingerprint === holdFingerprint + && (priorHold.recheckAt === null || Date.parse(priorHold.recheckAt) > now().getTime())) { + setTerminalWorkspaceDeadline(workspace.id, priorHold.recheckAt === null ? null : Date.parse(priorHold.recheckAt), pendingCompactionAt); + result.skippedBackoff += 1; + continue; + } if (opts.terminalWorkspaceInspectionBackoffMs) { const revision = createHash("sha256").update(JSON.stringify([ workspace.updatedAt, workspace.metadata, @@ -4094,6 +4172,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic ? JSON.stringify([workspace.companyId, workspacePath, workspace.baseRef, workspace.branchName, createdByRuntime]) : `workspace:${workspace.id}`; let inspection = readinessByWorkingTree.get(readinessKey); + const inspectionStartedAt = now().getTime(); if (!inspection) { inspection = inspectWorkspace(toExecutionWorkspace(workspace), recoveryPullRequest); readinessByWorkingTree.set(readinessKey, inspection); @@ -4134,18 +4213,33 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic result.skippedNonTerminalTree += 1; continue; } - if (assessment.workspaceDirty) { + const keepReason = assessment.workspaceDirty ? "local_data" as const + : assessment.deliveryState !== "merged_via_pr" && assessment.deliveryState !== "merged_by_ancestry" ? "unmerged" as const : null; + if (keepReason && holdFingerprint) { + const headSha = git?.headSha ?? null; + const checks = priorHold?.reason === keepReason && priorHold.fingerprint === holdFingerprint && priorHold.headSha === headSha + ? priorHold.checks + 1 : 1; + const recheckAt = keepReason === "unmerged" ? inspectionStartedAt + terminalWorkspaceRecheckDelayMs(checks) : null; + const hold: TerminalWorkspaceHold = { generation: holdGeneration, fingerprint: holdFingerprint, reason: keepReason, headSha, checks, + checkedAt: new Date(inspectionStartedAt).toISOString(), recheckAt: recheckAt === null ? null : new Date(recheckAt).toISOString() }; + await recordTerminalWorkspaceHold(db, workspace.id, hold); + setTerminalWorkspaceDeadline(workspace.id, recheckAt, pendingCompactionAt); + } + if (keepReason === "local_data") { result.skippedUndelivered += 1; continue; } - if ( - assessment.deliveryState !== "merged_via_pr" - && assessment.deliveryState !== "merged_by_ancestry" - ) { + if (keepReason === "unmerged") { if (remoteWorkspace) terminalWorkspaceDeadlines.delete(workspace.id); result.skippedUndelivered += 1; continue; } + if (workspace.metadata?.[TERMINAL_WORKSPACE_HOLD_KEY] !== undefined) { + await clearTerminalWorkspaceHold(db, workspace.id); + const metadata = { ...workspace.metadata }; + delete metadata[TERMINAL_WORKSPACE_HOLD_KEY]; + workspace = { ...workspace, metadata }; + } // Hold the archive during the cooldown window. The anchor is the most // recent terminal timestamp across the issue tree. A person can reopen // the work inside this window. A cooldown of 0 disables the check, so the @@ -4392,6 +4486,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }); } } + if (rotationCompleted && completedRotationStartedAt !== null) { + for (const [key, at] of terminalHoldResets) if (at < completedRotationStartedAt) terminalHoldResets.delete(key); + if (terminalHoldResetAllAt < completedRotationStartedAt) terminalHoldResetAllAt = Number.NEGATIVE_INFINITY; + } return result; } catch (error) { if (!options.targets) { @@ -4423,6 +4521,20 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic else for (const [id, skip] of taskDirectoryRowSkips) if (`issue:${skip.issueId}` === hint) taskDirectoryRowSkips.delete(id); } } + // A provider refresh (activation, upgrade, capability change, company + // re-enablement) can change how any plugin workspace inspects, and holds + // apply only to plugin workspaces, so it ends every hold. + const resetAt = now().getTime(); + if (!hints || hints.has(TERMINAL_WORKSPACE_PLUGIN_HINT)) terminalHoldResetAllAt = resetAt; + for (const hint of hints ?? []) { + if (!hint.startsWith("workspace:") && !hint.startsWith("issue:")) continue; + if (terminalHoldResets.size >= TERMINAL_HOLD_RESET_LIMIT) { + terminalHoldResets.clear(); + terminalHoldResetAllAt = resetAt; + break; + } + terminalHoldResets.set(hint, resetAt); + } if (!hints) { terminalInspections.clear(); return; } for (const [id, inspection] of terminalInspections) { if (hints.has(`workspace:${id}`) || [...inspection.issueIds].some(issueId => hints.has(`issue:${issueId}`))) { diff --git a/server/src/services/terminal-workspace-retry.ts b/server/src/services/terminal-workspace-retry.ts index 2db4a263c02..6531318b026 100644 --- a/server/src/services/terminal-workspace-retry.ts +++ b/server/src/services/terminal-workspace-retry.ts @@ -48,3 +48,46 @@ export async function clearTerminalWorkspaceRetry(db: Db, workspaceId: string) { .set({ metadata: sql`${executionWorkspaces.metadata} - ${TERMINAL_WORKSPACE_RETRY_KEY}::text` }) .where(and(eq(executionWorkspaces.id, workspaceId), sql`${executionWorkspaces.metadata} -> ${TERMINAL_WORKSPACE_RETRY_KEY}::text IS NOT NULL`)); } + +// A provider workspace that a sweep inspected and kept. `local_data` holds until +// the workspace or its task tree changes; `unmerged` re-checks on a growing +// interval because the remote can change without touching the workspace. +export const TERMINAL_WORKSPACE_HOLD_KEY = "terminalInspectionHold"; +export const TERMINAL_WORKSPACE_RECHECK_MS = 60 * 60 * 1000; +export const TERMINAL_WORKSPACE_RECHECK_MAX_MS = 24 * 60 * 60 * 1000; + +export type TerminalWorkspaceHold = { + generation: number; + fingerprint: string; + reason: "local_data" | "unmerged"; + headSha: string | null; + checks: number; + checkedAt: string; + recheckAt: string | null; +}; + +export function terminalWorkspaceRecheckDelayMs(checks: number) { + return Math.min(TERMINAL_WORKSPACE_RECHECK_MS * 2 ** Math.min(Math.max(checks - 1, 0), 16), TERMINAL_WORKSPACE_RECHECK_MAX_MS); +} + +export function readTerminalWorkspaceHold(metadata: Record | null | undefined, generation: number) { + const hold = metadata?.[TERMINAL_WORKSPACE_HOLD_KEY] as Partial | undefined; + return hold && typeof hold === "object" && hold.generation === generation && typeof hold.fingerprint === "string" + && (hold.reason === "local_data" || hold.reason === "unmerged") && Number.isInteger(hold.checks) + && typeof hold.checkedAt === "string" && (hold.recheckAt === null || typeof hold.recheckAt === "string") + ? hold as TerminalWorkspaceHold : null; +} + +/** Records the hold only while the workspace is still in the inspected lifecycle generation. */ +export async function recordTerminalWorkspaceHold(db: Db, workspaceId: string, hold: TerminalWorkspaceHold) { + await db.update(executionWorkspaces) + .set({ metadata: sql`coalesce(${executionWorkspaces.metadata}, '{}'::jsonb) || jsonb_build_object(${TERMINAL_WORKSPACE_HOLD_KEY}::text, ${JSON.stringify(hold)}::jsonb)` }) + .where(and(eq(executionWorkspaces.id, workspaceId), + sql`coalesce((${executionWorkspaces.metadata}->>'lifecycleGeneration')::int, 0) = ${hold.generation}`)); +} + +export async function clearTerminalWorkspaceHold(db: Db, workspaceId: string) { + await db.update(executionWorkspaces) + .set({ metadata: sql`${executionWorkspaces.metadata} - ${TERMINAL_WORKSPACE_HOLD_KEY}::text` }) + .where(and(eq(executionWorkspaces.id, workspaceId), sql`${executionWorkspaces.metadata} -> ${TERMINAL_WORKSPACE_HOLD_KEY}::text IS NOT NULL`)); +}