diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 3e07b832df4..4f1daf6b540 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -640,6 +640,16 @@ admitted once the hold releases, and stops parking at the same cutoff. The writer's final save commit and an unchanged release (below) make these waiters due immediately instead of at the next re-check. +When the unfinished-writer error was the task's last setup failure and +stranded-work recovery blocked the task with a board escalation, that block +belongs to the hold. The active recovery-action sweep settles the escalation +once the hold releases (the files were saved, or the task continued from its +last save): the task returns to `todo`, gets one plain notice, and its assignee +gets one wake. The sweep never changes the workspace. It keeps the task blocked +while the writer still holds the workspace, while a dependency forbids work, +after a newer run for the task, or when a later status decision chose the block. +A decision that only kept the Blocked status it found does not own it. + A writer that ended (failed, cancelled, interrupted or timed out) without changing its files releases the hold at once, with no snapshot. Its workspace keeps the saved version it started from. Preparation records that starting diff --git a/server/src/__tests__/released-writer-escalation.test.ts b/server/src/__tests__/released-writer-escalation.test.ts new file mode 100644 index 00000000000..e81be6042a1 --- /dev/null +++ b/server/src/__tests__/released-writer-escalation.test.ts @@ -0,0 +1,241 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { activityLog, agents, agentWakeupRequests, authUsers, companies, companyMemberships, createDb, executionWorkspaces, + heartbeatRuns, issueRecoveryActions, issueRelations, issues, statusDecisions } from "@paperclipai/db"; +import { heartbeatService } from "../services/heartbeat.js"; +import { REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE } from "../services/remote-workspace-writer-hold.js"; +import { continueTasksAfterWorkspaceWriterReleased, RELEASED_WORKSPACE_WRITER_OUTCOME } + from "../services/recovery/released-writer-escalation.js"; +import { recoveryActionSweepCondition } from "../services/recovery/blocker-stall-wakeups.js"; +import { recoveryService } from "../services/recovery/service.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; + +const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; +if (!recoveryPostgresSupport.supported) console.warn(`Skipping released-writer escalation tests: ${recoveryPostgresSupport.reason}`); + +// Recovery text never asks a person to inspect or fix anything, and never shows internal names. +const adminText = /sandbox|environment|administrator|board member|board operator|owner must|inspect the|epoch|writer run/i; + +/** + * The staging shape (Nov 12). The task's own remote workspace was + * held by its ended native run (its superseded export never saved the files). + * The task's queued runs then failed setup on that hold before any provider + * work, stranded-work recovery blocked the task with a board-owned escalation + * at 16:00:56.667Z, and a late native status decision 1.4 s later only kept + * that block ("prior_status_terminal_preserved"). Hours later the retained-file + * recovery released the hold (it continued from the last save and abandoned + * the run's workspace epoch), but the escalation and the Blocked status stayed. + */ +describePostgres("a task stranded behind its own workspace's ended writer", () => { + let temporary: Awaited>; + let db: ReturnType; + let heartbeat: ReturnType; + const oldHome = process.env.PAPERCLIP_HOME; + let home: string; + const wakes: Array<{ agentId: string; options: Record }> = []; + const enqueueWakeup = async (agentId: string, options: unknown) => { + wakes.push({ agentId, options: options as Record }); + return { id: randomUUID() }; + }; + + beforeAll(async () => { + temporary = await startRecoveryPostgres("released-writer-escalation-"); + db = createDb(temporary.connectionString); + home = await mkdtemp(path.join(tmpdir(), "paperclip-released-writer-")); + process.env.PAPERCLIP_HOME = home; + heartbeat = heartbeatService(db); + }, 240_000); + + afterAll(async () => { + await heartbeat?.drainActiveRunExecutions(); + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + if (oldHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = oldHome; + if (home) await rm(home, { recursive: true, force: true }); + }); + + async function seedStrandedTask(options: { laterDecision?: { fromStatus: string; toStatus: string } | null } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), userId = `owner-${companyId}`; + await db.insert(authUsers).values({ id: userId, name: "Owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Writer hold", defaultResponsibleUserId: userId, issuePrefix: `W${companyId.slice(0, 6)}` }); + await db.insert(companyMemberships).values({ companyId, principalId: userId, principalType: "user", status: "active", membershipRole: "owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "process", status: "idle", adapterConfig: {} }); + const [task] = await db.insert(issues).values({ companyId, title: "Ship the page", status: "blocked", assigneeAgentId: agentId, + responsibleUserId: userId, executionWorkspacePreference: "reuse_existing", issueNumber: 1 }).returning(); + const hour = 3_600_000, base = Date.now() - 9 * hour; + // The ended native run that still writes the task's own workspace. + const writerRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: writerRunId, companyId, agentId, issueId: task.id, nativeIssueId: task.id, + runtimeMode: "native", status: "failed", errorCode: "native_finalization_superseded", + createdAt: new Date(base - 4 * hour), finishedAt: new Date(base), contextSnapshot: { issueId: task.id } }); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: task.id, name: "Task workspace", + mode: "isolated_workspace", strategyType: "project_primary", status: "active", providerType: "cloud_sandbox", + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 3, snapshotId: randomUUID(), + source: { kind: "empty" }, writerRunId, dirty: true, location: null } } }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, task.id)); + // Its queued runs failed setup on that hold, the last one finishing at 16:00:52. + let latest: typeof heartbeatRuns.$inferSelect | undefined; + for (const minutes of [5, 30, 60]) { + [latest] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: task.id, runtimeMode: "legacy", + status: "failed", errorCode: "setup_failed", error: REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE, + createdAt: new Date(base + minutes * 60_000 - 13_000), startedAt: new Date(base + minutes * 60_000 - 10_000), + finishedAt: new Date(base + minutes * 60_000), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { issueId: task.id } }).returning(); + } + const blockedAt = new Date(latest!.finishedAt!.getTime() + 4_388); + await db.update(issues).set({ blockedTransitionAt: blockedAt, statusVersion: 7 }).where(eq(issues.id, task.id)); + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: task.id, kind: "stranded_assigned_issue", + status: "active", ownerType: "board", previousOwnerAgentId: agentId, returnOwnerAgentId: agentId, cause: "stranded_assigned_issue", + fingerprint: `source_scoped_recovery:${companyId}:${task.id}:stranded_assigned_issue`, + nextAction: "Board operator: inspect the evidence, repair the runtime if appropriate, then explicitly retry the original owner, reassign, or intentionally resolve the task.", + evidence: { latestRunId: latest!.id, retryReason: null, sourceRunId: null, recoveryCause: "stranded_assigned_issue", + routingPolicy: "board_escalation_no_takeover_v1", sourceIssueId: task.id, previousStatus: "in_progress", + latestRunStatus: "failed", latestIssueStatus: "in_progress", latestRunErrorCode: "setup_failed" }, + createdAt: blockedAt, updatedAt: blockedAt }).returning(); + const laterDecision = options.laterDecision === undefined ? { fromStatus: "blocked", toStatus: "blocked" } : options.laterDecision; + if (laterDecision) { + // A late result of an earlier run kept the Blocked status it found. + const [earlier] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: task.id, status: "succeeded", + createdAt: new Date(base - 5 * hour), finishedAt: new Date(base - 4 * hour), contextSnapshot: { issueId: task.id } }).returning(); + const decisionId = randomUUID(); + await db.transaction(async tx => { + await tx.execute(sql`set local session_replication_role = replica`); + await tx.insert(statusDecisions).values({ id: decisionId, companyId, issueId: task.id, runId: earlier.id, + assessmentId: randomUUID(), decisionVersion: 1, policyVersion: "test", ...laterDecision, + reasonCode: "prior_status_terminal_preserved", decisionJson: {}, decisionDigest: "test", + createdAt: new Date(blockedAt.getTime() + 1_390) }); + }); + await db.update(issues).set({ lastStatusDecisionId: decisionId }).where(eq(issues.id, task.id)); + } + /** What the retained-file recovery leaves when it releases the hold (continue from the last save). */ + const releaseWriter = async () => { + const [row] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace.id)); + const current = (row.metadata as { remoteWorkspace: Record }).remoteWorkspace; + await db.update(executionWorkspaces).set({ metadata: { ...row.metadata, remoteWorkspace: { ...current, + epoch: Number(current.epoch) + 1, writerRunId: null, dirty: false, location: null } } }) + .where(eq(executionWorkspaces.id, workspace.id)); + }; + return { companyId, agentId, task, workspaceId: workspace.id, writerRunId, actionId: action.id, releaseWriter }; + } + + /** The pass the scheduled issue-recovery worker runs for a company with active recovery actions. */ + async function scheduledRecoveryPass(companyId: string) { + const [selected] = await db.selectDistinct({ companyId: issueRecoveryActions.companyId }).from(issueRecoveryActions) + .where(and(recoveryActionSweepCondition(), eq(issueRecoveryActions.companyId, companyId))); + if (!selected) return false; + await recoveryService(db, { enqueueWakeup: enqueueWakeup as never }).reconcileActiveRecoveryActions({ companyId }); + return true; + } + + const issue = async (id: string) => (await db.select().from(issues).where(eq(issues.id, id)))[0]!; + const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; + const wakesFor = (issueId: string) => wakes.filter(wake => (wake.options.payload as { issueId?: string }).issueId === issueId); + + it("continues the task once the hold is released, settling the board escalation", async () => { + const f = await seedStrandedTask(); + const workspaceBefore = (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)))[0]; + + // While the ended run still holds the workspace, nothing moves: a new run could only fail setup again. + expect(await scheduledRecoveryPass(f.companyId)).toBe(true); + expect(await issue(f.task.id)).toMatchObject({ status: "blocked" }); + expect(await action(f.actionId)).toMatchObject({ status: "active", ownerType: "board" }); + expect(wakesFor(f.task.id)).toEqual([]); + expect((await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)))[0]).toEqual(workspaceBefore); + + await f.releaseWriter(); + expect(await scheduledRecoveryPass(f.companyId)).toBe(true); + + expect(await issue(f.task.id)).toMatchObject({ status: "todo", blockedTransitionAt: null, executionWorkspaceId: f.workspaceId }); + const resolved = await action(f.actionId); + expect(resolved).toMatchObject({ status: "resolved", outcome: "restored", ownerType: "system", + resolutionNote: RELEASED_WORKSPACE_WRITER_OUTCOME, returnOwnerAgentId: f.agentId }); + expect(resolved.evidence).toMatchObject({ releasedWorkspaceWriter: { wake: "delivered", workspaceEpoch: 4 } }); + expect(String(resolved.nextAction)).not.toMatch(adminText); + // The assignee is woken exactly once; its run resumes normally in the same workspace. + expect(wakesFor(f.task.id)).toHaveLength(1); + expect(wakesFor(f.task.id)[0]).toMatchObject({ agentId: f.agentId, options: { reason: "issue_recovery_action_restored", + issueStateGuard: { statuses: ["todo"], assigneeAgentId: f.agentId } } }); + expect(wakesFor(f.task.id)[0].options.contextSnapshot).not.toHaveProperty("forceFreshSession"); + const notices = await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.companyId), + eq(activityLog.action, "issue.workspace_writer_released_continued"))); + expect(notices.map(row => row.entityId)).toEqual([f.task.id]); + expect(String(notices[0].details?.note)).not.toMatch(adminText); + + // A later pass changes nothing. + expect(await scheduledRecoveryPass(f.companyId)).toBe(false); + expect((await continueTasksAfterWorkspaceWriterReleased(db, null, { companyId: f.companyId })).continued).toEqual([]); + expect(wakesFor(f.task.id)).toHaveLength(1); + }, 60_000); + + it("delivers a wake that a crash left pending, and keeps backing off a refused one", async () => { + const f = await seedStrandedTask({ laterDecision: null }); + await f.releaseWriter(); + expect((await continueTasksAfterWorkspaceWriterReleased(db, null, { companyId: f.companyId })).continued).toEqual([f.task.id]); + expect((await action(f.actionId)).evidence).toMatchObject({ releasedWorkspaceWriter: { wake: "pending" } }); + + const refused = await continueTasksAfterWorkspaceWriterReleased(db, (async () => null) as never, { companyId: f.companyId }); + expect((await action(f.actionId)).evidence).toMatchObject({ releasedWorkspaceWriter: { wake: "pending", wakeFailures: 1 } }); + expect(refused.nextWakeRetryAt).toBeGreaterThan(Date.now()); + + // The scheduled worker keeps selecting the company while the wake is owed. + await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{releasedWorkspaceWriter,wakeRetryAt}', ${JSON.stringify(Date.now() - 1)}::jsonb)` }).where(eq(issueRecoveryActions.id, f.actionId)); + expect(await scheduledRecoveryPass(f.companyId)).toBe(true); + expect((await action(f.actionId)).evidence).toMatchObject({ releasedWorkspaceWriter: { wake: "delivered" } }); + expect(wakesFor(f.task.id)).toHaveLength(1); + }, 60_000); + + it("never wakes twice when a crash came after the wake created its run", async () => { + const f = await seedStrandedTask(); + await f.releaseWriter(); + expect((await continueTasksAfterWorkspaceWriterReleased(db, null, { companyId: f.companyId })).continued).toEqual([f.task.id]); + // The wake committed and created its run, then the process stopped before recording it as delivered. + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + reason: "issue_recovery_action_restored", status: "completed", runId: randomUUID(), + payload: { issueId: f.task.id, recoveryActionId: f.actionId, mutation: "recovery_action_resolution" } }); + expect((await action(f.actionId)).evidence).toMatchObject({ releasedWorkspaceWriter: { wake: "pending" } }); + + expect(await scheduledRecoveryPass(f.companyId)).toBe(true); + + expect(wakesFor(f.task.id)).toEqual([]); + expect((await action(f.actionId)).evidence).toMatchObject({ releasedWorkspaceWriter: { wake: "delivered" } }); + }, 60_000); + + it("keeps the task when a later decision chose the block, a dependency forbids work, or a newer run exists", async () => { + const chosen = await seedStrandedTask({ laterDecision: { fromStatus: "in_progress", toStatus: "blocked" } }); + const dependent = await seedStrandedTask(); + const [cancelled] = await db.insert(issues).values({ companyId: dependent.companyId, title: "Cancelled prerequisite", + status: "cancelled", issueNumber: 9 }).returning(); + await db.insert(issueRelations).values({ companyId: dependent.companyId, issueId: cancelled.id, + relatedIssueId: dependent.task.id, type: "blocks" }); + const newer = await seedStrandedTask(); + await db.insert(heartbeatRuns).values({ companyId: newer.companyId, agentId: newer.agentId, issueId: newer.task.id, + status: "queued", contextSnapshot: { issueId: newer.task.id } }); + + for (const f of [chosen, dependent, newer]) { + await f.releaseWriter(); + await scheduledRecoveryPass(f.companyId); + expect(await issue(f.task.id)).toMatchObject({ status: "blocked" }); + expect(await action(f.actionId), f.task.title).not.toMatchObject({ resolutionNote: RELEASED_WORKSPACE_WRITER_OUTCOME }); + expect(wakesFor(f.task.id)).toEqual([]); + } + }, 60_000); + + it("keeps a task whose setup failed for another reason", async () => { + const f = await seedStrandedTask(); + const latestRunId = String((await action(f.actionId)).evidence.latestRunId); + await db.update(heartbeatRuns).set({ error: "Project access denied" }).where(eq(heartbeatRuns.id, latestRunId)); + await f.releaseWriter(); + + await scheduledRecoveryPass(f.companyId); + + expect(await issue(f.task.id)).toMatchObject({ status: "blocked" }); + expect(await action(f.actionId)).toMatchObject({ status: "active" }); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toEqual([]); + }, 60_000); +}); diff --git a/server/src/services/recovery/blocker-stall-wakeups.ts b/server/src/services/recovery/blocker-stall-wakeups.ts index d8c1766e621..5afc157e7e6 100644 --- a/server/src/services/recovery/blocker-stall-wakeups.ts +++ b/server/src/services/recovery/blocker-stall-wakeups.ts @@ -96,6 +96,9 @@ export function recoveryActionSweepCondition() { // The pass returns its retry time, which schedules the company again. and(eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}#>>'{legacyWorkspaceRehome,wake}' = 'pending'`), + // Likewise a task continued once its workspace's writer let go. + and(eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}#>>'{releasedWorkspaceWriter,wake}' = 'pending'`), )!; } diff --git a/server/src/services/recovery/legacy-workspace-rehome.ts b/server/src/services/recovery/legacy-workspace-rehome.ts index 1e20517e7bb..733e425e791 100644 --- a/server/src/services/recovery/legacy-workspace-rehome.ts +++ b/server/src/services/recovery/legacy-workspace-rehome.ts @@ -1,5 +1,5 @@ -import { and, asc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; -import { heartbeatRuns, issueRecoveryActions, issues, statusDecisions, type Db } from "@paperclipai/db"; +import { and, asc, eq, gt, inArray, isNotNull, isNull, sql } from "drizzle-orm"; +import { agentWakeupRequests, heartbeatRuns, issueRecoveryActions, issues, statusDecisions, type Db } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; import { persistActivity, publishActivity } from "../activity-log.js"; import { releaseAncestorWorkspaceBinding } from "../execution-workspaces.js"; @@ -21,7 +21,7 @@ export const LEGACY_WORKSPACE_REHOME_NOTE = "This task was set up in its parent' + "workspace was held by another task's unsaved files, so the task could not start. Paperclip moved the task to its own " + "workspace from the project base and continues it there. No work was lost: nothing ran in the shared workspace."; -type RehomeWakeup = (agentId: string, options: { +export type RehomeWakeup = (agentId: string, options: { source: "automation"; triggerDetail: "system"; reason: string; payload: Record; requestedByActorType: "system"; requestedByActorId: string; contextSnapshot: Record; issueStateGuard: { statuses: string[]; assigneeAgentId: string }; @@ -79,63 +79,80 @@ export async function rehomeStrandedLegacyWorkspaceTasks(db: Db, wake: RehomeWak } // The move records its wake as pending in the same transaction. Deliver every // due pending wake, including one that a crash after an earlier commit left behind. - const nextWakeRetryAt = wake ? await deliverPendingRehomeWakes(db, wake, scope) : null; + const nextWakeRetryAt = wake ? await deliverPendingRecoveryWakes(db, wake, scope, REHOME_WAKE) : null; return { rehomed, nextWakeRetryAt }; } -const wakeRetryAt = sql`coalesce((${issueRecoveryActions.evidence}#>>'{legacyWorkspaceRehome,wakeRetryAt}')::bigint, 0)`; +/** Which recovery pass owes a wake: the evidence key it records it under, and how the woken run starts. */ +export type PendingRecoveryWake = { key: string; actorId: string; forceFreshSession: boolean }; +// A session saved in the shared workspace never resumes in the new one. +const REHOME_WAKE: PendingRecoveryWake = { key: "legacyWorkspaceRehome", actorId: "legacy-workspace-rehome", forceFreshSession: true }; -/** Delivers due pending wakes; returns when the earliest one still pending is due (epoch ms), or null. */ -async function deliverPendingRehomeWakes(db: Db, wake: RehomeWakeup, scope: { companyId?: string; issueId?: string }) { +const wakeField = (key: string, field: string) => sql`(${issueRecoveryActions.evidence} #>> array[${key}::text, ${field}::text])`; +const wakeRetryAt = (key: string) => sql`coalesce(${wakeField(key, "wakeRetryAt")}::bigint, 0)`; + +/** + * Delivers the due wakes a stranded-escalation settlement recorded as pending + * under `kind.key`; returns when the earliest one still pending is due (epoch + * ms), or null. + */ +export async function deliverPendingRecoveryWakes(db: Db, wake: RehomeWakeup, scope: { companyId?: string; issueId?: string }, + kind: PendingRecoveryWake) { const pendingIn = and( scope.companyId ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined, scope.issueId ? eq(issueRecoveryActions.sourceIssueId, scope.issueId) : undefined, eq(issueRecoveryActions.kind, STRANDED_KIND), eq(issueRecoveryActions.status, "resolved"), - sql`${issueRecoveryActions.evidence}#>>'{legacyWorkspaceRehome,wake}' = 'pending'`); + sql`${wakeField(kind.key, "wake")} = 'pending'`); const pending = await db.select({ actionId: issueRecoveryActions.id, companyId: issueRecoveryActions.companyId, issueId: issueRecoveryActions.sourceIssueId, agentId: issues.assigneeAgentId, - wakeFailures: sql`coalesce((${issueRecoveryActions.evidence}#>>'{legacyWorkspaceRehome,wakeFailures}')::int, 0)`, + wakeFailures: sql`coalesce(${wakeField(kind.key, "wakeFailures")}::int, 0)`, }).from(issueRecoveryActions) .innerJoin(issues, and(eq(issues.companyId, issueRecoveryActions.companyId), eq(issues.id, issueRecoveryActions.sourceIssueId))) // A wake that was refused waits for its retry time, so it cannot hold up the others. - .where(and(pendingIn, sql`${wakeRetryAt} <= ${Date.now()}::bigint`)) - .orderBy(wakeRetryAt, issueRecoveryActions.id).limit(PAGE_SIZE); + .where(and(pendingIn, sql`${wakeRetryAt(kind.key)} <= ${Date.now()}::bigint`)) + .orderBy(wakeRetryAt(kind.key), issueRecoveryActions.id).limit(PAGE_SIZE); for (const row of pending) { - // A task that changed since the move no longer owes this wake. + // A task that changed since the settlement no longer owes this wake. const [task] = await db.select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId }).from(issues) .where(and(eq(issues.companyId, row.companyId), eq(issues.id, row.issueId))); const owed = task?.status === "todo" && Boolean(task.assigneeAgentId) && task.assigneeAgentId === row.agentId; - let accepted = !owed; + // The wake's identity is its recovery action: a run already created for it + // (a crash after the wake committed, before this record changed) is the + // delivery, so the continuation never runs twice. + const [delivered] = owed ? await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, row.companyId), sql`${agentWakeupRequests.payload}->>'issueId' = ${row.issueId}`, + sql`${agentWakeupRequests.payload}->>'recoveryActionId' = ${row.actionId}`, isNotNull(agentWakeupRequests.runId))).limit(1) : []; + let accepted = !owed || Boolean(delivered); try { // Only a created run settles the wake; a refusal (for example a paused // company or disabled on-demand wakes) returns null and is retried. - if (owed) accepted = Boolean(await wake(row.agentId!, { + if (!accepted) accepted = Boolean(await wake(row.agentId!, { source: "automation", triggerDetail: "system", reason: "issue_recovery_action_restored", payload: { issueId: row.issueId, recoveryActionId: row.actionId, mutation: "recovery_action_resolution" }, - requestedByActorType: "system", requestedByActorId: "legacy-workspace-rehome", - // A session saved in the shared workspace never resumes in the new one. + requestedByActorType: "system", requestedByActorId: kind.actorId, contextSnapshot: { issueId: row.issueId, taskId: row.issueId, wakeReason: "issue_recovery_action_restored", - source: "issue.recovery_action_resolution", recoveryActionId: row.actionId, forceFreshSession: true }, + source: "issue.recovery_action_resolution", recoveryActionId: row.actionId, + ...(kind.forceFreshSession ? { forceFreshSession: true } : {}) }, issueStateGuard: { statuses: ["todo"], assigneeAgentId: row.agentId! }, })); } catch (error) { - logger.warn({ err: error, issueId: row.issueId }, "Moved task wake failed; it stays pending"); + logger.warn({ err: error, issueId: row.issueId }, "Recovery wake failed; it stays pending"); } const owner = and(eq(issueRecoveryActions.companyId, row.companyId), eq(issueRecoveryActions.id, row.actionId), - sql`${issueRecoveryActions.evidence}#>>'{legacyWorkspaceRehome,wake}' = 'pending'`); + sql`${wakeField(kind.key, "wake")} = 'pending'`); if (accepted) { await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, - '{legacyWorkspaceRehome,wake}', ${JSON.stringify(owed ? "delivered" : "not_owed")}::jsonb)` }).where(owner); + array[${kind.key}::text, 'wake'], ${JSON.stringify(owed ? "delivered" : "not_owed")}::jsonb)` }).where(owner); continue; } // Back off, up to an hour, so a wake that keeps being refused yields to the others. const failures = row.wakeFailures + 1; const retryAt = Date.now() + Math.min(REHOME_WAKE_RETRY_MAX_MS, REHOME_WAKE_RETRY_BASE_MS * 2 ** (failures - 1)); await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(jsonb_set(${issueRecoveryActions.evidence}, - '{legacyWorkspaceRehome,wakeFailures}', ${JSON.stringify(failures)}::jsonb), - '{legacyWorkspaceRehome,wakeRetryAt}', ${JSON.stringify(retryAt)}::jsonb)` }).where(owner); + array[${kind.key}::text, 'wakeFailures'], ${JSON.stringify(failures)}::jsonb), + array[${kind.key}::text, 'wakeRetryAt'], ${JSON.stringify(retryAt)}::jsonb)` }).where(owner); } - const [next] = await db.select({ at: sql`min(${wakeRetryAt})` }).from(issueRecoveryActions).where(pendingIn); + const [next] = await db.select({ at: sql`min(${wakeRetryAt(kind.key)})` }).from(issueRecoveryActions).where(pendingIn); return next?.at === null || next?.at === undefined ? null : Math.max(Number(next.at), Date.now()); } diff --git a/server/src/services/recovery/released-writer-escalation.ts b/server/src/services/recovery/released-writer-escalation.ts new file mode 100644 index 00000000000..2fe660eee9e --- /dev/null +++ b/server/src/services/recovery/released-writer-escalation.ts @@ -0,0 +1,135 @@ +import { and, asc, eq, gt, inArray, isNull, ne, or, sql } from "drizzle-orm"; +import { executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, statusDecisions, type Db } from "@paperclipai/db"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import { logger } from "../../middleware/logger.js"; +import { persistActivity, publishActivity } from "../activity-log.js"; +import { issueService } from "../issues.js"; +import { readRemoteWorkspaceWriterHold, REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE } from "../remote-workspace-writer-hold.js"; +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { deliverPendingRecoveryWakes, type PendingRecoveryWake, type RehomeWakeup } from "./legacy-workspace-rehome.js"; + +export const RELEASED_WORKSPACE_WRITER_SCHEMA = "paperclip.released-workspace-writer.v1"; +export const RELEASED_WORKSPACE_WRITER_OUTCOME = "workspace_writer_released_continued"; +const STRANDED_KIND = "stranded_assigned_issue"; +const BOARD_ESCALATION_POLICY = "board_escalation_no_takeover_v1"; +const PAGE_SIZE = 25; +// A block set by the stranded escalation lands right after its failed run. +const BLOCK_OWNERSHIP_SLACK_MS = 60_000; +const WAKE: PendingRecoveryWake = { key: "releasedWorkspaceWriter", actorId: "released-workspace-writer", forceFreshSession: false }; + +export const RELEASED_WORKSPACE_WRITER_NOTE = "This task could not start while an earlier run still held its workspace. " + + "That workspace is free again, so Paperclip continues the task."; + +/** + * A task whose runs failed setup because an ended run still held its remote + * workspace, before any provider work, which stranded-work recovery then + * blocked with a board-owned escalation. That failure belongs to the hold, not + * to the task: once the hold settles (the held files were saved, or the task + * continues from its last save), nothing a person could do remains. This pass + * settles the escalation in one transaction: it resolves with a named outcome, + * the task returns to `todo`, and a pending wake is recorded and delivered + * after commit (a wake lost to a crash stays pending for the next pass). The + * workspace itself is never changed here: the next run's admission takes it + * through the normal writer rule. Bounded and idempotent: a resolved + * escalation is never revisited, and a failed guard leaves the task as it is. + */ +export async function continueTasksAfterWorkspaceWriterReleased(db: Db, wake: RehomeWakeup | null, + scope: { companyId?: string; issueId?: string } = {}) { + const continued: string[] = []; + let cursor: string | null = null; + for (;;) { + const candidates: Array<{ actionId: string; companyId: string; issueId: string }> = await db.select({ + actionId: issueRecoveryActions.id, companyId: issueRecoveryActions.companyId, issueId: issueRecoveryActions.sourceIssueId, + }).from(issueRecoveryActions) + .innerJoin(issues, and(eq(issues.companyId, issueRecoveryActions.companyId), eq(issues.id, issueRecoveryActions.sourceIssueId))) + .where(and( + scope.companyId ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined, + scope.issueId ? eq(issueRecoveryActions.sourceIssueId, scope.issueId) : undefined, + cursor ? gt(issueRecoveryActions.id, cursor) : undefined, + eq(issueRecoveryActions.kind, STRANDED_KIND), inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}->>'routingPolicy' = ${BOARD_ESCALATION_POLICY}`, + sql`${issueRecoveryActions.evidence}->>'latestRunErrorCode' = 'setup_failed'`, + eq(issues.status, "blocked"), isNull(issues.hiddenAt), isNull(issues.assigneeUserId), + sql`${issues.assigneeAgentId} is not null`, sql`${issues.executionWorkspaceId} is not null`, + )).orderBy(asc(issueRecoveryActions.id)).limit(PAGE_SIZE); + for (const candidate of candidates) { + let settled: Awaited>; + try { + settled = await db.transaction(tx => continueOne(tx as unknown as Db, candidate)); + } catch (error) { + logger.warn({ err: error, issueId: candidate.issueId }, "Task held by an earlier writer stays blocked until the next sweep"); + continue; + } + if (!settled) continue; + continued.push(candidate.issueId); + try { publishActivity(settled.activity.publication); } catch { logger.warn({ issueId: candidate.issueId }, "Recovery activity delivery deferred"); } + } + if (candidates.length < PAGE_SIZE) break; + cursor = candidates.at(-1)!.actionId; + } + const nextWakeRetryAt = wake ? await deliverPendingRecoveryWakes(db, wake, scope, WAKE) : null; + return { continued, nextWakeRetryAt }; +} + +async function continueOne(tx: Db, input: { actionId: string; companyId: string; issueId: string }) { + // Lock order: task, then recovery action. + const [issue] = await tx.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update"); + const [action] = await tx.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, input.companyId), + eq(issueRecoveryActions.id, input.actionId))).for("update"); + if (!issue || !action || issue.status !== "blocked" || issue.hiddenAt || issue.assigneeUserId || !issue.assigneeAgentId + || !issue.executionWorkspaceId || action.sourceIssueId !== issue.id || action.kind !== STRANDED_KIND + || !["active", "escalated"].includes(action.status) || action.ownerType !== "board" + || action.evidence.routingPolicy !== BOARD_ESCALATION_POLICY) return null; + const latestRunId = action.evidence.latestRunId; + if (typeof latestRunId !== "string") return null; + const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.id, latestRunId))); + // Only a setup failure on a held workspace, before any provider work, by the current assignee. + const recovery = (run?.resultJson?.executionRecovery ?? null) as { kind?: unknown; providerWorkStarted?: unknown } | null; + if (!run || run.agentId !== issue.assigneeAgentId || run.status !== "failed" || run.errorCode !== "setup_failed" + || run.error !== REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE || recovery?.kind !== "bootstrap" || recovery.providerWorkStarted !== false + || (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId) !== issue.id) return null; + // That failed run is still the task's latest: nothing newer has started or queued for it. + const [newer] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), ne(heartbeatRuns.id, run.id), gt(heartbeatRuns.createdAt, run.createdAt), + // Each binding has its own (company, task, created) index. + or(eq(heartbeatRuns.issueId, issue.id), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}::text`))) + .limit(1); + if (newer) return null; + // The Blocked status belongs to this escalation: set after that run, and no blocker keeps it. + const blockedAt = issue.blockedTransitionAt?.getTime(); + if (blockedAt === undefined || !run.finishedAt || blockedAt < run.finishedAt.getTime() - 1_000 + || blockedAt > action.updatedAt.getTime() + BLOCK_OWNERSHIP_SLACK_MS) return null; + // A status decision after the block owns the status now, unless it only kept + // the block it found (a late result never decides a status it did not set). + if (issue.lastStatusDecisionId) { + const [decision] = await tx.select({ createdAt: statusDecisions.createdAt, fromStatus: statusDecisions.fromStatus, + toStatus: statusDecisions.toStatus }).from(statusDecisions).where(and( + eq(statusDecisions.companyId, input.companyId), eq(statusDecisions.id, issue.lastStatusDecisionId))); + if (!decision || decision.createdAt.getTime() >= blockedAt + && !(decision.fromStatus === "blocked" && decision.toStatus === "blocked")) return null; + } + // The shared dependency rule: any blocker that still forbids work keeps the task blocked. + const readiness = await issueService(tx).listDependencyReadiness(input.companyId, [issue.id], tx); + if (readiness.get(issue.id)?.isDependencyReady !== true) return null; + // The hold that refused the run has settled: workspace reservation would now + // admit a new writer. A writer still saving or running keeps the task as it is. + const [workspace] = await tx.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, issue.executionWorkspaceId))); + const authority = workspace ? readRemoteWorkspaceAuthority(workspace.metadata) : null; + if (!authority || (await readRemoteWorkspaceWriterHold(tx, { companyId: input.companyId, authority })).kind !== "free") return null; + const now = new Date(); + await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: now, updatedAt: now, + ownerType: "system", ownerAgentId: null, ownerUserId: null, returnOwnerAgentId: issue.assigneeAgentId, + resolutionNote: RELEASED_WORKSPACE_WRITER_OUTCOME, nextAction: RELEASED_WORKSPACE_WRITER_NOTE, wakePolicy: null, monitorPolicy: null, + evidence: { ...action.evidence, [WAKE.key]: { schema: RELEASED_WORKSPACE_WRITER_SCHEMA, runId: run.id, + workspaceEpoch: authority.epoch, recordedAt: now.toISOString(), wake: "pending" } }, + }).where(and(eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, action.id))); + await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, + blockedOwnerNotifiedAt: null, updatedAt: now }).where(and(eq(issues.companyId, input.companyId), eq(issues.id, issue.id))); + await notifyStatusCardIssueChange(tx, input.companyId); + const activity = await persistActivity(tx, { companyId: input.companyId, actorType: "system", actorId: WAKE.actorId, + action: "issue.workspace_writer_released_continued", entityType: "issue", entityId: issue.id, runId: run.id, + details: { recoveryActionId: action.id, status: "todo", note: RELEASED_WORKSPACE_WRITER_NOTE, _previous: { status: "blocked" } } }); + return { activity }; +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 7ea066bc377..dc5eaffb7fb 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -184,6 +184,7 @@ import { withRecoveryContext } from "./status-only-context.js"; import { currentRunFailureText } from "../native-runtime/native-exact-resume-failure.js"; import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js"; import { rehomeStrandedLegacyWorkspaceTasks } from "./legacy-workspace-rehome.js"; +import { continueTasksAfterWorkspaceWriterReleased } from "./released-writer-escalation.js"; import { collectDispositionRepairSourceState, readDispositionRepairDurableWaitingState, @@ -3546,6 +3547,11 @@ export function recoveryService( // is an active recovery action, so this company-scoped sweep reaches it). const rehome = await rehomeStrandedLegacyWorkspaceTasks(db, deps.enqueueWakeup, { companyId: opts?.companyId }) .catch(error => { logger.warn({ err: error }, "Legacy shared-workspace rehome deferred to the next sweep"); return null; }); + // A task whose runs failed setup only because an earlier run held its own + // workspace continues once that hold settles (its board escalation is an + // active recovery action, so this company-scoped sweep reaches it). + const releasedWriter = await continueTasksAfterWorkspaceWriterReleased(db, deps.enqueueWakeup, { companyId: opts?.companyId }) + .catch(error => { logger.warn({ err: error }, "Released-writer continuation deferred to the next sweep"); return null; }); const rows = await db .select({ action: issueRecoveryActions, @@ -3718,7 +3724,7 @@ export function recoveryService( const stalls = await wakeWaitersOfStalledIssues(db, deps.enqueueWakeup as BlockerStallWakeup, { companyId: opts?.companyId }); // A failed delivery is reported so the recovery worker retries this pass. // The earliest owed wake of a moved legacy subtask is a deadline of this pass too. - const retryAts = [stalls.retryAt, rehome?.nextWakeRetryAt].filter((at): at is number => typeof at === "number"); + const retryAts = [stalls.retryAt, rehome?.nextWakeRetryAt, releasedWriter?.nextWakeRetryAt].filter((at): at is number => typeof at === "number"); return { ...result, blockerStallWakes: stalls.woken, blockerStallWakeFailures: stalls.failed, blockerStallRetryAt: retryAts.length ? Math.min(...retryAts) : null }; }