From cc4dfa1696682cdca1d381abbc7a342dca1926c6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 20:01:17 -0600 Subject: [PATCH] fix(recovery): run the isolated continuation before saved input; an isolated replacement releases the old executor's holds Saved input carries no isolation marker, so it cannot prepare the new workspace the fallback chose: on a projectless task admission refused it once the binding was cleared. The fallback now records its own isolated continuation first whenever the old writer is not proven stopped, and saved input follows it. When restart recovery had exhausted its retries, the manual-admission hold and the pending-cleanup check still stopped the follow-up behind a run the fallback replaced. Both protect the old executor's workspace, which an isolated continuation (new workspace or abandoned remote epoch) never reuses, so they no longer hold the task's next turn there; any other replacement still waits. Co-Authored-By: Claude Opus 5.5 --- .../wake-queue/adapters/postgres.test.ts | 34 +++++++++++++++++++ .../modules/wake-queue/adapters/postgres.ts | 11 ++++-- .../fresh-continuation-fallback.test.ts | 8 +++-- .../services/fresh-continuation-fallback.ts | 24 ++++++++++--- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 80cb1559e55..341cb239011 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -5,6 +5,7 @@ import { randomBytes, randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { Db } from "@paperclipai/db"; +import { createSystemRecovery } from "../../../services/system-recovery-state.js"; import { activityLog, agentWakeupRequests, @@ -504,6 +505,39 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { } }); + it.each([ + ["fresh_isolated_workspace", "promoted"], ["remote_epoch", "promoted"], ["writer_ended", "released"], + ] as const)("promotes saved input behind an exhausted restart hold once a %s continuation replaced the run (%s)", async (isolation, expected) => { + // Restart recovery exhausted its retries (systemRecovery.state exhausted, no stop receipt), then the + // fallback replaced the run. An isolated continuation reuses none of the old executor's workspace, + // so the manual-admission hold no longer stands between the task and the person's follow-up. + const companyId = await seedCompany(), agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, status: "in_progress", assigneeAgentId: agentId }); + const runId = await seedRun({ companyId, agentId, status: "failed", contextSnapshot: { issueId } }); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issueId, finishedAt: new Date(), + errorCode: "native_restart_recovery_blocked", resultJson: { freshContinuationSupersededAt: new Date().toISOString() }, + }).where(eq(heartbeatRuns.id, runId)); + await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "terminal_failure", + recoveryState: "blocked", failureCode: "native_restart_recovery_blocked", + failureDetail: { systemRecovery: { ...createSystemRecovery(runId), state: "exhausted" }, replacementDenied: "fresh_continuation" } }); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId, + cause: "fresh_continuation", fingerprint: `fresh-continuation:${runId}`, nextAction: "Continuing.", resolvedAt: new Date(), + evidence: { runId, freshContinuation: { sourceRunId: runId, isolation, delivery: "saved_input" }, continuationDelivery: "not_requested" } }); + const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "responsible-user", body: "Follow-up during the run" }).returning(); + const wakeId = await seedDeferredWake({ companyId, agentId, issueId, requestedByActorId: "responsible-user", + payload: { _paperclipWakeContext: { issueId, wakeReason: "issue_commented", wakeCommentIds: [comment.id] } }, + }); + const release = createReleaseIssueExecution({ issueLock: createPostgresWakeQueueAdapter(db, stubDeps), + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); + const result = await release({ companyId, runId, now: new Date(), suppressImmediateRecovery: true, resumeStrandedQueue: true }); + expect(result.outcome.kind).toBe(expected); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)); + if (expected === "promoted") expect(wake.status).not.toBe("deferred_issue_execution"); + else expect(wake.status).toBe("deferred_issue_execution"); + }); + it.each([ "completed", "multiple_comments", "repeated_reference", "parent_reference", "mixed_issue_references", "mixed_foreign_references", "mixed_unknown_references", "no_comments", "missing_comment", "human_comment", "other_run_comment", "foreign_comment", "other_issue_comment", "deleted_comment", diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 93e5c6c5496..36d479a679d 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -19,6 +19,7 @@ import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm" import { readReleaseRun, type ReleaseRun } from "./release-run-read.js"; import { isGovernancePauseStop } from "../../../services/governance-pause-stop.js"; import { agentRunSupersededByFreshContinuation } from "../../../agent-run-cancellation.js"; +import { freshContinuationIsolatesRun } from "../../../services/fresh-continuation-fallback.js"; import type { Db } from "@paperclipai/db"; import { deliveredQueuedCommentIds, EXTERNAL_CHAT_MESSAGE_WAKE_MUTATION, EXTERNAL_CHAT_MESSAGE_WAKE_REASON } from "../../../services/queued-comment-delivery.js"; import { extractIssueReferenceIdentifiers, isUuidLike } from "@paperclipai/shared"; @@ -1297,10 +1298,16 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd : []; if (!finalized) return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } - if (await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) { + // These two holds protect the old executor's workspace and resources. A + // continuation in a new isolated workspace (or an abandoned remote epoch) + // reuses none of them, so they no longer hold the task's next turn; any + // other replacement still waits for them. + const isolatedReplacement = agentRunSupersededByFreshContinuation(run) + && await freshContinuationIsolatesRun(tx, { companyId: input.companyId, runId: run.id }); + if (!isolatedReplacement && await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) { return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } - if (input.resumeStrandedQueue && await hasPendingRunCleanup(tx, input.companyId, run.id)) { + if (input.resumeStrandedQueue && !isolatedReplacement && await hasPendingRunCleanup(tx, input.companyId, run.id)) { return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 289fbd58d60..f88dbbb13cc 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -883,18 +883,20 @@ suite("fresh-continuation fallback", () => { }, 60_000); // The isolation decision is a property of the task's next run, whatever path starts it. - it("moves the task off the old writer's workspace for every next run, saved input included", async () => { + it("moves the task off the old writer's workspace and runs the isolated continuation before saved input", async () => { const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: f.issueId, mode: "isolated", strategyType: "git_worktree", name: "Task workspace", cwd: "/tmp/old-writer" }).returning(); await db.update(issues).set({ executionWorkspaceId: workspace!.id, workspaceBindingRevision: 2 }).where(eq(issues.id, f.issueId)); await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000), contextSnapshot: { issueId: f.issueId, executionWorkspaceId: workspace!.id } }).where(eq(heartbeatRuns.id, f.runId)); - // A person's saved message is waiting: it continues the task itself. + // A person's saved message is waiting. It carries no isolation marker, so it cannot prepare the + // new workspace (a projectless task would be refused at admission once the binding is cleared): + // the isolated continuation runs first and the saved message follows it. await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", status: "deferred_issue_execution", requestedByActorType: "user", requestedByActorId: userId, payload: { issueId: f.issueId } }); expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) - .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "saved_input" }); + .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "recorded" }); // No delivery path can reach the old directory: the task is no longer bound to it. expect(await issue(f.issueId)).toMatchObject({ executionWorkspaceId: null, workspaceBindingRevision: 3 }); const [kept] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace!.id)); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 68f035fb35f..08082214efc 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -320,6 +320,20 @@ export async function isFreshIsolatedContinuationWake(db: Db, input: { companyId contextSnapshot: input.contextSnapshot ?? {} }, input.issueId)) !== null; } +/** + * True when the fallback replaced this run and its continuation reuses none of + * the run's workspace: a new isolated workspace or an abandoned remote epoch. + * Holds that only protect the old executor's resources then no longer stand + * between the task and its next turn. + */ +export async function freshContinuationIsolatesRun(db: Db, input: { companyId: string; runId: string }): Promise { + const [record] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${input.runId}`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}' in ('fresh_isolated_workspace', 'remote_epoch')`)).limit(1); + return Boolean(record); +} + export async function ensureFreshContinuation(db: Db, input: Input): Promise { return db.transaction(tx => continueFreshInTransaction(tx as unknown as Db, input)); } @@ -518,10 +532,12 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< // Never claim a stop that was not proven: record what isolates the successor. const isolation: FreshContinuationIsolation = fenced || remoteSource ? "remote_epoch" : writer === "ended" && !waitBounded ? "writer_ended" : "fresh_isolated_workspace"; - // With no task workspace binding to move, only a run that carries the - // isolation in its context can avoid the shared directory: the recorded - // continuation runs first, and saved input follows it. - if (isolation === "fresh_isolated_workspace" && !issue.executionWorkspaceId && delivery === "saved_input") delivery = "recorded"; + // Only a run that carries the isolation in its context prepares the new + // workspace (create_new). Saved input carries no such marker: on a task with + // no project it would be refused at admission once the binding is cleared, + // and otherwise could resolve the old directory. So the recorded + // continuation runs first, binds the new workspace, and saved input follows. + if (isolation === "fresh_isolated_workspace" && delivery === "saved_input") delivery = "recorded"; const unverified = await unverifiedActions(tx, run); const note = freshContinuationNote({ runId: run.id, unverified, workspace: isolation === "writer_ended" ? "kept" : "restored_last_save" });