From 1791534795d60fb1416d3624606aa77171eeedb4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 20:18:33 -0600 Subject: [PATCH] fix(recovery): the isolated continuation runs before the saved follow-up Releasing a run the fallback replaced promoted the person's saved follow-up while the fallback's isolated continuation was still waiting for delivery. The follow-up then became the task's next run without the isolation marker (refused at admission on a projectless task), and the delivery worker counted it as the continuation and skipped the isolated run. Release now leaves the follow-up deferred while that continuation is pending; the continuation's own release promotes it. Co-Authored-By: Claude Opus 5.5 --- .../wake-queue/adapters/postgres.test.ts | 24 +++++++++++++++---- .../modules/wake-queue/adapters/postgres.ts | 11 ++++++++- .../services/fresh-continuation-fallback.ts | 13 +++++++--- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 341cb239011..aac74f1f3bc 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -2,7 +2,7 @@ import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/exter import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js"; import { instanceSettingsService } from "../../../services/instance-settings.js"; import { randomBytes, randomUUID } from "node:crypto"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { Db } from "@paperclipai/db"; import { createSystemRecovery } from "../../../services/system-recovery-state.js"; @@ -506,8 +506,12 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { }); it.each([ - ["fresh_isolated_workspace", "promoted"], ["remote_epoch", "promoted"], ["writer_ended", "released"], - ] as const)("promotes saved input behind an exhausted restart hold once a %s continuation replaced the run (%s)", async (isolation, expected) => { + ["fresh_isolated_workspace", "promoted", "not_requested"], ["remote_epoch", "promoted", "not_requested"], + ["writer_ended", "released", "not_requested"], + // The isolated continuation is still to be delivered: it runs first and binds the new workspace; + // promoting the follow-up now would make it the task's next run, without the isolation. + ["fresh_isolated_workspace", "released", "pending"], + ] as const)("promotes saved input behind an exhausted restart hold once a %s continuation replaced the run (%s, continuation %s)", async (isolation, expected, continuationDelivery) => { // Restart recovery exhausted its retries (systemRecovery.state exhausted, no stop receipt), then the // fallback replaced the run. An isolated continuation reuses none of the old executor's workspace, // so the manual-admission hold no longer stands between the task and the person's follow-up. @@ -523,7 +527,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId, cause: "fresh_continuation", fingerprint: `fresh-continuation:${runId}`, nextAction: "Continuing.", resolvedAt: new Date(), - evidence: { runId, freshContinuation: { sourceRunId: runId, isolation, delivery: "saved_input" }, continuationDelivery: "not_requested" } }); + evidence: { runId, freshContinuation: { sourceRunId: runId, isolation, + delivery: continuationDelivery === "pending" ? "recorded" : "saved_input" }, continuationDelivery } }); const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "responsible-user", body: "Follow-up during the run" }).returning(); const wakeId = await seedDeferredWake({ companyId, agentId, issueId, requestedByActorId: "responsible-user", payload: { _paperclipWakeContext: { issueId, wakeReason: "issue_commented", wakeCommentIds: [comment.id] } }, @@ -536,6 +541,17 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)); if (expected === "promoted") expect(wake.status).not.toBe("deferred_issue_execution"); else expect(wake.status).toBe("deferred_issue_execution"); + if (continuationDelivery !== "pending") return; + // The isolated continuation is delivered and runs; its release promotes the follow-up behind it. + const continuationRunId = await seedRun({ companyId, agentId, status: "succeeded", contextSnapshot: { issueId } }); + await db.update(heartbeatRuns).set({ finishedAt: new Date() }).where(eq(heartbeatRuns.id, continuationRunId)); + await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} + || ${JSON.stringify({ continuationDelivery: "delivered", continuationRunId })}::jsonb` }) + .where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect((await release({ companyId, runId: continuationRunId, now: new Date(), suppressImmediateRecovery: true })).outcome.kind) + .toBe("promoted"); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0].status) + .not.toBe("deferred_issue_execution"); }); it.each([ diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 36d479a679d..ff582a93cc6 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1303,7 +1303,7 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd // reuses none of them, so they no longer hold the task's next turn; any // other replacement still waits for them. const isolatedReplacement = agentRunSupersededByFreshContinuation(run) - && await freshContinuationIsolatesRun(tx, { companyId: input.companyId, runId: run.id }); + ? await freshContinuationIsolatesRun(tx, { companyId: input.companyId, runId: run.id }) : false; if (!isolatedReplacement && await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) { return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } @@ -1469,6 +1469,15 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd // A release must leave deferred messages intact while execution is held. // The finishing conversation may still own its lease until finally cleanup. // Allow only bounded retry planning in that case; admission stays gated. + // The fallback's isolated continuation runs before any saved input: it + // alone carries the marker that prepares the new workspace and binds it. + // Promoting the follow-up now would make it the task's next run without + // that isolation (and the delivery worker would then skip the + // continuation). The follow-up stays deferred and is promoted behind + // the continuation when that run is released. + if (isolatedReplacement && isolatedReplacement.continuationPending) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } const executionBlocker = await getExecutionBlocker(tx, issueRow.companyId, issueRow.id); const recoveryOnly = Boolean(executionBlocker && executionBlocker.cause === "execution_owner_active" && executionBlocker.runId === run.id && diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 08082214efc..adc6eb81df5 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -326,12 +326,19 @@ export async function isFreshIsolatedContinuationWake(db: Db, input: { companyId * Holds that only protect the old executor's resources then no longer stand * between the task and its next turn. */ -export async function freshContinuationIsolatesRun(db: Db, input: { companyId: string; runId: string }): Promise { - const [record] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( +export async function freshContinuationIsolatesRun(db: Db, input: { companyId: string; runId: string }): Promise { + const [record] = await db.select({ isolation: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}'`, + delivery: sql`${issueRecoveryActions.evidence}->>'continuationDelivery'` }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, input.companyId), sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${input.runId}`, sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}' in ('fresh_isolated_workspace', 'remote_epoch')`)).limit(1); - return Boolean(record); + if (!record) return false; + // Only the recorded continuation carries the isolation marker that prepares + // the new workspace, so while it is pending it must be the task's next run. + return { continuationPending: record.isolation === "fresh_isolated_workspace" && record.delivery === "pending" }; } export async function ensureFreshContinuation(db: Db, input: Input): Promise {