fix(recovery): run the isolated continuation before saved input; an isolated replacement releases the old executor's holds

Saved input carries no isolation marker, so it cannot prepare the new workspace
the fallback chose: on a projectless task admission refused it once the binding
was cleared. The fallback now records its own isolated continuation first
whenever the old writer is not proven stopped, and saved input follows it.

When restart recovery had exhausted its retries, the manual-admission hold and
the pending-cleanup check still stopped the follow-up behind a run the fallback
replaced. Both protect the old executor's workspace, which an isolated
continuation (new workspace or abandoned remote epoch) never reuses, so they no
longer hold the task's next turn there; any other replacement still waits.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 20:01:17 -06:00
1 parent 312922ed53
commit cc4dfa1696
4 files changed
+68 -9

No files matched your search

@@ -5,6 +5,7 @@ import { randomBytes, randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import { createSystemRecovery } from "../../../services/system-recovery-state.js";
import {
activityLog,
agentWakeupRequests,
@@ -504,6 +505,39 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
}
});
it.each([
["fresh_isolated_workspace", "promoted"], ["remote_epoch", "promoted"], ["writer_ended", "released"],
] as const)("promotes saved input behind an exhausted restart hold once a %s continuation replaced the run (%s)", async (isolation, expected) => {
// Restart recovery exhausted its retries (systemRecovery.state exhausted, no stop receipt), then the
// fallback replaced the run. An isolated continuation reuses none of the old executor's workspace,
// so the manual-admission hold no longer stands between the task and the person's follow-up.
const companyId = await seedCompany(), agentId = await seedAgent({ companyId });
const issueId = await seedIssue({ companyId, status: "in_progress", assigneeAgentId: agentId });
const runId = await seedRun({ companyId, agentId, status: "failed", contextSnapshot: { issueId } });
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issueId, finishedAt: new Date(),
errorCode: "native_restart_recovery_blocked", resultJson: { freshContinuationSupersededAt: new Date().toISOString() },
}).where(eq(heartbeatRuns.id, runId));
await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "terminal_failure",
recoveryState: "blocked", failureCode: "native_restart_recovery_blocked",
failureDetail: { systemRecovery: { ...createSystemRecovery(runId), state: "exhausted" }, replacementDenied: "fresh_continuation" } });
await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog",
status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId,
cause: "fresh_continuation", fingerprint: `fresh-continuation:${runId}`, nextAction: "Continuing.", resolvedAt: new Date(),
evidence: { runId, freshContinuation: { sourceRunId: runId, isolation, delivery: "saved_input" }, continuationDelivery: "not_requested" } });
const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "responsible-user", body: "Follow-up during the run" }).returning();
const wakeId = await seedDeferredWake({ companyId, agentId, issueId, requestedByActorId: "responsible-user",
payload: { _paperclipWakeContext: { issueId, wakeReason: "issue_commented", wakeCommentIds: [comment.id] } },
});
const release = createReleaseIssueExecution({ issueLock: createPostgresWakeQueueAdapter(db, stubDeps),
recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined },
});
const result = await release({ companyId, runId, now: new Date(), suppressImmediateRecovery: true, resumeStrandedQueue: true });
expect(result.outcome.kind).toBe(expected);
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId));
if (expected === "promoted") expect(wake.status).not.toBe("deferred_issue_execution");
else expect(wake.status).toBe("deferred_issue_execution");
});
it.each([
"completed", "multiple_comments", "repeated_reference", "parent_reference", "mixed_issue_references", "mixed_foreign_references", "mixed_unknown_references", "no_comments", "missing_comment",
"human_comment", "other_run_comment", "foreign_comment", "other_issue_comment", "deleted_comment",
@@ -19,6 +19,7 @@ import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm"
import { readReleaseRun, type ReleaseRun } from "./release-run-read.js";
import { isGovernancePauseStop } from "../../../services/governance-pause-stop.js";
import { agentRunSupersededByFreshContinuation } from "../../../agent-run-cancellation.js";
import { freshContinuationIsolatesRun } from "../../../services/fresh-continuation-fallback.js";
import type { Db } from "@paperclipai/db";
import { deliveredQueuedCommentIds, EXTERNAL_CHAT_MESSAGE_WAKE_MUTATION, EXTERNAL_CHAT_MESSAGE_WAKE_REASON } from "../../../services/queued-comment-delivery.js";
import { extractIssueReferenceIdentifiers, isUuidLike } from "@paperclipai/shared";
@@ -1297,10 +1298,16 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
: [];
if (!finalized) return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
}
if (await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) {
// These two holds protect the old executor's workspace and resources. A
// continuation in a new isolated workspace (or an abandoned remote epoch)
// reuses none of them, so they no longer hold the task's next turn; any
// other replacement still waits for them.
const isolatedReplacement = agentRunSupersededByFreshContinuation(run)
&& await freshContinuationIsolatesRun(tx, { companyId: input.companyId, runId: run.id });
if (!isolatedReplacement && await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) {
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
}
if (input.resumeStrandedQueue && await hasPendingRunCleanup(tx, input.companyId, run.id)) {
if (input.resumeStrandedQueue && !isolatedReplacement && await hasPendingRunCleanup(tx, input.companyId, run.id)) {
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
}
@@ -883,18 +883,20 @@ suite("fresh-continuation fallback", () => {
}, 60_000);
// The isolation decision is a property of the task's next run, whatever path starts it.
it("moves the task off the old writer's workspace for every next run, saved input included", async () => {
it("moves the task off the old writer's workspace and runs the isolated continuation before saved input", async () => {
const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" });
const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: f.issueId, mode: "isolated",
strategyType: "git_worktree", name: "Task workspace", cwd: "/tmp/old-writer" }).returning();
await db.update(issues).set({ executionWorkspaceId: workspace!.id, workspaceBindingRevision: 2 }).where(eq(issues.id, f.issueId));
await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000),
contextSnapshot: { issueId: f.issueId, executionWorkspaceId: workspace!.id } }).where(eq(heartbeatRuns.id, f.runId));
// A person's saved message is waiting: it continues the task itself.
// A person's saved message is waiting. It carries no isolation marker, so it cannot prepare the
// new workspace (a projectless task would be refused at admission once the binding is cleared):
// the isolated continuation runs first and the saved message follows it.
await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", status: "deferred_issue_execution",
requestedByActorType: "user", requestedByActorId: userId, payload: { issueId: f.issueId } });
expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" }))
.toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "saved_input" });
.toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "recorded" });
// No delivery path can reach the old directory: the task is no longer bound to it.
expect(await issue(f.issueId)).toMatchObject({ executionWorkspaceId: null, workspaceBindingRevision: 3 });
const [kept] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace!.id));
@@ -320,6 +320,20 @@ export async function isFreshIsolatedContinuationWake(db: Db, input: { companyId
contextSnapshot: input.contextSnapshot ?? {} }, input.issueId)) !== null;
}
/**
* True when the fallback replaced this run and its continuation reuses none of
* the run's workspace: a new isolated workspace or an abandoned remote epoch.
* Holds that only protect the old executor's resources then no longer stand
* between the task and its next turn.
*/
export async function freshContinuationIsolatesRun(db: Db, input: { companyId: string; runId: string }): Promise<boolean> {
const [record] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, input.companyId),
sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${input.runId}`,
sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}' in ('fresh_isolated_workspace', 'remote_epoch')`)).limit(1);
return Boolean(record);
}
export async function ensureFreshContinuation(db: Db, input: Input): Promise<FreshContinuationResult> {
return db.transaction(tx => continueFreshInTransaction(tx as unknown as Db, input));
}
@@ -518,10 +532,12 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise<
// Never claim a stop that was not proven: record what isolates the successor.
const isolation: FreshContinuationIsolation = fenced || remoteSource ? "remote_epoch"
: writer === "ended" && !waitBounded ? "writer_ended" : "fresh_isolated_workspace";
// With no task workspace binding to move, only a run that carries the
// isolation in its context can avoid the shared directory: the recorded
// continuation runs first, and saved input follows it.
if (isolation === "fresh_isolated_workspace" && !issue.executionWorkspaceId && delivery === "saved_input") delivery = "recorded";
// Only a run that carries the isolation in its context prepares the new
// workspace (create_new). Saved input carries no such marker: on a task with
// no project it would be refused at admission once the binding is cleared,
// and otherwise could resolve the old directory. So the recorded
// continuation runs first, binds the new workspace, and saved input follows.
if (isolation === "fresh_isolated_workspace" && delivery === "saved_input") delivery = "recorded";
const unverified = await unverifiedActions(tx, run);
const note = freshContinuationNote({ runId: run.id, unverified,
workspace: isolation === "writer_ended" ? "kept" : "restored_last_save" });