diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 80cb1559e55..341cb239011 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -5,6 +5,7 @@ import { randomBytes, randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { Db } from "@paperclipai/db"; +import { createSystemRecovery } from "../../../services/system-recovery-state.js"; import { activityLog, agentWakeupRequests, @@ -504,6 +505,39 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { } }); + it.each([ + ["fresh_isolated_workspace", "promoted"], ["remote_epoch", "promoted"], ["writer_ended", "released"], + ] as const)("promotes saved input behind an exhausted restart hold once a %s continuation replaced the run (%s)", async (isolation, expected) => { + // Restart recovery exhausted its retries (systemRecovery.state exhausted, no stop receipt), then the + // fallback replaced the run. An isolated continuation reuses none of the old executor's workspace, + // so the manual-admission hold no longer stands between the task and the person's follow-up. + const companyId = await seedCompany(), agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, status: "in_progress", assigneeAgentId: agentId }); + const runId = await seedRun({ companyId, agentId, status: "failed", contextSnapshot: { issueId } }); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issueId, finishedAt: new Date(), + errorCode: "native_restart_recovery_blocked", resultJson: { freshContinuationSupersededAt: new Date().toISOString() }, + }).where(eq(heartbeatRuns.id, runId)); + await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "terminal_failure", + recoveryState: "blocked", failureCode: "native_restart_recovery_blocked", + failureDetail: { systemRecovery: { ...createSystemRecovery(runId), state: "exhausted" }, replacementDenied: "fresh_continuation" } }); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId, + cause: "fresh_continuation", fingerprint: `fresh-continuation:${runId}`, nextAction: "Continuing.", resolvedAt: new Date(), + evidence: { runId, freshContinuation: { sourceRunId: runId, isolation, delivery: "saved_input" }, continuationDelivery: "not_requested" } }); + const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "responsible-user", body: "Follow-up during the run" }).returning(); + const wakeId = await seedDeferredWake({ companyId, agentId, issueId, requestedByActorId: "responsible-user", + payload: { _paperclipWakeContext: { issueId, wakeReason: "issue_commented", wakeCommentIds: [comment.id] } }, + }); + const release = createReleaseIssueExecution({ issueLock: createPostgresWakeQueueAdapter(db, stubDeps), + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); + const result = await release({ companyId, runId, now: new Date(), suppressImmediateRecovery: true, resumeStrandedQueue: true }); + expect(result.outcome.kind).toBe(expected); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)); + if (expected === "promoted") expect(wake.status).not.toBe("deferred_issue_execution"); + else expect(wake.status).toBe("deferred_issue_execution"); + }); + it.each([ "completed", "multiple_comments", "repeated_reference", "parent_reference", "mixed_issue_references", "mixed_foreign_references", "mixed_unknown_references", "no_comments", "missing_comment", "human_comment", "other_run_comment", "foreign_comment", "other_issue_comment", "deleted_comment", diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 93e5c6c5496..36d479a679d 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -19,6 +19,7 @@ import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm" import { readReleaseRun, type ReleaseRun } from "./release-run-read.js"; import { isGovernancePauseStop } from "../../../services/governance-pause-stop.js"; import { agentRunSupersededByFreshContinuation } from "../../../agent-run-cancellation.js"; +import { freshContinuationIsolatesRun } from "../../../services/fresh-continuation-fallback.js"; import type { Db } from "@paperclipai/db"; import { deliveredQueuedCommentIds, EXTERNAL_CHAT_MESSAGE_WAKE_MUTATION, EXTERNAL_CHAT_MESSAGE_WAKE_REASON } from "../../../services/queued-comment-delivery.js"; import { extractIssueReferenceIdentifiers, isUuidLike } from "@paperclipai/shared"; @@ -1297,10 +1298,16 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd : []; if (!finalized) return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } - if (await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) { + // These two holds protect the old executor's workspace and resources. A + // continuation in a new isolated workspace (or an abandoned remote epoch) + // reuses none of them, so they no longer hold the task's next turn; any + // other replacement still waits for them. + const isolatedReplacement = agentRunSupersededByFreshContinuation(run) + && await freshContinuationIsolatesRun(tx, { companyId: input.companyId, runId: run.id }); + if (!isolatedReplacement && await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) { return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } - if (input.resumeStrandedQueue && await hasPendingRunCleanup(tx, input.companyId, run.id)) { + if (input.resumeStrandedQueue && !isolatedReplacement && await hasPendingRunCleanup(tx, input.companyId, run.id)) { return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 289fbd58d60..f88dbbb13cc 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -883,18 +883,20 @@ suite("fresh-continuation fallback", () => { }, 60_000); // The isolation decision is a property of the task's next run, whatever path starts it. - it("moves the task off the old writer's workspace for every next run, saved input included", async () => { + it("moves the task off the old writer's workspace and runs the isolated continuation before saved input", async () => { const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: f.issueId, mode: "isolated", strategyType: "git_worktree", name: "Task workspace", cwd: "/tmp/old-writer" }).returning(); await db.update(issues).set({ executionWorkspaceId: workspace!.id, workspaceBindingRevision: 2 }).where(eq(issues.id, f.issueId)); await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000), contextSnapshot: { issueId: f.issueId, executionWorkspaceId: workspace!.id } }).where(eq(heartbeatRuns.id, f.runId)); - // A person's saved message is waiting: it continues the task itself. + // A person's saved message is waiting. It carries no isolation marker, so it cannot prepare the + // new workspace (a projectless task would be refused at admission once the binding is cleared): + // the isolated continuation runs first and the saved message follows it. await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", status: "deferred_issue_execution", requestedByActorType: "user", requestedByActorId: userId, payload: { issueId: f.issueId } }); expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) - .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "saved_input" }); + .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "recorded" }); // No delivery path can reach the old directory: the task is no longer bound to it. expect(await issue(f.issueId)).toMatchObject({ executionWorkspaceId: null, workspaceBindingRevision: 3 }); const [kept] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace!.id)); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 68f035fb35f..08082214efc 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -320,6 +320,20 @@ export async function isFreshIsolatedContinuationWake(db: Db, input: { companyId contextSnapshot: input.contextSnapshot ?? {} }, input.issueId)) !== null; } +/** + * True when the fallback replaced this run and its continuation reuses none of + * the run's workspace: a new isolated workspace or an abandoned remote epoch. + * Holds that only protect the old executor's resources then no longer stand + * between the task and its next turn. + */ +export async function freshContinuationIsolatesRun(db: Db, input: { companyId: string; runId: string }): Promise { + const [record] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${input.runId}`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}' in ('fresh_isolated_workspace', 'remote_epoch')`)).limit(1); + return Boolean(record); +} + export async function ensureFreshContinuation(db: Db, input: Input): Promise { return db.transaction(tx => continueFreshInTransaction(tx as unknown as Db, input)); } @@ -518,10 +532,12 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< // Never claim a stop that was not proven: record what isolates the successor. const isolation: FreshContinuationIsolation = fenced || remoteSource ? "remote_epoch" : writer === "ended" && !waitBounded ? "writer_ended" : "fresh_isolated_workspace"; - // With no task workspace binding to move, only a run that carries the - // isolation in its context can avoid the shared directory: the recorded - // continuation runs first, and saved input follows it. - if (isolation === "fresh_isolated_workspace" && !issue.executionWorkspaceId && delivery === "saved_input") delivery = "recorded"; + // Only a run that carries the isolation in its context prepares the new + // workspace (create_new). Saved input carries no such marker: on a task with + // no project it would be refused at admission once the binding is cleared, + // and otherwise could resolve the old directory. So the recorded + // continuation runs first, binds the new workspace, and saved input follows. + if (isolation === "fresh_isolated_workspace" && delivery === "saved_input") delivery = "recorded"; const unverified = await unverifiedActions(tx, run); const note = freshContinuationNote({ runId: run.id, unverified, workspace: isolation === "writer_ended" ? "kept" : "restored_last_save" });