diff --git a/server/src/services/chat-channels/failed-chat-retry-source-stop-wait.test.ts b/server/src/services/chat-channels/failed-chat-retry-source-stop-wait.test.ts new file mode 100644 index 00000000000..00708eb9edf --- /dev/null +++ b/server/src/services/chat-channels/failed-chat-retry-source-stop-wait.test.ts @@ -0,0 +1,161 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issues } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { conflict } from "../../errors.js"; +import { FailedChatRunRetryAuthorizationError } from "../durable-chat-wakeup.js"; +import { remoteTerminationReceipt } from "../remote-execution-termination.js"; +import { prepareStoppedSandboxForRecovery, restoreSandboxStopAfterFailedRestart } from "../sandbox-stop-and-retain.js"; +import { taskChatRetryExecutionStopped } from "./task-retry-authority.js"; +import { awaitFailedChatRetrySourceStop, failedChatRetrySourceStopPending } from "./failed-chat-retry-source-stop-wait.js"; + +describe("exact chat replay while workspace recovery restarts the failed run's sandbox", () => { + let postgres: Awaited>; + let db: ReturnType; + beforeAll(async () => { + postgres = await startEmbeddedPostgresTestDatabase("chat-retry-source-stop-"); + db = createDb(postgres.connectionString); + }, 30_000); + afterAll(async () => { await db?.$client.end(); await postgres?.cleanup(); }); + + /** A failed native run whose remote sandbox was stopped with a bound receipt. */ + async function stoppedSource() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Chat retry", issuePrefix: `C${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Reviewer", adapterType: "paperclip_runner", status: "idle" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Review", status: "blocked", assigneeAgentId: agentId }); + const [source] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "failed", + runtimeMode: "native", nativeIssueId: issueId, nativePhase: "terminal_failure", + errorCode: "native_continuation_requires_reconciliation", contextSnapshot: { issueId }, finishedAt: new Date() }).returning(); + const [environment] = await db.insert(environments).values({ name: runId, driver: "sandbox" }).returning(); + const identity = { id: randomUUID(), companyId, heartbeatRunId: runId, provider: "fake-sandbox", providerLeaseId: randomUUID() }; + const [lease] = await db.insert(environmentLeases).values({ ...identity, issueId, environmentId: environment.id, + status: "released", cleanupStatus: "success", releasedAt: new Date(), leasePolicy: "retain_on_failure", + metadata: { pluginId: "fake-plugin", remoteCwd: "/work", remoteExecutionTermination: + remoteTerminationReceipt(identity, { providerLeaseId: identity.providerLeaseId, state: "stopped" }) }, + }).returning(); + return { source: source!, lease: lease!, companyId }; + } + + /** Workspace recovery suspends the stop receipt and restarts the sandbox to save its files. */ + async function recoveryRestart(lease: typeof environmentLeases.$inferSelect) { + const pending = await prepareStoppedSandboxForRecovery(db, lease); + expect(pending).not.toBeNull(); + return pending!; + } + + async function restop(pending: typeof environmentLeases.$inferSelect) { + const attemptId = String(pending.metadata?.pendingCleanupAttemptId); + expect(await restoreSandboxStopAfterFailedRestart(db, pending, { attemptId })).not.toBeNull(); + } + + /** The containment part of the retry authority, as assertFailedNativeRetryState applies it. */ + const authorize = (sourceRunId: string) => async () => { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, sourceRunId)); + if (!await taskChatRetryExecutionStopped(db, run!)) + throw conflict("This chat request can't be repeated exactly as it was sent.", { code: "chat_failed_run_retry_not_authorized" }); + return true; + }; + + it("loses the source's containment proof while recovery holds the sandbox, then regains it", async () => { + const { source, lease, companyId } = await stoppedSource(); + expect(await taskChatRetryExecutionStopped(db, source)).toBe(true); + expect(await failedChatRetrySourceStopPending(db, companyId, source.id)).toBe(false); + + const pending = await recoveryRestart(lease); + // This is the state the replay's dispatch recheck saw in the incident. + expect(await taskChatRetryExecutionStopped(db, source)).toBe(false); + expect(await failedChatRetrySourceStopPending(db, companyId, source.id)).toBe(true); + + await restop(pending); + expect(await taskChatRetryExecutionStopped(db, source)).toBe(true); + expect(await failedChatRetrySourceStopPending(db, companyId, source.id)).toBe(false); + }); + + it("waits for the sandbox to stop again instead of failing the replay", async () => { + const { source, lease, companyId } = await stoppedSource(); + const pending = await recoveryRestart(lease); + await expect(authorize(source.id)()).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" } }); + + const onWait = vi.fn(); + const attempt = vi.fn(authorize(source.id)); + const restopped = new Promise(resolve => setTimeout(() => void restop(pending).then(resolve), 150)); + await expect(awaitFailedChatRetrySourceStop(db, { companyId, sourceRunId: source.id, + signal: new AbortController().signal, onWait, pollMs: 25 }, attempt)).resolves.toBe(true); + await restopped; + expect(onWait).toHaveBeenCalledTimes(1); + expect(attempt.mock.calls.length).toBeGreaterThan(1); + }); + + it("checks once more when the stop settles between the refusal and the pending-stop read", async () => { + const { source, lease, companyId } = await stoppedSource(); + const pending = await recoveryRestart(lease); + const check = authorize(source.id); + let first = true; + // The refused transaction has released its locks; recovery then commits the stop receipt. + const attempt = vi.fn(async () => { + if (!first) return check(); + first = false; + await check().catch(async error => { await restop(pending); throw error; }); + throw new Error("expected the first attempt to be refused"); + }); + const onWait = vi.fn(); + await expect(awaitFailedChatRetrySourceStop(db, { companyId, sourceRunId: source.id, + signal: new AbortController().signal, onWait, pollMs: 25 }, attempt)).resolves.toBe(true); + expect(attempt).toHaveBeenCalledTimes(2); + expect(onWait).not.toHaveBeenCalled(); + }); + + it("refuses at the deadline when the sandbox stays held", async () => { + const { source, lease, companyId } = await stoppedSource(); + await recoveryRestart(lease); + await expect(awaitFailedChatRetrySourceStop(db, { companyId, sourceRunId: source.id, + signal: new AbortController().signal, timeoutMs: 100, pollMs: 25 }, authorize(source.id))) + .rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" } }); + }); + + it("refuses without waiting when no stop of the source sandbox is in progress", async () => { + const { source, companyId } = await stoppedSource(); + // An unreleased lease without a stop intent: the old executor may still be running. + await db.update(environmentLeases).set({ status: "active", cleanupStatus: null, releasedAt: null, metadata: { pluginId: "fake-plugin" } }) + .where(eq(environmentLeases.heartbeatRunId, source.id)); + const attempt = vi.fn(authorize(source.id)); + const onWait = vi.fn(); + await expect(awaitFailedChatRetrySourceStop(db, { companyId, sourceRunId: source.id, + signal: new AbortController().signal, onWait, pollMs: 25 }, attempt)).rejects.toMatchObject({ status: 409 }); + expect(attempt).toHaveBeenCalledTimes(1); + expect(onWait).not.toHaveBeenCalled(); + }); + + it.each([ + ["a reasoned refusal", () => conflict("saved", { code: "chat_failed_run_retry_not_authorized", reason: "prior_result_publication" })], + ["a mismatched retry selector", () => new FailedChatRunRetryAuthorizationError()], + ["another error", () => new Error("database unavailable")], + ])("does not wait on %s", async (_label, error) => { + const { source, lease, companyId } = await stoppedSource(); + await recoveryRestart(lease); + const attempt = vi.fn(async () => { throw error(); }); + await expect(awaitFailedChatRetrySourceStop(db, { companyId, sourceRunId: source.id, + signal: new AbortController().signal, pollMs: 25 }, attempt)).rejects.toBeInstanceOf(Error); + expect(attempt).toHaveBeenCalledTimes(1); + }); + + it("does not wait when the run is not an exact replay of a failed run", async () => { + const { source, lease, companyId } = await stoppedSource(); + await recoveryRestart(lease); + const attempt = vi.fn(authorize(source.id)); + await expect(awaitFailedChatRetrySourceStop(db, { companyId, sourceRunId: null, + signal: new AbortController().signal, pollMs: 25 }, attempt)).rejects.toMatchObject({ status: 409 }); + expect(attempt).toHaveBeenCalledTimes(1); + }); + + it("stops waiting when the run is stopped", async () => { + const { source, lease, companyId } = await stoppedSource(); + await recoveryRestart(lease); + const control = new AbortController(); + const waiting = awaitFailedChatRetrySourceStop(db, { companyId, sourceRunId: source.id, + signal: control.signal, pollMs: 10_000, onWait: () => control.abort(new Error("stopped")) }, authorize(source.id)); + await expect(waiting).rejects.toThrow("stopped"); + }); +}); diff --git a/server/src/services/chat-channels/failed-chat-retry-source-stop-wait.ts b/server/src/services/chat-channels/failed-chat-retry-source-stop-wait.ts new file mode 100644 index 00000000000..c1db42aae3d --- /dev/null +++ b/server/src/services/chat-channels/failed-chat-retry-source-stop-wait.ts @@ -0,0 +1,85 @@ +import { and, eq } from "drizzle-orm"; +import { environmentLeases, type Db } from "@paperclipai/db"; +import { HttpError } from "../../errors.js"; +import { FailedChatRunRetryAuthorizationError } from "../durable-chat-wakeup.js"; +import { readStopOnlyCleanup } from "../sandbox-stop-and-retain.js"; + +/** Long enough for workspace recovery to save files and stop the sandbox again. */ +export const FAILED_CHAT_RETRY_SOURCE_STOP_WAIT_MS = 3 * 60_000; +export const FAILED_CHAT_RETRY_SOURCE_STOP_POLL_MS = 5_000; + +/** The containment refusal carries no reason. Reasoned refusals (an accepted + * result, a prior publication or interaction, source authorization) and a + * mismatched retry selector do not change while a sandbox stops. */ +function isSourceContainmentRefusal(error: unknown) { + if (!(error instanceof HttpError) || error instanceof FailedChatRunRetryAuthorizationError || + error.status !== 409) return false; + const details = error.details as Record | undefined; + return details?.code === "chat_failed_run_retry_not_authorized" && details.reason === undefined; +} + +/** A stop of the failed run's sandbox is in progress. Workspace recovery holds + * one while it restarts the sandbox to save its files, and removes the stop + * receipt until it stops it again. This only permits a wait: containment is + * still decided by the unchanged retry authority on each attempt. */ +export async function failedChatRetrySourceStopPending(db: Db, companyId: string, sourceRunId: string) { + const leases = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, sourceRunId), + )); + return leases.some(lease => lease.provider !== "local" && lease.status === "pending_cleanup" && + readStopOnlyCleanup(lease) !== null); +} + +function delay(ms: number, signal: AbortSignal) { + return new Promise((resolve, reject) => { + if (signal.aborted) return reject(signal.reason); + const onAbort = () => { clearTimeout(timer); reject(signal.reason); }; + const timer = setTimeout(() => { signal.removeEventListener("abort", onAbort); resolve(); }, ms); + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +/** Re-run an exact chat replay's authorization while the failed run's sandbox + * stop settles, instead of failing the replay on a transient missing receipt. + * The wait is bounded and stops with the run; the last refusal is returned. + * The pending stop is read before and after each refused attempt. A stop can + * settle after the refused transaction releases its locks, so a stop seen + * before the attempt earns one more attempt even when it is gone after. A + * refusal with no pending stop on either side is returned at once. */ +export async function awaitFailedChatRetrySourceStop(db: Db, input: { + companyId: string; + sourceRunId: string | null; + signal: AbortSignal; + onWait?: () => Promise | void; + timeoutMs?: number; + pollMs?: number; +}, attempt: () => Promise): Promise { + const sourceRunId = input.sourceRunId; + if (!sourceRunId) return attempt(); + const deadline = Date.now() + (input.timeoutMs ?? FAILED_CHAT_RETRY_SOURCE_STOP_WAIT_MS); + let announced = false; + let settledRetry = false; + for (;;) { + const pendingBefore = await failedChatRetrySourceStopPending(db, input.companyId, sourceRunId); + let pendingAfter: boolean; + try { + return await attempt(); + } catch (error) { + if (!isSourceContainmentRefusal(error)) throw error; + pendingAfter = await failedChatRetrySourceStopPending(db, input.companyId, sourceRunId); + if ((!pendingBefore && !pendingAfter) || Date.now() >= deadline) throw error; + } + // The stop settled during the refused attempt: read the authority again + // now, once. A later flip waits for the normal poll interval. + if (!pendingAfter && !settledRetry) { + settledRetry = true; + continue; + } + if (!announced) { + announced = true; + await input.onWait?.(); + } + await delay(Math.min(input.pollMs ?? FAILED_CHAT_RETRY_SOURCE_STOP_POLL_MS, Math.max(0, deadline - Date.now())), + input.signal); + } +} diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 200faf00f62..be7c0cf9ea1 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -314,6 +314,7 @@ import { watchLegacyControllerLease, } from "./legacy-controller-lease.js"; import { retryChatControlAdmission } from "./chat-control-admission-retry.js"; +import { awaitFailedChatRetrySourceStop } from "./chat-channels/failed-chat-retry-source-stop-wait.js"; import { @@ -2268,8 +2269,17 @@ export function heartbeatService( } const runtime = await timePreparation("runtime_state", () => ensureRuntimeState(agent)); const context = parseObject(run.contextSnapshot); + // Workspace recovery can restart the failed run's sandbox to save its + // files while this replay prepares. Wait for that stop to settle; the + // retry authority still decides each attempt. const authorizeFailedChatRetryExecution = (stage?: "dispatching") => - retryChatControlAdmission(async () => { + awaitFailedChatRetrySourceStop(db, { + companyId: run.companyId, + sourceRunId: Object.hasOwn(context, "chatFailedRunRetry") ? readNonEmptyString(context.retryOfRunId) : null, + signal: executionControl.controller.signal, + onWait: () => appendRunEvent(run, { eventType: "lifecycle", stream: "system", level: "info", + message: "Waiting for the failed run's sandbox to stop before repeating the chat request" }), + }, () => retryChatControlAdmission(async () => { executionControl.controller.signal.throwIfAborted(); await controllerLease.assertOwned(stage); return db.transaction((tx) => @@ -2282,7 +2292,7 @@ export function heartbeatService( runId: run.id, contextSnapshot: context, })); - }); + })); const isFailedChatRunRetry = await timePreparation("chat_retry_admission", () => authorizeFailedChatRetryExecution()); // Never adopt a chat-execution attestation supplied in a wake payload. // Reviewed chat turns rebuild it from the current durable owner below.