diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index 85a7d24242f..3e9490f9bf4 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -29,6 +29,7 @@ import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { heartbeatRunEvents } from "@paperclipai/db"; import { logger } from "../middleware/logger.js"; +import { RemoteWorkspacePreparationTimeoutError } from "../services/remote-workspace-transport.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_DIRECTORY_TEST_DATABASE_URL"); @@ -1505,8 +1506,9 @@ describe("persistent agent directories", () => { try { const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId }); - await expect(copies.prepare({ companyId, agentId, runId, cwd: home, target: sshTarget("/fixture/slow-task") })) - .rejects.toThrow("Could not exclude agent files from task Git staging (reason=timed_out)"); + const failure = copies.prepare({ companyId, agentId, runId, cwd: home, target: sshTarget("/fixture/slow-task") }); + await expect(failure).rejects.toThrow("Could not exclude agent files from task Git staging (reason=timed_out)"); + await expect(failure).rejects.toBeInstanceOf(RemoteWorkspacePreparationTimeoutError); expect(exclusions(shell)).toHaveLength(3); expect((await copies.get(companyId, runId))!.state).toBe("unavailable"); } finally { shell.mockRestore(); } diff --git a/server/src/services/agent-instruction-files.ts b/server/src/services/agent-instruction-files.ts index fa81c24c957..3de999a40b5 100644 --- a/server/src/services/agent-instruction-files.ts +++ b/server/src/services/agent-instruction-files.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; import { unprocessable } from "../errors.js"; import { runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError } from "./remote-workspace-transport.js"; export const MAX_INSTRUCTION_BYTES = 1024 * 1024; export function instructionPath(value: string): string { @@ -193,15 +194,18 @@ export function instructionGitExcludeFailureReason(result: { } const REMOTE_GIT_EXCLUDE_ATTEMPTS = 3; +const REMOTE_GIT_EXCLUDE_TIMEOUT_SEC = 30; export async function excludeRuntimeFromRemoteGit(runId: string, target: Exclude, failure: string) { const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; for (let attempt = 1; ; attempt++) { const result = await runAdapterExecutionTargetShellCommand(runId, target, - `node -e ${quote(instructionGitExcludeProgram)} ${quote(target.remoteCwd)}`, { cwd: target.remoteCwd, env: {}, timeoutSec: 30 }); + `node -e ${quote(instructionGitExcludeProgram)} ${quote(target.remoteCwd)}`, + { cwd: target.remoteCwd, env: {}, timeoutSec: REMOTE_GIT_EXCLUDE_TIMEOUT_SEC }); if (result.exitCode === 0 && !result.timedOut) return; - if (!result.timedOut || attempt === REMOTE_GIT_EXCLUDE_ATTEMPTS) { - throw new Error(`${failure} (reason=${instructionGitExcludeFailureReason(result)})`); - } + const message = `${failure} (reason=${instructionGitExcludeFailureReason(result)})`; + if (!result.timedOut) throw new Error(message); + if (attempt === REMOTE_GIT_EXCLUDE_ATTEMPTS) throw new RemoteWorkspacePreparationTimeoutError( + new RemoteWorkspaceOperationTimeoutError("prepare", REMOTE_GIT_EXCLUDE_TIMEOUT_SEC * 1000), message); } } diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index 7b1ec54fbbf..9358bc973c9 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -11,6 +11,7 @@ import { CONTROLLER_LEASE_LOST_FAILURE_CODE, legacyControllerBootId, legacyContr import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; import { readFailedPreparationSettlement } from "./failed-preparation-settlement.js"; +import { failedPreparationOwnsBoundedRetry } from "./failed-preparation-retry.js"; import { hasVerifiedPreparationLog } from "./verified-preparation-log.js"; import { executionStartedEventPredicate, isPreDispatchResourceWait, noRetainedExecutionPredicate } from "./pre-dispatch-execution-evidence.js"; @@ -19,7 +20,7 @@ type Run = Pick; + "controllerBootId" | "controllerLeaseExpiresAt"> & { runnerInstanceId?: string | null }; type Coordinator = typeof nativeRunFinalizations.$inferSelect; const EMPTY_LOG_SHA256 = createHash("sha256").digest("hex"); @@ -135,7 +136,9 @@ export function isInterruptedNativePreparationCandidate(run: Run): boolean { // Restart recovery records this certificate only after revoking a preparing // controller. A generic process-lost error cannot establish safe replay. const lostPreparation = run.status === "failed" - && (run.errorCode === "process_lost" || run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE) + && (run.errorCode === "process_lost" || run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE + || hasRequiredWorkspaceRecovery(run.resultJson) + && failedPreparationOwnsBoundedRetry({ ...run, runnerInstanceId: run.runnerInstanceId ?? null })) && recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; // Older shutdown records predate structured cancellation. Current records // identify unexpected system interruption without relying on an error label. diff --git a/server/src/services/failed-preparation-retry.ts b/server/src/services/failed-preparation-retry.ts new file mode 100644 index 00000000000..35fe4cd9b72 --- /dev/null +++ b/server/src/services/failed-preparation-retry.ts @@ -0,0 +1,50 @@ +import type { heartbeatRuns } from "@paperclipai/db"; +import { parseObject } from "../adapters/utils.js"; +import { isResolvedInteractionContinuationWakeContext } from "../modules/run-dispatch/index.js"; +import { STORAGE_UNAVAILABLE_CODE } from "../storage/storage-unavailable.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "./environment-provider-unavailable-error.js"; +import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "./plugin-worker-unavailable-error.js"; +import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "./remote-workspace-transport.js"; +import { REPOSITORY_CATALOG_UNAVAILABLE_CODE } from "./repository-catalog-unavailable-error.js"; +import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "./reusable-sandbox-resume-error.js"; +import { REVIEWED_CHAT_BINDING_NOT_READY } from "./reviewed-chat-binding-diagnostics.js"; +import { RUNNER_REVISION_MISMATCH_CODE } from "./runner-revision-mismatch.js"; +import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "./workspace-git-operation-scheduler.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; + +export const TRANSIENT_WORKSPACE_PREPARATION_CODES: readonly string[] = [ + WORKSPACE_GIT_SCAN_ERROR_CODES.timeout, WORKSPACE_GIT_SCAN_ERROR_CODES.saturated, + REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REUSABLE_SANDBOX_RESUME_FAILED_CODE, + REMOTE_WORKSPACE_RESTORE_FAILED_CODE, PLUGIN_WORKER_UNAVAILABLE_CODE, + RUNNER_REVISION_MISMATCH_CODE, ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE, + REPOSITORY_CATALOG_UNAVAILABLE_CODE, REVIEWED_CHAT_BINDING_NOT_READY, + STORAGE_UNAVAILABLE_CODE, +]; + +export function isTransientWorkspacePreparationCode(code: string | null | undefined): boolean { + return typeof code === "string" && TRANSIENT_WORKSPACE_PREPARATION_CODES.includes(code); +} + +/** + * The persisted form of the setup-failure decision in run completion: this + * failed legacy run stopped before provider work with a transient preparation + * code, so its own bounded retry (scheduleBoundedRetryForRun) continues it. + * The stranded-task sweep uses it to offer a run whose executor has not + * scheduled that retry (still settling, or its server stopped) to the same + * idempotent scheduler instead of escalating it. + */ +export function failedPreparationOwnsBoundedRetry(run: Pick): boolean { + if (run.status !== "failed" || run.runtimeMode !== "legacy" || run.nativeSessionId !== null || run.runnerInstanceId !== null + || !isTransientWorkspacePreparationCode(run.errorCode)) return false; + const recovery = parseObject(run.resultJson?.executionRecovery); + if (recovery.kind !== "bootstrap" || recovery.providerWorkStarted !== false) return false; + // A restore failure is retried only when its owner proved it retryable. + if (run.errorCode === REMOTE_WORKSPACE_RESTORE_FAILED_CODE) + return parseObject(run.resultJson?.remoteWorkspaceRestore).retryable === true; + // An accepted interaction keeps its own continuation retry. + return !((run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE || run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE + || run.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE || run.errorCode === REVIEWED_CHAT_BINDING_NOT_READY) + && isResolvedInteractionContinuationWakeContext(run.contextSnapshot)); +} diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 7425f402af0..8e45af94cde 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -32,9 +32,7 @@ import { serverRestartBudgetSpent, } from "../execution-recovery-attempt.js"; import { isReplacedPreparation, isServerReplacementInterruption } from "../server-replacement-interruption.js"; -import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js"; -import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "../remote-workspace-transport.js"; -import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; +import { failedPreparationOwnsBoundedRetry, isTransientWorkspacePreparationCode } from "../failed-preparation-retry.js"; import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; import { REPOSITORY_CATALOG_UNAVAILABLE_CODE } from "../repository-catalog-unavailable-error.js"; @@ -153,37 +151,7 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; -export function isTransientWorkspacePreparationCode(code: string | null | undefined): boolean { - return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated - || code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || code === REUSABLE_SANDBOX_RESUME_FAILED_CODE - || code === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || code === PLUGIN_WORKER_UNAVAILABLE_CODE - || code === RUNNER_REVISION_MISMATCH_CODE || code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE - || code === REPOSITORY_CATALOG_UNAVAILABLE_CODE || code === REVIEWED_CHAT_BINDING_NOT_READY - || code === STORAGE_UNAVAILABLE_CODE; -} - -/** - * The persisted form of the setup-failure decision in run completion: this - * failed legacy run stopped before provider work with a transient preparation - * code, so its own bounded retry (scheduleBoundedRetryForRun) continues it. - * The stranded-task sweep uses it to offer a run whose executor has not - * scheduled that retry (still settling, or its server stopped) to the same - * idempotent scheduler instead of escalating it. - */ -export function failedPreparationOwnsBoundedRetry(run: Pick): boolean { - if (run.status !== "failed" || run.runtimeMode !== "legacy" || run.nativeSessionId !== null || run.runnerInstanceId !== null - || !isTransientWorkspacePreparationCode(run.errorCode)) return false; - const recovery = parseObject(run.resultJson?.executionRecovery); - if (recovery.kind !== "bootstrap" || recovery.providerWorkStarted !== false) return false; - // A restore failure is retried only when its owner proved it retryable. - if (run.errorCode === REMOTE_WORKSPACE_RESTORE_FAILED_CODE) - return parseObject(run.resultJson?.remoteWorkspaceRestore).retryable === true; - // An accepted interaction keeps its own continuation retry. - return !((run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE || run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE - || run.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE || run.errorCode === REVIEWED_CHAT_BINDING_NOT_READY) - && isResolvedInteractionContinuationWakeContext(run.contextSnapshot)); -} +export { failedPreparationOwnsBoundedRetry, isTransientWorkspacePreparationCode }; export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index fb4cbeefc52..219b630feaa 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -1370,6 +1370,18 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); }); + it.each(["before_adapter", "adapter_entered", "untrusted", "native"] as const)("names a remote preparation timeout raised %s during execution", async scenario => { + const f = await fixture({ runtimeMode: scenario === "native" ? "native" : "legacy" }); + const cause = new RemoteWorkspaceOperationTimeoutError("prepare", 30_000); + const message = "Could not exclude agent files from task Git staging (reason=timed_out)"; + const error = scenario === "untrusted" ? new Error(message) : new RemoteWorkspacePreparationTimeoutError(cause, message); + await f.completion.failRun({ ...f.failure(error), requiredWorkspaceRestoreEvidence: {}, legacyAdapterEntered: scenario === "adapter_entered" }); + const failed = await read(f.run.id); + expect(failed).toMatchObject({ status: "failed", error: message, + errorCode: scenario === "before_adapter" ? "remote_workspace_prepare_timeout" : expect.not.stringMatching(/^remote_workspace_prepare_timeout$/) }); + if (scenario === "before_adapter") expect(failed.resultJson).toMatchObject({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }); + }); + it.each(["fresh", "started", "untrusted", "retained"] as const)("routes a %s preserved sandbox resume failure through bounded bootstrap recovery", async scenario => { const f = await fixture({ runtimeMode: scenario === "retained" ? "native" : "legacy" }); const providerError = Object.assign(new Error("Sandbox abc is not in a valid state: stopping /home/daytona/secret"), { statusCode: 400 }); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 87f95c21daf..bb81b2b97b6 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1323,6 +1323,9 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC ? { ...exactResumeCause, originalMessage: redactFailureText(exactResumeCause.originalMessage) } : null; const storageUnavailable = isStorageUnavailableError(err); + const remoteWorkspacePreparationTimeout = !legacyAdapterEntered && run.runtimeMode === "legacy" + && run.nativeSessionId === null && run.runnerInstanceId === null && err instanceof RemoteWorkspacePreparationTimeoutError + ? err : null; const failureErrorCode = workspaceValidationFailure?.code ?? configurationIncompleteFailure?.code ?? @@ -1330,6 +1333,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC recordedResponsibleUserDenialCode ?? nativeTerminalFailureCode ?? (storageUnavailable ? STORAGE_UNAVAILABLE_CODE : null) ?? + remoteWorkspacePreparationTimeout?.code ?? "adapter_failed"; // Only a `storage_unavailable` run gets a storage wait. Under any other code // (for example a native recovery hold that keeps the files without diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 305149fce90..fbffb95d319 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -133,10 +133,11 @@ export const REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE = "remote_workspace_prepa export class RemoteWorkspacePreparationTimeoutError extends RemoteWorkspaceOperationTimeoutError { readonly code = REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE; - constructor(error: RemoteWorkspaceOperationTimeoutError) { + constructor(error: RemoteWorkspaceOperationTimeoutError, message?: string) { super("prepare", error.timeoutMs); this.name = "RemoteWorkspacePreparationTimeoutError"; this.cause = error; + if (message) this.message = message; } } diff --git a/server/src/services/workspace-recovery-preparation-dispatch.test.ts b/server/src/services/workspace-recovery-preparation-dispatch.test.ts index 883c225319d..211d99892c1 100644 --- a/server/src/services/workspace-recovery-preparation-dispatch.test.ts +++ b/server/src/services/workspace-recovery-preparation-dispatch.test.ts @@ -419,6 +419,23 @@ describe("automatic retained workspace recovery", () => { } finally { await restarted.$client.end(); } }, 30_000); + it.each([["remote_workspace_prepare_timeout", 1], ["adapter_failed", 0]] as const)( + "continues a recovered preparation that failed with %s only when its bounded retry owns it", async (errorCode, continuations) => { + const f = await interruptedRemotePreparation("process_lost"); + await db.update(heartbeatRuns).set({ errorCode, error: "Could not exclude agent files from task Git staging (reason=timed_out)" }) + .where(eq(heartbeatRuns.id, f.runId)); + await recoverRetainedWorkspace(f.input); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true); + const listed = await retainedWorkspaceCandidates(db, undefined, false, true); + expect(listed.some(row => row.run.id === f.runId)).toBe(continuations === 1); + const heartbeat = heartbeatService(db); + await Promise.all([heartbeat.resumeRemoteStopComments(await run(f.runId)), + heartbeat.resumeRemoteStopComments(await run(f.runId))]); + const successors = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.runId)); + expect(successors).toHaveLength(continuations); + if (continuations) expect(successors[0]).toMatchObject({ status: "scheduled_retry", scheduledRetryAttempt: 1 }); + }, 30_000); + it.each(["restart", "pause", "claimed", "duplicate"] as const)("retains preparation continuation after settlement before enqueue (%s)", async mode => { const f = await interruptedRemotePreparation("process_lost"), withheld = await withheldPreparationRetry(f); await recoverRetainedWorkspace(f.input); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 7d652a35e1f..d48a9668b8f 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -22,7 +22,8 @@ import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { confirmSandboxRecoveryRestart, hasStopOnlyCleanup, prepareHistoricalSandboxStopVerification, prepareSandboxStopAndRetain, prepareStoppedSandboxForRecovery, restoreSandboxStopAfterFailedRestart, settleAbsentSandbox, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; import { requestSandboxAbsenceProof } from "./sandbox-absence-proof.js"; import { persistActivity, publishActivity } from "./activity-log.js"; -import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; +import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; +import { TRANSIENT_WORKSPACE_PREPARATION_CODES } from "./failed-preparation-retry.js"; import { hasVerifiedWorkspaceRepair, readWorkspaceRecovery, type WorkspaceRecovery } from "./workspace-recovery-receipt.js"; import { withNativeWorkspaceFinalizationOwnership } from "./native-runtime/native-workspace-finalization-ownership.js"; import { readNativeWorkspaceSyncReference, readNativeWorkspaceSyncLocalCwd, resumeNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js"; @@ -630,7 +631,8 @@ export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedO and ${agentWakeupRequests.payload}->>'issueId' = ${issueRecoveryActions.sourceIssueId}::text)`; const interruptedPreparation = sql`(${heartbeatRuns.runtimeMode} = 'legacy' and (${heartbeatRuns.status} = 'interrupted' or (${heartbeatRuns.status} = 'failed' - and ${heartbeatRuns.errorCode} = 'process_lost' + and (${heartbeatRuns.errorCode} = 'process_lost' or (${inArray(heartbeatRuns.errorCode, [...TRANSIENT_WORKSPACE_PREPARATION_CODES])} + and ${heartbeatRuns.resultJson}#>>'{workspaceRestoreRecovery,schema}' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA})) and ${heartbeatRuns.resultJson}#>>'{executionRecovery,kind}' = 'bootstrap' and ${heartbeatRuns.resultJson}#>'{executionRecovery,providerWorkStarted}' = 'false'::jsonb)) and ${heartbeatRuns.runtimeModeResolvedAt} is null and ${heartbeatRuns.executionStage} = 'preparing'