diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 7ae7a155af9..f17a4cdb0db 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -920,6 +920,23 @@ recovery path with the specific scan cause and an operator action. Generic stranded-work recovery must not grant another budget for these errors. This does not automatically replay historical generic `setup_failed` runs. +Fresh remote workspace preparation timeouts use that same bounded retry budget. +The workspace owner re-reads its company-scoped authority after the timeout. It +must still have the same writer and epoch, with no location, saved version, or +unsaved task work. Only the initial directory/clone operation qualifies; restoring +saved work, acquiring additional repositories, provisioning, and saving do not. +The failed allocation is not reused by a successor. Normal lease cleanup and +fresh admission retain their existing ownership and execution gates. A lost +preparation reply therefore cannot cause a successor to overwrite that allocation. +Provider work must not have started and the run must not have retained native +execution identity. For that persisted bootstrap-only timeout, a successor may +claim the still-unpublished reservation after every prior lease is terminal and +successfully cleaned up, including providers whose release returns no receipt. +This grants no provider-stop evidence: published or saved work, retained execution +identity, unknown failures, active leases, and incomplete cleanup still require +their normal reconciliation and termination proof. Unknown historical setup errors remain +outside this policy, and generic recovery cannot renew its exhausted budget. + ### Runner authentication and ACP startup handshake bounds The native Runner waits up to the ten-second connection setup budget for the diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 01c9ec1cb19..a2540946420 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -852,6 +852,36 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease }; } + it("gives a bootstrap retry a fresh allocation even when the failed lease was reusable", async () => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); + await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); + let acquisitions = 0; + const workerManager = { + isRunning: () => true, + call: vi.fn(async (_id: string, method: string) => { + if (method === "environmentAcquireLease") return { providerLeaseId: `fresh-bootstrap-${++acquisitions}`, + metadata: { provider: "fake-plugin", reuseLease: true, remoteCwd: "/workspace" } }; + throw new Error(`Unexpected provider operation: ${method}`); + }), + getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), + } as unknown as PluginWorkerManager; + const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + const input = { companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, + adapterType: "paperclip_runner", heartbeatRunId: seeded.runId, + persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" as const } }; + const first = await runtime.acquireRunLease(input); + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "remote_workspace_prepare_timeout" }) + .where(eq(heartbeatRuns.id, seeded.runId)); + await environmentService(db).releaseLease(first.lease.id, "failed"); + const retryRunId = randomUUID(); + await insertRunningManualRun(retryRunId, seeded.companyId, seeded.agentId); + const next = await runtime.acquireRunLease({ ...input, heartbeatRunId: retryRunId }); + expect(next.lease.providerLeaseId).toBe("fresh-bootstrap-2"); + expect(next.lease.providerLeaseId).not.toBe(first.lease.providerLeaseId); + expect(workerManager.call).toHaveBeenCalledTimes(2); + expect((await environmentService(db).getLeaseById(first.lease.id))?.status).toBe("failed"); + }); + it("acquires and resumes a reusable lease after a task changes from per-turn to warm", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index 1bb89e34429..fca54d223ba 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -494,6 +494,47 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { return { companyId, agentId, issueId, runId, now }; } + it.each(["resume", "paused", "Stop", "dependency"] as const)("bounds fresh remote workspace retries across restart and honors %s", async action => { + const { companyId, agentId, issueId, runId, now } = await seedMaxTurnFixture(); + await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol" } }) + .where(eq(agents.id, agentId)); + const resultJson = { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }; + await db.update(heartbeatRuns).set({ errorCode: "remote_workspace_prepare_timeout", resultJson }).where(eq(heartbeatRuns.id, runId)); + let predecessor = runId; + for (const attempt of [1, 2]) { + const restarted = heartbeatService(db); + const outcomes = await Promise.all([ + restarted.scheduleBoundedRetry(predecessor, { now, random: () => 0 }), + heartbeat.scheduleBoundedRetry(predecessor, { now, random: () => 0 }), + ]); + expect(outcomes.every(outcome => outcome.outcome === "scheduled")).toBe(true); + const children = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, predecessor)); + expect(children).toHaveLength(1); + const child = children[0]!; + expect(child).toMatchObject({ status: "scheduled_retry", scheduledRetryAttempt: attempt, scheduledRetryReason: "transient_failure" }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]?.executionRunId).toBe(child.id); + if (action !== "resume") { + if (action === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + if (action === "Stop") await restarted.cancelRun(child.id); + if (action === "dependency") { + const [blocker] = await db.insert(issues).values({ companyId, title: "Required dependency", status: "todo" }).returning(); + await db.insert(issueRelations).values({ companyId, issueId: blocker.id, relatedIssueId: issueId, type: "blocks" }); + } + const promotion = await createPostgresRunDispatchAdapter(db).promoteOrCancelDueRetry({ companyId, runId: child.id, + now: new Date(now.getTime() + 120_000) }); + expect(promotion.outcome).not.toBe("promoted"); + expect((await restarted.getRun(child.id))?.status).toBe("cancelled"); + return; + } + predecessor = child.id; + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "remote_workspace_prepare_timeout", resultJson, finishedAt: now }) + .where(eq(heartbeatRuns.id, predecessor)); + } + expect(await heartbeatService(db).scheduleBoundedRetry(predecessor, { now })).toMatchObject({ outcome: "retry_exhausted" }); + await heartbeatService(db).reconcileStrandedAssignedIssues(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(3); + }); + it("bounds interrupted conversations across restarts and concurrent scheduling", async () => { const { companyId, issueId, runId, now } = await seedMaxTurnFixture(); const resultJson = { conversationContinuation: "continue_conversation_v1" }; diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index f09aad46dec..60e80818bea 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -20,6 +20,7 @@ import { accountingForScheduledRetry, } from "../execution-recovery-attempt.js"; import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js"; +import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE } from "../remote-workspace-transport.js"; import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; import { randomUUID } from "node:crypto"; import { @@ -124,8 +125,9 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; -export function isTransientWorkspaceGitScanCode(code: string | null | undefined): boolean { - return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated; +export function isTransientWorkspacePreparationCode(code: string | null | undefined): boolean { + return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated + || code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE; } export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = @@ -565,7 +567,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen : baseSchedule; const requiresIssueGate = - isTransientWorkspaceGitScanCode(run.errorCode) || + isTransientWorkspacePreparationCode(run.errorCode) || hasConversationContinuationPolicy(run.resultJson) || (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) || retryReason === MAX_TURN_CONTINUATION_RETRY_REASON || diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 8b8416c1891..4f3b919f5a5 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -12,6 +12,7 @@ import { UnresolvedWorkspaceBaseRefError } from "../workspace-runtime.js"; import { LegacyControllerLeaseLostError } from "../legacy-controller-lease.js"; import { requestedRunCancellation } from "../run-cancellation.js"; import { ProjectAccessDeniedError } from "../project-access-denial.js"; +import { RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError } from "../remote-workspace-transport.js"; import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; import { createHeartbeatRunPreparation, ConfigurationIncompleteFailure } from "./run-preparation.js"; import { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js"; @@ -677,6 +678,20 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( } }); }); + it.each(["fresh", "started", "untrusted", "retained"] as const)("routes a %s workspace preparation timeout through bootstrap recovery", async scenario => { + const f = await fixture({ runtimeMode: scenario === "retained" ? "native" : "legacy" }); + const cause = new RemoteWorkspaceOperationTimeoutError("prepare", 300_000); + const error = scenario === "untrusted" ? new Error(cause.message) : new RemoteWorkspacePreparationTimeoutError(cause); + f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce({ outcome: "not_scheduled", reason: "Fixture scheduler boundary", issueId: null }); + await f.completion.failRunSetup(f.setup(error, scenario === "started")); + const failed = await read(f.run.id); + expect(failed).toMatchObject({ status: "failed", errorCode: scenario === "fresh" ? "remote_workspace_prepare_timeout" : "setup_failed" }); + if (scenario === "fresh") { + expect(failed.resultJson).toMatchObject({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }); + expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent); + } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + }); + it("records a lease lost before dispatch as a retryable host stop, not an opaque setup failure", async () => { const f = await fixture(); f.controller.abort(new LegacyControllerLeaseLostError("renewal_rejected")); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 17bf7a54e4f..720fe69d8ac 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -7,7 +7,8 @@ import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-acce import { pendingCaseHandoffForRun } from "./case-handoff-lifecycle.js"; import { NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, isHeartbeatRunTerminalStatus } from "./run-lifecycle.js"; import { ChatControlRecoveryUnresolvedError } from "./queue.js"; -import { readTransientRecoveryContractFromRun, readHeartbeatRunErrorFamily, isTransientWorkspaceGitScanCode } from "./retries.js"; +import { readTransientRecoveryContractFromRun, readHeartbeatRunErrorFamily, isTransientWorkspacePreparationCode } from "./retries.js"; +import { RemoteWorkspacePreparationTimeoutError } from "../remote-workspace-transport.js"; import { type RunSessionOutcome, resolveNextSessionState, normalizeUsageTotals, resolveCacheAdjustedCostUsd, resolveLedgerBiller, resolveLedgerCostStatus, normalizeLedgerBillingType } from "./run-state.js"; import { readGitConnectionFailure } from "../git-connection-failure.js"; import { CONFIGURATION_INCOMPLETE_FAILURE_CODE, ConfigurationIncompleteFailure } from "./run-preparation.js"; @@ -1350,8 +1351,12 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const nonRetryablePreflightCode = nonRetryablePreflightFailureCode(outerErr); const workspaceGitScanFailure = isWorkspaceGitScanError(outerErr) ? outerErr : null; + const remoteWorkspacePreparationTimeout = !providerDispatchStarted && run.runtimeMode === "legacy" + && run.nativeSessionId === null && run.runnerInstanceId === null && outerErr instanceof RemoteWorkspacePreparationTimeoutError + ? outerErr : null; const setupFailureErrorCode = (controllerLeaseLost ? CONTROLLER_LEASE_LOST_FAILURE_CODE : null) ?? + remoteWorkspacePreparationTimeout?.code ?? workspaceGitScanFailure?.code ?? workspaceValidationSetupFailure?.code ?? configurationIncompleteSetupFailure?.code ?? @@ -1377,7 +1382,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC workspaceGitScan: { code: workspaceGitScanFailure.code, phase: "workspace_setup", - retryable: isTransientWorkspaceGitScanCode(workspaceGitScanFailure.code), + retryable: isTransientWorkspacePreparationCode(workspaceGitScanFailure.code), }, } : null) ?? workspaceValidationSetupFailure?.resultJson ?? @@ -1493,11 +1498,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC () => undefined, ); } - // No provider work began. Retry temporary host scan failures with - // the existing durable failure budget, before releasing execution. - // Generic recovery must not grant a second budget on exhaustion. if (livenessRun.errorCode !== PROJECT_ACCESS_DENIED) { - await (isTransientWorkspaceGitScanCode(livenessRun.errorCode) + await (isTransientWorkspacePreparationCode(livenessRun.errorCode) ? scheduleBoundedRetryForRun(livenessRun, failedAgent) : scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent) ).catch((retryError) => { diff --git a/server/src/services/legacy-execution-recovery.test.ts b/server/src/services/legacy-execution-recovery.test.ts index b1563e07d1d..a57a29bf7dd 100644 --- a/server/src/services/legacy-execution-recovery.test.ts +++ b/server/src/services/legacy-execution-recovery.test.ts @@ -10,7 +10,7 @@ const stopped = { }, }; -it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated"])("does not invent unknown provider actions after exhausted %s bootstrap retries", (errorCode) => { +it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "remote_workspace_prepare_timeout"])("does not invent unknown provider actions after exhausted %s bootstrap retries", (errorCode) => { const run = { runtimeMode: "legacy", status: "failed", errorCode, scheduledRetryAttempt: 2, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }; expect(legacyExecutionNeedsReconciliation(run)).toBe(false); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index c462cdf1fca..68e61f9b02a 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -6,6 +6,7 @@ import { hasRequiredWorkspaceRecovery, preserveWorkspaceRestoreRecoveryMetadata, import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources, type LegacyWorkspaceRestoreSource } from "./legacy-workspace-restore-recovery.js"; import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js"; import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js"; +import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE } from "./remote-workspace-transport.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js"; import { claimedAdapterType, hasConversationContinuationPolicy } from "./conversation-continuation.js"; @@ -72,7 +73,8 @@ export function legacyExecutionNeedsReconciliation( // Setup owns the bounded retry budget for temporary workspace scans. Its // exhaustion needs workspace repair, not reconciliation of provider actions // that the bootstrap evidence proves never started. Keep unknown outcomes held. - if ((run.errorCode === "workspace_git_scan_timeout" || run.errorCode === "workspace_git_scan_saturated") && + if ((run.errorCode === "workspace_git_scan_timeout" || run.errorCode === "workspace_git_scan_saturated" + || run.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE) && evidence?.kind === "bootstrap" && evidence.providerWorkStarted === false) return false; if (executionFailureRetryCount(run) >= 2) return true; return !( diff --git a/server/src/services/recovery/service.pause-durability.test.ts b/server/src/services/recovery/service.pause-durability.test.ts index a3ad7a83b55..253bf1812d0 100644 --- a/server/src/services/recovery/service.pause-durability.test.ts +++ b/server/src/services/recovery/service.pause-durability.test.ts @@ -5,7 +5,7 @@ const run = (errorCode: string | null) => ({ errorCode } as unknown as Parameters[0]); describe("pause durability: continuation retry classification", () => { - it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_failed", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled"])("does not grant %s another recovery budget", (code) => { + it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_failed", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled", "remote_workspace_prepare_timeout"])("does not grant %s another recovery budget", (code) => { expect(classifyContinuationFailure(run(code))).toMatchObject({ kind: "non_retryable", maxAttempts: 0 }); }); it("agent_paused is retryable so work resumes (Option A: Resume Continues Work)", () => { diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 67d6eeae20c..f935b8a2df9 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -7,6 +7,7 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { boundRunIssueId } from "../run-task-binding.js"; +import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE } from "../remote-workspace-transport.js"; import { recoveryRunContextColumn, recoveryRunVersionColumn } from "./run-context-read.js"; import { recoveryRunResultColumn } from "./run-result-read.js"; import { readLegacyDispositionSource } from "./legacy-disposition-read.js"; @@ -552,6 +553,7 @@ const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set([ "workspace_git_scan_cancelled", "workspace_git_scan_output_limit", "workspace_git_scan_failed", + REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, "low_trust_isolation_unavailable", "low_trust_requires_isolated_workspace", "low_trust_boundary_mismatch", diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index c4f9e4f8346..0ec705aa7b6 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -49,6 +49,7 @@ const STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE: Record const WORKSPACE_SCAN_NOTICES: Record = { workspace_git_scan_timeout: { title: "Workspace scan timed out", nextAction: "Check repository access and server load, then retry the task." }, + remote_workspace_prepare_timeout: { title: "Workspace setup timed out", nextAction: "Inspect the failed run and check repository access and sandbox load before retrying the task." }, workspace_git_scan_saturated: { title: "Workspace scan queue is full", nextAction: "Check server load and the workspace scan queue, then retry the task." }, workspace_git_scan_output_limit: { title: "Workspace scan exceeded its limit", nextAction: "Check the repository size and workspace scan output limit before retrying the task." }, workspace_git_scan_failed: { title: "Workspace scan failed", nextAction: "Inspect the failed run and check repository access and integrity before retrying the task." }, diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 794a6d500f6..aa9df01d06c 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -30,12 +30,24 @@ const DISK_FULL_CHECK = String.raw`const diskFull=e=>['ENOSPC','EDQUOT'].include type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "restore" | "cleanup"; export class RemoteWorkspaceOperationTimeoutError extends Error { - constructor(operation: RemoteWorkspaceOperation, timeoutMs: number) { + constructor(readonly operation: RemoteWorkspaceOperation, readonly timeoutMs: number) { super(`remote_workspace_operation_timed_out: ${operation} (configured timeout ${timeoutMs}ms)`); this.name = "RemoteWorkspaceOperationTimeoutError"; } } +export const REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE = "remote_workspace_prepare_timeout"; + +export class RemoteWorkspacePreparationTimeoutError extends RemoteWorkspaceOperationTimeoutError { + readonly code = REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE; + + constructor(error: RemoteWorkspaceOperationTimeoutError) { + super("prepare", error.timeoutMs); + this.name = "RemoteWorkspacePreparationTimeoutError"; + this.cause = error; + } +} + async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspaceOperation, args: string[], env?: Record, timeoutMs = 300_000) { const result = await target.runner!.execute({ command: "node", args, cwd: target.remoteCwd, diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index e1c6e2d93c3..d87caddcc77 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -126,6 +126,138 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem companyId: ids.company, issueId: ids.issue, runId: ids.run, agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, existingWorkspaceId: null, source }); + it("keeps a fresh preparation timeout eligible for bounded bootstrap recovery", async () => { + const reserved = await reserve({ kind: "empty" }), remote = await target("timed-out-bootstrap"); + const execute = vi.spyOn(remote.runner!, "execute").mockResolvedValueOnce({ + exitCode: null, timedOut: true, stdout: "untrusted command output", stderr: "untrusted command output", signal: null, + pid: null, startedAt: new Date().toISOString(), + }); + await expect(prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: remote })) + .rejects.toMatchObject({ code: "remote_workspace_prepare_timeout" }); + expect(execute).toHaveBeenCalledOnce(); + expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata)).toMatchObject({ + writerRunId: ids.run, location: null, snapshotId: null, dirty: false, + }); + expect(await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.companyId, ids.company))).toHaveLength(0); + }); + + it.each(["dirty", "owner_changed"] as const)("does not grant bootstrap replay after the workspace becomes %s", async state => { + const reserved = await reserve({ kind: "empty" }), remote = await target(`changed-${state}`); + await fs.writeFile(path.join(remote.remoteCwd, "saved.txt"), "retain existing work\n"); + const authority = readRemoteWorkspaceAuthority(reserved.metadata)!; + vi.spyOn(remote.runner!, "execute").mockImplementationOnce(async () => { + await db.update(executionWorkspaces).set({ metadata: { ...reserved.metadata, remoteWorkspace: { + ...authority, ...(state === "dirty" ? { dirty: true } : { epoch: authority.epoch + 1, writerRunId: randomUUID() }), + } } }).where(eq(executionWorkspaces.id, reserved.id)); + return { exitCode: null, timedOut: true, stdout: "", stderr: "", signal: null, pid: null, startedAt: new Date().toISOString() }; + }); + await expect(prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: remote })) + .rejects.toMatchObject({ name: "RemoteWorkspaceOperationTimeoutError" }); + expect(await fs.readFile(path.join(remote.remoteCwd, "saved.txt"), "utf8")).toBe("retain existing work\n"); + }); + + async function releaseBootstrapLease(receipt: boolean) { + const pluginId = randomUUID(); + await db.update(environmentLeases).set({ metadata: { driver: "sandbox", pluginId, provider: "daytona", + sandboxProviderPlugin: true } }).where(eq(environmentLeases.id, ids.lease)); + const runtime = environmentRuntimeService(db, { pluginWorkerManager: { + isRunning: () => true, + getWorker: () => ({ supportedMethods: ["environmentReleaseLease"] }), + call: async (_id: string, method: string) => { + expect(method).toBe("environmentReleaseLease"); + return receipt ? { providerLeaseId: `allocation-${ids.lease}`, state: "stopped" } : undefined; + }, + } as never }); + await runtime.releaseRunLeases(ids.run, "failed"); + expect(await lease()).toMatchObject({ status: "failed", cleanupStatus: "success" }); + expect(Boolean((await lease()).metadata?.remoteExecutionTermination)).toBe(receipt); + } + + it.each(["receipt", "void"] as const)("recovers a lost preparation reply after %s cleanup and retains the resulting work", async cleanup => { + await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, issueId: ids.issue }) + .where(eq(heartbeatRuns.id, ids.run)); + const reserved = await reserve({ kind: "empty" }), first = await target("lost-bootstrap-reply"); + const actual = first.runner!.execute.bind(first.runner!); + const execute = vi.spyOn(first.runner!, "execute").mockImplementationOnce(async input => { + const result = await actual(input); + expect(result.exitCode).toBe(0); + return { ...result, timedOut: true }; + }); + await expect(prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first })) + .rejects.toMatchObject({ code: "remote_workspace_prepare_timeout" }); + const oldRoot = path.join(first.remoteCwd, ".paperclip-workspaces", reserved.id); + const oldReceipt = await fs.readFile(path.join(oldRoot, ".paperclip-runtime", "workspace-ready.json"), "utf8"); + const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, ids.run)); + await terminalizeLegacyExecution({ db, run: failed!, status: "failed", patch: { + errorCode: "remote_workspace_prepare_timeout", resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + } }); + await releaseBootstrapLease(cleanup === "receipt"); + const continueOn = async (name: string) => { + const runId = randomUUID(), leaseId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: ids.company, agentId: ids.agent, + nativeIssueId: ids.issue, runtimeMode: "native", status: "running" }); + const [nextLease] = await db.insert(environmentLeases).values({ id: leaseId, companyId: ids.company, + environmentId: ids.environment, heartbeatRunId: runId, issueId: ids.issue, provider: "daytona", + providerLeaseId: `allocation-${leaseId}`, status: "active" }).returning(); + const next = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, + runId, agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, + existingWorkspaceId: reserved.id, source: { kind: "empty" } }); + const remote = await target(name); + return prepareRemoteWorkspace({ db, storage, workspace: next, runId, lease: nextLease as EnvironmentLease, + target: remote }).then(prepared => ({ runId, prepared, remote })); + }; + const { runId, prepared, remote } = await continueOn("fresh-bootstrap-retry"); + await fs.writeFile(path.join(prepared.remoteRoot, "report.md"), "useful recovered result\n"); + const { snapshotId } = await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId, + lease: prepared.lease, target: remote, assertOwned: async () => undefined }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + await fs.rm(prepared.remoteRoot, { recursive: true }); + const restored = await continueOn("cold-bootstrap-continuation"); + expect(await fs.readFile(path.join(restored.prepared.remoteRoot, "report.md"), "utf8")).toBe("useful recovered result\n"); + expect(restored.prepared.reference.snapshotId).not.toBeNull(); + expect(snapshotId).toBeTruthy(); + expect(execute).toHaveBeenCalledOnce(); + expect(await fs.readFile(path.join(oldRoot, ".paperclip-runtime", "workspace-ready.json"), "utf8")).toBe(oldReceipt); + }, 60_000); + + it.each(["unclassified", "provider_started", "retained_native", "runner_identity", "native_mode", "saved", "published", "dirty", "cleanup_failed", "active_lease"] as const)( + "keeps receiptless workspace handoff blocked for %s", async state => { + const reserved = await reserve({ kind: "empty" }); + let savedAuthority: ReturnType = null; + let remoteRoot: string | undefined; + if (state === "saved" || state === "published") { + const remote = await target(`preserved-${state}`); + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: remote }); + remoteRoot = prepared.remoteRoot; + await fs.writeFile(path.join(remoteRoot, "saved.txt"), "keep prior files\n"); + await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId: ids.run, + lease: prepared.lease, target: remote, boundary: "initial", assertOwned: async () => undefined }); + savedAuthority = readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata); + } + await db.update(heartbeatRuns).set({ status: "failed", runtimeMode: state === "native_mode" ? "native" : "legacy", finishedAt: new Date(), + errorCode: state === "unclassified" ? "setup_failed" : "remote_workspace_prepare_timeout", + nativeSessionId: state === "retained_native" ? randomUUID() : null, + runnerInstanceId: state === "runner_identity" ? randomUUID() : null, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: state === "provider_started" } }, + }).where(eq(heartbeatRuns.id, ids.run)); + if (state !== "active_lease") await releaseBootstrapLease(false); + if (state === "cleanup_failed") await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, ids.lease)); + if (savedAuthority) await db.update(executionWorkspaces).set({ metadata: { ...reserved.metadata, + remoteWorkspace: { ...savedAuthority, dirty: false, snapshotId: state === "published" ? null : savedAuthority.snapshotId }, + } }).where(eq(executionWorkspaces.id, reserved.id)); + if (state === "dirty") await db.update(executionWorkspaces).set({ metadata: { ...reserved.metadata, + remoteWorkspace: { ...readRemoteWorkspaceAuthority(reserved.metadata)!, dirty: true }, + } }).where(eq(executionWorkspaces.id, reserved.id)); + const nextRun = randomUUID(); + await db.insert(heartbeatRuns).values({ id: nextRun, companyId: ids.company, agentId: ids.agent, + nativeIssueId: ids.issue, runtimeMode: "native", status: "running" }); + await expect(reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, + runId: nextRun, agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, + existingWorkspaceId: reserved.id, source: { kind: "empty" } })).rejects.toThrow("unfinished writer"); + expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata)?.writerRunId).toBe(ids.run); + if (remoteRoot) expect(await fs.readFile(path.join(remoteRoot, "saved.txt"), "utf8")).toBe("keep prior files\n"); + }); + it.each(["initial", "follow-up"] as const)("edits and cold-restores both an anchor and a referenced project's task copy (%s attachment)", async attachment => { vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "true"); vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE", "true"); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 231896c716c..6c7cdee8dab 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -25,7 +25,7 @@ import { canonicalExecutionUuid } from "./execution-identity.js"; import { persistActivity } from "./activity-log.js"; import { captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath, - RemoteWorkspaceOperationTimeoutError, requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, + REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, } from "./remote-workspace-transport.js"; import { beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot, @@ -196,6 +196,8 @@ export async function reserveRemoteWorkspace(input: { if (authority.writerRunId && authority.writerRunId !== input.runId) { const [prior] = await tx.select({ status: heartbeatRuns.status, agentId: heartbeatRuns.agentId, issueId: heartbeatRuns.issueId, nativeIssueId: heartbeatRuns.nativeIssueId, + runtimeMode: heartbeatRuns.runtimeMode, nativeSessionId: heartbeatRuns.nativeSessionId, runnerInstanceId: heartbeatRuns.runnerInstanceId, + errorCode: heartbeatRuns.errorCode, executionRecovery: sql`${heartbeatRuns.resultJson}->'executionRecovery'`, contextIssueId: sql`coalesce(${heartbeatRuns.contextSnapshot}->>'issueId', ${heartbeatRuns.contextSnapshot}->>'taskId')`, lastOutputAt: heartbeatRuns.lastOutputAt, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt, }).from(heartbeatRuns).where(and( @@ -211,8 +213,14 @@ export async function reserveRemoteWorkspace(input: { } const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.heartbeatRunId, authority.writerRunId))); + const bootstrap = object(prior?.executionRecovery); + const pristinePreparation = !authority.location && !authority.snapshotId && prior?.status === "failed" + && prior.runtimeMode === "legacy" && prior.nativeSessionId === null && prior.runnerInstanceId === null + && prior.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE + && bootstrap.kind === "bootstrap" && bootstrap.providerWorkStarted === false; const unusedReservation = !authority.dirty && prior && ["failed", "cancelled", "interrupted", "timed_out", "succeeded"].includes(prior.status) - && leases.every(hasRemoteTerminationReceipt); + && leases.every(lease => hasRemoteTerminationReceipt(lease) || (pristinePreparation + && lease.releasedAt && lease.cleanupStatus === "success" && ["released", "expired", "failed"].includes(lease.status))); if (!unusedReservation) throw conflict("The workspace still has an unfinished writer or save operation.", { code: "remote_workspace_writer_pending" }); } if (authority.writerRunId === input.runId) return existing; @@ -355,9 +363,23 @@ export async function prepareRemoteWorkspace(input: { restorationIdentity: preparationIdentity }); } finally { await rm(scratch, { recursive: true, force: true }); } } else { - await prepareRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, source: initial.source, - preparationIdentity, - resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) }); + try { + await prepareRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, source: initial.source, + preparationIdentity, + resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) }); + } catch (error) { + if (error instanceof RemoteWorkspaceOperationTimeoutError && error.operation === "prepare" + && !initial.dirty && !initial.location) { + const [latest] = await input.db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, input.workspace.companyId), eq(executionWorkspaces.id, input.workspace.id))); + const current = readRemoteWorkspaceAuthority(latest?.metadata); + if (current && current.epoch === initial.epoch && current.writerRunId === input.runId + && !current.dirty && !current.location && !current.snapshotId) { + throw new RemoteWorkspacePreparationTimeoutError(error); + } + } + throw error; + } } } for (const repository of (input.source ?? initial.source).repositories ?? []) {