diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 62379bdc484..1f5d3d40020 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -34,6 +34,9 @@ import { executionProjectionForRun } from "../services/execution-projection.js"; import { JsonRpcCallError } from "@paperclipai/plugin-sdk"; import { leaseFaultError } from "../services/environment-lease-fault-injection.js"; import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; +import { automaticRunBrake } from "../services/recovery/automatic-run-brake.js"; +import { environmentService } from "../services/environments.js"; +import { ENVIRONMENT_PROVIDER_WAIT, ENVIRONMENT_PROVIDER_WAIT_KEY, ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON } from "../services/environment-provider-wait.js"; import { requestTaskRetry } from "../services/task-retry.js"; import { adapterExecutionControls, createAdapterExecutionControl } from "../services/adapter-execution-control.js"; import { execFileSync, spawn, spawnSync, type ChildProcess } from "node:child_process"; @@ -1572,8 +1575,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .then((rows) => rows.map((row) => row.blockerIssueId)); } - async function seedQueuedIssueRunFixture() { - const companyId = randomUUID(); + async function seedQueuedIssueRunFixture(shared?: { companyId: string; issueNumber: number }) { + const companyId = shared?.companyId ?? randomUUID(); const agentId = randomUUID(); const runId = randomUUID(); const wakeupRequestId = randomUUID(); @@ -1581,12 +1584,12 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const now = new Date("2026-03-19T00:00:00.000Z"); const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; - await seedRecoveryCompany(companyId, issuePrefix); + if (!shared) await seedRecoveryCompany(companyId, issuePrefix); await db.insert(agents).values({ id: agentId, companyId, - name: "CodexCoder", + name: shared ? `CodexCoder ${shared.issueNumber}` : "CodexCoder", role: "engineer", status: "idle", adapterType: "codex_local", @@ -1643,8 +1646,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { checkoutRunId: runId, executionRunId: runId, responsibleUserId: "responsible-user", - issueNumber: 1, - identifier: `${issuePrefix}-1`, + issueNumber: shared?.issueNumber ?? 1, + identifier: `${issuePrefix}-${shared?.issueNumber ?? 1}`, startedAt: now, }); @@ -6306,9 +6309,18 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } }, 60_000); - it("explains the block with a working Retry once automatic retries are used up", async () => { + async function expireProviderWait(agentId: string) { + const [waiting] = (await runsFor(agentId)).filter(run => run.status === "scheduled_retry"); + expect(waiting).toMatchObject({ scheduledRetryReason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON }); + const wait = waiting!.contextSnapshot?.[ENVIRONMENT_PROVIDER_WAIT_KEY] as Record; + await db.update(heartbeatRuns).set({ contextSnapshot: { ...waiting!.contextSnapshot, [ENVIRONMENT_PROVIDER_WAIT_KEY]: { ...wait, + waitingSince: new Date(Date.now() - ENVIRONMENT_PROVIDER_WAIT.ceilingMs - 1_000).toISOString() } } }) + .where(eq(heartbeatRuns.id, waiting!.id)); + } + + it("waits for the provider without a Retry, and offers a working Retry only after the 30-minute ceiling", async () => { const { companyId, agentId, issueId } = await seedQueuedIssueRunFixture(); - const lease = failLeaseAcquisition(3); + const lease = failLeaseAcquisition(5); try { const heartbeat = heartbeatService(db); await heartbeat.resumeQueuedRuns(); @@ -6316,13 +6328,39 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { await promoteLatestScheduledRetry(heartbeat, agentId); await promoteLatestScheduledRetry(heartbeat, agentId); - const exhausted = await runsFor(agentId); - expect(exhausted.map(run => [run.status, run.errorCode])).toEqual([ - ["failed", "environment_provider_unavailable"], - ["failed", "environment_provider_unavailable"], - ["failed", "environment_provider_unavailable"], + const fastRetriesSpent = await runsFor(agentId); + expect(fastRetriesSpent.map(run => [run.status, run.errorCode, run.scheduledRetryReason])).toEqual([ + ["failed", "environment_provider_unavailable", null], + ["failed", "environment_provider_unavailable", "transient_failure"], + ["failed", "environment_provider_unavailable", "transient_failure"], + ["scheduled_retry", null, ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON], ]); - expect(lease.acquisitions.refused).toBe(3); + expect(fastRetriesSpent[3]).toMatchObject({ scheduledRetryAttempt: 1 }); + expect(fastRetriesSpent[3]!.scheduledRetryAt!.getTime() - fastRetriesSpent[3]!.createdAt.getTime()) + .toBeGreaterThanOrEqual(ENVIRONMENT_PROVIDER_WAIT.delaysMs[0] - 1_000); + const [waiting] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(waiting).toMatchObject({ status: "in_progress", executionRunId: fastRetriesSpent[3]!.id }); + expect(await taskInterruptionPresentation(db, waiting!)).toMatchObject({ state: "working", responsibility: "system", + canRetry: false, retryWait: { reason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + message: expect.stringMatching(/^Waiting for .+ to come back\. Paperclip continues this task automatically when it does\.$/) } }); + expect((await selectRecoveryActionsForIssue(issueId)).filter(action => action.status === "active")).toEqual([]); + + await promoteLatestScheduledRetry(heartbeat, agentId); + const stillWaiting = await runsFor(agentId); + expect(stillWaiting.at(-1)).toMatchObject({ status: "scheduled_retry", scheduledRetryReason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + scheduledRetryAttempt: 2 }); + expect(stillWaiting.at(-1)!.scheduledRetryAt!.getTime() - stillWaiting.at(-1)!.createdAt.getTime()) + .toBeGreaterThanOrEqual(ENVIRONMENT_PROVIDER_WAIT.delaysMs[1] - 1_000); + const [stillWorking] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(await taskInterruptionPresentation(db, stillWorking!)).toMatchObject({ state: "working", canRetry: false }); + + await expireProviderWait(agentId); + await promoteLatestScheduledRetry(heartbeat, agentId); + + const exhausted = await runsFor(agentId); + expect(exhausted.map(run => [run.status, run.errorCode])).toEqual( + Array(5).fill(["failed", "environment_provider_unavailable"])); + expect(lease.acquisitions.refused).toBe(5); expect(mockAdapterExecute).not.toHaveBeenCalled(); const blocked = await waitForValue(async () => { @@ -6333,12 +6371,13 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { // The blocked task names its cause, its next action, and a Retry. const presentation = await taskInterruptionPresentation(db, blocked!); expect(presentation).toMatchObject({ state: "failed", responsibility: "system", canRetry: true, - observedRunId: exhausted[2]!.id, + observedRunId: exhausted[4]!.id, providerFailure: { httpStatus: 502, phase: "startup", retryability: "transient" } }); expect(presentation?.providerFailure?.message).toContain("temporarily unavailable"); const notices = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); expect(notices.map(comment => (comment.presentation as { title?: string } | null)?.title)) .toContain("Environment provider was temporarily unavailable"); + expect(presentation?.retryWait).toBeUndefined(); // No no-replay hold: provider work never started. expect(await getExecutionBlocker(db, companyId, issueId)).toBeNull(); @@ -6393,7 +6432,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { it("rests a provider deadline that never clears on a working Retry, never on a board recovery action", async () => { const { companyId, agentId, issueId } = await seedQueuedIssueRunFixture(); - const lease = failLeaseAcquisition(BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS + 1, daytonaAcquisitionDeadline); + const lease = failLeaseAcquisition(BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS + 2, daytonaAcquisitionDeadline); try { const heartbeat = heartbeatService(db); await heartbeat.resumeQueuedRuns(); @@ -6401,9 +6440,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { for (let attempt = 0; attempt < BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS; attempt += 1) { await promoteLatestScheduledRetry(heartbeat, agentId); } + await expireProviderWait(agentId); + await promoteLatestScheduledRetry(heartbeat, agentId); const exhausted = await runsFor(agentId); expect(exhausted.map(run => run.errorCode)).toEqual( - Array(BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS + 1).fill("environment_provider_unavailable")); + Array(BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS + 2).fill("environment_provider_unavailable")); expect(mockAdapterExecute).not.toHaveBeenCalled(); const blocked = await waitForValue(async () => { @@ -6431,6 +6472,172 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { mockAdapterExecute.mockClear(); } }, 90_000); + + async function seedProviderWaiters(leaderWaitedMs: number, followerWaitedMs: number) { + const leader = await seedQueuedIssueRunFixture(); + const follower = await seedQueuedIssueRunFixture({ companyId: leader.companyId, issueNumber: 2 }); + const environment = await environmentService(db).ensureLocalEnvironment(leader.companyId); + const now = new Date(); + for (const [task, waitedMs] of [[leader, leaderWaitedMs], [follower, followerWaitedMs]] as const) { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, task.runId)); + await db.update(heartbeatRuns).set({ status: "scheduled_retry", scheduledRetryReason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + scheduledRetryAttempt: 1, scheduledRetryAt: new Date(now.getTime() - 1_000), createdAt: new Date(now.getTime() - 10_000), + contextSnapshot: { ...run!.contextSnapshot, [ENVIRONMENT_PROVIDER_WAIT_KEY]: { environmentId: environment.id, + environmentName: environment.name, waitingSince: new Date(now.getTime() - waitedMs).toISOString() } } }) + .where(eq(heartbeatRuns.id, task.runId)); + } + return { leader, follower, environment, now }; + } + + it("lets the next waiter probe when the first waiter cannot start because its agent is busy", async () => { + const { leader, follower, now } = await seedProviderWaiters(120_000, 60_000); + await db.update(heartbeatRuns).set({ status: "queued", scheduledRetryAt: null, + updatedAt: new Date(now.getTime() - 2 * ENVIRONMENT_PROVIDER_WAIT.probeStartGraceMs) }).where(eq(heartbeatRuns.id, leader.runId)); + await heartbeatService(db).promoteDueScheduledRetries(now); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, follower.runId)))[0]!.status).toBe("queued"); + }, 60_000); + + it("hands a parked task to a person with a working Retry at the 30-minute ceiling even while the leader stays queued", async () => { + const { leader, follower, now } = await seedProviderWaiters(ENVIRONMENT_PROVIDER_WAIT.ceilingMs + 120_000, + ENVIRONMENT_PROVIDER_WAIT.ceilingMs + 60_000); + await db.update(heartbeatRuns).set({ status: "queued", scheduledRetryAt: null, updatedAt: now }).where(eq(heartbeatRuns.id, leader.runId)); + const lease = failLeaseAcquisition(2); + try { + const heartbeat = heartbeatService(db); + await heartbeat.promoteDueScheduledRetries(now); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, follower.runId)))[0]!.status).not.toBe("scheduled_retry"); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + expect(lease.acquisitions.refused).toBeGreaterThanOrEqual(1); + expect((await runsFor(follower.agentId)).filter(run => run.status === "scheduled_retry")).toEqual([]); + const blocked = await waitForValue(async () => { + const [row] = await db.select().from(issues).where(eq(issues.id, follower.issueId)); + return row?.status === "blocked" ? row : null; + }); + expect(await taskInterruptionPresentation(db, blocked!)).toMatchObject({ state: "failed", canRetry: true }); + } finally { + lease.restore(); + mockAdapterExecute.mockClear(); + } + }, 60_000); + + it("lets one waiter per environment probe the provider and releases the others once a lease is acquired", async () => { + const { leader, follower, environment, now } = await seedProviderWaiters(120_000, 60_000); + const heartbeat = heartbeatService(db); + + await heartbeat.promoteDueScheduledRetries(now); + const [probe] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, leader.runId)); + const [parked] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, follower.runId)); + expect(probe!.status).toBe("queued"); + expect(parked).toMatchObject({ status: "scheduled_retry", scheduledRetryAttempt: 1 }); + expect(parked!.scheduledRetryAt!.getTime() - now.getTime()).toBe(ENVIRONMENT_PROVIDER_WAIT.parkedRecheckMs); + + await db.update(heartbeatRuns).set({ status: "running", startedAt: now }).where(eq(heartbeatRuns.id, leader.runId)); + const recheck = new Date(now.getTime() + ENVIRONMENT_PROVIDER_WAIT.parkedRecheckMs); + await heartbeat.promoteDueScheduledRetries(recheck); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, follower.runId)))[0]).toMatchObject({ + status: "scheduled_retry", scheduledRetryAt: new Date(recheck.getTime() + ENVIRONMENT_PROVIDER_WAIT.parkedRecheckMs) }); + + await db.insert(environmentLeases).values({ companyId: leader.companyId, environmentId: environment.id, + heartbeatRunId: leader.runId, issueId: leader.issueId, status: "active", provider: "local", acquiredAt: now }); + const released = new Date(recheck.getTime() + ENVIRONMENT_PROVIDER_WAIT.parkedRecheckMs); + await heartbeat.promoteDueScheduledRetries(released); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, follower.runId)))[0]!.status).toBe("queued"); + }, 60_000); + + it("resumes ten tasks by itself when a five-minute provider outage ends, one probe at a time and within the automatic-run brake", async () => { + const first = await seedQueuedIssueRunFixture(); + const tasks = [first]; + for (let issueNumber = 2; issueNumber <= 10; issueNumber += 1) + tasks.push(await seedQueuedIssueRunFixture({ companyId: first.companyId, issueNumber })); + const outageMs = 5 * 60_000; + let simulatedMs = 0; + const attempts: Array<{ at: number; refused: boolean; runId: string | null }> = []; + const originalFactory = environmentOrchestrator.environmentRunOrchestrator; + const factory = vi.spyOn(environmentOrchestrator, "environmentRunOrchestrator").mockImplementation((database, options = {}) => { + const runtime = originalFactory(database, options).runtime; + return originalFactory(database, { ...options, environmentRuntime: { ...runtime, acquireRunLease: async (input) => { + const refused = simulatedMs < outageMs; + attempts.push({ at: simulatedMs, refused, runId: input.heartbeatRunId }); + if (refused) throw daytonaCreate502(); + return runtime.acquireRunLease(input); + } } }); + }); + const advance = async (ms: number) => { + const by = `${ms} milliseconds`; + await db.execute(sql`update heartbeat_runs set created_at = created_at - ${by}::interval, updated_at = updated_at - ${by}::interval, + started_at = started_at - ${by}::interval, finished_at = finished_at - ${by}::interval, + scheduled_retry_at = scheduled_retry_at - ${by}::interval, last_output_at = last_output_at - ${by}::interval, + context_snapshot = case when context_snapshot #>> '{environmentProviderWait,waitingSince}' is null then context_snapshot + else jsonb_set(context_snapshot, '{environmentProviderWait,waitingSince}', to_jsonb(to_char( + ((context_snapshot #>> '{environmentProviderWait,waitingSince}')::timestamptz - ${by}::interval) at time zone 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))) end + where company_id = ${first.companyId}`); + await db.execute(sql`update agent_wakeup_requests set requested_at = requested_at - ${by}::interval, + claimed_at = claimed_at - ${by}::interval, finished_at = finished_at - ${by}::interval, + created_at = created_at - ${by}::interval, updated_at = updated_at - ${by}::interval where company_id = ${first.companyId}`); + await db.execute(sql`update environment_leases set acquired_at = acquired_at - ${by}::interval, created_at = created_at - ${by}::interval, + last_used_at = last_used_at - ${by}::interval, released_at = released_at - ${by}::interval where company_id = ${first.companyId}`); + await db.execute(sql`update automatic_run_brakes set state_changed_at = state_changed_at - ${by}::interval, + opened_at = opened_at - ${by}::interval, next_probe_at = next_probe_at - ${by}::interval, + release_window_started_at = release_window_started_at - ${by}::interval, next_creation_at = next_creation_at - ${by}::interval + where company_id = ${first.companyId}`); + automaticRunBrake.reset(); + simulatedMs += ms; + }; + const finished = async () => (await Promise.all(tasks.map(async task => + (await runsFor(task.agentId)).some(run => run.status === "succeeded")))).every(Boolean); + const ticks: Array<{ at: number; brake: string; brakeAfter: string; starts: number }> = []; + try { + const heartbeat = heartbeatService(db); + const tick = async () => { + await heartbeat.promoteDueScheduledRetries(new Date()); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + }; + await tick(); + for (let step = 0; step < 120 && !(await finished()); step += 1) { + await advance(30_000); + const brake = (await automaticRunBrake.read(db, first.companyId)).state; + const before = attempts.length; + await tick(); + automaticRunBrake.reset(); + const brakeAfter = (await automaticRunBrake.read(db, first.companyId)).state; + ticks.push({ at: simulatedMs, brake, brakeAfter, starts: attempts.length - before }); + for (const task of tasks) { + const taskRuns = await runsFor(task.agentId); + if (taskRuns.some(run => run.status === "succeeded")) continue; + const [issue] = await db.select().from(issues).where(eq(issues.id, task.issueId)); + const current = taskRuns.find(run => run.id === issue!.executionRunId); + if (current?.status === "scheduled_retry" && current.scheduledRetryReason !== ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON + && current.scheduledRetryAt && current.scheduledRetryAt.getTime() <= Date.now()) continue; + expect(issue!.status).not.toBe("blocked"); + expect((await taskInterruptionPresentation(db, issue!))?.canRetry ?? false).toBe(false); + } + } + + expect(await finished()).toBe(true); + expect(simulatedMs).toBeLessThan(ENVIRONMENT_PROVIDER_WAIT.ceilingMs); + const runs = (await Promise.all(tasks.map(task => runsFor(task.agentId)))).flat(); + const waitRunIds = new Set(runs.filter(run => run.scheduledRetryReason === ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON).map(run => run.id)); + const waitRefusalsByTick = new Map(); + for (const attempt of attempts.filter(entry => entry.refused && entry.runId !== null && waitRunIds.has(entry.runId))) + waitRefusalsByTick.set(attempt.at, (waitRefusalsByTick.get(attempt.at) ?? 0) + 1); + expect([...waitRefusalsByTick.values()].every(count => count === 1)).toBe(true); + expect(ticks.some(entry => entry.brake === "open")).toBe(true); + expect(ticks.filter(entry => entry.brake === "open" && entry.brakeAfter === "open").every(entry => entry.starts <= 1)).toBe(true); + expect(simulatedMs).toBeLessThanOrEqual(outageMs + 5 * 60_000); + for (const task of tasks) { + expect((await selectRecoveryActionsForIssue(task.issueId)) + .filter(action => action.status === "active" && action.cause === "environment_provider_unavailable")).toEqual([]); + } + expect(runs.filter(run => run.status === "succeeded" && run.errorCode === null).length).toBeGreaterThanOrEqual(10); + expect(attempts.filter(entry => !entry.refused).length).toBeGreaterThanOrEqual(10); + } finally { + factory.mockRestore(); + mockAdapterExecute.mockClear(); + } + }, 300_000); }); it("classifies only the installed-but-not-ready sandbox provider plugin message as a configuration gap", () => { diff --git a/server/src/modules/run-dispatch/adapters/environment-provider-wait.ts b/server/src/modules/run-dispatch/adapters/environment-provider-wait.ts new file mode 100644 index 00000000000..1195ad381ae --- /dev/null +++ b/server/src/modules/run-dispatch/adapters/environment-provider-wait.ts @@ -0,0 +1,45 @@ +import { and, eq, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, type Db } from "@paperclipai/db"; +import { parkScheduledRetryInTx } from "./scheduled-retry-park.js"; +import { + ENVIRONMENT_PROVIDER_WAIT, + ENVIRONMENT_PROVIDER_WAIT_KEY, + ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + readEnvironmentProviderWait, +} from "../../../services/environment-provider-wait.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; + +const waitEnvironmentSql = sql`${heartbeatRuns.contextSnapshot} -> ${ENVIRONMENT_PROVIDER_WAIT_KEY} ->> 'environmentId'`; +const waitingSinceSql = sql`${heartbeatRuns.contextSnapshot} -> ${ENVIRONMENT_PROVIDER_WAIT_KEY} ->> 'waitingSince'`; + +export async function parkEnvironmentProviderWaitInTx(tx: Db, run: HeartbeatRun, now: Date): Promise { + if (run.status !== "scheduled_retry" || run.scheduledRetryReason !== ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON) return false; + const wait = readEnvironmentProviderWait(run.contextSnapshot); + if (!wait) return false; + if (Date.parse(wait.waitingSince) + ENVIRONMENT_PROVIDER_WAIT.ceilingMs <= now.getTime()) return false; + const [recovered] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, run.companyId), + eq(environmentLeases.environmentId, wait.environmentId), + sql`${environmentLeases.acquiredAt} >= ${run.createdAt.toISOString()}::timestamptz`, + sql`${environmentLeases.status} <> 'pending_cleanup'`, + sql`${environmentLeases.failureReason} is null`, + )).limit(1); + if (recovered) return false; + const [earlier] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), + sql`${heartbeatRuns.id} <> ${run.id}`, + eq(heartbeatRuns.scheduledRetryReason, ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON), + sql`${heartbeatRuns.status} in ('scheduled_retry', 'queued', 'running')`, + sql`${waitEnvironmentSql} = ${wait.environmentId}`, + sql`((${waitingSinceSql})::timestamptz, ${heartbeatRuns.id}::text) < (${wait.waitingSince}::timestamptz, ${run.id}::text)`, + sql`(${heartbeatRuns.status} = 'scheduled_retry' + or (${heartbeatRuns.status} = 'queued' and ${heartbeatRuns.updatedAt} > ${new Date(now.getTime() - ENVIRONMENT_PROVIDER_WAIT.probeStartGraceMs).toISOString()}::timestamptz) + or (${heartbeatRuns.status} = 'running' and ${heartbeatRuns.startedAt} > ${new Date(now.getTime() - ENVIRONMENT_PROVIDER_WAIT.probeRunningBoundMs).toISOString()}::timestamptz + and not exists (select 1 from ${environmentLeases} l + where l.company_id = ${heartbeatRuns.companyId} and l.heartbeat_run_id = ${heartbeatRuns.id})))`, + )).limit(1); + if (!earlier) return false; + const recheckAt = new Date(now.getTime() + ENVIRONMENT_PROVIDER_WAIT.parkedRecheckMs); + return parkScheduledRetryInTx(tx, run, now, recheckAt); +} diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index e68204939ec..e2e24bd6c9f 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -102,6 +102,7 @@ import type { } from "../application/types.js"; import { RunDispatchApplicationError } from "../application/types.js"; import { parkWorkspaceBusyRetryInTx } from "./workspace-busy-wait.js"; +import { parkEnvironmentProviderWaitInTx } from "./environment-provider-wait.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type LoadGateFactsInput = { @@ -1005,6 +1006,9 @@ export function createPostgresRunDispatchAdapter( if (await parkWorkspaceBusyRetryInTx(tx, run, now)) { return { outcome: { outcome: "not_promoted" as const }, telemetryRun: null }; } + if (await parkEnvironmentProviderWaitInTx(tx, run, now)) { + return { outcome: { outcome: "not_promoted" as const }, telemetryRun: null }; + } const promoted = await promoteDueRetryInTx(tx as unknown as Db, { runId: input.runId, diff --git a/server/src/modules/run-dispatch/adapters/scheduled-retry-park.ts b/server/src/modules/run-dispatch/adapters/scheduled-retry-park.ts new file mode 100644 index 00000000000..33611b25948 --- /dev/null +++ b/server/src/modules/run-dispatch/adapters/scheduled-retry-park.ts @@ -0,0 +1,18 @@ +import { and, eq, sql } from "drizzle-orm"; +import { heartbeatRuns, type Db } from "@paperclipai/db"; +import { notifyScheduledRetryCancellation } from "../../../services/scheduled-retry-cancellation-notifications.js"; + +export async function parkScheduledRetryInTx(tx: Db, run: Pick, + now: Date, recheckAt: Date): Promise { + await notifyScheduledRetryCancellation(tx, { companyId: run.companyId, runId: run.id }); + const [parked] = await tx.update(heartbeatRuns).set({ + scheduledRetryAt: recheckAt, + updatedAt: now, + contextSnapshot: sql`jsonb_set(coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb), '{scheduledRetryAt}', to_jsonb(${recheckAt.toISOString()}::text))`, + }).where(and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.status, "scheduled_retry"), + )).returning({ id: heartbeatRuns.id }); + return Boolean(parked); +} diff --git a/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts b/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts index b9cdf5e13d7..17c2ffcb3c4 100644 --- a/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts +++ b/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts @@ -2,7 +2,7 @@ import { and, eq, gt, inArray, ne, sql } from "drizzle-orm"; import { executionWorkspaces, heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { notifyScheduledRetryDispatch } from "../../../services/run-dispatch-notifications.js"; -import { notifyScheduledRetryCancellation } from "../../../services/scheduled-retry-cancellation-notifications.js"; +import { parkScheduledRetryInTx } from "./scheduled-retry-park.js"; import { readRemoteWorkspaceWriterHold, runOwnerFenceHeldSql } from "../../../services/remote-workspace-writer-hold.js"; import { WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, @@ -80,21 +80,7 @@ export async function parkWorkspaceBusyRetryInTx(tx: Db, run: HeartbeatRun, now: now.getTime() + (holderLive && settlingSince ? WORKSPACE_BUSY_SETTLING_RECHECK_MS : WORKSPACE_BUSY_PARKED_RECHECK_MS), holderLive ? holderStaleAt! + 1_000 : Number.POSITIVE_INFINITY, )); - await notifyScheduledRetryCancellation(tx, { companyId: run.companyId, runId: run.id }); - const [parked] = await tx - .update(heartbeatRuns) - .set({ - scheduledRetryAt: recheckAt, - updatedAt: now, - contextSnapshot: sql`jsonb_set(coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb), '{scheduledRetryAt}', to_jsonb(${recheckAt.toISOString()}::text))`, - }) - .where(and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.companyId, run.companyId), - eq(heartbeatRuns.status, "scheduled_retry"), - )) - .returning({ id: heartbeatRuns.id }); - return Boolean(parked); + return parkScheduledRetryInTx(tx, run, now, recheckAt); } /** diff --git a/server/src/services/environment-provider-unavailable-error.ts b/server/src/services/environment-provider-unavailable-error.ts index 852c39b0b91..3f7ecba56c3 100644 --- a/server/src/services/environment-provider-unavailable-error.ts +++ b/server/src/services/environment-provider-unavailable-error.ts @@ -128,10 +128,12 @@ export class EnvironmentProviderUnavailableError extends Error { readonly code = ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE; readonly httpStatus: number | null; readonly errorCode: string | null; + readonly environmentId: string | null; + readonly environmentName: string; /** Read by the setup-failure path and shown on the task as the reason. */ readonly providerFailure: ProviderFailure; - constructor(input: { environmentName: string; driver: string; evidence: TransientProviderEvidence; cause: unknown }) { + constructor(input: { environmentId?: string; environmentName: string; driver: string; evidence: TransientProviderEvidence; cause: unknown }) { const status = input.evidence.httpStatus !== undefined ? `HTTP ${input.evidence.httpStatus}` : input.evidence.errorCode ?? (input.evidence.reason ? REASON_LABELS[input.evidence.reason] : undefined); super(status @@ -140,6 +142,8 @@ export class EnvironmentProviderUnavailableError extends Error { this.name = "EnvironmentProviderUnavailableError"; this.httpStatus = input.evidence.httpStatus ?? null; this.errorCode = input.evidence.errorCode ?? null; + this.environmentId = input.environmentId ?? null; + this.environmentName = input.environmentName; const providerCode = input.evidence.errorCode ?? (input.evidence.httpStatus === undefined ? input.evidence.reason : undefined); this.providerFailure = { schema: "paperclip.provider-failure.v1", @@ -149,7 +153,7 @@ export class EnvironmentProviderUnavailableError extends Error { message: (status ? `The provider for "${input.environmentName}" was temporarily unavailable while Paperclip was starting it (${status}). ` : `The sandbox for "${input.environmentName}" did not finish starting in time. `) - + "No agent work had started. Paperclip retries automatically; if those retries are used up, use Retry once the provider is available.", + + "No agent work had started. Paperclip keeps retrying automatically until the provider is back; if it stays unavailable for 30 minutes, use Retry once it is available.", ...(input.evidence.httpStatus !== undefined ? { httpStatus: input.evidence.httpStatus } : {}), ...(providerCode !== undefined ? { providerCode } : {}), retryability: "transient", diff --git a/server/src/services/environment-provider-wait.test.ts b/server/src/services/environment-provider-wait.test.ts new file mode 100644 index 00000000000..c1820274c09 --- /dev/null +++ b/server/src/services/environment-provider-wait.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { + ENVIRONMENT_PROVIDER_WAIT, + ENVIRONMENT_PROVIDER_WAIT_KEY, + ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + environmentProviderWaitFor, + environmentProviderWaitMessage, + environmentProviderWaitRetry, +} from "./environment-provider-wait.js"; + +const environmentId = "6f1c1e8a-3f5b-4b55-9a2e-6f0f6b1b8f11"; +const now = new Date("2026-11-17T19:45:00.000Z"); +const failed = { errorCode: "environment_provider_unavailable", + resultJson: { environmentProvider: { environmentId, environmentName: "Daytona" } } }; + +function waiting(attempt: number, waitedMs: number) { + return { ...failed, scheduledRetryReason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, scheduledRetryAttempt: attempt, + contextSnapshot: { [ENVIRONMENT_PROVIDER_WAIT_KEY]: { environmentId, environmentName: "Daytona", + waitingSince: new Date(now.getTime() - waitedMs).toISOString() } } }; +} + +describe("environment provider outage wait", () => { + it("starts the wait when the fast retries are spent", () => { + expect(environmentProviderWaitFor({ ...failed, scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 2 }, now)) + .toEqual({ environmentId, environmentName: "Daytona", waitingSince: now.toISOString() }); + expect(environmentProviderWaitRetry(failed, now)).toEqual({ retryReason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + wakeReason: "transient_failure_retry", maxAttempts: ENVIRONMENT_PROVIDER_WAIT.maxAttempts, delayMs: 60_000 }); + }); + + it("backs off from one minute to two and then five, keeping the original start", () => { + expect(environmentProviderWaitRetry(waiting(1, 60_000), now)?.delayMs).toBe(2 * 60_000); + expect(environmentProviderWaitRetry(waiting(2, 3 * 60_000), now)?.delayMs).toBe(5 * 60_000); + expect(environmentProviderWaitRetry(waiting(5, 18 * 60_000), now)?.delayMs).toBe(5 * 60_000); + expect(environmentProviderWaitFor(waiting(3, 8 * 60_000), now)?.waitingSince) + .toBe(new Date(now.getTime() - 8 * 60_000).toISOString()); + }); + + it("never waits past the ceiling and stops at it", () => { + expect(environmentProviderWaitRetry(waiting(6, 28 * 60_000), now)).toMatchObject({ delayMs: 2 * 60_000, + maxAttempts: ENVIRONMENT_PROVIDER_WAIT.maxAttempts }); + expect(environmentProviderWaitRetry(waiting(7, ENVIRONMENT_PROVIDER_WAIT.ceilingMs), now)).toMatchObject({ maxAttempts: 7 }); + }); + + it("needs the failed environment and names it in the wait", () => { + expect(environmentProviderWaitRetry({ errorCode: "environment_provider_unavailable", resultJson: {} }, now)).toBeNull(); + expect(environmentProviderWaitMessage("Daytona")) + .toBe("Waiting for \"Daytona\" to come back. Paperclip continues this task automatically when it does."); + expect(environmentProviderWaitMessage(null)).toContain("the sandbox provider"); + }); +}); diff --git a/server/src/services/environment-provider-wait.ts b/server/src/services/environment-provider-wait.ts new file mode 100644 index 00000000000..91218fb5194 --- /dev/null +++ b/server/src/services/environment-provider-wait.ts @@ -0,0 +1,73 @@ +export const ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON = "environment_provider_wait"; + +export const ENVIRONMENT_PROVIDER_WAIT = { + delaysMs: [60_000, 2 * 60_000, 5 * 60_000], + ceilingMs: 30 * 60_000, + maxAttempts: 12, + parkedRecheckMs: 30_000, + probeStartGraceMs: 60_000, + probeRunningBoundMs: 10 * 60_000, + wakeReason: "transient_failure_retry", +} as const; + +export const ENVIRONMENT_PROVIDER_WAIT_KEY = "environmentProviderWait"; + +export type EnvironmentProviderWait = { environmentId: string; environmentName: string | null; waitingSince: string }; + +type WaitRun = { + errorCode?: string | null; + scheduledRetryReason?: string | null; + scheduledRetryAttempt?: number | null; + contextSnapshot?: Record | null; + resultJson?: Record | null; +}; + +const record = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export function readEnvironmentProviderWait(contextSnapshot: unknown): EnvironmentProviderWait | null { + const wait = record(record(contextSnapshot)[ENVIRONMENT_PROVIDER_WAIT_KEY]); + const { environmentId, environmentName, waitingSince } = wait; + if (typeof environmentId !== "string" || !UUID.test(environmentId)) return null; + if (typeof waitingSince !== "string" || !Number.isFinite(Date.parse(waitingSince))) return null; + return { environmentId, environmentName: typeof environmentName === "string" && environmentName.trim() ? environmentName : null, + waitingSince }; +} + +function failedEnvironment(run: WaitRun): { environmentId: string; environmentName: string | null } | null { + const provider = record(record(run.resultJson).environmentProvider); + if (typeof provider.environmentId !== "string" || !UUID.test(provider.environmentId)) return null; + return { environmentId: provider.environmentId, + environmentName: typeof provider.environmentName === "string" && provider.environmentName.trim() ? provider.environmentName : null }; +} + +export function environmentProviderWaitFor(run: WaitRun, now: Date): EnvironmentProviderWait | null { + const environment = failedEnvironment(run); + if (!environment) return null; + const saved = run.scheduledRetryReason === ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON + ? readEnvironmentProviderWait(run.contextSnapshot) : null; + const waitingSince = saved && saved.environmentId === environment.environmentId ? saved.waitingSince : now.toISOString(); + return { ...environment, waitingSince }; +} + +export function environmentProviderWaitRetry(run: WaitRun, now: Date) { + const wait = environmentProviderWaitFor(run, now); + if (!wait) return null; + const consumed = run.scheduledRetryReason === ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON ? run.scheduledRetryAttempt ?? 0 : 0; + const remainingMs = Date.parse(wait.waitingSince) + ENVIRONMENT_PROVIDER_WAIT.ceilingMs - now.getTime(); + const delays = ENVIRONMENT_PROVIDER_WAIT.delaysMs; + const delayMs = Math.min(delays[Math.min(consumed, delays.length - 1)]!, Math.max(0, remainingMs)); + return { + retryReason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + wakeReason: ENVIRONMENT_PROVIDER_WAIT.wakeReason, + maxAttempts: remainingMs > 0 ? ENVIRONMENT_PROVIDER_WAIT.maxAttempts : consumed, + delayMs, + }; +} + +export function environmentProviderWaitMessage(environmentName: string | null): string { + const name = environmentName?.trim() ? `"${environmentName.trim()}"` : "the sandbox provider"; + return `Waiting for ${name} to come back. Paperclip continues this task automatically when it does.`; +} diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 6b4951d63c0..ccdb156c912 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -260,7 +260,7 @@ export function environmentRunOrchestrator( environmentId: input.environment.id, driver: input.environment.driver, cause: transient - ? new EnvironmentProviderUnavailableError({ environmentName: input.environment.name, + ? new EnvironmentProviderUnavailableError({ environmentId: input.environment.id, environmentName: input.environment.name, driver: input.environment.driver, evidence: transient, cause: err }) : err, }, diff --git a/server/src/services/execution-recovery-attempt.ts b/server/src/services/execution-recovery-attempt.ts index a46bf2b5c2f..98c9091806c 100644 --- a/server/src/services/execution-recovery-attempt.ts +++ b/server/src/services/execution-recovery-attempt.ts @@ -1,5 +1,6 @@ import { RUNNER_REVISION_WAIT_RETRY_REASON } from "./runner-revision-mismatch.js"; import { STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../storage/storage-unavailable.js"; +import { ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON } from "./environment-provider-wait.js"; import { PROVIDER_ATTEMPT_CREDIT_GENERATION_LIMIT } from "./native-runtime/provider-attempt-outcome.js"; import { AGENT_DIRECTORY_BUSY_RETRY_REASON } from "../modules/run-dispatch/index.js"; @@ -31,6 +32,7 @@ export const SERVER_RESTART_RETRY_WINDOW_MS = 60 * 60_000; const UNCHARGED_RETRY_LANES: ReadonlySet = new Set([ "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait", AGENT_DIRECTORY_WAIT_RETRY_REASON, RUNNER_REVISION_WAIT_RETRY_REASON, INTERRUPTED_PREPARATION_RETRY_REASON, STORAGE_UNAVAILABLE_WAIT_RETRY_REASON, SERVER_RESTART_RETRY_REASON, + ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, ]); type RetryRun = { diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 8e45af94cde..6e5ee8e85dc 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -40,6 +40,7 @@ import { REVIEWED_CHAT_BINDING_NOT_READY } from "../reviewed-chat-binding-diagno import { RUNNER_REVISION_MISMATCH_CODE, RUNNER_REVISION_MISMATCH_RETRY, RUNNER_REVISION_WAIT_RETRY_REASON } from "../runner-revision-mismatch.js"; import { STORAGE_UNAVAILABLE_CODE, STORAGE_UNAVAILABLE_RETRY, STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../../storage/storage-unavailable.js"; import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; +import { ENVIRONMENT_PROVIDER_WAIT_KEY, ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, environmentProviderWaitFor, environmentProviderWaitRetry } from "../environment-provider-wait.js"; import { randomUUID } from "node:crypto"; import { and, @@ -524,6 +525,12 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } if (!opts?.retryReason && run.errorCode === RUNNER_REVISION_MISMATCH_CODE) opts = { ...RUNNER_REVISION_MISMATCH_RETRY, ...opts }; if (!opts?.retryReason && run.errorCode === STORAGE_UNAVAILABLE_CODE) opts = { ...STORAGE_UNAVAILABLE_RETRY, ...opts }; + if (!opts?.retryReason && run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE + && (run.scheduledRetryReason === ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON + || executionFailureRetryCount(run) >= BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS)) { + const providerWait = environmentProviderWaitRetry(run, opts?.now ?? new Date()); + if (providerWait) opts = { ...providerWait, ...opts }; + } if (await hasCaseHandoffForRun(run)) return { outcome: "not_scheduled" as const, reason: "The durable case handoff owns successor work.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; if (["openai_dot", "muse"].includes(String(parseObject(agent.adapterConfig).provider)) @@ -781,6 +788,9 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen ? { preparationWait: { kind: retryReason, attempt: schedule.attempt, nextAttemptAt: schedule.dueAt.toISOString() } } : { wakeReason, retryReason }), ...(interactionRunnerWait ? { mutation: "interaction" } : {}), + ...(retryReason === ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON + ? { [ENVIRONMENT_PROVIDER_WAIT_KEY]: environmentProviderWaitFor(run, now) } + : {}), ...(retryReason === WORKSPACE_BUSY_RETRY_REASON ? { failureRetriesBeforeWorkspaceWait: @@ -831,6 +841,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen delete retryContextSnapshot.skillSelectionSnapshot; delete retryContextSnapshot.shadowedSkills; delete retryContextSnapshot.retryWorkspaceRefusal; + if (retryReason !== ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON) delete retryContextSnapshot[ENVIRONMENT_PROVIDER_WAIT_KEY]; const responsibleUserId = await resolveResponsibleUserIdForRunContext( run, retryContextSnapshot, diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index bb81b2b97b6..c76c9010cec 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1746,6 +1746,9 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC ...(providerFailure ? { providerFailure: { ...providerFailure, message: redactCurrentUserText(identityRedactor.redact(providerFailure.message), currentUserRedactionOptions) } } : {}), ...(connectionFailure ? { connectionFailure } : {}), + ...(environmentProviderUnavailable?.environmentId ? { environmentProvider: { + environmentId: environmentProviderUnavailable.environmentId, + environmentName: environmentProviderUnavailable.environmentName } } : {}), ...(workspaceProviderFailure ? { workspaceProviderFailure } : {}), // Names the configured setup command that exited non-zero, for the task page. ...setupCommandFailureResultJson(outerErr, readFailureReportSecrets()), diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 48c7ecded67..59da1fac7f8 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -75,7 +75,7 @@ const WORKSPACE_SCAN_NOTICES: Record & Partial> & { + environmentProviderWaitName?: string | null; /** Due time of the native coordinator's own same-run resume, if one is pending. */ nativeResumeAt?: Date | null; /** When the native coordinator saved this run's terminal failure, if it did. */ @@ -346,6 +348,9 @@ function projectTaskInterruptionState(issue: ProjectionIssue, input: ProjectionS return { ...base, state: "retry_scheduled", nextRetryAt: new Date(deadline).toISOString() }; if (!pending) return { ...base, state: "failed", canRetry: retryWait.preparationStatus !== "cancelled" }; } + if (run.status === "scheduled_retry" && run.scheduledRetryReason === ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON && !system) + return { ...base, state: "working", canRetry: false, retryWait: { reason: ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, + message: environmentProviderWaitMessage(run.environmentProviderWaitName ?? null) } }; if (run.status === "scheduled_retry" && resourceWaitRetryReasons.has(run.scheduledRetryReason ?? "") && !system) return { ...base, state: "working" }; if (run.status === "scheduled_retry" && run.scheduledRetryReason === INTERRUPTED_PREPARATION_RETRY_REASON && !system @@ -482,6 +487,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i rejectedModel: sql`case when ${heartbeatRuns.errorCode} = 'native_provider_model_rejected' then ${boundedJsonString(sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'provider'->'model'`, 200)} end`, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + environmentProviderWaitName: sql`case when ${heartbeatRuns.scheduledRetryReason} = ${ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON} + then ${boundedJsonString(sql`${heartbeatRuns.contextSnapshot}->${ENVIRONMENT_PROVIDER_WAIT_KEY}->'environmentName'`, 200)} end`, // Candidate only: the coordinator also pushes nextAttemptAt forward while // it defers an ineligible run. The scheduler's own admission check below // decides whether this resume is real. diff --git a/ui/src/components/IssueMonitorBanner.test.tsx b/ui/src/components/IssueMonitorBanner.test.tsx index 7e11fcaf134..acabfd2f541 100644 --- a/ui/src/components/IssueMonitorBanner.test.tsx +++ b/ui/src/components/IssueMonitorBanner.test.tsx @@ -40,6 +40,14 @@ describe("buildMonitorSurfaceCopy", () => { expect(copy!.workspaceWait).toBe(true); expect(copy!.bannerMeta.join(" ")).not.toMatch(/Attempt|overdue|retry/i); }); + it.each(["retrying", "due-now", "overdue"] as const)("shows a steady provider wait with no countdown when %s", (state) => { + const copy = buildMonitorSurfaceCopy(derived({ + state, source: "scheduled-retry", nextCheckAt: NOW.toISOString(), attemptCount: 3, + }), NOW, "environment_provider_wait", 0, "in_progress", true); + expect(copy!.bannerTitle).toBe("Waiting for the sandbox provider"); + expect(copy!.workspaceWait).toBe(true); + expect(copy!.bannerMeta.join(" ")).not.toMatch(/Attempt|overdue|retry|resumes/i); + }); it("leads with two-unit relative time while scheduled", () => { const copy = buildMonitorSurfaceCopy( derived({ diff --git a/ui/src/components/IssueMonitorBanner.tsx b/ui/src/components/IssueMonitorBanner.tsx index 22ca0474136..50575171494 100644 --- a/ui/src/components/IssueMonitorBanner.tsx +++ b/ui/src/components/IssueMonitorBanner.tsx @@ -105,6 +105,17 @@ export function buildMonitorSurfaceCopy( }; } + if (derived.source === "scheduled-retry" && scheduledRetryReason === "environment_provider_wait") { + return { + bannerTitle: "Waiting for the sandbox provider", + stripTitle: "Waiting for the sandbox provider", + bannerMeta: ["The sandbox provider is unavailable. Work continues automatically when it is back."], + stripMeta: ["Work continues automatically when the sandbox provider is back."], + tone: "info", + workspaceWait: true, + }; + } + if (derived.source === "scheduled-retry" && scheduledRetryReason === "workspace_busy") { return { bannerTitle: "Waiting for workspace", diff --git a/ui/src/components/IssueScheduledRetryCard.test.tsx b/ui/src/components/IssueScheduledRetryCard.test.tsx index 37270eb1d1d..1d2dfd4d4a0 100644 --- a/ui/src/components/IssueScheduledRetryCard.test.tsx +++ b/ui/src/components/IssueScheduledRetryCard.test.tsx @@ -126,6 +126,11 @@ describe("IssueScheduledRetryCard", () => { expect(container.textContent).toBe(""); expect(getRetryNowButton()).toBeNull(); }); + it("offers no Retry while the task waits for the sandbox provider to come back", () => { + renderWithProviders(); + expect(container.textContent).toBe(""); + expect(getRetryNowButton()).toBeNull(); + }); it("surfaces model capacity and the automatic retry without a generic failure", () => { renderWithProviders(); diff --git a/ui/src/components/IssueScheduledRetryCard.tsx b/ui/src/components/IssueScheduledRetryCard.tsx index 72f9712ca55..68b2c59e1df 100644 --- a/ui/src/components/IssueScheduledRetryCard.tsx +++ b/ui/src/components/IssueScheduledRetryCard.tsx @@ -35,7 +35,8 @@ export function IssueScheduledRetryCard({ // Workspace contention is a transient wait with nothing for the user to do. // The composer strip and properties row carry the quiet status instead. - if (scheduledRetry.scheduledRetryReason === "workspace_busy") return null; + if (scheduledRetry.scheduledRetryReason === "workspace_busy" + || scheduledRetry.scheduledRetryReason === "environment_provider_wait") return null; const continuation = isContinuationReason(scheduledRetry.scheduledRetryReason); const dueAtIso = scheduledRetry.scheduledRetryAt diff --git a/ui/src/components/issue-properties/IssueProperties.tsx b/ui/src/components/issue-properties/IssueProperties.tsx index 705fedb32e1..fbdaccece1b 100644 --- a/ui/src/components/issue-properties/IssueProperties.tsx +++ b/ui/src/components/issue-properties/IssueProperties.tsx @@ -2717,6 +2717,10 @@ export function IssueProperties({ Waiting for workspace + ) : showScheduledRetryRow && scheduledRetry?.scheduledRetryReason === "environment_provider_wait" ? ( + + Waiting for the provider + ) : showScheduledRetryRow && scheduledRetryContent ? (