diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index d2a40388ef5..38843d08951 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -68,7 +68,12 @@ async function failedNativePreparation(db: Db, run: Run, coordinator: Coordinato return typeof issueId === "string" && leases.length > 0 && leases.every(lease => lease.issueId === issueId && lease.metadata?.pendingCleanupInFlight !== true && hasRemoteTerminationReceipt(lease)) && await hasSettledFailedPreparationEvidence(db, run, historicalRetryAncestor) - && await hasVerifiedWorkspaceRepair(db, run); + && (!holdsPreparedFiles(run) || await hasVerifiedWorkspaceRepair(db, run)); +} + +function holdsPreparedFiles(run: Run) { + return run.runnerProfileJson?.remoteWorkspace != null || run.resultJson?.workspaceRestoreFailure != null + || hasRequiredWorkspaceRecovery(run.resultJson); } async function hasSettledFailedPreparationEvidence(db: Db, run: Run, historicalRetryAncestor = false): Promise { diff --git a/server/src/services/failed-preparation-settlement.ts b/server/src/services/failed-preparation-settlement.ts index 3fb3844b73a..bcb4a55118f 100644 --- a/server/src/services/failed-preparation-settlement.ts +++ b/server/src/services/failed-preparation-settlement.ts @@ -1,5 +1,5 @@ import { and, asc, eq, sql } from "drizzle-orm"; -import { heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; import { isUuidLike } from "@paperclipai/shared"; type Source = Pick; @@ -10,7 +10,7 @@ export async function readFailedPreparationSettlement(db: Db, source: Source, al if (typeof source.error !== "string" || !source.error.length) return null; const events = await db.select().from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, source.companyId), eq(heartbeatRunEvents.runId, source.id), - )).orderBy(asc(heartbeatRunEvents.seq)).limit(allowRunStarted ? 6 : 5); + )).orderBy(asc(heartbeatRunEvents.seq)).limit(allowRunStarted ? 7 : 6); const hostEvent = (event: typeof heartbeatRunEvents.$inferSelect) => event.agentId === source.agentId && event.stream === "system" && event.sourceInstanceId === null && event.sourceEventId === null && event.sourceSeq === null && event.sourcePayloadSha256 === null && event.protocolSchemaVersion === null; @@ -35,8 +35,9 @@ export async function readFailedPreparationSettlement(db: Db, source: Source, al if (!hostEvent(started) || started.seq !== 1 + offset || started.level !== "info" || started.payload !== null) return null; offset++; } - const [error, measured] = events.slice(offset); - if (events.length !== 2 + offset || !error || !measured || !hostEvent(error) || !hostEvent(measured) + const [error, measured, settledHold] = events.slice(offset); + if (settledHold && !(hostEvent(settledHold) && await isRecoveryHoldSettlement(db, source, settledHold, 3 + offset))) return null; + if (events.length !== (settledHold ? 3 : 2) + offset || !error || !measured || !hostEvent(error) || !hostEvent(measured) || error.seq !== 1 + offset || error.eventType !== "error" || error.level !== "error" || error.message !== source.error || error.payload !== null || measured.seq !== 2 + offset || measured.eventType !== "lifecycle" || measured.level !== "info" || measured.message !== "task.run.measured" @@ -73,3 +74,18 @@ async function matchesScheduledRetryPromotion(db: Db, source: Source, event: typ && event.payload.scheduledRetryReason === run.scheduledRetryReason && Object.keys(event.payload).every(key => ["scheduledRetryAt", "scheduledRetryAttempt", "scheduledRetryReason"].includes(key))); } + +async function isRecoveryHoldSettlement(db: Db, source: Source, event: typeof heartbeatRunEvents.$inferSelect, seq: number) { + const payload = event.payload; + if (event.seq !== seq || event.eventType !== "lifecycle" || event.level !== "warn" || !payload + || Object.keys(payload).some(key => !["recoveryActionId", "cause", "automaticRecovery", "replay"].includes(key)) + || payload.cause !== "legacy_execution_requires_reconciliation" || payload.automaticRecovery !== "preserve_without_replay_v1" + || payload.replay !== "blocked" || typeof payload.recoveryActionId !== "string" || !isUuidLike(payload.recoveryActionId)) return false; + const [hold] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, source.companyId), eq(issueRecoveryActions.id, payload.recoveryActionId), + eq(issueRecoveryActions.cause, "legacy_execution_requires_reconciliation"), eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'runId' = ${source.id}`, + sql`${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked'`, + )).limit(1); + return Boolean(hold); +} diff --git a/server/src/services/legacy-preparation-retry-hold.test.ts b/server/src/services/legacy-preparation-retry-hold.test.ts new file mode 100644 index 00000000000..4b7092950b6 --- /dev/null +++ b/server/src/services/legacy-preparation-retry-hold.test.ts @@ -0,0 +1,126 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, agentWakeupRequests, companies, createDb, environmentLeases, executionWorkspaces, + heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { requestTaskRetry } from "./task-retry.js"; +import { heartbeatService } from "./heartbeat.js"; +import { getExecutionBlocker } from "./execution-blocker.js"; + +describe("Retry of a legacy preparation that automatic recovery stopped", () => { + let postgres: Awaited>; + let db: ReturnType; + beforeAll(async () => { + postgres = await startEmbeddedPostgresTestDatabase("legacy-preparation-retry-"); + db = createDb(postgres.connectionString); + }, 30_000); + afterAll(async () => { await db?.$client.end(); await postgres?.cleanup(); }); + + async function seed() { + const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(); + const parentId = randomUUID(), sourceId = randomUUID(), actionId = randomUUID(); + const workspaceId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); + const actorId = "board", now = Date.now(), error = "Worker process exited (code=null, signal=SIGKILL)"; + const controllerProcess = { pid: 3, bootId: randomUUID(), hostHash: "f".repeat(32), processStartedAt: new Date(now - 3_600_000).toISOString() }; + await db.insert(companies).values({ id: companyId, name: "Legacy preparation", issuePrefix: `L${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "idle", + runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Repair a failed run", status: "blocked", assigneeAgentId: agentId }); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, + mode: "isolated_workspace", strategyType: "git_worktree", name: "Task workspace", providerType: "cloud_sandbox" }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issueId)); + await db.insert(heartbeatRuns).values({ id: parentId, companyId, agentId, issueId, status: "failed", + runtimeMode: "legacy", errorCode: "environment_provider_unavailable", executionStage: "preparing", + createdAt: new Date(now - 600_000), startedAt: new Date(now - 590_000), finishedAt: new Date(now - 400_000), + contextSnapshot: { issueId }, runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: { stopReason: "adapter_failed", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + const scheduledRetryAt = new Date(now - 300_000); + const [source] = await db.insert(heartbeatRuns).values({ id: sourceId, companyId, agentId, issueId, status: "failed", + runtimeMode: "legacy", errorCode: "setup_failed", error, executionStage: "preparing", invocationSource: "automation", + createdAt: new Date(now - 360_000), startedAt: new Date(now - 250_000), finishedAt: new Date(now - 200_000), + retryOfRunId: parentId, scheduledRetryAt, scheduledRetryAttempt: 2, scheduledRetryReason: "transient_failure", + controllerBootId: controllerProcess.bootId, controllerProcess, + usageJson: { ledgerScope: { issueId, projectId: null, billingCode: null } }, + contextSnapshot: { issueId, taskId: issueId, retryOfRunId: parentId, retryReason: "transient_failure", + scheduledRetryAt: scheduledRetryAt.toISOString(), scheduledRetryAttempt: 2, wakeReason: "transient_failure_retry" }, + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" }, + chatControlRecoveryAdmission: { phase: "required", runId: sourceId, agentId, issueId, version: 1, companyId } }, + resultJson: { stopReason: "adapter_failed", failureCause: "JsonRpcCallError", timeoutFired: false, timeoutSource: "config", + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, timeoutConfigured: false, effectiveTimeoutSec: 0 }, + }).returning(); + const event = { companyId, agentId, runId: sourceId, stream: "system" }; + await db.insert(heartbeatRunEvents).values([ + { ...event, seq: 1, eventType: "lifecycle", level: "info", createdAt: new Date(scheduledRetryAt.getTime() + 200), + message: "Scheduled retry became due and was promoted to the queued run pool", + payload: { scheduledRetryAt: scheduledRetryAt.toISOString(), scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 2 } }, + { ...event, seq: 2, eventType: "error", level: "error", message: error }, + { ...event, seq: 3, eventType: "lifecycle", level: "info", message: "task.run.measured", + payload: { phase: "task.run.measured", outcome: "failed", durationMs: 141_150, durableSettlement: true } }, + { ...event, seq: 4, eventType: "lifecycle", level: "warn", + message: "Automatic recovery stopped. Recorded work is preserved; actions with unverified outcomes will not be repeated.", + payload: { cause: "legacy_execution_requires_reconciliation", replay: "blocked", recoveryActionId: actionId, + automaticRecovery: "preserve_without_replay_v1" } }, + ]); + const releasedAt = new Date(now - 150_000); + await db.insert(environmentLeases).values({ id: leaseId, companyId, issueId, heartbeatRunId: sourceId, + executionWorkspaceId: workspaceId, provider: "daytona", providerLeaseId, status: "expired", + leasePolicy: "reuse_by_environment", releasedAt, cleanupStatus: "success", failureReason: "pending_cleanup_retry", + metadata: { pendingCleanupInFlight: false, remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", + companyId, runId: sourceId, leaseId, provider: "daytona", providerLeaseId, state: "destroyed", + confirmedAt: releasedAt.toISOString() } } }); + await db.insert(issueRecoveryActions).values({ id: actionId, companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", fingerprint: sourceId, + status: "resolved", outcome: "blocked", + nextAction: "Automatic recovery stopped. Recorded work is preserved; actions with unverified outcomes will not be repeated.", + evidence: { runId: sourceId, automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } } }); + await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }); + return { companyId, issueId, agentId, actorId, source, actionId, leaseId }; + } + + it("starts the person's Retry once the stopped preparation's sandbox is destroyed", async () => { + const f = await seed(); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + expect(blocker).toMatchObject({ cause: "legacy_execution_requires_reconciliation", canRetry: true }); + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID(), actorSource: "local_implicit" }); + expect(receipt).toMatchObject({ accepted: true }); + const heartbeat = heartbeatService(db); + await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake.payload?.executionWait).toBeUndefined(); + expect(wake).toMatchObject({ status: "coalesced", runId: expect.any(String) }); + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, wake.runId!)); + expect(successor.status).toBe("queued"); + expect(successor.contextSnapshot).toMatchObject({ previousRunId: f.source.id, taskRetryRequestId: receipt.requestId }); + }); + + it.each(["foreign_hold", "provider_event", "unstopped_lease", "unsaved_remote_workspace"] as const)( + "keeps the Retry held when the stopped preparation is not proven: %s", async gate => { + const f = await seed(); + if (gate === "foreign_hold") { + const [other] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", fingerprint: randomUUID(), + status: "resolved", outcome: "blocked", nextAction: "Other run.", + evidence: { runId: randomUUID(), automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } } }).returning(); + await db.update(heartbeatRunEvents).set({ payload: { cause: "legacy_execution_requires_reconciliation", replay: "blocked", + recoveryActionId: other.id, automaticRecovery: "preserve_without_replay_v1" } }) + .where(and(eq(heartbeatRunEvents.runId, f.source.id), eq(heartbeatRunEvents.seq, 4))); + } + if (gate === "provider_event") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, agentId: f.agentId, + runId: f.source.id, seq: 5, eventType: "tool.execution.started", stream: "system" }); + if (gate === "unstopped_lease") await db.update(environmentLeases).set({ metadata: { pendingCleanupInFlight: false } }) + .where(eq(environmentLeases.id, f.leaseId)); + if (gate === "unsaved_remote_workspace") await db.update(heartbeatRuns).set({ runnerProfileJson: { + ...f.source.runnerProfileJson, remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId: randomUUID(), + leaseId: f.leaseId, providerLeaseId: randomUUID(), remoteCwd: "/fixture", epoch: 1, state: "prepared", snapshotId: null } } }) + .where(eq(heartbeatRuns.id, f.source.id)); + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID(), actorSource: "local_implicit" }); + await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId!); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.status, "queued")))).toHaveLength(0); + }); +});