diff --git a/server/src/services/execution-control-work.ts b/server/src/services/execution-control-work.ts index eb3e1983964..125824b1193 100644 --- a/server/src/services/execution-control-work.ts +++ b/server/src/services/execution-control-work.ts @@ -11,6 +11,7 @@ import { nextAbandonedNativeOwnerReleaseAt } from "./native-runtime/native-faile import { hasPendingPauseResume } from "./issue-tree-resume.js"; import { hasPendingBudgetResume } from "./budget-resume-continuation.js"; import { continuationDeliveryDue } from "./recovery/continuation-delivery-attempt.js"; +import { nextNativeInterruptionCheckAt } from "./native-interruption-continuation.js"; export function registerExecutionControlWork( coordinator: ReturnType, @@ -113,13 +114,15 @@ export function executionControlSchedule(db: Db) { }, async disposition() { const now = new Date(); - const [conversation, execution, held, nextCheck] = await Promise.all([ + const [conversation, execution, held, nextCheck, nextInterruptionCheck] = await Promise.all([ db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(obsoleteExecutionConversationHold()).limit(1), unresolvedExecutionDispositions(db, 1), heldExecutionsAwaitingFreshContinuation(db, now, 1), nextFreshContinuationCheckAt(db), + nextNativeInterruptionCheckAt(db), ]); - return conversation.length || execution.length || held.length ? retry() : deadline([nextCheck ? new Date(nextCheck) : null]); + return conversation.length || execution.length || held.length ? retry() + : deadline([nextCheck ? new Date(nextCheck) : null, nextInterruptionCheck ? new Date(nextInterruptionCheck) : null]); }, async localLogin() { const rows = await db.select({ at: adapterAuthSessions.expiresAt }).from(adapterAuthSessions).where(and( diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index d3dc9e68a94..7b0cb23dd52 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -43,6 +43,7 @@ import { reserveAutomaticRunCreation } from "./recovery/automatic-run-brake.js"; import { continuationDeliveryDue, deferForUnwakeableAssignee, recordContinuationDeliveryAttempt } from "./recovery/continuation-delivery-attempt.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; +import { continueNativeInterruption, nativeInterruptionHold } from "./native-interruption-continuation.js"; import { alias } from "drizzle-orm/pg-core"; import { RECOVERY_BACKLOG_AGE_MS, RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; import { agentRecoveryQueueIsFull } from "./recovery-queue-admission.js"; @@ -680,7 +681,7 @@ export function unresolvedExecutionDispositions(db: Db, limit: number) { sql`${issueRecoveryActions.wakePolicy}->>'runId' = ${heartbeatRuns.id}::text`, exists(supersedingNativeWorkspaceHold(db, { companyId: heartbeatRuns.companyId, issueId: issueRecoveryActions.sourceIssueId, runId: heartbeatRuns.id })), - )), + ), nativeInterruptionHold()), inArray(heartbeatRuns.status, [ "failed", "timed_out", @@ -818,14 +819,14 @@ export async function settleUnrecoverableExecutions( ) return; if (!EXECUTION_RECONCILIATION_CAUSES.includes(action.cause as (typeof EXECUTION_RECONCILIATION_CAUSES)[number])) { - if (run.runtimeMode !== "native" || run.nativePhase !== "terminal_failure" - || coordinator?.phase !== "terminal_failure" || coordinator.leaseOwner || coordinator.resultId - || coordinator.nextAttemptAt || coordinator.failureDetail?.successorRunId - || action.wakePolicy?.kind !== "resume_native_run" || action.wakePolicy.runId !== run.id) return; - const [hold] = await supersedingNativeWorkspaceHold(tx as unknown as Db, { + const resumeHold = run.runtimeMode === "native" && run.nativePhase === "terminal_failure" + && coordinator?.phase === "terminal_failure" && !coordinator.leaseOwner && !coordinator.resultId + && !coordinator.nextAttemptAt && !coordinator.failureDetail?.successorRunId + && action.wakePolicy?.kind === "resume_native_run" && action.wakePolicy.runId === run.id; + const [hold] = resumeHold ? await supersedingNativeWorkspaceHold(tx as unknown as Db, { companyId: run.companyId, issueId: task.id, runId: run.id, - }).limit(1).for("update"); - if (!hold) return; + }).limit(1).for("update") : []; + if (!hold) return void await continueNativeInterruption(tx as unknown as Db, { action, run, coordinator, now }); await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 4dce0254bb3..12b197ed3d4 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -301,7 +301,8 @@ const blocksExecution = (hold: typeof issueRecoveryActions.$inferSelect) => /** A reconciliation hold that blocks the task and that this decision settles. */ function isOwnedReconciliationHold(hold: typeof issueRecoveryActions.$inferSelect) { if (hold.kind === "stranded_assigned_issue") return false; - if (!(EXECUTION_RECONCILIATION_CAUSES as readonly string[]).includes(hold.cause) || !blocksExecution(hold)) return false; + const interruption = hold.kind === "active_run_watchdog" && (RESTART_CLASS_FAILURE_CODES as readonly string[]).includes(hold.cause); + if ((!(EXECUTION_RECONCILIATION_CAUSES as readonly string[]).includes(hold.cause) && !interruption) || !blocksExecution(hold)) return false; if (isHumanGate(hold)) return false; // A restore hold is owned only once its saved-work recovery completed. return !isRestoreHold(hold) || readSystemRecovery(hold.evidence)?.state === "complete"; diff --git a/server/src/services/native-interruption-continuation.ts b/server/src/services/native-interruption-continuation.ts new file mode 100644 index 00000000000..fedea0357a9 --- /dev/null +++ b/server/src/services/native-interruption-continuation.ts @@ -0,0 +1,44 @@ +import { and, eq, inArray, isNull, not, sql } from "drizzle-orm"; +import { heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { RESTART_CLASS_FAILURE_CODES, isRestartClassFailureCode } from "./continuation-cause.js"; +import { canonicalExecutionUuid } from "./execution-identity.js"; +import { continueFreshInTransaction, recordFreshContinuationCheck } from "./fresh-continuation-fallback.js"; +import { nativeReplacementPending } from "./native-runtime/native-safe-replacement.js"; + +export function nativeInterruptionHold() { + return and( + inArray(issueRecoveryActions.cause, [...RESTART_CLASS_FAILURE_CODES]), + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.nativePhase, "terminal_failure"), + eq(nativeRunFinalizations.phase, "terminal_failure"), isNull(nativeRunFinalizations.nextAttemptAt), + not(nativeReplacementPending()!), + )!; +} + +export async function continueNativeInterruption(tx: Db, input: { + action: typeof issueRecoveryActions.$inferSelect; + run: typeof heartbeatRuns.$inferSelect; + coordinator: typeof nativeRunFinalizations.$inferSelect | undefined; + now: Date; +}): Promise { + const { action, run, coordinator, now } = input; + if (!isRestartClassFailureCode(action.cause) || run.runtimeMode !== "native" || run.nativePhase !== "terminal_failure" + || coordinator?.phase !== "terminal_failure" || coordinator.nextAttemptAt || coordinator.leaseOwner || coordinator.resultId) + return false; + const fresh = await continueFreshInTransaction(tx, { companyId: action.companyId, runId: run.id, trigger: "recovery_refused", now }); + await recordFreshContinuationCheck(tx, { companyId: action.companyId, actionId: action.id }, fresh, now); + return true; +} + +export async function nextNativeInterruptionCheckAt(db: Db): Promise { + const [row] = await db.select({ at: sql`min((${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}')::bigint)` }) + .from(issueRecoveryActions) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, issueRecoveryActions.companyId), + eq(heartbeatRuns.id, canonicalExecutionUuid(sql`${issueRecoveryActions.evidence}->>'runId'`)))) + .innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), + eq(nativeRunFinalizations.runId, heartbeatRuns.id))) + .where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}' is not null`, + inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), + isNull(nativeRunFinalizations.leaseOwner), isNull(nativeRunFinalizations.resultId), nativeInterruptionHold())); + return row?.at ? Number(row.at) : null; +} diff --git a/server/src/services/native-runtime/native-client-timeout-continuation.integration.test.ts b/server/src/services/native-runtime/native-client-timeout-continuation.integration.test.ts new file mode 100644 index 00000000000..f965260d097 --- /dev/null +++ b/server/src/services/native-runtime/native-client-timeout-continuation.integration.test.ts @@ -0,0 +1,154 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, agents, companies, completionContracts, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, +} from "@paperclipai/db"; +import type { ControlPlanePort, NativeExecutionInputV1, PrpEvent } from "@paperclipai/paperclip-runner"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; + +const provider = vi.hoisted(() => ({ execute: vi.fn() })); +vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ + ...(await importOriginal()), + executeNativeSession: provider.execute, +})); + +import { executePaperclipNativeSession } from "./native-session-executor.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; +import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "../execution-recovery-resolution.js"; +import { executionControlSchedule } from "../execution-control-work.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION } from "../recovery/continuation-ledger.js"; +import { FRESH_CONTINUATION_WINDOW_MS } from "../fresh-continuation-fallback.js"; + +const DAYTONA_CLIENT_TIMEOUT = "HTTP request timed out after 300000ms waiting for a response. This is a client-side deadline " + + "(DaytonaConfig.requestTimeoutMs, or the per-call operation/execution timeout); any operation already started on the " + + "server may still be running."; + +describe("a native turn cut off by a sandbox client timeout", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = randomUUID(); + const agentId = randomUUID(); + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-client-timeout-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Client timeout", issuePrefix: "NCT" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Builder", status: "active", adapterType: "paperclip_runner", + adapterConfig: { provider: "opencode", model: "openrouter/test-model" } }); + }, 240_000); + afterAll(async () => { + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + }); + + async function failTurnWithClientTimeout(issueId: string) { + const runId = randomUUID(); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", + contextSnapshot: { issueId, taskId: issueId, wakeReason: "issue_assigned" } }).returning(); + await db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, issueId)); + const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, + provider: { kind: "opencode", model: "openrouter/test-model" }, + task: { identifier: issueId, title: issue.title, description: null, prompt: "Build it.", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: "opencode_server", protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: buildNativeCompletionContract(issue, { revision: contract.revision }) }, + interactionResponses: [], credentialBindings: [], + }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); + const turnStarted: PrpEvent = { + schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId, + turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "turn-started", sourceSeq: 1, + sourceKind: "runner", eventType: "turn.started", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), + payload: {}, + }; + provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => { + await options.controlPlane.openRun({ identity: { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId }, + backendKind: "mock", sourceInstanceId: native.runnerInstanceId }); + await options.controlPlane.appendEvent(turnStarted); + throw new Error(DAYTONA_CLIENT_TIMEOUT); + }); + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { + descriptor: async () => { throw new Error("Unexpected provider call"); }, + openSession: async () => { throw new Error("Unexpected provider call"); }, + } }).catch(() => undefined); + return runId; + } + + const continuationsOf = (runId: string) => db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${runId}`)); + + it("continues the task automatically exactly once in a fresh run, without replaying the cut-off turn", async () => { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Build the feature", status: "in_progress", + workMode: "standard", assigneeAgentId: agentId }); + const runId = await failTurnWithClientTimeout(issueId); + + const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(failed).toMatchObject({ status: "failed", errorCode: "native_session_interrupted", error: DAYTONA_CLIENT_TIMEOUT }); + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_session_interrupted", nextAttemptAt: null, + failureDetail: expect.objectContaining({ recoveryMode: "ambiguous_state", providerEventsExist: true }) }); + + await settleUnrecoverableExecutions(db); + await settleUnrecoverableExecutions(db); + + const continuations = await continuationsOf(runId); + expect(continuations).toHaveLength(1); + expect(continuations[0]!.evidence).toMatchObject({ continuationDelivery: "pending", + freshContinuation: { sourceRunId: runId, failureCode: "native_session_interrupted" }, + freshContinuationSettlement: { replay: "none", decision: "continue_in_fresh_run" } }); + const [task] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(task).toMatchObject({ status: "todo", assigneeAgentId: agentId }); + const open = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId), + sql`${issueRecoveryActions.status} in ('active', 'escalated')`)); + expect(open).toEqual([]); + const [settled] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(settled.failureDetail).toMatchObject({ replacementDenied: "fresh_continuation" }); + expect(provider.execute).toHaveBeenCalledOnce(); + + const wake = vi.fn(async () => ({ id: randomUUID() }) as never); + await deliverReconciledExecutions(db, wake); + await deliverReconciledExecutions(db, wake); + expect(wake).toHaveBeenCalledOnce(); + expect(wake).toHaveBeenCalledWith(agentId, expect.objectContaining({ + payload: expect.objectContaining({ issueId }) })); + }, 120_000); + + it("registers the capped continuation's next check with the disposition worker and continues the task when it is due", async () => { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Build the capped feature", status: "in_progress", + workMode: "standard", assigneeAgentId: agentId }); + const runId = await failTurnWithClientTimeout(issueId); + const firstAt = new Date(Date.now() - 10 * 60_000); + const capped = await db.insert(activityLog).values(Array.from({ length: 6 }, (_, index) => ({ companyId, actorId: "test", + action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: issueId, details: { outcome: "queued" }, + createdAt: new Date(firstAt.getTime() + index * 1_000) }))).returning({ id: activityLog.id }); + + await settleUnrecoverableExecutions(db); + expect(await continuationsOf(runId)).toHaveLength(0); + const [hold] = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId), + eq(issueRecoveryActions.status, "active"))); + const nextCheckAt = firstAt.getTime() + FRESH_CONTINUATION_WINDOW_MS; + expect(hold?.evidence).toMatchObject({ freshContinuationCheck: { kind: "backoff", nextCheckAt } }); + expect(await executionControlSchedule(db).disposition()).toBe(nextCheckAt); + + await db.delete(activityLog).where(sql`${activityLog.id} in ${capped.map(row => row.id)}`); + await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuationCheck,nextCheckAt}', to_jsonb(${Date.now() - 1}::bigint))` }).where(eq(issueRecoveryActions.id, hold!.id)); + await settleUnrecoverableExecutions(db); + expect(await continuationsOf(runId)).toHaveLength(1); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "todo" }); + }, 120_000); +});