diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 41f8670bc5c..1e4190c8064 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1845,6 +1845,13 @@ mid-turn can leave provider or tool processes that the fence cannot find, so it keeps the fail-closed path below, and the task continues in a fresh session from the saved work. +On every `reattach_remote_runner` attempt, a turn the Runner reports as +`provider_turn_lost_on_restore` with `recoverable: true` continues once in the same +conversation, as on a local dead-runner resume. The Runner reports that cause only +after it proved the old process ended and restored the conversation. If the turn +still ends the run, the run keeps `native_provider_terminal_failed`, and the safe +replacement lane continues the task. + The first unanswered read can come on the last of the three provider attempts, for example when a server restart used the second attempt. The run then gets one more attempt for the replacement only: the failure stays retryable, the claim diff --git a/server/src/services/native-runtime/native-recoverable-lost-turn.integration.test.ts b/server/src/services/native-runtime/native-recoverable-lost-turn.integration.test.ts new file mode 100644 index 00000000000..7d20633bd38 --- /dev/null +++ b/server/src/services/native-runtime/native-recoverable-lost-turn.integration.test.ts @@ -0,0 +1,75 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, completionContracts, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; +import type { ControlPlanePort, NativeExecutionInputV1, PersistedNativeSession } from "@paperclipai/paperclip-runner"; +import { EXECUTION_RECONCILIATION_CAUSES } from "@paperclipai/shared"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { NativeProviderTerminalFailure } from "../../vendor/paperclip-runner/index.js"; + +const provider = vi.hoisted(() => ({ execute: vi.fn() })); +vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ + ...(await importOriginal()), executeNativeSession: provider.execute, +})); +import { executePaperclipNativeSession } from "./native-session-executor.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; + +const LOST_TURN_MESSAGE = "The runner restored the conversation after process loss, but the previous turn is no longer active."; + +describe("a lost turn that still ends the run after a restore", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = randomUUID(), agentId = randomUUID(); + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-recoverable-lost-turn-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Lost turn", issuePrefix: "LST" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Lost turn worker", status: "active", adapterType: "paperclip_runner", + adapterConfig: { provider: "codex", model: "test-model" } }); + }); + afterAll(async () => { await temporary?.cleanup(); }); + + it.each([true, false])("keeps a lost turn (recoverable %s) on the hold the replacement lane continues", async (recoverable) => { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Lost turn task", status: "in_progress", assigneeAgentId: agentId }); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: issueId, contextSnapshot: { issueId } }).returning(); + const [issue] = await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)).returning(); + const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + const contractInput = buildNativeCompletionContract(issue, { revision: contract.revision }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, + provider: { kind: "codex", model: "test-model" }, + task: { identifier: issueId, title: issue.title, description: null, prompt: "Continue the task.", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: contractInput }, + interactionResponses: [], credentialBindings: [], + }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); + const identity = { companyId, agentId, issueId, runId, sessionId: native.normalizedSessionId }; + const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: native.normalizedSessionId, identity, + providerSessionId: "provider-thread", activeTurnId: null, pendingRuntimeRequests: [], goal: null, semanticResult: null, + terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "failed", runTerminalState: "failed", reportedWorkDisposition: "yielded" } }; + provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => { + await options.controlPlane.openRun({ identity, backendKind: "mock", sourceInstanceId: native.runnerInstanceId }); + await options.controlPlane.checkpointSession?.(snapshot); + throw new NativeProviderTerminalFailure("provider_turn_lost_on_restore", recoverable, LOST_TURN_MESSAGE); + }); + const unexpected = async () => { throw new Error("Unexpected provider call"); }; + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { descriptor: unexpected, openSession: unexpected } }) + .catch(() => undefined); + expect(provider.execute).toHaveBeenCalledTimes(1); + const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(saved.runnerProfileJson).toMatchObject({ sessionCheckpoint: { terminal: { runTerminalState: "failed" } } }); + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_provider_terminal_failed", nextAttemptAt: null }); + const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(holds).toEqual([expect.objectContaining({ kind: "active_run_watchdog", cause: "native_provider_terminal_failed", + evidence: expect.objectContaining({ runId }) })]); + expect(EXECUTION_RECONCILIATION_CAUSES).toContain(holds[0]!.cause); + }); +}); diff --git a/server/src/services/native-runtime/native-scheduled-resume-continuation.integration.test.ts b/server/src/services/native-runtime/native-scheduled-resume-continuation.integration.test.ts index dd94207b60d..bf447bebdba 100644 --- a/server/src/services/native-runtime/native-scheduled-resume-continuation.integration.test.ts +++ b/server/src/services/native-runtime/native-scheduled-resume-continuation.integration.test.ts @@ -18,6 +18,7 @@ import { executePaperclipNativeSession } from "./native-session-executor.js"; import { buildNativeCompletionContract } from "./completion-contracts.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js"; +import { currentNativeControllerIdentity, type NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; const EXITED_PID = 4_194_304; @@ -40,7 +41,8 @@ describe("a scheduled same-run resume reaching the provider runtime", () => { checkpointExists: true, originalFailureCode: "native_session_recovery_failed" }; async function resumeOnce(options: { failureDetail: Record; stopRecorded: boolean; viaReconciler?: boolean; - failedRelaunch?: boolean; unidentifiedStopRecorded?: boolean; liveRunnerForRun?: boolean }) { + failedRelaunch?: boolean; unidentifiedStopRecorded?: boolean; liveRunnerForRun?: boolean; + remoteReattach?: { replaceUnansweredRunner?: true } }) { const issueId = randomUUID(); const runId = randomUUID(); const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Resume the lost turn", @@ -99,12 +101,23 @@ describe("a scheduled same-run resume reaching the provider runtime", () => { ? spawn(process.execPath, ["-e", "setTimeout(() => {}, 60_000)", "--", "--run-id", runId], { stdio: "ignore" }) : null; if (liveRunner) await once(liveRunner, "spawn"); let leaseOwner: string | undefined; + let restartRecovery: NativeRestartRecoveryClaim | undefined; + if (options.remoteReattach) { + const controller = await currentNativeControllerIdentity(); + leaseOwner = `reattach-${runId}`; + await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000), + controllerBootId: controller.bootId, controllerPid: controller.pid, controllerGeneration: 7 }) + .where(eq(nativeRunFinalizations.runId, runId)); + restartRecovery = { kind: "reattach_remote_runner", runId, leaseOwner, controllerGeneration: 7, providerAttempt: 2, + restartKind: "hard", recoveryRequestId: null, remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" }, + ...options.remoteReattach }; + } if (options.viaReconciler) { const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: native.runnerInstanceId, runIds: [runId] }); expect(claims).toEqual([expect.objectContaining({ runId })]); leaseOwner = claims[0]!.leaseOwner; } - await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, leaseOwner, backend: { + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, leaseOwner, restartRecovery, backend: { descriptor: async () => { throw new Error("Unexpected provider call"); }, openSession: async () => { throw new Error("Unexpected provider call"); }, } }).catch(() => undefined).finally(() => { liveRunner?.kill("SIGKILL"); }); @@ -135,6 +148,11 @@ describe("a scheduled same-run resume reaching the provider runtime", () => { expect(passed).toEqual([false]); }); + it.each([{}, { replaceUnansweredRunner: true as const }])("continues a lost turn on a same-run remote Runner reattachment (%j)", async (remoteReattach) => { + const { passed } = await resumeOnce({ failureDetail: exactResume, stopRecorded: false, remoteReattach }); + expect(passed).toEqual([true]); + }); + it("keeps the lost turn failing when the reconciler claimed a resume without a stop receipt", async () => { const { passed } = await resumeOnce({ failureDetail: exactResume, stopRecorded: false, viaReconciler: true }); expect(passed).toEqual([false]); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 3b6d3662c47..ef4122799ae 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -12321,7 +12321,8 @@ async function executePaperclipNativeSessionAttempt( trace.activate(runnerSessionStartupScope); const runExecution = () => executeNativeSession({ - resumeInterruptedTurn: input.restartRecovery?.kind === "resume_dead_runner" || scheduledResumeContinuation, + resumeInterruptedTurn: input.restartRecovery?.kind === "resume_dead_runner" + || input.restartRecovery?.kind === "reattach_remote_runner" || scheduledResumeContinuation, getFreshSessionHandoff: input.getFreshSessionHandoff, // A failed seal commits nothing: the messages stay queued for the // turn boundary instead of failing this run before its turn.