diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 08154d6ddbd..40903c46bee 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3020,6 +3020,18 @@ replacement behavior. When a replacement does archive remote continuity state and the remote move fails, the local state is not moved, and the run error keeps the bounded, redacted exit status and output of that move. +When the provider state backup fails to save after the run's result was +accepted, and the run used a per-turn sandbox (no reusable lease), that backup +was the only copy of the provider session. The server records +`nativeProviderStateUnsaved` with the code `native_provider_state_incompatible` +in that run's runner profile. The run's status, result and task outcome do not +change. The next run on the session does not rebind that checkpoint: it starts +a fresh provider session in the same workspace with the full task context, +without first trying to resume, and its transcript shows the session as started +rather than resumed. A failure before the result is accepted, a backup that +saves after a retry, a retained sandbox, an SSH host, and records saved without +the key keep the resume path described next. + A Runner that keeps no control-plane state for the saved session (none is live and no archived authority epoch matches it) reports a typed refusal. This happens when a follow-up run starts in a new sandbox and the previous run's diff --git a/server/src/services/native-runtime/native-provider-state-unsaved.ts b/server/src/services/native-runtime/native-provider-state-unsaved.ts new file mode 100644 index 00000000000..388213bdd51 --- /dev/null +++ b/server/src/services/native-runtime/native-provider-state-unsaved.ts @@ -0,0 +1,51 @@ +import { and, eq, sql } from "drizzle-orm"; +import { heartbeatRuns, nativeHarnessBackups, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; +import { NATIVE_PROVIDER_STATE_INCOMPATIBLE, NATIVE_PROVIDER_STATE_UNSAVED_KEY } from "./native-session-resume.js"; + +const REASON = "harness_backup_not_saved"; + +type RunBinding = { companyId: string; issueId: string; runId: string }; + +/** + * Mark the run's checkpoint not resumable, only when the run already holds an + * accepted result and no current durable backup. A failure before acceptance, + * or after the backup became current, changes nothing here. The run + * status, result, and outcome are untouched. + */ +export async function markAcceptedRunProviderStateUnsaved(db: Db, binding: RunBinding): Promise { + const mark = { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: NATIVE_PROVIDER_STATE_INCOMPATIBLE, reason: REASON, recordedAt: new Date().toISOString() } }; + const rows = await db.update(heartbeatRuns).set({ + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify(mark)}::jsonb`, + }).where(and( + eq(heartbeatRuns.id, binding.runId), + eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.nativeIssueId, binding.issueId), + sql`exists (select 1 from ${nativeRunFinalizations} inner join ${nativeRunResults} + on ${nativeRunResults.id} = ${nativeRunFinalizations.resultId} + and ${nativeRunResults.runId} = ${nativeRunFinalizations.runId} + and ${nativeRunResults.companyId} = ${nativeRunFinalizations.companyId} + and ${nativeRunResults.schemaStatus} = 'accepted' + where ${nativeRunFinalizations.runId} = ${binding.runId}::uuid + and ${nativeRunFinalizations.companyId} = ${binding.companyId}::uuid + and ${nativeRunFinalizations.issueId} = ${binding.issueId}::uuid)`, + // A failure after this run's durable backup became current (for example + // scratch cleanup) leaves the backup restorable: keep the resume path. + sql`not exists (select 1 from ${nativeHarnessBackups} + where ${nativeHarnessBackups.companyId} = ${binding.companyId}::uuid + and ${nativeHarnessBackups.runId} = ${binding.runId}::uuid + and ${nativeHarnessBackups.status} = 'current')`, + )).returning({ id: heartbeatRuns.id }); + return rows.length > 0; +} + +/** A later save of the same run's provider state restores the resume path. */ +export async function clearRunProviderStateUnsaved(db: Db, binding: RunBinding): Promise { + await db.update(heartbeatRuns).set({ + runnerProfileJson: sql`${heartbeatRuns.runnerProfileJson} - ${NATIVE_PROVIDER_STATE_UNSAVED_KEY}`, + }).where(and( + eq(heartbeatRuns.id, binding.runId), + eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.nativeIssueId, binding.issueId), + sql`${heartbeatRuns.runnerProfileJson} ? ${NATIVE_PROVIDER_STATE_UNSAVED_KEY}`, + )); +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b77374934e2..03e71ad6519 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -369,6 +369,15 @@ vi.mock("./native-runner-file-handoff.js", () => ({ state.renderNativeRunnerStagedAttachmentPrompt, })); +const providerStateUnsaved = vi.hoisted(() => ({ + mark: vi.fn(async (_db: unknown, _binding: unknown) => true), + clear: vi.fn(async (_db: unknown, _binding: unknown) => undefined), +})); +vi.mock("./native-provider-state-unsaved.js", () => ({ + markAcceptedRunProviderStateUnsaved: providerStateUnsaved.mark, + clearRunProviderStateUnsaved: providerStateUnsaved.clear, +})); + vi.mock("./current-wake-comments.js", async importOriginal => ({ ...await importOriginal(), resolveCurrentWakeCommentsBinding: state.resolveCurrentWakeCommentsBinding, @@ -2326,7 +2335,7 @@ describe("remote provider pack manifest", () => { return JSON.stringify(value); }; - it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { + it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-new-durable-store-failed", "opencode-new-durable-publish-failed", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { const firstDurable = provider.startsWith("opencode-new-durable"); // A sandbox connection drops once while the settled state is read out. let checkpointReadResets = provider === "opencode-new-durable-reset" ? 1 : 0; @@ -2422,6 +2431,10 @@ describe("remote provider pack manifest", () => { const capture = vi.spyOn(providerArtifactCapture, "captureRemoteNativeProviderArtifacts"); const backup = vi.spyOn(harnessStorage, "persistNativeHarnessBackup").mockImplementation(async input => { expect(JSON.parse(await readFile(join(input.root, "manifest.json"), "utf8")).providerArtifacts).toEqual(newReceipt); + // Only the first save fails; a later checkpoint of the same run saves. + if (provider === "opencode-new-durable-store-failed" && backup.mock.calls.length === 1) throw new Error("SlowDown: Please reduce your request rate."); + // The durable backup is stored; only the later local publication fails. + if (provider === "opencode-new-durable-publish-failed") await rm(input.root, { recursive: true, force: true }); return undefined as never; }); onTestFinished(() => { storage.mockRestore(); owner.mockRestore(); validate.mockRestore(); retain.mockRestore(); capture.mockRestore(); backup.mockRestore(); }); @@ -2525,7 +2538,27 @@ describe("remote provider pack manifest", () => { expect(providerArtifactStorage.retainNativeProviderArtifacts).toHaveBeenCalledWith(expect.objectContaining({ owner: expect.objectContaining({ runId: remoteExecution.binding.runId }), source: expect.objectContaining({ manifest: installed, launchRoot, accountUid: 1000 }) })); if (inspectionReset) inspectionResets = 1; + // Each case starts with no recorded mark or clear calls. + providerStateUnsaved.mark.mockClear(); + providerStateUnsaved.clear.mockClear(); + if (provider === "opencode-new-durable-store-failed" || provider === "opencode-new-durable-publish-failed") { + await expect(driver.mock.calls.at(-1)![0].checkpoint!("settled")).rejects.toThrow(); + // Only a backup that was never stored makes the next run start fresh. + if (provider === "opencode-new-durable-store-failed") { + expect(providerStateUnsaved.mark).toHaveBeenCalledWith(preparedDb, remoteExecution.binding); + // A later checkpoint of the same run that saves clears the mark once. + providerStateUnsaved.clear.mockClear(); + await driver.mock.calls.at(-1)![0].checkpoint!("settled"); + expect(providerStateUnsaved.clear).toHaveBeenCalledTimes(1); + expect(providerStateUnsaved.clear).toHaveBeenCalledWith(preparedDb, remoteExecution.binding); + } else { + expect(providerStateUnsaved.mark).not.toHaveBeenCalled(); + expect(providerStateUnsaved.clear).not.toHaveBeenCalled(); + } + return; + } await driver.mock.calls.at(-1)![0].checkpoint!("settled"); + expect(providerStateUnsaved.mark).not.toHaveBeenCalled(); expect(harnessStorage.persistNativeHarnessBackup).toHaveBeenCalledTimes(1); expect(checkpointReadResets).toBe(0); expect(inspectionResets).toBe(0); @@ -17745,6 +17778,70 @@ describe("runnerd provider runtime wiring", () => { } }); + it.each([ + ["a per-turn sandbox", false, true], + ["a retained sandbox", true, false], + ] as const)("marks the provider state unsaved when the close checkpoint fails in %s", async (_name, reusable, marked) => { + providerStateUnsaved.mark.mockClear(); + const remoteCwd = join(isolatedStateDirectory, `remote-unsaved-${reusable}`); + const current: NativeExecutionInputV1 = { ...execution, + binding: { ...execution.binding, companyId: randomUUID(), agentId: randomUUID(), issueId: randomUUID(), runId: randomUUID() }, + session: { ...execution.session, normalizedSessionId: randomUUID(), lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + }; + const runnerInstanceId = "unsaved-state-runner"; + const priorIdentity = { runnerInstanceId, environmentLeaseId: "lease-unsaved", runId: randomUUID(), + normalizedSessionId: current.session.normalizedSessionId!, turnId: "turn-prior", itemId: "item-prior" }; + const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { + if (command.args?.[0] === "--build-metadata") return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + }) }; + if (command.args?.[0] === "--version") return { exitCode: 0, timedOut: false, stdout: "codex-cli 0.160.0", stderr: "" }; + if (command.args?.[1]?.includes("base64")) { + try { return { exitCode: 0, timedOut: false, stdout: execFileSync("sh", command.args, { encoding: "utf8" }), stderr: "" }; } + catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } + } + return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + }); + const controllerArtifact = join(isolatedStateDirectory, `unsaved-fixture-runnerd-${reusable}`); + await writeFile(controllerArtifact, "fixture bootstrap artifact"); + const logs: string[] = []; + await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId, runnerIngressAuthorized: true, + runnerRemoteBinaryPath: controllerArtifact, runnerRemoteCodexPath: controllerArtifact, + onLog: async (_stream: string, chunk: string) => { logs.push(chunk); }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd, environmentId: "environment", + leaseId: priorIdentity.environmentLeaseId, providerKey: "daytona", reusableLeaseConfigured: reusable, + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "unsaved-sandbox" }, + runner: { execute: remoteExecute, syncIn: vi.fn(async () => undefined) }, + getRunnerIngressEndpoint: async () => ({ kind: "authenticated_websocket", websocketUrl: "wss://sandbox.invalid/runner", + secretHeaders: [{ name: "Authorization", value: "fixture-ingress" }], generation: "1", refresh: vi.fn(), close: vi.fn(async () => undefined) }), + } as never, + } as never); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { + controlPlaneRegistration: NonNullable; + }; + // The sandbox holds no suspended Runner state, so no backup can be saved. + const controller = new DurablePrpControlPlane({ stateDirectory: join(options.stateDirectory!, "control-plane"), + identity: options.prpIdentity! as ConstructorParameters[0]["identity"], + expectedRunnerVersion: "unsaved-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + let registration: Awaited> | undefined; + onTestFinished(async () => { await registration?.release(); await controller.stop(); }); + registration = await options.controlPlaneRegistration(controller); + await expect(registration.checkpoint!("settled")).rejects.toThrow("runner_remote_checkpoint_incomplete"); + if (marked) { + expect(providerStateUnsaved.mark).toHaveBeenCalledTimes(1); + expect(providerStateUnsaved.mark.mock.calls[0]![1]).toEqual(current.binding); + expect(logs.join("")).toContain("the next run starts a fresh provider session"); + } else { + // A retained sandbox keeps the live state, so the resume path stays. + expect(providerStateUnsaved.mark).not.toHaveBeenCalled(); + expect(logs.join("")).not.toContain("fresh provider session"); + } + }); + it("makes remote authority archival idempotent and returns the archived state", async () => { const remoteExecute = vi.fn(); const remoteTarget = { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index cfa74418b02..8ce1b6faf79 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -271,6 +271,7 @@ import { stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; import { NATIVE_PROVIDER_SIDE_EFFECT_PENDING, NATIVE_PROVIDER_STATE_INCOMPATIBLE, nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { clearRunProviderStateUnsaved, markAcceptedRunProviderStateUnsaved } from "./native-provider-state-unsaved.js"; import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; @@ -16655,15 +16656,50 @@ async function createRunnerdBackendWithinSessionClaim( ); }; + // A per-turn sandbox is released after the run, so the backup is the only + // copy of the provider session. When it is not saved after the result was + // accepted, mark the checkpoint not resumable: the next run starts a fresh + // provider session instead of failing once on the missing state. A retained + // sandbox or SSH host keeps the live state, so it keeps the resume path. + let providerStateUnsavedMarked = false; + // Set once the durable backup is stored. A later local publication, stamp, + // or cleanup failure leaves that backup restorable, so it is not marked. + let checkpointBackupStored = false; const checkpointRemoteRunner = async ( settlement: "settled" | "unsettled", ) => { + let saved: boolean; + checkpointBackupStored = false; + try { + saved = await persistRemoteRunnerCheckpoint(settlement); + } catch (error) { + if (!checkpointBackupStored && remoteTarget?.transport === "sandbox" && remoteTarget.reusableLeaseConfigured !== true) { + providerStateUnsavedMarked = await markAcceptedRunProviderStateUnsaved(input.db, input.execution.binding) + .catch(() => false) || providerStateUnsavedMarked; + // A log failure cannot replace the checkpoint error. + if (providerStateUnsavedMarked) await Promise.resolve(input.onLog?.( + "stderr", + "[paperclip-runner] provider state was not saved after the accepted result; the next run starts a fresh provider session\n", + )).catch(() => undefined); + } + throw error; + } + // A later save of this run's state restores the resume path. If clearing + // fails, the next run still starts fresh, which is safe. + if (saved && providerStateUnsavedMarked) { + await clearRunProviderStateUnsaved(input.db, input.execution.binding).catch(() => undefined); + providerStateUnsavedMarked = false; + } + }; + const persistRemoteRunnerCheckpoint = async ( + settlement: "settled" | "unsettled", + ): Promise => { if ( !remoteCommandRunner || !remoteStateDirectory || !remoteRunnerFilesystemRoot ) - return; + return false; const durableStorage = durableNativeHarnessStorage(); const currentAuthority = sessionToolAuthorityEpochs.get(sessionScopeId); const backupOwner = durableStorage && currentAuthority ? sessionHarnessBackupOwners.get(currentAuthority) : null; @@ -16704,7 +16740,7 @@ async function createRunnerdBackendWithinSessionClaim( `[paperclip-runner] remote checkpoint ${incompleteFailure ? "failed" : "skipped"}: exact suspended harness state unavailable (process=${settlement} reason=${checkpointable.incompleteReason})\n`, ); if (incompleteFailure) throw incompleteFailure; - return; + return false; } const backupSpanAttributes = { provider: input.execution.provider.kind, @@ -16802,6 +16838,10 @@ async function createRunnerdBackendWithinSessionClaim( scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), root: pendingRoot, signal: AbortSignal.timeout(300_000) }); } + // Without durable storage the local publication below is the + // save; its result is uncertain once it starts, so keep the + // resume path rather than discard a backup that may exist. + checkpointBackupStored = true; await publishNativeHarnessBackup({ db: input.db, companyId: input.execution.binding.companyId, scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), backupRoot, pendingRoot, @@ -16833,6 +16873,7 @@ async function createRunnerdBackendWithinSessionClaim( ); throw error; } + return true; }; const prepareExternalRunnerState = diff --git a/server/src/services/native-runtime/native-session-resume-projection.test.ts b/server/src/services/native-runtime/native-session-resume-projection.test.ts index 2838347e5bd..e1a1b931cd2 100644 --- a/server/src/services/native-runtime/native-session-resume-projection.test.ts +++ b/server/src/services/native-runtime/native-session-resume-projection.test.ts @@ -60,7 +60,8 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport( const saved = source(); await db.insert(heartbeatRuns).values([saved, run(1), run(2)]); const result = await select(); - expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null }, + expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null, + nativeProviderStateUnsaved: null }, contextSnapshot: { paperclipWake, paperclipIssue } }); const stored = await db.select({ profile: heartbeatRuns.runnerProfileJson, context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(eq(heartbeatRuns.id, saved.id)); expect(stored[0].profile).toEqual(profile); diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index ce3f8314711..1f88c898e36 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -21,6 +21,8 @@ import { heartbeatRunEvents, heartbeatRuns, issues, + nativeHarnessBackups, + nativeRunFinalizations, nativeRunResults, } from "@paperclipai/db"; import { describe, expect, it, vi } from "vitest"; @@ -59,11 +61,13 @@ import { REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION, } from "./chat-attachment-reuse.js"; import { buildNativeExecutionInput } from "./native-execution-input.js"; +import { clearRunProviderStateUnsaved, markAcceptedRunProviderStateUnsaved } from "./native-provider-state-unsaved.js"; import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, NATIVE_PROVIDER_SIDE_EFFECT_PENDING, NATIVE_PROVIDER_STATE_INCOMPATIBLE, + NATIVE_PROVIDER_STATE_UNSAVED_KEY, NATIVE_TOOL_CONTRACT_FINGERPRINT, isUnusedLegacyNativeRetryReplacement, nativeToolContractFingerprintForTarget, @@ -310,6 +314,48 @@ describe("a source whose saved provider session the runner proved incompatible", }); }); +describe("a source whose provider state was not saved after its result was accepted", () => { + const unsaved = { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + reason: "harness_backup_not_saved", recordedAt: "2026-11-11T00:00:00.000Z" } }; + const withPolicy = (policy: "resume_failure" | "governed_wait" | "same_session", overrides: Record = {}) => { + const source = previousRun(overrides); + const checkpoint = (source.runnerProfileJson as Record).sessionCheckpoint as Record; + if (policy === "governed_wait") checkpoint.semanticResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", + summary: "Waiting for a response.", continuation: { kind: "response_wake", idempotencyKey: "interaction-response:one" } }; + if (policy === "same_session") checkpoint.goal = { status: "paused", objective: "Keep working" }; + return source; + }; + + it.each(["resume_failure", "governed_wait", "same_session"] as const)( + "starts the next run in a fresh provider session directly (%s policy)", (policy) => { + const current = execution(currentRunId); + expect(rebindNativeSessionCheckpoint({ previousRun: withPolicy(policy, unsaved), currentExecution: current })).toBeNull(); + const buildExecution = vi.fn(({ normalizedSessionId: sessionId, resumedSession }) => { + const built = execution(currentRunId); + built.session.normalizedSessionId = sessionId; + built.task.prompt = resumedSession ? "wake delta" : "original goal and complete handoff"; + return built; + }); + const result = buildNativeExecutionWithCheckpoint({ previousRun: withPolicy(policy, unsaved), normalizedSessionId, buildExecution }); + expect(result.checkpoint).toBeNull(); + expect(result.normalizedSessionId).not.toBe(normalizedSessionId); + expect(result.execution.task.prompt).toContain("complete handoff"); + expect(result.execution.workspace).toEqual(current.workspace); + }); + + it.each(["resume_failure", "governed_wait", "same_session"] as const)( + "keeps the resume path for a record without the mark (%s policy)", (policy) => { + const expected = { resume_failure: "allow_replacement_after_resume_failure", + governed_wait: "allow_replacement_after_governed_wait", same_session: "same_session_only" }[policy]; + // Records written before the mark existed, and runs whose backup saved. + for (const overrides of [{}, { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: null }, + { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: "some_other_code" } }]) { + expect(rebindNativeSessionCheckpoint({ previousRun: withPolicy(policy, overrides), currentExecution: execution(currentRunId) })) + .toMatchObject({ providerRecoveryPolicy: expected, identity: { runId: currentRunId } }); + } + }); +}); + it("wires exact-session recovery and guarded selected identity into heartbeat persistence", () => { const source = readFileSync( new URL("../heartbeat/runtime-selection.ts", import.meta.url), @@ -1112,6 +1158,67 @@ const recoveryFakeCodex = resolve( (databaseSupported ? describe : describe.skip)( "native session recovery database orchestration", () => { + it.each(["accepted", "no_result", "rejected", "backup_current"] as const)( + "marks the checkpoint not resumable only after an accepted result (%s)", async (outcome) => { + const database = await startEmbeddedPostgresTestDatabase("paperclip-native-unsaved-state-"); + const db = createDb(database.connectionString); + try { + await db.insert(companies).values({ id: companyId, name: "Unsaved provider state", issuePrefix: "NPS" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Native runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Keep the task" }); + const source = previousRun(); + const sourceExecution = source.runnerProfileJson.nativeExecutionInput; + await db.insert(completionContracts).values({ id: sourceExecution.completionContract.id, companyId, issueId, + revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "unsaved-state-v1", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { ...sourceExecution.completionContract.contract }, canonicalSha256: sourceExecution.completionContract.sha256, + createdByActorType: "system", createdByActorId: "test" }); + const resultJson = { result: { summary: "Done." }, terminal: { runTerminalState: "succeeded" } }; + await db.insert(heartbeatRuns).values({ id: previousRunId, companyId, agentId, status: "succeeded", + runtimeMode: "native", nativeIssueId: issueId, nativeSessionId: normalizedSessionId, runnerInstanceId: randomUUID(), + completionContractId: sourceExecution.completionContract.id, completionContractSha256: sourceExecution.completionContract.sha256, + createdAt: new Date(Date.UTC(2026, 10, 11, 0, 0, 0)), resultJson, runnerProfileJson: source.runnerProfileJson }); + const resultId = randomUUID(); + if (outcome !== "no_result") await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: previousRunId, + completionContractId: sourceExecution.completionContract.id, serverFingerprint: "test", + schemaStatus: outcome === "rejected" ? "rejected" : "accepted", resultJson, canonicalSha256: "a".repeat(64) }); + await db.insert(nativeRunFinalizations).values({ runId: previousRunId, companyId, issueId, + phase: outcome === "accepted" || outcome === "backup_current" ? "committed" : "observed", + resultId: outcome === "no_result" ? null : resultId }); + // The durable backup became current before a later step failed. + if (outcome === "backup_current") await db.insert(nativeHarnessBackups).values({ companyId, issueId, runId: previousRunId, + scopeDigest: "b".repeat(64), manifestSha256: `sha256:${"c".repeat(64)}`, status: "current", reference: {} }); + const binding = { companyId, issueId, runId: previousRunId }; + const selection = { companyId, agentId, issueId, normalizedSessionId, currentRunId, + beforeCreatedAt: new Date(Date.UTC(2026, 10, 11, 0, 1, 0)) }; + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, previousRunId)); + + expect(await markAcceptedRunProviderStateUnsaved(db, binding)).toBe(outcome === "accepted"); + const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, previousRunId)); + // The accepted result and the run outcome are untouched. + expect({ ...after, runnerProfileJson: null }).toEqual({ ...before, runnerProfileJson: null }); + const selected = await findNativeSessionResumeRun(db, selection); + expect(selected?.id).toBe(previousRunId); + const rebound = rebindNativeSessionCheckpoint({ previousRun: selected!, currentExecution: execution(currentRunId) }); + if (outcome === "accepted") { + expect(after.runnerProfileJson).toMatchObject({ ...source.runnerProfileJson, + [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: NATIVE_PROVIDER_STATE_INCOMPATIBLE, reason: "harness_backup_not_saved" } }); + expect(rebound).toBeNull(); + // A later save of the same run's state restores the resume path. + await clearRunProviderStateUnsaved(db, binding); + const restored = await findNativeSessionResumeRun(db, selection); + expect(rebindNativeSessionCheckpoint({ previousRun: restored!, currentExecution: execution(currentRunId) })) + .toMatchObject({ providerRecoveryPolicy: "allow_replacement_after_resume_failure" }); + } else { + // A failure before acceptance, or after the backup became current, keeps today's path. + expect(after.runnerProfileJson).toEqual(before.runnerProfileJson); + expect(rebound).toMatchObject({ providerRecoveryPolicy: "allow_replacement_after_resume_failure" }); + } + } finally { + await database.cleanup(); + } + }, 60_000); + it("uses scoped latest checkpoints, preserves progress barriers, and guards bootstrap persistence after the row lock", async () => { const database = await startEmbeddedPostgresTestDatabase( "paperclip-native-resume-selection-", diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 7708c725ce2..9cd6fddbe16 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -101,6 +101,19 @@ export const NATIVE_TOOL_CONTRACT_FINGERPRINT = /** The runner proved that this run's saved provider session cannot be * restored by the current runner build. Its checkpoint is not resumable. */ export const NATIVE_PROVIDER_STATE_INCOMPATIBLE = "native_provider_state_incompatible"; +/** + * `runnerProfileJson` key on a run whose result was accepted but whose + * provider state backup was not saved. Its per-turn sandbox is released after + * the run, so that backup was the only copy of the provider session. The next + * run on the session starts a fresh provider session instead of resuming. + * Records without the key, including every record written before it existed, + * keep the ordinary resume path. + */ +export const NATIVE_PROVIDER_STATE_UNSAVED_KEY = "nativeProviderStateUnsaved"; + +export function isNativeProviderStateUnsaved(profile: unknown): boolean { + return record(record(profile)[NATIVE_PROVIDER_STATE_UNSAVED_KEY]).code === NATIVE_PROVIDER_STATE_INCOMPATIBLE; +} /** The saved session owned a side-effecting call without a delivered result. * Recovery requires an explicit reconciliation of that action. */ export const NATIVE_PROVIDER_SIDE_EFFECT_PENDING = "native_provider_side_effect_pending"; @@ -362,11 +375,13 @@ async function selectNativeSessionResumeBarrier(db: Db, input: NativeSessionResu nativeExecutionInput: unknown; sessionCheckpoint: unknown; nativeControllerRecovery: unknown; + nativeProviderStateUnsaved?: unknown; }>`jsonb_build_object( 'nativeToolContractFingerprint', ${heartbeatRuns.runnerProfileJson}->'nativeToolContractFingerprint', 'nativeExecutionInput', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput', 'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint', - 'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery' + 'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery', + '${sql.raw(NATIVE_PROVIDER_STATE_UNSAVED_KEY)}', ${heartbeatRuns.runnerProfileJson}->'${sql.raw(NATIVE_PROVIDER_STATE_UNSAVED_KEY)}' )`, contextSnapshot: sql<{ paperclipWake: unknown; @@ -625,6 +640,9 @@ export function rebindNativeSessionCheckpoint(input: { // reconciled continuation then starts a fresh session in the same workspace. if (input.previousRun.errorCode === NATIVE_PROVIDER_SIDE_EFFECT_PENDING) return null; const previousProfile = record(input.previousRun.runnerProfileJson); + // The source's result was accepted, but its provider state was never saved + // and its sandbox is gone. Resuming would only fail, so start fresh now. + if (isNativeProviderStateUnsaved(previousProfile)) return null; if ( previousProfile.nativeToolContractFingerprint !== nativeToolContractFingerprintForTarget(input.executionTargetKind ?? "local", input.currentExecution.provider.kind)