From c8ae02e5ce42c254e1d56dd4d1b34bd73e1a415b Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:36:00 -0600 Subject: [PATCH] fix(native): start a fresh session when an accepted run's provider state was not saved (#20811) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - A native run in a per-turn remote sandbox saves a provider state backup after its result is accepted. The sandbox is then released, so the backup is the only copy of the provider session. > - When that backup fails to save (for example, #20772: storage throttling), the run keeps its accepted result. Nothing records that its checkpoint can no longer be resumed. > - The next run on the session rebinds that checkpoint and asks the Runner to resume. The Runner has no state and refuses (#20757). With the normal policy, the run opens a replacement session after that failed attempt. With a governed-wait or unfinished-goal checkpoint, the run fails with `native_provider_state_incompatible`, and only Retry or the next message starts fresh. > - The server already knew the state was gone when the backup failed. The failed resume is wasted work. > - This pull request marks the accepted run's checkpoint as not resumable at the moment the backup fails. The next run then starts a fresh provider session directly. > - The benefit is that a lost backup no longer costs a failed run or a failed resume attempt before the task continues. ## Linked Issues or Issue Description Refs #20757 (the missing-state refusal that the next run reaches today). Refs #20772 (the throttled backup upload that loses the backup). Refs #20809 (a related fix for governed-wait follow-ups that reach the missing state; this PR stops most of them from reaching it). **What happened?** 1. A native run in a per-turn sandbox finished, and its result was accepted. 2. `harness_state.backup.persist` failed. The run recorded `native.post_result_failure`. Its run row kept a normal `sessionCheckpoint`. 3. The sandbox was released. 4. The next run on the same session selected that run as its resume source, and `rebindNativeSessionCheckpoint` returned the checkpoint. 5. The Runner found no saved state and refused with `PRP provider resume state is unavailable`. The run then opened a replacement session after the failed attempt, or (governed-wait or unfinished-goal checkpoint) failed with `native_provider_state_incompatible`. Retry or the next message then started fresh. **Expected behavior** The next run starts a fresh provider session at once, because the server already knows the saved state is not there. The accepted result and the task outcome stay as they are. **Steps to reproduce** 1. Give a run row an accepted result and a normal `sessionCheckpoint`. 2. Make its close checkpoint fail in a per-turn sandbox (`reusableLeaseConfigured: false`). 3. Select the next run's resume source with `findNativeSessionResumeRun` and call `rebindNativeSessionCheckpoint`. 4. On `master` it returns the checkpoint, so the next run tries to resume state that does not exist. **Paperclip version or commit** `master` at `556b249897`. ## What Changed - `server/src/services/native-runtime/native-provider-state-unsaved.ts` (new): `markAcceptedRunProviderStateUnsaved` adds `nativeProviderStateUnsaved: { code: "native_provider_state_incompatible", reason: "harness_backup_not_saved" }` to the run's `runnerProfileJson`. One conditional `UPDATE` writes it only when the run's coordinator holds a result with `schema_status = 'accepted'`. `clearRunProviderStateUnsaved` removes it. - `server/src/services/native-runtime/native-session-executor.ts`: the remote close checkpoint (`checkpointRemoteRunner`) now wraps the existing body. When the checkpoint throws in a sandbox without a reusable lease, it calls the marker, writes one run-log line, and throws the original error. A marker or log failure cannot replace that error. If a later checkpoint of the same run saves, it clears the mark. - `server/src/services/native-runtime/native-session-resume.ts`: `rebindNativeSessionCheckpoint` returns `null` for a source with the mark, as it already does for `errorCode = native_provider_state_incompatible`. `findNativeSessionResumeRun` projects the new key. - `doc/execution-semantics.md`: one additive paragraph. - Tests in `native-session-resume.test.ts` and `native-session-executor.test.ts`. Guard conditions. The mark is written only when all of these are true: - The close checkpoint threw before the durable backup was stored (after the upload's own bounded retries). A later local publication, stamp or cleanup failure does not mark the run. - The target is a remote `sandbox` with `reusableLeaseConfigured !== true`. A retained sandbox or an SSH host keeps the live state, so its resume path is unchanged. - The run holds an accepted result. A failure before acceptance changes nothing. Saved state from the previous release: run rows without the key keep today's resume path. An older server that reads a row with the key ignores it, and so it behaves as it does today. No schema change. User-visible: the next run's transcript shows "Paperclip session started" instead of "resumed", with a new session id. The source run's log shows `provider state was not saved after the accepted result; the next run starts a fresh provider session`. The fresh session gets the full task context, as for other fresh starts. ## Verification - `cd server && npx vitest run src/services/native-runtime/native-session-resume.test.ts -t "not resumable only after|provider state was not saved"`: 9 pass (6 pure, 3 embedded Postgres). - `cd server && npx vitest run src/services/native-runtime/native-session-executor.test.ts -t "marks the provider state unsaved|uses the bundled Linux image authority for remote opencode-new-durable"`: 8 pass. They include a durable store failure (marked) and a local publication failure after the store (not marked). - Red on `master` (source files reverted, tests kept): the 3 "starts the next run in a fresh provider session directly" cases fail (`expected {…} to be null`), the DB "accepted" case fails the same way, and the executor per-turn case fails (`mark` called 0 times). - Guard removal checks: - Sandbox guard replaced with `true`: the "retained sandbox" case fails (`mark` called once). - `!checkpointBackupStored` removed: the "publish-failed" case fails. - `schema_status = 'accepted'` removed from the marker: the "rejected" case fails (`expected true to be false`). - Negative cases that pass: no result, rejected result, retained sandbox, records without the key or with another code (all three recovery policies), and a later save that clears the mark. - `cd server && npx tsc --noEmit -p .`: clean. - Not related to this change and failing on `master` in this worktree too: `runtime-selection.test.ts` connector skill cases (missing `.claude/skills/*/SKILL.md`) and `native-session-resume.test.ts` "archives a damaged prior epoch" (`ENOENT … authority-epochs`). Required CI is the authority for those suites. ## Risks - Low. The change only skips a resume that cannot succeed. The fresh session uses the same workspace and full task context, like a Retry after `native_provider_state_incompatible`. - If the backup failed but the provider state were still somewhere recoverable, the next run would lose provider context. For a per-turn sandbox without a reusable lease, the restore path uses only the backup, so this does not apply. Retained sandboxes and SSH hosts are excluded. - One extra conditional `UPDATE` runs only on the failure path. The normal path before `turn.start` has no new waits or work. - `doc/execution-semantics.md` is also edited by #20809. The paragraphs are at different places; whichever merges second rebases. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code, with tool use (shell, file edit, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude --- doc/execution-semantics.md | 12 ++ .../native-provider-state-unsaved.ts | 51 +++++++++ .../native-session-executor.test.ts | 99 +++++++++++++++- .../native-runtime/native-session-executor.ts | 45 +++++++- .../native-session-resume-projection.test.ts | 3 +- .../native-session-resume.test.ts | 107 ++++++++++++++++++ .../native-runtime/native-session-resume.ts | 20 +++- 7 files changed, 332 insertions(+), 5 deletions(-) create mode 100644 server/src/services/native-runtime/native-provider-state-unsaved.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 08154d6ddbd..40903c46bee 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3020,6 +3020,18 @@ replacement behavior. When a replacement does archive remote continuity state and the remote move fails, the local state is not moved, and the run error keeps the bounded, redacted exit status and output of that move. +When the provider state backup fails to save after the run's result was +accepted, and the run used a per-turn sandbox (no reusable lease), that backup +was the only copy of the provider session. The server records +`nativeProviderStateUnsaved` with the code `native_provider_state_incompatible` +in that run's runner profile. The run's status, result and task outcome do not +change. The next run on the session does not rebind that checkpoint: it starts +a fresh provider session in the same workspace with the full task context, +without first trying to resume, and its transcript shows the session as started +rather than resumed. A failure before the result is accepted, a backup that +saves after a retry, a retained sandbox, an SSH host, and records saved without +the key keep the resume path described next. + A Runner that keeps no control-plane state for the saved session (none is live and no archived authority epoch matches it) reports a typed refusal. This happens when a follow-up run starts in a new sandbox and the previous run's diff --git a/server/src/services/native-runtime/native-provider-state-unsaved.ts b/server/src/services/native-runtime/native-provider-state-unsaved.ts new file mode 100644 index 00000000000..388213bdd51 --- /dev/null +++ b/server/src/services/native-runtime/native-provider-state-unsaved.ts @@ -0,0 +1,51 @@ +import { and, eq, sql } from "drizzle-orm"; +import { heartbeatRuns, nativeHarnessBackups, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; +import { NATIVE_PROVIDER_STATE_INCOMPATIBLE, NATIVE_PROVIDER_STATE_UNSAVED_KEY } from "./native-session-resume.js"; + +const REASON = "harness_backup_not_saved"; + +type RunBinding = { companyId: string; issueId: string; runId: string }; + +/** + * Mark the run's checkpoint not resumable, only when the run already holds an + * accepted result and no current durable backup. A failure before acceptance, + * or after the backup became current, changes nothing here. The run + * status, result, and outcome are untouched. + */ +export async function markAcceptedRunProviderStateUnsaved(db: Db, binding: RunBinding): Promise { + const mark = { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: NATIVE_PROVIDER_STATE_INCOMPATIBLE, reason: REASON, recordedAt: new Date().toISOString() } }; + const rows = await db.update(heartbeatRuns).set({ + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify(mark)}::jsonb`, + }).where(and( + eq(heartbeatRuns.id, binding.runId), + eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.nativeIssueId, binding.issueId), + sql`exists (select 1 from ${nativeRunFinalizations} inner join ${nativeRunResults} + on ${nativeRunResults.id} = ${nativeRunFinalizations.resultId} + and ${nativeRunResults.runId} = ${nativeRunFinalizations.runId} + and ${nativeRunResults.companyId} = ${nativeRunFinalizations.companyId} + and ${nativeRunResults.schemaStatus} = 'accepted' + where ${nativeRunFinalizations.runId} = ${binding.runId}::uuid + and ${nativeRunFinalizations.companyId} = ${binding.companyId}::uuid + and ${nativeRunFinalizations.issueId} = ${binding.issueId}::uuid)`, + // A failure after this run's durable backup became current (for example + // scratch cleanup) leaves the backup restorable: keep the resume path. + sql`not exists (select 1 from ${nativeHarnessBackups} + where ${nativeHarnessBackups.companyId} = ${binding.companyId}::uuid + and ${nativeHarnessBackups.runId} = ${binding.runId}::uuid + and ${nativeHarnessBackups.status} = 'current')`, + )).returning({ id: heartbeatRuns.id }); + return rows.length > 0; +} + +/** A later save of the same run's provider state restores the resume path. */ +export async function clearRunProviderStateUnsaved(db: Db, binding: RunBinding): Promise { + await db.update(heartbeatRuns).set({ + runnerProfileJson: sql`${heartbeatRuns.runnerProfileJson} - ${NATIVE_PROVIDER_STATE_UNSAVED_KEY}`, + }).where(and( + eq(heartbeatRuns.id, binding.runId), + eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.nativeIssueId, binding.issueId), + sql`${heartbeatRuns.runnerProfileJson} ? ${NATIVE_PROVIDER_STATE_UNSAVED_KEY}`, + )); +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b77374934e2..03e71ad6519 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -369,6 +369,15 @@ vi.mock("./native-runner-file-handoff.js", () => ({ state.renderNativeRunnerStagedAttachmentPrompt, })); +const providerStateUnsaved = vi.hoisted(() => ({ + mark: vi.fn(async (_db: unknown, _binding: unknown) => true), + clear: vi.fn(async (_db: unknown, _binding: unknown) => undefined), +})); +vi.mock("./native-provider-state-unsaved.js", () => ({ + markAcceptedRunProviderStateUnsaved: providerStateUnsaved.mark, + clearRunProviderStateUnsaved: providerStateUnsaved.clear, +})); + vi.mock("./current-wake-comments.js", async importOriginal => ({ ...await importOriginal(), resolveCurrentWakeCommentsBinding: state.resolveCurrentWakeCommentsBinding, @@ -2326,7 +2335,7 @@ describe("remote provider pack manifest", () => { return JSON.stringify(value); }; - it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { + it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-new-durable-store-failed", "opencode-new-durable-publish-failed", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { const firstDurable = provider.startsWith("opencode-new-durable"); // A sandbox connection drops once while the settled state is read out. let checkpointReadResets = provider === "opencode-new-durable-reset" ? 1 : 0; @@ -2422,6 +2431,10 @@ describe("remote provider pack manifest", () => { const capture = vi.spyOn(providerArtifactCapture, "captureRemoteNativeProviderArtifacts"); const backup = vi.spyOn(harnessStorage, "persistNativeHarnessBackup").mockImplementation(async input => { expect(JSON.parse(await readFile(join(input.root, "manifest.json"), "utf8")).providerArtifacts).toEqual(newReceipt); + // Only the first save fails; a later checkpoint of the same run saves. + if (provider === "opencode-new-durable-store-failed" && backup.mock.calls.length === 1) throw new Error("SlowDown: Please reduce your request rate."); + // The durable backup is stored; only the later local publication fails. + if (provider === "opencode-new-durable-publish-failed") await rm(input.root, { recursive: true, force: true }); return undefined as never; }); onTestFinished(() => { storage.mockRestore(); owner.mockRestore(); validate.mockRestore(); retain.mockRestore(); capture.mockRestore(); backup.mockRestore(); }); @@ -2525,7 +2538,27 @@ describe("remote provider pack manifest", () => { expect(providerArtifactStorage.retainNativeProviderArtifacts).toHaveBeenCalledWith(expect.objectContaining({ owner: expect.objectContaining({ runId: remoteExecution.binding.runId }), source: expect.objectContaining({ manifest: installed, launchRoot, accountUid: 1000 }) })); if (inspectionReset) inspectionResets = 1; + // Each case starts with no recorded mark or clear calls. + providerStateUnsaved.mark.mockClear(); + providerStateUnsaved.clear.mockClear(); + if (provider === "opencode-new-durable-store-failed" || provider === "opencode-new-durable-publish-failed") { + await expect(driver.mock.calls.at(-1)![0].checkpoint!("settled")).rejects.toThrow(); + // Only a backup that was never stored makes the next run start fresh. + if (provider === "opencode-new-durable-store-failed") { + expect(providerStateUnsaved.mark).toHaveBeenCalledWith(preparedDb, remoteExecution.binding); + // A later checkpoint of the same run that saves clears the mark once. + providerStateUnsaved.clear.mockClear(); + await driver.mock.calls.at(-1)![0].checkpoint!("settled"); + expect(providerStateUnsaved.clear).toHaveBeenCalledTimes(1); + expect(providerStateUnsaved.clear).toHaveBeenCalledWith(preparedDb, remoteExecution.binding); + } else { + expect(providerStateUnsaved.mark).not.toHaveBeenCalled(); + expect(providerStateUnsaved.clear).not.toHaveBeenCalled(); + } + return; + } await driver.mock.calls.at(-1)![0].checkpoint!("settled"); + expect(providerStateUnsaved.mark).not.toHaveBeenCalled(); expect(harnessStorage.persistNativeHarnessBackup).toHaveBeenCalledTimes(1); expect(checkpointReadResets).toBe(0); expect(inspectionResets).toBe(0); @@ -17745,6 +17778,70 @@ describe("runnerd provider runtime wiring", () => { } }); + it.each([ + ["a per-turn sandbox", false, true], + ["a retained sandbox", true, false], + ] as const)("marks the provider state unsaved when the close checkpoint fails in %s", async (_name, reusable, marked) => { + providerStateUnsaved.mark.mockClear(); + const remoteCwd = join(isolatedStateDirectory, `remote-unsaved-${reusable}`); + const current: NativeExecutionInputV1 = { ...execution, + binding: { ...execution.binding, companyId: randomUUID(), agentId: randomUUID(), issueId: randomUUID(), runId: randomUUID() }, + session: { ...execution.session, normalizedSessionId: randomUUID(), lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + }; + const runnerInstanceId = "unsaved-state-runner"; + const priorIdentity = { runnerInstanceId, environmentLeaseId: "lease-unsaved", runId: randomUUID(), + normalizedSessionId: current.session.normalizedSessionId!, turnId: "turn-prior", itemId: "item-prior" }; + const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { + if (command.args?.[0] === "--build-metadata") return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + }) }; + if (command.args?.[0] === "--version") return { exitCode: 0, timedOut: false, stdout: "codex-cli 0.160.0", stderr: "" }; + if (command.args?.[1]?.includes("base64")) { + try { return { exitCode: 0, timedOut: false, stdout: execFileSync("sh", command.args, { encoding: "utf8" }), stderr: "" }; } + catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } + } + return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + }); + const controllerArtifact = join(isolatedStateDirectory, `unsaved-fixture-runnerd-${reusable}`); + await writeFile(controllerArtifact, "fixture bootstrap artifact"); + const logs: string[] = []; + await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId, runnerIngressAuthorized: true, + runnerRemoteBinaryPath: controllerArtifact, runnerRemoteCodexPath: controllerArtifact, + onLog: async (_stream: string, chunk: string) => { logs.push(chunk); }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd, environmentId: "environment", + leaseId: priorIdentity.environmentLeaseId, providerKey: "daytona", reusableLeaseConfigured: reusable, + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "unsaved-sandbox" }, + runner: { execute: remoteExecute, syncIn: vi.fn(async () => undefined) }, + getRunnerIngressEndpoint: async () => ({ kind: "authenticated_websocket", websocketUrl: "wss://sandbox.invalid/runner", + secretHeaders: [{ name: "Authorization", value: "fixture-ingress" }], generation: "1", refresh: vi.fn(), close: vi.fn(async () => undefined) }), + } as never, + } as never); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { + controlPlaneRegistration: NonNullable; + }; + // The sandbox holds no suspended Runner state, so no backup can be saved. + const controller = new DurablePrpControlPlane({ stateDirectory: join(options.stateDirectory!, "control-plane"), + identity: options.prpIdentity! as ConstructorParameters[0]["identity"], + expectedRunnerVersion: "unsaved-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + let registration: Awaited> | undefined; + onTestFinished(async () => { await registration?.release(); await controller.stop(); }); + registration = await options.controlPlaneRegistration(controller); + await expect(registration.checkpoint!("settled")).rejects.toThrow("runner_remote_checkpoint_incomplete"); + if (marked) { + expect(providerStateUnsaved.mark).toHaveBeenCalledTimes(1); + expect(providerStateUnsaved.mark.mock.calls[0]![1]).toEqual(current.binding); + expect(logs.join("")).toContain("the next run starts a fresh provider session"); + } else { + // A retained sandbox keeps the live state, so the resume path stays. + expect(providerStateUnsaved.mark).not.toHaveBeenCalled(); + expect(logs.join("")).not.toContain("fresh provider session"); + } + }); + it("makes remote authority archival idempotent and returns the archived state", async () => { const remoteExecute = vi.fn(); const remoteTarget = { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index cfa74418b02..8ce1b6faf79 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -271,6 +271,7 @@ import { stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; import { NATIVE_PROVIDER_SIDE_EFFECT_PENDING, NATIVE_PROVIDER_STATE_INCOMPATIBLE, nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { clearRunProviderStateUnsaved, markAcceptedRunProviderStateUnsaved } from "./native-provider-state-unsaved.js"; import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; @@ -16655,15 +16656,50 @@ async function createRunnerdBackendWithinSessionClaim( ); }; + // A per-turn sandbox is released after the run, so the backup is the only + // copy of the provider session. When it is not saved after the result was + // accepted, mark the checkpoint not resumable: the next run starts a fresh + // provider session instead of failing once on the missing state. A retained + // sandbox or SSH host keeps the live state, so it keeps the resume path. + let providerStateUnsavedMarked = false; + // Set once the durable backup is stored. A later local publication, stamp, + // or cleanup failure leaves that backup restorable, so it is not marked. + let checkpointBackupStored = false; const checkpointRemoteRunner = async ( settlement: "settled" | "unsettled", ) => { + let saved: boolean; + checkpointBackupStored = false; + try { + saved = await persistRemoteRunnerCheckpoint(settlement); + } catch (error) { + if (!checkpointBackupStored && remoteTarget?.transport === "sandbox" && remoteTarget.reusableLeaseConfigured !== true) { + providerStateUnsavedMarked = await markAcceptedRunProviderStateUnsaved(input.db, input.execution.binding) + .catch(() => false) || providerStateUnsavedMarked; + // A log failure cannot replace the checkpoint error. + if (providerStateUnsavedMarked) await Promise.resolve(input.onLog?.( + "stderr", + "[paperclip-runner] provider state was not saved after the accepted result; the next run starts a fresh provider session\n", + )).catch(() => undefined); + } + throw error; + } + // A later save of this run's state restores the resume path. If clearing + // fails, the next run still starts fresh, which is safe. + if (saved && providerStateUnsavedMarked) { + await clearRunProviderStateUnsaved(input.db, input.execution.binding).catch(() => undefined); + providerStateUnsavedMarked = false; + } + }; + const persistRemoteRunnerCheckpoint = async ( + settlement: "settled" | "unsettled", + ): Promise => { if ( !remoteCommandRunner || !remoteStateDirectory || !remoteRunnerFilesystemRoot ) - return; + return false; const durableStorage = durableNativeHarnessStorage(); const currentAuthority = sessionToolAuthorityEpochs.get(sessionScopeId); const backupOwner = durableStorage && currentAuthority ? sessionHarnessBackupOwners.get(currentAuthority) : null; @@ -16704,7 +16740,7 @@ async function createRunnerdBackendWithinSessionClaim( `[paperclip-runner] remote checkpoint ${incompleteFailure ? "failed" : "skipped"}: exact suspended harness state unavailable (process=${settlement} reason=${checkpointable.incompleteReason})\n`, ); if (incompleteFailure) throw incompleteFailure; - return; + return false; } const backupSpanAttributes = { provider: input.execution.provider.kind, @@ -16802,6 +16838,10 @@ async function createRunnerdBackendWithinSessionClaim( scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), root: pendingRoot, signal: AbortSignal.timeout(300_000) }); } + // Without durable storage the local publication below is the + // save; its result is uncertain once it starts, so keep the + // resume path rather than discard a backup that may exist. + checkpointBackupStored = true; await publishNativeHarnessBackup({ db: input.db, companyId: input.execution.binding.companyId, scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), backupRoot, pendingRoot, @@ -16833,6 +16873,7 @@ async function createRunnerdBackendWithinSessionClaim( ); throw error; } + return true; }; const prepareExternalRunnerState = diff --git a/server/src/services/native-runtime/native-session-resume-projection.test.ts b/server/src/services/native-runtime/native-session-resume-projection.test.ts index 2838347e5bd..e1a1b931cd2 100644 --- a/server/src/services/native-runtime/native-session-resume-projection.test.ts +++ b/server/src/services/native-runtime/native-session-resume-projection.test.ts @@ -60,7 +60,8 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport( const saved = source(); await db.insert(heartbeatRuns).values([saved, run(1), run(2)]); const result = await select(); - expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null }, + expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null, + nativeProviderStateUnsaved: null }, contextSnapshot: { paperclipWake, paperclipIssue } }); const stored = await db.select({ profile: heartbeatRuns.runnerProfileJson, context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(eq(heartbeatRuns.id, saved.id)); expect(stored[0].profile).toEqual(profile); diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index ce3f8314711..1f88c898e36 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -21,6 +21,8 @@ import { heartbeatRunEvents, heartbeatRuns, issues, + nativeHarnessBackups, + nativeRunFinalizations, nativeRunResults, } from "@paperclipai/db"; import { describe, expect, it, vi } from "vitest"; @@ -59,11 +61,13 @@ import { REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION, } from "./chat-attachment-reuse.js"; import { buildNativeExecutionInput } from "./native-execution-input.js"; +import { clearRunProviderStateUnsaved, markAcceptedRunProviderStateUnsaved } from "./native-provider-state-unsaved.js"; import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, NATIVE_PROVIDER_SIDE_EFFECT_PENDING, NATIVE_PROVIDER_STATE_INCOMPATIBLE, + NATIVE_PROVIDER_STATE_UNSAVED_KEY, NATIVE_TOOL_CONTRACT_FINGERPRINT, isUnusedLegacyNativeRetryReplacement, nativeToolContractFingerprintForTarget, @@ -310,6 +314,48 @@ describe("a source whose saved provider session the runner proved incompatible", }); }); +describe("a source whose provider state was not saved after its result was accepted", () => { + const unsaved = { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + reason: "harness_backup_not_saved", recordedAt: "2026-11-11T00:00:00.000Z" } }; + const withPolicy = (policy: "resume_failure" | "governed_wait" | "same_session", overrides: Record = {}) => { + const source = previousRun(overrides); + const checkpoint = (source.runnerProfileJson as Record).sessionCheckpoint as Record; + if (policy === "governed_wait") checkpoint.semanticResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", + summary: "Waiting for a response.", continuation: { kind: "response_wake", idempotencyKey: "interaction-response:one" } }; + if (policy === "same_session") checkpoint.goal = { status: "paused", objective: "Keep working" }; + return source; + }; + + it.each(["resume_failure", "governed_wait", "same_session"] as const)( + "starts the next run in a fresh provider session directly (%s policy)", (policy) => { + const current = execution(currentRunId); + expect(rebindNativeSessionCheckpoint({ previousRun: withPolicy(policy, unsaved), currentExecution: current })).toBeNull(); + const buildExecution = vi.fn(({ normalizedSessionId: sessionId, resumedSession }) => { + const built = execution(currentRunId); + built.session.normalizedSessionId = sessionId; + built.task.prompt = resumedSession ? "wake delta" : "original goal and complete handoff"; + return built; + }); + const result = buildNativeExecutionWithCheckpoint({ previousRun: withPolicy(policy, unsaved), normalizedSessionId, buildExecution }); + expect(result.checkpoint).toBeNull(); + expect(result.normalizedSessionId).not.toBe(normalizedSessionId); + expect(result.execution.task.prompt).toContain("complete handoff"); + expect(result.execution.workspace).toEqual(current.workspace); + }); + + it.each(["resume_failure", "governed_wait", "same_session"] as const)( + "keeps the resume path for a record without the mark (%s policy)", (policy) => { + const expected = { resume_failure: "allow_replacement_after_resume_failure", + governed_wait: "allow_replacement_after_governed_wait", same_session: "same_session_only" }[policy]; + // Records written before the mark existed, and runs whose backup saved. + for (const overrides of [{}, { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: null }, + { [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: "some_other_code" } }]) { + expect(rebindNativeSessionCheckpoint({ previousRun: withPolicy(policy, overrides), currentExecution: execution(currentRunId) })) + .toMatchObject({ providerRecoveryPolicy: expected, identity: { runId: currentRunId } }); + } + }); +}); + it("wires exact-session recovery and guarded selected identity into heartbeat persistence", () => { const source = readFileSync( new URL("../heartbeat/runtime-selection.ts", import.meta.url), @@ -1112,6 +1158,67 @@ const recoveryFakeCodex = resolve( (databaseSupported ? describe : describe.skip)( "native session recovery database orchestration", () => { + it.each(["accepted", "no_result", "rejected", "backup_current"] as const)( + "marks the checkpoint not resumable only after an accepted result (%s)", async (outcome) => { + const database = await startEmbeddedPostgresTestDatabase("paperclip-native-unsaved-state-"); + const db = createDb(database.connectionString); + try { + await db.insert(companies).values({ id: companyId, name: "Unsaved provider state", issuePrefix: "NPS" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Native runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Keep the task" }); + const source = previousRun(); + const sourceExecution = source.runnerProfileJson.nativeExecutionInput; + await db.insert(completionContracts).values({ id: sourceExecution.completionContract.id, companyId, issueId, + revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "unsaved-state-v1", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { ...sourceExecution.completionContract.contract }, canonicalSha256: sourceExecution.completionContract.sha256, + createdByActorType: "system", createdByActorId: "test" }); + const resultJson = { result: { summary: "Done." }, terminal: { runTerminalState: "succeeded" } }; + await db.insert(heartbeatRuns).values({ id: previousRunId, companyId, agentId, status: "succeeded", + runtimeMode: "native", nativeIssueId: issueId, nativeSessionId: normalizedSessionId, runnerInstanceId: randomUUID(), + completionContractId: sourceExecution.completionContract.id, completionContractSha256: sourceExecution.completionContract.sha256, + createdAt: new Date(Date.UTC(2026, 10, 11, 0, 0, 0)), resultJson, runnerProfileJson: source.runnerProfileJson }); + const resultId = randomUUID(); + if (outcome !== "no_result") await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: previousRunId, + completionContractId: sourceExecution.completionContract.id, serverFingerprint: "test", + schemaStatus: outcome === "rejected" ? "rejected" : "accepted", resultJson, canonicalSha256: "a".repeat(64) }); + await db.insert(nativeRunFinalizations).values({ runId: previousRunId, companyId, issueId, + phase: outcome === "accepted" || outcome === "backup_current" ? "committed" : "observed", + resultId: outcome === "no_result" ? null : resultId }); + // The durable backup became current before a later step failed. + if (outcome === "backup_current") await db.insert(nativeHarnessBackups).values({ companyId, issueId, runId: previousRunId, + scopeDigest: "b".repeat(64), manifestSha256: `sha256:${"c".repeat(64)}`, status: "current", reference: {} }); + const binding = { companyId, issueId, runId: previousRunId }; + const selection = { companyId, agentId, issueId, normalizedSessionId, currentRunId, + beforeCreatedAt: new Date(Date.UTC(2026, 10, 11, 0, 1, 0)) }; + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, previousRunId)); + + expect(await markAcceptedRunProviderStateUnsaved(db, binding)).toBe(outcome === "accepted"); + const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, previousRunId)); + // The accepted result and the run outcome are untouched. + expect({ ...after, runnerProfileJson: null }).toEqual({ ...before, runnerProfileJson: null }); + const selected = await findNativeSessionResumeRun(db, selection); + expect(selected?.id).toBe(previousRunId); + const rebound = rebindNativeSessionCheckpoint({ previousRun: selected!, currentExecution: execution(currentRunId) }); + if (outcome === "accepted") { + expect(after.runnerProfileJson).toMatchObject({ ...source.runnerProfileJson, + [NATIVE_PROVIDER_STATE_UNSAVED_KEY]: { code: NATIVE_PROVIDER_STATE_INCOMPATIBLE, reason: "harness_backup_not_saved" } }); + expect(rebound).toBeNull(); + // A later save of the same run's state restores the resume path. + await clearRunProviderStateUnsaved(db, binding); + const restored = await findNativeSessionResumeRun(db, selection); + expect(rebindNativeSessionCheckpoint({ previousRun: restored!, currentExecution: execution(currentRunId) })) + .toMatchObject({ providerRecoveryPolicy: "allow_replacement_after_resume_failure" }); + } else { + // A failure before acceptance, or after the backup became current, keeps today's path. + expect(after.runnerProfileJson).toEqual(before.runnerProfileJson); + expect(rebound).toMatchObject({ providerRecoveryPolicy: "allow_replacement_after_resume_failure" }); + } + } finally { + await database.cleanup(); + } + }, 60_000); + it("uses scoped latest checkpoints, preserves progress barriers, and guards bootstrap persistence after the row lock", async () => { const database = await startEmbeddedPostgresTestDatabase( "paperclip-native-resume-selection-", diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 7708c725ce2..9cd6fddbe16 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -101,6 +101,19 @@ export const NATIVE_TOOL_CONTRACT_FINGERPRINT = /** The runner proved that this run's saved provider session cannot be * restored by the current runner build. Its checkpoint is not resumable. */ export const NATIVE_PROVIDER_STATE_INCOMPATIBLE = "native_provider_state_incompatible"; +/** + * `runnerProfileJson` key on a run whose result was accepted but whose + * provider state backup was not saved. Its per-turn sandbox is released after + * the run, so that backup was the only copy of the provider session. The next + * run on the session starts a fresh provider session instead of resuming. + * Records without the key, including every record written before it existed, + * keep the ordinary resume path. + */ +export const NATIVE_PROVIDER_STATE_UNSAVED_KEY = "nativeProviderStateUnsaved"; + +export function isNativeProviderStateUnsaved(profile: unknown): boolean { + return record(record(profile)[NATIVE_PROVIDER_STATE_UNSAVED_KEY]).code === NATIVE_PROVIDER_STATE_INCOMPATIBLE; +} /** The saved session owned a side-effecting call without a delivered result. * Recovery requires an explicit reconciliation of that action. */ export const NATIVE_PROVIDER_SIDE_EFFECT_PENDING = "native_provider_side_effect_pending"; @@ -362,11 +375,13 @@ async function selectNativeSessionResumeBarrier(db: Db, input: NativeSessionResu nativeExecutionInput: unknown; sessionCheckpoint: unknown; nativeControllerRecovery: unknown; + nativeProviderStateUnsaved?: unknown; }>`jsonb_build_object( 'nativeToolContractFingerprint', ${heartbeatRuns.runnerProfileJson}->'nativeToolContractFingerprint', 'nativeExecutionInput', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput', 'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint', - 'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery' + 'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery', + '${sql.raw(NATIVE_PROVIDER_STATE_UNSAVED_KEY)}', ${heartbeatRuns.runnerProfileJson}->'${sql.raw(NATIVE_PROVIDER_STATE_UNSAVED_KEY)}' )`, contextSnapshot: sql<{ paperclipWake: unknown; @@ -625,6 +640,9 @@ export function rebindNativeSessionCheckpoint(input: { // reconciled continuation then starts a fresh session in the same workspace. if (input.previousRun.errorCode === NATIVE_PROVIDER_SIDE_EFFECT_PENDING) return null; const previousProfile = record(input.previousRun.runnerProfileJson); + // The source's result was accepted, but its provider state was never saved + // and its sandbox is gone. Resuming would only fail, so start fresh now. + if (isNativeProviderStateUnsaved(previousProfile)) return null; if ( previousProfile.nativeToolContractFingerprint !== nativeToolContractFingerprintForTarget(input.executionTargetKind ?? "local", input.currentExecution.provider.kind)