From 4d554f08218b04df49ec0f2a0d5c5dfdfaf65660 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:52:38 -0600 Subject: [PATCH 1/5] test(runner): set default inputs in workflow target fixture (#19602) ## Thinking Path > - Paperclip manages work performed by AI agents. > - Its Runner support tests verify workflow authorization and target pinning. > - The workflow now passes two Boolean inputs to its shell guard. > - The target fixture runs that guard with `set -u` but omits those inputs. > - Five cases stop on an unset variable before they reach their assertions. > - This change supplies the workflow's normal false defaults in the fixture. > - The production guard and all target-pinning assertions remain unchanged. ## Linked Issues or Issue Description Refs #19537, #19576. **What happened?** The required Runner support suite fails five `workflow-target.test.ts` cases with `IMAGE_ONLY: unbound variable`. The real workflow supplies `IMAGE_ONLY` and `TESTOPS_REQUESTED`. The test fixture does not supply either value. **Expected behavior** The fixture uses the normal workflow defaults and reaches the target authorization assertions. **Steps to reproduce** Run `pnpm exec vitest run --config tests/runner-e2e/vitest.config.ts workflow-target.test.ts` on master `4938a930d66846094810cbed45d2fd33c0d14c2d`. ## What Changed - Set `IMAGE_ONLY` and `TESTOPS_REQUESTED` to `false` in the target fixture's child environment. ## Verification - RED: the unchanged current-master fixture had five failures and four passes. Bash stopped on `IMAGE_ONLY: unbound variable` before target authorization ran (220ms). - GREEN: all nine cases pass after the one-line fixture repair (168ms). Existing checks still reject a custom bounded target and preserve the queued commit for scheduled and bounded manual runs. - Local evidence uses the actual test and workflow files with installed Vitest 5.0.3 and a temporary focused config. It omits the unrelated provider-bundle global setup. No provider, external service, or paid workflow runs. - The [original required CI failure](https://github.com/paperclipai/paperclip/actions/runs/42288042001/job/128134318743) has the same five Bash failures. Its evidence is retained; no unchanged retry was used. - Whitespace checks pass. Exact head `f1d178ad483a49f8dd087a1c6d22bf43c06a39e4` has 56 successful checks, three expected skips, one neutral advisory status and successful Snyk. No check is pending or failed. The [full Runner support suite](https://github.com/paperclipai/paperclip/actions/runs/42289536842/job/128139308510), typecheck, build and [aggregate verification](https://github.com/paperclipai/paperclip/actions/runs/42289536842/job/128145908574) pass. Greptile and Paperclip Review both rate this exact head 5/5, with no findings or unresolved threads. - Environment: macOS arm64, Node 26.4.0, Vitest 5.0.3. ## Risks Low risk. This changes only fixture inputs for normal target selection. It does not change the production workflow guard, branch policy, credentials, paid execution, timeouts or assertions. Product documentation needs no update for test-only default inputs. ## Model Used OpenAI Codex, GPT-6 family, with repository analysis, code editing and tool execution. The exact runtime model ID and context window are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub references) - [x] My branch name describes the change and contains no internal Paperclip ticket id - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (not needed: fixture-only default inputs) - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- tests/runner-e2e/workflow-target.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/runner-e2e/workflow-target.test.ts b/tests/runner-e2e/workflow-target.test.ts index a4e1c57998b..d02c01d647b 100644 --- a/tests/runner-e2e/workflow-target.test.ts +++ b/tests/runner-e2e/workflow-target.test.ts @@ -16,6 +16,7 @@ async function resolveTarget(event: string, cadence: string, branch = "master") const output = path.join(directory, "output"); const result = spawnSync("bash", ["-c", `gh() { printf '%s\\n' "$MOCK_BRANCH_SHA"; }\n${script}`], { encoding: "utf8", env: { ...process.env, EVENT_NAME: event, EVAL_CADENCE: cadence, + IMAGE_ONLY: "false", TESTOPS_REQUESTED: "false", TARGET_BRANCH: branch, DEFAULT_BRANCH: "master", GITHUB_SHA: queuedSha, MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip" }, }); From 4ef3fa427194d34e0086dbb02aa49fba23b1187b Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 6 Nov 2026 13:05:37 -0800 Subject: [PATCH 2/5] docs(release): prepare v2026.1106.0 stable notes before publish (#19607) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Each stable release publishes curated release notes as its GitHub Release body. > - The stable release workflow reads `releases/beta/v2026.1023.0-beta.0.md` from master when it starts. It publishes that exact revision. > - The notes on master show v2026.1103.0 and the date 2026-11-03. That version did not ship. > - The promotion of `2026.1023.0-beta.0` now starts on 2026-11-06, so the version is `2026.1106.0`. > - This pull request sets the header to the correct version and date before the stable release starts. > - The benefit is that the release page shows the version and date that actually ship. ## Linked Issues or Issue Description Refs #18842 (the previous pre-publish fix of these notes, for v2026.1103.0). Refs #17388 (the curated notes for the `2026.1023.0-beta.0` soak). **Problem** The stable notes for the promotion of `2026.1023.0-beta.0` show `# Paperclip v2026.1103.0` and `> Released: 2026-11-03`. The release for that version did not publish. `scripts/release.sh stable --print-version --date 2026-11-06` on master gives `2026.1106.0`. **Expected result** The notes show `# Paperclip v2026.1106.0` and `> Released: 2026-11-06`. The intro paragraph names v2026.1106.0. **Impact** The stable workflow publishes these notes as the GitHub Release body. If this pull request does not merge before the release starts, the release page shows the incorrect version and date. ## What Changed - Set the H1 to `# Paperclip v2026.1106.0`. - Set the release line to `> Released: 2026-11-06`. - Set the version in the first sentence of the intro paragraph to v2026.1106.0. - No other text changes. The diff is 3 lines in one file. ## Verification - `scripts/release.sh stable --print-version --date 2026-11-06` on master `d728be8fbffc95a903ffd58bc527c7e8f1a1c704` gives `2026.1106.0`. - The file has no other `1103` or `2026-11-03` strings after the change. - The audit of the content against `v2026.1023.0..d69631b01b9c658f80bf8bed389863dce82643e1` did not change since #18842. I ran it again: - `git rev-list --count` = 515. This matches the notes. - All 255 cited PR numbers are in the range. Five cited PRs (#14180, #16356, #16401, #16403, #16976) are squash commits without a `(#N)` suffix. - `git shortlog -sn` without `github-actions[bot]` gives 27 contributors. This matches the notes. The 21 credited handles do not change. - The migrations and the defaults in the Upgrade Guide were checked in #18842. The range did not change. - The file on this branch has blob `16f05f22d4d71de3a568b493de2ddcb8eadb7859`. This is identical to the locally checked edit. ## Risks - Low risk. This is a change to documentation only, in one file. - If the UTC date changes before the release starts, the release workflow keeps `stable_date=2026-11-06` to match this header. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use (shell, git, GitHub API). Used to check the version, run the audit again, and write the change. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass (documentation only; no tests apply) - [ ] I have added or updated tests where applicable (not applicable) - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- releases/beta/v2026.1023.0-beta.0.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/releases/beta/v2026.1023.0-beta.0.md b/releases/beta/v2026.1023.0-beta.0.md index ef11a77a4b2..16f05f22d4d 100644 --- a/releases/beta/v2026.1023.0-beta.0.md +++ b/releases/beta/v2026.1023.0-beta.0.md @@ -1,8 +1,8 @@ -# Paperclip v2026.1103.0 +# Paperclip v2026.1106.0 -> Released: 2026-11-03 +> Released: 2026-11-06 -Paperclip v2026.1103.0 carries 515 commits, promoted from `2026.1023.0-beta.0`. The headline is that the Paperclip Runner is now the default for new agents: new Claude, Codex, OpenCode, and Pi agents start on the native Runner on qualified macOS and Linux hosts, existing agents get a one-click "Upgrade runner" action, and GitHub Copilot joins as a native harness. Around that: project repositories that automatically become skill sources, model-backed prompt injection prevention, agent observability to Braintrust or any OTLP endpoint, a unified Google Workspace connection, bulk edits on the Agents roster, workspaces that compact and expire themselves with encrypted restore points, and Grok Bot and Muse as experimental Runner providers. Underneath, server restarts report ready sooner, the Docker image boots from a precompiled bundle, task pages load faster, and more background services stop polling the database. +Paperclip v2026.1106.0 carries 515 commits, promoted from `2026.1023.0-beta.0`. The headline is that the Paperclip Runner is now the default for new agents: new Claude, Codex, OpenCode, and Pi agents start on the native Runner on qualified macOS and Linux hosts, existing agents get a one-click "Upgrade runner" action, and GitHub Copilot joins as a native harness. Around that: project repositories that automatically become skill sources, model-backed prompt injection prevention, agent observability to Braintrust or any OTLP endpoint, a unified Google Workspace connection, bulk edits on the Agents roster, workspaces that compact and expire themselves with encrypted restore points, and Grok Bot and Muse as experimental Runner providers. Underneath, server restarts report ready sooner, the Docker image boots from a precompiled bundle, task pages load faster, and more background services stop polling the database. ## Breaking Changes From e28158b353fd274f0322be76d62cd223e0df599f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:08:43 +0000 Subject: [PATCH 3/5] chore(lockfile): refresh pnpm lockfiles (#19617) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip uses a dependency lock for reproducible builds. > - Dependency changes on master need matching lock entries. > - Paid eval reports also pin a reviewed offline correction. > - Reports must install before paid model work starts. > - This automated PR keeps the lock and report proof together. ## Linked Issues or Issue Description **What happened?** Dependency manifests on master changed after the last committed lock refresh. **Expected behavior** Frozen installs must match those manifests. Eval reports must retain matching offline lock proof. **Steps to reproduce** Run a frozen install before and after this lock refresh. ## What Changed - Refresh the dependency lock from the manifests on master. - Refresh the sandbox provider lockfiles from their manifests on master. - Include the matching offline report correction and hashes. ## Verification - Pinned pnpm resolution completed with lifecycle scripts disabled. - The report proof refresh accepted the reviewed committed baseline. - The workflow rejected changes outside the lock and report proof files. ## Risks The lockfile bot merges this PR without CI or review when every added package version is at least 3 days old on npm. Otherwise the PR waits for a maintainer, and the daily refresh tries again. The bot merges only the exact commit this workflow generated from manifests already reviewed on master. Unknown report lock baselines remain blocked. This update changes no permissions or paid enable gates. ## Model Used None — generated by pnpm and the deterministic report-lock updater. ## Checklist - [x] Thinking path and problem description included - [x] Generated lock and matching report proof included - [x] Release-age gate decides between bot merge and maintainer review Co-authored-by: lockfile-bot --- .../sandbox-providers/daytona/pnpm-lock.yaml | 4 +++- pnpm-lock.yaml | 4 +++- tests/runner-e2e/report-lock.json | 6 ++--- tests/runner-e2e/report-lock.patch | 23 ++++++++++--------- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml b/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml index c4483ecd951..38f45f94b93 100644 --- a/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml +++ b/packages/plugins/sandbox-providers/daytona/pnpm-lock.yaml @@ -483,7 +483,9 @@ packages: '@smithy/core@3.33.3': resolution: {integrity: sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==} engines: {node: '>=18.0.0'} - deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. + deprecated: |- + Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this + version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393. '@smithy/credential-provider-imds@4.5.2': resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a87f8e749f5..97cbf7b33af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5048,7 +5048,9 @@ packages: '@smithy/core@3.35.1': resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==} engines: {node: '>=18.0.0'} - deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. + deprecated: |- + Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this + version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393. '@smithy/credential-provider-imds@4.5.2': resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} diff --git a/tests/runner-e2e/report-lock.json b/tests/runner-e2e/report-lock.json index bd6c65c89b8..d6740c79df5 100644 --- a/tests/runner-e2e/report-lock.json +++ b/tests/runner-e2e/report-lock.json @@ -1,6 +1,6 @@ { - "inputSha256": "14049cb20d48f448b6f24e5445a75bf8d22ed3052fe9ff885f4c127775d958d3", - "outputSha256": "53f6c7c7f09a5cf0442ee2992a656eb2336ab6b13e665219a0cd1247475f13d9", - "patchSha256": "eb4adb2f3c8a2ebefe65d3100b581c5765318023d96aa8a80394c848d7299b9d", + "inputSha256": "53f6c7c7f09a5cf0442ee2992a656eb2336ab6b13e665219a0cd1247475f13d9", + "outputSha256": "5e2b012b50bb8b198384da4f5427c27acd948af405f89cde15bfa48bb10c0ad4", + "patchSha256": "2abdd6ec8e374bfb20b7e168ef010858493a50a044360175e1af5a2ca393f909", "basis": "Generated with the trusted master lockfile refresh. Review this offline correction and its hashes with the dependency changes." } diff --git a/tests/runner-e2e/report-lock.patch b/tests/runner-e2e/report-lock.patch index 6e5bb651368..090ee867256 100644 --- a/tests/runner-e2e/report-lock.patch +++ b/tests/runner-e2e/report-lock.patch @@ -1,14 +1,15 @@ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml -index 45b3aaf..a87f8e7 100644 +index a87f8e7..97cbf7b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml -@@ -1306,6 +1306,9 @@ importers: - lucide-react: - specifier: ^1.48.0 - version: 1.48.0(react@19.2.8) -+ mdast-util-from-markdown: -+ specifier: ^2.0.3 -+ version: 2.0.3 - mermaid: - specifier: ^12.1.0 - version: 12.1.0 +@@ -5048,7 +5048,9 @@ packages: + '@smithy/core@3.35.1': + resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==} + engines: {node: '>=18.0.0'} +- deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. ++ deprecated: |- ++ Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this ++ version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393. + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} From 586caa5f4935c0d79eed5c1a4456ce7fe171b1fc Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:10:44 -0600 Subject: [PATCH 4/5] fix(native): hand a shutdown-interrupted startup to the next controller (#19596) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The native Runner path runs an agent turn through a runnerd process that the control plane owns > - A restart of the control plane must keep the task going without a click, if that is safe > - A graceful shutdown closes the HTTP listener first and then drains the heartbeat runs > - A native run that was between provider dispatch and runnerd backend setup in that gap failed with `local_runtime_api_listener_unavailable`, so the drain did not see it as running and did not suspend it for the next boot > - This pull request makes that failure a restart detach: the run stays running with its claim, the drain suspends it, and the next controller adopts it > - The benefit is that a SIGTERM in this window no longer spends a failure and a same-run retry, and the task continues by itself ## Linked Issues or Issue Description Refs #19530 (the same restart window, before provider dispatch). **What happened?** In a restart chaos replay (ACPX Claude, local runner), a restart-recovered run got SIGTERM while `createRunnerdBackendWithinSessionClaim` built the runnerd backend. `resolveLocalRuntimeApiUrl()` threw `local_runtime_api_listener_unavailable` because shutdown had already closed the listener. The run settled `failed`. The drain then logged `restartSuspendedRunIds: []`. The next boot used the one bootstrap retry. A second SIGTERM hit the same window, and the task stopped at a manual Retry. **Expected behavior** A run that this process cannot continue because it is shutting down is handed to the next controller. It is not settled as failed. **Steps to reproduce** 1. Start a native Runner task and restart the server (K1) so that startup recovery resumes the run. 2. Send SIGTERM while the resumed run sets up its runnerd backend. 3. See the run fail with `local_runtime_api_listener_unavailable` and the drain report no suspended runs. ## What Changed - `server/src/runtime-api.ts`: `resolveLocalRuntimeApiUrl()` throws the new `LocalRuntimeApiListenerClosedError` when this process had a listener and closed it (the `listening` flag is false or the `close` event fired). The error message stays `local_runtime_api_listener_unavailable`, so the existing classifiers do not change. A listener that never started still throws the plain error. - `server/src/services/native-runtime/native-session-executor.ts`: for a local Runner, the startup failure handler treats `LocalRuntimeApiListenerClosedError` like a restart detach. It stops lease renewal, keeps the claim and throws `NativeControllerDetachedForRestartError`. Heartbeat already keeps such a run `running`. A remote Runner keeps the same-run bootstrap retry from #19462, because startup recovery has no process evidence to adopt a claimed remote startup (`native-runner-restart-preparation.integration.test.ts` still covers that path). - `doc/execution-semantics.md`: describe the new rule. - Tests: a heartbeat regression that closes the real runtime API listener before backend setup, then drains and restarts; a unit test for the closed versus never-started listener; two `vi.mock` factories keep the real module exports. ## Verification - Red before the fix: `pnpm exec vitest run src/__tests__/heartbeat-native-restart-suspension.test.ts -t listener` failed with `restartSuspendedRunIds: []` and the run `failed` (`local_runtime_api_listener_unavailable`, `bootstrap_retry`). - Green after the fix: the same test passes. The drain suspends the run, the run stays `running` with `awaiting_runner_reattach`, and the next controller claims it and reaches the backend once. - `pnpm exec vitest run src/__tests__/heartbeat-native-restart-suspension.test.ts src/http/server.test.ts src/__tests__/server-startup-feedback-export.test.ts src/services/native-runtime/native-session-executor.test.ts src/services/native-runtime/native-recovered-owner-followup.test.ts`: 1333 tests pass. One timing test in `native-session-executor.test.ts` failed once under host load average ~30 and passed alone. - `pnpm exec tsc --noEmit -p server/tsconfig.json`: pass. - Live, restart chaos harness, local ACPX Claude (`claude-sonnet-5`), `--rc06-masked`, this branch at a35ec8b845 (before the remote-scope commit 5dc63f24fb, which does not change the local path). The original replay schedule no longer reached the window, because the task now ends before the second fault. So I moved the faults earlier: K1 at 10 s, then K2 as soon as the recovered server is ready. - K1 + K2, 2 runs: both runs PASS with **R1 pass**. Each drain logged `restartSuspendedRunIds: []` (before: `[]`). The next boot classified the run `resume_dead_runner` and the same run completed. No `local_runtime_api_listener_unavailable` failure. - K1 + K2 + K2, 2 runs: each drain suspended the run. The fourth boot blocked the run with `native_restart_recovery_blocked`, because three restart claims used the three-attempt recovery budget. Retry then completed the task (R3 pass). That budget count is a separate open item. This pull request does not change it. ## Risks - Low. The new path applies only after this process closed its own listener, which happens only during shutdown. The run then waits for the drain or the next boot, as for the other restart detaches. - If shutdown does not reach the drain (for example SIGKILL after the listener closes), the run keeps its live claim until the claim expires. Then startup recovery handles it as a hard restart. This is the same as any run that is running at a hard kill. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, through Claude Code with tool use (shell, file edits, GitHub CLI). Extended reasoning on. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 7 +++ ...eartbeat-native-restart-suspension.test.ts | 51 +++++++++++++++++++ .../src/__tests__/helpers/native-restart.ts | 2 +- server/src/http/server.test.ts | 26 +++++++++- server/src/runtime-api.ts | 38 ++++++++++++-- .../native-recovered-owner-followup.test.ts | 2 +- .../native-session-executor.test.ts | 2 +- .../native-runtime/native-session-executor.ts | 11 +++- tests/runner-e2e/workflow-target.test.ts | 3 +- 9 files changed, 131 insertions(+), 11 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 986c65cbbf5..5c71fd55f82 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3317,6 +3317,13 @@ A graceful shutdown can also suspend a restart-recovery claim whose execution has not yet reached provider dispatch. That execution has lost its authority: it leaves the run running, its budget hold held and its workspace unmarked, and the next controller adopts the same run. It does not settle the run as failed. +Shutdown closes the runtime API listener before it drains runs. A local Runner +execution that then needs the listener (for example, to give a new runner its +callback origin) detaches the same way instead of failing. It keeps its claim, +the drain suspends it, and the next controller adopts the run. A remote Runner +execution has no process evidence on the controller host, so it keeps the +same-run bootstrap retry. A listener that never started is still a startup +failure. Remote Runner processes use their authenticated connection lease as the bound on disconnected lifetime. The controller's shorter connection-recovery budget limits its own attempt; it must not terminate the remote owner while another diff --git a/server/src/__tests__/heartbeat-native-restart-suspension.test.ts b/server/src/__tests__/heartbeat-native-restart-suspension.test.ts index 892cb063735..654f152bcf3 100644 --- a/server/src/__tests__/heartbeat-native-restart-suspension.test.ts +++ b/server/src/__tests__/heartbeat-native-restart-suspension.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { authUsers, budgetReservations, companyMemberships, completionContracts, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations } from "@paperclipai/db"; import { expect, it, vi } from "vitest"; +import { resolveLocalRuntimeApiUrl } from "../runtime-api.js"; import { heartbeatService } from "../services/heartbeat.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { currentNativeControllerIdentity } from "../services/native-runtime/native-restart-recovery.js"; @@ -77,6 +78,56 @@ describeEmbeddedPostgres("heartbeat native recovery interrupted by graceful shut 90_000, ); + it("hands a recovery to the next controller when shutdown closed the listener before backend setup", async () => { + const f = await seedRecoverableRun("LISTENER-CLOSED"); + const listener = fixture.current.server!; + const address = listener.address(); + if (!address || typeof address === "string") throw new Error("Expected a TCP runtime API listener"); + let listenerClosed: Promise | undefined; + // Graceful shutdown closes HTTP admission before it drains heartbeat runs. + // Backend setup then asks for the callback origin, as runnerd setup does. + const dying = vi.fn(() => ({ + descriptor: async () => { resolveLocalRuntimeApiUrl(); throw new Error("unreachable"); }, + openSession: async () => { resolveLocalRuntimeApiUrl(); throw new Error("unreachable"); }, + })); + const heartbeat = heartbeatService(f.db, { + nativeSessionBackendFactory: dying as never, + beforeNativeRuntimeSelection: async () => { + listener.closeIdleConnections(); + listenerClosed = new Promise(resolve => listener.close(() => resolve())); + }, + }); + let drained: unknown; + try { + try { + const { claims } = await heartbeat.recoverNativeRunsAfterRestart(new Date(), [f.runId]); + expect(claims).toEqual([expect.objectContaining({ runId: f.runId })]); + } finally { await heartbeat.drainActiveRunExecutions(); } + expect(dying).toHaveBeenCalled(); + drained = await heartbeat.drainRunningRunsForShutdown("SIGTERM", new Date(), [f.runId]); + await heartbeat.drainActiveRunExecutions(); + } finally { + await listenerClosed; + await new Promise(resolve => listener.listen(address.port, "127.0.0.1", resolve)); + } + expect(drained).toMatchObject({ restartSuspendedRunIds: [f.runId] }); + const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [owner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect({ status: run.status, errorCode: run.errorCode, finishedAt: run.finishedAt }).toEqual({ + status: "running", errorCode: null, finishedAt: null }); + expect(owner).toMatchObject({ phase: "observed", resultId: null, recoveryState: "awaiting_runner_reattach" }); + + await f.db.update(nativeRunFinalizations).set({ controllerPid: 2_000_000_001 }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const resumed = vi.fn(() => { throw new Error("next boot reached provider dispatch"); }); + const next = heartbeatService(f.db, { nativeSessionBackendFactory: resumed }); + try { + const { claims } = await next.recoverNativeRunsAfterRestart(new Date(), [f.runId]); + expect(claims).toEqual([expect.objectContaining({ runId: f.runId })]); + } finally { await next.drainActiveRunExecutions(); } + expect(resumed).toHaveBeenCalledOnce(); + }, 90_000); + it("still settles a recovery whose claim merely expired without a shutdown suspension", async () => { const f = await seedRecoverableRun("EXPIRED-NOT-SUSPENDED"); const dying = vi.fn(() => { throw new Error("An expired claim must not open a provider session"); }); diff --git a/server/src/__tests__/helpers/native-restart.ts b/server/src/__tests__/helpers/native-restart.ts index deccd592af1..8d642f4ca12 100644 --- a/server/src/__tests__/helpers/native-restart.ts +++ b/server/src/__tests__/helpers/native-restart.ts @@ -244,7 +244,7 @@ export function nativeRestartFixture() { }; } - return { seedRun, transportOptions, get current() { return { temporary, runtimeRoot, paperclipHome, apiUrl, companyId, agentId, successor }; } }; + return { seedRun, transportOptions, get current() { return { temporary, runtimeRoot, paperclipHome, apiUrl, companyId, agentId, successor, server }; } }; } export function remotePreparedWorkspace() { diff --git a/server/src/http/server.test.ts b/server/src/http/server.test.ts index b548858d23f..c4616ec6751 100644 --- a/server/src/http/server.test.ts +++ b/server/src/http/server.test.ts @@ -8,7 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import express from "express"; import { DurablePrpControlPlane } from "../vendor/paperclip-runner/index.js"; import { registerRunnerPrpAuthority, runnerPrpWebSocketInternals } from "../realtime/runner-prp-ws.js"; -import { resolveLocalRuntimeApiUrl } from "../runtime-api.js"; +import { LocalRuntimeApiListenerClosedError, resolveLocalRuntimeApiUrl } from "../runtime-api.js"; import { createPaperclipHttpServer } from "./server.js"; import { privateHostnameGuard } from "../middleware/private-hostname-guard.js"; @@ -206,6 +206,30 @@ describe("Paperclip HTTP server runner transport", () => { } }); + it("distinguishes a listener closed for shutdown from one that never served", async () => { + expect(() => resolveLocalRuntimeApiUrl()).toThrow("local_runtime_api_listener_unavailable"); + expect(() => resolveLocalRuntimeApiUrl()).not.toThrow(LocalRuntimeApiListenerClosedError); + const server = createPaperclipHttpServer((_req, res) => res.end("closing"), { apiUrl: "http://ignored.example.test" }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const client = await fetch(resolveLocalRuntimeApiUrl(), { keepalive: true }); + // `close()` stops admission at once; its close event waits for open requests. + const closed = new Promise(resolve => server.close(() => resolve())); + expect(() => resolveLocalRuntimeApiUrl()).toThrow(LocalRuntimeApiListenerClosedError); + await client.text(); + server.closeAllConnections(); + await closed; + expect(() => resolveLocalRuntimeApiUrl()).toThrow(LocalRuntimeApiListenerClosedError); + expect(() => resolveLocalRuntimeApiUrl()).toThrow("local_runtime_api_listener_unavailable"); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + try { + expect(resolveLocalRuntimeApiUrl()).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + } finally { + await new Promise(resolve => server.close(() => resolve())); + } + }); + it.each(["127.0.0.1", "::1", "::ffff:127.0.0.1"])("routes a guarded native read through the real %s listener", async host => { await guardedNativeRead(host); }); diff --git a/server/src/runtime-api.ts b/server/src/runtime-api.ts index 4c4f87868a6..b7d1a3d64f4 100644 --- a/server/src/runtime-api.ts +++ b/server/src/runtime-api.ts @@ -4,15 +4,37 @@ import { isIP } from "node:net"; const runtimeApiListeners = new Set(); const registeredRuntimeApiListeners = new WeakSet(); +let runtimeApiListenerClosed = false; + +/** + * The process had a runtime API listener and has since closed it, which only + * happens while it shuts down. Work that needs a callback origin must hand off + * to the next controller instead of failing as a provider or startup error. + */ +export class LocalRuntimeApiListenerClosedError extends Error { + constructor() { + // Keep the established failure code for classifiers and persisted rows. + super("local_runtime_api_listener_unavailable"); + this.name = "LocalRuntimeApiListenerClosedError"; + } +} /** Track listener ownership without treating configured or advertised URLs as authority. */ export function registerLocalRuntimeApiServer(server: Server): void { if (!registeredRuntimeApiListeners.has(server)) { registeredRuntimeApiListeners.add(server); - server.on("listening", () => runtimeApiListeners.add(server)); - server.on("close", () => runtimeApiListeners.delete(server)); + server.on("listening", () => { + runtimeApiListeners.add(server); + runtimeApiListenerClosed = false; + }); + server.on("close", () => { + if (runtimeApiListeners.delete(server)) runtimeApiListenerClosed = true; + }); + } + if (server.listening) { + runtimeApiListeners.add(server); + runtimeApiListenerClosed = false; } - if (server.listening) runtimeApiListeners.add(server); } function normalizeHost(value: string | null | undefined): string { @@ -94,6 +116,11 @@ export function chooseLocalRuntimeApiUrl(input: { export function resolveLocalRuntimeApiUrl(): string { const listeners = [...runtimeApiListeners].filter(server => server.listening); + // `server.close()` clears `listening` at once; its close event waits for + // admitted requests. Either state means this process stopped serving. + if (listeners.length === 0 && (runtimeApiListenerClosed || runtimeApiListeners.size > 0)) { + throw new LocalRuntimeApiListenerClosedError(); + } if (listeners.length !== 1) { throw new Error(listeners.length ? "local_runtime_api_listener_ambiguous" : "local_runtime_api_listener_unavailable"); } @@ -103,7 +130,10 @@ export function resolveLocalRuntimeApiUrl(): string { } export const runtimeApiInternals = { - resetForTests(): void { runtimeApiListeners.clear(); }, + resetForTests(): void { + runtimeApiListeners.clear(); + runtimeApiListenerClosed = false; + }, }; export function choosePrimaryRuntimeApiUrl(input: { diff --git a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts index f12aad1871f..8185661619d 100644 --- a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts +++ b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts @@ -25,7 +25,7 @@ import { import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; const toolDefinitions = vi.hoisted(() => vi.fn()); -vi.mock("../../runtime-api.js", () => ({ resolveLocalRuntimeApiUrl: () => "http://127.0.0.1:43219" })); +vi.mock("../../runtime-api.js", async importOriginal => ({ ...await importOriginal(), resolveLocalRuntimeApiUrl: () => "http://127.0.0.1:43219" })); vi.mock("./paperclip-runner-tool-authority.js", () => ({ PaperclipRunnerToolAuthority: class { readonly binding = {}; diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c568d5350f2..dd9be079322 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -275,7 +275,7 @@ const state = vi.hoisted(() => ({ release: null as null | (() => void), })); -vi.mock("../../runtime-api.js", () => ({ resolveLocalRuntimeApiUrl: state.localRuntimeApiUrl })); +vi.mock("../../runtime-api.js", async importOriginal => ({ ...await importOriginal(), resolveLocalRuntimeApiUrl: state.localRuntimeApiUrl })); const accountingEvents = vi.hoisted(() => ({ committed: undefined as undefined | ((event: PrpEvent) => Promise), duplicate: undefined as undefined | ((event: PrpEvent) => Promise) })); vi.mock("./paperclip-control-plane-port.js", async importOriginal => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c7fddc8388b..fd1216fe57c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5,7 +5,7 @@ import { logger } from "../../middleware/logger.js"; import { canRestoreMigratedNativeSession } from "./native-workspace-migration.js"; import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES, nativeHarnessBackupManifestSha256, retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js"; import { prepareCursorCloudRuntime } from "./cursor-cloud-runtime.js"; -import { resolveLocalRuntimeApiUrl } from "../../runtime-api.js"; +import { LocalRuntimeApiListenerClosedError, resolveLocalRuntimeApiUrl } from "../../runtime-api.js"; import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; import { readProviderFailure } from "@paperclipai/shared"; import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForNativeCancellation } from "./native-cancellation-delivery.js"; @@ -10676,7 +10676,14 @@ async function executePaperclipNativeSessionWithinScope( } else { error = identityRedactor.redact(error); } - if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) { + // A closed runtime API listener means this process is shutting down; it + // closes HTTP admission before it drains runs. For a local Runner, keep the + // claim for the drain to suspend: the next controller proves the Runner + // process identity and adopts the run without spending a failure or a + // same-run retry. A remote Runner has no process evidence on this host, so + // it keeps the bootstrap retry below. + if (nativeRunsDetachingForRestart.has(input.execution.binding.runId) + || (error instanceof LocalRuntimeApiListenerClosedError && input.runnerExecutionTarget?.kind !== "remote")) { await leaseRenewal.stop().catch(() => undefined); liveQuestions.close(); activeNativeSessions.delete(input.execution.binding.runId); diff --git a/tests/runner-e2e/workflow-target.test.ts b/tests/runner-e2e/workflow-target.test.ts index d02c01d647b..7f7aab50686 100644 --- a/tests/runner-e2e/workflow-target.test.ts +++ b/tests/runner-e2e/workflow-target.test.ts @@ -18,7 +18,8 @@ async function resolveTarget(event: string, cadence: string, branch = "master") encoding: "utf8", env: { ...process.env, EVENT_NAME: event, EVAL_CADENCE: cadence, IMAGE_ONLY: "false", TESTOPS_REQUESTED: "false", TARGET_BRANCH: branch, DEFAULT_BRANCH: "master", GITHUB_SHA: queuedSha, - MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip" }, + MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip", + IMAGE_ONLY: "false", TESTOPS_REQUESTED: "false" }, }); return { status: result.status, stderr: result.stderr, output: await readFile(output, "utf8").catch(() => "") }; From da9a4cb96c46945d693f0ddffdcb0e0743649a01 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:11:00 -0600 Subject: [PATCH 5/5] fix(native): retire a finished local Runner that stopped before suspending (#19597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The native Runner path runs each per-turn agent turn through a runnerd process, and the next turn must prove that the previous Runner is finished before it takes over the saved session root > - Under load, a local turn can have its result accepted while its `runner.suspend` command does not finish, so the Runner's own receipt stays unsuspended > - The retirement proof that handles the same case for sandbox Runners refused every local Runner with `target_not_sandbox_runner` > - So the next turn failed with `runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate` and its session root was quarantined > - This pull request extends the retirement to a local Runner when the prior Runner process is proven stopped by exact PID and start time > - The benefit is that a follow-up turn (for example, after a user answers a question) continues instead of failing ## Linked Issues or Issue Description Refs #19525 (keeps the accepted result in the same incident; this is the next run). **What happened?** In the runner E2E cell `local-session-integrity.runner-acpx-claude.local.structured-question-restart-resume`, the first turn asked a question and its result was accepted. The Runner then logged `runner did not prove durable suspension before checkpoint` (`runner.suspend` pending, Runner lifecycle `ready`). After a server restart and the answer, the next run failed with `runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate; retirement_refused=target_not_sandbox_runner`, and its root was quarantined. **Expected behavior** When the prior local Runner is provably stopped and every other retirement proof holds, the next run retires that owner as a continuity break and continues. If the stop cannot be proven, the run keeps the named rejection. **Steps to reproduce** 1. Run a local per-turn native turn whose `runner.suspend` does not finish after its result is accepted (host load average ~40). 2. Start the next turn for the same session. 3. See `retirement_refused=target_not_sandbox_runner`. ## What Changed - `server/src/services/native-runtime/native-restart-recovery.ts`: add `proveNativeLocalRunnerStopped`. It uses the existing restart-recovery checks: the recorded Runner PID is dead or now belongs to a process with a different start time; its recorded process group is not alive; each recorded provider process is dead or recycled. A missing PID, start time or process group, a live recorded process, or a start time that cannot be compared is not proof. When settlement already cleared the run's process fields, the saved `native.local_process_stopped` receipt is the Runner proof. Extract `nativeProviderProcessIdentities` (checkpoint plus run events) so restart recovery and this proof read provider PIDs the same way. - `server/src/services/native-runtime/native-session-executor.ts` (`retireFinishedPerTurnRunnerdOwner`, `readSettledRunnerdProjection`): a local target is now eligible. For a local Runner, the projection accepts the Runner's own unsuspended receipt for the same identity and unfinished Runner lifecycle commands (`runner.suspend`, `runner.drain`, `runner.shutdown`) only. All other proofs are unchanged: committed result, released lease, no recovery state, stopped controller, delivered terminal, other commands complete, tool inputs answered, quarantine copies. The process proof runs before retirement and again just before the archive. - `doc/execution-semantics.md`: describe the local case. - Tests: seven integration cases in `native-recovered-owner-followup.test.ts` and ten unit cases in `native-restart-recovery.test.ts`. ## Verification - Red before the fix: the four new integration cases failed with `retirement_refused=target_not_sandbox_runner`, which is the incident error. - Green after the fix: `pnpm exec vitest run src/services/native-runtime/native-recovered-owner-followup.test.ts` 35/35. The new cases prove that a stopped Runner is retired (root archived under `continuity-breaks`, no quarantine), and that a live recorded Runner (`runner_process_alive`), a missing identity (`runner_process_identity_missing`), and a pending non-lifecycle command (`journal_commands_unsettled(turn.start.pending)`) keep the rejection. All existing sandbox cases still pass, including "keeps an unsuspended Runner receipt decisive" for sandbox targets. - `native-restart-recovery.test.ts` unit cases: recycled PID, live process, unknown start time, live group, live provider, missing PID or start time. - `native-session-executor.test.ts` + `native-restart-recovery.test.ts`: 1190 pass. `native-runner-restart-recovery.integration.test.ts` + `native-restart-recovery-isolation.test.ts` (shared provider-PID extraction): 81 pass. `tsc --noEmit -p server/tsconfig.json`: pass. - Live: `pnpm test:e2e:runner -- --max-parallel 1 --max-automatic-retries 0 --id local-session-integrity.runner-acpx-claude.local.structured-question-restart-resume`, 3 runs at eec57dd5ed: 3/3 PASS. The host load average was ~15 (the incident had ~40), and `runner.suspend` finished in all three runs, so these runs show no regression but did not exercise the new retirement path. The integration cases cover that path. ## Risks - The new path lets a follow-up archive a local root whose Runner receipt is unsuspended. This is safe only if the Runner is stopped, so the proof uses exact PID and start-time identity and fails closed on any uncertainty. `ps` start times have one-second precision; a sub-second difference is treated as the same process (not proof). - A zombie-only process group on macOS can report as alive. That keeps the rejection, as before. - No quarantine behavior changes. A refused retirement still quarantines the root as before. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, through Claude Code with tool use (shell, file edits, GitHub CLI). Extended reasoning on. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 16 ++ .../native-recovered-owner-followup.test.ts | 104 +++++++++- .../native-restart-recovery.test.ts | 40 ++++ .../native-runtime/native-restart-recovery.ts | 183 +++++++++++------- .../native-runtime/native-session-executor.ts | 78 ++++++-- 5 files changed, 338 insertions(+), 83 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 5c71fd55f82..e5451eb7ace 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2379,6 +2379,22 @@ All of these must be true: - Each quarantined root for the scope passes the same journal and local Runner evidence checks for the same owner. +A local per-turn Runner can end the same way: the result was accepted, but its +`runner.suspend` command did not finish, so its own receipt is still +unsuspended. The next local run retires that owner with the same checks and +these differences: + +- The Runner's own receipt may be unsuspended. It may not be unreadable or + name another owner. +- Only the Runner's own lifecycle commands (`runner.suspend`, `runner.drain`, + `runner.shutdown`) may be unfinished. All other commands must be complete. +- The prior run's recorded Runner process is proven stopped by exact PID and + start time, as in restart recovery. A PID that now belongs to another + process counts as stopped. A live recorded process, a live process group, a + live recorded provider process, a missing identity, or a start time that + cannot be compared keeps the rejection, for example + `retirement_refused=runner_process_alive`. A later Retry checks again. + The provider session then follows the normal resume-or-replace policy. A resumed sandbox, a warm session, a plugin task, a restart claim, or any missing proof keeps the existing `prior_owner_terminal_state_indeterminate` diff --git a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts index 8185661619d..e969b20d0c2 100644 --- a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts +++ b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts @@ -185,7 +185,11 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered runner: { execute: vi.fn(async () => { throw new Error("admission must not reach the sandbox"); }) }, sandboxLeaseAcquisition: { outcome, providerLeaseId: `sandbox-${runId}` } } as never, }).then(() => "constructed", (error: Error) => error.message); - return { runId, logs, start, selectedRunId: selected?.id ?? null }; + // A local Runner keeps its controller root on this host between turns. + const startLocal = () => createRunnerdBackend({ + db, execution, runnerInstanceId: next.runnerInstanceId, onLog: async (_stream, chunk) => { logs.push(chunk); }, + }).then(() => "constructed", (error: Error) => error.message); + return { runId, logs, start, startLocal, selectedRunId: selected?.id ?? null }; } async function quarantined() { @@ -465,6 +469,104 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered expect(followUp.logs.join("")).not.toContain("finished without a saved runner checkpoint"); }, 60_000); + /** A spawned process that stands in for the prior local Runner. */ + async function runnerProcess() { + const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60_000)"], { stdio: "ignore" }); + await new Promise(resolve => child.once("spawn", resolve)); + const startedAt = await readProcessStartedAt(child.pid!); + expect(startedAt).toBeTruthy(); + return { child, pid: child.pid!, processStartedAt: new Date(startedAt!) }; + } + + async function stoppedRunnerProcess() { + const runner = await runnerProcess(); + const exited = new Promise(resolve => runner.child.once("exit", resolve)); + runner.child.kill("SIGKILL"); + await exited; + return runner; + } + + /** The state a local per-turn Runner leaves when its result was accepted but + * its suspend command never finished: the controller journal holds a pending + * runner.suspend and the Runner's own receipt is still `ready`. */ + async function seedUnsuspendedLocalOwner(runner: { pid: number; processStartedAt: Date }) { + const prior = await seedRecoveredPriorRun("pending-command"); + await db.update(heartbeatRuns).set({ processPid: runner.pid, processGroupId: runner.pid, processStartedAt: runner.processStartedAt }) + .where(eq(heartbeatRuns.id, prior.runId)); + const followUp = await admitFollowUp(prior); + // Learn the scope key from a held attempt, then keep its root in place. + expect(await followUp.start()).toMatch(/^runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate/); + const [copy] = await quarantined(); + const root = join(stateDirectory, copy!.split(".")[0]!); + await rename(join(stateDirectory, "quarantine", copy!), root); + await rm(join(stateDirectory, "quarantine"), { recursive: true, force: true }); + await mkdir(join(root, "runner"), { recursive: true }); + await writeFile(join(root, "runner", "runner-state.json"), JSON.stringify({ + schema: "paperclip.runner.durable.state.v1", lifecycle: "ready", ...prior.identity })); + return { prior, followUp, root }; + } + + it("retires a finished local owner whose Runner stopped before it proved suspension", async () => { + const { prior, followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + expect(await followUp.startLocal()).toBe(ADMITTED); + expect(await quarantined()).toEqual([]); + expect(existsSync(join(root, "runner"))).toBe(false); + expect(await archivedOwners(root)).toEqual([prior.runId]); + expect(followUp.logs.join("")).toContain(`Run ${prior.runId} finished without a saved runner checkpoint`); + }, 60_000); + + it("keeps a finished local owner held while its recorded Runner process is alive", async () => { + const runner = await runnerProcess(); + try { + const { followUp } = await seedUnsuspendedLocalOwner(runner); + expect(await followUp.startLocal()).toBe(stranded("runner_process_alive")); + expect(followUp.logs.join("")).not.toContain("finished without a saved runner checkpoint"); + } finally { + runner.child.kill("SIGKILL"); + } + }, 60_000); + + it.each(["no identity", "no process group"] as const)("keeps a finished local owner held with %s", async scenario => { + const { prior, followUp } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + await db.update(heartbeatRuns).set(scenario === "no identity" + ? { processPid: null, processGroupId: null, processStartedAt: null } : { processGroupId: null }) + .where(eq(heartbeatRuns.id, prior.runId)); + expect(await followUp.startLocal()).toBe(stranded("runner_process_identity_missing")); + }, 60_000); + + it("retires a finished local owner from its saved stop receipt after settlement cleared the process fields", async () => { + const runner = await stoppedRunnerProcess(); + const { prior, followUp, root } = await seedUnsuspendedLocalOwner(runner); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: prior.runId, seq: 900, eventType: "native.local_process_stopped", + stream: "system", payload: { processPid: runner.pid, processGroupId: runner.pid, processStartedAt: runner.processStartedAt.toISOString() } }); + await db.update(heartbeatRuns).set({ processPid: null, processGroupId: null, processStartedAt: null }) + .where(eq(heartbeatRuns.id, prior.runId)); + expect(await followUp.startLocal()).toBe(ADMITTED); + expect(await archivedOwners(root)).toEqual([prior.runId]); + }, 60_000); + + it("keeps a stopped local owner held while a provider process from its run events is alive", async () => { + const provider = await runnerProcess(); + try { + const { prior, followUp } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: prior.runId, seq: 901, eventType: "provider.event", + stream: "system", payload: { prpEvent: { payload: { processId: provider.pid, + processStartedAt: provider.processStartedAt.toISOString() } } } }); + expect(await followUp.startLocal()).toBe(stranded("provider_process_alive")); + } finally { + provider.child.kill("SIGKILL"); + } + }, 60_000); + + it("keeps a stopped local owner held by an unfinished non-lifecycle command", async () => { + const { followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + const path = join(root, "control-plane", "control-plane-state.json"); + const state = JSON.parse(await readFile(path, "utf8")); + state.commands.find((command: { type: string }) => command.type === "turn.start").status = "pending"; + await writeFile(path, JSON.stringify(state)); + expect(await followUp.startLocal()).toBe(stranded("journal_commands_unsettled(turn.start.pending)")); + }, 60_000); + it.each(["uncommitted result", "committed without result", "live controller lease", "unexpired controller lease", "recovery pending", "pending command", "unanswered tool input", "foreign quarantine"] as const)("keeps the prior owner held with %s", async scenario => { const prior = await seedRecoveredPriorRun(scenario === "pending command" ? "pending-command" diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index fd79245b6e5..52d8d15c9b7 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -5,8 +5,48 @@ import { evaluateNativeControllerTakeover, evaluateNativeProviderProcesses, nextNativeProviderAttempt, + proveNativeLocalRunnerStopped, } from "./native-restart-recovery.js"; +describe("local runner stop proof", () => { + const startedAt = new Date("2026-11-06T17:58:54.000Z"); + const base = { processPid: 72454, processGroupId: 72454, processStartedAt: startedAt }; + const probe = (alive: number[], starts: Record, groups: number[] = []) => ({ + isProcessAlive: (pid: number) => alive.includes(pid), + isProcessGroupAlive: (group: number) => groups.includes(group), + readProcessStartedAt: async (pid: number) => starts[pid] ?? null, + }); + + it.each([ + ["the runner and its group exited", base, probe([], {}), { stopped: true }], + ["the runner PID now belongs to another process", base, + probe([72454], { 72454: new Date(startedAt.getTime() + 60_000) }), { stopped: true }], + ["the recorded runner is alive", base, probe([72454], { 72454: startedAt }), + { stopped: false, reason: "runner_process_alive" }], + ["the live PID start time is unknown", base, probe([72454], {}), + { stopped: false, reason: "runner_process_identity_unverifiable" }], + ["the runner group is alive", base, probe([], {}, [72454]), + { stopped: false, reason: "runner_process_group_alive" }], + ["no runner PID was recorded", { ...base, processPid: null }, probe([], {}), + { stopped: false, reason: "runner_process_identity_missing" }], + ["no runner start time was recorded", { ...base, processStartedAt: null }, probe([], {}), + { stopped: false, reason: "runner_process_identity_missing" }], + ["no runner process group was recorded", { ...base, processGroupId: null }, probe([], {}), + { stopped: false, reason: "runner_process_identity_missing" }], + ["a stop receipt proved the runner absent", { ...base, processStartedAt: null, runnerStopRecorded: true }, + probe([], {}), { stopped: true }], + ] as const)("decides when %s", async (_case, identity, checks, expected) => { + await expect(proveNativeLocalRunnerStopped({ ...identity, ...checks })).resolves.toEqual(expected); + }); + + it("refuses while a recorded provider process is alive", async () => { + const providerStartedAt = new Date("2026-11-06T17:58:56.000Z"); + await expect(proveNativeLocalRunnerStopped({ ...base, + providerProcesses: [{ pid: 72460, processStartedAt: providerStartedAt }], + ...probe([72460], { 72460: providerStartedAt }) })).resolves.toEqual({ stopped: false, reason: "provider_process_alive" }); + }); +}); + describe("native restart recovery classification", () => { it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => { const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false, diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index d750a83a141..089d421450e 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -336,6 +336,118 @@ export async function evaluateNativeProviderProcesses(input: { }; } +/** Run event types that can carry a provider process identity. */ +export const NATIVE_PROVIDER_PROCESS_EVENT_TYPES = [ + "harness.ready", + "session.started", + "session.resumed", + "session.updated", + "turn.started", + "provider.event", + "provider.rpc_result", +] as const; + +function parsedProcessStart(value: unknown): Date | null { + const parsed = typeof value === "string" ? new Date(value) : null; + return parsed && !Number.isNaN(parsed.getTime()) ? parsed : null; +} + +/** Provider process identities from a saved checkpoint and recent run events. */ +export function nativeProviderProcessIdentities( + checkpoint: Record, + providerEvents: ReadonlyArray<{ payload: Record | null }>, +): NativeProviderProcessIdentity[] { + const identities: NativeProviderProcessIdentity[] = []; + const checkpointProcess = + checkpoint.process && typeof checkpoint.process === "object" && !Array.isArray(checkpoint.process) + ? (checkpoint.process as Record) + : {}; + for (const [field, startedAtField] of [ + ["providerPid", "providerProcessStartedAt"], + ["codexPid", "codexProcessStartedAt"], + ["sidecarPid", "sidecarProcessStartedAt"], + ["agentPid", "agentProcessStartedAt"], + ] as const) { + const value = checkpointProcess[field]; + if (typeof value === "number" && Number.isInteger(value) && value > 0) { + identities.push({ pid: value, processStartedAt: parsedProcessStart(checkpointProcess[startedAtField]) }); + } + } + for (const providerEvent of providerEvents) { + const prpEvent = providerEvent.payload?.prpEvent; + const event = prpEvent && typeof prpEvent === "object" && !Array.isArray(prpEvent) + ? (prpEvent as Record) : {}; + const eventPayload = event.payload && typeof event.payload === "object" && !Array.isArray(event.payload) + ? (event.payload as Record) : {}; + const processId = eventPayload.processId; + if (typeof processId === "number" && Number.isInteger(processId) && processId > 0) { + identities.push({ pid: processId, + processStartedAt: parsedProcessStart(eventPayload.processStartedAt ?? eventPayload.providerProcessStartedAt) }); + } + } + return identities; +} + +export type NativeLocalRunnerStopProof = + | { stopped: true } + | { + stopped: false; + reason: + | "runner_process_identity_missing" + | "runner_process_alive" + | "runner_process_identity_unverifiable" + | "runner_process_group_alive" + | "provider_process_alive" + | "provider_process_identity_unverifiable"; + }; + +/** + * Proves that a finished local runner and its recorded provider processes + * stopped, with the same exact PID and start-time checks as restart recovery. + * A PID that now belongs to a different process counts as stopped. A live + * process with the recorded identity, a live process group, or a live PID + * whose start time cannot be compared is not proof. + */ +export async function proveNativeLocalRunnerStopped(input: { + processPid: number | null; + processGroupId: number | null; + processStartedAt: Date | null; + providerProcesses?: readonly NativeProviderProcessIdentity[]; + /** A current, server-authored stop receipt already proved this exact Runner and group absent. */ + runnerStopRecorded?: boolean; + isProcessAlive?: (pid: number) => boolean; + isProcessGroupAlive?: (processGroupId: number) => boolean; + readProcessStartedAt?: (pid: number) => Promise; +}): Promise { + const refuse = (reason: Extract["reason"]) => + ({ stopped: false as const, reason }); + // A local Runner leads its own process group. Without both identities, a + // live child in that group could outlive the Runner unseen. + if (!input.processPid || !Number.isInteger(input.processPid) || input.processPid <= 1 || !input.processGroupId + || (!input.processStartedAt && !input.runnerStopRecorded)) { + return refuse("runner_process_identity_missing"); + } + if (!input.runnerStopRecorded) { + const runner = await evaluateNativeProviderProcesses({ + identities: [{ pid: input.processPid, processStartedAt: input.processStartedAt }], + isProcessAlive: input.isProcessAlive, + readProcessStartedAt: input.readProcessStartedAt, + }); + if (runner.livePids.length > 0) return refuse("runner_process_alive"); + if (runner.ambiguousLivePids.length > 0) return refuse("runner_process_identity_unverifiable"); + } + const groupAlive = input.isProcessGroupAlive ?? processGroupIsAlive; + if (groupAlive(input.processGroupId)) return refuse("runner_process_group_alive"); + const providers = await evaluateNativeProviderProcesses({ + identities: (input.providerProcesses ?? []).filter(identity => identity.pid !== input.processPid), + isProcessAlive: input.isProcessAlive, + readProcessStartedAt: input.readProcessStartedAt, + }); + if (providers.livePids.length > 0) return refuse("provider_process_alive"); + if (providers.ambiguousLivePids.length > 0) return refuse("provider_process_identity_unverifiable"); + return { stopped: true }; +} + export function classifyNativeRunnerRecoveryEvidence(input: { runnerPidAlive: boolean; runnerGroupAlive: boolean; @@ -898,79 +1010,12 @@ export async function claimNativeRestartRecoveries(input: { .where( and( eq(heartbeatRunEvents.runId, row.run.id), - inArray(heartbeatRunEvents.eventType, [ - "harness.ready", - "session.started", - "session.resumed", - "session.updated", - "turn.started", - "provider.event", - "provider.rpc_result", - ]), + inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_PROCESS_EVENT_TYPES]), ), ) .orderBy(desc(heartbeatRunEvents.id)) .limit(100); - const checkpointProcess = - checkpointRecord.process && - typeof checkpointRecord.process === "object" && - !Array.isArray(checkpointRecord.process) - ? (checkpointRecord.process as Record) - : {}; - const providerProcessIdentities: NativeProviderProcessIdentity[] = []; - const checkpointProcessFields = [ - ["providerPid", "providerProcessStartedAt"], - ["codexPid", "codexProcessStartedAt"], - ["sidecarPid", "sidecarProcessStartedAt"], - ["agentPid", "agentProcessStartedAt"], - ] as const; - for (const [field, startedAtField] of checkpointProcessFields) { - const value = checkpointProcess[field]; - if (typeof value === "number" && Number.isInteger(value) && value > 0) { - const rawStartedAt = checkpointProcess[startedAtField]; - const parsedStartedAt = - typeof rawStartedAt === "string" ? new Date(rawStartedAt) : null; - providerProcessIdentities.push({ - pid: value, - processStartedAt: - parsedStartedAt && !Number.isNaN(parsedStartedAt.getTime()) - ? parsedStartedAt - : null, - }); - } - } - for (const providerEvent of providerEvents) { - const prpEvent = providerEvent.payload?.prpEvent; - const event = - prpEvent && typeof prpEvent === "object" && !Array.isArray(prpEvent) - ? (prpEvent as Record) - : {}; - const eventPayload = - event.payload && - typeof event.payload === "object" && - !Array.isArray(event.payload) - ? (event.payload as Record) - : {}; - const processId = eventPayload.processId; - if ( - typeof processId === "number" && - Number.isInteger(processId) && - processId > 0 - ) { - const rawStartedAt = - eventPayload.processStartedAt ?? - eventPayload.providerProcessStartedAt; - const parsedStartedAt = - typeof rawStartedAt === "string" ? new Date(rawStartedAt) : null; - providerProcessIdentities.push({ - pid: processId, - processStartedAt: - parsedStartedAt && !Number.isNaN(parsedStartedAt.getTime()) - ? parsedStartedAt - : null, - }); - } - } + const providerProcessIdentities = nativeProviderProcessIdentities(checkpointRecord, providerEvents); const providerProcesses = await evaluateNativeProviderProcesses({ identities: (remoteProcess ? [] : providerProcessIdentities).filter( (identity) => identity.pid !== row.run.processPid, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index fd1216fe57c..98fee67a06c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -292,6 +292,10 @@ import { import { currentNativeControllerIdentity, evaluateNativeControllerTakeover, + proveNativeLocalRunnerStopped, + nativeProviderProcessIdentities, + NATIVE_PROVIDER_PROCESS_EVENT_TYPES, + type NativeLocalRunnerStopProof, isCurrentNativeControllerProcess, nextNativeProviderAttempt, type NativeControllerIdentity, @@ -5286,15 +5290,25 @@ function moveNativeContinuityState(root: string, archiveRoot: string): void { * Runner outbox can hold only post-terminal telemetry. Local lifecycle * evidence, even an unsuspended receipt, and unknown entries are owner * claims that this proof never overrides. + * + * `localRunner` names a local Runner that wrote its own lifecycle receipt in + * this root. Its unsuspended receipt and its own unfinished lifecycle commands + * (suspend, drain, shutdown) are claims of that one process. The caller must + * separately prove that process stopped before it relies on this projection. */ type SettledRunnerdProjection = { bytes: string; refused?: never } | { bytes?: never; refused: string }; -function readSettledRunnerdProjection(root: string, identity: Record): SettledRunnerdProjection { +const LOCAL_RUNNER_LIFECYCLE_COMMANDS = new Set(["runner.suspend", "runner.drain", "runner.shutdown"]); + +function readSettledRunnerdProjection(root: string, identity: Record, + options: { localRunner?: boolean } = {}): SettledRunnerdProjection { const refuse = (refused: string): SettledRunnerdProjection => ({ refused }); try { if (!isSafeNativeStateDirectory(root)) return refuse("root_unsafe"); const lifecycle = runnerdAuthorityLifecycle(root, identity as RunnerdDurableIdentity); - if (lifecycle !== "absent") return refuse(`runner_lifecycle_${lifecycle}`); + if (lifecycle !== "absent" && !(options.localRunner && lifecycle === "not_suspended")) { + return refuse(`runner_lifecycle_${lifecycle}`); + } // Every legitimate writer's entry is classified in native-session-root-entries.ts. // Held entries are known evidence that needs its own settlement; anything // else is writer drift. Name each kind separately. @@ -5306,6 +5320,11 @@ function readSettledRunnerdProjection(root: string, identity: Record !LOCAL_RUNNER_LIFECYCLE_COMMANDS.has(String(command.type)) || command.status === "completed") + : commands; + const settledControl = settledCommands === commands ? control : { ...control, commands: settledCommands }; const answered = new Set(commands .filter(command => command.type === "semantic_tool.result" && command.status === "completed") .map(command => record(command.payload).sourceEventId)); @@ -5315,11 +5334,11 @@ function readSettledRunnerdProjection(root: string, identity: Record record(control.identity)[key] !== value)) { return refuse("journal_identity_mismatch"); } - if (!isSettledRemoteWarmControlState(control, identity)) { + if (!isSettledRemoteWarmControlState(settledControl, identity)) { // Closed vocabulary only: known command types and statuses, never payloads. - const pending = commands.filter(command => command.status !== "completed") + const unsettled = settledCommands.filter(command => command.status !== "completed") .map(command => `${retirementReasonName("commandType", command.type)}.${retirementReasonName("commandStatus", command.status)}`); - return refuse(pending.length > 0 ? `journal_commands_unsettled(${[...new Set(pending)].slice(0, 4).join(",")})` + return refuse(unsettled.length > 0 ? `journal_commands_unsettled(${[...new Set(unsettled)].slice(0, 4).join(",")})` : "journal_terminal_unsettled"); } if ((control.committedEvents as unknown[]).map(record).some(event => @@ -5363,6 +5382,12 @@ function retirementReasonName(kind: keyof typeof RETIREMENT_REASON_NAMES, value: * that authority as a continuity break instead. This run then bootstraps its * own authority, and the provider session follows the ordinary * resume-or-replace policy. Any missing proof keeps the original rejection. + * + * A local per-turn Runner can end the same way: the result is accepted but its + * suspend command does not finish, so its own receipt stays unsuspended. That + * receipt is a claim of one recorded Runner process. Once that process and its + * recorded provider processes are proven stopped by exact PID and start time, + * the claim has no owner left and the same retirement applies. */ async function retireFinishedPerTurnRunnerdOwner(input: { db: Db; @@ -5377,11 +5402,14 @@ async function retireFinishedPerTurnRunnerdOwner(input: { // existing recovery path reads. Plugin tasks own a separate saved checkpoint. const refuse = (refused: string) => ({ retired: false as const, refused }); if (input.execution.session.lifecyclePolicy.mode !== "per_turn") return refuse("lifecycle_not_per_turn"); - if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) { - return refuse("target_not_sandbox_runner"); - } - if (!["created", "replacement"].includes(target.sandboxLeaseAcquisition?.outcome ?? "")) { - return refuse(`sandbox_${retirementReasonName("sandboxOutcome", target.sandboxLeaseAcquisition?.outcome)}`); + const localRunner = !target || target.kind === "local"; + if (!localRunner) { + if (target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) { + return refuse("target_not_sandbox_runner"); + } + if (!["created", "replacement"].includes(target.sandboxLeaseAcquisition?.outcome ?? "")) { + return refuse(`sandbox_${retirementReasonName("sandboxOutcome", target.sandboxLeaseAcquisition?.outcome)}`); + } } const identity: Record = {}; for (const key of ["runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId"]) { @@ -5389,7 +5417,7 @@ async function retireFinishedPerTurnRunnerdOwner(input: { if (typeof value !== "string" || !value) return refuse("durable_identity_incomplete"); identity[key] = value; } - const projection = readSettledRunnerdProjection(input.root, identity); + const projection = readSettledRunnerdProjection(input.root, identity, { localRunner }); if (projection.refused !== undefined) return refuse(projection.refused); const controlBytes = projection.bytes; const { binding } = input.execution; @@ -5421,6 +5449,29 @@ async function retireFinishedPerTurnRunnerdOwner(input: { remoteLeaseFenced: readNativeControllerRecovery(profile.nativeControllerRecovery, profile.sessionCheckpoint) !== null }); if (!stopped.allowed) return refuse(`controller_${retirementReasonName("controllerReason", stopped.reason)}`); const provenFinishedHere = finishedHere(); + const proveLocalRunnerStopped = async (): Promise => { + const run = prior!.run; + // Settlement may already have cleared the process fields after it saved a + // server-authored stop receipt; that receipt is the identity to check. + const released = run.processPid === null && run.processGroupId === null && run.processStartedAt === null; + const receipt = released ? await readNativeLocalProcessStop(input.db, run.companyId, run.id) : null; + if (released && !receipt) return { stopped: false, reason: "runner_process_identity_missing" }; + const providerEvents = await input.db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), + inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_PROCESS_EVENT_TYPES]))) + .orderBy(desc(heartbeatRunEvents.id)).limit(100); + return proveNativeLocalRunnerStopped({ + processPid: receipt?.processPid ?? run.processPid, + processGroupId: receipt?.processGroupId ?? run.processGroupId, + processStartedAt: receipt ? null : run.processStartedAt, + runnerStopRecorded: receipt !== null, + providerProcesses: nativeProviderProcessIdentities(record(profile.sessionCheckpoint), providerEvents), + }); + }; + if (localRunner) { + const runnerStopped = await proveLocalRunnerStopped(); + if (!runnerStopped.stopped) return refuse(runnerStopped.reason); + } // Earlier rejected admissions can quarantine the same projection. Each copy // must pass the same proof: a rebuilt root never carries the local Runner // evidence that held an earlier attempt, so that evidence stays decisive. @@ -5429,13 +5480,14 @@ async function retireFinishedPerTurnRunnerdOwner(input: { if (!isSafeNativeStateDirectory(quarantine)) return refuse("quarantine_unsafe"); for (const name of readdirSync(quarantine)) { if (!name.startsWith(`${basename(input.root)}.`)) continue; - const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity); + const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity, { localRunner }); if (copy.refused !== undefined) return refuse(`quarantine_${copy.refused}`); } } // Database reads can await. Retire only the exact evidence proven above. - if (readSettledRunnerdProjection(input.root, identity).bytes !== controlBytes) return refuse("journal_changed"); + if (readSettledRunnerdProjection(input.root, identity, { localRunner }).bytes !== controlBytes) return refuse("journal_changed"); if (provenFinishedHere && !finishedHere()) return refuse("controller_restarted_here"); + if (localRunner && !(await proveLocalRunnerStopped()).stopped) return refuse("runner_process_restarted"); const archivedAt = Date.now(); const archiveRoot = resolve(input.root, "continuity-breaks", `${archivedAt}-${randomUUID()}`); mkdirSync(archiveRoot, { recursive: true, mode: 0o700 });