From 2e593bf3cda94f8b56c6845784e5c6f53074004e Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:35:15 -0600 Subject: [PATCH] fix(native): drop a repeated identity-record check and prove an unsettled predecessor stays an ordinary refusal (#21086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - #21001 sends a re-wake to a fresh sandbox when live adoption of a crashed predecessor's warm Runner is refused, and lets an inherited Runner identity pass the "workspace unchanged" proof > - The reviewer found two gaps on #21001 that were not answered before it merged > - The unchanged proof checked for a `native.process_identity_recorded` event twice, and no test proved that an unsettled predecessor stays an ordinary refusal > - This pull request removes the repeated check and adds that proof > - The benefit is simpler proof code and a test that stops the fresh-sandbox fallback from widening to unsettled predecessors ## Linked Issues or Issue Description Refs #21001, #20863 **What happened?** Two review warnings on #21001: 1. `carriesOnlyInheritedRunnerIdentity` in `remote-workspace-unchanged-proof.ts` looked for the run's own `native.process_identity_recorded` event. `executionStartedEventPredicate()`, later in the same proof, already refuses a run with that event (`execution_recorded`). 2. The existing denial cases in `native-controller-recovery.integration.test.ts` use `toThrow(denied)`. That also matches the wrapped `NativePredecessorRetirementUnprovenError`, so a regression that wraps an unsettled predecessor's refusal would pass. **Expected behavior** The proof has one check per fact, and the tests fail when an unsettled predecessor goes to the fresh-sandbox fallback. **Steps to reproduce** 1. Read `proveRemoteWorkspaceUnchanged` and `executionStartedEventPredicate`. 2. Change `remote-warm-session-recovery.ts` to wrap every refusal; the old denial cases still pass. ## What Changed - `server/src/services/remote-workspace-unchanged-proof.ts`: delete `carriesOnlyInheritedRunnerIdentity` and its `PROCESS_IDENTITY_RECORDED` import; call `carriesInheritedRunnerIdentity(tx, run)` directly. A run with its own identity record is still refused, now by `execution_recorded`. - `server/src/services/native-runtime/native-controller-recovery.integration.test.ts`: - The denial cases for an unsettled predecessor (other company, other execution, finalization lease held, run still `running`) now assert the error is not a `NativePredecessorRetirementUnprovenError`. - The pending-command case after the predecessor is `succeeded` and `committed` now asserts it is one (the #21001 behavior). - The crash-restart case adds a predecessor run that is `failed` or `running`: an unsettled journal is refused with `warm_recovery_refused=owner_unsettled`, not wrapped. ## Verification - `cd server && npx vitest run src/services/native-runtime/native-controller-recovery.integration.test.ts src/services/remote-workspace-unchanged-release.test.ts`: 50 passed. The three inherited-identity cases from #21001 return the same results (released / refused / refused). - `cd server && npx tsc --noEmit -p tsconfig.json`: passes. ## Risks - Low risk. No behavior change: the removed check is covered by `executionStartedEventPredicate()`. The rest is tests. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use (shell, file edits, test runs). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- ...ve-controller-recovery.integration.test.ts | 26 ++++++++++++++----- .../remote-workspace-unchanged-proof.ts | 12 +-------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index b682b02f785..49cac6d9fbb 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -480,18 +480,23 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d loadCheckpoint: () => checkpoint, canReuseProcess: () => true, ...overrides, }); const denied = "runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate"; - await expect(inspect({ execution: { ...current, binding: { ...current.binding, companyId: randomUUID() } } })) - .rejects.toThrow(denied); - await expect(inspect({ matchesExecution: () => false })).rejects.toThrow(denied); + const ordinary = async (attempt: Promise) => { + const error = await attempt.then(() => null, failure => failure); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain(denied); + expect(error).not.toBeInstanceOf(NativePredecessorRetirementUnprovenError); + }; + await ordinary(inspect({ execution: { ...current, binding: { ...current.binding, companyId: randomUUID() } } })); + await ordinary(inspect({ matchesExecution: () => false })); await expect(readSandbox({ remoteCwd: "/unrelated" })).rejects.toThrow("retained_sandbox_binding"); for (const leaseExpiresAt of [null, new Date(Date.now() - 1_000)]) { await db.update(nativeRunFinalizations).set({ leaseExpiresAt }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); - await expect(inspect()).rejects.toThrow(denied); + await ordinary(inspect()); await expect(readSandbox()).rejects.toThrow("retained_sandbox_binding"); } await db.update(nativeRunFinalizations).set({ leaseExpiresAt: deadline }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); - await expect(inspect()).rejects.toThrow(denied); + await ordinary(inspect()); await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); await expect(readSandbox()).rejects.toMatchObject({ retryAt: deadline }); const waiting = await inspect().catch(error => error); @@ -505,7 +510,7 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d .where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); await expect(readSandbox()).resolves.toEqual({ remoteCwd: "/workspace" }); await expect(inspect({ readControlState: () => ({ identity, commands: [{ status: "pending" }], - committedEvents: [{ eventType: "run.terminal", envelope: identity }] }) })).rejects.toThrow(denied); + committedEvents: [{ eventType: "run.terminal", envelope: identity }] }) })).rejects.toBeInstanceOf(NativePredecessorRetirementUnprovenError); // A created or replacement sandbox holds no Runner to adopt. Live adoption // steps aside so ordinary admission can retire the finished owner, instead // of denying with prior_owner_terminal_state_indeterminate (Nov 10 staging). @@ -582,6 +587,15 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d fallback(await inspect(() => ({ ...parked, commands: [...parked.commands, { type: "runner.suspend", status: "pending" }] })) .catch(error => error), "journal_unsettled"); + for (const status of ["failed", "running"] as const) { + await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + const unsettled = await inspect(() => ({ ...parked, commands: [...parked.commands, { type: "runner.suspend", status: "pending" }] })) + .catch(error => error); + expect(unsettled).not.toBeInstanceOf(NativePredecessorRetirementUnprovenError); + expect(unsettled.message).toBe("runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate; warm_recovery_refused=owner_unsettled"); + } + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + const recovered = (await inspect())!; await expect(recovered.revalidate("replacement-controller")).resolves.toBeUndefined(); await db.update(nativeRunFinalizations).set({ leaseOwner: "native-finalizer:replacement", leaseExpiresAt: new Date(Date.now() + 60_000) }) diff --git a/server/src/services/remote-workspace-unchanged-proof.ts b/server/src/services/remote-workspace-unchanged-proof.ts index 7c0fd82a5d1..67477468de5 100644 --- a/server/src/services/remote-workspace-unchanged-proof.ts +++ b/server/src/services/remote-workspace-unchanged-proof.ts @@ -7,7 +7,6 @@ import { remoteWorkspaceRunReferenceSchema, type RemoteWorkspaceAuthority } from import { executionStartedEventPredicate } from "./pre-dispatch-execution-evidence.js"; import { runOwnerFenceHeldSql } from "./remote-workspace-writer-hold.js"; import { carriesInheritedRunnerIdentity } from "./native-runtime/native-inherited-runner-retry.js"; -import { PROCESS_IDENTITY_RECORDED } from "./native-local-process-stop.js"; /** * A writer's prepared files equal its saved version only while nothing ran in @@ -46,15 +45,6 @@ const FILE_NEUTRAL_OPERATION_PHASES = ["remote_workspace_restore", "workspace_fi /** Terminal states of a run that ended without a result. */ const ENDED_WITHOUT_RESULT = ["failed", "cancelled", "interrupted", "timed_out"]; -async function carriesOnlyInheritedRunnerIdentity(tx: Pick, - run: Parameters[1]): Promise { - if (!await carriesInheritedRunnerIdentity(tx, run)) return false; - const [own] = await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( - eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), - eq(heartbeatRunEvents.eventType, PROCESS_IDENTITY_RECORDED))).limit(1); - return !own; -} - export type UnchangedProofRefusal = | "baseline_missing" | "authority_mismatch" | "writer_not_ended" | "writer_owner_live" | "process_recorded" | "reference_mismatch" | "provider_session_started" | "execution_recorded" | "result_recorded" @@ -90,7 +80,7 @@ export async function proveRemoteWorkspaceUnchanged(tx: Pick, inpu if (!input.callerOwnsWriter && run.ownerFenceHeld) return { proven: false, reason: "writer_owner_live" }; if (run.sessionIdAfter !== null) return { proven: false, reason: "process_recorded" }; if ((run.processPid !== null || run.processGroupId !== null || run.processStartedAt !== null) - && !await carriesOnlyInheritedRunnerIdentity(tx, run)) return { proven: false, reason: "process_recorded" }; + && !await carriesInheritedRunnerIdentity(tx, run)) return { proven: false, reason: "process_recorded" }; const profile = object(run.runnerProfileJson); const reference = remoteWorkspaceRunReferenceSchema.safeParse(profile.remoteWorkspace); if (profile.nativeWorkspaceSync != null || !reference.success || reference.data.state !== "prepared"