diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index c321e01b5e2..986c65cbbf5 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3329,6 +3329,15 @@ receipt. The existing sandbox marker, process identity and authenticated PRP checkpoint still establish the original execution. Adoption uses that admitted artifact identity and does not stage or launch provider assets. Fresh launches and recovery without persisted admission retain their release and pack checks. +Ordinary per-turn OpenCode continuation also retains its recorded qualified +provider pack when its verified durable session binding and pack receipt remain +available. A new controller release or diagnostics setting does not replace the +proxy behind the saved provider state. If the exact artifacts are unavailable, +continuation stays blocked; this rule grants no fresh-session or replay authority. +A newly launched daemon binds its current Runner artifact while retaining the +provider pack. Only a surviving daemon retains its previous Runner artifact. +An authorized continuity break releases the pack pin after archiving the old +state, so the fresh session uses normal qualified pack selection. Native ACPX semantic tool calls wait on their durable PRP invocation across a controller outage. Their provider-side MCP bridge does not impose a separate request deadline that could make the provider retry an already queued write. diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index e83b70871aa..78e3b3e85f7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -2437,6 +2437,62 @@ fn rejects_opencode_launch_profile_drift_across_fresh_recovery() { fs::remove_dir_all(directory).unwrap(); } +#[test] +#[cfg_attr(windows, ignore = "verified launch is not supported on Windows yet")] +fn retains_opencode_state_when_a_valid_new_proxy_replaces_the_admitted_bytes() { + let directory = temporary_directory("opencode-proxy-upgrade-recovery"); + let config = opencode_config(&directory); + let mut first = NativeProviderCommandExecutor::with_runner_config(&directory, &config); + first + .execute(&command( + 1, + "run.prepare", + opencode_prepare_payload(&directory), + )) + .unwrap(); + first.shutdown().unwrap(); + drop(first); + let state_path = directory.join("codex-provider-state.json"); + let saved_state = fs::read(&state_path).unwrap(); + let proxy = directory.join("qualified-opencode-proxy-script"); + let original = fs::read(&proxy).unwrap(); + #[cfg(unix)] + fs::set_permissions(&proxy, fs::Permissions::from_mode(0o700)).unwrap(); + let mut upgraded = original.clone(); + upgraded.extend_from_slice(b"\n# qualified next proxy release\n"); + fs::write(&proxy, &upgraded).unwrap(); + #[cfg(unix)] + fs::set_permissions(&proxy, fs::Permissions::from_mode(0o500)).unwrap(); + let mut next = config.clone(); + next.opencode_launch_profile.as_mut().unwrap().proxy_script = qualified_artifact(proxy.clone()); + let mut recovered = NativeProviderCommandExecutor::with_runner_config(&directory, &next); + let error = recovered + .execute(&command( + 2, + "run.prepare", + opencode_prepare_payload(&directory), + )) + .unwrap_err(); + assert_eq!(error.classification(), Some(PROVIDER_STATE_INCOMPATIBLE)); + assert_eq!(fs::read(&state_path).unwrap(), saved_state); + drop(recovered); + #[cfg(unix)] + fs::set_permissions(&proxy, fs::Permissions::from_mode(0o700)).unwrap(); + fs::write(&proxy, original).unwrap(); + #[cfg(unix)] + fs::set_permissions(&proxy, fs::Permissions::from_mode(0o500)).unwrap(); + let mut pinned = NativeProviderCommandExecutor::with_runner_config(&directory, &config); + pinned + .execute(&command( + 3, + "run.prepare", + opencode_prepare_payload(&directory), + )) + .unwrap(); + pinned.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + #[test] fn rejects_pi_without_an_explicit_model_before_starting_a_sidecar() { let directory = temporary_directory("pi-model"); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 47e155341a4..c568d5350f2 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -116,6 +116,7 @@ import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } f import { admittedNativeProviderPack, nativeProviderPackAdmissionPending, rememberAdmittedNativeProviderPack } from "./native-provider-pack-admission.js"; import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; import * as providerPackAdmission from "./native-provider-pack-admission.js"; +import * as nativeExecutionDriver from "./native-runner-execution-driver.js"; vi.mock("node:fs", async (importOriginal) => { const actual = await importOriginal(); @@ -1878,7 +1879,7 @@ describe("remote provider pack manifest", () => { return JSON.stringify(value); }; - it.each(["opencode", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { + it.each(["opencode", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { const architecture = provider === "claude-arm64" ? "arm64" : "x64"; const claude = provider.startsWith("claude-"); const opencode = provider.startsWith("opencode"); @@ -1957,6 +1958,7 @@ describe("remote provider pack manifest", () => { const transport = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { controlPlaneRegistration: (authority: unknown) => Promise; environment?: NodeJS.ProcessEnv; + runnerArtifact?: { version: string; digest: string }; }; expect(transport.runnerBinary).toBe(state.bundledRunnerBinary); if (opencode) { @@ -1977,7 +1979,7 @@ describe("remote provider pack manifest", () => { expect(transport).toMatchObject({ providerPackAuthorityDigest: installed.digest, providerNodeCommand: "/opt/paperclip-runner/provider-pack/node_modules/node/bin/node", providerNodeCommandSha256: installed.payload.artifacts.nodeCommand.sha256 }); - if (provider === "opencode" || provider === "opencode-dirty") { + if (provider === "opencode" || provider.startsWith("opencode-continuation") || provider === "opencode-dirty") { const identity = { ...remoteExecution.binding, sessionId: remoteExecution.session.normalizedSessionId! }; const session = { identity: () => identity } as never; const backend = state.createBackend.mock.results.at(-1)!.value; @@ -2002,6 +2004,75 @@ describe("remote provider pack manifest", () => { expect(execute.mock.calls.filter(([call]) => call.args?.[0] === "--build-metadata")).toHaveLength(1); expect(execute.mock.calls.some(([call]) => call.args?.[0] === "-e" || call.args?.[0] === "--version")).toBe(false); expect(syncIn.mock.calls.every(([operations]) => operations.every(operation => operation.files.every(file => file.kind === "file")))).toBe(true); + if (provider.startsWith("opencode-continuation")) { + const identity = transport.prpIdentity!; + await mkdir(join(transport.stateDirectory!, "control-plane"), { recursive: true }); + await writeFile(join(transport.stateDirectory!, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity))); + await mkdir(join(transport.stateDirectory!, "runner"), { recursive: true }); + await writeFile(join(transport.stateDirectory!, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended"))); + const next = structuredClone(manifest); + next.payload.runnerSourceRevision = "b".repeat(40); + next.payload.artifacts.opencodeProxy.sha256 = "sha256:" + "f".repeat(64); + next.payload.bridgeDigest = "sha256:" + createHash("sha256") + .update(next.payload.artifacts.opencodeProxy.sha256 + "\n" + digest + "\n" + next.payload.distDigest).digest("hex"); + next.digest = "sha256:" + createHash("sha256").update(canonical(next.payload)).digest("hex"); + await writeFile(state.bundledPackManifestPath!, JSON.stringify(next)); + const currentDaemon = Buffer.concat([daemon, Buffer.from("newly qualified daemon bytes")]); + await writeFile(state.bundledRunnerBinary!, currentDaemon); + // A normal next turn restores this provider session; it is not a live + // daemon reattachment. Enabling diagnostics must not replace the + // already admitted proxy behind that retained provider state. + const receiptPath = join(transport.stateDirectory!, "provider-pack-authority.json"); + const receiptBefore = await readFile(receiptPath, "utf8"); + if (provider === "opencode-continuation-no-receipt") await rm(receiptPath); + const prepareRemote = vi.spyOn(nativeExecutionDriver, "prepareNativeRemoteRunner"); + const backendsBefore = state.createBackend.mock.calls.length; + const selectedExecute = provider === "opencode-continuation-missing" + ? async (command: { command: string; args?: string[] }) => command.args?.[0] === "--build-metadata" + ? { ...await execute(command), stdout: JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + providerPacks: [{ root: "/opt/paperclip-runner/provider-pack", manifest: next, immutable: true }], + }) } : execute(command) + : execute; + const continuation = createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution, + runnerInstanceId: identity.runnerInstanceId, runnerIngressAuthorized: true, + runnerEnvironment: configuredEnvironmentProjection(provider === "opencode-continuation-replacement" ? {} : { PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS: "1" }), + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + environmentId: "environment", leaseId: "lease", providerKey: "daytona", + effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute: selectedExecute, syncIn } } as never }); + if (provider === "opencode-continuation-missing" || provider === "opencode-continuation-no-receipt") { + await expect(continuation).rejects.toThrow("image provider pack did not match the bundled release identity"); + expect(state.createBackend).toHaveBeenCalledTimes(backendsBefore); + if (provider === "opencode-continuation-missing") expect(await readFile(receiptPath, "utf8")).toBe(receiptBefore); + else expect(existsSync(receiptPath)).toBe(false); + return; + } + const continued = await continuation; + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ + providerPackAuthorityDigest: installed.digest, + opencodeProxySha256: installed.payload.artifacts.opencodeProxy.sha256, + opencodeProxyPath: "/opt/paperclip-runner/provider-pack/dist/cli/opencode-app-server-proxy.cjs", + runnerArtifact: { version: transport.runnerArtifact!.version, digest: "sha256:" + createHash("sha256").update(currentDaemon).digest("hex") }, + }); + if (provider === "opencode-continuation-replacement") { + const preparation = prepareRemote.mock.calls.at(-1)![0]; + const backend = state.createBackend.mock.results.at(-1)!.value; + Object.assign(backend, { openSession: async () => { + await nativeExecutionDriver.prepareNativeRemoteRunner(preparation); + return { identity: () => ({ ...remoteExecution.binding, sessionId: remoteExecution.session.normalizedSessionId! }) }; + } }); + const priorIdentity = { ...remoteExecution.binding, sessionId: remoteExecution.session.normalizedSessionId! }; + await continued.openReplacementSession!({ identity: priorIdentity }, { + backendKind: "runner", identity: priorIdentity, sessionId: priorIdentity.sessionId, + providerSessionId: "prior-provider-session", activeTurnId: null, terminal: null, + }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ providerPackAuthorityDigest: installed.digest }); + } + } if (provider === "claude-x64" || provider === "opencode") { const admitted = opencode ? installed : manifest; const identity = transport.prpIdentity!; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 27f2abb0856..5d2321c14e6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -13906,8 +13906,12 @@ async function createRunnerdBackendWithinSessionClaim( })(); return currentProviderPackManifest; }; + // Ordinary per-turn OpenCode continuation restores the same durable provider + // state even when no daemon survived. Its launch profile binds the admitted + // artifact bytes and paths just as a live reattachment does. const reattachProviderPack = input.restartRecovery?.kind === "reattach_remote_runner" || input.remoteWarmRecovery !== undefined; - if (requiresRemoteProviderPack && !reattachProviderPack) resolveCurrentProviderPackManifest(); + let retainProviderPack = reattachProviderPack || input.execution.provider.kind === "opencode" && durableBinding !== null; + if (requiresRemoteProviderPack && !retainProviderPack) resolveCurrentProviderPackManifest(); // The transport hashes runnerBinary on the controller before an external // launcher starts runnerd. Keep that artifact identity in the controller's // filesystem; the remote launcher separately owns the sandbox command path. @@ -13930,6 +13934,7 @@ async function createRunnerdBackendWithinSessionClaim( runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId, remoteCwd: remoteTarget!.remoteCwd, providerDigest: nativeSha256(input.execution.provider) }, reattach, + retainProviderPack: reattach || retainProviderPack, runnerArtifact: () => readRunnerdArtifactBinding(controllerRunnerBinary), current: resolveCurrentProviderPackManifest, }) : null; @@ -14064,11 +14069,11 @@ async function createRunnerdBackendWithinSessionClaim( runner: remoteCommandRunner, cwd: remoteTarget.remoteCwd, packRoot: installation.root, manifest: expected, installation, provider: input.execution.provider as { kind: "opencode" } | { kind: "acpx"; agent: string }, - requireExactIdentity: reattachProviderPack || explicitProviderPackSelection, - requireCurrentOpenCodeProxy: !reattachProviderPack + requireExactIdentity: retainProviderPack || explicitProviderPackSelection, + requireCurrentOpenCodeProxy: !retainProviderPack && runnerEnvironment?.PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS === "1", })); - if (!reattachProviderPack) providerPackAuthority!.select(manifest); + if (!retainProviderPack) providerPackAuthority!.select(manifest); expectedProviderPackManifest = manifest; activeRemoteProviderPackRoot = installation.root; providerPackVerified = true; @@ -15045,6 +15050,7 @@ async function createRunnerdBackendWithinSessionClaim( } } remotePrepared = false; + retainProviderPack = false; providerPackAuthority = selectProviderPackAuthority(false); expectedProviderPackManifest = providerPackAuthority?.manifest ?? null; providerPackVerified = false; diff --git a/server/src/services/native-runtime/remote-provider-pack-authority.test.ts b/server/src/services/native-runtime/remote-provider-pack-authority.test.ts index 8530105addc..0a9f0712a82 100644 --- a/server/src/services/native-runtime/remote-provider-pack-authority.test.ts +++ b/server/src/services/native-runtime/remote-provider-pack-authority.test.ts @@ -93,6 +93,16 @@ it("retains the authenticated daemon identity across controller daemon replaceme expect(recovered.runnerArtifact).toEqual(runnerArtifact()); expect(changedRunner).not.toHaveBeenCalled(); }); +it("retains the admitted provider pack but binds a newly launched daemon to its current bytes", () => { + const { input } = fixture(); + resolveRemoteProviderPackAuthority(input).verified(); + const currentRunner = { version: "new", digest: "sha256:" + "d".repeat(64) }; + const continued = resolveRemoteProviderPackAuthority({ ...input, reattach: false, retainProviderPack: true, + current: () => manifest("b"), runnerArtifact: () => currentRunner } as Parameters[0]); + expect(continued.manifest).toEqual(manifest("a")); + expect(continued.runnerArtifact).toEqual(currentRunner); + expect(() => continued.select(manifest("b"))).toThrow("cannot replace"); +}); it.each([{}, { version: "0.0.0", digest: "not-a-sha256" }])("rejects an invalid saved daemon identity: %j", runnerArtifact => { const { input, path } = fixture(); resolveRemoteProviderPackAuthority(input).verified(); diff --git a/server/src/services/native-runtime/remote-provider-pack-authority.ts b/server/src/services/native-runtime/remote-provider-pack-authority.ts index b1c30f55407..283ef123d8b 100644 --- a/server/src/services/native-runtime/remote-provider-pack-authority.ts +++ b/server/src/services/native-runtime/remote-provider-pack-authority.ts @@ -31,6 +31,7 @@ export function resolveRemoteProviderPackAuthority T; runnerArtifact: () => RunnerArtifactIdentity; }): { manifest: T; runnerArtifact: RunnerArtifactIdentity; select: (manifest: T) => void; verified: () => void } { @@ -41,7 +42,7 @@ export function resolveRemoteProviderPackAuthority { - if (input.reattach) throw new Error("runner_remote_provider_artifact_incompatible: cannot replace an attached pack authority"); + if (input.reattach || input.retainProviderPack) throw new Error("runner_remote_provider_artifact_incompatible: cannot replace an attached pack authority"); assertManifest(manifest); const next = structuredClone(manifest); receipt = serialize(next);