From bf9dbd18a8d9a539f79ca2e3ccb10b6df31c6334 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:57:26 -0500 Subject: [PATCH] fix(runner): recover incompatible Codex models before launch (#15519) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The Runner prepares and saves each task's provider configuration before launch. > - A sandbox image can have a supported Codex CLI that is too old for the selected model. > - The current version check stops that task even when the image can run a similar model. > - This pull request selects a compatible model before it saves a fresh execution. > - The task continues with a visible warning, and recovery uses the saved effective model. ## Linked Issues or Issue Description - Refs #15053. This fixes the same CLI and model mismatch for fresh remote Runner tasks. It does not change subscription onboarding. - Refs #14721. This adds a bounded startup choice for Codex CLI compatibility. It does not add user-defined fallback chains or quota failover. - Related PR: #15399 added the model-specific CLI checks that this change preserves at launch. ## What Changed - Check the preinstalled remote Codex CLI before saving a fresh execution that uses a model with a verified CLI minimum. - Select the closest compatible older model in the same class, then the stable Runner default. Consider each candidate once. - Save the effective model before checkpoint selection. Keep the requested agent and task settings unchanged. - Add a task warning and a local `runner.model_fallback` run-log event with both models and the CLI version. - Share executable discovery with the launch verifier. Preserve explicit artifact and install settings, saved executions, and existing artifact checks. - Add regression tests and document the selection and warning behavior. ## Verification - Red: the three new provider-configuration regression cases failed before the fix. They kept the incompatible requested model. - Red: both rejected-probe cases failed before the review fix. They now defer to launch verification. - Green: targeted provider configuration, remote preflight, task warning, and launch-verifier tests pass (680 tests). - `pnpm -r typecheck` passes. - [Full CI](https://github.com/paperclipai/paperclip/actions/runs/37713759744) passes on `eb273fc32661526d90a864b42c73a7da61dd62da`: 47 successful jobs, including all general and serialized Vitest groups, browser shards, Runner verification, typecheck, build, and the canary dry run. - Local extended verification: three serialized shards pass (115 suites). HTTP route tests hit intermittent 15-second timeouts. The authorization suite passes on rerun (130 tests). The document suite passes on both master and the PR head in the same isolated setup (6 tests). The local general run was stopped after the equivalent full CI matrix passed. - `pnpm build` passes. - Server typecheck and build pass again after the probe-error review fix. - `git diff --check` and `pnpm check:module-boundaries` pass. - Example: request `gpt-6.1-sol` on Codex 0.158.0 to use `gpt-6-sol`; on 0.156.0, use `gpt-5.6-sol`. The warning names the requested model, effective model, and CLI version. - This PR has not been deployed to staging. ## Risks - A fallback can have different capabilities. The task warning makes the substitution visible. Later runs can use the requested model after the image CLI is updated. - Preparation adds two remote commands for models with a verified CLI minimum. Failed or invalid version probes retain the existing launch checks. - This only handles known CLI and model mismatches before a fresh provider launch. Authentication, capacity, and artifact failures keep their existing behavior. - No database migration is required. ## Model Used OpenAI GPT-6 through Codex, with tool use and code execution. This session does not expose the exact deployment model ID, context window size, or reasoning setting. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/DEVELOPING.md | 9 ++ doc/SPEC-implementation.md | 10 ++ doc/execution-semantics.md | 18 ++++ doc/run-log-events.md | 6 ++ .../heartbeat-model-fallback-warning.test.ts | 19 ++++ .../heartbeat-runner-provider-config.test.ts | 22 +++++ server/src/services/heartbeat.ts | 82 +++++++++++++--- .../codex-model-fallback.test.ts | 88 +++++++++++++++++ .../native-runtime/codex-model-fallback.ts | 95 +++++++++++++++++++ .../native-session-executor.test.ts | 35 ++++--- .../native-runtime/native-session-executor.ts | 34 +------ .../native-runtime/provider-profile.ts | 5 +- 12 files changed, 365 insertions(+), 58 deletions(-) create mode 100644 server/src/__tests__/heartbeat-model-fallback-warning.test.ts create mode 100644 server/src/services/native-runtime/codex-model-fallback.test.ts create mode 100644 server/src/services/native-runtime/codex-model-fallback.ts diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 0151a641fda..43d0b7e67c1 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1289,6 +1289,15 @@ agent workspace. The host `HOME` itself, a directory that contains it, a filesystem root, a `CODEX_HOME` overlap, or a canonical path outside the assigned workspace is rejected before provider startup. +Fresh remote Codex Runner runs recover from a supported image CLI that is too old +for the selected model. Preparation selects the closest compatible older model +of the same class, then the stable Runner default. The task shows a warning with +the requested model, effective model, and CLI version. Agent and task settings +stay unchanged. Update the image CLI to restore the requested model on later +runs. Explicit `PAPERCLIP_RUNNER_REMOTE_CODEX_PATH` and +`PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC` settings take precedence. See +[execution semantics](execution-semantics.md#remote-codex-model-compatibility). + ### Sandbox ACP input delivery The legacy sandbox process bridge retries recognized Daytona and Cloudflare diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 1ad4d830359..df952bf5250 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -2080,6 +2080,16 @@ remove stale instructions after edits or access revocation. See [Connection instructions](connections/CONNECTION-INSTRUCTIONS.md) for contracts, UI conventions, custom adapter integration, and initial memory templates. +### Remote Codex model compatibility + +Fresh remote Codex Runner preparation can replace a model whose verified CLI +minimum exceeds the supported image CLI version. It selects a compatible older +model in the same class, then the stable Runner default, and saves that effective +model before checkpoint selection. A visible task warning and local run-log event +record the substitution. This does not change agent settings, rewrite admitted +executions, or bypass artifact, ownership, permission, and budget gates. See +[remote Codex compatibility](execution-semantics.md#remote-codex-model-compatibility). + ### Native provider capacity retry Committed, run-bound Codex `serverOverloaded` terminal failures display the model diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index c923e18c66f..fe62494d668 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -517,6 +517,24 @@ Warm sandbox reuse must match the current host Git tip and branch as well as the file snapshot and saved stamp, including managed nested repositories. A history or branch mismatch restages the host before the next run begins. +### Remote Codex model compatibility + +Fresh remote Codex Runner runs check the selected image CLI before saving the +native execution input. If the CLI is in the supported app-server version window +but below the selected model's verified minimum, preparation chooses the newest +compatible older model of the same class (`sol`, `luna`, `astra`, or `terra`), +then the stable Runner default. Each candidate is considered once; this selection +does not replay a provider turn or consume a failure-retry attempt. + +The effective model is saved in the execution input before checkpoint selection, +so launch, recovery, and usage accounting share that identity. The requested +agent and task settings stay unchanged. A task warning and `runner.model_fallback` +run-log event name both models and the image CLI version. An explicit remote +Codex artifact or npm install pin retains precedence. Persisted executions are +not rewritten, and unknown models, invalid CLI builds, and artifact failures keep +their existing verification and recovery rules. Capacity and authentication +failures do not trigger this startup substitution. + ### Native provider model capacity A committed Codex `turn.failed` event with `codexErrorInfo: serverOverloaded`, diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 10557d8b6bb..9641c092e72 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -7,6 +7,12 @@ event needs no operator endpoint. ## Native PRP Run-Log Events +Fresh remote Codex model substitution emits `runner.model_fallback` on the +system stream with `warn` severity. Its payload contains `requestedModel`, +`effectiveModel`, and `codexCliVersion`. It records a preparation choice before +provider launch, not a failed turn or a retry. The same substitution appears as +a system warning in the task conversation. It remains local run-log data. + The hidden native coordinator writes each validated PRP event to the bound run's existing event stream before it acknowledges the runner. The row keeps the PRP `eventType`, source instance, source event ID, source sequence, protocol diff --git a/server/src/__tests__/heartbeat-model-fallback-warning.test.ts b/server/src/__tests__/heartbeat-model-fallback-warning.test.ts new file mode 100644 index 00000000000..d9c07db02fe --- /dev/null +++ b/server/src/__tests__/heartbeat-model-fallback-warning.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it, vi } from "vitest"; +import { postNativeModelFallbackWarning } from "../services/heartbeat.js"; + +describe("model fallback warning", () => { + it("records the substitution and shows a run-attributed warning on the task", async () => { + const addComment = vi.fn(async () => ({} as never)); + const onEvent = vi.fn(async () => undefined); + await postNativeModelFallbackWarning({ issuesSvc: { addComment }, onEvent, + issueId: "issue", runId: "run", requestedModel: "gpt-6.1-sol", effectiveModel: "gpt-5.6-sol", codexCliVersion: "0.156.0" }); + expect(onEvent).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ + eventType: "runner.model_fallback", stream: "system", level: "warn", + payload: { requestedModel: "gpt-6.1-sol", effectiveModel: "gpt-5.6-sol", codexCliVersion: "0.156.0" }, + })); + expect(addComment).toHaveBeenCalledExactlyOnceWith("issue", expect.stringContaining("Work will continue with the compatible model"), + { runId: "run" }, { authorType: "system", + presentation: { kind: "system_notice", tone: "warning", title: "Using gpt-5.6-sol", density: "compact", detailsDefaultOpen: false } }); + expect(onEvent.mock.invocationCallOrder[0]).toBeLessThan(addComment.mock.invocationCallOrder[0]!); + }); +}); diff --git a/server/src/services/heartbeat-runner-provider-config.test.ts b/server/src/services/heartbeat-runner-provider-config.test.ts index 1787dbcc89f..0da343e0a37 100644 --- a/server/src/services/heartbeat-runner-provider-config.test.ts +++ b/server/src/services/heartbeat-runner-provider-config.test.ts @@ -8,6 +8,28 @@ import { } from "./native-runtime/provider-profile.js"; describe("Paperclip Runner native provider configuration", () => { + it.each([ + ["gpt-6.1-sol", "0.158.0", "gpt-6-sol"], + ["gpt-6.1-sol", "0.156.0", "gpt-5.6-sol"], + ["gpt-6-luna", "0.156.0", "gpt-5.6-luna"], + ])("recovers %s on Codex %s with compatible model %s", (model, codexCliVersion, expectedModel) => { + const input = { + backend: "codex_app_server" as const, + adapterConfig: { provider: "codex", model, modelReasoningEffort: "high", codexPermissionMode: "never" }, + codexCliVersion, + }; + expect(resolvePaperclipRunnerNativeProviderInput(input)).toEqual({ + provider: "codex", model: expectedModel, codexApprovalPolicy: "never", codexReasoningEffort: "high", + }); + expect(input.adapterConfig.model).toBe(model); + }); + + it.each([ + ["gpt-6.1-sol", "0.159.0"], ["gpt-6-sol", "0.157.0"], ["gpt-5.6-sol", "0.156.0"], + ])("keeps a supported %s selection on Codex %s", (model, codexCliVersion) => { + expect(resolvePaperclipRunnerNativeProviderInput({ backend: "codex_app_server", + adapterConfig: { provider: "codex", model }, codexCliVersion })).toMatchObject({ model }); + }); it.each([undefined, "approve-all", "approve-paperclip", "approve-reads", "deny-all"])( "passes Grok's full-auto default or explicit %s policy to the native runner", (acpxPermissionMode) => { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 9c89809c661..c47132dbf2c 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -287,6 +287,7 @@ import { projectPaperclipRunnerTaskConfig, resolvePaperclipRunnerNativeProviderInput, } from "./native-runtime/provider-profile.js"; +import { readRemoteCodexModelCliVersion } from "./native-runtime/codex-model-fallback.js"; import { buildNativeHeartbeatPreparationSpans, buildNativeWakeIngressSpan, @@ -9640,6 +9641,39 @@ export function postWorkspaceReadyComment(input: { ); } +export async function postNativeModelFallbackWarning(input: { + issuesSvc: Pick, "addComment">; + onEvent: (event: AdapterRuntimeEvent) => Promise; + issueId: string; + runId: string; + requestedModel: string | null; + effectiveModel: string | null; + codexCliVersion: string; +}): Promise { + const message = `Using ${input.effectiveModel} because the sandbox's Codex ${input.codexCliVersion} does not support ${input.requestedModel}. Work will continue with the compatible model. Update the sandbox's Codex CLI to use the requested model.`; + await input.onEvent({ + eventType: "runner.model_fallback", + stream: "system", + level: "warn", + message, + payload: { + requestedModel: input.requestedModel, + effectiveModel: input.effectiveModel, + codexCliVersion: input.codexCliVersion, + }, + }); + await input.issuesSvc.addComment(input.issueId, message, { runId: input.runId }, { + authorType: "system", + presentation: { + kind: "system_notice", + tone: "warning", + title: `Using ${input.effectiveModel}`, + density: "compact", + detailsDefaultOpen: false, + }, + }); +} + function isTruthyRuntimeEnvValue(value: string | undefined) { return value === "true" || value === "1" || value === "yes" || value === "on"; } @@ -24501,6 +24535,38 @@ export function heartbeatService( instructionWorkingCopy: instructionCopy ? { rootPath: instructionCopy.executionRoot, entryPath: instructionCopy.entryFile, ...(isAgentDirectoryCopy(instructionCopy) ? { kind: "agent_files" as const } : {}) } : undefined, }); getNativeFreshSessionHandoff = nativeReviewRequest ? undefined : getFreshSessionHandoff; + const nativeProviderConfig = nativeRuntimeResolution.profile.backend === "codex_app_server" + || nativeRuntimeResolution.profile.backend === "opencode_server" + ? projectPaperclipRunnerTaskConfig( + nativeRuntimeResolution.profile.backend, + agent.adapterConfig, + issueAssigneeOverrides?.adapterConfig, + managedAiRuntime ? readNonEmptyString(resolvedConfig.model) ?? undefined : undefined, + ) + : agent.adapterConfig; + const requestedNativeProvider = resolvePaperclipRunnerNativeProviderInput({ + backend: nativeRuntimeResolution.profile.backend, + adapterConfig: nativeProviderConfig, managedProfile, agentCoreProfile, dotBinding, + }); + const codexCliVersion = agent.adapterType === "paperclip_runner" && requestedNativeProvider.provider === "codex" + ? await readRemoteCodexModelCliVersion({ + model: requestedNativeProvider.model, + target: executionTarget, + remoteCodexPath: runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH, + remoteCodexNpmSpec: runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC, + }) : null; + const nativeProvider = codexCliVersion + ? resolvePaperclipRunnerNativeProviderInput({ + backend: nativeRuntimeResolution.profile.backend, + adapterConfig: nativeProviderConfig, codexCliVersion, managedProfile, agentCoreProfile, dotBinding, + }) : requestedNativeProvider; + if (nativeProvider.model !== requestedNativeProvider.model) { + await postNativeModelFallbackWarning({ + issuesSvc, onEvent: onAdapterEvent, issueId: issueRef.id, runId: run.id, + requestedModel: requestedNativeProvider.model, effectiveModel: nativeProvider.model, + codexCliVersion: codexCliVersion!, + }); + } const buildExecution = ({ normalizedSessionId, resumedSession }: { normalizedSessionId: string; resumedSession: boolean }) => buildNativeExecutionInput({ agentKeyId: agentIdentity?.keyId, @@ -24575,21 +24641,7 @@ export function heartbeatService( : {}, } : null, - ...resolvePaperclipRunnerNativeProviderInput({ - backend: nativeRuntimeResolution.profile.backend, - adapterConfig: nativeRuntimeResolution.profile.backend === "codex_app_server" - || nativeRuntimeResolution.profile.backend === "opencode_server" - ? projectPaperclipRunnerTaskConfig( - nativeRuntimeResolution.profile.backend, - agent.adapterConfig, - issueAssigneeOverrides?.adapterConfig, - managedAiRuntime ? readNonEmptyString(resolvedConfig.model) ?? undefined : undefined, - ) - : agent.adapterConfig, - managedProfile, - agentCoreProfile, - dotBinding, - }), + ...nativeProvider, lifecyclePolicy: effectiveLifecyclePolicy, interactionResponses, completionContract: { diff --git a/server/src/services/native-runtime/codex-model-fallback.test.ts b/server/src/services/native-runtime/codex-model-fallback.test.ts new file mode 100644 index 00000000000..6967c29f6ab --- /dev/null +++ b/server/src/services/native-runtime/codex-model-fallback.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, vi } from "vitest"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { + codexModelFallbackCandidates, + compatibleCodexModel, + readRemoteCodexModelCliVersion, + selectCodexModelFallback, +} from "./codex-model-fallback.js"; + +describe("bounded Codex model fallback", () => { + it("keeps capability classes separate and appends the stable default once", () => { + expect(codexModelFallbackCandidates("gpt-6.1-sol")).toEqual(["gpt-6-sol", "gpt-5.6-sol"]); + expect(codexModelFallbackCandidates("gpt-6-luna")).toEqual(["gpt-5.6-luna", "gpt-5.6-sol"]); + expect(codexModelFallbackCandidates("gpt-6-astra")).toEqual(["gpt-5.6-sol"]); + expect(codexModelFallbackCandidates("gpt-5.6-sol")).toEqual([]); + }); + + it("uses the default after same-class candidates are unavailable, then stops", () => { + const compatible = vi.fn((model: string) => model === "gpt-5.6-sol"); + expect(selectCodexModelFallback("gpt-6-luna", compatible)).toBe("gpt-5.6-sol"); + expect(compatible.mock.calls).toEqual([["gpt-5.6-luna"], ["gpt-5.6-sol"]]); + const unavailable = vi.fn(() => false); + expect(selectCodexModelFallback("gpt-6.1-sol", unavailable)).toBeNull(); + expect(unavailable.mock.calls).toEqual([["gpt-6-sol"], ["gpt-5.6-sol"]]); + }); + + it.each([ + ["gpt-6.1-sol", "0.159.0"], ["gpt-6-sol", "0.157.0"], ["gpt-5.6-sol", "0.156.0"], + ["team-model", "0.156.0"], [null, "0.156.0"], ["gpt-6.1-sol", null], + ["gpt-6.1-sol", "0.148.9"], ["gpt-6.1-sol", "0.161.0"], ["gpt-6.1-sol", "0.160.0-alpha.1"], + ])("preserves %s on %s when no safe startup substitution applies", (model, version) => { + expect(compatibleCodexModel(model, version)).toBe(model); + }); +}); + +function remoteTarget(stdout = "codex-cli 0.156.0", overrides = {}) { + const execute = vi.fn(async (command: { command: string; args?: string[] }) => ({ + exitCode: 0, signal: null, timedOut: false, stderr: "", + stdout: command.command === "sh" ? "/opt/paperclip-runner/bin/codex\n" : stdout, + ...(command.command !== "sh" ? overrides : {}), + })); + const target = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + runner: { execute } } as unknown as AdapterExecutionTarget; + return { target, execute }; +} + +describe("remote model compatibility preflight", () => { + it.each(["discovery", "version"])("defers to launch verification when the %s probe rejects", async (phase) => { + const { target, execute } = remoteTarget(); + if (phase === "version") { + execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, + stderr: "", stdout: "/opt/paperclip-runner/bin/codex\n" }); + } + execute.mockRejectedValueOnce(new Error("temporary sandbox command failure")); + await expect(readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target })).resolves.toBeNull(); + expect(execute).toHaveBeenCalledTimes(phase === "version" ? 2 : 1); + }); + + it("checks the image CLI without invoking a provider turn or installing software", async () => { + const { target, execute } = remoteTarget(); + const version = await readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target }); + expect(version).toBe("0.156.0"); + expect(compatibleCodexModel("gpt-6.1-sol", version)).toBe("gpt-5.6-sol"); + expect(execute).toHaveBeenCalledTimes(2); + expect(execute).toHaveBeenLastCalledWith({ command: "/opt/paperclip-runner/bin/codex", + args: ["--version"], cwd: "/workspace", bypassSession: true, timeoutMs: 30_000 }); + }); + + it.each([ + { remoteCodexPath: "/controller/codex" }, + { remoteCodexNpmSpec: "@openai/codex@0.160.0" }, + { model: "gpt-5.6-sol" }, + { model: "custom-model" }, + { target: { kind: "local" } as AdapterExecutionTarget }, + ])("preserves explicit artifacts, install pins, and unqualified models: %j", async (overrides) => { + const { target, execute } = remoteTarget(); + expect(await readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target, ...overrides })).toBeNull(); + expect(execute).not.toHaveBeenCalled(); + }); + + it.each([ + ["codex-cli 0.148.9", {}], ["codex-cli 0.161.0", {}], ["codex-cli 0.160.0-alpha.1", {}], + ["codex-cli 0.156.0\ncodex-cli 0.160.0", {}], ["codex-cli 0.156.0", { timedOut: true }], + ["codex-cli 0.156.0", { exitCode: 1 }], + ])("keeps the launch verifier authoritative for invalid probes: %s %j", async (stdout, overrides) => { + expect(await readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target: remoteTarget(stdout, overrides).target })).toBeNull(); + }); +}); diff --git a/server/src/services/native-runtime/codex-model-fallback.ts b/server/src/services/native-runtime/codex-model-fallback.ts new file mode 100644 index 00000000000..dbf95b1b0d2 --- /dev/null +++ b/server/src/services/native-runtime/codex-model-fallback.ts @@ -0,0 +1,95 @@ +import { posix } from "node:path"; +import { + codexCliVersionAtLeast, + DEFAULT_CODEX_LOCAL_MODEL, + minimumCodexCliVersionForModel, + models, + normalizeCodexModel, +} from "@paperclipai/adapter-codex-local"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js"; +import { isSupportedRemoteCodexVersion, parseCodexCliVersion } from "./codex-runtime-compatibility.js"; + +/** Newest older model in the same class first, then the stable Runner default. */ +export function codexModelFallbackCandidates(requestedModel: string): string[] { + const requested = normalizeCodexModel(requestedModel); + const identity = /^gpt-(\d+(?:\.\d+)?)-(astra|sol|terra|luna)$/.exec(requested); + const sameClass = identity ? models.flatMap(({ id }) => { + const candidate = /^gpt-(\d+(?:\.\d+)?)-(astra|sol|terra|luna)$/.exec(id); + return candidate && candidate[2] === identity[2] && Number(candidate[1]) < Number(identity[1]) + ? [id] : []; + }) : []; + return [...new Set([...sameClass, DEFAULT_CODEX_LOCAL_MODEL])].filter((model) => model !== requested); +} + +export function compatibleCodexModel(model: string | null, version: string | null): string | null { + if (!version || !isSupportedRemoteCodexVersion(version)) return model; + const minimum = minimumCodexCliVersionForModel(model); + if (!minimum || codexCliVersionAtLeast(version, minimum)) return model; + return selectCodexModelFallback(model!, (candidate) => { + const floor = minimumCodexCliVersionForModel(candidate); + return !floor || codexCliVersionAtLeast(version, floor); + }) ?? model; +} + +export function selectCodexModelFallback(requestedModel: string, compatible: (model: string) => boolean): string | null { + for (const candidate of codexModelFallbackCandidates(requestedModel)) { + if (compatible(candidate)) return candidate; + } + // Keep the verifier's existing actionable failure when no candidate is safe. + return null; +} + +export function parseRemoteExecutableCandidate(stdout: string): string | null { + const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + if (lines.length !== 1) return null; + const candidate = lines[0]!; + if (!candidate.startsWith("/") || candidate.length > 4_096 || !/^\/[A-Za-z0-9_./+@-]+$/.test(candidate)) return null; + return posix.normalize(candidate); +} + +/** Shared with the launch verifier so preparation checks the same image CLI. */ +export async function discoverRemoteExecutable( + runner: CommandManagedRuntimeRunner, + cwd: string, + name: "paperclip-runnerd" | "codex", +): Promise { + const result = await runner.execute({ + command: "sh", + args: ["-c", `for candidate in /opt/paperclip-runner/bin/${name} "$HOME/.local/bin/${name}"; do ` + + `if [ -x "$candidate" ]; then printf '%s\\n' "$candidate"; exit 0; fi; done; ` + + `command -v ${name} 2>/dev/null || true`], + cwd, bypassSession: true, timeoutMs: 10_000, + }); + return result.exitCode === 0 && !result.timedOut ? parseRemoteExecutableCandidate(result.stdout) : null; +} + +export async function readRemoteCodexModelCliVersion(input: { + model: string | null; + target?: AdapterExecutionTarget | null; + remoteCodexPath?: string | null; + remoteCodexNpmSpec?: string | null; +}): Promise { + const target = input.target; + // An explicit artifact or install pin takes precedence over the image CLI. + if (target?.kind !== "remote" || !minimumCodexCliVersionForModel(input.model) + || input.remoteCodexPath?.trim() || input.remoteCodexNpmSpec?.trim()) return null; + try { + const runner = target.transport === "ssh" + ? createNativeSshCommandRunner({ spec: target.spec, defaultCwd: target.remoteCwd }) + : target.runner; + if (!runner) return null; + const executable = await discoverRemoteExecutable(runner, target.remoteCwd, "codex"); + if (!executable) return null; + const result = await runner.execute({ command: executable, args: ["--version"], + cwd: target.remoteCwd, bypassSession: true, timeoutMs: 30_000 }); + if (result.exitCode !== 0 || result.timedOut) return null; + const version = parseCodexCliVersion(`${result.stdout}\n${result.stderr}`); + return version && isSupportedRemoteCodexVersion(version) ? version : null; + } catch { + // Optional preparation probes must not bypass launch verification or its + // existing recovery policy when a remote command rejects or times out. + return null; + } +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index acfdca2bb40..da86c153195 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -56,6 +56,7 @@ import { } from "../../vendor/paperclip-runner/index.js"; import * as issueServiceModule from "../issues.js"; import { NativePermissionDeclinedError } from "./native-permission-decline.js"; +import { resolvePaperclipRunnerNativeProviderInput } from "./provider-profile.js"; import { createNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp, @@ -12061,24 +12062,31 @@ describe("runnerd provider runtime wiring", () => { }); it.each([ - { version: "0.156.0", model: "gpt-6.1-sol", floor: "0.159.0" }, - { version: "0.158.0", model: "gpt-6.1-sol", floor: "0.159.0" }, - { version: "0.156.0", model: "gpt-6-sol", floor: "0.157.0" }, - { version: "0.156.0", model: "gpt-6-luna", floor: "0.157.0" }, - { version: "0.159.0", model: "gpt-6.1-sol", floor: null }, - { version: "0.160.0", model: "gpt-6.1-sol", floor: null }, - { version: "0.157.0", model: "gpt-6-sol", floor: null }, - { version: "0.156.0", model: "gpt-5.6-sol", floor: null }, - { version: "0.156.0", model: null, floor: null }, - ])("rejects a sandbox Codex $version below the ChatGPT floor for $model before launch (floor=$floor)", async ({ version, model, floor }) => { + ...[ + { version: "0.156.0", model: "gpt-6.1-sol", floor: "0.159.0" }, + { version: "0.158.0", model: "gpt-6.1-sol", floor: "0.159.0" }, + { version: "0.156.0", model: "gpt-6-sol", floor: "0.157.0" }, + { version: "0.156.0", model: "gpt-6-luna", floor: "0.157.0" }, + { version: "0.159.0", model: "gpt-6.1-sol", floor: null }, + { version: "0.160.0", model: "gpt-6.1-sol", floor: null }, + { version: "0.157.0", model: "gpt-6-sol", floor: null }, + { version: "0.156.0", model: "gpt-5.6-sol", floor: null }, + { version: "0.156.0", model: null, floor: null }, + ].map((scenario) => ({ ...scenario, recover: false })), + { version: "0.156.0", model: "gpt-6.1-sol", floor: null, recover: true }, + { version: "0.158.0", model: "gpt-6.1-sol", floor: null, recover: true }, + ])("verifies sandbox Codex $version against the prepared $model selection (floor=$floor, recover=$recover)", async ({ version, model, floor, recover }) => { // A preinstalled Codex inside the compatibility window can still be too // old for the configured model: the ChatGPT backend rejects every turn // with "not supported when using Codex with a ChatGPT account". The // verifier names the stale image instead of letting the run fail as an // account problem. + const preparedModel = recover ? resolvePaperclipRunnerNativeProviderInput({ + backend: "codex_app_server", adapterConfig: { provider: "codex", model }, codexCliVersion: version, + }).model : model; const gatedExecution = { ...execution, - provider: { kind: "codex", model, approvalPolicy: "never" }, + provider: { kind: "codex", model: preparedModel, approvalPolicy: "never" }, binding: { ...execution.binding, runId: `run-codex-floor-${version}-${model ?? "default"}` }, } as NativeExecutionInputV1; const onLog = vi.fn(async () => undefined); @@ -12136,6 +12144,11 @@ describe("runnerd provider runtime wiring", () => { .calls[0]![0] as RunnerTransportOptions & { controlPlaneRegistration: (authority: unknown) => Promise; }; + if (recover) { + expect(state.createBackend.mock.calls.at(-1)![0].provider.model).toBe( + version === "0.158.0" ? "gpt-6-sol" : "gpt-5.6-sol", + ); + } await expect(transport.controlPlaneRegistration({})).rejects.toThrow( floor ? `runner_remote_provider_artifact_incompatible: ${model} requires Codex ${floor} or newer with ChatGPT sign-in, received ${version} from the sandbox image; promote a sandbox image with Codex 0.160.0 or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@0.160.0` diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 6d7104b2378..a85a64bd0e7 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -112,6 +112,8 @@ import { } from "../../vendor/paperclip-runner/index.js"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js"; +import { discoverRemoteExecutable, parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; +export { parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import { resolvePaperclipRunnerTransport, @@ -9743,23 +9745,6 @@ export function buildRemoteCodexLauncherCommand( ); } -export function parseRemoteExecutableCandidate(stdout: string): string | null { - const lines = stdout - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - if (lines.length !== 1) return null; - const candidate = lines[0]!; - if ( - !candidate.startsWith("/") || - candidate.length > 4_096 || - !/^\/[A-Za-z0-9_./+@-]+$/.test(candidate) - ) { - return null; - } - return posix.normalize(candidate); -} - export function mayUsePreinstalledRunnerArtifact( configuredRemoteBinaryPath: string | null | undefined, ): boolean { @@ -11550,20 +11535,7 @@ async function createRunnerdBackendWithinSessionClaim( name: "paperclip-runnerd" | "codex", ) => { if (!remoteTarget || !remoteCommandRunner) return null; - const result = await remoteCommandRunner.execute({ - command: "sh", - args: [ - "-c", - `for candidate in /opt/paperclip-runner/bin/${name} "$HOME/.local/bin/${name}"; do ` + - `if [ -x "$candidate" ]; then printf '%s\\n' "$candidate"; exit 0; fi; done; ` + - `command -v ${name} 2>/dev/null || true`, - ], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 10_000, - }); - if (result.exitCode !== 0 || result.timedOut) return null; - return parseRemoteExecutableCandidate(result.stdout); + return discoverRemoteExecutable(remoteCommandRunner, remoteTarget.remoteCwd, name); }; const linkPreinstalledExecutable = async ( diff --git a/server/src/services/native-runtime/provider-profile.ts b/server/src/services/native-runtime/provider-profile.ts index 0a2e0a89886..114963a68a0 100644 --- a/server/src/services/native-runtime/provider-profile.ts +++ b/server/src/services/native-runtime/provider-profile.ts @@ -16,6 +16,7 @@ import { CLAUDE_MANAGED_QUALIFIED_MODEL, } from "../provider-profile-qualification.js"; import { resolveAcpxQualification, type AcpxQualificationCandidate } from "./acpx-qualification.js"; +import { compatibleCodexModel } from "./codex-model-fallback.js"; export const QUALIFIED_OPENCODE_RUNNER_VERSION = "1.18.34" as const; export const DEFAULT_OPENCODE_RUNNER_MODEL = @@ -520,6 +521,8 @@ export function resolvePaperclipRunnerProviderProfile( export function resolvePaperclipRunnerNativeProviderInput(input: { backend: PaperclipRunnerProviderProfile["backend"]; adapterConfig: unknown; + /** Verified image CLI version for a fresh remote Codex run only. */ + codexCliVersion?: string | null; dotBinding?: import("../../vendor/paperclip-runner/index.js").DotBindingSnapshot; managedProfile?: { id: string; @@ -738,7 +741,7 @@ export function resolvePaperclipRunnerNativeProviderInput(input: { } return { provider: "codex", - model: profile.model, + model: compatibleCodexModel(profile.model, input.codexCliVersion ?? null), codexApprovalPolicy: resolvePaperclipRunnerPermissionMode( "codex", config.codexPermissionMode,