diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 0151a641fda..43d0b7e67c1 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1289,6 +1289,15 @@ agent workspace. The host `HOME` itself, a directory that contains it, a filesystem root, a `CODEX_HOME` overlap, or a canonical path outside the assigned workspace is rejected before provider startup. +Fresh remote Codex Runner runs recover from a supported image CLI that is too old +for the selected model. Preparation selects the closest compatible older model +of the same class, then the stable Runner default. The task shows a warning with +the requested model, effective model, and CLI version. Agent and task settings +stay unchanged. Update the image CLI to restore the requested model on later +runs. Explicit `PAPERCLIP_RUNNER_REMOTE_CODEX_PATH` and +`PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC` settings take precedence. See +[execution semantics](execution-semantics.md#remote-codex-model-compatibility). + ### Sandbox ACP input delivery The legacy sandbox process bridge retries recognized Daytona and Cloudflare diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 1ad4d830359..df952bf5250 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -2080,6 +2080,16 @@ remove stale instructions after edits or access revocation. See [Connection instructions](connections/CONNECTION-INSTRUCTIONS.md) for contracts, UI conventions, custom adapter integration, and initial memory templates. +### Remote Codex model compatibility + +Fresh remote Codex Runner preparation can replace a model whose verified CLI +minimum exceeds the supported image CLI version. It selects a compatible older +model in the same class, then the stable Runner default, and saves that effective +model before checkpoint selection. A visible task warning and local run-log event +record the substitution. This does not change agent settings, rewrite admitted +executions, or bypass artifact, ownership, permission, and budget gates. See +[remote Codex compatibility](execution-semantics.md#remote-codex-model-compatibility). + ### Native provider capacity retry Committed, run-bound Codex `serverOverloaded` terminal failures display the model diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index c923e18c66f..fe62494d668 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -517,6 +517,24 @@ Warm sandbox reuse must match the current host Git tip and branch as well as the file snapshot and saved stamp, including managed nested repositories. A history or branch mismatch restages the host before the next run begins. +### Remote Codex model compatibility + +Fresh remote Codex Runner runs check the selected image CLI before saving the +native execution input. If the CLI is in the supported app-server version window +but below the selected model's verified minimum, preparation chooses the newest +compatible older model of the same class (`sol`, `luna`, `astra`, or `terra`), +then the stable Runner default. Each candidate is considered once; this selection +does not replay a provider turn or consume a failure-retry attempt. + +The effective model is saved in the execution input before checkpoint selection, +so launch, recovery, and usage accounting share that identity. The requested +agent and task settings stay unchanged. A task warning and `runner.model_fallback` +run-log event name both models and the image CLI version. An explicit remote +Codex artifact or npm install pin retains precedence. Persisted executions are +not rewritten, and unknown models, invalid CLI builds, and artifact failures keep +their existing verification and recovery rules. Capacity and authentication +failures do not trigger this startup substitution. + ### Native provider model capacity A committed Codex `turn.failed` event with `codexErrorInfo: serverOverloaded`, diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 10557d8b6bb..9641c092e72 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -7,6 +7,12 @@ event needs no operator endpoint. ## Native PRP Run-Log Events +Fresh remote Codex model substitution emits `runner.model_fallback` on the +system stream with `warn` severity. Its payload contains `requestedModel`, +`effectiveModel`, and `codexCliVersion`. It records a preparation choice before +provider launch, not a failed turn or a retry. The same substitution appears as +a system warning in the task conversation. It remains local run-log data. + The hidden native coordinator writes each validated PRP event to the bound run's existing event stream before it acknowledges the runner. The row keeps the PRP `eventType`, source instance, source event ID, source sequence, protocol diff --git a/server/src/__tests__/heartbeat-model-fallback-warning.test.ts b/server/src/__tests__/heartbeat-model-fallback-warning.test.ts new file mode 100644 index 00000000000..d9c07db02fe --- /dev/null +++ b/server/src/__tests__/heartbeat-model-fallback-warning.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it, vi } from "vitest"; +import { postNativeModelFallbackWarning } from "../services/heartbeat.js"; + +describe("model fallback warning", () => { + it("records the substitution and shows a run-attributed warning on the task", async () => { + const addComment = vi.fn(async () => ({} as never)); + const onEvent = vi.fn(async () => undefined); + await postNativeModelFallbackWarning({ issuesSvc: { addComment }, onEvent, + issueId: "issue", runId: "run", requestedModel: "gpt-6.1-sol", effectiveModel: "gpt-5.6-sol", codexCliVersion: "0.156.0" }); + expect(onEvent).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ + eventType: "runner.model_fallback", stream: "system", level: "warn", + payload: { requestedModel: "gpt-6.1-sol", effectiveModel: "gpt-5.6-sol", codexCliVersion: "0.156.0" }, + })); + expect(addComment).toHaveBeenCalledExactlyOnceWith("issue", expect.stringContaining("Work will continue with the compatible model"), + { runId: "run" }, { authorType: "system", + presentation: { kind: "system_notice", tone: "warning", title: "Using gpt-5.6-sol", density: "compact", detailsDefaultOpen: false } }); + expect(onEvent.mock.invocationCallOrder[0]).toBeLessThan(addComment.mock.invocationCallOrder[0]!); + }); +}); diff --git a/server/src/services/heartbeat-runner-provider-config.test.ts b/server/src/services/heartbeat-runner-provider-config.test.ts index 1787dbcc89f..0da343e0a37 100644 --- a/server/src/services/heartbeat-runner-provider-config.test.ts +++ b/server/src/services/heartbeat-runner-provider-config.test.ts @@ -8,6 +8,28 @@ import { } from "./native-runtime/provider-profile.js"; describe("Paperclip Runner native provider configuration", () => { + it.each([ + ["gpt-6.1-sol", "0.158.0", "gpt-6-sol"], + ["gpt-6.1-sol", "0.156.0", "gpt-5.6-sol"], + ["gpt-6-luna", "0.156.0", "gpt-5.6-luna"], + ])("recovers %s on Codex %s with compatible model %s", (model, codexCliVersion, expectedModel) => { + const input = { + backend: "codex_app_server" as const, + adapterConfig: { provider: "codex", model, modelReasoningEffort: "high", codexPermissionMode: "never" }, + codexCliVersion, + }; + expect(resolvePaperclipRunnerNativeProviderInput(input)).toEqual({ + provider: "codex", model: expectedModel, codexApprovalPolicy: "never", codexReasoningEffort: "high", + }); + expect(input.adapterConfig.model).toBe(model); + }); + + it.each([ + ["gpt-6.1-sol", "0.159.0"], ["gpt-6-sol", "0.157.0"], ["gpt-5.6-sol", "0.156.0"], + ])("keeps a supported %s selection on Codex %s", (model, codexCliVersion) => { + expect(resolvePaperclipRunnerNativeProviderInput({ backend: "codex_app_server", + adapterConfig: { provider: "codex", model }, codexCliVersion })).toMatchObject({ model }); + }); it.each([undefined, "approve-all", "approve-paperclip", "approve-reads", "deny-all"])( "passes Grok's full-auto default or explicit %s policy to the native runner", (acpxPermissionMode) => { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 9c89809c661..c47132dbf2c 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -287,6 +287,7 @@ import { projectPaperclipRunnerTaskConfig, resolvePaperclipRunnerNativeProviderInput, } from "./native-runtime/provider-profile.js"; +import { readRemoteCodexModelCliVersion } from "./native-runtime/codex-model-fallback.js"; import { buildNativeHeartbeatPreparationSpans, buildNativeWakeIngressSpan, @@ -9640,6 +9641,39 @@ export function postWorkspaceReadyComment(input: { ); } +export async function postNativeModelFallbackWarning(input: { + issuesSvc: Pick, "addComment">; + onEvent: (event: AdapterRuntimeEvent) => Promise; + issueId: string; + runId: string; + requestedModel: string | null; + effectiveModel: string | null; + codexCliVersion: string; +}): Promise { + const message = `Using ${input.effectiveModel} because the sandbox's Codex ${input.codexCliVersion} does not support ${input.requestedModel}. Work will continue with the compatible model. Update the sandbox's Codex CLI to use the requested model.`; + await input.onEvent({ + eventType: "runner.model_fallback", + stream: "system", + level: "warn", + message, + payload: { + requestedModel: input.requestedModel, + effectiveModel: input.effectiveModel, + codexCliVersion: input.codexCliVersion, + }, + }); + await input.issuesSvc.addComment(input.issueId, message, { runId: input.runId }, { + authorType: "system", + presentation: { + kind: "system_notice", + tone: "warning", + title: `Using ${input.effectiveModel}`, + density: "compact", + detailsDefaultOpen: false, + }, + }); +} + function isTruthyRuntimeEnvValue(value: string | undefined) { return value === "true" || value === "1" || value === "yes" || value === "on"; } @@ -24501,6 +24535,38 @@ export function heartbeatService( instructionWorkingCopy: instructionCopy ? { rootPath: instructionCopy.executionRoot, entryPath: instructionCopy.entryFile, ...(isAgentDirectoryCopy(instructionCopy) ? { kind: "agent_files" as const } : {}) } : undefined, }); getNativeFreshSessionHandoff = nativeReviewRequest ? undefined : getFreshSessionHandoff; + const nativeProviderConfig = nativeRuntimeResolution.profile.backend === "codex_app_server" + || nativeRuntimeResolution.profile.backend === "opencode_server" + ? projectPaperclipRunnerTaskConfig( + nativeRuntimeResolution.profile.backend, + agent.adapterConfig, + issueAssigneeOverrides?.adapterConfig, + managedAiRuntime ? readNonEmptyString(resolvedConfig.model) ?? undefined : undefined, + ) + : agent.adapterConfig; + const requestedNativeProvider = resolvePaperclipRunnerNativeProviderInput({ + backend: nativeRuntimeResolution.profile.backend, + adapterConfig: nativeProviderConfig, managedProfile, agentCoreProfile, dotBinding, + }); + const codexCliVersion = agent.adapterType === "paperclip_runner" && requestedNativeProvider.provider === "codex" + ? await readRemoteCodexModelCliVersion({ + model: requestedNativeProvider.model, + target: executionTarget, + remoteCodexPath: runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH, + remoteCodexNpmSpec: runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC, + }) : null; + const nativeProvider = codexCliVersion + ? resolvePaperclipRunnerNativeProviderInput({ + backend: nativeRuntimeResolution.profile.backend, + adapterConfig: nativeProviderConfig, codexCliVersion, managedProfile, agentCoreProfile, dotBinding, + }) : requestedNativeProvider; + if (nativeProvider.model !== requestedNativeProvider.model) { + await postNativeModelFallbackWarning({ + issuesSvc, onEvent: onAdapterEvent, issueId: issueRef.id, runId: run.id, + requestedModel: requestedNativeProvider.model, effectiveModel: nativeProvider.model, + codexCliVersion: codexCliVersion!, + }); + } const buildExecution = ({ normalizedSessionId, resumedSession }: { normalizedSessionId: string; resumedSession: boolean }) => buildNativeExecutionInput({ agentKeyId: agentIdentity?.keyId, @@ -24575,21 +24641,7 @@ export function heartbeatService( : {}, } : null, - ...resolvePaperclipRunnerNativeProviderInput({ - backend: nativeRuntimeResolution.profile.backend, - adapterConfig: nativeRuntimeResolution.profile.backend === "codex_app_server" - || nativeRuntimeResolution.profile.backend === "opencode_server" - ? projectPaperclipRunnerTaskConfig( - nativeRuntimeResolution.profile.backend, - agent.adapterConfig, - issueAssigneeOverrides?.adapterConfig, - managedAiRuntime ? readNonEmptyString(resolvedConfig.model) ?? undefined : undefined, - ) - : agent.adapterConfig, - managedProfile, - agentCoreProfile, - dotBinding, - }), + ...nativeProvider, lifecyclePolicy: effectiveLifecyclePolicy, interactionResponses, completionContract: { diff --git a/server/src/services/native-runtime/codex-model-fallback.test.ts b/server/src/services/native-runtime/codex-model-fallback.test.ts new file mode 100644 index 00000000000..6967c29f6ab --- /dev/null +++ b/server/src/services/native-runtime/codex-model-fallback.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, vi } from "vitest"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { + codexModelFallbackCandidates, + compatibleCodexModel, + readRemoteCodexModelCliVersion, + selectCodexModelFallback, +} from "./codex-model-fallback.js"; + +describe("bounded Codex model fallback", () => { + it("keeps capability classes separate and appends the stable default once", () => { + expect(codexModelFallbackCandidates("gpt-6.1-sol")).toEqual(["gpt-6-sol", "gpt-5.6-sol"]); + expect(codexModelFallbackCandidates("gpt-6-luna")).toEqual(["gpt-5.6-luna", "gpt-5.6-sol"]); + expect(codexModelFallbackCandidates("gpt-6-astra")).toEqual(["gpt-5.6-sol"]); + expect(codexModelFallbackCandidates("gpt-5.6-sol")).toEqual([]); + }); + + it("uses the default after same-class candidates are unavailable, then stops", () => { + const compatible = vi.fn((model: string) => model === "gpt-5.6-sol"); + expect(selectCodexModelFallback("gpt-6-luna", compatible)).toBe("gpt-5.6-sol"); + expect(compatible.mock.calls).toEqual([["gpt-5.6-luna"], ["gpt-5.6-sol"]]); + const unavailable = vi.fn(() => false); + expect(selectCodexModelFallback("gpt-6.1-sol", unavailable)).toBeNull(); + expect(unavailable.mock.calls).toEqual([["gpt-6-sol"], ["gpt-5.6-sol"]]); + }); + + it.each([ + ["gpt-6.1-sol", "0.159.0"], ["gpt-6-sol", "0.157.0"], ["gpt-5.6-sol", "0.156.0"], + ["team-model", "0.156.0"], [null, "0.156.0"], ["gpt-6.1-sol", null], + ["gpt-6.1-sol", "0.148.9"], ["gpt-6.1-sol", "0.161.0"], ["gpt-6.1-sol", "0.160.0-alpha.1"], + ])("preserves %s on %s when no safe startup substitution applies", (model, version) => { + expect(compatibleCodexModel(model, version)).toBe(model); + }); +}); + +function remoteTarget(stdout = "codex-cli 0.156.0", overrides = {}) { + const execute = vi.fn(async (command: { command: string; args?: string[] }) => ({ + exitCode: 0, signal: null, timedOut: false, stderr: "", + stdout: command.command === "sh" ? "/opt/paperclip-runner/bin/codex\n" : stdout, + ...(command.command !== "sh" ? overrides : {}), + })); + const target = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + runner: { execute } } as unknown as AdapterExecutionTarget; + return { target, execute }; +} + +describe("remote model compatibility preflight", () => { + it.each(["discovery", "version"])("defers to launch verification when the %s probe rejects", async (phase) => { + const { target, execute } = remoteTarget(); + if (phase === "version") { + execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, + stderr: "", stdout: "/opt/paperclip-runner/bin/codex\n" }); + } + execute.mockRejectedValueOnce(new Error("temporary sandbox command failure")); + await expect(readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target })).resolves.toBeNull(); + expect(execute).toHaveBeenCalledTimes(phase === "version" ? 2 : 1); + }); + + it("checks the image CLI without invoking a provider turn or installing software", async () => { + const { target, execute } = remoteTarget(); + const version = await readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target }); + expect(version).toBe("0.156.0"); + expect(compatibleCodexModel("gpt-6.1-sol", version)).toBe("gpt-5.6-sol"); + expect(execute).toHaveBeenCalledTimes(2); + expect(execute).toHaveBeenLastCalledWith({ command: "/opt/paperclip-runner/bin/codex", + args: ["--version"], cwd: "/workspace", bypassSession: true, timeoutMs: 30_000 }); + }); + + it.each([ + { remoteCodexPath: "/controller/codex" }, + { remoteCodexNpmSpec: "@openai/codex@0.160.0" }, + { model: "gpt-5.6-sol" }, + { model: "custom-model" }, + { target: { kind: "local" } as AdapterExecutionTarget }, + ])("preserves explicit artifacts, install pins, and unqualified models: %j", async (overrides) => { + const { target, execute } = remoteTarget(); + expect(await readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target, ...overrides })).toBeNull(); + expect(execute).not.toHaveBeenCalled(); + }); + + it.each([ + ["codex-cli 0.148.9", {}], ["codex-cli 0.161.0", {}], ["codex-cli 0.160.0-alpha.1", {}], + ["codex-cli 0.156.0\ncodex-cli 0.160.0", {}], ["codex-cli 0.156.0", { timedOut: true }], + ["codex-cli 0.156.0", { exitCode: 1 }], + ])("keeps the launch verifier authoritative for invalid probes: %s %j", async (stdout, overrides) => { + expect(await readRemoteCodexModelCliVersion({ model: "gpt-6.1-sol", target: remoteTarget(stdout, overrides).target })).toBeNull(); + }); +}); diff --git a/server/src/services/native-runtime/codex-model-fallback.ts b/server/src/services/native-runtime/codex-model-fallback.ts new file mode 100644 index 00000000000..dbf95b1b0d2 --- /dev/null +++ b/server/src/services/native-runtime/codex-model-fallback.ts @@ -0,0 +1,95 @@ +import { posix } from "node:path"; +import { + codexCliVersionAtLeast, + DEFAULT_CODEX_LOCAL_MODEL, + minimumCodexCliVersionForModel, + models, + normalizeCodexModel, +} from "@paperclipai/adapter-codex-local"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js"; +import { isSupportedRemoteCodexVersion, parseCodexCliVersion } from "./codex-runtime-compatibility.js"; + +/** Newest older model in the same class first, then the stable Runner default. */ +export function codexModelFallbackCandidates(requestedModel: string): string[] { + const requested = normalizeCodexModel(requestedModel); + const identity = /^gpt-(\d+(?:\.\d+)?)-(astra|sol|terra|luna)$/.exec(requested); + const sameClass = identity ? models.flatMap(({ id }) => { + const candidate = /^gpt-(\d+(?:\.\d+)?)-(astra|sol|terra|luna)$/.exec(id); + return candidate && candidate[2] === identity[2] && Number(candidate[1]) < Number(identity[1]) + ? [id] : []; + }) : []; + return [...new Set([...sameClass, DEFAULT_CODEX_LOCAL_MODEL])].filter((model) => model !== requested); +} + +export function compatibleCodexModel(model: string | null, version: string | null): string | null { + if (!version || !isSupportedRemoteCodexVersion(version)) return model; + const minimum = minimumCodexCliVersionForModel(model); + if (!minimum || codexCliVersionAtLeast(version, minimum)) return model; + return selectCodexModelFallback(model!, (candidate) => { + const floor = minimumCodexCliVersionForModel(candidate); + return !floor || codexCliVersionAtLeast(version, floor); + }) ?? model; +} + +export function selectCodexModelFallback(requestedModel: string, compatible: (model: string) => boolean): string | null { + for (const candidate of codexModelFallbackCandidates(requestedModel)) { + if (compatible(candidate)) return candidate; + } + // Keep the verifier's existing actionable failure when no candidate is safe. + return null; +} + +export function parseRemoteExecutableCandidate(stdout: string): string | null { + const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + if (lines.length !== 1) return null; + const candidate = lines[0]!; + if (!candidate.startsWith("/") || candidate.length > 4_096 || !/^\/[A-Za-z0-9_./+@-]+$/.test(candidate)) return null; + return posix.normalize(candidate); +} + +/** Shared with the launch verifier so preparation checks the same image CLI. */ +export async function discoverRemoteExecutable( + runner: CommandManagedRuntimeRunner, + cwd: string, + name: "paperclip-runnerd" | "codex", +): Promise { + const result = await runner.execute({ + command: "sh", + args: ["-c", `for candidate in /opt/paperclip-runner/bin/${name} "$HOME/.local/bin/${name}"; do ` + + `if [ -x "$candidate" ]; then printf '%s\\n' "$candidate"; exit 0; fi; done; ` + + `command -v ${name} 2>/dev/null || true`], + cwd, bypassSession: true, timeoutMs: 10_000, + }); + return result.exitCode === 0 && !result.timedOut ? parseRemoteExecutableCandidate(result.stdout) : null; +} + +export async function readRemoteCodexModelCliVersion(input: { + model: string | null; + target?: AdapterExecutionTarget | null; + remoteCodexPath?: string | null; + remoteCodexNpmSpec?: string | null; +}): Promise { + const target = input.target; + // An explicit artifact or install pin takes precedence over the image CLI. + if (target?.kind !== "remote" || !minimumCodexCliVersionForModel(input.model) + || input.remoteCodexPath?.trim() || input.remoteCodexNpmSpec?.trim()) return null; + try { + const runner = target.transport === "ssh" + ? createNativeSshCommandRunner({ spec: target.spec, defaultCwd: target.remoteCwd }) + : target.runner; + if (!runner) return null; + const executable = await discoverRemoteExecutable(runner, target.remoteCwd, "codex"); + if (!executable) return null; + const result = await runner.execute({ command: executable, args: ["--version"], + cwd: target.remoteCwd, bypassSession: true, timeoutMs: 30_000 }); + if (result.exitCode !== 0 || result.timedOut) return null; + const version = parseCodexCliVersion(`${result.stdout}\n${result.stderr}`); + return version && isSupportedRemoteCodexVersion(version) ? version : null; + } catch { + // Optional preparation probes must not bypass launch verification or its + // existing recovery policy when a remote command rejects or times out. + return null; + } +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index acfdca2bb40..da86c153195 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -56,6 +56,7 @@ import { } from "../../vendor/paperclip-runner/index.js"; import * as issueServiceModule from "../issues.js"; import { NativePermissionDeclinedError } from "./native-permission-decline.js"; +import { resolvePaperclipRunnerNativeProviderInput } from "./provider-profile.js"; import { createNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp, @@ -12061,24 +12062,31 @@ describe("runnerd provider runtime wiring", () => { }); it.each([ - { version: "0.156.0", model: "gpt-6.1-sol", floor: "0.159.0" }, - { version: "0.158.0", model: "gpt-6.1-sol", floor: "0.159.0" }, - { version: "0.156.0", model: "gpt-6-sol", floor: "0.157.0" }, - { version: "0.156.0", model: "gpt-6-luna", floor: "0.157.0" }, - { version: "0.159.0", model: "gpt-6.1-sol", floor: null }, - { version: "0.160.0", model: "gpt-6.1-sol", floor: null }, - { version: "0.157.0", model: "gpt-6-sol", floor: null }, - { version: "0.156.0", model: "gpt-5.6-sol", floor: null }, - { version: "0.156.0", model: null, floor: null }, - ])("rejects a sandbox Codex $version below the ChatGPT floor for $model before launch (floor=$floor)", async ({ version, model, floor }) => { + ...[ + { version: "0.156.0", model: "gpt-6.1-sol", floor: "0.159.0" }, + { version: "0.158.0", model: "gpt-6.1-sol", floor: "0.159.0" }, + { version: "0.156.0", model: "gpt-6-sol", floor: "0.157.0" }, + { version: "0.156.0", model: "gpt-6-luna", floor: "0.157.0" }, + { version: "0.159.0", model: "gpt-6.1-sol", floor: null }, + { version: "0.160.0", model: "gpt-6.1-sol", floor: null }, + { version: "0.157.0", model: "gpt-6-sol", floor: null }, + { version: "0.156.0", model: "gpt-5.6-sol", floor: null }, + { version: "0.156.0", model: null, floor: null }, + ].map((scenario) => ({ ...scenario, recover: false })), + { version: "0.156.0", model: "gpt-6.1-sol", floor: null, recover: true }, + { version: "0.158.0", model: "gpt-6.1-sol", floor: null, recover: true }, + ])("verifies sandbox Codex $version against the prepared $model selection (floor=$floor, recover=$recover)", async ({ version, model, floor, recover }) => { // A preinstalled Codex inside the compatibility window can still be too // old for the configured model: the ChatGPT backend rejects every turn // with "not supported when using Codex with a ChatGPT account". The // verifier names the stale image instead of letting the run fail as an // account problem. + const preparedModel = recover ? resolvePaperclipRunnerNativeProviderInput({ + backend: "codex_app_server", adapterConfig: { provider: "codex", model }, codexCliVersion: version, + }).model : model; const gatedExecution = { ...execution, - provider: { kind: "codex", model, approvalPolicy: "never" }, + provider: { kind: "codex", model: preparedModel, approvalPolicy: "never" }, binding: { ...execution.binding, runId: `run-codex-floor-${version}-${model ?? "default"}` }, } as NativeExecutionInputV1; const onLog = vi.fn(async () => undefined); @@ -12136,6 +12144,11 @@ describe("runnerd provider runtime wiring", () => { .calls[0]![0] as RunnerTransportOptions & { controlPlaneRegistration: (authority: unknown) => Promise; }; + if (recover) { + expect(state.createBackend.mock.calls.at(-1)![0].provider.model).toBe( + version === "0.158.0" ? "gpt-6-sol" : "gpt-5.6-sol", + ); + } await expect(transport.controlPlaneRegistration({})).rejects.toThrow( floor ? `runner_remote_provider_artifact_incompatible: ${model} requires Codex ${floor} or newer with ChatGPT sign-in, received ${version} from the sandbox image; promote a sandbox image with Codex 0.160.0 or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@0.160.0` diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 6d7104b2378..a85a64bd0e7 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -112,6 +112,8 @@ import { } from "../../vendor/paperclip-runner/index.js"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js"; +import { discoverRemoteExecutable, parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; +export { parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import { resolvePaperclipRunnerTransport, @@ -9743,23 +9745,6 @@ export function buildRemoteCodexLauncherCommand( ); } -export function parseRemoteExecutableCandidate(stdout: string): string | null { - const lines = stdout - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - if (lines.length !== 1) return null; - const candidate = lines[0]!; - if ( - !candidate.startsWith("/") || - candidate.length > 4_096 || - !/^\/[A-Za-z0-9_./+@-]+$/.test(candidate) - ) { - return null; - } - return posix.normalize(candidate); -} - export function mayUsePreinstalledRunnerArtifact( configuredRemoteBinaryPath: string | null | undefined, ): boolean { @@ -11550,20 +11535,7 @@ async function createRunnerdBackendWithinSessionClaim( name: "paperclip-runnerd" | "codex", ) => { if (!remoteTarget || !remoteCommandRunner) return null; - const result = await remoteCommandRunner.execute({ - command: "sh", - args: [ - "-c", - `for candidate in /opt/paperclip-runner/bin/${name} "$HOME/.local/bin/${name}"; do ` + - `if [ -x "$candidate" ]; then printf '%s\\n' "$candidate"; exit 0; fi; done; ` + - `command -v ${name} 2>/dev/null || true`, - ], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 10_000, - }); - if (result.exitCode !== 0 || result.timedOut) return null; - return parseRemoteExecutableCandidate(result.stdout); + return discoverRemoteExecutable(remoteCommandRunner, remoteTarget.remoteCwd, name); }; const linkPreinstalledExecutable = async ( diff --git a/server/src/services/native-runtime/provider-profile.ts b/server/src/services/native-runtime/provider-profile.ts index 0a2e0a89886..114963a68a0 100644 --- a/server/src/services/native-runtime/provider-profile.ts +++ b/server/src/services/native-runtime/provider-profile.ts @@ -16,6 +16,7 @@ import { CLAUDE_MANAGED_QUALIFIED_MODEL, } from "../provider-profile-qualification.js"; import { resolveAcpxQualification, type AcpxQualificationCandidate } from "./acpx-qualification.js"; +import { compatibleCodexModel } from "./codex-model-fallback.js"; export const QUALIFIED_OPENCODE_RUNNER_VERSION = "1.18.34" as const; export const DEFAULT_OPENCODE_RUNNER_MODEL = @@ -520,6 +521,8 @@ export function resolvePaperclipRunnerProviderProfile( export function resolvePaperclipRunnerNativeProviderInput(input: { backend: PaperclipRunnerProviderProfile["backend"]; adapterConfig: unknown; + /** Verified image CLI version for a fresh remote Codex run only. */ + codexCliVersion?: string | null; dotBinding?: import("../../vendor/paperclip-runner/index.js").DotBindingSnapshot; managedProfile?: { id: string; @@ -738,7 +741,7 @@ export function resolvePaperclipRunnerNativeProviderInput(input: { } return { provider: "codex", - model: profile.model, + model: compatibleCodexModel(profile.model, input.codexCliVersion ?? null), codexApprovalPolicy: resolvePaperclipRunnerPermissionMode( "codex", config.codexPermissionMode,