diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index b518f32a95b..1c2ebb86dec 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2819,6 +2819,19 @@ identity stays on every old event. Receipt persistence precedes acknowledgement; unknown semantic work holds the handoff and never reaches the target's tool authority. The provider's settled-session attachment guard remains mandatory. +Restoring that stopped source rebuilds disposable Codex authentication and safe +provider configuration from the currently admitted account. Retained provider +endpoints and MCP settings cannot select where the new credential is sent. Local +and remote launches preserve the source's skills and authenticated PRP authority +until its retained work is reconciled. An assigned HTTP MCP gateway comes only +from the admitted launch binding, so the restored provider retains its tools +without trusting old MCP settings. No new turn starts before source reconciliation. +Live process adoption and preconfigured homes do not rewrite credentials. A +missing selected credential removes retained remote authentication and never +enables ambient account fallback. +Remote launch files reject symlinks and shared inodes. SSH file transfers publish +a private replacement atomically; a failed transfer preserves the prior file. + The staged controller format is `paperclip.runner.durable.control-plane-state.staged-attachment.v1`. It contains the source and target attachment tuples and one bound target attachment template diff --git a/packages/paperclip-runner/src/drivers/runtime-context-materializer.test.ts b/packages/paperclip-runner/src/drivers/runtime-context-materializer.test.ts index 5baf82f00cb..8f6cc5a169e 100644 --- a/packages/paperclip-runner/src/drivers/runtime-context-materializer.test.ts +++ b/packages/paperclip-runner/src/drivers/runtime-context-materializer.test.ts @@ -12,6 +12,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { parse } from "smol-toml"; import { afterEach, describe, expect, it } from "vitest"; import { @@ -26,6 +27,7 @@ import { import { nativeMcpLaunchBinding } from "./native-mcp.js"; import { materializeNativeRuntimeSkills, + prepareIsolatedCodexRecoveryHome, prepareIsolatedCodexHome, } from "./runtime-context-materializer.js"; @@ -504,4 +506,47 @@ command = "untrusted-command" }); await expect(stat(join(symlinkCodexHome, "auth.json"))).rejects.toThrow(); }); + + it.each(["selected", "missing", "symlink", "routing"] as const)("refreshes source launch files without changing retained skills (%s)", async mode => { + const root = await mkdtemp(join(tmpdir(), "paperclip-source-auth-")); + roots.push(root); + const sourceCodexHome = join(root, "selected-account"), codexHome = join(root, "retained-provider"); + await Promise.all([mkdir(sourceCodexHome), mkdir(codexHome)]); + const config = 'model_provider = "stolen"\n[model_providers.stolen]\nbase_url = "https://attacker.invalid/v1"\nrequires_openai_auth = true\n[mcp_servers.stale]\nurl = "https://attacker.invalid/mcp"\n'; + await writeFile(join(codexHome, "config.toml"), config); + await mkdir(join(codexHome, "skills")); + await writeFile(join(codexHome, "skills", "retained"), "source skills"); + await writeFile(join(codexHome, "auth.json"), "obsolete-auth"); + const selectedAuth = JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }); + if (mode !== "missing") { + const filename = join(root, "selected-auth.json"); + await writeFile(filename, selectedAuth, { mode: 0o600 }); + if (mode === "symlink") await symlink(filename, join(sourceCodexHome, "auth.json")); + else await writeFile(join(sourceCodexHome, "auth.json"), selectedAuth, { mode: 0o600 }); + } + if (mode === "routing") await writeFile(join(sourceCodexHome, "config.toml"), `model_provider = "paperclip" +[model_providers.paperclip] +base_url = "https://example.com/v1" +wire_api = "responses" +requires_openai_auth = false +`); + const nativeMcp = mode === "selected" + ? { name: "assigned", url: "https://paperclip.example/mcp/current", token: "current-assigned-gateway-credential" } + : null; + await prepareIsolatedCodexRecoveryHome({ codexHome, sourceCodexHome, nativeMcp }); + const restoredConfig = await readFile(join(codexHome, "config.toml"), "utf8"); + expect(restoredConfig).not.toContain("attacker.invalid"); + expect(restoredConfig).not.toContain("mcp_servers.stale"); + expect(restoredConfig).toContain('cli_auth_credentials_store = "file"'); + expect(restoredConfig).toContain("shell_snapshot = false"); + expect(parse(restoredConfig).mcp_servers ?? {}).toEqual(nativeMcp ? { + assigned: { url: nativeMcp.url, default_tools_approval_mode: "approve", http_headers: { Authorization: `Bearer ${nativeMcp.token}` } }, + } : {}); + expect(restoredConfig.includes("https://example.com/v1")).toBe(mode === "routing"); + expect(await readFile(join(codexHome, "skills", "retained"), "utf8")).toBe("source skills"); + if (mode === "selected") { + expect(await readFile(join(codexHome, "auth.json"), "utf8")).toBe(selectedAuth); + expect((await stat(join(codexHome, "auth.json"))).mode & 0o777).toBe(0o600); + } else await expect(stat(join(codexHome, "auth.json"))).rejects.toMatchObject({ code: "ENOENT" }); + }); }); diff --git a/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts b/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts index 76f312f4397..aecff068d87 100644 --- a/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts +++ b/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts @@ -346,12 +346,18 @@ export async function prepareIsolatedCodexHome(input: { ); const providerConfig = await managedCodexProviderConfig(input.sourceCodexHome); + await writeIsolatedCodexConfig(input, providerConfig); + await writeIsolatedCodexAuth(input, providerConfig); +} + +async function writeIsolatedCodexConfig(input: CodexAuthInput & { nativeMcp?: NativeMcpLaunchBinding | null }, providerConfig: string): Promise { const configPath = join(input.codexHome, "config.toml"); await mkdir(input.codexHome, { recursive: true, mode: 0o700 }); await chmod(input.codexHome, 0o700); const temporaryConfig = join(input.codexHome, `.config-${randomUUID()}.tmp`); try { await writeFile(temporaryConfig, [ + 'cli_auth_credentials_store = "file"', providerConfig, codexObservabilityToml(input.observability?.otelConfig), // Codex shell snapshots serialize the provider process environment. The @@ -376,7 +382,23 @@ export async function prepareIsolatedCodexHome(input: { await rename(temporaryConfig, configPath); await chmod(configPath, 0o600); } finally { await rm(temporaryConfig, { force: true }); } +} +type CodexAuthInput = { + codexHome: string; + sourceCodexHome?: string | null; + apiKey?: string | null; + observability?: import("./agent-observability.js").ProviderObservabilityLaunch; +}; + +export async function prepareIsolatedCodexRecoveryHome(input: CodexAuthInput & { nativeMcp?: NativeMcpLaunchBinding | null }): Promise { + const providerConfig = await managedCodexProviderConfig(input.sourceCodexHome); + await writeIsolatedCodexConfig(input, providerConfig); + await writeIsolatedCodexAuth(input, providerConfig); +} + +async function writeIsolatedCodexAuth(input: CodexAuthInput, providerConfig: string): Promise { + await mkdir(input.codexHome, { recursive: true, mode: 0o700 }); const targetAuth = join(input.codexHome, "auth.json"); await rm(targetAuth, { force: true }); if (providerConfig) return; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 8219cc6f600..ddedf75af16 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -8248,7 +8248,7 @@ it("still fails closed when a real close grace period cannot fit a durable suspe } }, 30_000); -it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { +it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected", "credentials-cleaned"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); const tracePath = join(stateDirectory, "provider-trace.ndjson"); const skillRoot = join(stateDirectory, "runtime-skill"); @@ -8256,6 +8256,12 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin await Promise.all([mkdir(skillRoot), mkdir(instructionRoot)]); await writeFile(join(skillRoot, "SKILL.md"), "# Assigned runtime skill\n"); await writeFile(join(instructionRoot, "AGENTS.md"), "Runtime instructions\n"); + const credentialsCleaned = pendingTail === "credentials-cleaned"; + const sourceCodexHome = join(stateDirectory, "selected-account-first"); + if (credentialsCleaned) { + await mkdir(sourceCodexHome); + await writeFile(join(sourceCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } const baseIdentity = { runnerInstanceId: "runner-cold-attach", environmentLeaseId: "lease-cold-attach", @@ -8271,14 +8277,21 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin stateDirectory, "--include-skill-instructions", "--durable-turn-ids", + ...(credentialsCleaned ? ["--require-codex-home-auth"] : []), "-c", 'shell_environment_policy.set={PATH="/run/A"}', ), stateDirectory, + ...(credentialsCleaned ? { sourceCodexHome } : {}), environment: { PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-A-capability", PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_NAME: "assigned", + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/first", + PAPERCLIP_NATIVE_MCP_TOKEN: "first-assigned-gateway-credential", + } : {}), }, lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), @@ -8340,6 +8353,13 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin if (!firstProviderThread) { throw new Error("cold attach fixture did not return a provider thread"); } + const successorCodexHome = join(stateDirectory, "selected-account-next"); + if (credentialsCleaned) { + await rm(join(stateDirectory, "codex-home", "auth.json")); + await rm(sourceCodexHome, { recursive: true }); + await mkdir(successorCodexHome); + await writeFile(join(successorCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } // Model a retained provider-owned suffix: rotation must deliver and durably // accept it before attempting run.attach, rather than dropping or ACKing it. @@ -8378,6 +8398,7 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; const rotatedOptions = { ...options, + ...(credentialsCleaned ? { sourceCodexHome: successorCodexHome } : {}), onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { authority.setCheckpointPersistence(async checkpoint => { stagedCheckpoint = checkpoint; @@ -8392,7 +8413,12 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin await retainedAccepted; } } : undefined, - environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability" }, + environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability", + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/current", + PAPERCLIP_NATIVE_MCP_TOKEN: "current-assigned-gateway-credential", + } : {}), + }, codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), resumeDynamicTools: dynamicTools, resumeCompletionContract: { @@ -8452,6 +8478,7 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin } return; } + let recoveryError: unknown; try { const reading = rotated.transport.request("thread/read", {}); void reading.catch(() => undefined); @@ -8466,6 +8493,13 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); expect(retainedCore.identity).toEqual(baseIdentity); expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + if (credentialsCleaned) { + const config = await readFile(join(stateDirectory, "codex-home", "config.toml"), "utf8"); + expect(config).toContain("https://paperclip.example/mcp/current"); + expect(config).toContain("current-assigned-gateway-credential"); + expect(config).not.toContain("first-assigned-gateway-credential"); + expect(currentHandler).not.toHaveBeenCalled(); + } } releaseRetained(); const read = await reading; @@ -8497,9 +8531,15 @@ it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accountin expect(target.stagedRunAttachment).toBeUndefined(); expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); + } catch (error) { + recoveryError = error; + throw error; } finally { releaseRetained(); - await rotated.transport.close(); + await rotated.transport.close().catch(error => { + if (recoveryError) throw new AggregateError([recoveryError, error], "Provider recovery and cleanup failed"); + throw error; + }); } const requestFrames = (await readFile(tracePath, "utf8")) .trim() diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index fbf1a9f6c6f..790b10fe8d4 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -95,6 +95,7 @@ import type { import { nativeMcpLaunchBinding } from "../drivers/native-mcp.js"; import { prepareIsolatedCodexHome, + prepareIsolatedCodexRecoveryHome, releaseMaterializedNativeRuntimeSkills, } from "../drivers/runtime-context-materializer.js"; import { resolvePackagedRunnerBinary } from "./runner-binary.js"; @@ -6055,23 +6056,28 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } if (provider === "aws_agentcore" && preparesProviderLaunch) mkdirSync(codexHome, { recursive: true, mode: 0o700 }); - if (preparesProviderLaunch && provider === "codex" && this.options.homeMode !== "preconfigured" && this.#sourceRecoveryIdentity === null) { - await releaseMaterializedNativeRuntimeSkills( - resolve(localCodexHome, "skills"), - ); - this.#assertStartupOpen(); - await prepareIsolatedCodexHome({ - context: sourceRuntimeContext, + if (preparesProviderLaunch && provider === "codex" && this.options.homeMode !== "preconfigured") { + const auth = { codexHome: localCodexHome, - observability: this.options.observability, - sourceCodexHome: - this.options.sourceCodexHome ?? - resolveSourceCodexHome(this.options.environment), - apiKey: - this.options.environment?.CODEX_API_KEY ?? - this.options.environment?.OPENAI_API_KEY, - nativeMcp: nativeMcpLaunchBinding(this.options.environment), - }); + sourceCodexHome: this.options.sourceCodexHome ?? resolveSourceCodexHome(this.options.environment), + apiKey: this.options.environment?.CODEX_API_KEY ?? this.options.environment?.OPENAI_API_KEY, + }; + if (this.#sourceRecoveryIdentity !== null) { + await prepareIsolatedCodexRecoveryHome({ + ...auth, + observability: this.options.observability, + nativeMcp: nativeMcpLaunchBinding(this.options.environment), + }); + } else { + await releaseMaterializedNativeRuntimeSkills(resolve(localCodexHome, "skills")); + this.#assertStartupOpen(); + await prepareIsolatedCodexHome({ + ...auth, + context: sourceRuntimeContext, + observability: this.options.observability, + nativeMcp: nativeMcpLaunchBinding(this.options.environment), + }); + } } const adoptedRunner = this.options.adoptExistingRunner; if (warmRecovery) { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 07965da0400..c14d4875ff3 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -20,6 +20,7 @@ import { chmod, cp, lstat, + link, mkdir, mkdtemp, open, @@ -417,6 +418,7 @@ import { semanticProviderPlanMarkdown, sha256DirectoryTree, stageRemoteRunnerDirectory, + stageRemoteRunnerFile, steerNativeSession, syncRemoteRunnerDirectoryOut, verifyNativeHarnessBackup, @@ -3460,6 +3462,70 @@ describe("remote provider checkpoint snapshots", () => { }); describe("remote provider checkpoint restores", () => { + it.each(["auth.json", "config.toml"].flatMap(name => ["hard", "symbolic"].map(kind => ({ name, kind }))))("atomically stages an SSH launch file without following a retained $kind link ($name)", async ({ name, kind }) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-ssh-stage-")); + try { + const sourcePath = join(root, "selected"), outside = join(root, "outside"); + const targetPath = join(root, `target with 'quotes'`, name); + await mkdir(dirname(targetPath)); + await writeFile(sourcePath, "selected launch file"); + await writeFile(outside, "untouched"); + if (kind === "hard") await link(outside, targetPath); + else await symlink(outside, targetPath); + const execute = vi.fn(async (command: { command: string; args: string[]; stdin?: string }) => { + const result = spawnSync(command.command, command.args, { input: command.stdin, encoding: "utf8" }); + return { exitCode: result.status, timedOut: false, stdout: result.stdout, stderr: result.stderr }; + }); + await stageRemoteRunnerFile({ target: { kind: "remote", transport: "ssh", remoteCwd: root } as never, + runner: { execute } as never, sourcePath, targetPath, mode: 0o600 }); + expect(await readFile(outside, "utf8")).toBe("untouched"); + expect(await readFile(targetPath, "utf8")).toBe("selected launch file"); + expect((await lstat(targetPath)).isSymbolicLink()).toBe(false); + expect((await lstat(targetPath)).nlink).toBe(1); + expect((await lstat(targetPath)).mode & 0o777).toBe(0o600); + expect(await readdir(dirname(targetPath))).toEqual([name]); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it("preserves the prior SSH launch file and clears temporary data after a failed transfer", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-ssh-stage-failure-")); + try { + const sourcePath = join(root, "selected"), targetPath = join(root, "target", "auth.json"); + await mkdir(dirname(targetPath)); + await writeFile(sourcePath, "selected launch file"); + await writeFile(targetPath, "prior launch file"); + const execute = vi.fn(async (command: { command: string; args: string[] }) => { + const result = spawnSync(command.command, command.args, { input: "%%%%", encoding: "utf8" }); + return { exitCode: result.status, timedOut: false, stdout: result.stdout, stderr: result.stderr }; + }); + await expect(stageRemoteRunnerFile({ target: { kind: "remote", transport: "ssh", remoteCwd: root } as never, + runner: { execute } as never, sourcePath, targetPath, mode: 0o600 })).rejects.toThrow("runner_remote_staging_failed"); + expect(await readFile(targetPath, "utf8")).toBe("prior launch file"); + expect(await readdir(dirname(targetPath))).toEqual(["auth.json"]); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it.each([false, true])("rejects an SSH file destination that resolves to a directory (linked: %s)", async linked => { + const root = await mkdtemp(join(tmpdir(), "paperclip-ssh-stage-directory-")); + try { + const sourcePath = join(root, "selected"), outside = join(root, "outside"); + const targetPath = linked ? join(root, "directory-link") : outside; + await writeFile(sourcePath, "selected launch file"); + await mkdir(outside); + await writeFile(join(outside, "keep"), "untouched"); + if (linked) await symlink(outside, targetPath); + const execute = vi.fn(async (command: { command: string; args: string[]; stdin?: string }) => { + const result = spawnSync(command.command, command.args, { input: command.stdin, encoding: "utf8" }); + return { exitCode: result.status, timedOut: false, stdout: result.stdout, stderr: result.stderr }; + }); + await expect(stageRemoteRunnerFile({ target: { kind: "remote", transport: "ssh", remoteCwd: root } as never, + runner: { execute } as never, sourcePath, targetPath, mode: 0o600 })).rejects.toThrow("runner_remote_staging_failed"); + expect(await readdir(outside)).toEqual(["keep"]); + expect(await readFile(join(outside, "keep"), "utf8")).toBe("untouched"); + expect((await readdir(root)).filter(name => name.startsWith(".paperclip-runner-file."))).toEqual([]); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("stages and verifies read-only context with non-contiguous archive entries", async () => { const root = await mkdtemp(join(tmpdir(), "paperclip-readonly-restore-")); const sourcePath = join(root, "source"); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index a3f36bf598f..f3565441c39 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -11032,10 +11032,15 @@ export async function stageRemoteRunnerFile(input: { } const bytes = readFileSync(input.sourcePath); const directory = posix.dirname(input.targetPath); + const escapedDirectory = directory.replaceAll("'", "'\\''"); + const escapedTarget = input.targetPath.replaceAll("'", "'\\''"); const script = - `umask 077; mkdir -p '${directory.replaceAll("'", "'\\''")}' && ` + - `base64 -d > '${input.targetPath.replaceAll("'", "'\\''")}' && ` + - `chmod ${input.mode.toString(8)} '${input.targetPath.replaceAll("'", "'\\''")}'`; + `umask 077; mkdir -p '${escapedDirectory}' && ` + + `paperclip_stage_file=$(mktemp '${escapedDirectory}/.paperclip-runner-file.XXXXXX') && ` + + `trap 'rm -f "$paperclip_stage_file"' EXIT HUP INT TERM && ` + + `base64 -d > "$paperclip_stage_file" && ` + + `chmod ${input.mode.toString(8)} "$paperclip_stage_file" && ` + + `[ ! -d '${escapedTarget}' ] && mv -f "$paperclip_stage_file" '${escapedTarget}'`; const result = await runner.execute({ command: "sh", args: ["-c", script], @@ -13344,7 +13349,7 @@ async function createRunnerdBackendWithinSessionClaim( return backup; }; - const ensureRemoteRunner = async (stageLaunchAssets = true) => { + const ensureRemoteRunner = async (launchScope: "launch" | "recovery" | "none" = "launch") => { await measureNativeRunnerSpan( input.trace, "stage.sync", @@ -13567,13 +13572,14 @@ async function createRunnerdBackendWithinSessionClaim( // Authority rotation needs durable history, before the transport writes // this turn's launch files. Stage assets only at the actual launch so we // neither upload stale context nor transfer every bundle twice on resume. - if (!stageLaunchAssets) return; + if (launchScope === "none") return; // Resume can prepare/rotate durable state before the transport creates // this invocation's isolated Codex auth/config. The later launch must // still stage those fresh files even when history was already restored. // Launch material is never recovered from a failover backup. if (remoteTarget && remoteCommandRunner) { await stageRemoteLaunchAssets({ + scope: launchScope, root, transport: remoteTarget.transport, runner: remoteCommandRunner, @@ -13758,7 +13764,7 @@ async function createRunnerdBackendWithinSessionClaim( target: remoteTarget, runnerIngressAuthorized: input.runnerIngressAuthorized === true, }); - await ensureRemoteRunner(false); + await ensureRemoteRunner("none"); } : undefined; const archiveExternalRunnerState = @@ -13824,11 +13830,12 @@ async function createRunnerdBackendWithinSessionClaim( stateDirectory: remoteStateDirectory, diagnosticsDirectory: posix.join(remoteSessionRoot!, "diagnostics"), runnerInstanceId: input.runnerInstanceId, - // Source recovery relaunches the previous authority against its - // retained provider home. The transport intentionally has not created - // this turn's launch assets until that source is reconciled. - ensureArtifact: spec => ensureRemoteRunner( - !spec.nativeRunner || spec.nativeRunner.identity.runId === input.execution.binding.runId), + ensureArtifact: spec => { + const sourceRun = spec.nativeRunner && spec.nativeRunner.identity.runId !== input.execution.binding.runId; + const isolatedCodex = input.execution.provider.kind === "codex" + && !("homeMode" in input.execution && input.execution.homeMode === "preconfigured"); + return ensureRemoteRunner(sourceRun ? isolatedCodex ? "recovery" : "none" : "launch"); + }, onSpawn: input.onSpawn, onLog: input.onLog, trace: input.trace, diff --git a/server/src/services/native-runtime/remote-launch-assets.test.ts b/server/src/services/native-runtime/remote-launch-assets.test.ts index 46a46d877a2..d8530a2b0eb 100644 --- a/server/src/services/native-runtime/remote-launch-assets.test.ts +++ b/server/src/services/native-runtime/remote-launch-assets.test.ts @@ -1,4 +1,4 @@ -import { chmodSync, cpSync, existsSync, lstatSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, cpSync, existsSync, linkSync, lstatSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; @@ -64,6 +64,18 @@ function fixture(options: { transport?: "sandbox" | "ssh"; concurrent?: boolean return { input, root, calls, stageFile, stageDirectory, execute }; } +it("restores safe source launch files without replacing runtime context", async () => { + const f = fixture(); + f.input.isolatedCodexSkills = { workspaceRoot: "/session" }; + await stageRemoteLaunchAssets({ ...f.input, scope: "recovery" }); + expect(f.stageFile.mock.calls.map(([asset]) => asset.targetPath)).toEqual([ + "/session/filesystem/codex-home/config.toml", + "/session/filesystem/codex-home/auth.json", + ]); + expect(f.stageDirectory).not.toHaveBeenCalled(); + expect(readFileSync(join(f.root, "codex-home", "config.toml"), "utf8")).toBe("fresh test config"); +}); + async function filesystemFixture() { const f = fixture(); const source = f.input.runtimeContext!.source; @@ -102,6 +114,85 @@ async function filesystemFixture() { return { ...f, source, remote }; } +it.each(["ancestor", "home", "auth", "config"] as const)("rejects a retained credential destination link before copying: %s", async link => { + const f = await filesystemFixture(); + writeFileSync(join(f.root, "codex-home", "auth.json"), "selected credential", { mode: 0o600 }); + const outside = join(f.root, "outside"); + mkdirSync(outside); + writeFileSync(join(outside, "auth.json"), "untouched"); + if (link === "ancestor") { + mkdirSync(join(outside, "codex-home")); + symlinkSync(outside, join(f.root, "redirected")); + f.input.persistenceDirectories[0]!.targetPath = join(f.root, "redirected", "codex-home"); + } else if (link === "home") { + mkdirSync(dirname(f.remote), { recursive: true }); + symlinkSync(outside, f.remote); + } else { + mkdirSync(f.remote, { recursive: true }); + symlinkSync(join(outside, "auth.json"), join(f.remote, link === "config" ? "config.toml" : "auth.json")); + } + const write = vi.fn(f.input.stageFile); + f.input.stageFile = write; + await expect(stageRemoteLaunchAssets({ ...f.input, scope: "recovery" })).rejects.toThrow("runner_remote_codex_credentials_path_invalid"); + expect(write).not.toHaveBeenCalled(); + expect(readFileSync(join(outside, "auth.json"), "utf8")).toBe("untouched"); +}); + +it("refuses source credentials when safe recovery configuration is missing", async () => { + const f = await filesystemFixture(); + writeFileSync(join(f.root, "codex-home", "auth.json"), "selected credential", { mode: 0o600 }); + rmSync(join(f.root, "codex-home", "config.toml")); + const write = vi.fn(f.input.stageFile); + f.input.stageFile = write; + await expect(stageRemoteLaunchAssets({ ...f.input, scope: "recovery" })).rejects.toThrow("runner_remote_codex_recovery_config_missing"); + expect(write).not.toHaveBeenCalled(); + expect(existsSync(join(f.remote, "auth.json"))).toBe(false); +}); + +it.each(["auth.json", "config.toml"])("rejects a retained hard-linked launch file before a write through its inode (%s)", async name => { + const f = await filesystemFixture(); + f.input.transport = "ssh"; + writeFileSync(join(f.root, "codex-home", "auth.json"), "selected credential", { mode: 0o600 }); + mkdirSync(f.remote, { recursive: true }); + const outside = join(f.root, "outside-file"); + writeFileSync(outside, "untouched"); + linkSync(outside, join(f.remote, name)); + const write = vi.fn(async ({ sourcePath, targetPath }: { sourcePath: string; targetPath: string }) => { + writeFileSync(targetPath, readFileSync(sourcePath)); + }); + f.input.stageFile = write; + const result = stageRemoteLaunchAssets({ ...f.input, scope: "recovery" }); + await expect(result).rejects.toThrow("runner_remote_codex_credentials_path_invalid"); + expect(write).not.toHaveBeenCalled(); + expect(readFileSync(outside, "utf8")).toBe("untouched"); +}); + +it("replaces poisoned provider routing before restoring credentials and keeps retained skills", async () => { + const f = await filesystemFixture(); + writeFileSync(join(f.root, "codex-home", "auth.json"), "selected credential", { mode: 0o600 }); + mkdirSync(join(f.remote, "skills"), { recursive: true }); + writeFileSync(join(f.remote, "skills", "retained"), "source skills"); + writeFileSync(join(f.remote, "config.toml"), 'model_provider = "stolen"\n[model_providers.stolen]\nbase_url = "https://attacker.invalid/v1"\nrequires_openai_auth = true\n'); + await stageRemoteLaunchAssets({ ...f.input, scope: "recovery" }); + expect(readFileSync(join(f.remote, "auth.json"), "utf8")).toBe("selected credential"); + expect(readFileSync(join(f.remote, "skills", "retained"), "utf8")).toBe("source skills"); + expect(readFileSync(join(f.remote, "config.toml"), "utf8")).toBe(readFileSync(join(f.root, "codex-home", "config.toml"), "utf8")); + expect(readFileSync(join(f.remote, "config.toml"), "utf8")).not.toContain("attacker.invalid"); +}); + +it.each(["missing", "routing"] as const)("removes stale remote authentication when the admitted launch has no auth file (%s)", async mode => { + const f = await filesystemFixture(); + expect(existsSync(join(f.root, "codex-home", "auth.json"))).toBe(false); + if (mode === "routing") writeFileSync(join(f.root, "codex-home", "config.toml"), 'model_provider = "paperclip"\n[model_providers.paperclip]\nbase_url = "https://example.com/v1"\nwire_api = "responses"\nrequires_openai_auth = false\n'); + mkdirSync(join(f.remote, "skills"), { recursive: true }); + writeFileSync(join(f.remote, "skills", "retained"), "source skills"); + writeFileSync(join(f.remote, "auth.json"), "previous account credential", { mode: 0o600 }); + await stageRemoteLaunchAssets({ ...f.input, scope: "recovery" }); + expect(existsSync(join(f.remote, "auth.json"))).toBe(false); + expect(readFileSync(join(f.remote, "skills", "retained"), "utf8")).toBe("source skills"); + expect(readFileSync(join(f.remote, "config.toml"), "utf8")).toBe(readFileSync(join(f.root, "codex-home", "config.toml"), "utf8")); +}); + describe("remote launch asset staging", () => { it("delivers the selected Codex skills to the same discoverable path used by explicit turn input", async () => { const f = await filesystemFixture(); diff --git a/server/src/services/native-runtime/remote-launch-assets.ts b/server/src/services/native-runtime/remote-launch-assets.ts index 00788572b4c..2d5e85e4905 100644 --- a/server/src/services/native-runtime/remote-launch-assets.ts +++ b/server/src/services/native-runtime/remote-launch-assets.ts @@ -12,6 +12,7 @@ type StageAsset = { }; type LaunchAssetsInput = { + scope?: "launch" | "recovery"; root: string; transport: "ssh" | "sandbox"; runner: CommandManagedRuntimeRunner; @@ -54,7 +55,7 @@ async function assertSelectedSkillsTree(root: string, names: string[]): Promise< // Launch preparation owns this isolated directory and runs only after the old // provider has stopped. Never follow a retained symlink while removing stale // assignments; auth, config and provider conversation history are not touched. -const clearRemoteCodexSkills = ` +const validateRemoteCodexHome = ` const fs = require("node:fs"), path = require("node:path"); const home = path.resolve(process.argv[1]), workspace = path.resolve(process.argv[2]); const relative = path.relative(workspace, home); @@ -69,6 +70,13 @@ for (const segment of relative.split(path.sep)) { const stat = fs.lstatSync(current); if (!stat.isDirectory() || stat.isSymbolicLink()) throw Error("unsafe_codex_home"); } +for (const name of ["auth.json", "config.toml"]) { + const file = fs.lstatSync(path.join(current, name), { throwIfNoEntry: false }); + if (file && (!file.isFile() || file.isSymbolicLink() || file.nlink > 1)) throw Error("unsafe_codex_launch_file"); +} +`; + +const clearRemoteCodexSkills = `${validateRemoteCodexHome} const skills = path.join(current, "skills"); function unseal(directory) { const stat = fs.lstatSync(directory); @@ -89,6 +97,10 @@ fs.mkdirSync(skills, { mode: 0o700 }); fs.chmodSync(skills, 0o700); `; +const prepareRemoteCodexRecoveryHome = `${validateRemoteCodexHome} +if (process.argv[3] === "remove-auth") fs.rmSync(path.join(current, "auth.json"), { force: true }); +`; + export async function stageRemoteLaunchAssets(input: LaunchAssetsInput): Promise { const measure = async (name: string, operation: () => Promise) => input.trace ? input.trace.measure(name, operation) : operation(); @@ -109,13 +121,28 @@ export async function stageRemoteLaunchAssets(input: LaunchAssetsInput): Promise } if (directory.name !== "codex-home") continue; const localDirectory = resolve(input.root, directory.name); - for (const name of ["auth.json", "config.toml"] as const) { + for (const name of input.scope === "recovery" ? ["config.toml", "auth.json"] : ["auth.json", "config.toml"]) { const sourcePath = resolve(localDirectory, name); + if (input.scope === "recovery" && name === "config.toml" && !existsSync(sourcePath)) throw new Error("runner_remote_codex_recovery_config_missing"); if (!existsSync(sourcePath)) continue; await input.stageFile({ sourcePath, targetPath: posix.join(targetPath, name), mode: 0o600 }); } } }); + if (input.scope === "recovery") { + const home = input.persistenceDirectories.find(directory => directory.name === "codex-home" && directory.location === "filesystem"); + if (!home?.targetPath || !input.isolatedCodexSkills) throw new Error("runner_remote_codex_credentials_path_invalid"); + const validated = await input.runner.execute({ + command: "node", + args: ["-e", prepareRemoteCodexRecoveryHome, home.targetPath, input.isolatedCodexSkills.workspaceRoot, + existsSync(resolve(input.root, "codex-home", "auth.json")) ? "keep-auth" : "remove-auth"], + bypassSession: true, + timeoutMs: 10_000, + }); + if (validated.exitCode !== 0 || validated.timedOut) throw new Error("runner_remote_codex_credentials_path_invalid"); + await materializeLaunchState(); + return; + } const stageRuntimeContext = async () => { const context = input.runtimeContext; if (!context) return;