From 0631cf1ec3442105271caa7c5533e432be70fded Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 12 Nov 2026 19:47:14 -0800 Subject: [PATCH] Distinguish missing SSH credential files from read failures Routed Codex accounts do not create auth.json. Report a confirmed missing file as ENOENT so the existing guarded credential copyback can skip it. Require the remote marker and exit status together, and preserve connection and other read errors. Reproduced the failure with the real shell command. Verified 325 tests across the SSH runtime, Codex probes and credential merge, Codex remote execution, and ACP executor. Adapter-utils and Codex typechecks passed. The routed probe retains its config-only home and still removes temporary staging files. Co-Authored-By: Paperclip --- doc/connections/AI-CONNECTIONS.md | 4 +- .../src/remote-managed-runtime.test.ts | 45 ++++++++++++++++++- .../src/remote-managed-runtime.ts | 20 +++++++-- .../src/server/test.remote.test.ts | 19 +++++--- 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/doc/connections/AI-CONNECTIONS.md b/doc/connections/AI-CONNECTIONS.md index 8ba64f3f933..34baa5f3fce 100644 --- a/doc/connections/AI-CONNECTIONS.md +++ b/doc/connections/AI-CONNECTIONS.md @@ -458,7 +458,9 @@ per-turn directories are removed, including after provider or setup failure. Environment tests use the selected account and remove their temporary homes after probing. Managed Codex probes copy refreshed credentials to that account's local home before cleanup, even when the provider request fails. The merge rejects -credentials from another account. A copyback failure fails the probe. A disconnected target can prevent remote cleanup; the run reports +credentials from another account. Routed accounts can omit `auth.json`; the SSH +reader reports confirmed file absence as `ENOENT`, which needs no copyback. +Connection and file-read failures still fail the probe. A disconnected target can prevent remote cleanup; the run reports the resulting failure rather than silently selecting another account. Codex CLI runs in an in-place SSH workspace keep conversation files and native diff --git a/packages/adapter-utils/src/remote-managed-runtime.test.ts b/packages/adapter-utils/src/remote-managed-runtime.test.ts index b2da51fdcbb..d2850d865c3 100644 --- a/packages/adapter-utils/src/remote-managed-runtime.test.ts +++ b/packages/adapter-utils/src/remote-managed-runtime.test.ts @@ -1,3 +1,5 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -176,12 +178,53 @@ describe("remote managed runtime", () => { expect(restoreWorkspaceFromSshExecution).not.toHaveBeenCalled(); expect(runSshCommand).toHaveBeenCalledWith( expect.anything(), - "base64 < '/app/.paperclip-runtime/codex/home/auth.json'", + expect.stringContaining("base64 < '/app/.paperclip-runtime/codex/home/auth.json'"), { maxBuffer: 1024 * 1024 }, ); expect(restoredAuth).toBe('{"token":"remote"}\n'); }); + it.each(["missing", "present", "read failure", "connection failure"])( + "preserves SSH file-read error semantics for %s files", + async outcome => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-file-read-")); + cleanupDirs.push(root); + const remoteCwd = path.join(root, "remote's workspace"); + const assetDir = path.join(remoteCwd, ".paperclip-runtime", "codex", "home"); + await mkdir(assetDir, { recursive: true }); + const content = Buffer.from([0, 255, 1, 10]); + if (outcome !== "missing") await writeFile(path.join(assetDir, "auth.json"), content); + const transportError = Object.assign(new Error("fixture SSH connection failed"), { code: 255 }); + runSshCommand.mockImplementationOnce(async (_spec, command) => { + if (outcome === "connection failure") throw transportError; + const script = outcome === "read failure" + ? `base64() { printf '%s\\n' 'fixture read failure' >&2; return 44; }; ${command}` + : command!; + return promisify(execFile)("sh", ["-c", script!]); + }); + let bytes: Buffer | undefined; + const prepared = await prepareRemoteManagedRuntime({ + spec: { host: "ssh.example.test", port: 22, username: "agent", remoteWorkspacePath: remoteCwd, + remoteCwd, privateKey: null, knownHosts: null, strictHostKeyChecking: true }, + runId: "read-file", adapterKey: "codex", workspaceLocalDir: root, workspaceRemoteDir: remoteCwd, + syncWorkspace: false, + assets: [{ key: "home", localDir: root, restore: async ({ assetDir, readFile }) => { + bytes = await readFile(path.posix.join(assetDir, "auth.json")); + } }], + }); + if (outcome === "present") { + await prepared.restoreWorkspace(); + expect(bytes).toEqual(content); + } else if (outcome === "missing") { + await expect(prepared.restoreWorkspace()).rejects.toMatchObject({ code: "ENOENT" }); + } else if (outcome === "connection failure") { + await expect(prepared.restoreWorkspace()).rejects.toBe(transportError); + } else { + await expect(prepared.restoreWorkspace()).rejects.toMatchObject({ code: 44, stderr: "fixture read failure\n" }); + } + }, + ); + it("stages each additional project into its own isolated SSH dir, isolating one failure", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-additional-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/remote-managed-runtime.ts b/packages/adapter-utils/src/remote-managed-runtime.ts index b7c97ac4953..c59b2a100e2 100644 --- a/packages/adapter-utils/src/remote-managed-runtime.ts +++ b/packages/adapter-utils/src/remote-managed-runtime.ts @@ -76,10 +76,22 @@ function shellQuote(value: string): string { } async function readRemoteFile(spec: SshRemoteExecutionSpec, remotePath: string): Promise { - const result = await runSshCommand(spec, `base64 < ${shellQuote(remotePath)}`, { - maxBuffer: 1024 * 1024, - }); - return Buffer.from(result.stdout.replace(/\s+/g, ""), "base64"); + const missingMarker = "paperclip:remote-file-missing"; + const missingExitCode = 44; + const file = shellQuote(remotePath); + const parent = shellQuote(path.posix.dirname(remotePath)); + /* Confirm the parent is searchable so a permission error cannot masquerade as absence. */ + const command = `if [ -d ${parent} ] && [ -x ${parent} ] && [ ! -e ${file} ]; then printf '%s\\n' '${missingMarker}' >&2; exit ${missingExitCode}; fi; base64 < ${file}`; + try { + const result = await runSshCommand(spec, command, { maxBuffer: 1024 * 1024 }); + return Buffer.from(result.stdout.replace(/\s+/g, ""), "base64"); + } catch (error) { + const failure = error as { code?: unknown; stderr?: unknown } | null; + if (failure?.code === missingExitCode && typeof failure.stderr === "string" && failure.stderr.trim() === missingMarker) { + throw Object.assign(new Error("Remote runtime file does not exist"), { code: "ENOENT" }); + } + throw error; + } } export function buildRemoteExecutionSessionIdentity(spec: SshRemoteExecutionSpec | null) { diff --git a/packages/adapters/codex-local/src/server/test.remote.test.ts b/packages/adapters/codex-local/src/server/test.remote.test.ts index 048701c9291..de349a4ba7f 100644 --- a/packages/adapters/codex-local/src/server/test.remote.test.ts +++ b/packages/adapters/codex-local/src/server/test.remote.test.ts @@ -512,7 +512,7 @@ describe("codex remote environment diagnostics", () => { last_refresh: lastRefresh, }); - it.each(["success", "provider failure", "foreign account", "copyback failure"])( + it.each(["success", "provider failure", "foreign account", "copyback failure", "routed account"])( "preserves managed SSH probe credentials through %s before cleanup", async outcome => { const selectedHome = await makeScratchDir("paperclip-test-selected-codex-"); @@ -520,7 +520,9 @@ describe("codex remote environment diagnostics", () => { const refreshed = subscriptionAuth(outcome === "foreign account" ? "acct-other" : "acct-selected", "refreshed", "2020-01-02T00:00:00Z"); const shared = subscriptionAuth("acct-shared", "untouched", "2020-01-01T00:00:00Z"); const authPath = path.join(selectedHome, "auth.json"); - await fs.writeFile(authPath, original); + if (outcome === "routed account") { + await fs.writeFile(path.join(selectedHome, "config.toml"), 'model_provider = "paperclip"\n[model_providers.paperclip]\nrequires_openai_auth = false\nenv_key = "PAPERCLIP_AI_PROVIDER_KEY"\n'); + } else await fs.writeFile(authPath, original); await fs.writeFile(path.join(process.env.CODEX_HOME!, "auth.json"), shared); const basePrepare = prepareAdapterExecutionTargetRuntime.getMockImplementation()!; let stagingDir = ""; @@ -537,11 +539,14 @@ describe("codex remote environment diagnostics", () => { expect(remotePath).toBe(path.posix.join(prepared.assetDirs.home, "auth.json")); assetRestoreCalls += 1; await expect(fs.stat(stagingDir)).resolves.toBeDefined(); + if (outcome === "routed account") throw Object.assign(new Error("fixture auth file absent"), { code: "ENOENT" }); if (outcome === "copyback failure") throw new Error("fixture credential read failed"); return Buffer.from(refreshed); }, } as Parameters>[0]); - expect(await fs.readFile(authPath, "utf8")).toBe(outcome === "foreign account" ? original : refreshed); + expect(await fs.readFile(authPath, "utf8").catch(() => null)).toBe( + outcome === "routed account" ? null : outcome === "foreign account" ? original : refreshed, + ); } finally { await restoreWorkspace(); } @@ -552,7 +557,8 @@ describe("codex remote environment diagnostics", () => { runAdapterExecutionTargetProcess.mockResolvedValueOnce({ ...success, exitCode: 1, stdout: "", stderr: "fixture provider unavailable" }); } const probe = testEnvironment({ companyId: "company-1", adapterType: "codex_local", - config: { engine: "cli", command: "codex", managedAiConnection: { identity: "selected-account" }, env: { CODEX_HOME: selectedHome } }, + config: { engine: "cli", command: "codex", managedAiConnection: { identity: "selected-account" }, env: { CODEX_HOME: selectedHome, + ...(outcome === "routed account" ? { PAPERCLIP_AI_PROVIDER_KEY: "fixture-route-key" } : {}) } }, executionTarget: { kind: "remote", transport: "ssh", remoteCwd: "/remote/workspace", spec: { host: "ssh.example.test", port: 22, username: "agent", privateKey: null, knownHosts: null, strictHostKeyChecking: true, remoteCwd: "/remote/workspace", remoteWorkspacePath: "/remote/workspace", useHostAuth: false } }, @@ -567,7 +573,10 @@ describe("codex remote environment diagnostics", () => { expect(restoreWorkspace).toHaveBeenCalledOnce(); await expect(fs.stat(stagingDir)).rejects.toMatchObject({ code: "ENOENT" }); expect(await fs.readFile(path.join(process.env.CODEX_HOME!, "auth.json"), "utf8")).toBe(shared); - expect(await fs.readFile(authPath, "utf8")).toBe(["foreign account", "copyback failure"].includes(outcome) ? original : refreshed); + expect(await fs.readFile(authPath, "utf8").catch(() => null)).toBe( + outcome === "routed account" ? null : ["foreign account", "copyback failure"].includes(outcome) ? original : refreshed, + ); + if (outcome === "routed account") expect(capturedHomeAssetFiles.value).toEqual(["config.toml"]); }, );