From 03e654e87005985583aee53643f8d4a06b141a76 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 12 Nov 2026 17:42:03 -0800 Subject: [PATCH] Preserve SSH conversations and select authentication before pools Keep only account-scoped Codex conversation and native index state between in-place SSH turns. Refresh current credentials and managed files for each turn, copy authentication back before cleanup, and start a named fresh session when saved state is unavailable or the account changes. Resolve SSH authentication before pool selection and native home selection. Preserve admitted native recovery inputs. Update transport test fixtures for the stdin command envelope and refresh the committed SDK source pins. Validated 164 execution/routing/probe tests, 91 CI regression tests, 58 SDK support tests, and Codex, Grok and server typechecks. Physical shell tests cover two turns, refresh, account changes, old saved parameters, and failure cleanup. Co-Authored-By: Paperclip --- doc/connections/AI-CONNECTIONS.md | 14 +++ .../src/server/execute.remote.test.ts | 109 +++++++++++++++++- .../codex-local/src/server/execute.ts | 35 +++++- .../src/server/ssh-session-state.ts | 41 +++++++ .../grok-local/src/server/execute.test.ts | 5 +- .../__tests__/ai-connection-router.test.ts | 48 ++++++++ .../heartbeat-execution-configuration.test.ts | 17 ++- .../__tests__/kimi-connection-probe.test.ts | 2 +- server/src/routes/agents.ts | 2 +- server/src/services/heartbeat.ts | 13 ++- .../heartbeat/environment-selection.ts | 19 ++- .../heartbeat/execution-configuration.ts | 2 +- .../native-ssh-command-runner.test.ts | 2 +- .../runner-e2e/source-sdk-public-inputs.json | 2 +- 14 files changed, 293 insertions(+), 18 deletions(-) create mode 100644 packages/adapters/codex-local/src/server/ssh-session-state.ts diff --git a/doc/connections/AI-CONNECTIONS.md b/doc/connections/AI-CONNECTIONS.md index 7d88165009b..eee20ba3f81 100644 --- a/doc/connections/AI-CONNECTIONS.md +++ b/doc/connections/AI-CONNECTIONS.md @@ -459,6 +459,20 @@ Environment tests use the selected account and remove their temporary homes after probing. A disconnected target can prevent remote cleanup; the run reports the resulting failure rather than silently selecting another account. +Codex CLI runs in an in-place SSH workspace keep conversation files and native +SQLite indexes in a private remote home scoped to the company, agent, AI account, +and conversation. The home path stays stable across turns and token refreshes. +Each turn installs the current managed files, copies refreshed authentication +back, and removes everything except conversation state, including on provider +failure. Credentials and managed MCP configuration do not remain there between +turns. Switching accounts selects a different home and starts a new conversation. +The cache lives below the environment's remote workspace root. Removing that +root also removes the retained conversation state. +Older SSH host-auth sessions keep their existing remote home. A managed SSH +session saved before conversation retention has no retained state reference and +starts a fresh conversation instead of reusing an unavailable thread. This reset +is recorded as `codex_ssh_session_state_unavailable`; existing saved files remain. + ACP SSH sessions use the existing remote process bridge and session reuse lifecycle. A compatible session can retain its private remote home; workspace sync and credential refresh run at turn teardown, and the session disposal hook diff --git a/packages/adapters/codex-local/src/server/execute.remote.test.ts b/packages/adapters/codex-local/src/server/execute.remote.test.ts index fd36719149c..4329b22afc5 100644 --- a/packages/adapters/codex-local/src/server/execute.remote.test.ts +++ b/packages/adapters/codex-local/src/server/execute.remote.test.ts @@ -1,4 +1,6 @@ -import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -27,7 +29,7 @@ const { prepareWorkspaceForSshExecution: vi.fn(async () => ({ gitBacked: false })), restoreWorkspaceFromSshExecution: vi.fn(async () => undefined), runSshCommand: vi.fn(async (_spec?: unknown, _command?: string) => ({ stdout: Buffer.from("{}").toString("base64"), stderr: "" })), - syncDirectoryToSsh: vi.fn(async () => undefined), + syncDirectoryToSsh: vi.fn(async (_input?: { localDir: string; remoteDir: string }) => undefined), startAdapterExecutionTargetPaperclipBridge: vi.fn(async () => ({ env: { PAPERCLIP_API_URL: "http://127.0.0.1:4310", @@ -88,6 +90,109 @@ describe("codex remote execution", () => { } }); + it.each([false, true])("retains a managed SSH Codex conversation across turns and removes credentials (second turn throws=%s)", async fails => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-ssh-resume-")); + cleanupDirs.push(root); + const workspace = path.join(root, "workspace"), home = path.join(root, "account"), secondHome = path.join(root, "next-invocation"); + await mkdir(workspace); await mkdir(home); await mkdir(secondHome); + const auth = (account: string, marker: string, last_refresh: string) => JSON.stringify({ tokens: { + account_id: account, access_token: `fixture-access-${marker}`, refresh_token: `fixture-refresh-${marker}`, id_token: `fixture-id-${marker}`, + }, last_refresh }); + const credential = auth("selected", "before", "2000-01-01T00:00:00Z"); + const refreshed = auth("selected", "after", new Date().toISOString()); + const otherCredential = auth("other", "other", "2000-01-01T00:00:00Z"); + await writeFile(path.join(home, "auth.json"), credential); + await writeFile(path.join(secondHome, "auth.json"), credential); + const executeShell = promisify(execFile); + const originalRun = runChildProcess.getMockImplementation()!; + let firstRemoteHome: string | undefined; + let otherRemoteHome: string | undefined; + const privateRoots: string[] = []; + let calls = 0; + runSshCommand.mockImplementation(async (_spec, command) => { + const result = await executeShell("sh", ["-c", command!]); + if (command!.includes("mktemp -d")) privateRoots.push(result.stdout.trim()); + return result; + }); + syncDirectoryToSsh.mockImplementation(async input => { await cp(input!.localDir, input!.remoteDir, { recursive: true }); }); + runChildProcess.mockImplementation(async (...args: unknown[]) => { + const env = (args[3] as { env: Record }).env; + const remoteHome = env.CODEX_HOME!; + calls++; + expect(await readFile(path.join(remoteHome, "auth.json"), "utf8")).toBe(calls === 1 ? credential : calls === 2 ? refreshed : otherCredential); + if (calls === 1) { + firstRemoteHome = remoteHome; + await mkdir(path.join(remoteHome, "sessions")); + await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "fixture conversation"); + await writeFile(path.join(remoteHome, "state_5.sqlite"), "fixture native index"); + await writeFile(path.join(remoteHome, "auth.json"), refreshed); + } else if (calls === 2) { + expect(remoteHome).toBe(firstRemoteHome); + expect(args[2]).toContain("resume"); + expect(args[2]).toContain("ssh-thread"); + expect(await readFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation"); + expect(await readFile(path.join(remoteHome, "state_5.sqlite"), "utf8")).toBe("fixture native index"); + if (fails) throw new Error("fixture provider failed"); + } else { + if (calls === 3) otherRemoteHome = remoteHome; + expect(remoteHome).not.toBe(firstRemoteHome); + expect(args[2]).not.toContain("resume"); + await expect(readFile(path.join(remoteHome, "sessions", "rollout.jsonl"))).rejects.toThrow("ENOENT"); + if (calls === 3) { + await mkdir(path.join(remoteHome, "sessions")); + await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "other conversation"); + } + } + return { exitCode: 0, signal: null, timedOut: false, stdout: JSON.stringify({ type: "thread.started", thread_id: calls < 3 ? "ssh-thread" : "other-thread" })+'\n{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}\n', stderr: "", pid: 123, startedAt: new Date().toISOString() }; + }); + const ctx = { + agent: { id: "agent-1", companyId: "company-1", name: "Selected agent", adapterType: "codex_local", adapterConfig: {} }, + config: { engine: "cli", command: "codex", managedAiConnection: { identity: "generation-one", sessionIdentity: "selected-account", provider: "openai", method: "subscription" }, env: { CODEX_HOME: home, OPENAI_API_KEY: "" } }, + context: { paperclipWorkspace: { cwd: workspace, source: "project_primary" } }, + executionTarget: { kind: "remote" as const, transport: "ssh" as const, remoteCwd: workspace, + spec: { useHostAuth: false, host: "127.0.0.1", port: 2222, username: "fixture", remoteCwd: workspace, remoteWorkspacePath: workspace, privateKey: null, knownHosts: null, strictHostKeyChecking: true }, + workspaceRealization: { mode: "in_place" as const, authoritativeRoot: workspace, pathAliases: [], outboundRestorePaths: [] } }, onLog: async () => {}, + }; + try { + const first = await execute({ ...ctx, runId: "first", runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: "task-one" } }); + expect(first.sessionId).toBe("ssh-thread"); + expect(await readFile(path.join(home, "auth.json"), "utf8")).toBe(refreshed); + await writeFile(path.join(secondHome, "auth.json"), refreshed); + expect(await readdir(firstRemoteHome!)).toEqual(expect.arrayContaining(["sessions", "state_5.sqlite"])); + expect(await readdir(firstRemoteHome!)).not.toContain("auth.json"); + const second = execute({ ...ctx, runId: "second", config: { ...ctx.config, + managedAiConnection: { ...ctx.config.managedAiConnection, identity: "generation-two" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } }, + runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } }); + if (fails) await expect(second).rejects.toThrow("fixture provider failed"); + else expect((await second).sessionId).toBe("ssh-thread"); + expect(calls).toBe(2); + expect((await readdir(firstRemoteHome!)).sort()).toEqual(["sessions", "state_5.sqlite"]); + expect((await stat(firstRemoteHome!)).mode & 0o077).toBe(0); + if (!fails) { + await writeFile(path.join(secondHome, "auth.json"), otherCredential); + const switched = await execute({ ...ctx, runId: "other-account", config: { ...ctx.config, + managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } }, + runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } }); + expect(switched.sessionId).toBe("other-thread"); + expect(switched.sessionParams?.sshCodexAccountScope).not.toBe(first.sessionParams?.sshCodexAccountScope); + expect(switched.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } }); + expect(await readdir(otherRemoteHome!)).not.toContain("auth.json"); + expect(await readFile(path.join(firstRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation"); + const oldShape = { sessionId: switched.sessionId!, cwd: workspace, remoteExecution: switched.sessionParams!.remoteExecution }; + const legacy = await execute({ ...ctx, runId: "old-saved-shape", config: { ...ctx.config, + managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } }, + runtime: { sessionId: switched.sessionId!, sessionParams: oldShape, sessionDisplayId: switched.sessionId!, taskKey: "task-one" } }); + expect(legacy.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } }); + expect(await readFile(path.join(otherRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("other conversation"); + } + for (const dir of privateRoots) await expect(stat(dir)).rejects.toThrow("ENOENT"); + } finally { + runChildProcess.mockImplementation(originalRun); + runSshCommand.mockReset().mockResolvedValue({ stdout: Buffer.from("{}").toString("base64"), stderr: "" }); + syncDirectoryToSsh.mockReset().mockResolvedValue(undefined); + } + }); + it("stages only the selected agent subscription and returns SSH refreshes to that account", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-selected-ssh-")); cleanupDirs.push(root); diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts index 499ade3b259..8788a399b02 100644 --- a/packages/adapters/codex-local/src/server/execute.ts +++ b/packages/adapters/codex-local/src/server/execute.ts @@ -10,6 +10,7 @@ import { fileURLToPath } from "node:url"; import { inferOpenAiCompatibleBiller, resolveManagedOpenAiBilling, type AdapterExecutionContext, type AdapterExecutionResult } from "@paperclipai/adapter-utils"; import { buildCodexAuthInboundProvision } from "./codex-auth-merge-scripts.js"; import { copyBackCodexAuth } from "./codex-auth-copyback.js"; +import { prepareCodexSshSessionState } from "./ssh-session-state.js"; import { ensureCodexAuthCacheEntryDir, isCodexAuthCacheEnabled, @@ -783,6 +784,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise> | null = null; let observabilityProfile: Awaited> | null = null; let observabilityPreparationFailed = false; try { @@ -894,9 +896,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise - void (await copyBackCodexAuth({ - readSandboxAuth: () => readFile(path.posix.join(assetDir, "auth.json")), + restore: async ({ assetDir, readFile }) => { + try { await copyBackCodexAuth({ + readSandboxAuth: () => readFile(path.posix.join(sshSessionState?.home ?? assetDir, "auth.json")), hostAuthPath: path.join(config.managedAiConnection ? effectiveCodexHome : resolveSharedCodexHomeDir(process.env), "auth.json"), log: (line) => onLog("stdout", `${line}\n`), // Additive cache write (sandbox to host): also cache the @@ -907,7 +909,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise ensureCodexAuthCacheEntryDir(process.env, accountId, agent.companyId), env: process.env, - })), + }); } finally { await sshSessionState?.cleanup(); } + }, // No `exclude` denylist: `stagedCodexHomeDir` already contains // ONLY the allowlisted files (auth/config/skills), so there is // nothing to filter out. @@ -928,10 +931,26 @@ export async function execute(ctx: AdapterExecutionContext): Promise> = null; let executionError: unknown = null; try { - const remoteCodexHome = executionTargetIsRemote && !useSshHostAuth + let remoteCodexHome = executionTargetIsRemote && !useSshHostAuth ? preparedExecutionTargetRuntime?.assetDirs.home ?? path.posix.join(effectiveExecutionCwd, ".paperclip-runtime", "codex", "home") : null; + if (executionTarget?.kind === "remote" && executionTarget.transport === "ssh" && executionTarget.spec.useHostAuth === false + && config.managedAiConnection && targetWorkspaceRealization?.mode === "in_place" && remoteCodexHome) { + const account = parseObject(config.managedAiConnection); + const identity = asString(account.sessionIdentity, asString(account.identity, "")); + if (!identity) throw new Error("Managed SSH Codex execution requires an account identity"); + sshSessionState = await prepareCodexSshSessionState({ + remoteRoot: executionTarget.spec.remoteWorkspacePath, companyId: agent.companyId, agentId: agent.id, + accountIdentity: identity, executionIdentity: adapterExecutionTargetSessionIdentity(runtimeExecutionTarget), + runId, saved: parseObject(runtime.sessionParams), stagedHome: remoteCodexHome, + run: async command => { + const result = await runAdapterExecutionTargetShellCommand(runId, runtimeExecutionTarget, command, { cwd: effectiveExecutionCwd, env: {}, timeoutSec: 15 }); + if (result.exitCode !== 0 || result.timedOut) throw new Error("Cannot prepare or clean up SSH Codex conversation state"); + }, + }); + remoteCodexHome = sshSessionState.home; + } await emitSandboxAuthPrecedenceWarningIfNeeded({ runId, target: runtimeExecutionTarget, @@ -1138,6 +1157,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise 0 && + (!sshSessionState || sshSessionState.canResume) && (runtimeSessionCwd.length === 0 || path.resolve(runtimeSessionCwd) === path.resolve(effectiveExecutionCwd)) && adapterExecutionTargetSessionMatches(runtimeRemoteExecution, runtimeExecutionTarget); const codexTransientFallbackMode = readCodexTransientFallbackMode(context); @@ -1525,7 +1545,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise; + stagedHome: string; + run: (command: string) => Promise; +}) { + const hash = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); + const scope = hash([input.companyId, input.agentId, input.accountIdentity, input.executionIdentity]); + const savedId = input.saved.sshCodexConversationId; + const canResume = input.saved.sshCodexAccountScope === scope && typeof savedId === "string" && /^[a-f0-9]{64}$/.test(savedId); + const conversationId = canResume ? savedId : hash(input.runId); + const home = path.posix.join(input.remoteRoot, ".paperclip-runtime", "codex", "conversations", scope, conversationId); + const prefix = `set -eu\numask 077\nhome=${shellQuote(home)}\n[ ! -L "$home" ]\n`; + const cleanup = () => input.run(`${prefix}if [ -d "$home" ]; then\n${STATE_ONLY}\nfi`); + try { + await input.run(`${prefix}mkdir -p "$home"\nchmod 700 "$home"\n${STATE_ONLY}\ncp -RP ${shellQuote(input.stagedHome + "/.")} "$home/"`); + } catch (error) { + await cleanup(); + throw error; + } + return { home, scope, conversationId, canResume, cleanup }; +} diff --git a/packages/adapters/grok-local/src/server/execute.test.ts b/packages/adapters/grok-local/src/server/execute.test.ts index ba18635e386..b741f0640c1 100644 --- a/packages/adapters/grok-local/src/server/execute.test.ts +++ b/packages/adapters/grok-local/src/server/execute.test.ts @@ -64,7 +64,10 @@ vi.mock("@paperclipai/adapter-utils/execution-target", () => ({ prepareAdapterExecutionTargetRuntime: (...args: unknown[]) => (mocks.prepareRuntimeMock as (...args: unknown[]) => unknown)(...args), readAdapterExecutionTarget: (input: { executionTarget?: unknown }) => input.executionTarget ?? - (mocks.state.isRemote ? { kind: "remote", transport: "ssh" } : { kind: "local" }), + (mocks.state.isRemote ? { kind: "remote", transport: "ssh", remoteCwd: "/remote/workspace", spec: { + host: "fixture.invalid", port: 22, username: "fixture", remoteCwd: "/remote/workspace", remoteWorkspacePath: "/remote/workspace", + privateKey: null, knownHosts: null, strictHostKeyChecking: true, + } } : { kind: "local" }), resolveAdapterExecutionTargetCommandForLogs: (...args: unknown[]) => (mocks.resolveCommandForLogsMock as (...args: unknown[]) => unknown)(...args), resolveAdapterExecutionTargetTimeoutSec: (_target: unknown, timeoutSec: number) => timeoutSec, diff --git a/server/src/__tests__/ai-connection-router.test.ts b/server/src/__tests__/ai-connection-router.test.ts index 329feb58dbc..6f6ca01fc94 100644 --- a/server/src/__tests__/ai-connection-router.test.ts +++ b/server/src/__tests__/ai-connection-router.test.ts @@ -27,6 +27,8 @@ import { legacyExecutionNeedsReconciliation } from "../services/legacy-execution import { getServerAdapter, registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; import { dotRunnerBroker } from "../services/dot-runner-broker.js"; +import { environmentService } from "../services/environments.js"; +import * as executionConfiguration from "../services/heartbeat/execution-configuration.js"; const captureRunFailure = vi.hoisted(() => vi.fn()); vi.mock("../sentry.js", async (importOriginal) => ({ @@ -446,6 +448,52 @@ describe("durable, authorized connection routing", () => { expect(resolveTaskSessionConfigFreshness({ hasTaskSession: true, configuredModel: String(selected.runtimeConfig.model), taskSessionParams: { __paperclipConfiguredModel: selected.runtimeConfig.model, __paperclipConfigFingerprint: previous.fingerprint, __paperclipConfigFingerprintVersion: previous.version, __paperclipConfigCategoryFingerprints: previous.categoryFingerprints }, configMetadata: current, compatibleConfigMetadata }).reset).toBe(false); } finally { await before.cleanup(); await after.cleanup(); } }); + it.each([ + { useHostAuth: undefined, nativeHomeMode: undefined }, { useHostAuth: true, nativeHomeMode: undefined }, + { useHostAuth: false, nativeHomeMode: undefined }, { useHostAuth: false, nativeHomeMode: "preconfigured" }, + ])("applies SSH authentication before selecting a busy pool (%j)", async ({ useHostAuth, nativeHomeMode }) => { + const original = await instanceSettingsService(db).getExperimental(); + await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true, enableNativeRunner: true, enableEnvironments: true, enableWorktreeRunExecution: true }); + const pool = await makePool({ members: [members[0]!] }); + const environment = await environmentService(db).create(companyId, { name: `SSH routing fixture ${useHostAuth} ${nativeHomeMode}`, driver: "ssh", status: "active", envVars: {}, config: { + host: "ssh.example.test", username: "fixture", remoteWorkspacePath: "/workspace", ...(useHostAuth === undefined ? {} : { useHostAuth }), + } }); + const configured = { provider: "acpx", acpxAgent: "claude", model: "fixture-host-model", ...(nativeHomeMode ? { homeMode: nativeHomeMode } : {}) }; + const [agent] = await db.insert(agents).values({ companyId, name: "SSH routing fixture", status: "idle", adapterType: "paperclip_runner", + adapterConfig: configured, defaultEnvironmentId: environment.id, + runtimeConfig: { aiConnection: { mode: "router", connectionId: pool.id }, heartbeat: { enabled: false, wakeOnDemand: true } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "SSH pool routing fixture", status: "todo", assigneeAgentId: agent.id, responsibleUserId: "alice", createdByUserId: "alice" }).returning(); + proposal.mockClear(); + proposal.mockImplementation(async () => ({ kind: "exhausted", retryAt: new Date(Date.now() + 60_000).toISOString(), skipped: {} })); + const preparation = vi.spyOn(executionConfiguration, "prepareHeartbeatExecutionConfiguration").mockRejectedValue(new Error("fixture stopped before provider configuration")); + const heartbeat = heartbeatService(db, { pluginWorkerManager: worker, runtimeEnv: {} }); + try { + const run = await heartbeat.wakeup(agent.id, { source: "assignment", reason: "issue_assigned", triggerDetail: "system", + payload: { issueId: issue.id }, contextSnapshot: { issueId: issue.id, responsibleUserId: "alice" } }); + expect(run).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + const saved = await heartbeat.getRun(run!.id); + if (useHostAuth === false) { + expect(proposal).toHaveBeenCalled(); + expect(preparation).not.toHaveBeenCalled(); + expect(saved?.errorCode).toBe("ai_connection_pool_exhausted"); + } else { + expect(proposal).not.toHaveBeenCalled(); + expect(saved?.contextSnapshot?.aiRouterSelection).toBeUndefined(); + expect(saved?.error).toContain("fixture stopped before provider configuration"); + expect(preparation).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ config: expect.objectContaining({ + aiBinding: undefined, sourceWorkspaceConfig: expect.objectContaining(configured), + }) })); + } + } finally { + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agent.id)); + await heartbeat.drainActiveRunExecutions(); + preparation.mockRestore(); + proposal.mockImplementation(defaultProposal); + await instanceSettingsService(db).updateExperimental(original); + } + }, 30_000); + it("admits a healthy pool member when the saved Grok connection is held", async () => { const original = await instanceSettingsService(db).getExperimental(); await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true, diff --git a/server/src/__tests__/heartbeat-execution-configuration.test.ts b/server/src/__tests__/heartbeat-execution-configuration.test.ts index 028535af21a..84fefd182f4 100644 --- a/server/src/__tests__/heartbeat-execution-configuration.test.ts +++ b/server/src/__tests__/heartbeat-execution-configuration.test.ts @@ -25,7 +25,7 @@ import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js"; import * as sentry from "../sentry.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; -import { selectHeartbeatEnvironment, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js"; +import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js"; import { environmentService } from "../services/environments.js"; import { executionWorkspaceService } from "../services/execution-workspaces.js"; import { buildExecutionWorkspaceAdapterConfig } from "../services/execution-workspace-policy.js"; @@ -41,6 +41,21 @@ function deferred() { const skillFailures = vi.hoisted(() => new Map()); const skillWaits = vi.hoisted(() => new Map Promise>()); +describe("SSH authentication before account routing", () => { + it.each([ + { agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "agent", expected: "host" }, + { agentDefaultEnvironmentId: null, lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "instance", expected: "host" }, + { agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: "sandbox", executionMode: "any", expectedId: "sandbox", expected: null }, + { agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "kubernetes", expectedId: null, expected: null }, + ] as const)("honors environment precedence and forced execution policy: %j", async value => { + const get = vi.fn(async () => ({ defaultEnvironmentId: "instance", general: { executionMode: value.executionMode } })); + const getById = vi.fn(async (id: string) => ({ id, driver: id === "sandbox" ? "sandbox" : "ssh", config: {} } as Environment)); + expect(await resolveSshAuthenticationBeforeRouting({ ...value, instanceSettings: { get }, environmentsSvc: { getById } })).toBe(value.expected); + if (value.expectedId) expect(getById).toHaveBeenCalledExactlyOnceWith(value.expectedId); + else expect(getById).not.toHaveBeenCalled(); + }); +}); + describe("Runner home mode during recovery preparation", () => { const fingerprint = (effectiveAdapterConfig: Record) => buildEffectiveRunSessionConfigMetadata({ adapterType: "paperclip_runner", effectiveAdapterConfig, agentRuntimeConfig: {}, issueOverrides: null, diff --git a/server/src/__tests__/kimi-connection-probe.test.ts b/server/src/__tests__/kimi-connection-probe.test.ts index b7c05ae1118..a7a89991e94 100644 --- a/server/src/__tests__/kimi-connection-probe.test.ts +++ b/server/src/__tests__/kimi-connection-probe.test.ts @@ -24,7 +24,7 @@ vi.mock("node:child_process", async importOriginal => { spawn: (file: string, args: string[], options: Parameters[2]) => { if (file !== "ssh") return actual.spawn(file, args, options); mocks.wire.push({ file, args, stdin: true }); - return actual.spawn(process.execPath, ["-e", 'process.stdout.write(require("node:fs").readFileSync(0,"utf8"))'], options); + return actual.spawn(process.execPath, ["-e", 'const wire = require("node:fs").readFileSync(0); process.stdout.write(wire.subarray(wire.indexOf(10) + 1))'], options); }, }; }); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index c7e91e9cc2f..8892fe8ddc2 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -3391,7 +3391,7 @@ export function agentRoutes( }); type = selection.adapterType; adapterConfigForTest = { ...selection.adapterConfig, - ...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" && requestedEnvironment.config.useHostAuth !== false ? { homeMode: "preconfigured" } : {}) }; + ...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" ? { homeMode: requestedEnvironment.config.useHostAuth === false ? "isolated" : "preconfigured" } : {}) }; if (requestedEnvironment) await assertAdapterTestEnvironmentForCompany(companyId, requestedEnvironment.id, type); const adapter = requireServerAdapter(type); if (!retainsSavedExecution) await assertSelectableAdapterType(type, adapterConfigForTest, companyId); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 32481819937..5374759701e 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -58,7 +58,7 @@ import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery. import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability"; import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js"; import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js"; -import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js"; +import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting } from "./heartbeat/environment-selection.js"; import { reconcileMuseNativeTerminalRun } from "./native-runtime/muse-terminal-reconciliation.js"; import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js"; import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js"; @@ -2666,8 +2666,17 @@ export function heartbeatService( if ("homeMode" in homeConfig) config = { ...config, homeMode: homeConfig.homeMode }; } const persistedRouterPoolId = routerHasPersistedInput ? readNonEmptyString(parseObject(context.aiRouterSelection).poolId) : null; + const sshAuthentication = routerHasPersistedInput ? null : await resolveSshAuthenticationBeforeRouting({ + agentDefaultEnvironmentId: agent.defaultEnvironmentId, + lowTrustIssueEnvironmentId: trustPreset.kind === "low_trust_review" + ? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings, { includeEnvironmentId: true })?.environmentId ?? null : null, + environmentsSvc, instanceSettings, + }); + if (agent.adapterType === "paperclip_runner" && sshAuthentication) { + config = { ...config, homeMode: sshAuthentication === "host" ? "preconfigured" : "isolated" }; + } const preconfiguredHome = agent.adapterType === "paperclip_runner" && config.homeMode === "preconfigured"; - const requestedAiBinding = preconfiguredHome ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined; + const requestedAiBinding = preconfiguredHome || sshAuthentication === "host" ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined; let aiBinding = requestedAiBinding?.mode === "router" ? undefined : requestedAiBinding; const originalAiIssueOverrides = issueAssigneeOverrides; if (requestedAiBinding?.mode === "router") { diff --git a/server/src/services/heartbeat/environment-selection.ts b/server/src/services/heartbeat/environment-selection.ts index ab0581c87f0..78f9e922b82 100644 --- a/server/src/services/heartbeat/environment-selection.ts +++ b/server/src/services/heartbeat/environment-selection.ts @@ -3,7 +3,7 @@ import { heartbeatExternalProvider, nativeExecutionUsesTaskWorkspace } from "./e import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; import { readGitCacheExpiration } from "../git-workspace-restoration.js"; -import type { ExecutionWorkspace, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared"; +import type { ExecutionWorkspace, InstanceSettings, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared"; import { parseObject } from "../../adapters/utils.js"; import { logger } from "../../middleware/logger.js"; import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; @@ -39,6 +39,23 @@ import { resolveNativeRecoveryExecutionWorkspaceBinding, resolveExecutionWorkspa type Issue = typeof issues.$inferSelect; +export async function resolveSshAuthenticationBeforeRouting(input: { + agentDefaultEnvironmentId: string | null; + lowTrustIssueEnvironmentId: string | null; + environmentsSvc: Pick, "getById">; + instanceSettings: { get(): Promise<{ + defaultEnvironmentId: InstanceSettings["defaultEnvironmentId"]; + general: Pick; + }> }; +}): Promise<"host" | "managed" | null> { + const settings = await input.instanceSettings.get(); + if (isExecutionForcedToKubernetes({ executionMode: settings.general.executionMode })) return null; + const id = input.lowTrustIssueEnvironmentId ?? input.agentDefaultEnvironmentId ?? settings.defaultEnvironmentId; + if (!id) return null; + const environment = await input.environmentsSvc.getById(id); + return environment?.driver === "ssh" ? environment.config.useHostAuth === false ? "managed" : "host" : null; +} + /** Selects policy and reuse inputs; lease acquisition and cleanup belong to the executor. */ export interface HeartbeatEnvironmentSelectionInput { run: Pick { }); it("still delivers every byte when SSH consumes the upload", async () => { - await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst hash = require('node:crypto').createHash('sha256');\nprocess.stdin.on('data', chunk => hash.update(chunk));\nprocess.stdin.on('end', () => process.stdout.write(hash.digest('hex')));\n`); + await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst chunks = [];\nprocess.stdin.on('data', chunk => chunks.push(chunk));\nprocess.stdin.on('end', () => { const wire = Buffer.concat(chunks); const end = wire.indexOf(10); if (end < 0) process.exit(1); process.stdout.write(require('node:crypto').createHash('sha256').update(wire.subarray(end + 1)).digest('hex')); });\n`); const stdin = "synthetic upload bytes\0".repeat(32_768); const result = await createRunner().execute({ command: "node", args: ["unused"], stdin, timeoutMs: 5000 }); expect(result.exitCode).toBe(0); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 2623962bee4..7fbe53f120e 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -392,7 +392,7 @@ "packages/shared/src/types/decision.ts": "b0d40077d481ce6180de6b02bad6df5bb0fbf977", "packages/shared/src/types/document-annotation.ts": "7769a42f1ccd707a2259774e7fe6aa536b5d183f", "packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc", - "packages/shared/src/types/environment.ts": "a380b9ce9d2d1763011e9adaa58615a3ef1e56c5", + "packages/shared/src/types/environment.ts": "16b9d9598abfa2583602181b44483b0fa24d5bc2", "packages/shared/src/types/execution-continuation.ts": "5afe62afd227efa03ff847d8f73291c14707ab5b", "packages/shared/src/types/execution-projection.ts": "e72e641602b296f6d3e16d07700629df3039a7b1", "packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1",